<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"

	"os"
	"strconv"

	termbox "github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	tuikit "github.com/sgeb/go-tuikit"
	db "github.com/sgeb/go-tuikit/databinding"
)

func main() {
	repaint := make(chan struct{}, 1)
	quit := make(chan struct{}, 1)

	if err := tuikit.Init(); err != nil {
		panic(err)
	}
	defer tuikit.Close()

	fmt.Fprintln(os.Stderr, "-----\nStarting")
	w := newWindow()
	w.SetPaintSubscriber(func() { repaint <- struct{}{} })
	tuikit.SetPainter(w)
	repaint <- struct{}{}

	for i := 0; ; i++ {
		select {
		case ev := <-tuikit.Events:
			if ev.Handled || ev.Type != termbox.EventKey {
				continue
			}
			if ev.Ch == 'q' {
				quit <- struct{}{}
			}
		case <-repaint:
			tuikit.Paint()
		case <-quit:
			return
		}
		//		fmt.Fprintf(os.Stderr, "[%d] nbr of goroutines: %d\n", i, runtime.NumGoroutine())
	}
}

//----------------------------------------------------------------------------
// window
//----------------------------------------------------------------------------

type window struct {
	*tuikit.BaseView
	lastPaintRect tulib.Rect
	views         []*tuikit.TextView
}

func newWindow() *window {
	return &window{
		BaseView: tuikit.NewBaseView(),
	}
}

func (w *window) PaintTo(buffer *tulib.Buffer, rect tulib.Rect) error {
	if w.lastPaintRect.Width != rect.Width ||
		w.lastPaintRect.Height != rect.Height {
		for _, v := range w.views {
			w.DetachChild(v)
		}

		ns := rect.Width * rect.Height
		diff := ns - len(w.views)
		if diff > 0 {
			for i := 0; i < diff; i++ {
				tv := tuikit.NewTextView()
				rs := newRandomString()
				c := rs.Subscribe()
				go func() {
					for _ = range c {
						tv.SetText(rs.Get())
					}
				}()
				rs.startRandomness()
				w.views = append(w.views, tv)
			}
		} else {
			w.views = w.views[:ns]
		}

		for i, v := range w.views {
			dx := int(i % rect.Width)
			dy := int(i / rect.Width)
			w.AttachChild(v, tulib.Rect{dx, dy, 1, 1})
		}

		w.lastPaintRect = rect
	}

	return w.BaseView.PaintTo(buffer, rect)
}

//----------------------------------------------------------------------------
// randomString
//----------------------------------------------------------------------------

type randomString struct {
	db.StringProperty
}

func newRandomString() *randomString {
	return &randomString{db.NewStringProperty()}
}

func (rs *randomString) startRandomness() {
	go func() {
		sleep := time.Duration(rand.Float64() * 2.0 * 1e9)

		for i := uint64(0); ; i++ {
			rs.Set(strconv.Itoa(int(i % 10)))
			time.Sleep(sleep)
		}
	}()
}
<commit_msg>Fix goroutine leak<commit_after>package main

import (
	"fmt"
	"math/rand"
	"strconv"
	"time"

	"os"

	"runtime"

	termbox "github.com/nsf/termbox-go"
	"github.com/nsf/tulib"
	tuikit "github.com/sgeb/go-tuikit"
	db "github.com/sgeb/go-tuikit/databinding"
)

func main() {
	repaint := make(chan struct{}, 1)
	quit := make(chan struct{}, 1)

	if err := tuikit.Init(); err != nil {
		panic(err)
	}
	defer tuikit.Close()

	fmt.Fprintln(os.Stderr, "-----\nStarting")
	w := newWindow()
	w.SetPaintSubscriber(func() { repaint <- struct{}{} })
	tuikit.SetPainter(w)
	repaint <- struct{}{}

	go func() {
		for _ = range time.Tick(time.Second) {
			fmt.Fprintf(os.Stderr, "Nbr of goroutines: %v\n", runtime.NumGoroutine())
		}
	}()

	for {
		select {
		case ev := <-tuikit.Events:
			if ev.Handled || ev.Type != termbox.EventKey {
				continue
			}
			if ev.Ch == 'q' {
				quit <- struct{}{}
			}
		case <-repaint:
			tuikit.Paint()
		case <-quit:
			return
		}
	}
}

//----------------------------------------------------------------------------
// window
//----------------------------------------------------------------------------

type window struct {
	*tuikit.BaseView
	lastPaintRect tulib.Rect
	views         []*tuikit.TextView
	randomStrings []*randomString
}

func newWindow() *window {
	return &window{
		BaseView: tuikit.NewBaseView(),
	}
}

func (w *window) PaintTo(buffer *tulib.Buffer, rect tulib.Rect) error {
	if w.lastPaintRect.Width != rect.Width ||
		w.lastPaintRect.Height != rect.Height {
		for _, v := range w.views {
			w.DetachChild(v)
		}

		ns := rect.Width * rect.Height
		diff := ns - len(w.views)
		if diff > 0 {
			for i := 0; i < diff; i++ {
				rs := newRandomString()
				tv := tuikit.NewTextView()

				go func() {
					for _ = range rs.Subscribe() {
						tv.SetText(rs.Get())
					}
				}()
				rs.startRandomness()

				w.randomStrings = append(w.randomStrings, rs)
				w.views = append(w.views, tv)
			}
		} else {
			for _, rs := range w.randomStrings[ns:] {
				rs.Dispose()
			}
			w.randomStrings = w.randomStrings[:ns]
			w.views = w.views[:ns]
		}

		for i, v := range w.views {
			dx := int(i % rect.Width)
			dy := int(i / rect.Width)
			w.AttachChild(v, tulib.Rect{dx, dy, 1, 1})
		}

		w.lastPaintRect = rect
	}

	return w.BaseView.PaintTo(buffer, rect)
}

//----------------------------------------------------------------------------
// randomString
//----------------------------------------------------------------------------

type randomString struct {
	db.StringProperty
	stopRandom chan struct{}
}

func newRandomString() *randomString {
	return &randomString{
		db.NewStringProperty(),
		make(chan struct{}),
	}
}

func (rs *randomString) startRandomness() {
	go func() {
		tick := time.Tick(time.Duration(rand.Float64() * 2.0 * 1e9))
		i := uint64(1)

		for {
			select {
			case <-rs.stopRandom:
				return
			case <-tick:
				rs.Set(strconv.Itoa(int(i % 10)))
				i++
			}
		}
	}()
}

func (rs *randomString) Dispose() {
	rs.stopRandom <- struct{}{}
	rs.StringProperty.Dispose()
}
<|endoftext|>


<commit_before>package actions

import (
	"net/http"
	"strconv"

	"github.com/stellar/go-stellar-base/xdr"
	"github.com/stellar/horizon/assets"
	"github.com/stellar/horizon/db"
	"github.com/stellar/horizon/render/problem"
)

const (
	// ParamCursor is a query string param name
	ParamCursor = "cursor"
	// ParamOrder is a query string param name
	ParamOrder = "order"
	// ParamLimit is a query string param name
	ParamLimit = "limit"
)

// OrderBookParams is a helper struct that encapsulates the specification for
// an order book
type OrderBookParams struct {
	SellingType   xdr.AssetType
	SellingIssuer string
	SellingCode   string
	BuyingType    xdr.AssetType
	BuyingIssuer  string
	BuyingCode    string
}

// GetString retrieves a string from either the URLParams, form or query string.
// This method uses the priority (URLParams, Form, Query).
func (base *Base) GetString(name string) string {
	if base.Err != nil {
		return ""
	}

	fromURL, ok := base.GojiCtx.URLParams[name]

	if ok {
		return fromURL
	}

	fromForm := base.R.FormValue(name)

	if fromForm != "" {
		return fromForm
	}

	return base.R.URL.Query().Get(name)
}

// GetInt64 retrieves an int64 from the action parameter of the given name.
// Populates err if the value is not a valid int64
func (base *Base) GetInt64(name string) int64 {
	if base.Err != nil {
		return 0
	}

	asStr := base.GetString(name)

	if asStr == "" {
		return 0
	}

	asI64, err := strconv.ParseInt(asStr, 10, 64)

	if err != nil {
		base.Err = err
		return 0
	}

	return asI64
}

// ValidateInt64 populates err if the value is not a valid int64
func (base *Base) ValidateInt64(name string) {
	_ = base.GetInt64(name)
}

// GetInt32 retrieves an int32 from the action parameter of the given name.
// Populates err if the value is not a valid int32
func (base *Base) GetInt32(name string) int32 {
	if base.Err != nil {
		return 0
	}

	asStr := base.GetString(name)

	if asStr == "" {
		return 0
	}

	asI64, err := strconv.ParseInt(asStr, 10, 32)

	if err != nil {
		base.Err = err
		return 0
	}

	return int32(asI64)
}

// GetPagingParams returns the cursor/order/limit triplet that is the
// standard way of communicating paging data to a horizon endpoint.
func (base *Base) GetPagingParams() (cursor string, order string, limit int32) {
	if base.Err != nil {
		return
	}

	cursor = base.GetString(ParamCursor)
	order = base.GetString(ParamOrder)
	limit = base.GetInt32(ParamLimit)

	if lei := base.R.Header.Get("Last-Event-ID"); lei != "" {
		cursor = lei
	}

	return
}

// GetPageQuery is a helper that returns a new db.PageQuery struct initialized
// using the results from a call to GetPagingParams()
func (base *Base) GetPageQuery() db.PageQuery {
	if base.Err != nil {
		return db.PageQuery{}
	}

	r, err := db.NewPageQuery(base.GetPagingParams())

	if err != nil {
		base.Err = err
	}

	return r
}

// GetAssetType is a helper that returns a xdr.AssetType by reading a string
func (base *Base) GetAssetType(name string) xdr.AssetType {
	if base.Err != nil {
		return xdr.AssetTypeAssetTypeNative
	}

	r, err := assets.Parse(base.GetString(name))

	if base.Err != nil {
		return xdr.AssetTypeAssetTypeNative
	}

	if err != nil {
		base.Err = err
	}

	return r
}

// GetOrderBook returns an OrderBookParams from the url params
func (base *Base) GetOrderBook() (result OrderBookParams) {
	if base.Err != nil {
		return
	}

	result = OrderBookParams{
		SellingType:   base.GetAssetType("selling_asset_type"),
		SellingIssuer: base.GetString("selling_asset_issuer"),
		SellingCode:   base.GetString("selling_asset_code"),
		BuyingType:    base.GetAssetType("buying_asset_type"),
		BuyingIssuer:  base.GetString("buying_asset_issuer"),
		BuyingCode:    base.GetString("buying_asset_code"),
	}

	if base.Err != nil {
		goto InvalidOrderBook
	}

	if result.SellingType != xdr.AssetTypeAssetTypeNative {
		if result.SellingCode == "" {
			goto InvalidOrderBook
		}

		if result.SellingIssuer == "" {
			goto InvalidOrderBook
		}
	}

	if result.BuyingType != xdr.AssetTypeAssetTypeNative {
		if result.BuyingCode == "" {
			goto InvalidOrderBook
		}

		if result.BuyingIssuer == "" {
			goto InvalidOrderBook
		}
	}

	return

InvalidOrderBook:
	base.Err = &problem.P{
		Type:   "invalid_order_book",
		Title:  "Invalid Order Book Parameters",
		Status: http.StatusBadRequest,
		Detail: "The parameters that specify what order book to view are invalid in some way. " +
			"Please ensure that your type parameters (selling_asset_type and buying_asset_type) are one the " +
			"following valid values: native, credit_alphanum4, credit_alphanum12.  Also ensure that you " +
			"have specified selling_asset_code and selling_issuer if selling_asset_type is not 'native', as well " +
			"as buying_asset_code and buying_issuer if buying_asset_type is not 'native'",
	}

	return
}

// Path returns the current action's path, as determined by the http.Request of
// this action
func (base *Base) Path() string {
	return base.R.URL.Path
}
<commit_msg>Add some wrapped errors to action helpers<commit_after>package actions

import (
	"net/http"
	"strconv"

	"github.com/go-errors/errors"
	"github.com/stellar/go-stellar-base/xdr"
	"github.com/stellar/horizon/assets"
	"github.com/stellar/horizon/db"
	"github.com/stellar/horizon/render/problem"
)

const (
	// ParamCursor is a query string param name
	ParamCursor = "cursor"
	// ParamOrder is a query string param name
	ParamOrder = "order"
	// ParamLimit is a query string param name
	ParamLimit = "limit"
)

// OrderBookParams is a helper struct that encapsulates the specification for
// an order book
type OrderBookParams struct {
	SellingType   xdr.AssetType
	SellingIssuer string
	SellingCode   string
	BuyingType    xdr.AssetType
	BuyingIssuer  string
	BuyingCode    string
}

// GetString retrieves a string from either the URLParams, form or query string.
// This method uses the priority (URLParams, Form, Query).
func (base *Base) GetString(name string) string {
	if base.Err != nil {
		return ""
	}

	fromURL, ok := base.GojiCtx.URLParams[name]

	if ok {
		return fromURL
	}

	fromForm := base.R.FormValue(name)

	if fromForm != "" {
		return fromForm
	}

	return base.R.URL.Query().Get(name)
}

// GetInt64 retrieves an int64 from the action parameter of the given name.
// Populates err if the value is not a valid int64
func (base *Base) GetInt64(name string) int64 {
	if base.Err != nil {
		return 0
	}

	asStr := base.GetString(name)

	if asStr == "" {
		return 0
	}

	asI64, err := strconv.ParseInt(asStr, 10, 64)

	if err != nil {
		base.Err = errors.Wrap(err, 1)
		return 0
	}

	return asI64
}

// ValidateInt64 populates err if the value is not a valid int64
func (base *Base) ValidateInt64(name string) {
	_ = base.GetInt64(name)
}

// GetInt32 retrieves an int32 from the action parameter of the given name.
// Populates err if the value is not a valid int32
func (base *Base) GetInt32(name string) int32 {
	if base.Err != nil {
		return 0
	}

	asStr := base.GetString(name)

	if asStr == "" {
		return 0
	}

	asI64, err := strconv.ParseInt(asStr, 10, 32)

	if err != nil {
		base.Err = errors.Wrap(err, 1)
		return 0
	}

	return int32(asI64)
}

// GetPagingParams returns the cursor/order/limit triplet that is the
// standard way of communicating paging data to a horizon endpoint.
func (base *Base) GetPagingParams() (cursor string, order string, limit int32) {
	if base.Err != nil {
		return
	}

	cursor = base.GetString(ParamCursor)
	order = base.GetString(ParamOrder)
	limit = base.GetInt32(ParamLimit)

	if lei := base.R.Header.Get("Last-Event-ID"); lei != "" {
		cursor = lei
	}

	return
}

// GetPageQuery is a helper that returns a new db.PageQuery struct initialized
// using the results from a call to GetPagingParams()
func (base *Base) GetPageQuery() db.PageQuery {
	if base.Err != nil {
		return db.PageQuery{}
	}

	r, err := db.NewPageQuery(base.GetPagingParams())

	if err != nil {
		base.Err = err
	}

	return r
}

// GetAssetType is a helper that returns a xdr.AssetType by reading a string
func (base *Base) GetAssetType(name string) xdr.AssetType {
	if base.Err != nil {
		return xdr.AssetTypeAssetTypeNative
	}

	r, err := assets.Parse(base.GetString(name))

	if base.Err != nil {
		return xdr.AssetTypeAssetTypeNative
	}

	if err != nil {
		base.Err = err
	}

	return r
}

// GetOrderBook returns an OrderBookParams from the url params
func (base *Base) GetOrderBook() (result OrderBookParams) {
	if base.Err != nil {
		return
	}

	result = OrderBookParams{
		SellingType:   base.GetAssetType("selling_asset_type"),
		SellingIssuer: base.GetString("selling_asset_issuer"),
		SellingCode:   base.GetString("selling_asset_code"),
		BuyingType:    base.GetAssetType("buying_asset_type"),
		BuyingIssuer:  base.GetString("buying_asset_issuer"),
		BuyingCode:    base.GetString("buying_asset_code"),
	}

	if base.Err != nil {
		goto InvalidOrderBook
	}

	if result.SellingType != xdr.AssetTypeAssetTypeNative {
		if result.SellingCode == "" {
			goto InvalidOrderBook
		}

		if result.SellingIssuer == "" {
			goto InvalidOrderBook
		}
	}

	if result.BuyingType != xdr.AssetTypeAssetTypeNative {
		if result.BuyingCode == "" {
			goto InvalidOrderBook
		}

		if result.BuyingIssuer == "" {
			goto InvalidOrderBook
		}
	}

	return

InvalidOrderBook:
	base.Err = &problem.P{
		Type:   "invalid_order_book",
		Title:  "Invalid Order Book Parameters",
		Status: http.StatusBadRequest,
		Detail: "The parameters that specify what order book to view are invalid in some way. " +
			"Please ensure that your type parameters (selling_asset_type and buying_asset_type) are one the " +
			"following valid values: native, credit_alphanum4, credit_alphanum12.  Also ensure that you " +
			"have specified selling_asset_code and selling_issuer if selling_asset_type is not 'native', as well " +
			"as buying_asset_code and buying_issuer if buying_asset_type is not 'native'",
	}

	return
}

// Path returns the current action's path, as determined by the http.Request of
// this action
func (base *Base) Path() string {
	return base.R.URL.Path
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
	"k8s.io/apiserver/pkg/authentication/user"
	apifilters "k8s.io/apiserver/pkg/endpoints/filters"
	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/server/mux"
	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

const (
	decisionNoQueuingExecute = iota
	decisionQueuingExecute
	decisionCancelWait
	decisionReject
	decisionSkipFilter
)

type fakeApfFilter struct {
	mockDecision int
	postEnqueue  func()
	postDequeue  func()
}

func (t fakeApfFilter) MaintainObservations(stopCh <-chan struct{}) {
}

func (t fakeApfFilter) Handle(ctx context.Context,
	requestDigest utilflowcontrol.RequestDigest,
	noteFn func(fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration),
	queueNoteFn fq.QueueNoteFn,
	execFn func(),
) {
	if t.mockDecision == decisionSkipFilter {
		panic("Handle should not be invoked")
	}
	noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault)
	switch t.mockDecision {
	case decisionNoQueuingExecute:
		execFn()
	case decisionQueuingExecute:
		queueNoteFn(true)
		t.postEnqueue()
		queueNoteFn(false)
		t.postDequeue()
		execFn()
	case decisionCancelWait:
		queueNoteFn(true)
		t.postEnqueue()
		queueNoteFn(false)
		t.postDequeue()
	case decisionReject:
		return
	}
}

func (t fakeApfFilter) Run(stopCh <-chan struct{}) error {
	return nil
}

func (t fakeApfFilter) Install(c *mux.PathRecorderMux) {
}

func newApfServer(decision int, t *testing.T) *httptest.Server {
	requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
	longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy"))

	apfHandler := WithPriorityAndFairness(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if decision == decisionCancelWait {
			t.Errorf("execute should not be invoked")
		}
		if decision != decisionSkipFilter && atomicReadOnlyExecuting != 1 {
			t.Errorf("Wanted %d requests executing, got %d", 1, atomicReadOnlyExecuting)
		}
	}), longRunningRequestCheck, fakeApfFilter{
		mockDecision: decision,
		postEnqueue: func() {
			if atomicReadOnlyWaiting != 1 {
				t.Errorf("Wanted %d requests in queue, got %d", 1, atomicReadOnlyWaiting)
			}
		},
		postDequeue: func() {
			if atomicReadOnlyWaiting != 0 {
				t.Errorf("Wanted %d requests in queue, got %d", 0, atomicReadOnlyWaiting)
			}
		},
	})

	handler := apifilters.WithRequestInfo(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
			Groups: []string{user.AllUnauthenticated},
		}))
		apfHandler.ServeHTTP(w, r)
		if atomicReadOnlyExecuting != 0 {
			t.Errorf("Wanted %d requests executing, got %d", 0, atomicReadOnlyExecuting)
		}
	}), requestInfoFactory)

	apfServer := httptest.NewServer(handler)
	return apfServer
}

func TestApfSkipLongRunningRequest(t *testing.T) {
	epmetrics.Register()

	server := newApfServer(decisionSkipFilter, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces?watch=true", server.URL), http.StatusOK); err != nil {
		// request should not be rejected
		t.Error(err)
	}
}

func TestApfRejectRequest(t *testing.T) {
	epmetrics.Register()

	server := newApfServer(decisionReject, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_request_terminations_total",
		"apiserver_dropped_requests_total",
	})
}

func TestApfExemptRequest(t *testing.T) {
	epmetrics.Register()
	fcmetrics.Register()

	// wait the first sampleAndWaterMark metrics to be collected
	time.Sleep(time.Millisecond * 50)

	server := newApfServer(decisionNoQueuingExecute, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_flowcontrol_read_vs_write_request_count_watermarks",
		"apiserver_flowcontrol_read_vs_write_request_count_samples",
	})
}

func TestApfExecuteRequest(t *testing.T) {
	epmetrics.Register()
	fcmetrics.Register()

	// wait the first sampleAndWaterMark metrics to be collected
	time.Sleep(time.Millisecond * 50)

	server := newApfServer(decisionQueuingExecute, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_current_inqueue_requests",
		"apiserver_flowcontrol_read_vs_write_request_count_watermarks",
		"apiserver_flowcontrol_read_vs_write_request_count_samples",
	})
}

func TestApfCancelWaitRequest(t *testing.T) {
	epmetrics.Register()

	server := newApfServer(decisionCancelWait, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_request_terminations_total",
		"apiserver_dropped_requests_total",
	})
}

// wait async metrics to be collected
func checkForExpectedMetricsWithRetry(t *testing.T, expectedMetrics []string) {
	maxRetries := 5
	var checkErrors []error
	for i := 0; i < maxRetries; i++ {
		t.Logf("Check for expected metrics with retry %d", i)
		metricsFamily, err := legacyregistry.DefaultGatherer.Gather()
		if err != nil {
			t.Fatalf("Failed to gather metrics %v", err)
		}

		metrics := map[string]interface{}{}
		for _, mf := range metricsFamily {
			mf := mf
			metrics[*mf.Name] = mf
		}

		checkErrors = checkForExpectedMetrics(expectedMetrics, metrics)
		if checkErrors == nil {
			return
		}

		time.Sleep(1 * time.Second)
	}
	for _, checkError := range checkErrors {
		t.Error(checkError)
	}
}

func checkForExpectedMetrics(expectedMetrics []string, metrics map[string]interface{}) []error {
	var errs []error
	for _, metricName := range expectedMetrics {
		if _, ok := metrics[metricName]; !ok {
			if !ok {
				errs = append(errs, errors.New("Scraped metrics did not include expected metric "+metricName))
			}
		}
	}
	return errs
}
<commit_msg>Add multi request test<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
	"k8s.io/apiserver/pkg/authentication/user"
	apifilters "k8s.io/apiserver/pkg/endpoints/filters"
	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/server/mux"
	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

type mockDecision int

const (
	decisionNoQueuingExecute mockDecision = iota
	decisionQueuingExecute
	decisionCancelWait
	decisionReject
	decisionSkipFilter
)

type fakeApfFilter struct {
	mockDecision mockDecision
	postEnqueue  func()
	postDequeue  func()
}

func (t fakeApfFilter) MaintainObservations(stopCh <-chan struct{}) {
}

func (t fakeApfFilter) Handle(ctx context.Context,
	requestDigest utilflowcontrol.RequestDigest,
	noteFn func(fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration),
	queueNoteFn fq.QueueNoteFn,
	execFn func(),
) {
	if t.mockDecision == decisionSkipFilter {
		panic("Handle should not be invoked")
	}
	noteFn(bootstrap.SuggestedFlowSchemaGlobalDefault, bootstrap.SuggestedPriorityLevelConfigurationGlobalDefault)
	switch t.mockDecision {
	case decisionNoQueuingExecute:
		execFn()
	case decisionQueuingExecute:
		queueNoteFn(true)
		t.postEnqueue()
		queueNoteFn(false)
		t.postDequeue()
		execFn()
	case decisionCancelWait:
		queueNoteFn(true)
		t.postEnqueue()
		queueNoteFn(false)
		t.postDequeue()
	case decisionReject:
		return
	}
}

func (t fakeApfFilter) Run(stopCh <-chan struct{}) error {
	return nil
}

func (t fakeApfFilter) Install(c *mux.PathRecorderMux) {
}

func newApfServerWithSingleRequest(decision mockDecision, t *testing.T) *httptest.Server {
	onExecuteFunc := func() {
		if decision == decisionCancelWait {
			t.Errorf("execute should not be invoked")
		}
		// atomicReadOnlyExecuting can be either 0 or 1 as we test one request at a time.
		if decision != decisionSkipFilter && atomicReadOnlyExecuting != 1 {
			t.Errorf("Wanted %d requests executing, got %d", 1, atomicReadOnlyExecuting)
		}
	}
	postExecuteFunc := func() {}
	// atomicReadOnlyWaiting can be either 0 or 1 as we test one request at a time.
	postEnqueueFunc := func() {
		if atomicReadOnlyWaiting != 1 {
			t.Errorf("Wanted %d requests in queue, got %d", 1, atomicReadOnlyWaiting)
		}
	}
	postDequeueFunc := func() {
		if atomicReadOnlyWaiting != 0 {
			t.Errorf("Wanted %d requests in queue, got %d", 0, atomicReadOnlyWaiting)
		}
	}
	return newApfServerWithHooks(decision, onExecuteFunc, postExecuteFunc, postEnqueueFunc, postDequeueFunc, t)
}

func newApfServerWithHooks(decision mockDecision, onExecute, postExecute, postEnqueue, postDequeue func(), t *testing.T) *httptest.Server {
	requestInfoFactory := &apirequest.RequestInfoFactory{APIPrefixes: sets.NewString("apis", "api"), GrouplessAPIPrefixes: sets.NewString("api")}
	longRunningRequestCheck := BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString("proxy"))

	apfHandler := WithPriorityAndFairness(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		onExecute()
	}), longRunningRequestCheck, fakeApfFilter{
		mockDecision: decision,
		postEnqueue:  postEnqueue,
		postDequeue:  postDequeue,
	})

	handler := apifilters.WithRequestInfo(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r = r.WithContext(apirequest.WithUser(r.Context(), &user.DefaultInfo{
			Groups: []string{user.AllUnauthenticated},
		}))
		apfHandler.ServeHTTP(w, r)
		postExecute()
		if atomicReadOnlyExecuting != 0 {
			t.Errorf("Wanted %d requests executing, got %d", 0, atomicReadOnlyExecuting)
		}
	}), requestInfoFactory)

	apfServer := httptest.NewServer(handler)
	return apfServer
}

func TestApfSkipLongRunningRequest(t *testing.T) {
	epmetrics.Register()

	server := newApfServerWithSingleRequest(decisionSkipFilter, t)
	defer server.Close()

	// send a watch request to test skipping long running request
	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces?watch=true", server.URL), http.StatusOK); err != nil {
		// request should not be rejected
		t.Error(err)
	}
}

func TestApfRejectRequest(t *testing.T) {
	epmetrics.Register()

	server := newApfServerWithSingleRequest(decisionReject, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_request_terminations_total",
		"apiserver_dropped_requests_total",
	})
}

func TestApfExemptRequest(t *testing.T) {
	epmetrics.Register()
	fcmetrics.Register()

	// Wait for at least one sampling window to pass since creation of metrics.ReadWriteConcurrencyObserverPairGenerator,
	// so that an observation will cause some data to go into the Prometheus metrics.
	time.Sleep(time.Millisecond * 50)

	server := newApfServerWithSingleRequest(decisionNoQueuingExecute, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_flowcontrol_read_vs_write_request_count_watermarks",
		"apiserver_flowcontrol_read_vs_write_request_count_samples",
	})
}

func TestApfExecuteRequest(t *testing.T) {
	epmetrics.Register()
	fcmetrics.Register()

	// Wait for at least one sampling window to pass since creation of metrics.ReadWriteConcurrencyObserverPairGenerator,
	// so that an observation will cause some data to go into the Prometheus metrics.
	time.Sleep(time.Millisecond * 50)

	server := newApfServerWithSingleRequest(decisionQueuingExecute, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_current_inqueue_requests",
		"apiserver_flowcontrol_read_vs_write_request_count_watermarks",
		"apiserver_flowcontrol_read_vs_write_request_count_samples",
	})
}

func TestApfExecuteMultipleRequests(t *testing.T) {
	epmetrics.Register()
	fcmetrics.Register()

	// Wait for at least one sampling window to pass since creation of metrics.ReadWriteConcurrencyObserverPairGenerator,
	// so that an observation will cause some data to go into the Prometheus metrics.
	time.Sleep(time.Millisecond * 50)

	concurrentRequests := 5
	var preStartExecute, postStartExecute, preEnqueue, postEnqueue, preDequeue, postDequeue, finishExecute sync.WaitGroup
	for _, wg := range []*sync.WaitGroup{&preStartExecute, &postStartExecute, &preEnqueue, &postEnqueue, &preDequeue, &postDequeue, &finishExecute} {
		wg.Add(concurrentRequests)
	}

	onExecuteFunc := func() {
		preStartExecute.Done()
		preStartExecute.Wait()
		if int(atomicReadOnlyExecuting) != concurrentRequests {
			t.Errorf("Wanted %d requests executing, got %d", concurrentRequests, atomicReadOnlyExecuting)
		}
		postStartExecute.Done()
		postStartExecute.Wait()
	}

	postEnqueueFunc := func() {
		preEnqueue.Done()
		preEnqueue.Wait()
		if int(atomicReadOnlyWaiting) != concurrentRequests {
			t.Errorf("Wanted %d requests in queue, got %d", 1, atomicReadOnlyWaiting)

		}
		postEnqueue.Done()
		postEnqueue.Wait()
	}

	postDequeueFunc := func() {
		preDequeue.Done()
		preDequeue.Wait()
		if atomicReadOnlyWaiting != 0 {
			t.Errorf("Wanted %d requests in queue, got %d", 0, atomicReadOnlyWaiting)
		}
		postDequeue.Done()
		postDequeue.Wait()
	}

	postExecuteFunc := func() {
		finishExecute.Done()
		finishExecute.Wait()
	}

	server := newApfServerWithHooks(decisionQueuingExecute, onExecuteFunc, postExecuteFunc, postEnqueueFunc, postDequeueFunc, t)
	defer server.Close()

	for i := 0; i < concurrentRequests; i++ {
		var err error
		go func() {
			err = expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusOK)
		}()
		if err != nil {
			t.Error(err)
		}
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_current_inqueue_requests",
		"apiserver_flowcontrol_read_vs_write_request_count_watermarks",
		"apiserver_flowcontrol_read_vs_write_request_count_samples",
	})
}

func TestApfCancelWaitRequest(t *testing.T) {
	epmetrics.Register()

	server := newApfServerWithSingleRequest(decisionCancelWait, t)
	defer server.Close()

	if err := expectHTTPGet(fmt.Sprintf("%s/api/v1/namespaces/default", server.URL), http.StatusTooManyRequests); err != nil {
		t.Error(err)
	}

	checkForExpectedMetricsWithRetry(t, []string{
		"apiserver_current_inflight_requests",
		"apiserver_request_terminations_total",
		"apiserver_dropped_requests_total",
	})
}

// wait async metrics to be collected
func checkForExpectedMetricsWithRetry(t *testing.T, expectedMetrics []string) {
	maxRetries := 5
	var checkErrors []error
	for i := 0; i < maxRetries; i++ {
		t.Logf("Check for expected metrics with retry %d", i)
		metricsFamily, err := legacyregistry.DefaultGatherer.Gather()
		if err != nil {
			t.Fatalf("Failed to gather metrics %v", err)
		}

		metrics := map[string]interface{}{}
		for _, mf := range metricsFamily {
			metrics[*mf.Name] = mf
		}

		checkErrors = checkForExpectedMetrics(expectedMetrics, metrics)
		if checkErrors == nil {
			return
		}

		time.Sleep(1 * time.Second)
	}
	for _, checkError := range checkErrors {
		t.Error(checkError)
	}
}

func checkForExpectedMetrics(expectedMetrics []string, metrics map[string]interface{}) []error {
	var errs []error
	for _, metricName := range expectedMetrics {
		if _, ok := metrics[metricName]; !ok {
			if !ok {
				errs = append(errs, errors.New("Scraped metrics did not include expected metric "+metricName))
			}
		}
	}
	return errs
}
<|endoftext|>


<commit_before>// +build !ignore_autogenerated

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file was autogenerated by deepcopy-gen. Do not edit it manually!

package unversioned

import (
	conversion "k8s.io/kubernetes/pkg/conversion"
	time "time"
)

func DeepCopy_unversioned_Duration(in Duration, out *Duration, c *conversion.Cloner) error {
	out.Duration = in.Duration
	return nil
}

func DeepCopy_unversioned_GroupKind(in GroupKind, out *GroupKind, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Kind = in.Kind
	return nil
}

func DeepCopy_unversioned_GroupResource(in GroupResource, out *GroupResource, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Resource = in.Resource
	return nil
}

func DeepCopy_unversioned_GroupVersion(in GroupVersion, out *GroupVersion, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Version = in.Version
	return nil
}

func DeepCopy_unversioned_GroupVersionKind(in GroupVersionKind, out *GroupVersionKind, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Version = in.Version
	out.Kind = in.Kind
	return nil
}

func DeepCopy_unversioned_GroupVersionResource(in GroupVersionResource, out *GroupVersionResource, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Version = in.Version
	out.Resource = in.Resource
	return nil
}

func DeepCopy_unversioned_LabelSelector(in LabelSelector, out *LabelSelector, c *conversion.Cloner) error {
	if in.MatchLabels != nil {
		in, out := in.MatchLabels, &out.MatchLabels
		*out = make(map[string]string)
		for key, val := range in {
			(*out)[key] = val
		}
	} else {
		out.MatchLabels = nil
	}
	if in.MatchExpressions != nil {
		in, out := in.MatchExpressions, &out.MatchExpressions
		*out = make([]LabelSelectorRequirement, len(in))
		for i := range in {
			if err := DeepCopy_unversioned_LabelSelectorRequirement(in[i], &(*out)[i], c); err != nil {
				return err
			}
		}
	} else {
		out.MatchExpressions = nil
	}
	return nil
}

func DeepCopy_unversioned_LabelSelectorRequirement(in LabelSelectorRequirement, out *LabelSelectorRequirement, c *conversion.Cloner) error {
	out.Key = in.Key
	out.Operator = in.Operator
	if in.Values != nil {
		in, out := in.Values, &out.Values
		*out = make([]string, len(in))
		copy(*out, in)
	} else {
		out.Values = nil
	}
	return nil
}

func DeepCopy_unversioned_ListMeta(in ListMeta, out *ListMeta, c *conversion.Cloner) error {
	out.SelfLink = in.SelfLink
	out.ResourceVersion = in.ResourceVersion
	return nil
}

func DeepCopy_unversioned_Time(in Time, out *Time, c *conversion.Cloner) error {
	if newVal, err := c.DeepCopy(in.Time); err != nil {
		return err
	} else {
		out.Time = newVal.(time.Time)
	}
	return nil
}

func DeepCopy_unversioned_Timestamp(in Timestamp, out *Timestamp, c *conversion.Cloner) error {
	out.Seconds = in.Seconds
	out.Nanos = in.Nanos
	return nil
}

func DeepCopy_unversioned_TypeMeta(in TypeMeta, out *TypeMeta, c *conversion.Cloner) error {
	out.Kind = in.Kind
	out.APIVersion = in.APIVersion
	return nil
}
<commit_msg>add api group 'federation' and 'cluster' object<commit_after>// +build !ignore_autogenerated

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file was autogenerated by deepcopy-gen. Do not edit it manually!

package unversioned

import (
	conversion "k8s.io/kubernetes/pkg/conversion"
	time "time"
)

func DeepCopy_unversioned_Duration(in Duration, out *Duration, c *conversion.Cloner) error {
	out.Duration = in.Duration
	return nil
}

func DeepCopy_unversioned_GroupKind(in GroupKind, out *GroupKind, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Kind = in.Kind
	return nil
}

func DeepCopy_unversioned_GroupResource(in GroupResource, out *GroupResource, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Resource = in.Resource
	return nil
}

func DeepCopy_unversioned_GroupVersion(in GroupVersion, out *GroupVersion, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Version = in.Version
	return nil
}

func DeepCopy_unversioned_GroupVersionKind(in GroupVersionKind, out *GroupVersionKind, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Version = in.Version
	out.Kind = in.Kind
	return nil
}

func DeepCopy_unversioned_GroupVersionResource(in GroupVersionResource, out *GroupVersionResource, c *conversion.Cloner) error {
	out.Group = in.Group
	out.Version = in.Version
	out.Resource = in.Resource
	return nil
}

func DeepCopy_unversioned_LabelSelector(in LabelSelector, out *LabelSelector, c *conversion.Cloner) error {
	if in.MatchLabels != nil {
		in, out := in.MatchLabels, &out.MatchLabels
		*out = make(map[string]string)
		for key, val := range in {
			(*out)[key] = val
		}
	} else {
		out.MatchLabels = nil
	}
	if in.MatchExpressions != nil {
		in, out := in.MatchExpressions, &out.MatchExpressions
		*out = make([]LabelSelectorRequirement, len(in))
		for i := range in {
			if err := DeepCopy_unversioned_LabelSelectorRequirement(in[i], &(*out)[i], c); err != nil {
				return err
			}
		}
	} else {
		out.MatchExpressions = nil
	}
	return nil
}

func DeepCopy_unversioned_LabelSelectorRequirement(in LabelSelectorRequirement, out *LabelSelectorRequirement, c *conversion.Cloner) error {
	out.Key = in.Key
	out.Operator = in.Operator
	if in.Values != nil {
		in, out := in.Values, &out.Values
		*out = make([]string, len(in))
		copy(*out, in)
	} else {
		out.Values = nil
	}
	return nil
}

func DeepCopy_unversioned_ListMeta(in ListMeta, out *ListMeta, c *conversion.Cloner) error {
	out.SelfLink = in.SelfLink
	out.ResourceVersion = in.ResourceVersion
	return nil
}

func DeepCopy_unversioned_ServerAddressByClientCIDR(in ServerAddressByClientCIDR, out *ServerAddressByClientCIDR, c *conversion.Cloner) error {
	out.ClientCIDR = in.ClientCIDR
	out.ServerAddress = in.ServerAddress
	return nil
}

func DeepCopy_unversioned_Time(in Time, out *Time, c *conversion.Cloner) error {
	if newVal, err := c.DeepCopy(in.Time); err != nil {
		return err
	} else {
		out.Time = newVal.(time.Time)
	}
	return nil
}

func DeepCopy_unversioned_Timestamp(in Timestamp, out *Timestamp, c *conversion.Cloner) error {
	out.Seconds = in.Seconds
	out.Nanos = in.Nanos
	return nil
}

func DeepCopy_unversioned_TypeMeta(in TypeMeta, out *TypeMeta, c *conversion.Cloner) error {
	out.Kind = in.Kind
	out.APIVersion = in.APIVersion
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"context"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/backup"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// Create a new backup.
func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instance.Instance) error {
	// Create the database entry.
	err := s.Cluster.ContainerBackupCreate(args)
	if err != nil {
		if err == db.ErrAlreadyDefined {
			return fmt.Errorf("backup '%s' already exists", args.Name)
		}

		return errors.Wrap(err, "Insert backup info into database")
	}

	revert := true
	defer func() {
		if !revert {
			return
		}
		s.Cluster.ContainerBackupRemove(args.Name)
	}()

	// Get the backup struct.
	b, err := instance.BackupLoadByName(s, sourceInst.Project(), args.Name)
	if err != nil {
		return errors.Wrap(err, "Load backup object")
	}

	b.SetCompressionAlgorithm(args.CompressionAlgorithm)

	// Create a temporary path for the backup.
	tmpPath, err := ioutil.TempDir(shared.VarPath("backups"), "lxd_backup_")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpPath)

	pool, err := storagePools.GetPoolByInstance(s, sourceInst)
	if err != nil {
		return errors.Wrap(err, "Load instance storage pool")
	}

	err = pool.BackupInstance(sourceInst, tmpPath, b.OptimizedStorage(), !b.InstanceOnly(), nil)
	if err != nil {
		return errors.Wrap(err, "Backup create")
	}

	// Pack the backup.
	err = backupCreateTarball(s, tmpPath, *b, sourceInst)
	if err != nil {
		return err
	}

	revert = false
	return nil
}

func pruneExpiredContainerBackupsTask(d *Daemon) (task.Func, task.Schedule) {
	f := func(ctx context.Context) {
		opRun := func(op *operations.Operation) error {
			return pruneExpiredContainerBackups(ctx, d)
		}

		op, err := operations.OperationCreate(d.State(), "", operations.OperationClassTask, db.OperationBackupsExpire, nil, nil, opRun, nil, nil)
		if err != nil {
			logger.Error("Failed to start expired instance backups operation", log.Ctx{"err": err})
			return
		}

		logger.Info("Pruning expired instance backups")
		_, err = op.Run()
		if err != nil {
			logger.Error("Failed to expire instance backups", log.Ctx{"err": err})
		}
		logger.Info("Done pruning expired instance backups")
	}

	f(context.Background())

	first := true
	schedule := func() (time.Duration, error) {
		interval := time.Hour

		if first {
			first = false
			return interval, task.ErrSkip
		}

		return interval, nil
	}

	return f, schedule
}

func pruneExpiredContainerBackups(ctx context.Context, d *Daemon) error {
	// Get the list of expired backups.
	backups, err := d.cluster.ContainerBackupsGetExpired()
	if err != nil {
		return errors.Wrap(err, "Unable to retrieve the list of expired instance backups")
	}

	for _, b := range backups {
		inst, err := instance.LoadByID(d.State(), b.InstanceID)
		if err != nil {
			return errors.Wrapf(err, "Error deleting instance backup %s", b.Name)
		}

		err = backup.DoBackupDelete(d.State(), inst.Project(), b.Name, inst.Name())
		if err != nil {
			return errors.Wrapf(err, "Error deleting instance backup %s", b.Name)
		}
	}

	return nil
}
<commit_msg>lxd/backup: Updates instance backup to use tar writer rather than tar cmd<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"context"

	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/backup"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/revert"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/idmap"
	"github.com/lxc/lxd/shared/instancewriter"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// Create a new backup.
func backupCreate(s *state.State, args db.InstanceBackupArgs, sourceInst instance.Instance) error {
	logger := logging.AddContext(logger.Log, log.Ctx{"project": sourceInst.Project(), "instance": sourceInst.Name(), "name": args.Name})
	logger.Debug("Instance backup started")
	defer logger.Debug("Instance backup finished")

	if sourceInst.Type() != instancetype.Container {
		return fmt.Errorf("Instance type must be container")
	}

	revert := revert.New()
	defer revert.Fail()

	// Get storage pool.
	pool, err := storagePools.GetPoolByInstance(s, sourceInst)
	if err != nil {
		return errors.Wrap(err, "Load instance storage pool")
	}

	// Create the database entry.
	err = s.Cluster.InstanceBackupCreate(args)
	if err != nil {
		if err == db.ErrAlreadyDefined {
			return fmt.Errorf("Backup %q already exists", args.Name)
		}

		return errors.Wrap(err, "Insert backup info into database")
	}

	revert.Add(func() { s.Cluster.InstanceBackupRemove(args.Name) })

	// Get the backup struct.
	b, err := instance.BackupLoadByName(s, sourceInst.Project(), args.Name)
	if err != nil {
		return errors.Wrap(err, "Load backup object")
	}

	// Detect compression method.
	var compress string
	b.SetCompressionAlgorithm(args.CompressionAlgorithm)
	if b.CompressionAlgorithm() != "" {
		compress = b.CompressionAlgorithm()
	} else {
		compress, err = cluster.ConfigGetString(s.Cluster, "backups.compression_algorithm")
		if err != nil {
			return err
		}
	}

	// Create the target path if needed.
	backupsPath := shared.VarPath("backups", project.Instance(sourceInst.Project(), sourceInst.Name()))
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}

		revert.Add(func() { os.Remove(backupsPath) })
	}

	target := shared.VarPath("backups", project.Instance(sourceInst.Project(), b.Name()))

	// Create temp dir for storing transient files that will be removed at end.
	tmpDirPath := fmt.Sprintf("%s_tmp", target)
	logger.Debug("Creating temporary backup directory", log.Ctx{"path": tmpDirPath})
	err = os.Mkdir(tmpDirPath, 0700)
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDirPath)

	// Setup the tarball writer.
	logger.Debug("Opening backup tarball for writing", log.Ctx{"path": target})
	tarFileWriter, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return errors.Wrapf(err, "Error opening backup tarball for writing %q", target)
	}
	defer tarFileWriter.Close()
	revert.Add(func() { os.Remove(target) })

	// Get IDMap to unshift container as the tarball is created.
	var idmap *idmap.IdmapSet
	if sourceInst.Type() == instancetype.Container {
		c := sourceInst.(instance.Container)
		idmap, err = c.DiskIdmap()
		if err != nil {
			return errors.Wrap(err, "Error getting container IDMAP")
		}
	}

	// Create the tarball.
	tarPipeReader, tarPipeWriter := io.Pipe()
	defer tarPipeWriter.Close() // Ensure that go routine below always ends.
	tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, idmap)

	// Setup tar writer go routine, with optional compression.
	tarWriterRes := make(chan error, 0)
	go func(resCh chan<- error) {
		logger.Debug("Started backup tarball writer")
		defer logger.Debug("Finished backup tarball writer")
		if compress != "none" {
			err = compressFile(compress, tarPipeReader, tarFileWriter)
		} else {
			_, err = io.Copy(tarFileWriter, tarPipeReader)
		}
		resCh <- err
	}(tarWriterRes)

	// Write index file.
	indexFile := filepath.Join(tmpDirPath, "index.yaml")
	logger.Debug("Adding backup index file", log.Ctx{"path": indexFile})
	err = backupWriteIndex(sourceInst, pool, b.InstanceOnly(), indexFile, tarWriter)
	if err != nil {
		return errors.Wrapf(err, "Error writing backup index file")
	}

	err = pool.BackupInstance(sourceInst, tarWriter, b.OptimizedStorage(), !b.InstanceOnly(), nil)
	if err != nil {
		return errors.Wrap(err, "Backup create")
	}

	// Close off the tarball file.
	err = tarWriter.Close()
	if err != nil {
		return errors.Wrap(err, "Error closing tarball writer")
	}

	// Close off the tarball pipe writer (this will end the go routine above).
	err = tarPipeWriter.Close()
	if err != nil {
		return errors.Wrap(err, "Error closing tarball pipe writer")
	}

	err = <-tarWriterRes
	if err != nil {
		return errors.Wrap(err, "Error writing tarball")
	}

	revert.Success()
	return nil
}

// backupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball.
func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, instanceOnly bool, indexFile string, tarWriter *instancewriter.InstanceTarWriter) error {
	indexInfo := backup.Info{
		Name:       sourceInst.Name(),
		Privileged: sourceInst.IsPrivileged(),
		Pool:       pool.Name(),
		Snapshots:  []string{},
		Backend:    pool.Driver().Info().Name,
	}

	if !instanceOnly {
		snaps, err := sourceInst.Snapshots()
		if err != nil {
			return err
		}

		for _, snap := range snaps {
			_, snapName, _ := shared.InstanceGetParentAndSnapshotName(snap.Name())
			indexInfo.Snapshots = append(indexInfo.Snapshots, snapName)
		}
	}

	// Convert to JSON.
	indexData, err := yaml.Marshal(&indexInfo)
	if err != nil {
		return err
	}

	// Write index JSON to file.
	err = ioutil.WriteFile(indexFile, indexData, 0644)
	if err != nil {
		return err
	}
	defer os.Remove(indexFile)

	indexFileInfo, err := os.Lstat(indexFile)
	if err != nil {
		return err
	}

	// Write to tarball.
	err = tarWriter.WriteFile("backup/index.yaml", indexFile, indexFileInfo)
	if err != nil {
		return err
	}

	return nil
}

func pruneExpiredContainerBackupsTask(d *Daemon) (task.Func, task.Schedule) {
	f := func(ctx context.Context) {
		opRun := func(op *operations.Operation) error {
			return pruneExpiredContainerBackups(ctx, d)
		}

		op, err := operations.OperationCreate(d.State(), "", operations.OperationClassTask, db.OperationBackupsExpire, nil, nil, opRun, nil, nil)
		if err != nil {
			logger.Error("Failed to start expired instance backups operation", log.Ctx{"err": err})
			return
		}

		logger.Info("Pruning expired instance backups")
		_, err = op.Run()
		if err != nil {
			logger.Error("Failed to expire instance backups", log.Ctx{"err": err})
		}
		logger.Info("Done pruning expired instance backups")
	}

	f(context.Background())

	first := true
	schedule := func() (time.Duration, error) {
		interval := time.Hour

		if first {
			first = false
			return interval, task.ErrSkip
		}

		return interval, nil
	}

	return f, schedule
}

func pruneExpiredContainerBackups(ctx context.Context, d *Daemon) error {
	// Get the list of expired backups.
	backups, err := d.cluster.ContainerBackupsGetExpired()
	if err != nil {
		return errors.Wrap(err, "Unable to retrieve the list of expired instance backups")
	}

	for _, b := range backups {
		inst, err := instance.LoadByID(d.State(), b.InstanceID)
		if err != nil {
			return errors.Wrapf(err, "Error deleting instance backup %s", b.Name)
		}

		err = backup.DoBackupDelete(d.State(), inst.Project(), b.Name, inst.Name())
		if err != nil {
			return errors.Wrapf(err, "Error deleting instance backup %s", b.Name)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package manager

import (
	"fmt"
	"net/http"
	"time"

	"github.com/goharbor/harbor/src/common/dao/notification"
	commonhttp "github.com/goharbor/harbor/src/common/http"
	"github.com/goharbor/harbor/src/common/models"
	"github.com/goharbor/harbor/src/lib/log"
)

// DefaultManager ...
type DefaultManager struct {
}

// NewDefaultManger ...
func NewDefaultManger() *DefaultManager {
	return &DefaultManager{}
}

// Create notification policy
func (m *DefaultManager) Create(policy *models.NotificationPolicy) (int64, error) {
	t := time.Now()
	policy.CreationTime = t
	policy.UpdateTime = t

	err := policy.ConvertToDBModel()
	if err != nil {
		return 0, err
	}
	return notification.AddNotificationPolicy(policy)
}

// List the notification policies, returns the policy list and error
func (m *DefaultManager) List(projectID int64) ([]*models.NotificationPolicy, error) {
	policies := []*models.NotificationPolicy{}
	persisPolicies, err := notification.GetNotificationPolicies(projectID)
	if err != nil {
		return nil, err
	}

	for _, policy := range persisPolicies {
		err := policy.ConvertFromDBModel()
		if err != nil {
			return nil, err
		}
		policies = append(policies, policy)
	}

	return policies, nil
}

// Get notification policy with specified ID
func (m *DefaultManager) Get(id int64) (*models.NotificationPolicy, error) {
	policy, err := notification.GetNotificationPolicy(id)
	if err != nil {
		return nil, err
	}
	if policy == nil {
		return nil, nil
	}
	err = policy.ConvertFromDBModel()
	return policy, err
}

// GetByNameAndProjectID notification policy by the name and projectID
func (m *DefaultManager) GetByNameAndProjectID(name string, projectID int64) (*models.NotificationPolicy, error) {
	policy, err := notification.GetNotificationPolicyByName(name, projectID)
	if err != nil {
		return nil, err
	}
	err = policy.ConvertFromDBModel()
	return policy, err
}

// Update the specified notification policy
func (m *DefaultManager) Update(policy *models.NotificationPolicy) error {
	policy.UpdateTime = time.Now()
	err := policy.ConvertToDBModel()
	if err != nil {
		return err
	}
	return notification.UpdateNotificationPolicy(policy)
}

// Delete the specified notification policy
func (m *DefaultManager) Delete(policyID int64) error {
	return notification.DeleteNotificationPolicy(policyID)
}

// Test the specified notification policy, just test for network connection without request body
func (m *DefaultManager) Test(policy *models.NotificationPolicy) error {
	for _, target := range policy.Targets {
		switch target.Type {
		case "http":
			return m.policyHTTPTest(target.Address, target.SkipCertVerify)
		default:
			return fmt.Errorf("invalid policy target type: %s", target.Type)
		}
	}
	return nil
}

func (m *DefaultManager) policyHTTPTest(address string, skipCertVerify bool) error {
	req, err := http.NewRequest(http.MethodPost, address, nil)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/json")
	client := http.Client{
		Transport: commonhttp.GetHTTPTransportByInsecure(skipCertVerify),
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	log.Debugf("policy test success with address %s, skip cert verify :%v", address, skipCertVerify)

	return nil
}

// GetRelatedPolices get policies including event type in project
func (m *DefaultManager) GetRelatedPolices(projectID int64, eventType string) ([]*models.NotificationPolicy, error) {
	policies, err := m.List(projectID)
	if err != nil {
		return nil, fmt.Errorf("failed to get notification policies with projectID %d: %v", projectID, err)
	}

	var result []*models.NotificationPolicy

	for _, ply := range policies {
		if !ply.Enabled {
			continue
		}
		for _, t := range ply.EventTypes {
			if t != eventType {
				continue
			}
			result = append(result, ply)
		}
	}
	return result, nil
}
<commit_msg>fix webhook slack test error<commit_after>package manager

import (
	"fmt"
	"net/http"
	"time"

	"github.com/goharbor/harbor/src/common/dao/notification"
	commonhttp "github.com/goharbor/harbor/src/common/http"
	"github.com/goharbor/harbor/src/common/models"
	"github.com/goharbor/harbor/src/lib/log"
	"github.com/goharbor/harbor/src/pkg/notifier/model"
)

// DefaultManager ...
type DefaultManager struct {
}

// NewDefaultManger ...
func NewDefaultManger() *DefaultManager {
	return &DefaultManager{}
}

// Create notification policy
func (m *DefaultManager) Create(policy *models.NotificationPolicy) (int64, error) {
	t := time.Now()
	policy.CreationTime = t
	policy.UpdateTime = t

	err := policy.ConvertToDBModel()
	if err != nil {
		return 0, err
	}
	return notification.AddNotificationPolicy(policy)
}

// List the notification policies, returns the policy list and error
func (m *DefaultManager) List(projectID int64) ([]*models.NotificationPolicy, error) {
	policies := []*models.NotificationPolicy{}
	persisPolicies, err := notification.GetNotificationPolicies(projectID)
	if err != nil {
		return nil, err
	}

	for _, policy := range persisPolicies {
		err := policy.ConvertFromDBModel()
		if err != nil {
			return nil, err
		}
		policies = append(policies, policy)
	}

	return policies, nil
}

// Get notification policy with specified ID
func (m *DefaultManager) Get(id int64) (*models.NotificationPolicy, error) {
	policy, err := notification.GetNotificationPolicy(id)
	if err != nil {
		return nil, err
	}
	if policy == nil {
		return nil, nil
	}
	err = policy.ConvertFromDBModel()
	return policy, err
}

// GetByNameAndProjectID notification policy by the name and projectID
func (m *DefaultManager) GetByNameAndProjectID(name string, projectID int64) (*models.NotificationPolicy, error) {
	policy, err := notification.GetNotificationPolicyByName(name, projectID)
	if err != nil {
		return nil, err
	}
	err = policy.ConvertFromDBModel()
	return policy, err
}

// Update the specified notification policy
func (m *DefaultManager) Update(policy *models.NotificationPolicy) error {
	policy.UpdateTime = time.Now()
	err := policy.ConvertToDBModel()
	if err != nil {
		return err
	}
	return notification.UpdateNotificationPolicy(policy)
}

// Delete the specified notification policy
func (m *DefaultManager) Delete(policyID int64) error {
	return notification.DeleteNotificationPolicy(policyID)
}

// Test the specified notification policy, just test for network connection without request body
func (m *DefaultManager) Test(policy *models.NotificationPolicy) error {
	for _, target := range policy.Targets {
		switch target.Type {
		case model.NotifyTypeHTTP, model.NotifyTypeSlack:
			return m.policyHTTPTest(target.Address, target.SkipCertVerify)
		default:
			return fmt.Errorf("invalid policy target type: %s", target.Type)
		}
	}
	return nil
}

func (m *DefaultManager) policyHTTPTest(address string, skipCertVerify bool) error {
	req, err := http.NewRequest(http.MethodPost, address, nil)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/json")
	client := http.Client{
		Transport: commonhttp.GetHTTPTransportByInsecure(skipCertVerify),
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	log.Debugf("policy test success with address %s, skip cert verify :%v", address, skipCertVerify)

	return nil
}

// GetRelatedPolices get policies including event type in project
func (m *DefaultManager) GetRelatedPolices(projectID int64, eventType string) ([]*models.NotificationPolicy, error) {
	policies, err := m.List(projectID)
	if err != nil {
		return nil, fmt.Errorf("failed to get notification policies with projectID %d: %v", projectID, err)
	}

	var result []*models.NotificationPolicy

	for _, ply := range policies {
		if !ply.Enabled {
			continue
		}
		for _, t := range ply.EventTypes {
			if t != eventType {
				continue
			}
			result = append(result, ply)
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modfetch

import (
	"errors"
	pathpkg "path"
	"sort"
	"strings"
	"time"

	"cmd/go/internal/modfetch/bitbucket"
	"cmd/go/internal/modfetch/codehost"
	"cmd/go/internal/modfetch/github"
	"cmd/go/internal/modfetch/googlesource"
	"cmd/go/internal/module"
	"cmd/go/internal/semver"
)

// A Repo represents a repository storing all versions of a single module.
type Repo interface {
	// ModulePath returns the module path.
	ModulePath() string

	// Versions lists all known versions with the given prefix.
	// Pseudo-versions are not included.
	// Versions should be returned sorted in semver order
	// (implementations can use SortVersions).
	Versions(prefix string) (tags []string, err error)

	// Stat returns information about the revision rev.
	// A revision can be any identifier known to the underlying service:
	// commit hash, branch, tag, and so on.
	Stat(rev string) (*RevInfo, error)

	// LatestAt returns the latest revision at the given time.
	// If branch is non-empty, it restricts the query to revisions
	// on the named branch. The meaning of "branch" depends
	// on the underlying implementation.
	LatestAt(t time.Time, branch string) (*RevInfo, error)

	// GoMod returns the go.mod file for the given version.
	GoMod(version string) (data []byte, err error)

	// Zip downloads a zip file for the given version
	// to a new file in a given temporary directory.
	// It returns the name of the new file.
	// The caller should remove the file when finished with it.
	Zip(version, tmpdir string) (tmpfile string, err error)
}

// A Rev describes a single revision in a module repository.
type RevInfo struct {
	Version string    // version string
	Name    string    // complete ID in underlying repository
	Short   string    // shortened ID, for use in pseudo-version
	Time    time.Time // commit time
}

// Lookup returns the module with the given module path.
func Lookup(path string) (Repo, error) {
	if proxyURL != "" {
		return lookupProxy(path)
	}
	if code, err := lookupCodeHost(path, false); err != errNotHosted {
		if err != nil {
			return nil, err
		}
		return newCodeRepo(code, path)
	}
	return lookupCustomDomain(path)
}

func Import(path string, allowed func(module.Version) bool) (Repo, *RevInfo, error) {
	try := func(path string) (Repo, *RevInfo, error) {
		r, err := Lookup(path)
		if err != nil {
			return nil, nil, err
		}
		info, err := Query(path, "latest", allowed)
		if err != nil {
			return nil, nil, err
		}
		_, err = r.GoMod(info.Version)
		if err != nil {
			return nil, nil, err
		}
		return r, info, nil
	}

	var firstErr error
	for {
		r, info, err := try(path)
		if err == nil {
			return r, info, nil
		}
		if firstErr == nil {
			firstErr = err
		}
		p := pathpkg.Dir(path)
		if p == "." {
			break
		}
		path = p
	}
	return nil, nil, firstErr
}

var errNotHosted = errors.New("not hosted")

var isTest bool

func lookupCodeHost(path string, customDomain bool) (codehost.Repo, error) {
	switch {
	case strings.HasPrefix(path, "github.com/"):
		return github.Lookup(path)
	case strings.HasPrefix(path, "bitbucket.org/"):
		// Special case Bitbucket paths ending in ".git" for backwards compatibility
		// with go get.
		path = strings.TrimSuffix(path, ".git")
		return bitbucket.Lookup(path)
	case customDomain && strings.HasSuffix(path[:strings.Index(path, "/")+1], ".googlesource.com/") ||
		isTest && strings.HasPrefix(path, "go.googlesource.com/scratch"):
		return googlesource.Lookup(path)
	case strings.HasPrefix(path, "gopkg.in/"):
		return gopkginLookup(path)
	}
	return nil, errNotHosted
}

func SortVersions(list []string) {
	sort.Slice(list, func(i, j int) bool {
		cmp := semver.Compare(list[i], list[j])
		if cmp != 0 {
			return cmp < 0
		}
		return list[i] < list[j]
	})
}
<commit_msg>cmd/go/internal/modfetch/github: trim trailing '.git' suffix in repo name<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modfetch

import (
	"errors"
	pathpkg "path"
	"sort"
	"strings"
	"time"

	"cmd/go/internal/modfetch/bitbucket"
	"cmd/go/internal/modfetch/codehost"
	"cmd/go/internal/modfetch/github"
	"cmd/go/internal/modfetch/googlesource"
	"cmd/go/internal/module"
	"cmd/go/internal/semver"
)

// A Repo represents a repository storing all versions of a single module.
type Repo interface {
	// ModulePath returns the module path.
	ModulePath() string

	// Versions lists all known versions with the given prefix.
	// Pseudo-versions are not included.
	// Versions should be returned sorted in semver order
	// (implementations can use SortVersions).
	Versions(prefix string) (tags []string, err error)

	// Stat returns information about the revision rev.
	// A revision can be any identifier known to the underlying service:
	// commit hash, branch, tag, and so on.
	Stat(rev string) (*RevInfo, error)

	// LatestAt returns the latest revision at the given time.
	// If branch is non-empty, it restricts the query to revisions
	// on the named branch. The meaning of "branch" depends
	// on the underlying implementation.
	LatestAt(t time.Time, branch string) (*RevInfo, error)

	// GoMod returns the go.mod file for the given version.
	GoMod(version string) (data []byte, err error)

	// Zip downloads a zip file for the given version
	// to a new file in a given temporary directory.
	// It returns the name of the new file.
	// The caller should remove the file when finished with it.
	Zip(version, tmpdir string) (tmpfile string, err error)
}

// A Rev describes a single revision in a module repository.
type RevInfo struct {
	Version string    // version string
	Name    string    // complete ID in underlying repository
	Short   string    // shortened ID, for use in pseudo-version
	Time    time.Time // commit time
}

// Lookup returns the module with the given module path.
func Lookup(path string) (Repo, error) {
	if proxyURL != "" {
		return lookupProxy(path)
	}
	if code, err := lookupCodeHost(path, false); err != errNotHosted {
		if err != nil {
			return nil, err
		}
		return newCodeRepo(code, path)
	}
	return lookupCustomDomain(path)
}

func Import(path string, allowed func(module.Version) bool) (Repo, *RevInfo, error) {
	try := func(path string) (Repo, *RevInfo, error) {
		r, err := Lookup(path)
		if err != nil {
			return nil, nil, err
		}
		info, err := Query(path, "latest", allowed)
		if err != nil {
			return nil, nil, err
		}
		_, err = r.GoMod(info.Version)
		if err != nil {
			return nil, nil, err
		}
		return r, info, nil
	}

	var firstErr error
	for {
		r, info, err := try(path)
		if err == nil {
			return r, info, nil
		}
		if firstErr == nil {
			firstErr = err
		}
		p := pathpkg.Dir(path)
		if p == "." {
			break
		}
		path = p
	}
	return nil, nil, firstErr
}

var errNotHosted = errors.New("not hosted")

var isTest bool

func lookupCodeHost(path string, customDomain bool) (codehost.Repo, error) {
	switch {
	case strings.HasPrefix(path, "github.com/"):
		// Special case GitHub paths ending in ".git" for backwards compatibility
		// with go get.
		path = strings.TrimSuffix(path, ".git")
		return github.Lookup(path)
	case strings.HasPrefix(path, "bitbucket.org/"):
		// Special case Bitbucket paths ending in ".git" for backwards compatibility
		// with go get.
		path = strings.TrimSuffix(path, ".git")
		return bitbucket.Lookup(path)
	case customDomain && strings.HasSuffix(path[:strings.Index(path, "/")+1], ".googlesource.com/") ||
		isTest && strings.HasPrefix(path, "go.googlesource.com/scratch"):
		return googlesource.Lookup(path)
	case strings.HasPrefix(path, "gopkg.in/"):
		return gopkginLookup(path)
	}
	return nil, errNotHosted
}

func SortVersions(list []string) {
	sort.Slice(list, func(i, j int) bool {
		cmp := semver.Compare(list[i], list[j])
		if cmp != 0 {
			return cmp < 0
		}
		return list[i] < list[j]
	})
}
<|endoftext|>


<commit_before>// +build lambdabinary

package cloudwatch

import (
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	awsCloudWatch "github.com/aws/aws-sdk-go/service/cloudwatch"
	sparta "github.com/mweagle/Sparta"
	spartaAWS "github.com/mweagle/Sparta/aws"
	"github.com/rs/zerolog"
	gopsutilCPU "github.com/shirou/gopsutil/cpu"
	gopsutilDisk "github.com/shirou/gopsutil/disk"
	gopsutilHost "github.com/shirou/gopsutil/host"
	gopsutilLoad "github.com/shirou/gopsutil/load"
	gopsutilNet "github.com/shirou/gopsutil/net"
)

// publishMetrics is the actual metric publishing logic. T
func publishMetrics(customDimensionMap map[string]string) {
	currentTime := time.Now()

	// https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html
	functionName := os.Getenv("AWS_LAMBDA_FUNCTION_NAME")
	cpuMetrics, cpuMetricsErr := gopsutilCPU.Percent(0, false)
	// https://docs.aws.amazon.com/lambda/latest/dg/limits.html
	diskMetrics, diskMetricsErr := gopsutilDisk.Usage("/tmp")
	uptime, uptimeErr := gopsutilHost.Uptime()
	loadMetrics, loadMetricsErr := gopsutilLoad.Avg()
	netMetrics, netMetricsErr := gopsutilNet.IOCounters(false)

	// For now, just log everything...
	logger, _ := sparta.NewLogger(zerolog.InfoLevel.String())
	if logger != nil {
		logger.Info().
			Str("functionName", functionName).
			Interface("cpuMetrics", cpuMetrics).
			Interface("cpuMetricsErr", cpuMetricsErr).
			Interface("diskMetrics", diskMetrics).
			Interface("diskMetricsErr", diskMetricsErr).
			Interface("uptime", uptime).
			Interface("uptimeErr", uptimeErr).
			Interface("loadMetrics", loadMetrics).
			Interface("loadMetricsErr", loadMetricsErr).
			Interface("netMetrics", netMetrics).
			Interface("netMetricsErr", netMetricsErr).
			Msg("Metric info")
	}
	// Return the array of metricDatum for the item
	metricDatum := func(name string, value float64, unit MetricUnit) []*awsCloudWatch.MetricDatum {
		defaultDatum := []*awsCloudWatch.MetricDatum{{
			MetricName: aws.String(name),
			Dimensions: []*awsCloudWatch.Dimension{{
				Name:  aws.String("Name"),
				Value: aws.String(sparta.StampedServiceName),
			}},
			Value:     aws.Float64(value),
			Timestamp: &currentTime,
			Unit:      aws.String(string(unit)),
		},
		}
		if len(customDimensionMap) != 0 {
			metricDimension := []*awsCloudWatch.Dimension{{
				Name:  aws.String("Name"),
				Value: aws.String(sparta.StampedServiceName),
			}}
			for eachKey, eachValue := range customDimensionMap {
				metricDimension = append(metricDimension, &awsCloudWatch.Dimension{
					Name:  aws.String(eachKey),
					Value: aws.String(eachValue),
				})
			}
			defaultDatum = append(defaultDatum, &awsCloudWatch.MetricDatum{
				MetricName: aws.String(name),
				Dimensions: metricDimension,
				Value:      aws.Float64(value),
				Timestamp:  &currentTime,
				Unit:       aws.String(string(unit)),
			})
		}
		return defaultDatum
	}
	// Publish all the metrics...
	// https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html
	metricData := []*awsCloudWatch.MetricDatum{}
	// CPU?
	if len(cpuMetrics) == 1 {
		metricData = append(metricData, metricDatum("CPUPercent", cpuMetrics[0], UnitPercent)...)
	}
	if diskMetricsErr == nil {
		metricData = append(metricData, metricDatum("DiskUsedPercent", diskMetrics.UsedPercent, UnitPercent)...)
	}
	if uptimeErr == nil {
		metricData = append(metricData, metricDatum("Uptime", float64(uptime), UnitMilliseconds)...)
	}
	if loadMetricsErr == nil {
		metricData = append(metricData, metricDatum("Load1", loadMetrics.Load1, UnitNone)...)
		metricData = append(metricData, metricDatum("Load5", loadMetrics.Load5, UnitNone)...)
		metricData = append(metricData, metricDatum("Load15", loadMetrics.Load15, UnitNone)...)
	}
	if netMetricsErr == nil && len(netMetrics) == 1 {
		metricData = append(metricData, metricDatum("NetBytesSent", float64(netMetrics[0].BytesSent), UnitBytes)...)
		metricData = append(metricData, metricDatum("NetBytesRecv", float64(netMetrics[0].BytesRecv), UnitBytes)...)
		metricData = append(metricData, metricDatum("NetErrin", float64(netMetrics[0].Errin), UnitCount)...)
		metricData = append(metricData, metricDatum("NetErrout", float64(netMetrics[0].Errout), UnitCount)...)
	}
	putMetricInput := &awsCloudWatch.PutMetricDataInput{
		MetricData: metricData,
		Namespace:  aws.String(sparta.ProperName),
	}
	session := spartaAWS.NewSession(logger)
	awsCloudWatchSvc := awsCloudWatch.New(session)
	putMetricResponse, putMetricResponseErr := awsCloudWatchSvc.PutMetricData(putMetricInput)
	if putMetricResponseErr != nil {
		logger.Error().Err(putMetricResponseErr).Msg("Failed to submit CloudWatch Metric data")
	} else {
		logger.Info().Interface("Response", putMetricResponse).Msg("CloudWatch Metric response")
	}
}

// RegisterLambdaUtilizationMetricPublisher installs a periodic task
// to publish the current system metrics to CloudWatch Metrics. See
// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html
// for more information.
func RegisterLambdaUtilizationMetricPublisher(customDimensionMap map[string]string) {

	// Publish when we start
	publishMetrics(customDimensionMap)

	ticker := time.NewTicker(1 * time.Minute)
	quit := make(chan struct{})
	go func() {
		for {
			select {
			case <-ticker.C:
				publishMetrics(customDimensionMap)
			case <-quit:
				ticker.Stop()
				return
			}
		}
	}()
}
<commit_msg>Upgrade to v3 of gopsutil<commit_after>// +build lambdabinary

package cloudwatch

import (
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	awsCloudWatch "github.com/aws/aws-sdk-go/service/cloudwatch"
	sparta "github.com/mweagle/Sparta"
	spartaAWS "github.com/mweagle/Sparta/aws"
	"github.com/rs/zerolog"
	gopsutilCPU "github.com/shirou/gopsutil/v3/cpu"
	gopsutilDisk "github.com/shirou/gopsutil/v3/disk"
	gopsutilHost "github.com/shirou/gopsutil/v3/host"
	gopsutilLoad "github.com/shirou/gopsutil/v3/load"
	gopsutilNet "github.com/shirou/gopsutil/v3/net"
)

// publishMetrics is the actual metric publishing logic. T
func publishMetrics(customDimensionMap map[string]string) {
	currentTime := time.Now()

	// https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html
	functionName := os.Getenv("AWS_LAMBDA_FUNCTION_NAME")
	cpuMetrics, cpuMetricsErr := gopsutilCPU.Percent(0, false)
	// https://docs.aws.amazon.com/lambda/latest/dg/limits.html
	diskMetrics, diskMetricsErr := gopsutilDisk.Usage("/tmp")
	uptime, uptimeErr := gopsutilHost.Uptime()
	loadMetrics, loadMetricsErr := gopsutilLoad.Avg()
	netMetrics, netMetricsErr := gopsutilNet.IOCounters(false)

	// For now, just log everything...
	logger, _ := sparta.NewLogger(zerolog.InfoLevel.String())
	if logger != nil {
		logger.Info().
			Str("functionName", functionName).
			Interface("cpuMetrics", cpuMetrics).
			Interface("cpuMetricsErr", cpuMetricsErr).
			Interface("diskMetrics", diskMetrics).
			Interface("diskMetricsErr", diskMetricsErr).
			Interface("uptime", uptime).
			Interface("uptimeErr", uptimeErr).
			Interface("loadMetrics", loadMetrics).
			Interface("loadMetricsErr", loadMetricsErr).
			Interface("netMetrics", netMetrics).
			Interface("netMetricsErr", netMetricsErr).
			Msg("Metric info")
	}
	// Return the array of metricDatum for the item
	metricDatum := func(name string, value float64, unit MetricUnit) []*awsCloudWatch.MetricDatum {
		defaultDatum := []*awsCloudWatch.MetricDatum{{
			MetricName: aws.String(name),
			Dimensions: []*awsCloudWatch.Dimension{{
				Name:  aws.String("Name"),
				Value: aws.String(sparta.StampedServiceName),
			}},
			Value:     aws.Float64(value),
			Timestamp: &currentTime,
			Unit:      aws.String(string(unit)),
		},
		}
		if len(customDimensionMap) != 0 {
			metricDimension := []*awsCloudWatch.Dimension{{
				Name:  aws.String("Name"),
				Value: aws.String(sparta.StampedServiceName),
			}}
			for eachKey, eachValue := range customDimensionMap {
				metricDimension = append(metricDimension, &awsCloudWatch.Dimension{
					Name:  aws.String(eachKey),
					Value: aws.String(eachValue),
				})
			}
			defaultDatum = append(defaultDatum, &awsCloudWatch.MetricDatum{
				MetricName: aws.String(name),
				Dimensions: metricDimension,
				Value:      aws.Float64(value),
				Timestamp:  &currentTime,
				Unit:       aws.String(string(unit)),
			})
		}
		return defaultDatum
	}
	// Publish all the metrics...
	// https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html
	metricData := []*awsCloudWatch.MetricDatum{}
	// CPU?
	if len(cpuMetrics) == 1 {
		metricData = append(metricData, metricDatum("CPUPercent", cpuMetrics[0], UnitPercent)...)
	}
	if diskMetricsErr == nil {
		metricData = append(metricData, metricDatum("DiskUsedPercent", diskMetrics.UsedPercent, UnitPercent)...)
	}
	if uptimeErr == nil {
		metricData = append(metricData, metricDatum("Uptime", float64(uptime), UnitMilliseconds)...)
	}
	if loadMetricsErr == nil {
		metricData = append(metricData, metricDatum("Load1", loadMetrics.Load1, UnitNone)...)
		metricData = append(metricData, metricDatum("Load5", loadMetrics.Load5, UnitNone)...)
		metricData = append(metricData, metricDatum("Load15", loadMetrics.Load15, UnitNone)...)
	}
	if netMetricsErr == nil && len(netMetrics) == 1 {
		metricData = append(metricData, metricDatum("NetBytesSent", float64(netMetrics[0].BytesSent), UnitBytes)...)
		metricData = append(metricData, metricDatum("NetBytesRecv", float64(netMetrics[0].BytesRecv), UnitBytes)...)
		metricData = append(metricData, metricDatum("NetErrin", float64(netMetrics[0].Errin), UnitCount)...)
		metricData = append(metricData, metricDatum("NetErrout", float64(netMetrics[0].Errout), UnitCount)...)
	}
	putMetricInput := &awsCloudWatch.PutMetricDataInput{
		MetricData: metricData,
		Namespace:  aws.String(sparta.ProperName),
	}
	session := spartaAWS.NewSession(logger)
	awsCloudWatchSvc := awsCloudWatch.New(session)
	putMetricResponse, putMetricResponseErr := awsCloudWatchSvc.PutMetricData(putMetricInput)
	if putMetricResponseErr != nil {
		logger.Error().Err(putMetricResponseErr).Msg("Failed to submit CloudWatch Metric data")
	} else {
		logger.Info().Interface("Response", putMetricResponse).Msg("CloudWatch Metric response")
	}
}

// RegisterLambdaUtilizationMetricPublisher installs a periodic task
// to publish the current system metrics to CloudWatch Metrics. See
// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html
// for more information.
func RegisterLambdaUtilizationMetricPublisher(customDimensionMap map[string]string) {

	// Publish when we start
	publishMetrics(customDimensionMap)

	ticker := time.NewTicker(1 * time.Minute)
	quit := make(chan struct{})
	go func() {
		for {
			select {
			case <-ticker.C:
				publishMetrics(customDimensionMap)
			case <-quit:
				ticker.Stop()
				return
			}
		}
	}()
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"qiniu/api.v6/auth/digest"
	"qiniu/api.v6/conf"
	fio "qiniu/api.v6/io"
	rio "qiniu/api.v6/resumable/io"
	"qiniu/api.v6/rs"
	"qiniu/rpc"
	"qshell"
	"strconv"
	"strings"
	"sync"
	"time"
)

type PutRet struct {
	Key      string `json:"key"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
}

var upSettings = rio.Settings{
	ChunkSize: 4 * 1024 * 1024,
	TryTimes:  3,
}

func FormPut(cmd string, params ...string) {
	if len(params) >= 3 && len(params) <= 7 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		upHost := ""
		overwrite := false
		fileType := 0

		optionalParams := params[3:]
		for _, param := range optionalParams {

			if ft, err := strconv.Atoi(param); err == nil {
				if ft == 1 || ft == 0 {
					fileType = ft
					continue
				} else {
					fmt.Println("Wrong Filetype, It should be 0 or 1 ")
					os.Exit(qshell.STATUS_ERROR)
				}
			}
			if val, pErr := strconv.ParseBool(param); pErr == nil {
				overwrite = val
				continue
			}
			if strings.HasPrefix(param, "http://") || strings.HasPrefix(param, "https://") {
				upHost = param
				continue
			}

			mimeType = param
		}

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		//upload settings
		mac := digest.Mac{account.AccessKey, []byte(account.SecretKey)}
		if upHost == "" {
			//get bucket zone info
			bucketInfo, gErr := qshell.GetBucketInfo(&mac, bucket)
			if gErr != nil {
				fmt.Println("Get bucket region info error,", gErr)
				os.Exit(qshell.STATUS_ERROR)
			}

			//set up host
			qshell.SetZone(bucketInfo.Region)
		} else {
			conf.UP_HOST = upHost
		}

		//create uptoken
		policy := rs.PutPolicy{}
		if overwrite {
			policy.Scope = fmt.Sprintf("%s:%s", bucket, key)
		} else {
			policy.Scope = bucket
		}
		policy.FileType = fileType
		policy.Expires = 7 * 24 * 3600
		policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`
		putExtra := fio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}

		uptoken := policy.Token(&mac)

		//start to upload
		putRet := PutRet{}
		startTime := time.Now()
		fStat, statErr := os.Stat(localFile)
		if statErr != nil {
			fmt.Println("Local file error", statErr)
			os.Exit(qshell.STATUS_ERROR)
		}
		fsize := fStat.Size()
		putClient := rpc.NewClient("")
		fmt.Printf("Uploading %s => %s : %s ...\n", localFile, bucket, key)
		doneSignal := make(chan bool)
		go func(ch chan bool) {
			progressSigns := []string{"|", "/", "-", "\\", "|"}
			for {
				for _, p := range progressSigns {
					fmt.Print("\rProgress: ", p)
					os.Stdout.Sync()
					select {
					case <-ch:
						return
					case <-time.After(time.Millisecond * 50):
						continue
					}
				}
			}
		}(doneSignal)

		err := fio.PutFile(putClient, nil, &putRet, uptoken, key, localFile, &putExtra)
		doneSignal <- true
		fmt.Print("\rProgress: 100%")
		os.Stdout.Sync()
		fmt.Println()

		if err != nil {
			if v, ok := err.(*rpc.ErrorInfo); ok {
				fmt.Printf("Put file error, %d %s, Reqid: %s\n", v.Code, v.Err, v.Reqid)
			} else {
				fmt.Println("Put file error,", err)
			}
		} else {
			fmt.Println("Put file", localFile, "=>", bucket, ":", putRet.Key, "success!")
			fmt.Println("Hash:", putRet.Hash)
			fmt.Println("Fsize:", putRet.Fsize, "(", FormatFsize(fsize), ")")
			fmt.Println("MimeType:", putRet.MimeType)
		}
		lastNano := time.Now().UnixNano() - startTime.UnixNano()
		lastTime := fmt.Sprintf("%.2f", float32(lastNano)/1e9)
		avgSpeed := fmt.Sprintf("%.1f", float32(fsize)*1e6/float32(lastNano))
		fmt.Println("Last time:", lastTime, "s, Average Speed:", avgSpeed, "KB/s")

		if err != nil {
			os.Exit(qshell.STATUS_ERROR)
		}
	} else {
		CmdHelp(cmd)
	}
}

func ResumablePut(cmd string, params ...string) {
	if len(params) >= 3 && len(params) <= 7 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		upHost := ""
		overwrite := false
		fileType := 0

		optionalParams := params[3:]
		for _, param := range optionalParams {

			if ft, err := strconv.Atoi(param); err == nil {
				if ft == 1 || ft == 0 {
					fileType = ft
					continue
				} else {
					fmt.Println("Wrong Filetype, It should be 0 or 1 ")
					os.Exit(qshell.STATUS_ERROR)
				}

			}
			if val, pErr := strconv.ParseBool(param); pErr == nil {
				overwrite = val
				continue
			}
			if strings.HasPrefix(param, "http://") || strings.HasPrefix(param, "https://") {
				upHost = param
				continue
			}

			mimeType = param
		}

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		fStat, statErr := os.Stat(localFile)
		if statErr != nil {
			fmt.Println("Local file error", statErr)
			os.Exit(qshell.STATUS_ERROR)
		}
		fsize := fStat.Size()

		//upload settings
		mac := digest.Mac{account.AccessKey, []byte(account.SecretKey)}
		if upHost == "" {
			//get bucket zone info
			bucketInfo, gErr := qshell.GetBucketInfo(&mac, bucket)
			if gErr != nil {
				fmt.Println("Get bucket region info error,", gErr)
				os.Exit(qshell.STATUS_ERROR)
			}

			//set up host
			qshell.SetZone(bucketInfo.Region)
		} else {
			conf.UP_HOST = upHost
		}
		rio.SetSettings(&upSettings)

		//create uptoken
		policy := rs.PutPolicy{}
		if overwrite {
			policy.Scope = fmt.Sprintf("%s:%s", bucket, key)
		} else {
			policy.Scope = bucket
		}
		policy.FileType = fileType
		policy.Expires = 7 * 24 * 3600
		policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`

		putExtra := rio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}

		progressHandler := ProgressHandler{
			rwLock:  &sync.RWMutex{},
			fsize:   fsize,
			offsets: make(map[int]int64, 0),
		}

		putExtra.Notify = progressHandler.Notify
		putExtra.NotifyErr = progressHandler.NotifyErr
		uptoken := policy.Token(&mac)

		//start to upload
		putRet := PutRet{}
		startTime := time.Now()

		putClient := rio.NewClient(uptoken, "")
		fmt.Printf("Uploading %s => %s : %s ...\n", localFile, bucket, key)
		err := rio.PutFile(putClient, nil, &putRet, key, localFile, &putExtra)
		fmt.Println()
		if err != nil {
			if v, ok := err.(*rpc.ErrorInfo); ok {
				fmt.Printf("Put file error, %d %s, Reqid: %s\n", v.Code, v.Err, v.Reqid)
			} else {
				fmt.Println("Put file error,", err)
			}
		} else {
			fmt.Println("Put file", localFile, "=>", bucket, ":", putRet.Key, "success!")
			fmt.Println("Hash:", putRet.Hash)
			fmt.Println("Fsize:", putRet.Fsize, "(", FormatFsize(fsize), ")")
			fmt.Println("MimeType:", putRet.MimeType)
		}
		lastNano := time.Now().UnixNano() - startTime.UnixNano()
		lastTime := fmt.Sprintf("%.2f", float32(lastNano)/1e9)
		avgSpeed := fmt.Sprintf("%.1f", float32(fsize)*1e6/float32(lastNano))
		fmt.Println("Last time:", lastTime, "s, Average Speed:", avgSpeed, "KB/s")

		if err != nil {
			os.Exit(qshell.STATUS_ERROR)
		}
	} else {
		CmdHelp(cmd)
	}
}

type ProgressHandler struct {
	rwLock  *sync.RWMutex
	offsets map[int]int64
	fsize   int64
}

func (this *ProgressHandler) Notify(blkIdx int, blkSize int, ret *rio.BlkputRet) {
	this.rwLock.Lock()
	defer this.rwLock.Unlock()

	this.offsets[blkIdx] = int64(ret.Offset)
	var uploaded int64
	for _, offset := range this.offsets {
		uploaded += offset
	}

	percent := fmt.Sprintf("\rProgress: %.2f%%", float64(uploaded)/float64(this.fsize)*100)
	fmt.Print(percent)
	os.Stdout.Sync()
}

func (this *ProgressHandler) NotifyErr(blkIdx int, blkSize int, err error) {

}
<commit_msg>update rput block put routine count<commit_after>package cli

import (
	"fmt"
	"os"
	"qiniu/api.v6/auth/digest"
	"qiniu/api.v6/conf"
	fio "qiniu/api.v6/io"
	rio "qiniu/api.v6/resumable/io"
	"qiniu/api.v6/rs"
	"qiniu/rpc"
	"qshell"
	"strconv"
	"strings"
	"sync"
	"time"
)

type PutRet struct {
	Key      string `json:"key"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
}

var upSettings = rio.Settings{
	Workers:   16,
	ChunkSize: 4 * 1024 * 1024,
	TryTimes:  3,
}

func FormPut(cmd string, params ...string) {
	if len(params) >= 3 && len(params) <= 7 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		upHost := ""
		overwrite := false
		fileType := 0

		optionalParams := params[3:]
		for _, param := range optionalParams {

			if ft, err := strconv.Atoi(param); err == nil {
				if ft == 1 || ft == 0 {
					fileType = ft
					continue
				} else {
					fmt.Println("Wrong Filetype, It should be 0 or 1 ")
					os.Exit(qshell.STATUS_ERROR)
				}
			}
			if val, pErr := strconv.ParseBool(param); pErr == nil {
				overwrite = val
				continue
			}
			if strings.HasPrefix(param, "http://") || strings.HasPrefix(param, "https://") {
				upHost = param
				continue
			}

			mimeType = param
		}

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		//upload settings
		mac := digest.Mac{account.AccessKey, []byte(account.SecretKey)}
		if upHost == "" {
			//get bucket zone info
			bucketInfo, gErr := qshell.GetBucketInfo(&mac, bucket)
			if gErr != nil {
				fmt.Println("Get bucket region info error,", gErr)
				os.Exit(qshell.STATUS_ERROR)
			}

			//set up host
			qshell.SetZone(bucketInfo.Region)
		} else {
			conf.UP_HOST = upHost
		}

		//create uptoken
		policy := rs.PutPolicy{}
		if overwrite {
			policy.Scope = fmt.Sprintf("%s:%s", bucket, key)
		} else {
			policy.Scope = bucket
		}
		policy.FileType = fileType
		policy.Expires = 7 * 24 * 3600
		policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`
		putExtra := fio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}

		uptoken := policy.Token(&mac)

		//start to upload
		putRet := PutRet{}
		startTime := time.Now()
		fStat, statErr := os.Stat(localFile)
		if statErr != nil {
			fmt.Println("Local file error", statErr)
			os.Exit(qshell.STATUS_ERROR)
		}
		fsize := fStat.Size()
		putClient := rpc.NewClient("")
		fmt.Printf("Uploading %s => %s : %s ...\n", localFile, bucket, key)
		doneSignal := make(chan bool)
		go func(ch chan bool) {
			progressSigns := []string{"|", "/", "-", "\\", "|"}
			for {
				for _, p := range progressSigns {
					fmt.Print("\rProgress: ", p)
					os.Stdout.Sync()
					select {
					case <-ch:
						return
					case <-time.After(time.Millisecond * 50):
						continue
					}
				}
			}
		}(doneSignal)

		err := fio.PutFile(putClient, nil, &putRet, uptoken, key, localFile, &putExtra)
		doneSignal <- true
		fmt.Print("\rProgress: 100%")
		os.Stdout.Sync()
		fmt.Println()

		if err != nil {
			if v, ok := err.(*rpc.ErrorInfo); ok {
				fmt.Printf("Put file error, %d %s, Reqid: %s\n", v.Code, v.Err, v.Reqid)
			} else {
				fmt.Println("Put file error,", err)
			}
		} else {
			fmt.Println("Put file", localFile, "=>", bucket, ":", putRet.Key, "success!")
			fmt.Println("Hash:", putRet.Hash)
			fmt.Println("Fsize:", putRet.Fsize, "(", FormatFsize(fsize), ")")
			fmt.Println("MimeType:", putRet.MimeType)
		}
		lastNano := time.Now().UnixNano() - startTime.UnixNano()
		lastTime := fmt.Sprintf("%.2f", float32(lastNano)/1e9)
		avgSpeed := fmt.Sprintf("%.1f", float32(fsize)*1e6/float32(lastNano))
		fmt.Println("Last time:", lastTime, "s, Average Speed:", avgSpeed, "KB/s")

		if err != nil {
			os.Exit(qshell.STATUS_ERROR)
		}
	} else {
		CmdHelp(cmd)
	}
}

func ResumablePut(cmd string, params ...string) {
	if len(params) >= 3 && len(params) <= 7 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		upHost := ""
		overwrite := false
		fileType := 0

		optionalParams := params[3:]
		for _, param := range optionalParams {

			if ft, err := strconv.Atoi(param); err == nil {
				if ft == 1 || ft == 0 {
					fileType = ft
					continue
				} else {
					fmt.Println("Wrong Filetype, It should be 0 or 1 ")
					os.Exit(qshell.STATUS_ERROR)
				}

			}
			if val, pErr := strconv.ParseBool(param); pErr == nil {
				overwrite = val
				continue
			}
			if strings.HasPrefix(param, "http://") || strings.HasPrefix(param, "https://") {
				upHost = param
				continue
			}

			mimeType = param
		}

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		fStat, statErr := os.Stat(localFile)
		if statErr != nil {
			fmt.Println("Local file error", statErr)
			os.Exit(qshell.STATUS_ERROR)
		}
		fsize := fStat.Size()

		//upload settings
		mac := digest.Mac{account.AccessKey, []byte(account.SecretKey)}
		if upHost == "" {
			//get bucket zone info
			bucketInfo, gErr := qshell.GetBucketInfo(&mac, bucket)
			if gErr != nil {
				fmt.Println("Get bucket region info error,", gErr)
				os.Exit(qshell.STATUS_ERROR)
			}

			//set up host
			qshell.SetZone(bucketInfo.Region)
		} else {
			conf.UP_HOST = upHost
		}
		rio.SetSettings(&upSettings)

		//create uptoken
		policy := rs.PutPolicy{}
		if overwrite {
			policy.Scope = fmt.Sprintf("%s:%s", bucket, key)
		} else {
			policy.Scope = bucket
		}
		policy.FileType = fileType
		policy.Expires = 7 * 24 * 3600
		policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`

		putExtra := rio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}

		progressHandler := ProgressHandler{
			rwLock:  &sync.RWMutex{},
			fsize:   fsize,
			offsets: make(map[int]int64, 0),
		}

		putExtra.Notify = progressHandler.Notify
		putExtra.NotifyErr = progressHandler.NotifyErr
		uptoken := policy.Token(&mac)

		//start to upload
		putRet := PutRet{}
		startTime := time.Now()

		putClient := rio.NewClient(uptoken, "")
		fmt.Printf("Uploading %s => %s : %s ...\n", localFile, bucket, key)
		err := rio.PutFile(putClient, nil, &putRet, key, localFile, &putExtra)
		fmt.Println()
		if err != nil {
			if v, ok := err.(*rpc.ErrorInfo); ok {
				fmt.Printf("Put file error, %d %s, Reqid: %s\n", v.Code, v.Err, v.Reqid)
			} else {
				fmt.Println("Put file error,", err)
			}
		} else {
			fmt.Println("Put file", localFile, "=>", bucket, ":", putRet.Key, "success!")
			fmt.Println("Hash:", putRet.Hash)
			fmt.Println("Fsize:", putRet.Fsize, "(", FormatFsize(fsize), ")")
			fmt.Println("MimeType:", putRet.MimeType)
		}
		lastNano := time.Now().UnixNano() - startTime.UnixNano()
		lastTime := fmt.Sprintf("%.2f", float32(lastNano)/1e9)
		avgSpeed := fmt.Sprintf("%.1f", float32(fsize)*1e6/float32(lastNano))
		fmt.Println("Last time:", lastTime, "s, Average Speed:", avgSpeed, "KB/s")

		if err != nil {
			os.Exit(qshell.STATUS_ERROR)
		}
	} else {
		CmdHelp(cmd)
	}
}

type ProgressHandler struct {
	rwLock  *sync.RWMutex
	offsets map[int]int64
	fsize   int64
}

func (this *ProgressHandler) Notify(blkIdx int, blkSize int, ret *rio.BlkputRet) {
	this.rwLock.Lock()
	defer this.rwLock.Unlock()

	this.offsets[blkIdx] = int64(ret.Offset)
	var uploaded int64
	for _, offset := range this.offsets {
		uploaded += offset
	}

	percent := fmt.Sprintf("\rProgress: %.2f%%", float64(uploaded)/float64(this.fsize)*100)
	fmt.Print(percent)
	os.Stdout.Sync()
}

func (this *ProgressHandler) NotifyErr(blkIdx int, blkSize int, err error) {

}
<|endoftext|>