<commit_before>package router

import (
	"github.com/appleboy/gin-jwt"
	"github.com/gin-gonic/gin"
	"github.com/orange-jacky/albums/data"
	"github.com/orange-jacky/albums/util"
	"time"
)

func GetAuthMiddleware() *jwt.GinJWTMiddleware {
	return &jwt.GinJWTMiddleware{
		Realm:      "test zone",
		Key:        []byte("secret key"),
		Timeout:    time.Hour,
		MaxRefresh: time.Hour,
		Authenticator: func(username string, password string, c *gin.Context) (string, bool) {
			user := util.GetUser()
			err := user.CheckUser(username)
			if err != nil {
				return username, false
			}
			if err != nil {
				return username, false
			}
			return username, true
		},
		Authorizator: func(username string, c *gin.Context) bool {
			user := util.GetUser()
			if err := user.CheckUser(username); err != nil {
				return false
			}
			return true
		},
		Unauthorized: func(c *gin.Context, code int, message string) {
			resp := data.Response{Status: code, Data: message}
			c.JSON(code, resp)
		},
		// TokenLookup is a string in the form of "<source>:<name>" that is used
		// to extract token from the request.
		// Optional. Default value "header:Authorization".
		// Possible values:
		// - "header:<name>"
		// - "query:<name>"
		// - "cookie:<name>"
		TokenLookup: "header:Authorization",
		// TokenLookup: "query:token",
		// TokenLookup: "cookie:token",

		// TokenHeadName is a string in the header. Default value is "Bearer"
		TokenHeadName: "Bearer",
		// TimeFunc provides the current time. You can override it to use another time value. This is useful for testing or if your server uses a different time zone than your tokens.
		TimeFunc: time.Now,
	}
}
<commit_msg>jwt未授权的, http 响应状态改成200,真实错误状态放在status字段里<commit_after>package router

import (
	"github.com/appleboy/gin-jwt"
	"github.com/gin-gonic/gin"
	"github.com/orange-jacky/albums/data"
	"github.com/orange-jacky/albums/util"
	"net/http"
	"time"
)

func GetAuthMiddleware() *jwt.GinJWTMiddleware {
	return &jwt.GinJWTMiddleware{
		Realm:      "test zone",
		Key:        []byte("secret key"),
		Timeout:    time.Hour,
		MaxRefresh: time.Hour,
		Authenticator: func(username string, password string, c *gin.Context) (string, bool) {
			user := util.GetUser()
			err := user.CheckUser(username)
			if err != nil {
				return username, false
			}
			if err != nil {
				return username, false
			}
			return username, true
		},
		Authorizator: func(username string, c *gin.Context) bool {
			user := util.GetUser()
			if err := user.CheckUser(username); err != nil {
				return false
			}
			return true
		},
		Unauthorized: func(c *gin.Context, code int, message string) {
			resp := data.Response{Status: code, Data: message}
			//c.JSON(code, resp)
			//没授权的http resp状态也为200, 具体错误信息再resp中
			c.JSON(http.StatusOK, resp)
		},
		// TokenLookup is a string in the form of "<source>:<name>" that is used
		// to extract token from the request.
		// Optional. Default value "header:Authorization".
		// Possible values:
		// - "header:<name>"
		// - "query:<name>"
		// - "cookie:<name>"
		TokenLookup: "header:Authorization",
		// TokenLookup: "query:token",
		// TokenLookup: "cookie:token",

		// TokenHeadName is a string in the header. Default value is "Bearer"
		TokenHeadName: "Bearer",
		// TimeFunc provides the current time. You can override it to use another time value. This is useful for testing or if your server uses a different time zone than your tokens.
		TimeFunc: time.Now,
	}
}
<|endoftext|>


<commit_before>package mux

import (
	"bytes"
	"io"
	"net/http"
	"net/http/httptest"
	"testing"
)

type PathTest struct {
	title string
	path  string
}

func TestHost(t *testing.T) {

	tests := []PathTest{
		{
			title: "Path route with single path , match",
			path:  "/api/",
		},
	}

	for _, test := range tests {
		code, ok := testGET(test)

		if !ok {
			t.Errorf("Expected status code 200, Actucal status code %v", code)
		}
	}
}

func testGET(pt PathTest) (int, bool) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello World!"))
	}
	r := NewRouter()

	r.Get(pt.path, handler)

	req, _ := http.NewRequest("GET", "http://localhost"+pt.path, nil)
	res := httptest.NewRecorder()
	r.ServeHTTP(res, req)

	var content bytes.Buffer
	_, err := io.Copy(&content, res.Body)

	if err != nil {
		return -1, false
	}

	if res.Code != 200 && content.String() == "Hello World!" {
		return res.Code, false
	}

	return res.Code, true
}
<commit_msg>Added test cases for Routes<commit_after>package mux

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"testing"
)

type routeTest struct {
	title      string
	path       string
	method     string
	statusCode int
}

func TestPath(t *testing.T) {

	tests := []routeTest{
		{
			title:      "Path route with single path",
			path:       "/api/",
			method:     http.MethodGet,
			statusCode: http.StatusOK,
		},
		{
			title:      "Path route with single path",
			path:       "/api/users/",
			method:     http.MethodGet,
			statusCode: http.StatusOK,
		},
		{
			title:      "Path route with single path",
			path:       "/api/echo",
			method:     http.MethodPost,
			statusCode: http.StatusOK,
		},
		{
			title:      "Path route with single path",
			path:       "/api/echo",
			method:     http.MethodDelete,
			statusCode: http.StatusOK,
		},
	}

	for _, test := range tests {
		t.Run(fmt.Sprintf("Test: %s path: %s method %s", test.title, test.path, test.method), func(t *testing.T) {
			code, ok := testRoute(test)
			if !ok {
				t.Errorf("Expected status code %v, Actucal status code %v", test.statusCode, code)
			}
		})
	}
}

func testRoute(rt routeTest) (int, bool) {
	handler := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("succesfully"))
	}
	r := NewRouter()

	switch rt.method {
	case http.MethodGet:
		r.Get(rt.path, handler)
	case http.MethodPost:
		r.Post(rt.path, handler)
	case http.MethodDelete:
		r.Delete(rt.path, handler)
	}

	req, _ := http.NewRequest(rt.method, "http://localhost"+rt.path, nil)
	res := httptest.NewRecorder()
	r.ServeHTTP(res, req)

	var content bytes.Buffer
	_, err := io.Copy(&content, res.Body)

	if err != nil {
		return -1, false
	}

	if res.Code != rt.statusCode && content.String() == "succesfully" {
		return res.Code, false
	}

	return res.Code, true
}
<|endoftext|>


<commit_before>package Golf

import (
	"reflect"
	"testing"
)

func handler(ctx *Context) {}

func TestParsePatternWithOneParam(t *testing.T) {
	cases := []struct {
		in, regex, param string
	}{
		{"/:id/", `^/([\w-%]+)/$`, "id"},
	}

	for _, c := range cases {
		route := newRoute(routerMethodGet, c.in, handler)
		if route.regex.String() != c.regex {
			t.Errorf("regex of %q  == %q, want %q", c.in, route.regex.String(), c.regex)
		}
		if len(route.params) != 1 {
			t.Errorf("%q is supposed to have 1 parameter", c.in)
		}
		if route.params[0] != "id" {
			t.Errorf("params[0] == %q, want %q", c.in, c.param)
		}
	}
}

func TestParsePatternWithThreeParam(t *testing.T) {
	cases := []struct {
		in, regex string
		params    []string
	}{
		{
			"/:year/:month/:day/",
			`^/([\w-%]+)/([\w-%]+)/([\w-%]+)/$`,
			[]string{"year", "month", "day"},
		},
	}

	for _, c := range cases {
		route := newRoute(routerMethodGet, c.in, handler)
		if route.regex.String() != c.regex {
			t.Errorf("regex == %q, want %q", c.in, route.regex.String())
		}
		if !reflect.DeepEqual(route.params, c.params) {
			t.Errorf("parameters not match: %v != %v", route.params, c.params)
		}
	}
}

func TestRouterMatch(t *testing.T) {
	router := newRouter()
	cases := []struct {
		pattern string
		url     string
		params  map[string]string
	}{
		{
			"/:year/:month/:day/",
			"/2015/11/15/",
			map[string]string{"year": "2015", "month": "11", "day": "15"},
		},
		{
			"/user/:id/",
			"/user/foobar/",
			map[string]string{"id": "foobar"},
		},
	}
	for _, c := range cases {
		router.get(c.pattern, handler)
		params, _ := router.match(c.url, routerMethodGet)
		if !reflect.DeepEqual(params, c.params) {
			t.Errorf("parameters not match: %v != %v", params, c.params)
		}
	}
}
<commit_msg>[test] Add test cases for different HTTP methods in the router<commit_after>package Golf

import (
	"reflect"
	"testing"
)

func handler(ctx *Context) {}

func TestParsePatternWithOneParam(t *testing.T) {
	cases := []struct {
		method, in, regex, param string
	}{
		{routerMethodGet, "/:id/", `^/([\w-%]+)/$`, "id"},
		{routerMethodPost, "/:id/", `^/([\w-%]+)/$`, "id"},
		{routerMethodPut, "/:id/", `^/([\w-%]+)/$`, "id"},
		{routerMethodDelete, "/:id/", `^/([\w-%]+)/$`, "id"},
	}

	for _, c := range cases {
		route := newRoute(c.method, c.in, handler)
		if route.regex.String() != c.regex {
			t.Errorf("regex of %q  == %q, want %q", c.in, route.regex.String(), c.regex)
		}
		if len(route.params) != 1 {
			t.Errorf("%q is supposed to have 1 parameter", c.in)
		}
		if route.params[0] != "id" {
			t.Errorf("params[0] == %q, want %q", c.in, c.param)
		}
	}
}

func TestParsePatternWithThreeParam(t *testing.T) {
	cases := []struct {
		in, regex string
		params    []string
	}{
		{
			"/:year/:month/:day/",
			`^/([\w-%]+)/([\w-%]+)/([\w-%]+)/$`,
			[]string{"year", "month", "day"},
		},
	}

	for _, c := range cases {
		route := newRoute(routerMethodGet, c.in, handler)
		if route.regex.String() != c.regex {
			t.Errorf("regex == %q, want %q", c.in, route.regex.String())
		}
		if !reflect.DeepEqual(route.params, c.params) {
			t.Errorf("parameters not match: %v != %v", route.params, c.params)
		}
	}
}

func TestRouterMatch(t *testing.T) {
	router := newRouter()
	cases := []struct {
		pattern string
		url     string
		params  map[string]string
	}{
		{
			"/:year/:month/:day/",
			"/2015/11/15/",
			map[string]string{"year": "2015", "month": "11", "day": "15"},
		},
		{
			"/user/:id/",
			"/user/foobar/",
			map[string]string{"id": "foobar"},
		},
	}
	for _, c := range cases {
		router.get(c.pattern, handler)
		params, _ := router.match(c.url, routerMethodGet)
		if !reflect.DeepEqual(params, c.params) {
			t.Errorf("parameters not match: %v != %v", params, c.params)
		}
	}
}
<|endoftext|>


<commit_before>package multihash

import (
	"crypto/sha1"
	"crypto/sha512"
	"errors"
	"fmt"

	keccak "github.com/gxed/hashland/keccakpg"
	blake2b "github.com/minio/blake2b-simd"
	sha256 "github.com/minio/sha256-simd"
	murmur3 "github.com/spaolacci/murmur3"
	blake2s "golang.org/x/crypto/blake2s"
	sha3 "golang.org/x/crypto/sha3"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	m := Multihash{}
	err := error(nil)
	if !ValidCode(code) {
		return m, fmt.Errorf("invalid multihash code %d", code)
	}

	if length < 0 {
		var ok bool
		length, ok = DefaultLengths[code]
		if !ok {
			return m, fmt.Errorf("no default length for code %d", code)
		}
	}

	var d []byte
	switch {
	case isBlake2s(code):
		olen := code - BLAKE2S_MIN + 1
		switch olen {
		case 32:
			out := blake2s.Sum256(data)
			d = out[:]
		default:
			return nil, fmt.Errorf("unsupported length for blake2s: %d", olen)
		}
	case isBlake2b(code):
		olen := uint8(code - BLAKE2B_MIN + 1)
		d = sumBlake2b(olen, data)
	default:
		switch code {
		case ID:
			d = sumID(data)
		case SHA1:
			d = sumSHA1(data)
		case SHA2_256:
			d = sumSHA256(data)
		case SHA2_512:
			d = sumSHA512(data)
		case KECCAK_224:
			d = sumKeccak224(data)
		case KECCAK_256:
			d = sumKeccak256(data)
		case KECCAK_384:
			d = sumKeccak384(data)
		case KECCAK_512:
			d = sumKeccak512(data)
		case SHA3_224:
			d = sumSHA3_224(data)
		case SHA3_256:
			d = sumSHA3_256(data)
		case SHA3_384:
			d = sumSHA3_384(data)
		case SHA3_512:
			d = sumSHA3_512(data)
		case DBL_SHA2_256:
			d = sumSHA256(sumSHA256(data))
		case MURMUR3:
			d, err = sumMURMUR3(data)
		case SHAKE_128:
			d = sumSHAKE128(data)
		case SHAKE_256:
			d = sumSHAKE256(data)
		default:
			return m, ErrSumNotSupported
		}
	}
	if err != nil {
		return m, err
	}
	if length >= 0 {
		d = d[:length]
	}
	return Encode(d, code)
}

func isBlake2s(code uint64) bool {
	return code >= BLAKE2S_MIN && code <= BLAKE2S_MAX
}
func isBlake2b(code uint64) bool {
	return code >= BLAKE2B_MIN && code <= BLAKE2B_MAX
}

func sumBlake2b(size uint8, data []byte) []byte {
	hasher, err := blake2b.New(&blake2b.Config{Size: size})
	if err != nil {
		panic(err)
	}

	if _, err := hasher.Write(data); err != nil {
		panic(err)
	}

	return hasher.Sum(nil)[:]
}

func sumID(data []byte) []byte {
	return data
}

func sumSHA1(data []byte) []byte {
	a := sha1.Sum(data)
	return a[0:20]
}

func sumSHA256(data []byte) []byte {
	a := sha256.Sum256(data)
	return a[0:32]
}

func sumSHA512(data []byte) []byte {
	a := sha512.Sum512(data)
	return a[0:64]
}

func sumKeccak224(data []byte) []byte {
	h := keccak.New224()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak256(data []byte) []byte {
	h := keccak.New256()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak384(data []byte) []byte {
	h := keccak.New384()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak512(data []byte) []byte {
	h := keccak.New512()
	h.Write(data)
	return h.Sum(nil)
}

func sumSHA3(data []byte) ([]byte, error) {
	h := sha3.New512()
	if _, err := h.Write(data); err != nil {
		return nil, err
	}
	return h.Sum(nil), nil
}

func sumSHA3_512(data []byte) []byte {
	a := sha3.Sum512(data)
	return a[:]
}

func sumMURMUR3(data []byte) ([]byte, error) {
	number := murmur3.Sum32(data)
	bytes := make([]byte, 4)
	for i := range bytes {
		bytes[i] = byte(number & 0xff)
		number >>= 8
	}
	return bytes, nil
}

func sumSHAKE128(data []byte) []byte {
	bytes := make([]byte, 32)
	sha3.ShakeSum128(bytes, data)
	return bytes
}

func sumSHAKE256(data []byte) []byte {
	bytes := make([]byte, 64)
	sha3.ShakeSum256(bytes, data)
	return bytes
}

func sumSHA3_384(data []byte) []byte {
	a := sha3.Sum384(data)
	return a[:]
}

func sumSHA3_256(data []byte) []byte {
	a := sha3.Sum256(data)
	return a[:]
}

func sumSHA3_224(data []byte) []byte {
	a := sha3.Sum224(data)
	return a[:]
}
<commit_msg>sum: check length of identity hash<commit_after>package multihash

import (
	"crypto/sha1"
	"crypto/sha512"
	"errors"
	"fmt"

	keccak "github.com/gxed/hashland/keccakpg"
	blake2b "github.com/minio/blake2b-simd"
	sha256 "github.com/minio/sha256-simd"
	murmur3 "github.com/spaolacci/murmur3"
	blake2s "golang.org/x/crypto/blake2s"
	sha3 "golang.org/x/crypto/sha3"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	m := Multihash{}
	err := error(nil)
	if !ValidCode(code) {
		return m, fmt.Errorf("invalid multihash code %d", code)
	}

	if length < 0 {
		var ok bool
		length, ok = DefaultLengths[code]
		if !ok {
			return m, fmt.Errorf("no default length for code %d", code)
		}
	}

	if code == ID && length != len(data) {
		return m, fmt.Errorf("the length of the identity hash (%d) must be equal to the length of the data (%d)",
			length, len(data))
	}

	var d []byte
	switch {
	case isBlake2s(code):
		olen := code - BLAKE2S_MIN + 1
		switch olen {
		case 32:
			out := blake2s.Sum256(data)
			d = out[:]
		default:
			return nil, fmt.Errorf("unsupported length for blake2s: %d", olen)
		}
	case isBlake2b(code):
		olen := uint8(code - BLAKE2B_MIN + 1)
		d = sumBlake2b(olen, data)
	default:
		switch code {
		case ID:
			d = sumID(data)
		case SHA1:
			d = sumSHA1(data)
		case SHA2_256:
			d = sumSHA256(data)
		case SHA2_512:
			d = sumSHA512(data)
		case KECCAK_224:
			d = sumKeccak224(data)
		case KECCAK_256:
			d = sumKeccak256(data)
		case KECCAK_384:
			d = sumKeccak384(data)
		case KECCAK_512:
			d = sumKeccak512(data)
		case SHA3_224:
			d = sumSHA3_224(data)
		case SHA3_256:
			d = sumSHA3_256(data)
		case SHA3_384:
			d = sumSHA3_384(data)
		case SHA3_512:
			d = sumSHA3_512(data)
		case DBL_SHA2_256:
			d = sumSHA256(sumSHA256(data))
		case MURMUR3:
			d, err = sumMURMUR3(data)
		case SHAKE_128:
			d = sumSHAKE128(data)
		case SHAKE_256:
			d = sumSHAKE256(data)
		default:
			return m, ErrSumNotSupported
		}
	}
	if err != nil {
		return m, err
	}
	if length >= 0 {
		d = d[:length]
	}
	return Encode(d, code)
}

func isBlake2s(code uint64) bool {
	return code >= BLAKE2S_MIN && code <= BLAKE2S_MAX
}
func isBlake2b(code uint64) bool {
	return code >= BLAKE2B_MIN && code <= BLAKE2B_MAX
}

func sumBlake2b(size uint8, data []byte) []byte {
	hasher, err := blake2b.New(&blake2b.Config{Size: size})
	if err != nil {
		panic(err)
	}

	if _, err := hasher.Write(data); err != nil {
		panic(err)
	}

	return hasher.Sum(nil)[:]
}

func sumID(data []byte) []byte {
	return data
}

func sumSHA1(data []byte) []byte {
	a := sha1.Sum(data)
	return a[0:20]
}

func sumSHA256(data []byte) []byte {
	a := sha256.Sum256(data)
	return a[0:32]
}

func sumSHA512(data []byte) []byte {
	a := sha512.Sum512(data)
	return a[0:64]
}

func sumKeccak224(data []byte) []byte {
	h := keccak.New224()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak256(data []byte) []byte {
	h := keccak.New256()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak384(data []byte) []byte {
	h := keccak.New384()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak512(data []byte) []byte {
	h := keccak.New512()
	h.Write(data)
	return h.Sum(nil)
}

func sumSHA3(data []byte) ([]byte, error) {
	h := sha3.New512()
	if _, err := h.Write(data); err != nil {
		return nil, err
	}
	return h.Sum(nil), nil
}

func sumSHA3_512(data []byte) []byte {
	a := sha3.Sum512(data)
	return a[:]
}

func sumMURMUR3(data []byte) ([]byte, error) {
	number := murmur3.Sum32(data)
	bytes := make([]byte, 4)
	for i := range bytes {
		bytes[i] = byte(number & 0xff)
		number >>= 8
	}
	return bytes, nil
}

func sumSHAKE128(data []byte) []byte {
	bytes := make([]byte, 32)
	sha3.ShakeSum128(bytes, data)
	return bytes
}

func sumSHAKE256(data []byte) []byte {
	bytes := make([]byte, 64)
	sha3.ShakeSum256(bytes, data)
	return bytes
}

func sumSHA3_384(data []byte) []byte {
	a := sha3.Sum384(data)
	return a[:]
}

func sumSHA3_256(data []byte) []byte {
	a := sha3.Sum256(data)
	return a[:]
}

func sumSHA3_224(data []byte) []byte {
	a := sha3.Sum224(data)
	return a[:]
}
<|endoftext|>


<commit_before>/*
Package swp implements the same Sliding Window Protocol that
TCP uses for flow-control and reliable, ordered delivery.

The Nats event bus (https://nats.io/) is a
software model of a hardware multicast
switch. Nats provides multicast, but no guarantees of delivery
and no flow-control. This works fine as long as your
downstream read/subscribe capacity is larger than your
publishing rate.

If your nats publisher evers produces
faster than your subscriber can keep up, you may overrun
your buffers and drop messages. If your sender is local
and replaying a disk file of traffic over nats, you are
guanateed to exhaust even the largest of the internal
nats client buffers. In addition you may wish guaranteed
order of delivery (even with dropped messages), which
swp provides.

Hence swp was built to provide flow-control and reliable, ordered
delivery on top of the nats event bus. It reproduces the
TCP sliding window and flow-control mechanism in a
Session between two nats clients. It provides flow
control between exactly two nats endpoints; in many
cases this is sufficient to allow all subscribers to
keep up.  If you have a wide variation in consumer
performance, establish the rate-controlling
`swp` Session between your producer and your
slowest consumer.

There is also a Session.RegisterAsap() API that can be
used to obtain possibly-out-of-order and possibly-duplicated
but as-soon-as-possible delivery (similar to that which
nats give you natively), while retaining the
flow-control required to avoid client-buffer overrun.
This can be used in tandem with the main always-ordered-and-lossless
API if so desired.
*/
package swp

import (
	"fmt"
	"sync/atomic"
	"time"
)

// sliding window protocol
//
// Reference: pp118-120, Computer Networks: A Systems Approach
//  by Peterson and Davie, Morgan Kaufmann Publishers, 1996.
//
// In addition to sliding window, we implement flow-control
// similar to how tcp does for throttling senders.
// See pp296-301 of Peterson and Davie.
//
// Most of the implementation is in sender.go and recv.go.

//go:generate msgp

//msgp:ignore TxqSlot RxqSlot Semaphore SenderState RecvState SWP Session NatsNet SimNet

// Packet is what is transmitted between Sender A and
// Recver B, where A and B are the two endpoints in a
// given Session. (Endpoints are specified by the strings localInbox and
// destInbox in the NewSession constructor.)
//
// Packets also flow symmetrically from Sender B to Recver A.
//
// Special packets are AckOnly and KeepAlive
// flagged; otherwise normal packets are data
// segments that have neither of these flags
// set. Only normal data packets are tracked
// for timeout and retry purposes.
type Packet struct {
	From string
	Dest string

	SeqNum    int64
	AckNum    int64
	AckOnly   bool
	KeepAlive bool

	// AvailReaderByteCap and AvailReaderMsgCap are
	// like the byte count AdvertisedWindow in TCP, but
	// since nats has both byte and message count
	// limits, we want convey these instead.
	AvailReaderBytesCap int64
	AvailReaderMsgCap   int64

	// CumulBytesTransmitted should give the total accumulated
	// count of bytes ever transmitted on this session
	// from `From` to `Dest`.
	// On data payloads, CumulBytesTransmitted allows
	// the receiver to figure out how
	// big any gaps are, so as to give accurate flow control
	// byte count info. The CumulBytesTransmitted count
	// should include this packet's len(Data), assuming
	// this is a data packet.
	CumulBytesTransmitted int64

	Data []byte
}

// TxqSlot is the sender's sliding window element.
type TxqSlot struct {
	RetryDeadline time.Time
	Pack          *Packet
}

// RxqSlot is the receiver's sliding window element.
type RxqSlot struct {
	Received bool
	Pack     *Packet
}

// SWP holds the Sliding Window Protocol state
type SWP struct {
	Sender *SenderState
	Recver *RecvState
}

// NewSWP makes a new sliding window protocol manager, holding
// both sender and receiver components.
func NewSWP(net Network, windowMsgCount int64, windowByteCount int64,
	timeout time.Duration, inbox string, destInbox string) *SWP {

	snd := NewSenderState(net, windowMsgCount, timeout, inbox, destInbox)
	rcv := NewRecvState(net, windowMsgCount, windowByteCount, timeout, inbox, snd)
	swp := &SWP{
		Sender: snd,
		Recver: rcv,
	}

	return swp
}

// Session tracks a given point-to-point sesssion and its
// sliding window state for one of the end-points.
type Session struct {
	Swp         *SWP
	Destination string
	MyInbox     string

	Net            Network
	ReadMessagesCh chan InOrderSeq

	packetsConsumed uint64
	packetsSent     uint64
}

// NewSession makes a new Session, and calls
// Swp.Start to begin the sliding-window-protocol.
//
// If windowByteSz is negative or less than windowMsgSz,
// we estimate a byte size based on 10kb messages and the given windowMsgSz.
//
func NewSession(net Network,
	localInbox string,
	destInbox string,
	windowMsgSz int64,
	windowByteSz int64,
	timeout time.Duration) (*Session, error) {

	if windowMsgSz < 1 {
		return nil, fmt.Errorf("windowMsgSz must be 1 or more")
	}

	if windowByteSz < windowMsgSz {
		// guestimate
		windowByteSz = windowMsgSz * 10 * 1024
	}

	sess := &Session{
		Swp:         NewSWP(net, windowMsgSz, windowByteSz, timeout, localInbox, destInbox),
		MyInbox:     localInbox,
		Destination: destInbox,
		Net:         net,
	}
	sess.Swp.Start()
	sess.ReadMessagesCh = sess.Swp.Recver.ReadMessagesCh

	return sess, nil
}

// Push sends a message packet, blocking until that is done.
// You can use sess.CountPacketsSentForTransfer() to get
// the total count of packets Push()-ed so far.
func (sess *Session) Push(pack *Packet) {
	select {
	case sess.Swp.Sender.BlockingSend <- pack:
		q("%v Push succeeded on payload '%s' into BlockingSend", sess.MyInbox, string(pack.Data))
		sess.IncrPacketsSentForTransfer(1)
	case <-sess.Swp.Sender.ReqStop:
		// give up, Sender is shutting down.
	}
}

// SelfConsumeForTesting sets up a reader to read all produced
// messages automatically. You can use CountPacketsReadConsumed() to
// see the total number consumed thus far.
func (sess *Session) SelfConsumeForTesting() {
	go func() {
		for {
			select {
			case <-sess.Swp.Recver.ReqStop:
				return
			case read := <-sess.ReadMessagesCh:
				sess.IncrPacketsReadConsumed(int64(len(read.Seq)))
			}
		}
	}()
}

// InWindow returns true iff seqno is in [min, max].
func InWindow(seqno, min, max int64) bool {
	if seqno < min {
		return false
	}
	if seqno > max {
		return false
	}
	return true
}

// Stop shutsdown the session
func (s *Session) Stop() {
	s.Swp.Stop()
}

// Stop the sliding window protocol
func (s *SWP) Stop() {
	s.Recver.Stop()
	s.Sender.Stop()
}

// Start the sliding window protocol
func (s *SWP) Start() {
	//q("SWP Start() called")
	s.Recver.Start()
	s.Sender.Start()
}

// CountPacketsReadConsumed reports on how many packets
// the application has read from the session.
func (sess *Session) CountPacketsReadConsumed() int64 {
	return int64(atomic.LoadUint64(&sess.packetsConsumed))
}

// IncrPacketsReadConsumed increment packetsConsumed and return the new total.
func (sess *Session) IncrPacketsReadConsumed(n int64) int64 {
	return int64(atomic.AddUint64(&sess.packetsConsumed, uint64(n)))
}

// CountPacketsSentForTransfer reports on how many packets.
// the application has written to the session.
func (sess *Session) CountPacketsSentForTransfer() int64 {
	return int64(atomic.LoadUint64(&sess.packetsSent))
}

// IncrPacketsSentForTransfer increment packetsConsumed and return the new total.
func (sess *Session) IncrPacketsSentForTransfer(n int64) int64 {
	return int64(atomic.AddUint64(&sess.packetsSent, uint64(n)))
}

// RegisterAsap registers a call back channel,
// rcvUnordered, which will get *Packet that are
// unordered and possibly
// have gaps in their sequence (where packets
// where dropped). However the channel will see
// the packets as soon as possible. The session
// will still be flow controlled however, so
// if the receiver throttles the sender, packets
// may be delayed. Clients should be prepared
// to deal with duplicated, dropped, and mis-ordered packets
// on the rcvUnordered channel.
func (s *Session) RegisterAsap(rcvUnordered chan *Packet) error {
	s.Swp.Recver.setAsapHelper <- NewAsapHelper(rcvUnordered)
	return nil
}
<commit_msg>atg. docs++<commit_after>/*
Package swp implements the same Sliding Window Protocol that
TCP uses for flow-control and reliable, ordered delivery.

The Nats event bus (https://nats.io/) is a
software model of a hardware multicast
switch. Nats provides multicast, but no guarantees of delivery
and no flow-control. This works fine as long as your
downstream read/subscribe capacity is larger than your
publishing rate.

If your nats publisher evers produces
faster than your subscriber can keep up, you may overrun
your buffers and drop messages. If your sender is local
and replaying a disk file of traffic over nats, you are
guanateed to exhaust even the largest of the internal
nats client buffers. In addition you may wish guaranteed
order of delivery (even with dropped messages), which
swp provides.

Hence swp was built to provide flow-control and reliable, ordered
delivery on top of the nats event bus. It reproduces the
TCP sliding window and flow-control mechanism in a
Session between two nats clients. It provides flow
control between exactly two nats endpoints; in many
cases this is sufficient to allow all subscribers to
keep up.  If you have a wide variation in consumer
performance, establish the rate-controlling
swp Session between your producer and your
slowest consumer.

There is also a Session.RegisterAsap() API that can be
used to obtain possibly-out-of-order and possibly-duplicated
but as-soon-as-possible delivery (similar to that which
nats give you natively), while retaining the
flow-control required to avoid client-buffer overrun.
This can be used in tandem with the main always-ordered-and-lossless
API if so desired.
*/
package swp

import (
	"fmt"
	"sync/atomic"
	"time"
)

// sliding window protocol
//
// Reference: pp118-120, Computer Networks: A Systems Approach
//  by Peterson and Davie, Morgan Kaufmann Publishers, 1996.
//
// In addition to sliding window, we implement flow-control
// similar to how tcp does for throttling senders.
// See pp296-301 of Peterson and Davie.
//
// Most of the implementation is in sender.go and recv.go.

//go:generate msgp

//msgp:ignore TxqSlot RxqSlot Semaphore SenderState RecvState SWP Session NatsNet SimNet

// Packet is what is transmitted between Sender A and
// Recver B, where A and B are the two endpoints in a
// given Session. (Endpoints are specified by the strings localInbox and
// destInbox in the NewSession constructor.)
//
// Packets also flow symmetrically from Sender B to Recver A.
//
// Special packets are AckOnly and KeepAlive
// flagged; otherwise normal packets are data
// segments that have neither of these flags
// set. Only normal data packets are tracked
// for timeout and retry purposes.
type Packet struct {
	From string
	Dest string

	SeqNum    int64
	AckNum    int64
	AckOnly   bool
	KeepAlive bool

	// AvailReaderByteCap and AvailReaderMsgCap are
	// like the byte count AdvertisedWindow in TCP, but
	// since nats has both byte and message count
	// limits, we want convey these instead.
	AvailReaderBytesCap int64
	AvailReaderMsgCap   int64

	// CumulBytesTransmitted should give the total accumulated
	// count of bytes ever transmitted on this session
	// from `From` to `Dest`.
	// On data payloads, CumulBytesTransmitted allows
	// the receiver to figure out how
	// big any gaps are, so as to give accurate flow control
	// byte count info. The CumulBytesTransmitted count
	// should include this packet's len(Data), assuming
	// this is a data packet.
	CumulBytesTransmitted int64

	Data []byte
}

// TxqSlot is the sender's sliding window element.
type TxqSlot struct {
	RetryDeadline time.Time
	Pack          *Packet
}

// RxqSlot is the receiver's sliding window element.
type RxqSlot struct {
	Received bool
	Pack     *Packet
}

// SWP holds the Sliding Window Protocol state
type SWP struct {
	Sender *SenderState
	Recver *RecvState
}

// NewSWP makes a new sliding window protocol manager, holding
// both sender and receiver components.
func NewSWP(net Network, windowMsgCount int64, windowByteCount int64,
	timeout time.Duration, inbox string, destInbox string) *SWP {

	snd := NewSenderState(net, windowMsgCount, timeout, inbox, destInbox)
	rcv := NewRecvState(net, windowMsgCount, windowByteCount, timeout, inbox, snd)
	swp := &SWP{
		Sender: snd,
		Recver: rcv,
	}

	return swp
}

// Session tracks a given point-to-point sesssion and its
// sliding window state for one of the end-points.
type Session struct {
	Swp         *SWP
	Destination string
	MyInbox     string

	Net            Network
	ReadMessagesCh chan InOrderSeq

	packetsConsumed uint64
	packetsSent     uint64
}

// NewSession makes a new Session, and calls
// Swp.Start to begin the sliding-window-protocol.
//
// If windowByteSz is negative or less than windowMsgSz,
// we estimate a byte size based on 10kb messages and the given windowMsgSz.
//
func NewSession(net Network,
	localInbox string,
	destInbox string,
	windowMsgSz int64,
	windowByteSz int64,
	timeout time.Duration) (*Session, error) {

	if windowMsgSz < 1 {
		return nil, fmt.Errorf("windowMsgSz must be 1 or more")
	}

	if windowByteSz < windowMsgSz {
		// guestimate
		windowByteSz = windowMsgSz * 10 * 1024
	}

	sess := &Session{
		Swp:         NewSWP(net, windowMsgSz, windowByteSz, timeout, localInbox, destInbox),
		MyInbox:     localInbox,
		Destination: destInbox,
		Net:         net,
	}
	sess.Swp.Start()
	sess.ReadMessagesCh = sess.Swp.Recver.ReadMessagesCh

	return sess, nil
}

// Push sends a message packet, blocking until that is done.
// You can use sess.CountPacketsSentForTransfer() to get
// the total count of packets Push()-ed so far.
func (sess *Session) Push(pack *Packet) {
	select {
	case sess.Swp.Sender.BlockingSend <- pack:
		q("%v Push succeeded on payload '%s' into BlockingSend", sess.MyInbox, string(pack.Data))
		sess.IncrPacketsSentForTransfer(1)
	case <-sess.Swp.Sender.ReqStop:
		// give up, Sender is shutting down.
	}
}

// SelfConsumeForTesting sets up a reader to read all produced
// messages automatically. You can use CountPacketsReadConsumed() to
// see the total number consumed thus far.
func (sess *Session) SelfConsumeForTesting() {
	go func() {
		for {
			select {
			case <-sess.Swp.Recver.ReqStop:
				return
			case read := <-sess.ReadMessagesCh:
				sess.IncrPacketsReadConsumed(int64(len(read.Seq)))
			}
		}
	}()
}

// InWindow returns true iff seqno is in [min, max].
func InWindow(seqno, min, max int64) bool {
	if seqno < min {
		return false
	}
	if seqno > max {
		return false
	}
	return true
}

// Stop shutsdown the session
func (s *Session) Stop() {
	s.Swp.Stop()
}

// Stop the sliding window protocol
func (s *SWP) Stop() {
	s.Recver.Stop()
	s.Sender.Stop()
}

// Start the sliding window protocol
func (s *SWP) Start() {
	//q("SWP Start() called")
	s.Recver.Start()
	s.Sender.Start()
}

// CountPacketsReadConsumed reports on how many packets
// the application has read from the session.
func (sess *Session) CountPacketsReadConsumed() int64 {
	return int64(atomic.LoadUint64(&sess.packetsConsumed))
}

// IncrPacketsReadConsumed increment packetsConsumed and return the new total.
func (sess *Session) IncrPacketsReadConsumed(n int64) int64 {
	return int64(atomic.AddUint64(&sess.packetsConsumed, uint64(n)))
}

// CountPacketsSentForTransfer reports on how many packets.
// the application has written to the session.
func (sess *Session) CountPacketsSentForTransfer() int64 {
	return int64(atomic.LoadUint64(&sess.packetsSent))
}

// IncrPacketsSentForTransfer increment packetsConsumed and return the new total.
func (sess *Session) IncrPacketsSentForTransfer(n int64) int64 {
	return int64(atomic.AddUint64(&sess.packetsSent, uint64(n)))
}

// RegisterAsap registers a call back channel,
// rcvUnordered, which will get *Packet that are
// unordered and possibly
// have gaps in their sequence (where packets
// where dropped). However the channel will see
// the packets as soon as possible. The session
// will still be flow controlled however, so
// if the receiver throttles the sender, packets
// may be delayed. Clients should be prepared
// to deal with duplicated, dropped, and mis-ordered packets
// on the rcvUnordered channel.
func (s *Session) RegisterAsap(rcvUnordered chan *Packet) error {
	s.Swp.Recver.setAsapHelper <- NewAsapHelper(rcvUnordered)
	return nil
}
<|endoftext|>


<commit_before>package registry

import (
	"errors"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/jonboulle/clockwork"
)

const (
	initialBackoff = 500 * time.Millisecond
	maxBackoff     = 10 * time.Second
)

var (
	ErrTimeout = errors.New("http request timeout")
)

type backoffRoundTripper struct {
	roundTripper               http.RoundTripper
	initialBackoff, maxBackoff time.Duration
	clock                      clockwork.Clock
}

// BackoffRoundTripper is a http.RoundTripper which adds a backoff for
// throttling to requests. To add a total request timeout, use Request.WithContext.
//
// r              -- upstream roundtripper
// initialBackoff -- initial length to backoff to when request request fails
// maxBackoff     -- maximum length to backoff to between request attempts
func BackoffRoundTripper(r http.RoundTripper, initialBackoff, maxBackoff time.Duration, clock clockwork.Clock) http.RoundTripper {
	return &backoffRoundTripper{
		roundTripper:   r,
		initialBackoff: initialBackoff,
		maxBackoff:     maxBackoff,
		clock:          clock,
	}
}

func (c *backoffRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	b := &backoff{
		initial: c.initialBackoff,
		max:     c.maxBackoff,
	}
	for {
		// Try the request
		resp, err := c.roundTripper.RoundTrip(r)
		switch {
		case err != nil && strings.Contains(err.Error(), "Too Many Requests (HAP429)."):
			// Catch the terrible dockerregistry error here. Eugh. :(
			fallthrough
		case resp != nil && resp.StatusCode == http.StatusTooManyRequests:
			// Request rate-limited, backoff and retry.
			b.Failure()
			// Wait until the next time we are allowed to make a request
			c.clock.Sleep(b.Wait())
		default:
			return resp, err
		}
	}
}

// backoff calculates an exponential backoff. This is used to
// calculate wait times for future requests.
type backoff struct {
	initial time.Duration
	max     time.Duration

	current time.Duration
	sync.RWMutex
}

// Failure should be called each time a request fails.
func (b *backoff) Failure() {
	b.Lock()
	defer b.Unlock()
	b.current *= 2
	if b.current == 0 {
		b.current = b.initial
	} else if b.current > b.max {
		b.current = b.max
	}
}

// Wait how long to sleep before *actually* starting the request.
func (b *backoff) Wait() time.Duration {
	b.RLock()
	defer b.RUnlock()
	return b.current
}
<commit_msg>Removing locking from registry.backoff, as it's only used from 1 thread<commit_after>package registry

import (
	"errors"
	"net/http"
	"strings"
	"time"

	"github.com/jonboulle/clockwork"
)

const (
	initialBackoff = 500 * time.Millisecond
	maxBackoff     = 10 * time.Second
)

var (
	ErrTimeout = errors.New("http request timeout")
)

type backoffRoundTripper struct {
	roundTripper               http.RoundTripper
	initialBackoff, maxBackoff time.Duration
	clock                      clockwork.Clock
}

// BackoffRoundTripper is a http.RoundTripper which adds a backoff for
// throttling to requests. To add a total request timeout, use Request.WithContext.
//
// r              -- upstream roundtripper
// initialBackoff -- initial length to backoff to when request request fails
// maxBackoff     -- maximum length to backoff to between request attempts
func BackoffRoundTripper(r http.RoundTripper, initialBackoff, maxBackoff time.Duration, clock clockwork.Clock) http.RoundTripper {
	return &backoffRoundTripper{
		roundTripper:   r,
		initialBackoff: initialBackoff,
		maxBackoff:     maxBackoff,
		clock:          clock,
	}
}

func (c *backoffRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	b := &backoff{
		initial: c.initialBackoff,
		max:     c.maxBackoff,
	}
	for {
		// Try the request
		resp, err := c.roundTripper.RoundTrip(r)
		switch {
		case err != nil && strings.Contains(err.Error(), "Too Many Requests (HAP429)."):
			// Catch the terrible dockerregistry error here. Eugh. :(
			fallthrough
		case resp != nil && resp.StatusCode == http.StatusTooManyRequests:
			// Request rate-limited, backoff and retry.
			b.Failure()
			// Wait until the next time we are allowed to make a request
			c.clock.Sleep(b.Wait())
		default:
			return resp, err
		}
	}
}

// backoff calculates an exponential backoff. This is used to
// calculate wait times for future requests.
type backoff struct {
	initial time.Duration
	max     time.Duration

	current time.Duration
}

// Failure should be called each time a request fails.
func (b *backoff) Failure() {
	b.current *= 2
	if b.current == 0 {
		b.current = b.initial
	} else if b.current > b.max {
		b.current = b.max
	}
}

// Wait how long to sleep before *actually* starting the request.
func (b *backoff) Wait() time.Duration {
	return b.current
}
<|endoftext|>


<commit_before>package registry

import (
	"archive/tar"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/docker/docker/graph"
	"github.com/vbatts/docker-utils/sum"
)

/*
From a tar input, push it to the registry.Registry r
*/
func ExtractTar(r *Registry, in io.Reader) error {
	return extractTar(r, in, true)
}

func ExtractTarWithoutTarsums(r *Registry, in io.Reader) error {
	return extractTar(r, in, false)
}

func extractTar(r *Registry, in io.Reader, tarsums bool) error {
	t := tar.NewReader(in)

	for {
		hdr, err := t.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		basename := filepath.Base(hdr.Name)
		hashid := filepath.Dir(hdr.Name)
		// The json file comes first
		if basename == "json" {
			if r != nil && r.HasImage(hashid) {
				continue
			}
			err = os.MkdirAll(filepath.Dir(r.JsonFileName(hashid)), 0755)
			if err != nil {
				return err
			}
			json_fh, err := os.Create(r.JsonFileName(hashid))
			if err != nil {
				return err
			}
			if _, err = io.Copy(json_fh, t); err != nil {
				return err
			}
			if err = json_fh.Close(); err != nil {
				return err
			}
		} else if basename == "layer.tar" {
			if r != nil && r.HasImage(hashid) {
				continue
			}
			err = os.MkdirAll(filepath.Dir(r.JsonFileName(hashid)), 0755)
			if err != nil {
				return err
			}
			layer_fh, err := os.Create(r.LayerFileName(hashid))
			if err != nil {
				return err
			}
			if !tarsums {
				if _, err = io.Copy(layer_fh, t); err != nil {
					return err
				}
				if err = layer_fh.Close(); err != nil {
					return err
				}
				fmt.Printf("Extracted Layer: %s\n", hashid)
				continue
			}
			json_fh, err := os.Open(r.JsonFileName(hashid))
			if err != nil {
				return err
			}
			str, err := sum.SumTarLayer(t, json_fh, layer_fh)
			if err != nil {
				return err
			}
			if err = layer_fh.Close(); err != nil {
				return err
			}
			if err = json_fh.Close(); err != nil {
				return err
			}

			tarsum_fh, err := os.Create(r.TarsumFileName(hashid))
			if err != nil {
				return err
			}
			if _, err = tarsum_fh.WriteString(str); err != nil {
				return err
			}
			if err = tarsum_fh.Close(); err != nil {
				return err
			}
			fmt.Printf("Extracted Layer: %s [%s]\n", hashid, str)
		} else if basename == "repositories" {
			repoMap := map[string]graph.Repository{}
			repositoriesJson, err := ioutil.ReadAll(t)
			if err != nil {
				return err
			}
			if err = json.Unmarshal(repositoriesJson, &repoMap); err != nil {
				return err
			}

			for repo, set := range repoMap {
				fmt.Println(repo)
				var (
					images_fh *os.File
					tags_fh   *os.File
					images    = []Image{}
					tags      = []Tag{}
				)
				err = r.EnsureRepoReady(repo)
				if err != nil {
					return err
				}
				if r.HasRepository(repo) {
					images_fh, err = os.Open(r.ImagesFileName(repo))
					if err != nil {
						return err
					}
					imagesJson, err := ioutil.ReadAll(images_fh)
					if err != nil {
						return err
					}
					images_fh.Seek(0, 0) // this will be added to, so the result will always be longer
					if err = json.Unmarshal(imagesJson, &images); err != nil {
						return err
					}

					tags_fh, err = os.Open(r.TagsFileName(repo))
					if err != nil {
						return err
					}
					tagsJson, err := ioutil.ReadAll(tags_fh)
					if err != nil {
						return err
					}
					tags_fh.Seek(0, 0) // this will be added to, so the result will always be longer
					if err = json.Unmarshal(tagsJson, &tags); err != nil {
						return err
					}
				} else {
					images_fh, err = os.Create(r.ImagesFileName(repo))
					if err != nil {
						return err
					}
					tags_fh, err = os.Create(r.TagsFileName(repo))
					if err != nil {
						return err
					}
				}
				for tag, hashid := range set {
					fmt.Printf("  %s :: %s\n", tag, hashid)

					// visting existing tags for this repo. update and merge
					tagExisted := false
					for _, e_tag := range tags {
						if e_tag.Name == tag {
							e_tag.Layer = hashid
							tagExisted = true
						}
					}
					if !tagExisted {
						tags = append(tags, Tag{Name: tag, Layer: hashid})
					}

					imageExisted := false

					var checksum string
					if tarsums {
						checksum, err = r.LayerTarsum(hashid)
						if err != nil {
							return err
						}
					}
					for _, e_image := range images {
						if e_image.Id == hashid {
							imageExisted = true
						}
					}
					if !imageExisted {
						images = append(images, Image{Id: hashid, Checksum: checksum})
					}
				}

				// ensure that each image tagged has an ancestry file
				for _, tag := range tags {
					if _, err = os.Stat(r.AncestryFileName(tag.Layer)); os.IsNotExist(err) {
						r.CreateAncestry(tag.Layer)
					}
				}

				// Write back the new data
				tagsJson, err := json.Marshal(TagsMap(tags))
				if err != nil {
					return err
				}
				imagesJson, err := json.Marshal(images)
				if err != nil {
					return err
				}
				if _, err = tags_fh.Write(tagsJson); err != nil {
					return err
				}
				if tags_fh.Close(); err != nil {
					return err
				}
				if _, err = images_fh.Write(imagesJson); err != nil {
					return err
				}
				if images_fh.Close(); err != nil {
					return err
				}
			}
		}
	}

	return nil
}
<commit_msg>registry: Gzip extracted layers when not using tarsums<commit_after>package registry

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/docker/docker/graph"
	"github.com/vbatts/docker-utils/sum"
)

/*
From a tar input, push it to the registry.Registry r
*/
func ExtractTar(r *Registry, in io.Reader) error {
	return extractTar(r, in, true)
}

func ExtractTarWithoutTarsums(r *Registry, in io.Reader) error {
	return extractTar(r, in, false)
}

func extractTar(r *Registry, in io.Reader, tarsums bool) error {
	t := tar.NewReader(in)

	for {
		hdr, err := t.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		basename := filepath.Base(hdr.Name)
		hashid := filepath.Dir(hdr.Name)
		// The json file comes first
		if basename == "json" {
			if r != nil && r.HasImage(hashid) {
				continue
			}
			err = os.MkdirAll(filepath.Dir(r.JsonFileName(hashid)), 0755)
			if err != nil {
				return err
			}
			json_fh, err := os.Create(r.JsonFileName(hashid))
			if err != nil {
				return err
			}
			if _, err = io.Copy(json_fh, t); err != nil {
				return err
			}
			if err = json_fh.Close(); err != nil {
				return err
			}
		} else if basename == "layer.tar" {
			if r != nil && r.HasImage(hashid) {
				continue
			}
			err = os.MkdirAll(filepath.Dir(r.JsonFileName(hashid)), 0755)
			if err != nil {
				return err
			}
			layer_fh, err := os.Create(r.LayerFileName(hashid))
			if err != nil {
				return err
			}
			if !tarsums {
				// generating tarsums also gzip compresses the archive, so we need
				// to do that manually if not using tarsums
				layer_gz, err := gzip.NewWriterLevel(layer_fh, gzip.BestCompression)
				if err != nil {
					return err
				}
				if _, err = io.Copy(layer_gz, t); err != nil {
					return err
				}
				if err = layer_gz.Close(); err != nil {
					return err
				}
				if err = layer_fh.Close(); err != nil {
					return err
				}
				fmt.Printf("Extracted Layer: %s\n", hashid)
				continue
			}
			json_fh, err := os.Open(r.JsonFileName(hashid))
			if err != nil {
				return err
			}
			str, err := sum.SumTarLayer(t, json_fh, layer_fh)
			if err != nil {
				return err
			}
			if err = layer_fh.Close(); err != nil {
				return err
			}
			if err = json_fh.Close(); err != nil {
				return err
			}

			tarsum_fh, err := os.Create(r.TarsumFileName(hashid))
			if err != nil {
				return err
			}
			if _, err = tarsum_fh.WriteString(str); err != nil {
				return err
			}
			if err = tarsum_fh.Close(); err != nil {
				return err
			}
			fmt.Printf("Extracted Layer: %s [%s]\n", hashid, str)
		} else if basename == "repositories" {
			repoMap := map[string]graph.Repository{}
			repositoriesJson, err := ioutil.ReadAll(t)
			if err != nil {
				return err
			}
			if err = json.Unmarshal(repositoriesJson, &repoMap); err != nil {
				return err
			}

			for repo, set := range repoMap {
				fmt.Println(repo)
				var (
					images_fh *os.File
					tags_fh   *os.File
					images    = []Image{}
					tags      = []Tag{}
				)
				err = r.EnsureRepoReady(repo)
				if err != nil {
					return err
				}
				if r.HasRepository(repo) {
					images_fh, err = os.Open(r.ImagesFileName(repo))
					if err != nil {
						return err
					}
					imagesJson, err := ioutil.ReadAll(images_fh)
					if err != nil {
						return err
					}
					images_fh.Seek(0, 0) // this will be added to, so the result will always be longer
					if err = json.Unmarshal(imagesJson, &images); err != nil {
						return err
					}

					tags_fh, err = os.Open(r.TagsFileName(repo))
					if err != nil {
						return err
					}
					tagsJson, err := ioutil.ReadAll(tags_fh)
					if err != nil {
						return err
					}
					tags_fh.Seek(0, 0) // this will be added to, so the result will always be longer
					if err = json.Unmarshal(tagsJson, &tags); err != nil {
						return err
					}
				} else {
					images_fh, err = os.Create(r.ImagesFileName(repo))
					if err != nil {
						return err
					}
					tags_fh, err = os.Create(r.TagsFileName(repo))
					if err != nil {
						return err
					}
				}
				for tag, hashid := range set {
					fmt.Printf("  %s :: %s\n", tag, hashid)

					// visting existing tags for this repo. update and merge
					tagExisted := false
					for _, e_tag := range tags {
						if e_tag.Name == tag {
							e_tag.Layer = hashid
							tagExisted = true
						}
					}
					if !tagExisted {
						tags = append(tags, Tag{Name: tag, Layer: hashid})
					}

					imageExisted := false

					var checksum string
					if tarsums {
						checksum, err = r.LayerTarsum(hashid)
						if err != nil {
							return err
						}
					}
					for _, e_image := range images {
						if e_image.Id == hashid {
							imageExisted = true
						}
					}
					if !imageExisted {
						images = append(images, Image{Id: hashid, Checksum: checksum})
					}
				}

				// ensure that each image tagged has an ancestry file
				for _, tag := range tags {
					if _, err = os.Stat(r.AncestryFileName(tag.Layer)); os.IsNotExist(err) {
						r.CreateAncestry(tag.Layer)
					}
				}

				// Write back the new data
				tagsJson, err := json.Marshal(TagsMap(tags))
				if err != nil {
					return err
				}
				imagesJson, err := json.Marshal(images)
				if err != nil {
					return err
				}
				if _, err = tags_fh.Write(tagsJson); err != nil {
					return err
				}
				if tags_fh.Close(); err != nil {
					return err
				}
				if _, err = images_fh.Write(imagesJson); err != nil {
					return err
				}
				if images_fh.Close(); err != nil {
					return err
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package registry // import "github.com/docker/docker/registry"

import (
	"context"
	"crypto/tls"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/docker/api/types"
	registrytypes "github.com/docker/docker/api/types/registry"
	"github.com/docker/docker/errdefs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// DefaultSearchLimit is the default value for maximum number of returned search results.
	DefaultSearchLimit = 25
)

// Service is the interface defining what a registry service should implement.
type Service interface {
	Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error)
	LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error)
	LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error)
	ResolveRepository(name reference.Named) (*RepositoryInfo, error)
	Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error)
	ServiceConfig() *registrytypes.ServiceConfig
	TLSConfig(hostname string) (*tls.Config, error)
	LoadAllowNondistributableArtifacts([]string) error
	LoadMirrors([]string) error
	LoadInsecureRegistries([]string) error
}

// DefaultService is a registry service. It tracks configuration data such as a list
// of mirrors.
type DefaultService struct {
	config *serviceConfig
	mu     sync.Mutex
}

// NewService returns a new instance of DefaultService ready to be
// installed into an engine.
func NewService(options ServiceOptions) (*DefaultService, error) {
	config, err := newServiceConfig(options)

	return &DefaultService{config: config}, err
}

// ServiceConfig returns the public registry service configuration.
func (s *DefaultService) ServiceConfig() *registrytypes.ServiceConfig {
	s.mu.Lock()
	defer s.mu.Unlock()

	servConfig := registrytypes.ServiceConfig{
		AllowNondistributableArtifactsCIDRs:     make([]*(registrytypes.NetIPNet), 0),
		AllowNondistributableArtifactsHostnames: make([]string, 0),
		InsecureRegistryCIDRs:                   make([]*(registrytypes.NetIPNet), 0),
		IndexConfigs:                            make(map[string]*(registrytypes.IndexInfo)),
		Mirrors:                                 make([]string, 0),
	}

	// construct a new ServiceConfig which will not retrieve s.Config directly,
	// and look up items in s.config with mu locked
	servConfig.AllowNondistributableArtifactsCIDRs = append(servConfig.AllowNondistributableArtifactsCIDRs, s.config.ServiceConfig.AllowNondistributableArtifactsCIDRs...)
	servConfig.AllowNondistributableArtifactsHostnames = append(servConfig.AllowNondistributableArtifactsHostnames, s.config.ServiceConfig.AllowNondistributableArtifactsHostnames...)
	servConfig.InsecureRegistryCIDRs = append(servConfig.InsecureRegistryCIDRs, s.config.ServiceConfig.InsecureRegistryCIDRs...)

	for key, value := range s.config.ServiceConfig.IndexConfigs {
		servConfig.IndexConfigs[key] = value
	}

	servConfig.Mirrors = append(servConfig.Mirrors, s.config.ServiceConfig.Mirrors...)

	return &servConfig
}

// LoadAllowNondistributableArtifacts loads allow-nondistributable-artifacts registries for Service.
func (s *DefaultService) LoadAllowNondistributableArtifacts(registries []string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.config.LoadAllowNondistributableArtifacts(registries)
}

// LoadMirrors loads registry mirrors for Service
func (s *DefaultService) LoadMirrors(mirrors []string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.config.LoadMirrors(mirrors)
}

// LoadInsecureRegistries loads insecure registries for Service
func (s *DefaultService) LoadInsecureRegistries(registries []string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.config.LoadInsecureRegistries(registries)
}

// Auth contacts the public registry with the provided credentials,
// and returns OK if authentication was successful.
// It can be used to verify the validity of a client's credentials.
func (s *DefaultService) Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error) {
	// TODO Use ctx when searching for repositories
	var registryHostName = IndexHostname

	if authConfig.ServerAddress != "" {
		serverAddress := authConfig.ServerAddress
		if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") {
			serverAddress = "https://" + serverAddress
		}
		u, err := url.Parse(serverAddress)
		if err != nil {
			return "", "", errdefs.InvalidParameter(errors.Errorf("unable to parse server address: %v", err))
		}
		registryHostName = u.Host
	}

	// Lookup endpoints for authentication using "LookupPushEndpoints", which
	// excludes mirrors to prevent sending credentials of the upstream registry
	// to a mirror.
	endpoints, err := s.LookupPushEndpoints(registryHostName)
	if err != nil {
		return "", "", errdefs.InvalidParameter(err)
	}

	for _, endpoint := range endpoints {
		status, token, err = loginV2(authConfig, endpoint, userAgent)
		if err == nil {
			return
		}
		if fErr, ok := err.(fallbackError); ok {
			logrus.WithError(fErr.err).Infof("Error logging in to endpoint, trying next endpoint")
			continue
		}

		return "", "", err
	}

	return "", "", err
}

// splitReposSearchTerm breaks a search term into an index name and remote name
func splitReposSearchTerm(reposName string) (string, string) {
	nameParts := strings.SplitN(reposName, "/", 2)
	var indexName, remoteName string
	if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") &&
		!strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") {
		// This is a Docker Index repos (ex: samalba/hipache or ubuntu)
		// 'docker.io'
		indexName = IndexName
		remoteName = reposName
	} else {
		indexName = nameParts[0]
		remoteName = nameParts[1]
	}
	return indexName, remoteName
}

// Search queries the public registry for images matching the specified
// search terms, and returns the results.
func (s *DefaultService) Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) {
	// TODO Use ctx when searching for repositories
	if err := validateNoScheme(term); err != nil {
		return nil, err
	}

	indexName, remoteName := splitReposSearchTerm(term)

	// Search is a long-running operation, just lock s.config to avoid block others.
	s.mu.Lock()
	index, err := newIndexInfo(s.config, indexName)
	s.mu.Unlock()

	if err != nil {
		return nil, err
	}

	// *TODO: Search multiple indexes.
	endpoint, err := NewV1Endpoint(index, userAgent, http.Header(headers))
	if err != nil {
		return nil, err
	}

	var client *http.Client
	if authConfig != nil && authConfig.IdentityToken != "" && authConfig.Username != "" {
		creds := NewStaticCredentialStore(authConfig)
		scopes := []auth.Scope{
			auth.RegistryScope{
				Name:    "catalog",
				Actions: []string{"search"},
			},
		}

		modifiers := Headers(userAgent, nil)
		v2Client, foundV2, err := v2AuthHTTPClient(endpoint.URL, endpoint.client.Transport, modifiers, creds, scopes)
		if err != nil {
			if fErr, ok := err.(fallbackError); ok {
				logrus.Errorf("Cannot use identity token for search, v2 auth not supported: %v", fErr.err)
			} else {
				return nil, err
			}
		} else if foundV2 {
			// Copy non transport http client features
			v2Client.Timeout = endpoint.client.Timeout
			v2Client.CheckRedirect = endpoint.client.CheckRedirect
			v2Client.Jar = endpoint.client.Jar

			logrus.Debugf("using v2 client for search to %s", endpoint.URL)
			client = v2Client
		}
	}

	if client == nil {
		client = endpoint.client
		if err := authorizeClient(client, authConfig, endpoint); err != nil {
			return nil, err
		}
	}

	r := newSession(client, authConfig, endpoint)

	if index.Official {
		localName := remoteName
		if strings.HasPrefix(localName, "library/") {
			// If pull "library/foo", it's stored locally under "foo"
			localName = strings.SplitN(localName, "/", 2)[1]
		}

		return r.SearchRepositories(localName, limit)
	}
	return r.SearchRepositories(remoteName, limit)
}

// ResolveRepository splits a repository name into its components
// and configuration of the associated registry.
func (s *DefaultService) ResolveRepository(name reference.Named) (*RepositoryInfo, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return newRepositoryInfo(s.config, name)
}

// APIEndpoint represents a remote API endpoint
type APIEndpoint struct {
	Mirror                         bool
	URL                            *url.URL
	Version                        APIVersion
	AllowNondistributableArtifacts bool
	Official                       bool
	TrimHostname                   bool
	TLSConfig                      *tls.Config
}

// ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint
// Deprecated: this function is deprecated and will be removed in a future update
func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) *V1Endpoint {
	return newV1Endpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders)
}

// TLSConfig constructs a client TLS configuration based on server defaults
func (s *DefaultService) TLSConfig(hostname string) (*tls.Config, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	return newTLSConfig(hostname, isSecureIndex(s.config, hostname))
}

// tlsConfig constructs a client TLS configuration based on server defaults
func (s *DefaultService) tlsConfig(hostname string) (*tls.Config, error) {
	return newTLSConfig(hostname, isSecureIndex(s.config, hostname))
}

func (s *DefaultService) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) {
	return s.tlsConfig(mirrorURL.Host)
}

// LookupPullEndpoints creates a list of v2 endpoints to try to pull from, in order of preference.
// It gives preference to mirrors over the actual registry, and HTTPS over plain HTTP.
func (s *DefaultService) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.lookupV2Endpoints(hostname)
}

// LookupPushEndpoints creates a list of v2 endpoints to try to push to, in order of preference.
// It gives preference to HTTPS over plain HTTP. Mirrors are not included.
func (s *DefaultService) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	allEndpoints, err := s.lookupV2Endpoints(hostname)
	if err == nil {
		for _, endpoint := range allEndpoints {
			if !endpoint.Mirror {
				endpoints = append(endpoints, endpoint)
			}
		}
	}
	return endpoints, err
}
<commit_msg>registry: minor cleanup in search code<commit_after>package registry // import "github.com/docker/docker/registry"

import (
	"context"
	"crypto/tls"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/docker/api/types"
	registrytypes "github.com/docker/docker/api/types/registry"
	"github.com/docker/docker/errdefs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// DefaultSearchLimit is the default value for maximum number of returned search results.
	DefaultSearchLimit = 25
)

// Service is the interface defining what a registry service should implement.
type Service interface {
	Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error)
	LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error)
	LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error)
	ResolveRepository(name reference.Named) (*RepositoryInfo, error)
	Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error)
	ServiceConfig() *registrytypes.ServiceConfig
	TLSConfig(hostname string) (*tls.Config, error)
	LoadAllowNondistributableArtifacts([]string) error
	LoadMirrors([]string) error
	LoadInsecureRegistries([]string) error
}

// DefaultService is a registry service. It tracks configuration data such as a list
// of mirrors.
type DefaultService struct {
	config *serviceConfig
	mu     sync.Mutex
}

// NewService returns a new instance of DefaultService ready to be
// installed into an engine.
func NewService(options ServiceOptions) (*DefaultService, error) {
	config, err := newServiceConfig(options)

	return &DefaultService{config: config}, err
}

// ServiceConfig returns the public registry service configuration.
func (s *DefaultService) ServiceConfig() *registrytypes.ServiceConfig {
	s.mu.Lock()
	defer s.mu.Unlock()

	servConfig := registrytypes.ServiceConfig{
		AllowNondistributableArtifactsCIDRs:     make([]*(registrytypes.NetIPNet), 0),
		AllowNondistributableArtifactsHostnames: make([]string, 0),
		InsecureRegistryCIDRs:                   make([]*(registrytypes.NetIPNet), 0),
		IndexConfigs:                            make(map[string]*(registrytypes.IndexInfo)),
		Mirrors:                                 make([]string, 0),
	}

	// construct a new ServiceConfig which will not retrieve s.Config directly,
	// and look up items in s.config with mu locked
	servConfig.AllowNondistributableArtifactsCIDRs = append(servConfig.AllowNondistributableArtifactsCIDRs, s.config.ServiceConfig.AllowNondistributableArtifactsCIDRs...)
	servConfig.AllowNondistributableArtifactsHostnames = append(servConfig.AllowNondistributableArtifactsHostnames, s.config.ServiceConfig.AllowNondistributableArtifactsHostnames...)
	servConfig.InsecureRegistryCIDRs = append(servConfig.InsecureRegistryCIDRs, s.config.ServiceConfig.InsecureRegistryCIDRs...)

	for key, value := range s.config.ServiceConfig.IndexConfigs {
		servConfig.IndexConfigs[key] = value
	}

	servConfig.Mirrors = append(servConfig.Mirrors, s.config.ServiceConfig.Mirrors...)

	return &servConfig
}

// LoadAllowNondistributableArtifacts loads allow-nondistributable-artifacts registries for Service.
func (s *DefaultService) LoadAllowNondistributableArtifacts(registries []string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.config.LoadAllowNondistributableArtifacts(registries)
}

// LoadMirrors loads registry mirrors for Service
func (s *DefaultService) LoadMirrors(mirrors []string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.config.LoadMirrors(mirrors)
}

// LoadInsecureRegistries loads insecure registries for Service
func (s *DefaultService) LoadInsecureRegistries(registries []string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.config.LoadInsecureRegistries(registries)
}

// Auth contacts the public registry with the provided credentials,
// and returns OK if authentication was successful.
// It can be used to verify the validity of a client's credentials.
func (s *DefaultService) Auth(ctx context.Context, authConfig *types.AuthConfig, userAgent string) (status, token string, err error) {
	// TODO Use ctx when searching for repositories
	var registryHostName = IndexHostname

	if authConfig.ServerAddress != "" {
		serverAddress := authConfig.ServerAddress
		if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") {
			serverAddress = "https://" + serverAddress
		}
		u, err := url.Parse(serverAddress)
		if err != nil {
			return "", "", errdefs.InvalidParameter(errors.Errorf("unable to parse server address: %v", err))
		}
		registryHostName = u.Host
	}

	// Lookup endpoints for authentication using "LookupPushEndpoints", which
	// excludes mirrors to prevent sending credentials of the upstream registry
	// to a mirror.
	endpoints, err := s.LookupPushEndpoints(registryHostName)
	if err != nil {
		return "", "", errdefs.InvalidParameter(err)
	}

	for _, endpoint := range endpoints {
		status, token, err = loginV2(authConfig, endpoint, userAgent)
		if err == nil {
			return
		}
		if fErr, ok := err.(fallbackError); ok {
			logrus.WithError(fErr.err).Infof("Error logging in to endpoint, trying next endpoint")
			continue
		}

		return "", "", err
	}

	return "", "", err
}

// splitReposSearchTerm breaks a search term into an index name and remote name
func splitReposSearchTerm(reposName string) (string, string) {
	nameParts := strings.SplitN(reposName, "/", 2)
	if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") &&
		!strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") {
		// This is a Docker Hub repository (ex: samalba/hipache or ubuntu),
		// use the default Docker Hub registry (docker.io)
		return IndexName, reposName
	}
	return nameParts[0], nameParts[1]
}

// Search queries the public registry for images matching the specified
// search terms, and returns the results.
func (s *DefaultService) Search(ctx context.Context, term string, limit int, authConfig *types.AuthConfig, userAgent string, headers map[string][]string) (*registrytypes.SearchResults, error) {
	// TODO Use ctx when searching for repositories
	if err := validateNoScheme(term); err != nil {
		return nil, err
	}

	indexName, remoteName := splitReposSearchTerm(term)

	// Search is a long-running operation, just lock s.config to avoid block others.
	s.mu.Lock()
	index, err := newIndexInfo(s.config, indexName)
	s.mu.Unlock()

	if err != nil {
		return nil, err
	}

	// *TODO: Search multiple indexes.
	endpoint, err := NewV1Endpoint(index, userAgent, headers)
	if err != nil {
		return nil, err
	}

	var client *http.Client
	if authConfig != nil && authConfig.IdentityToken != "" && authConfig.Username != "" {
		creds := NewStaticCredentialStore(authConfig)
		scopes := []auth.Scope{
			auth.RegistryScope{
				Name:    "catalog",
				Actions: []string{"search"},
			},
		}

		modifiers := Headers(userAgent, nil)
		v2Client, foundV2, err := v2AuthHTTPClient(endpoint.URL, endpoint.client.Transport, modifiers, creds, scopes)
		if err != nil {
			if fErr, ok := err.(fallbackError); ok {
				logrus.Errorf("Cannot use identity token for search, v2 auth not supported: %v", fErr.err)
			} else {
				return nil, err
			}
		} else if foundV2 {
			// Copy non transport http client features
			v2Client.Timeout = endpoint.client.Timeout
			v2Client.CheckRedirect = endpoint.client.CheckRedirect
			v2Client.Jar = endpoint.client.Jar

			logrus.Debugf("using v2 client for search to %s", endpoint.URL)
			client = v2Client
		}
	}

	if client == nil {
		client = endpoint.client
		if err := authorizeClient(client, authConfig, endpoint); err != nil {
			return nil, err
		}
	}

	r := newSession(client, authConfig, endpoint)

	if index.Official {
		// If pull "library/foo", it's stored locally under "foo"
		remoteName = strings.TrimPrefix(remoteName, "library/")
	}
	return r.SearchRepositories(remoteName, limit)
}

// ResolveRepository splits a repository name into its components
// and configuration of the associated registry.
func (s *DefaultService) ResolveRepository(name reference.Named) (*RepositoryInfo, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return newRepositoryInfo(s.config, name)
}

// APIEndpoint represents a remote API endpoint
type APIEndpoint struct {
	Mirror                         bool
	URL                            *url.URL
	Version                        APIVersion
	AllowNondistributableArtifacts bool
	Official                       bool
	TrimHostname                   bool
	TLSConfig                      *tls.Config
}

// ToV1Endpoint returns a V1 API endpoint based on the APIEndpoint
// Deprecated: this function is deprecated and will be removed in a future update
func (e APIEndpoint) ToV1Endpoint(userAgent string, metaHeaders http.Header) *V1Endpoint {
	return newV1Endpoint(*e.URL, e.TLSConfig, userAgent, metaHeaders)
}

// TLSConfig constructs a client TLS configuration based on server defaults
func (s *DefaultService) TLSConfig(hostname string) (*tls.Config, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	return newTLSConfig(hostname, isSecureIndex(s.config, hostname))
}

// tlsConfig constructs a client TLS configuration based on server defaults
func (s *DefaultService) tlsConfig(hostname string) (*tls.Config, error) {
	return newTLSConfig(hostname, isSecureIndex(s.config, hostname))
}

func (s *DefaultService) tlsConfigForMirror(mirrorURL *url.URL) (*tls.Config, error) {
	return s.tlsConfig(mirrorURL.Host)
}

// LookupPullEndpoints creates a list of v2 endpoints to try to pull from, in order of preference.
// It gives preference to mirrors over the actual registry, and HTTPS over plain HTTP.
func (s *DefaultService) LookupPullEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	return s.lookupV2Endpoints(hostname)
}

// LookupPushEndpoints creates a list of v2 endpoints to try to push to, in order of preference.
// It gives preference to HTTPS over plain HTTP. Mirrors are not included.
func (s *DefaultService) LookupPushEndpoints(hostname string) (endpoints []APIEndpoint, err error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	allEndpoints, err := s.lookupV2Endpoints(hostname)
	if err == nil {
		for _, endpoint := range allEndpoints {
			if !endpoint.Mirror {
				endpoints = append(endpoints, endpoint)
			}
		}
	}
	return endpoints, err
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/coreos/etcd/clientv3"
)

// KeyValue defines key-value pair.
type KeyValue struct {
	Key   string
	Value string
}

// ClientRequest defines client requests.
type ClientRequest struct {
	Action      string // 'write', 'stress', 'delete', 'get', 'stop-node', 'restart-node'
	RangePrefix bool   // 'delete', 'get'
	Endpoints   []string
	KeyValue    KeyValue
}

// ClientResponse translates client's GET response in frontend-friendly format.
type ClientResponse struct {
	ClientRequest ClientRequest
	Success       bool
	Result        string
	ResultLines   []string
	KeyValues     []KeyValue
}

var (
	minScaleToDisplay = time.Millisecond
	// ErrNoEndpoint is returned when client request has no target endpoint.
	ErrNoEndpoint = "no endpoint is given"
)

// clientRequestHandler handles writes, reads, deletes, kill, restart operations.
func clientRequestHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
	switch req.Method {
	case "POST":
		cresp := ClientResponse{Success: true}
		if rmsg, ok := globalClientRequestLimiter.Check(); !ok {
			cresp.Success = false
			cresp.Result = "client request " + rmsg
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}
		globalClientRequestLimiter.Advance()

		creq := ClientRequest{}
		if err := json.NewDecoder(req.Body).Decode(&creq); err != nil {
			cresp.Success = false
			cresp.Result = err.Error()
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}
		defer req.Body.Close()
		cresp.ClientRequest = creq

		if len(creq.Endpoints) == 0 {
			cresp.Success = false
			cresp.Result = ErrNoEndpoint
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}

		idx := globalCluster.FindIndex(creq.Endpoints[0])
		if idx == -1 {
			cresp.Success = false
			cresp.Result = fmt.Sprintf("wrong endpoints are given (%v)", creq.Endpoints)
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}

		cctx, ccancel := context.WithTimeout(ctx, 3*time.Second)
		defer ccancel()

		reqStart := time.Now()

		switch creq.Action {
		case "write":
			if creq.KeyValue.Key == "" {
				cresp.Success = false
				cresp.Result = fmt.Sprint("'write' request got empty key")
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			cresp.KeyValues = []KeyValue{creq.KeyValue}
			if _, err := cli.Put(cctx, creq.KeyValue.Key, creq.KeyValue.Value); err != nil {
				cresp.Success = false
				cresp.Result = err.Error()
				cresp.ResultLines = []string{cresp.Result}
			} else {
				cresp.Success = true
				cresp.Result = fmt.Sprintf("'write' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, 1)
				for i := range lines {
					ks, vs := cresp.KeyValues[i].Key, cresp.KeyValues[i].Value
					if len(ks) > 7 {
						ks = ks[:7] + "..."
					}
					if len(vs) > 7 {
						vs = vs[:7] + "..."
					}
					lines[i] = fmt.Sprintf("'write' success (key: %s, value: %s)", ks, vs)
				}
				cresp.ResultLines = lines
			}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "stress":
			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			cresp.KeyValues = multiRandKeyValues("foo", "bar", 3, 3)
			for _, kv := range cresp.KeyValues {
				if _, err := cli.Put(cctx, kv.Key, kv.Value); err != nil {
					cresp.Success = false
					cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
					cresp.ResultLines = []string{cresp.Result}
					break
				}
			}

			if cresp.Success {
				cresp.Result = fmt.Sprintf("'stress' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, 3)
				for i := range lines {
					ks, vs := cresp.KeyValues[i].Key, cresp.KeyValues[i].Value
					if len(ks) > 7 {
						ks = ks[:7] + "..."
					}
					if len(vs) > 7 {
						vs = vs[:7] + "..."
					}
					lines[i] = fmt.Sprintf("'stress' success (key: %s, value: %s)", ks, vs)
				}
				cresp.ResultLines = lines
			}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "delete":
			if creq.KeyValue.Key == "" {
				cresp.Success = false
				cresp.Result = fmt.Sprint("'delete' request got empty key")
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			var opts []clientv3.OpOption
			if creq.RangePrefix {
				opts = append(opts, clientv3.WithPrefix(), clientv3.WithPrevKV())
			}
			dresp, err := cli.Delete(cctx, creq.KeyValue.Key, opts...)
			if err != nil {
				cresp.Success = false
				cresp.Result = err.Error()
			}
			kvs := make([]KeyValue, len(dresp.PrevKvs))
			for i := range dresp.PrevKvs {
				kvs[i] = KeyValue{Key: string(dresp.PrevKvs[i].Key), Value: string(dresp.PrevKvs[i].Value)}
			}
			cresp.KeyValues = kvs

			if cresp.Success {
				cresp.Result = fmt.Sprintf("'delete' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, len(cresp.KeyValues))
				for i := range lines {
					lines[i] = fmt.Sprintf("'delete' success (key: %s, value: %s)", cresp.KeyValues[i].Key, cresp.KeyValues[i].Value)
				}
				cresp.ResultLines = lines
			}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "get":
			if creq.KeyValue.Key == "" {
				cresp.Success = false
				cresp.Result = fmt.Sprint("'get' request got empty key")
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			// TODO: get all keys and by prefix

			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			var opts []clientv3.OpOption
			if creq.RangePrefix {
				opts = append(opts, clientv3.WithPrefix(), clientv3.WithPrevKV())
			}
			gresp, err := cli.Get(cctx, creq.KeyValue.Key, opts...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
			}
			kvs := make([]KeyValue, len(gresp.Kvs))
			for i := range gresp.Kvs {
				kvs[i] = KeyValue{Key: string(gresp.Kvs[i].Key), Value: string(gresp.Kvs[i].Value)}
			}
			cresp.KeyValues = kvs

			if err == nil {
				cresp.Result = fmt.Sprintf("'get' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, len(cresp.KeyValues))
				for i := range lines {
					lines[i] = fmt.Sprintf("'get' success (key: %s, value: %s)", cresp.KeyValues[i].Key, cresp.KeyValues[i].Value)
				}
				if len(lines) == 0 {
					lines = append(lines, fmt.Sprintf("key %q does not exist", creq.KeyValue.Key))
				}
				cresp.ResultLines = lines
			}

			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "stop-node":
			if rmsg, ok := globalStopRestartLimiter.Check(); !ok {
				cresp.Success = false
				cresp.Result = "'stop-node' request " + rmsg
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			globalStopRestartLimiter.Advance()

			if globalCluster.IsStopped(idx) {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("%s is already stopped (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			globalCluster.Stop(idx)

			cresp.Result = fmt.Sprintf("stopped %s (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
			cresp.ResultLines = []string{cresp.Result}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "restart-node":
			if rmsg, ok := globalStopRestartLimiter.Check(); !ok {
				cresp.Success = false
				cresp.Result = "'restart-node' request " + rmsg
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			globalStopRestartLimiter.Advance()

			if !globalCluster.IsStopped(idx) {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("%s is already started (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			if rerr := globalCluster.Restart(idx); rerr != nil {
				cresp.Success = false
				cresp.Result = rerr.Error()
			} else {
				cresp.Success = true
				cresp.Result = fmt.Sprintf("restarted %s (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
			}

			cresp.ResultLines = []string{cresp.Result}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		default:
			return fmt.Errorf("unknown action %q", creq.Action)
		}

	default:
		http.Error(w, "Method Not Allowed", 405)
	}

	return nil
}
<commit_msg>backend: escape HTML strings to prevent xss<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"time"

	"github.com/coreos/etcd/clientv3"
)

// KeyValue defines key-value pair.
type KeyValue struct {
	Key   string
	Value string
}

// ClientRequest defines client requests.
type ClientRequest struct {
	Action      string // 'write', 'stress', 'delete', 'get', 'stop-node', 'restart-node'
	RangePrefix bool   // 'delete', 'get'
	Endpoints   []string
	KeyValue    KeyValue
}

// ClientResponse translates client's GET response in frontend-friendly format.
type ClientResponse struct {
	ClientRequest ClientRequest
	Success       bool
	Result        string
	ResultLines   []string
	KeyValues     []KeyValue
}

var (
	minScaleToDisplay = time.Millisecond
	// ErrNoEndpoint is returned when client request has no target endpoint.
	ErrNoEndpoint = "no endpoint is given"
)

// clientRequestHandler handles writes, reads, deletes, kill, restart operations.
func clientRequestHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
	switch req.Method {
	case "POST":
		cresp := ClientResponse{Success: true}
		if rmsg, ok := globalClientRequestLimiter.Check(); !ok {
			cresp.Success = false
			cresp.Result = "client request " + rmsg
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}
		globalClientRequestLimiter.Advance()

		creq := ClientRequest{}
		if err := json.NewDecoder(req.Body).Decode(&creq); err != nil {
			cresp.Success = false
			cresp.Result = err.Error()
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}
		defer req.Body.Close()

		if creq.KeyValue.Key != "" {
			creq.KeyValue.Key = template.HTMLEscapeString(creq.KeyValue.Key)
		}
		if creq.KeyValue.Value != "" {
			creq.KeyValue.Value = template.HTMLEscapeString(creq.KeyValue.Value)
		}

		cresp.ClientRequest = creq

		if len(creq.Endpoints) == 0 {
			cresp.Success = false
			cresp.Result = ErrNoEndpoint
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}

		idx := globalCluster.FindIndex(creq.Endpoints[0])
		if idx == -1 {
			cresp.Success = false
			cresp.Result = fmt.Sprintf("wrong endpoints are given (%v)", creq.Endpoints)
			cresp.ResultLines = []string{cresp.Result}
			return json.NewEncoder(w).Encode(cresp)
		}

		cctx, ccancel := context.WithTimeout(ctx, 3*time.Second)
		defer ccancel()

		reqStart := time.Now()

		switch creq.Action {
		case "write":
			if creq.KeyValue.Key == "" {
				cresp.Success = false
				cresp.Result = fmt.Sprint("'write' request got empty key")
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			cresp.KeyValues = []KeyValue{creq.KeyValue}
			if _, err := cli.Put(cctx, creq.KeyValue.Key, creq.KeyValue.Value); err != nil {
				cresp.Success = false
				cresp.Result = err.Error()
				cresp.ResultLines = []string{cresp.Result}
			} else {
				cresp.Success = true
				cresp.Result = fmt.Sprintf("'write' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, 1)
				for i := range lines {
					ks, vs := cresp.KeyValues[i].Key, cresp.KeyValues[i].Value
					if len(ks) > 7 {
						ks = ks[:7] + "..."
					}
					if len(vs) > 7 {
						vs = vs[:7] + "..."
					}
					lines[i] = fmt.Sprintf("'write' success (key: %s, value: %s)", ks, vs)
				}
				cresp.ResultLines = lines
			}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "stress":
			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			cresp.KeyValues = multiRandKeyValues("foo", "bar", 3, 3)
			for _, kv := range cresp.KeyValues {
				if _, err := cli.Put(cctx, kv.Key, kv.Value); err != nil {
					cresp.Success = false
					cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
					cresp.ResultLines = []string{cresp.Result}
					break
				}
			}

			if cresp.Success {
				cresp.Result = fmt.Sprintf("'stress' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, 3)
				for i := range lines {
					ks, vs := cresp.KeyValues[i].Key, cresp.KeyValues[i].Value
					if len(ks) > 7 {
						ks = ks[:7] + "..."
					}
					if len(vs) > 7 {
						vs = vs[:7] + "..."
					}
					lines[i] = fmt.Sprintf("'stress' success (key: %s, value: %s)", ks, vs)
				}
				cresp.ResultLines = lines
			}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "delete":
			if creq.KeyValue.Key == "" {
				cresp.Success = false
				cresp.Result = fmt.Sprint("'delete' request got empty key")
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			var opts []clientv3.OpOption
			if creq.RangePrefix {
				opts = append(opts, clientv3.WithPrefix(), clientv3.WithPrevKV())
			}
			dresp, err := cli.Delete(cctx, creq.KeyValue.Key, opts...)
			if err != nil {
				cresp.Success = false
				cresp.Result = err.Error()
			}
			kvs := make([]KeyValue, len(dresp.PrevKvs))
			for i := range dresp.PrevKvs {
				kvs[i] = KeyValue{Key: string(dresp.PrevKvs[i].Key), Value: string(dresp.PrevKvs[i].Value)}
			}
			cresp.KeyValues = kvs

			if cresp.Success {
				cresp.Result = fmt.Sprintf("'delete' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, len(cresp.KeyValues))
				for i := range lines {
					lines[i] = fmt.Sprintf("'delete' success (key: %s, value: %s)", cresp.KeyValues[i].Key, cresp.KeyValues[i].Value)
				}
				cresp.ResultLines = lines
			}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "get":
			if creq.KeyValue.Key == "" {
				cresp.Success = false
				cresp.Result = fmt.Sprint("'get' request got empty key")
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			// TODO: get all keys and by prefix

			cli, _, err := globalCluster.Client(creq.Endpoints...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			defer cli.Close()

			var opts []clientv3.OpOption
			if creq.RangePrefix {
				opts = append(opts, clientv3.WithPrefix(), clientv3.WithPrevKV())
			}
			gresp, err := cli.Get(cctx, creq.KeyValue.Key, opts...)
			if err != nil {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("client error %v (took %v)", err, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
			}
			kvs := make([]KeyValue, len(gresp.Kvs))
			for i := range gresp.Kvs {
				kvs[i] = KeyValue{Key: string(gresp.Kvs[i].Key), Value: string(gresp.Kvs[i].Value)}
			}
			cresp.KeyValues = kvs

			if err == nil {
				cresp.Result = fmt.Sprintf("'get' success (took %v)", roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				lines := make([]string, len(cresp.KeyValues))
				for i := range lines {
					lines[i] = fmt.Sprintf("'get' success (key: %s, value: %s)", cresp.KeyValues[i].Key, cresp.KeyValues[i].Value)
				}
				if len(lines) == 0 {
					lines = append(lines, fmt.Sprintf("key %q does not exist", creq.KeyValue.Key))
				}
				cresp.ResultLines = lines
			}

			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "stop-node":
			if rmsg, ok := globalStopRestartLimiter.Check(); !ok {
				cresp.Success = false
				cresp.Result = "'stop-node' request " + rmsg
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			globalStopRestartLimiter.Advance()

			if globalCluster.IsStopped(idx) {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("%s is already stopped (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			globalCluster.Stop(idx)

			cresp.Result = fmt.Sprintf("stopped %s (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
			cresp.ResultLines = []string{cresp.Result}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		case "restart-node":
			if rmsg, ok := globalStopRestartLimiter.Check(); !ok {
				cresp.Success = false
				cresp.Result = "'restart-node' request " + rmsg
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}
			globalStopRestartLimiter.Advance()

			if !globalCluster.IsStopped(idx) {
				cresp.Success = false
				cresp.Result = fmt.Sprintf("%s is already started (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
				cresp.ResultLines = []string{cresp.Result}
				return json.NewEncoder(w).Encode(cresp)
			}

			if rerr := globalCluster.Restart(idx); rerr != nil {
				cresp.Success = false
				cresp.Result = rerr.Error()
			} else {
				cresp.Success = true
				cresp.Result = fmt.Sprintf("restarted %s (took %v)", globalCluster.NodeStatus(idx).Name, roundDownDuration(time.Since(reqStart), minScaleToDisplay))
			}

			cresp.ResultLines = []string{cresp.Result}
			if err := json.NewEncoder(w).Encode(cresp); err != nil {
				return err
			}

		default:
			return fmt.Errorf("unknown action %q", creq.Action)
		}

	default:
		http.Error(w, "Method Not Allowed", 405)
	}

	return nil
}
<|endoftext|>