<commit_before>// Package multihash is the Go implementation of
// https://github.com/multiformats/multihash, or self-describing
// hashes.
package multihash

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"math"

	b58 "github.com/mr-tron/base58/base58"
)

// errors
var (
	ErrUnknownCode      = errors.New("unknown multihash code")
	ErrTooShort         = errors.New("multihash too short. must be >= 2 bytes")
	ErrTooLong          = errors.New("multihash too long. must be < 129 bytes")
	ErrLenNotSupported  = errors.New("multihash does not yet support digests longer than 127 bytes")
	ErrInvalidMultihash = errors.New("input isn't valid multihash")

	ErrVarintBufferShort = errors.New("uvarint: buffer too small")
	ErrVarintTooLong     = errors.New("uvarint: varint too big (max 64bit)")
)

// ErrInconsistentLen is returned when a decoded multihash has an inconsistent length
type ErrInconsistentLen struct {
	dm *DecodedMultihash
}

func (e ErrInconsistentLen) Error() string {
	return fmt.Sprintf("multihash length inconsistent: %v", e.dm)
}

// constants
const (
	ID         = 0x00
	SHA1       = 0x11
	SHA2_256   = 0x12
	SHA2_512   = 0x13
	SHA3_224   = 0x17
	SHA3_256   = 0x16
	SHA3_384   = 0x15
	SHA3_512   = 0x14
	SHA3       = SHA3_512
	KECCAK_224 = 0x1A
	KECCAK_256 = 0x1B
	KECCAK_384 = 0x1C
	KECCAK_512 = 0x1D

	SHAKE_128 = 0x18
	SHAKE_256 = 0x19

	BLAKE2B_MIN = 0xb201
	BLAKE2B_MAX = 0xb240
	BLAKE2S_MIN = 0xb241
	BLAKE2S_MAX = 0xb260

	DBL_SHA2_256 = 0x56

	MURMUR3 = 0x22

	X11 = 0x1100
)

func init() {
	// Add blake2b (64 codes)
	for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
		n := c - BLAKE2B_MIN + 1
		name := fmt.Sprintf("blake2b-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}

	// Add blake2s (32 codes)
	for c := uint64(BLAKE2S_MIN); c <= BLAKE2S_MAX; c++ {
		n := c - BLAKE2S_MIN + 1
		name := fmt.Sprintf("blake2s-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}
}

// Names maps the name of a hash to the code
var Names = map[string]uint64{
	"id":           ID,
	"sha1":         SHA1,
	"sha2-256":     SHA2_256,
	"sha2-512":     SHA2_512,
	"sha3":         SHA3_512,
	"sha3-224":     SHA3_224,
	"sha3-256":     SHA3_256,
	"sha3-384":     SHA3_384,
	"sha3-512":     SHA3_512,
	"dbl-sha2-256": DBL_SHA2_256,
	"murmur3":      MURMUR3,
	"keccak-224":   KECCAK_224,
	"keccak-256":   KECCAK_256,
	"keccak-384":   KECCAK_384,
	"keccak-512":   KECCAK_512,
	"shake-128":    SHAKE_128,
	"shake-256":    SHAKE_256,
	"x11":          X11,
}

// Codes maps a hash code to it's name
var Codes = map[uint64]string{
	ID:           "id",
	SHA1:         "sha1",
	SHA2_256:     "sha2-256",
	SHA2_512:     "sha2-512",
	SHA3_224:     "sha3-224",
	SHA3_256:     "sha3-256",
	SHA3_384:     "sha3-384",
	SHA3_512:     "sha3-512",
	DBL_SHA2_256: "dbl-sha2-256",
	MURMUR3:      "murmur3",
	KECCAK_224:   "keccak-224",
	KECCAK_256:   "keccak-256",
	KECCAK_384:   "keccak-384",
	KECCAK_512:   "keccak-512",
	SHAKE_128:    "shake-128",
	SHAKE_256:    "shake-256",
	X11:          "x11",
}

// DefaultLengths maps a hash code to it's default length
var DefaultLengths = map[uint64]int{
	ID:           -1,
	SHA1:         20,
	SHA2_256:     32,
	SHA2_512:     64,
	SHA3_224:     28,
	SHA3_256:     32,
	SHA3_384:     48,
	SHA3_512:     64,
	DBL_SHA2_256: 32,
	KECCAK_224:   28,
	KECCAK_256:   32,
	MURMUR3:      4,
	KECCAK_384:   48,
	KECCAK_512:   64,
	SHAKE_128:    32,
	SHAKE_256:    64,
	X11:          64,
}

func uvarint(buf []byte) (uint64, []byte, error) {
	n, c := binary.Uvarint(buf)

	if c == 0 {
		return n, buf, ErrVarintBufferShort
	} else if c < 0 {
		return n, buf[-c:], ErrVarintTooLong
	} else {
		return n, buf[c:], nil
	}
}

// DecodedMultihash represents a parsed multihash and allows
// easy access to the different parts of a multihash.
type DecodedMultihash struct {
	Code   uint64
	Name   string
	Length int    // Length is just int as it is type of len() opearator
	Digest []byte // Digest holds the raw multihash bytes
}

// Multihash is byte slice with the following form:
// <hash function code><digest size><hash function output>.
// See the spec for more information.
type Multihash []byte

// HexString returns the hex-encoded representation of a multihash.
func (m *Multihash) HexString() string {
	return hex.EncodeToString([]byte(*m))
}

// String is an alias to HexString().
func (m *Multihash) String() string {
	return m.HexString()
}

// FromHexString parses a hex-encoded multihash.
func FromHexString(s string) (Multihash, error) {
	b, err := hex.DecodeString(s)
	if err != nil {
		return Multihash{}, err
	}

	return Cast(b)
}

// B58String returns the B58-encoded representation of a multihash.
func (m Multihash) B58String() string {
	return b58.Encode([]byte(m))
}

// FromB58String parses a B58-encoded multihash.
func FromB58String(s string) (m Multihash, err error) {
	b, err := b58.Decode(s)
	if err != nil {
		return Multihash{}, ErrInvalidMultihash
	}

	return Cast(b)
}

// Cast casts a buffer onto a multihash, and returns an error
// if it does not work.
func Cast(buf []byte) (Multihash, error) {
	dm, err := Decode(buf)
	if err != nil {
		return Multihash{}, err
	}

	if !ValidCode(dm.Code) {
		return Multihash{}, ErrUnknownCode
	}

	return Multihash(buf), nil
}

// Decode parses multihash bytes into a DecodedMultihash.
func Decode(buf []byte) (*DecodedMultihash, error) {

	if len(buf) < 2 {
		return nil, ErrTooShort
	}

	var err error
	var code, length uint64

	code, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	length, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	if length > math.MaxInt32 {
		return nil, errors.New("digest too long, supporting only <= 2^31-1")
	}

	dm := &DecodedMultihash{
		Code:   code,
		Name:   Codes[code],
		Length: int(length),
		Digest: buf,
	}

	if len(dm.Digest) != dm.Length {
		return nil, ErrInconsistentLen{dm}
	}

	return dm, nil
}

// Encode a hash digest along with the specified function code.
// Note: the length is derived from the length of the digest itself.
func Encode(buf []byte, code uint64) ([]byte, error) {

	if !ValidCode(code) {
		return nil, ErrUnknownCode
	}

	start := make([]byte, 2*binary.MaxVarintLen64)
	spot := start
	n := binary.PutUvarint(spot, code)
	spot = start[n:]
	n += binary.PutUvarint(spot, uint64(len(buf)))

	return append(start[:n], buf...), nil
}

// EncodeName is like Encode() but providing a string name
// instead of a numeric code. See Names for allowed values.
func EncodeName(buf []byte, name string) ([]byte, error) {
	return Encode(buf, Names[name])
}

// ValidCode checks whether a multihash code is valid.
func ValidCode(code uint64) bool {
	if AppCode(code) {
		return true
	}

	if _, ok := Codes[code]; ok {
		return true
	}

	return false
}

// AppCode checks whether a multihash code is part of the App range.
func AppCode(code uint64) bool {
	return code >= 0 && code < 0x10
}
<commit_msg>reduce allocations in encode function by pre-allocating the buffer<commit_after>// Package multihash is the Go implementation of
// https://github.com/multiformats/multihash, or self-describing
// hashes.
package multihash

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"math"

	b58 "github.com/mr-tron/base58/base58"
)

// errors
var (
	ErrUnknownCode      = errors.New("unknown multihash code")
	ErrTooShort         = errors.New("multihash too short. must be >= 2 bytes")
	ErrTooLong          = errors.New("multihash too long. must be < 129 bytes")
	ErrLenNotSupported  = errors.New("multihash does not yet support digests longer than 127 bytes")
	ErrInvalidMultihash = errors.New("input isn't valid multihash")

	ErrVarintBufferShort = errors.New("uvarint: buffer too small")
	ErrVarintTooLong     = errors.New("uvarint: varint too big (max 64bit)")
)

// ErrInconsistentLen is returned when a decoded multihash has an inconsistent length
type ErrInconsistentLen struct {
	dm *DecodedMultihash
}

func (e ErrInconsistentLen) Error() string {
	return fmt.Sprintf("multihash length inconsistent: %v", e.dm)
}

// constants
const (
	ID         = 0x00
	SHA1       = 0x11
	SHA2_256   = 0x12
	SHA2_512   = 0x13
	SHA3_224   = 0x17
	SHA3_256   = 0x16
	SHA3_384   = 0x15
	SHA3_512   = 0x14
	SHA3       = SHA3_512
	KECCAK_224 = 0x1A
	KECCAK_256 = 0x1B
	KECCAK_384 = 0x1C
	KECCAK_512 = 0x1D

	SHAKE_128 = 0x18
	SHAKE_256 = 0x19

	BLAKE2B_MIN = 0xb201
	BLAKE2B_MAX = 0xb240
	BLAKE2S_MIN = 0xb241
	BLAKE2S_MAX = 0xb260

	DBL_SHA2_256 = 0x56

	MURMUR3 = 0x22

	X11 = 0x1100
)

func init() {
	// Add blake2b (64 codes)
	for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
		n := c - BLAKE2B_MIN + 1
		name := fmt.Sprintf("blake2b-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}

	// Add blake2s (32 codes)
	for c := uint64(BLAKE2S_MIN); c <= BLAKE2S_MAX; c++ {
		n := c - BLAKE2S_MIN + 1
		name := fmt.Sprintf("blake2s-%d", n*8)
		Names[name] = c
		Codes[c] = name
		DefaultLengths[c] = int(n)
	}
}

// Names maps the name of a hash to the code
var Names = map[string]uint64{
	"id":           ID,
	"sha1":         SHA1,
	"sha2-256":     SHA2_256,
	"sha2-512":     SHA2_512,
	"sha3":         SHA3_512,
	"sha3-224":     SHA3_224,
	"sha3-256":     SHA3_256,
	"sha3-384":     SHA3_384,
	"sha3-512":     SHA3_512,
	"dbl-sha2-256": DBL_SHA2_256,
	"murmur3":      MURMUR3,
	"keccak-224":   KECCAK_224,
	"keccak-256":   KECCAK_256,
	"keccak-384":   KECCAK_384,
	"keccak-512":   KECCAK_512,
	"shake-128":    SHAKE_128,
	"shake-256":    SHAKE_256,
	"x11":          X11,
}

// Codes maps a hash code to it's name
var Codes = map[uint64]string{
	ID:           "id",
	SHA1:         "sha1",
	SHA2_256:     "sha2-256",
	SHA2_512:     "sha2-512",
	SHA3_224:     "sha3-224",
	SHA3_256:     "sha3-256",
	SHA3_384:     "sha3-384",
	SHA3_512:     "sha3-512",
	DBL_SHA2_256: "dbl-sha2-256",
	MURMUR3:      "murmur3",
	KECCAK_224:   "keccak-224",
	KECCAK_256:   "keccak-256",
	KECCAK_384:   "keccak-384",
	KECCAK_512:   "keccak-512",
	SHAKE_128:    "shake-128",
	SHAKE_256:    "shake-256",
	X11:          "x11",
}

// DefaultLengths maps a hash code to it's default length
var DefaultLengths = map[uint64]int{
	ID:           -1,
	SHA1:         20,
	SHA2_256:     32,
	SHA2_512:     64,
	SHA3_224:     28,
	SHA3_256:     32,
	SHA3_384:     48,
	SHA3_512:     64,
	DBL_SHA2_256: 32,
	KECCAK_224:   28,
	KECCAK_256:   32,
	MURMUR3:      4,
	KECCAK_384:   48,
	KECCAK_512:   64,
	SHAKE_128:    32,
	SHAKE_256:    64,
	X11:          64,
}

func uvarint(buf []byte) (uint64, []byte, error) {
	n, c := binary.Uvarint(buf)

	if c == 0 {
		return n, buf, ErrVarintBufferShort
	} else if c < 0 {
		return n, buf[-c:], ErrVarintTooLong
	} else {
		return n, buf[c:], nil
	}
}

// DecodedMultihash represents a parsed multihash and allows
// easy access to the different parts of a multihash.
type DecodedMultihash struct {
	Code   uint64
	Name   string
	Length int    // Length is just int as it is type of len() opearator
	Digest []byte // Digest holds the raw multihash bytes
}

// Multihash is byte slice with the following form:
// <hash function code><digest size><hash function output>.
// See the spec for more information.
type Multihash []byte

// HexString returns the hex-encoded representation of a multihash.
func (m *Multihash) HexString() string {
	return hex.EncodeToString([]byte(*m))
}

// String is an alias to HexString().
func (m *Multihash) String() string {
	return m.HexString()
}

// FromHexString parses a hex-encoded multihash.
func FromHexString(s string) (Multihash, error) {
	b, err := hex.DecodeString(s)
	if err != nil {
		return Multihash{}, err
	}

	return Cast(b)
}

// B58String returns the B58-encoded representation of a multihash.
func (m Multihash) B58String() string {
	return b58.Encode([]byte(m))
}

// FromB58String parses a B58-encoded multihash.
func FromB58String(s string) (m Multihash, err error) {
	b, err := b58.Decode(s)
	if err != nil {
		return Multihash{}, ErrInvalidMultihash
	}

	return Cast(b)
}

// Cast casts a buffer onto a multihash, and returns an error
// if it does not work.
func Cast(buf []byte) (Multihash, error) {
	dm, err := Decode(buf)
	if err != nil {
		return Multihash{}, err
	}

	if !ValidCode(dm.Code) {
		return Multihash{}, ErrUnknownCode
	}

	return Multihash(buf), nil
}

// Decode parses multihash bytes into a DecodedMultihash.
func Decode(buf []byte) (*DecodedMultihash, error) {

	if len(buf) < 2 {
		return nil, ErrTooShort
	}

	var err error
	var code, length uint64

	code, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	length, buf, err = uvarint(buf)
	if err != nil {
		return nil, err
	}

	if length > math.MaxInt32 {
		return nil, errors.New("digest too long, supporting only <= 2^31-1")
	}

	dm := &DecodedMultihash{
		Code:   code,
		Name:   Codes[code],
		Length: int(length),
		Digest: buf,
	}

	if len(dm.Digest) != dm.Length {
		return nil, ErrInconsistentLen{dm}
	}

	return dm, nil
}

// Encode a hash digest along with the specified function code.
// Note: the length is derived from the length of the digest itself.
func Encode(buf []byte, code uint64) ([]byte, error) {

	if !ValidCode(code) {
		return nil, ErrUnknownCode
	}

	start := make([]byte, 2*binary.MaxVarintLen64, 2*binary.MaxVarintLen64+len(buf))
	spot := start
	n := binary.PutUvarint(spot, code)
	spot = start[n:]
	n += binary.PutUvarint(spot, uint64(len(buf)))

	return append(start[:n], buf...), nil
}

// EncodeName is like Encode() but providing a string name
// instead of a numeric code. See Names for allowed values.
func EncodeName(buf []byte, name string) ([]byte, error) {
	return Encode(buf, Names[name])
}

// ValidCode checks whether a multihash code is valid.
func ValidCode(code uint64) bool {
	if AppCode(code) {
		return true
	}

	if _, ok := Codes[code]; ok {
		return true
	}

	return false
}

// AppCode checks whether a multihash code is part of the App range.
func AppCode(code uint64) bool {
	return code >= 0 && code < 0x10
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package unitassigner

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/apiserver/params"
)

var _ = gc.Suite(testsuite{})

type testsuite struct{}

func (testsuite) TestAssignUnits(c *gc.C) {
	f := &fakeAssignCaller{c: c, response: params.ErrorResults{
		Results: []params.ErrorResult{
			{},
			{},
		}}}
	api := New(f)
	ids := []string{"mysql/0", "mysql/1"}
	errs, err := api.AssignUnits(ids)
	c.Assert(f.request, gc.Equals, "AssignUnits")
	c.Assert(f.params, gc.DeepEquals,
		params.Entities{[]params.Entity{
			{Tag: "unit-mysql-0"},
			{Tag: "unit-mysql-1"},
		}},
	)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(errs, gc.DeepEquals, []error{nil, nil})
}

func (testsuite) TestWatchUnitAssignment(c *gc.C) {
	f := &fakeWatchCaller{c: c}
	api := New(f)
	w, err := api.WatchUnitAssignments()
	c.Assert(f.request, gc.Equals, "WatchUnitAssignments")
	c.Assert(f.params, gc.IsNil)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(w, gc.NotNil)
}

type fakeAssignCaller struct {
	base.APICaller
	request  string
	params   interface{}
	response params.ErrorResults
	err      error
	c        *gc.C
}

func (f *fakeAssignCaller) APICall(objType string, version int, id, request string, param, response interface{}) error {
	f.request = request
	f.params = param
	res, ok := response.(*params.ErrorResults)
	if !ok {
		f.c.Errorf("Expected *params.ErrorResults as response, but was %#v", response)
	}
	*res = f.response
	return f.err

}

func (fakeAssignCaller) BestFacadeVersion(facade string) int {
	return 1
}

type fakeWatchCaller struct {
	base.APICaller
	request  string
	params   interface{}
	response params.StringsWatchResult
	err      error
	c        *gc.C
}

func (f *fakeWatchCaller) APICall(objType string, version int, id, request string, param, response interface{}) error {
	f.request = request
	f.params = param
	res, ok := response.(*params.StringsWatchResult)
	if !ok {
		f.c.Errorf("Expected *params.StringsWatchResult as response, but was %#v", response)
	}
	*res = f.response
	return f.err

}

func (fakeWatchCaller) BestFacadeVersion(facade string) int {
	return 1
}
<commit_msg>fix the tests<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package unitassigner

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/apiserver/params"
)

var _ = gc.Suite(testsuite{})

type testsuite struct{}

func (testsuite) TestAssignUnits(c *gc.C) {
	f := &fakeAssignCaller{c: c, response: params.ErrorResults{
		Results: []params.ErrorResult{
			{},
			{},
		}}}
	api := New(f)
	ids := []string{"mysql/0", "mysql/1"}
	errs, err := api.AssignUnits(ids)
	c.Assert(f.request, gc.Equals, "AssignUnits")
	c.Assert(f.params, gc.DeepEquals,
		params.Entities{[]params.Entity{
			{Tag: "unit-mysql-0"},
			{Tag: "unit-mysql-1"},
		}},
	)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(errs, gc.DeepEquals, []error{nil, nil})
}

func (testsuite) TestWatchUnitAssignment(c *gc.C) {
	f := &fakeWatchCaller{
		c:        c,
		response: params.StringsWatchResult{},
	}
	api := New(f)
	w, err := api.WatchUnitAssignments()
	c.Assert(f.request, gc.Equals, "WatchUnitAssignments")
	c.Assert(f.params, gc.IsNil)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(w, gc.NotNil)
}

type fakeAssignCaller struct {
	base.APICaller
	request  string
	params   interface{}
	response params.ErrorResults
	err      error
	c        *gc.C
}

func (f *fakeAssignCaller) APICall(objType string, version int, id, request string, param, response interface{}) error {
	f.request = request
	f.params = param
	res, ok := response.(*params.ErrorResults)
	if !ok {
		f.c.Errorf("Expected *params.ErrorResults as response, but was %#v", response)
	}
	*res = f.response
	return f.err

}

func (fakeAssignCaller) BestFacadeVersion(facade string) int {
	return 1
}

type fakeWatchCaller struct {
	base.APICaller
	request  string
	params   interface{}
	response params.StringsWatchResult
	err      error
	c        *gc.C
}

func (f *fakeWatchCaller) APICall(objType string, version int, id, request string, param, response interface{}) error {
	f.request = request
	f.params = param
	res, ok := response.(*params.StringsWatchResult)
	if !ok {
		f.c.Errorf("Expected *params.StringsWatchResult as response, but was %#v", response)
	}
	*res = f.response
	return f.err

}

func (fakeWatchCaller) BestFacadeVersion(facade string) int {
	return 1
}
<|endoftext|>


<commit_before>package halgo

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Navigator is a mechanism for navigating HAL-compliant REST APIs. You
// start by creating a Navigator with a base URI, then Follow the links
// exposed by the API until you reach the place where you want to perform
// an action.
//
// For example, to request an API exposed at api.example.com and follow a
// link named products and GET the resulting page you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Get()
//
// To do the same thing but POST to the products page, you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Post("application/json", someContent)
//
// Multiple links followed in sequence.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Follow("next")
//       Get()
//
// Links can also be expanded with Followf if they are URI templates.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Followf("page", halgo.P{"number": 10})
//       Get()
//
// Navigation of relations is lazy. Requests will only be triggered when
// you execute a method which returns a result. For example, this doesn't
// perform any HTTP requests.
//
//     Navigator("http://api.example.com").
//       Follow("products")
//
// It's only when you add a call to Get, Post, PostForm, Patch, or
// Unmarshal to the end will any requests be triggered.
//
// By default a Navigator will use http.DefaultClient as its mechanism for
// making HTTP requests. If you want to supply your own HttpClient, you
// can assign to nav.HttpClient after creation.
//
//     nav := Navigator("http://api.example.com")
//     nav.HttpClient = MyHttpClient{}
//
// Any Client you supply must implement halgo.HttpClient, which
// http.Client does implicitly. By creating decorators for the HttpClient,
// logging and caching clients are trivial. See LoggingHttpClient for an
// example.
func Navigator(uri string) navigator {
	return navigator{
		rootUri:    uri,
		path:       []relation{},
		HttpClient: http.DefaultClient,
	}
}

// relation is an instruction of a relation to follow and any params to
// expand with when executed.
type relation struct {
	rel    string
	params P
}

// navigator is the API navigator
type navigator struct {
	// HttpClient is used to execute requests. By default it's
	// http.DefaultClient. By decorating a HttpClient instance you can
	// easily write loggers or caching mechanisms.
	HttpClient HttpClient

	// path is the follow queue.
	path []relation

	// rootUri is where the navigation will begin from.
	rootUri string
}

// Follow adds a relation to the follow queue of the navigator.
func (n navigator) Follow(rel string) navigator {
	return n.Followf(rel, nil)
}

// Followf adds a relation to the follow queue of the navigator, with a
// set of parameters to expand on execution.
func (n navigator) Followf(rel string, params P) navigator {
	relations := make([]relation, 0, len(n.path)+1)
	copy(n.path, relations)
	relations = append(relations, relation{rel: rel, params: params})

	return navigator{
		HttpClient: n.HttpClient,
		path:       relations,
		rootUri:    n.rootUri,
	}
}

// url returns the URL of the tip of the follow queue. Will follow the
// usual pattern of requests.
func (n navigator) url() (string, error) {
	url := n.rootUri

	for _, link := range n.path {
		links, err := n.getLinks(url)
		if err != nil {
			return "", err
		}

		if _, ok := links.Items[link.rel]; !ok {
			return "", LinkNotFoundError{link.rel}
		}

		url, err = links.HrefParams(link.rel, link.params)
		if err != nil {
			return "", err
		}

		if url == "" {
			return "", InvalidUrlError{url}
		}

		url, err = makeAbsoluteIfNecessary(url, n.rootUri)
		if err != nil {
			return "", err
		}
	}

	return url, nil
}

// makeAbsoluteIfNecessary takes the current url and the root url, and
// will make the current URL absolute by using the root's Host, Scheme,
// and credentials if current isn't already absolute.
func makeAbsoluteIfNecessary(current, root string) (string, error) {
	currentUri, err := url.Parse(current)
	if err != nil {
		return "", err
	}

	if currentUri.IsAbs() {
		return current, nil
	}

	rootUri, err := url.Parse(root)
	if err != nil {
		return "", err
	}

	currentUri.Scheme = rootUri.Scheme
	currentUri.Host = rootUri.Host
	currentUri.User = rootUri.User

	return currentUri.String(), nil
}

// Get performs a GET request on the tip of the follow queue.
//
// When a navigator is evaluated it will first request the root, then
// request each relation on the queue until it reaches the tip. Once the
// tip is reached it will defer to the calling method. In the case of GET
// the last request will just be returned. For Post it will issue a post
// to the URL of the last relation. Any error along the way will terminate
// the walk and return immediately.
func (n navigator) Get() (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	return n.HttpClient.Do(req)
}

// PostForm performs a POST request on the tip of the follow queue with
// the given form data.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) PostForm(data url.Values) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	return n.HttpClient.Do(req)
}

// Patch parforms a PATCH request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Patch(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Post performs a POST request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Post(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Unmarshal is a shorthand for Get followed by json.Unmarshal. Handles
// closing the response body and unmarshalling the body.
func (n navigator) Unmarshal(v interface{}) error {
	res, err := n.Get()
	if err != nil {
		return err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	return json.Unmarshal(body, &v)
}

func newHalRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Accept", "application/hal+json, application/json")

	return req, nil
}

// getLinks does a GET on a particular URL and try to deserialise it into
// a HAL links collection.
func (n navigator) getLinks(uri string) (Links, error) {
	req, err := newHalRequest("GET", uri, nil)
	if err != nil {
		return Links{}, err
	}

	res, err := n.HttpClient.Do(req)
	if err != nil {
		return Links{}, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return Links{}, err
	}

	var m Links

	if err := json.Unmarshal(body, &m); err != nil {
		return Links{}, err
	}

	return m, nil
}
<commit_msg>Adding a utility function to process Location headers<commit_after>package halgo

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Navigator is a mechanism for navigating HAL-compliant REST APIs. You
// start by creating a Navigator with a base URI, then Follow the links
// exposed by the API until you reach the place where you want to perform
// an action.
//
// For example, to request an API exposed at api.example.com and follow a
// link named products and GET the resulting page you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Get()
//
// To do the same thing but POST to the products page, you'd do this:
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Post("application/json", someContent)
//
// Multiple links followed in sequence.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Follow("next")
//       Get()
//
// Links can also be expanded with Followf if they are URI templates.
//
//     res, err := Navigator("http://api.example.com").
//       Follow("products").
//       Followf("page", halgo.P{"number": 10})
//       Get()
//
// Navigation of relations is lazy. Requests will only be triggered when
// you execute a method which returns a result. For example, this doesn't
// perform any HTTP requests.
//
//     Navigator("http://api.example.com").
//       Follow("products")
//
// It's only when you add a call to Get, Post, PostForm, Patch, or
// Unmarshal to the end will any requests be triggered.
//
// By default a Navigator will use http.DefaultClient as its mechanism for
// making HTTP requests. If you want to supply your own HttpClient, you
// can assign to nav.HttpClient after creation.
//
//     nav := Navigator("http://api.example.com")
//     nav.HttpClient = MyHttpClient{}
//
// Any Client you supply must implement halgo.HttpClient, which
// http.Client does implicitly. By creating decorators for the HttpClient,
// logging and caching clients are trivial. See LoggingHttpClient for an
// example.
func Navigator(uri string) navigator {
	return navigator{
		rootUri:    uri,
		path:       []relation{},
		HttpClient: http.DefaultClient,
	}
}

// relation is an instruction of a relation to follow and any params to
// expand with when executed.
type relation struct {
	rel    string
	params P
}

// navigator is the API navigator
type navigator struct {
	// HttpClient is used to execute requests. By default it's
	// http.DefaultClient. By decorating a HttpClient instance you can
	// easily write loggers or caching mechanisms.
	HttpClient HttpClient

	// path is the follow queue.
	path []relation

	// rootUri is where the navigation will begin from.
	rootUri string
}

// Follow adds a relation to the follow queue of the navigator.
func (n navigator) Follow(rel string) navigator {
	return n.Followf(rel, nil)
}

// Followf adds a relation to the follow queue of the navigator, with a
// set of parameters to expand on execution.
func (n navigator) Followf(rel string, params P) navigator {
	relations := make([]relation, 0, len(n.path)+1)
	copy(n.path, relations)
	relations = append(relations, relation{rel: rel, params: params})

	return navigator{
		HttpClient: n.HttpClient,
		path:       relations,
		rootUri:    n.rootUri,
	}
}

// Location follows the Location header from a response.  It makes the URI
// absolute, if necessary.
func (n navigator) Location(resp *http.Response) (navigator, error) {
	_, exists := resp.Header["Location"]
	if !exists {
		return n, fmt.Errorf("Response didn't contain a Location header")
	}
	loc := resp.Header.Get("Location")
	lurl, err := makeAbsoluteIfNecessary(loc, n.rootUri)
	if err != nil {
		return n, err
	}
	return navigator{
		HttpClient: n.HttpClient,
		path:       []relation{},
		rootUri:    lurl,
	}, nil
}

// url returns the URL of the tip of the follow queue. Will follow the
// usual pattern of requests.
func (n navigator) url() (string, error) {
	url := n.rootUri

	for _, link := range n.path {
		links, err := n.getLinks(url)
		if err != nil {
			return "", err
		}

		if _, ok := links.Items[link.rel]; !ok {
			return "", LinkNotFoundError{link.rel}
		}

		url, err = links.HrefParams(link.rel, link.params)
		if err != nil {
			return "", err
		}

		if url == "" {
			return "", InvalidUrlError{url}
		}

		url, err = makeAbsoluteIfNecessary(url, n.rootUri)
		if err != nil {
			return "", err
		}
	}

	return url, nil
}

// makeAbsoluteIfNecessary takes the current url and the root url, and
// will make the current URL absolute by using the root's Host, Scheme,
// and credentials if current isn't already absolute.
func makeAbsoluteIfNecessary(current, root string) (string, error) {
	currentUri, err := url.Parse(current)
	if err != nil {
		return "", err
	}

	if currentUri.IsAbs() {
		return current, nil
	}

	rootUri, err := url.Parse(root)
	if err != nil {
		return "", err
	}

	currentUri.Scheme = rootUri.Scheme
	currentUri.Host = rootUri.Host
	currentUri.User = rootUri.User

	return currentUri.String(), nil
}

// Get performs a GET request on the tip of the follow queue.
//
// When a navigator is evaluated it will first request the root, then
// request each relation on the queue until it reaches the tip. Once the
// tip is reached it will defer to the calling method. In the case of GET
// the last request will just be returned. For Post it will issue a post
// to the URL of the last relation. Any error along the way will terminate
// the walk and return immediately.
func (n navigator) Get() (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	return n.HttpClient.Do(req)
}

// PostForm performs a POST request on the tip of the follow queue with
// the given form data.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) PostForm(data url.Values) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")

	return n.HttpClient.Do(req)
}

// Patch parforms a PATCH request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Patch(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("PATCH", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Post performs a POST request on the tip of the follow queue with the
// given bodyType and body content.
//
// See GET for a note on how the navigator executes requests.
func (n navigator) Post(bodyType string, body io.Reader) (*http.Response, error) {
	url, err := n.url()
	if err != nil {
		return nil, err
	}

	req, err := newHalRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", bodyType)

	return n.HttpClient.Do(req)
}

// Unmarshal is a shorthand for Get followed by json.Unmarshal. Handles
// closing the response body and unmarshalling the body.
func (n navigator) Unmarshal(v interface{}) error {
	res, err := n.Get()
	if err != nil {
		return err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	return json.Unmarshal(body, &v)
}

func newHalRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Accept", "application/hal+json, application/json")

	return req, nil
}

// getLinks does a GET on a particular URL and try to deserialise it into
// a HAL links collection.
func (n navigator) getLinks(uri string) (Links, error) {
	req, err := newHalRequest("GET", uri, nil)
	if err != nil {
		return Links{}, err
	}

	res, err := n.HttpClient.Do(req)
	if err != nil {
		return Links{}, err
	}
	defer res.Body.Close()

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return Links{}, err
	}

	var m Links

	if err := json.Unmarshal(body, &m); err != nil {
		return Links{}, err
	}

	return m, nil
}
<|endoftext|>


<commit_before>package gitbackup

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type repository struct {
	name     string
	cloneURL string
}

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target Target, backupDirectory string) error {
	fmt.Printf("########## Backing up target %s ##########\n\n", target.Name)

	// Retrieve a list of all the git repositories available from the target.
	var repoList []repository
	var err error
	switch target.Source {
	case "github":
		repoList, err = getGitHubRepoList(target, backupDirectory)
	case "bitbucket":
		repoList, err = getBitBucketRepoList(target, backupDirectory)
	default:
		err = fmt.Errorf(`"%s" is not a recognized source type`, target.Source)
	}
	if err != nil {
		return err
	}

	// Back up each repository found.
	backupSuccesses := 0
	for _, repo := range repoList {
		fmt.Println(fmt.Sprintf("#> %s", repo.name))
		if includeRepository(repo.name, target) {
			backupSuccess := backupRepository(
				target.Name,
				repo.name,
				repo.cloneURL,
				backupDirectory,
			)
			if backupSuccess {
				backupSuccesses++
			}
			fmt.Println("")
		} else {
			fmt.Print("Skipped.\n\n")
		}
	}

	// Print out summary of the backup actions performed for this target.
	fmt.Printf(
		"Backed up %d/%d repositories successfully.\n\n\n",
		backupSuccesses,
		len(repoList),
	)

	return nil
}

// getGitHubRepoList finds all the repositories belonging to a given user or
// organization on GitHub.
func getGitHubRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	requestURL := fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target.Type,
		target.Entity,
		target.Token,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var data []map[string]interface{}
	if err := json.Unmarshal(contents, &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	var repoList []repository
	for _, repo := range data {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Token),
			1,
		)
		repo := repository{name: repoName, cloneURL: cloneURL}
		repoList = append(repoList,repo)
	}

	// No errors.
	return repoList, nil
}

// getBitBucketRepoList finds all the repositories belonging to a given user on
// BitBucket.
func getBitBucketRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	// TODO: support pagination.
	requestURL := fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target.Entity,
		target.Password,
		target.Entity,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	repoList := make([]repository, len(data))
	for i, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return nil, fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Determine URL for cloning.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target.Entity),
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Password),
			1,
		)

		repoList[i] = repository{name: repoName, cloneURL: cloneURL}
	}

	// No errors.
	return repoList, nil
}

// backupRepository takes a remote git repository and backs it up locally.
// Note that this makes a mirror repository - in other words, the backup only
// contains the content of a normal .git repository but no working directory,
// which saves space. You can always get a normal repository from the backup by
// doing a normal git clone of the backup itself.
func backupRepository(targetName string, repoName string, cloneURL string, backupDirectory string) bool {
	cloneDirectory := filepath.Join(backupDirectory, targetName, repoName)

	if _, err := os.Stat(cloneDirectory); os.IsNotExist(err) {
		// The repo doesn't exist locally, clone it.
		cmd := exec.Command("git", "clone", "--mirror", cloneURL, cloneDirectory)
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error cloning the repository:", err)
			return false
		}
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Cloned repository.")
	} else {
		// The repo already exists.
		// Set the origin again in case the credentials have changed.
		cmd := exec.Command("git", "remote", "set-url", "origin", cloneURL)
		cmd.Dir = cloneDirectory
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error setting remote URL:", err)
			return false
		}

		// Pull updates.
		cmd = exec.Command("git", "remote", "update", "-p")
		cmd.Dir = cloneDirectory
		cmdOut, err = cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error pulling in the repository:", err)
			return false
		}

		// Display pulled information.
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Pulled latest updates in the repository.")
	}

	return true
}

// includeRepository takes a repository name and the information about the
// target it is part of, and determines whether the repository should be backed
// up or not.
func includeRepository(repoName string, target Target) bool {
	if target.Skip != "" {
		r, err := regexp.Compile(target.Skip)
		if err != nil {
			log.Fatalf(`"skip" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == false
	}
	if target.Only != "" {
		r, err := regexp.Compile(target.Only)
		if err != nil {
			log.Fatalf(`"only" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == true
	}

	return true
}
<commit_msg>Formatting.<commit_after>package gitbackup

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type repository struct {
	name     string
	cloneURL string
}

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target Target, backupDirectory string) error {
	fmt.Printf("########## Backing up target %s ##########\n\n", target.Name)

	// Retrieve a list of all the git repositories available from the target.
	var repoList []repository
	var err error
	switch target.Source {
	case "github":
		repoList, err = getGitHubRepoList(target, backupDirectory)
	case "bitbucket":
		repoList, err = getBitBucketRepoList(target, backupDirectory)
	default:
		err = fmt.Errorf(`"%s" is not a recognized source type`, target.Source)
	}
	if err != nil {
		return err
	}

	// Back up each repository found.
	backupSuccesses := 0
	for _, repo := range repoList {
		fmt.Println(fmt.Sprintf("#> %s", repo.name))
		if includeRepository(repo.name, target) {
			backupSuccess := backupRepository(
				target.Name,
				repo.name,
				repo.cloneURL,
				backupDirectory,
			)
			if backupSuccess {
				backupSuccesses++
			}
			fmt.Println("")
		} else {
			fmt.Print("Skipped.\n\n")
		}
	}

	// Print out summary of the backup actions performed for this target.
	fmt.Printf(
		"Backed up %d/%d repositories successfully.\n\n\n",
		backupSuccesses,
		len(repoList),
	)

	return nil
}

// getGitHubRepoList finds all the repositories belonging to a given user or
// organization on GitHub.
func getGitHubRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	requestURL := fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target.Type,
		target.Entity,
		target.Token,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var data []map[string]interface{}
	if err := json.Unmarshal(contents, &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	var repoList []repository
	for _, repo := range data {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Token),
			1,
		)
		repo := repository{name: repoName, cloneURL: cloneURL}
		repoList = append(repoList, repo)
	}

	// No errors.
	return repoList, nil
}

// getBitBucketRepoList finds all the repositories belonging to a given user on
// BitBucket.
func getBitBucketRepoList(target Target, backupDirectory string) ([]repository, error) {
	// Create URL to request list of repos.
	// TODO: support pagination.
	requestURL := fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target.Entity,
		target.Password,
		target.Entity,
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return nil, fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Make a list of repositories.
	repoList := make([]repository, len(data))
	for i, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return nil, fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return nil, fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Determine URL for cloning.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target.Entity),
			fmt.Sprintf("https://%s:%s@", target.Entity, target.Password),
			1,
		)

		repoList[i] = repository{name: repoName, cloneURL: cloneURL}
	}

	// No errors.
	return repoList, nil
}

// backupRepository takes a remote git repository and backs it up locally.
// Note that this makes a mirror repository - in other words, the backup only
// contains the content of a normal .git repository but no working directory,
// which saves space. You can always get a normal repository from the backup by
// doing a normal git clone of the backup itself.
func backupRepository(targetName string, repoName string, cloneURL string, backupDirectory string) bool {
	cloneDirectory := filepath.Join(backupDirectory, targetName, repoName)

	if _, err := os.Stat(cloneDirectory); os.IsNotExist(err) {
		// The repo doesn't exist locally, clone it.
		cmd := exec.Command("git", "clone", "--mirror", cloneURL, cloneDirectory)
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error cloning the repository:", err)
			return false
		}
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Cloned repository.")
	} else {
		// The repo already exists.
		// Set the origin again in case the credentials have changed.
		cmd := exec.Command("git", "remote", "set-url", "origin", cloneURL)
		cmd.Dir = cloneDirectory
		cmdOut, err := cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error setting remote URL:", err)
			return false
		}

		// Pull updates.
		cmd = exec.Command("git", "remote", "update", "-p")
		cmd.Dir = cloneDirectory
		cmdOut, err = cmd.CombinedOutput()
		if err != nil {
			fmt.Println("Error pulling in the repository:", err)
			return false
		}

		// Display pulled information.
		if len(cmdOut) > 0 {
			fmt.Printf(string(cmdOut))
		}
		fmt.Println("Pulled latest updates in the repository.")
	}

	return true
}

// includeRepository takes a repository name and the information about the
// target it is part of, and determines whether the repository should be backed
// up or not.
func includeRepository(repoName string, target Target) bool {
	if target.Skip != "" {
		r, err := regexp.Compile(target.Skip)
		if err != nil {
			log.Fatalf(`"skip" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == false
	}
	if target.Only != "" {
		r, err := regexp.Compile(target.Only)
		if err != nil {
			log.Fatalf(`"only" does not specify a valid regular expression: %s`, err)
		}
		return r.MatchString(repoName) == true
	}

	return true
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2014 Ashley Jeffs

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, sub to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package net

import (
	"time"

	"github.com/jeffail/leaps/lib"
	"github.com/jeffail/leaps/lib/store"
)

/*--------------------------------------------------------------------------------------------------
 */

/*
LeapLocator - An interface capable of locating and creating leaps documents. This can either be a
curator, which deals with documents on the local service, or a TBD, which load balances between
servers of curators.
*/
type LeapLocator interface {
	// EditDocument - Find and return a binder portal to an existing document
	EditDocument(string, string) (lib.BinderPortal, error)

	// ReadDocument - Find and return a binder portal to an existing document with read only
	// priviledges
	ReadDocument(string, string) (lib.BinderPortal, error)

	// CreateDocument - Create and return a binder portal to a new document
	CreateDocument(string, string, store.Document) (lib.BinderPortal, error)

	// Close - Close the LeapLocator
	Close()
}

/*
LeapAdmin - An interface for performing privileged actions around the curation of leaps documents
such as user kicking and getting full lists of connected users per document.
*/
type LeapAdmin interface {
	// Kick a user from a document, needs the documentID and userID.
	KickUser(documentID, userID string, timeout time.Duration) error

	// Get the list of all users connected to all open binders.
	GetUsers(timeout time.Duration) (map[string][]string, error)
}

/*--------------------------------------------------------------------------------------------------
 */
<commit_msg>Update LeapLocator interface<commit_after>/*
Copyright (c) 2014 Ashley Jeffs

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, sub to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package net

import (
	"time"

	"github.com/jeffail/leaps/lib"
	"github.com/jeffail/leaps/lib/store"
)

/*--------------------------------------------------------------------------------------------------
 */

/*
LeapLocator - An interface capable of locating and creating leaps documents. This can either be a
curator, which deals with documents on the local service, or a TBD, which load balances between
servers of curators.
*/
type LeapLocator interface {
	// EditDocument - Find and return a binder portal to an existing document
	EditDocument(userID, token, documentID string) (lib.BinderPortal, error)

	// ReadDocument - Find and return a binder portal to an existing document with read only
	// priviledges
	ReadDocument(userID, token, documentID string) (lib.BinderPortal, error)

	// CreateDocument - Create and return a binder portal to a new document
	CreateDocument(userID, token string, document store.Document) (lib.BinderPortal, error)

	// Close - Close the LeapLocator
	Close()
}

/*
LeapAdmin - An interface for performing privileged actions around the curation of leaps documents
such as user kicking and getting full lists of connected users per document.
*/
type LeapAdmin interface {
	// Kick a user from a document, needs the documentID and userID.
	KickUser(documentID, userID string, timeout time.Duration) error

	// Get the list of all users connected to all open binders.
	GetUsers(timeout time.Duration) (map[string][]string, error)
}

/*--------------------------------------------------------------------------------------------------
 */
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gnmit is a single-target gNMI collector implementation that can be
// used as an on-device/fake device implementation. It supports the Subscribe RPC
// using the libraries from openconfig/gnmi.
package gnmit

import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"reflect"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/openconfig/gnmi/cache"
	"github.com/openconfig/lemming/gnmi/subscribe"
	"github.com/openconfig/ygot/ygot"
	"github.com/openconfig/ygot/ytypes"
	"google.golang.org/grpc"
	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/proto"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
)

var (
	// metadataUpdatePeriod is the period of time after which the metadata for the collector
	// is updated to the client.
	metadataUpdatePeriod = 30 * time.Second
	// sizeUpdatePeriod is the period of time after which the storage size information for
	// the collector is updated to the client.
	sizeUpdatePeriod = 30 * time.Second
)

// periodic runs the function fn every period.
func periodic(period time.Duration, fn func()) {
	if period == 0 {
		return
	}
	t := time.NewTicker(period)
	defer t.Stop()
	for range t.C {
		fn()
	}
}

// Queue is an interface that represents a possibly coalescing queue of updates.
type Queue interface {
	Next(ctx context.Context) (interface{}, uint32, error)
	Len() int
	Close()
}

// UpdateFn is a function that takes in a gNMI Notification object and updates
// a gNMI datastore with it.
type UpdateFn func(*gpb.Notification) error

// TaskRoutine is a reactor function that listens for updates from a queue,
// emits updates via an update function. It does this on a target (string
// parameter), and also has a final clean-up function to call when it finishes
// processing.
type TaskRoutine func(Queue, UpdateFn, string, func()) error

// Task defines a particular task that runs on the gNMI datastore.
type Task struct {
	Run    TaskRoutine
	Paths  []ygot.PathStruct
	Prefix *gpb.Path
}

// GNMIServer implements the gNMI server interface.
type GNMIServer struct {
	// The subscribe Server implements only Subscribe for gNMI.
	*subscribe.Server
	c *Collector
}

// RegisterTask starts up a task on the gNMI datastore.
func (s *GNMIServer) RegisterTask(task Task) error {
	var paths []*gpb.Path
	for _, p := range task.Paths {
		path, _, err := ygot.ResolvePath(p)
		if err != nil {
			return fmt.Errorf("gnmit: cannot register task: %v", err)
		}
		paths = append(paths, path)
	}
	queue, remove, err := s.Server.SubscribeLocal(s.c.name, paths, task.Prefix)
	if err != nil {
		return err
	}
	return task.Run(queue, s.c.cache.GnmiUpdate, s.c.name, remove)
}

// New returns a new collector server implementation that can be registered on
// an existing gRPC server. It takes a string indicating the hostname of the
// target, a boolean indicating whether metadata should be sent, and a slice of
// tasks that are to be launched to run on the server.
func NewServer(ctx context.Context, hostname string, sendMeta bool, tasks []Task) (*Collector, *GNMIServer, error) {
	c := &Collector{
		inCh: make(chan *gpb.SubscribeResponse),
		name: hostname,
	}

	c.cache = cache.New([]string{hostname})
	t := c.cache.GetTarget(hostname)

	if sendMeta {
		go periodic(metadataUpdatePeriod, c.cache.UpdateMetadata)
		go periodic(sizeUpdatePeriod, c.cache.UpdateSize)
	}
	t.Connect()

	// start our single collector from the input channel.
	go func() {
		for {
			select {
			case msg := <-c.inCh:
				if err := c.handleUpdate(msg); err != nil {
					return
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	subscribeSrv, err := subscribe.NewServer(c.cache)
	if err != nil {
		return nil, nil, fmt.Errorf("could not instantiate gNMI server: %v", err)
	}

	gnmiserver := &GNMIServer{
		Server: subscribeSrv, // use the 'subscribe' implementation.
		c:      c,
	}

	for _, t := range tasks {
		if err := gnmiserver.RegisterTask(t); err != nil {
			return nil, nil, err
		}
	}
	c.cache.SetClient(subscribeSrv.Update)

	return c, gnmiserver, nil
}

// New returns a new collector that listens on the specified addr (in the form host:port),
// supporting a single downstream target named hostname. sendMeta controls whether the
// metadata *other* than meta/sync and meta/connected is sent by the collector.
//
// New returns the new collector, the address it is listening on in the form hostname:port
// or any errors encounted whilst setting it up.
func New(ctx context.Context, addr, hostname string, sendMeta bool, tasks []Task, opts ...grpc.ServerOption) (*Collector, string, error) {
	c, gnmiserver, err := NewServer(ctx, hostname, sendMeta, tasks)
	if err != nil {
		return nil, "", err
	}
	srv := grpc.NewServer(opts...)
	gpb.RegisterGNMIServer(srv, gnmiserver)
	// Forward streaming updates to clients.
	// Register listening port and start serving.
	lis, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, "", fmt.Errorf("failed to listen: %v", err)
	}

	go func() {
		if err := srv.Serve(lis); err != nil {
			log.Errorf("Error while serving gnmi target: %v", err)
		}
	}()
	c.stopFn = srv.GracefulStop
	return c, lis.Addr().String(), nil
}

type populateDefaultser interface {
	PopulateDefaults()
}

// New returns a new collector that listens on the specified addr (in the form host:port),
// supporting a single downstream target named hostname. sendMeta controls whether the
// metadata *other* than meta/sync and meta/connected is sent by the collector.
//
// New returns the new collector, the address it is listening on in the form hostname:port
// or any errors encounted whilst setting it up.
//
// NewSettable is different from New in that the returned collector is
// schema-aware and supports gNMI Set. Currently it is not possible to change
// the schema of a Collector after it is created.
func NewSettable(ctx context.Context, addr string, hostname string, sendMeta bool, schema *ytypes.Schema, tasks []Task, opts ...grpc.ServerOption) (*Collector, string, error) {
	if !schema.IsValid() {
		return nil, "", fmt.Errorf("cannot obtain valid schema for GoStructs: %v", schema)
	}

	vr, ok := schema.Root.(ygot.ValidatedGoStruct)
	if err != nil {
		return nil, "", fmt.Errorf("invalid schema root, %v", ok)
	}
	// Initialize the root with default values.
	schema.Root.(populateDefaultser).PopulateDefaults()
	if err := vr.Validate(); err != nil {
		return nil, "", fmt.Errorf("default root of input schema fails validation: %v", err)
	}

	// FIXME(wenbli): initialize the collector with default values.
	collector, addr, err := New(ctx, addr, hostname, sendMeta, tasks, opts...)
	if err != nil {
		return nil, "", err
	}
	collector.schema = schema
	return collector, addr, nil
}

// Stop halts the running collector.
func (c *Collector) Stop() {
	c.stopFn()
}

// handleUpdate handles an input gNMI SubscribeResponse that is received by
// the target.
func (c *Collector) handleUpdate(resp *gpb.SubscribeResponse) error {
	t := c.cache.GetTarget(c.name)
	switch v := resp.Response.(type) {
	case *gpb.SubscribeResponse_Update:
		return t.GnmiUpdate(v.Update)
	case *gpb.SubscribeResponse_SyncResponse:
		t.Sync()
	case *gpb.SubscribeResponse_Error:
		return fmt.Errorf("error in response: %s", v)
	default:
		return fmt.Errorf("unknown response %T: %s", v, v)
	}
	return nil
}

// Collector is a basic gNMI target that supports only the Subscribe
// RPC, and acts as a cache for exactly one target.
type Collector struct {
	cache *cache.Cache

	smu    sync.Mutex
	schema *ytypes.Schema

	// name is the hostname of the client.
	name string
	// inCh is a channel use to write new SubscribeResponses to the client.
	inCh chan *gpb.SubscribeResponse
	// stopFn is the function used to stop the server.
	stopFn func()
}

func (c *Collector) Schema() *ytypes.Schema {
	c.smu.Lock()
	defer c.smu.Unlock()
	return c.schema
}

// TargetUpdate provides an input gNMI SubscribeResponse to update the
// cache and clients with.
func (c *Collector) TargetUpdate(m *gpb.SubscribeResponse) {
	c.inCh <- m
}

// setNode is a function that is able to unmarshal either a JSON-encoded value or a gNMI-encoded value.
// TODO(wenbli): This functionality should be upstreamed to ygot somehow.
func setNode(schema *ytypes.Schema, goStruct ygot.GoStruct, update *gpb.Update) error {
	nodeName := reflect.TypeOf(goStruct).Elem().Name()
	nodeI, targetSchema, err := ytypes.GetOrCreateNode(schema.SchemaTree[nodeName], goStruct, update.Path)
	if err != nil {
		return fmt.Errorf("gnmit: failed to GetOrCreate a node: %v", err)
	}

	// TODO(wenbli): Populate default values using PopulateDefaults.
	jsonUpdate := update.Val.GetJsonIetfVal()
	if jsonUpdate == nil {
		if err := ytypes.SetNode(schema.SchemaTree[nodeName], goStruct, update.Path, update.Val); err != nil {
			return fmt.Errorf("gnmit: SetNode failed on leaf node: %v", err)
		}
		return nil
	}

	node, ok := nodeI.(ygot.GoStruct)
	path := proto.Clone(update.Path).(*gpb.Path)
	for i := len(path.Elem) - 1; i >= 0 && !ok; i-- {
		path.Elem = path.Elem[:i]
		nodeI, _, err := ytypes.GetOrCreateNode(schema.SchemaTree[nodeName], goStruct, path)
		if err != nil {
			continue
		}
		node, ok = nodeI.(ygot.GoStruct)
	}
	if ok {
		var jsonTree interface{}
		if err := json.Unmarshal(jsonUpdate, &jsonTree); err != nil {
			return err
		}
		return ytypes.Unmarshal(targetSchema, node, jsonTree)
	}

	return fmt.Errorf("gnmit: cannot find GoStruct parent into which to ummarshal update message: %s", prototext.Format(update))
}

// Set is a prototype for a gNMI Set operation.
// TODO(wenbli): This function is too complex to stand on its own. We should
// split this into stages, each encapsulated within its own function.
func (s *GNMIServer) Set(ctx context.Context, req *gpb.SetRequest) (*gpb.SetResponse, error) {
	if s.c.schema == nil {
		return s.UnimplementedGNMIServer.Set(ctx, req)
	}
	// Create a copy so that we can rollback the transaction when validation fails.
	dirtyRootG, err := ygot.DeepCopy(s.c.Schema().Root)
	if err != nil {
		return nil, fmt.Errorf("gnmit: failed to ygot.DeepCopy the cached root object: %v", err)
	}
	dirtyRoot, ok := dirtyRootG.(ygot.ValidatedGoStruct)
	if !ok {
		return nil, fmt.Errorf("gnmit: cannot convert root object to ValidatedGoStruct")
	}
	// Operate at the prefix level.
	nodeI, _, err := ytypes.GetOrCreateNode(s.c.Schema().RootSchema(), dirtyRoot, req.Prefix)
	if err != nil {
		return nil, fmt.Errorf("gnmit: failed to GetOrCreate the prefix node: %v", err)
	}
	node, ok := nodeI.(ygot.GoStruct)
	if !ok {
		return nil, fmt.Errorf("gnmit: prefix path points to a non-GoStruct, this is not allowed: %T, %v", nodeI, nodeI)
	}
	nodeName := reflect.TypeOf(nodeI).Elem().Name()

	// TODO(wenbli): Reject paths that try to modify read-only values.
	// TODO(wenbli): Question: what to do if there are operational-state values in a container that is specified to be replaced or deleted?

	// Process deletes, then replace, then updates.
	for _, path := range req.Delete {
		if err := ytypes.DeleteNode(s.c.Schema().SchemaTree[nodeName], node, path); err != nil {
			return nil, fmt.Errorf("gnmit: DeleteNode error: %v", err)
		}
	}
	for _, update := range req.Replace {
		if err := ytypes.DeleteNode(s.c.Schema().SchemaTree[nodeName], node, update.Path); err != nil {
			return nil, fmt.Errorf("gnmit: DeleteNode error: %v", err)
		}
		if err := setNode(s.c.schema, node, update); err != nil {
			return nil, err
		}
	}
	for _, update := range req.Update {
		if err := setNode(s.c.schema, node, update); err != nil {
			return nil, err
		}
	}

	if err := dirtyRoot.Validate(); err != nil {
		return nil, fmt.Errorf("gnmit: invalid SetRequest: %v", err)
	}

	n, err := ygot.Diff(s.c.Schema().Root, dirtyRoot)
	if err != nil {
		return nil, fmt.Errorf("gnmit: error while creating update notification for Set: %v", err)
	}
	n.Timestamp = time.Now().UnixNano()
	n.Prefix = &gpb.Path{Origin: req.Prefix.Origin, Target: s.c.name}
	if n.Prefix.Origin == "" {
		n.Prefix.Origin = "openconfig"
	}

	// Update cache
	t := s.c.cache.GetTarget(s.c.name)
	if err := t.GnmiUpdate(n); err != nil {
		return nil, err
	}
	// TODO(wenbli): Should handle updates one at a time to avoid partial updates being reflected in the cache when an error occurs. AKA we should support transactional semantics.
	s.c.Schema().Root = dirtyRoot
	// TODO(wenbli): Currently the SetResponse is not filled.
	return &gpb.SetResponse{}, nil
}
<commit_msg>Commit corrected version of file.<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gnmit is a single-target gNMI collector implementation that can be
// used as an on-device/fake device implementation. It supports the Subscribe RPC
// using the libraries from openconfig/gnmi.
package gnmit

import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"reflect"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/openconfig/gnmi/cache"
	"github.com/openconfig/lemming/gnmi/subscribe"
	"github.com/openconfig/ygot/ygot"
	"github.com/openconfig/ygot/ytypes"
	"google.golang.org/grpc"
	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/proto"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
)

var (
	// metadataUpdatePeriod is the period of time after which the metadata for the collector
	// is updated to the client.
	metadataUpdatePeriod = 30 * time.Second
	// sizeUpdatePeriod is the period of time after which the storage size information for
	// the collector is updated to the client.
	sizeUpdatePeriod = 30 * time.Second
)

// periodic runs the function fn every period.
func periodic(period time.Duration, fn func()) {
	if period == 0 {
		return
	}
	t := time.NewTicker(period)
	defer t.Stop()
	for range t.C {
		fn()
	}
}

// Queue is an interface that represents a possibly coalescing queue of updates.
type Queue interface {
	Next(ctx context.Context) (interface{}, uint32, error)
	Len() int
	Close()
}

// UpdateFn is a function that takes in a gNMI Notification object and updates
// a gNMI datastore with it.
type UpdateFn func(*gpb.Notification) error

// TaskRoutine is a reactor function that listens for updates from a queue,
// emits updates via an update function. It does this on a target (string
// parameter), and also has a final clean-up function to call when it finishes
// processing.
type TaskRoutine func(Queue, UpdateFn, string, func()) error

// Task defines a particular task that runs on the gNMI datastore.
type Task struct {
	Run    TaskRoutine
	Paths  []ygot.PathStruct
	Prefix *gpb.Path
}

// GNMIServer implements the gNMI server interface.
type GNMIServer struct {
	// The subscribe Server implements only Subscribe for gNMI.
	*subscribe.Server
	c *Collector
}

// RegisterTask starts up a task on the gNMI datastore.
func (s *GNMIServer) RegisterTask(task Task) error {
	var paths []*gpb.Path
	for _, p := range task.Paths {
		path, _, err := ygot.ResolvePath(p)
		if err != nil {
			return fmt.Errorf("gnmit: cannot register task: %v", err)
		}
		paths = append(paths, path)
	}
	queue, remove, err := s.Server.SubscribeLocal(s.c.name, paths, task.Prefix)
	if err != nil {
		return err
	}
	return task.Run(queue, s.c.cache.GnmiUpdate, s.c.name, remove)
}

// New returns a new collector server implementation that can be registered on
// an existing gRPC server. It takes a string indicating the hostname of the
// target, a boolean indicating whether metadata should be sent, and a slice of
// tasks that are to be launched to run on the server.
func NewServer(ctx context.Context, hostname string, sendMeta bool, tasks []Task) (*Collector, *GNMIServer, error) {
	c := &Collector{
		inCh: make(chan *gpb.SubscribeResponse),
		name: hostname,
	}

	c.cache = cache.New([]string{hostname})
	t := c.cache.GetTarget(hostname)

	if sendMeta {
		go periodic(metadataUpdatePeriod, c.cache.UpdateMetadata)
		go periodic(sizeUpdatePeriod, c.cache.UpdateSize)
	}
	t.Connect()

	// start our single collector from the input channel.
	go func() {
		for {
			select {
			case msg := <-c.inCh:
				if err := c.handleUpdate(msg); err != nil {
					return
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	subscribeSrv, err := subscribe.NewServer(c.cache)
	if err != nil {
		return nil, nil, fmt.Errorf("could not instantiate gNMI server: %v", err)
	}

	gnmiserver := &GNMIServer{
		Server: subscribeSrv, // use the 'subscribe' implementation.
		c:      c,
	}

	for _, t := range tasks {
		if err := gnmiserver.RegisterTask(t); err != nil {
			return nil, nil, err
		}
	}
	c.cache.SetClient(subscribeSrv.Update)

	return c, gnmiserver, nil
}

// New returns a new collector that listens on the specified addr (in the form host:port),
// supporting a single downstream target named hostname. sendMeta controls whether the
// metadata *other* than meta/sync and meta/connected is sent by the collector.
//
// New returns the new collector, the address it is listening on in the form hostname:port
// or any errors encounted whilst setting it up.
func New(ctx context.Context, addr, hostname string, sendMeta bool, tasks []Task, opts ...grpc.ServerOption) (*Collector, string, error) {
	c, gnmiserver, err := NewServer(ctx, hostname, sendMeta, tasks)
	if err != nil {
		return nil, "", err
	}
	srv := grpc.NewServer(opts...)
	gpb.RegisterGNMIServer(srv, gnmiserver)
	// Forward streaming updates to clients.
	// Register listening port and start serving.
	lis, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, "", fmt.Errorf("failed to listen: %v", err)
	}

	go func() {
		if err := srv.Serve(lis); err != nil {
			log.Errorf("Error while serving gnmi target: %v", err)
		}
	}()
	c.stopFn = srv.GracefulStop
	return c, lis.Addr().String(), nil
}

type populateDefaultser interface {
	PopulateDefaults()
}

// New returns a new collector that listens on the specified addr (in the form host:port),
// supporting a single downstream target named hostname. sendMeta controls whether the
// metadata *other* than meta/sync and meta/connected is sent by the collector.
//
// New returns the new collector, the address it is listening on in the form hostname:port
// or any errors encounted whilst setting it up.
//
// NewSettable is different from New in that the returned collector is
// schema-aware and supports gNMI Set. Currently it is not possible to change
// the schema of a Collector after it is created.
func NewSettable(ctx context.Context, addr string, hostname string, sendMeta bool, schema *ytypes.Schema, tasks []Task, opts ...grpc.ServerOption) (*Collector, string, error) {
	if !schema.IsValid() {
		return nil, "", fmt.Errorf("cannot obtain valid schema for GoStructs: %v", schema)
	}

	vr, ok := schema.Root.(ygot.ValidatedGoStruct)
	if !ok {
		return nil, "", fmt.Errorf("invalid schema root, %v", schema.Root)
	}
	// Initialize the root with default values.
	schema.Root.(populateDefaultser).PopulateDefaults()
	if err := vr.Validate(); err != nil {
		return nil, "", fmt.Errorf("default root of input schema fails validation: %v", err)
	}

	// FIXME(wenbli): initialize the collector with default values.
	collector, addr, err := New(ctx, addr, hostname, sendMeta, tasks, opts...)
	if err != nil {
		return nil, "", err
	}
	collector.schema = schema
	return collector, addr, nil
}

// Stop halts the running collector.
func (c *Collector) Stop() {
	c.stopFn()
}

// handleUpdate handles an input gNMI SubscribeResponse that is received by
// the target.
func (c *Collector) handleUpdate(resp *gpb.SubscribeResponse) error {
	t := c.cache.GetTarget(c.name)
	switch v := resp.Response.(type) {
	case *gpb.SubscribeResponse_Update:
		return t.GnmiUpdate(v.Update)
	case *gpb.SubscribeResponse_SyncResponse:
		t.Sync()
	case *gpb.SubscribeResponse_Error:
		return fmt.Errorf("error in response: %s", v)
	default:
		return fmt.Errorf("unknown response %T: %s", v, v)
	}
	return nil
}

// Collector is a basic gNMI target that supports only the Subscribe
// RPC, and acts as a cache for exactly one target.
type Collector struct {
	cache *cache.Cache

	smu    sync.Mutex
	schema *ytypes.Schema

	// name is the hostname of the client.
	name string
	// inCh is a channel use to write new SubscribeResponses to the client.
	inCh chan *gpb.SubscribeResponse
	// stopFn is the function used to stop the server.
	stopFn func()
}

func (c *Collector) Schema() *ytypes.Schema {
	c.smu.Lock()
	defer c.smu.Unlock()
	return c.schema
}

// TargetUpdate provides an input gNMI SubscribeResponse to update the
// cache and clients with.
func (c *Collector) TargetUpdate(m *gpb.SubscribeResponse) {
	c.inCh <- m
}

// setNode is a function that is able to unmarshal either a JSON-encoded value or a gNMI-encoded value.
// TODO(wenbli): This functionality should be upstreamed to ygot somehow.
func setNode(schema *ytypes.Schema, goStruct ygot.GoStruct, update *gpb.Update) error {
	nodeName := reflect.TypeOf(goStruct).Elem().Name()
	nodeI, targetSchema, err := ytypes.GetOrCreateNode(schema.SchemaTree[nodeName], goStruct, update.Path)
	if err != nil {
		return fmt.Errorf("gnmit: failed to GetOrCreate a node: %v", err)
	}

	// TODO(wenbli): Populate default values using PopulateDefaults.
	jsonUpdate := update.Val.GetJsonIetfVal()
	if jsonUpdate == nil {
		if err := ytypes.SetNode(schema.SchemaTree[nodeName], goStruct, update.Path, update.Val); err != nil {
			return fmt.Errorf("gnmit: SetNode failed on leaf node: %v", err)
		}
		return nil
	}

	node, ok := nodeI.(ygot.GoStruct)
	path := proto.Clone(update.Path).(*gpb.Path)
	for i := len(path.Elem) - 1; i >= 0 && !ok; i-- {
		path.Elem = path.Elem[:i]
		nodeI, _, err := ytypes.GetOrCreateNode(schema.SchemaTree[nodeName], goStruct, path)
		if err != nil {
			continue
		}
		node, ok = nodeI.(ygot.GoStruct)
	}
	if ok {
		var jsonTree interface{}
		if err := json.Unmarshal(jsonUpdate, &jsonTree); err != nil {
			return err
		}
		return ytypes.Unmarshal(targetSchema, node, jsonTree)
	}

	return fmt.Errorf("gnmit: cannot find GoStruct parent into which to ummarshal update message: %s", prototext.Format(update))
}

// Set is a prototype for a gNMI Set operation.
// TODO(wenbli): This function is too complex to stand on its own. We should
// split this into stages, each encapsulated within its own function.
func (s *GNMIServer) Set(ctx context.Context, req *gpb.SetRequest) (*gpb.SetResponse, error) {
	if s.c.schema == nil {
		return s.UnimplementedGNMIServer.Set(ctx, req)
	}
	// Create a copy so that we can rollback the transaction when validation fails.
	dirtyRootG, err := ygot.DeepCopy(s.c.Schema().Root)
	if err != nil {
		return nil, fmt.Errorf("gnmit: failed to ygot.DeepCopy the cached root object: %v", err)
	}
	dirtyRoot, ok := dirtyRootG.(ygot.ValidatedGoStruct)
	if !ok {
		return nil, fmt.Errorf("gnmit: cannot convert root object to ValidatedGoStruct")
	}
	// Operate at the prefix level.
	nodeI, _, err := ytypes.GetOrCreateNode(s.c.Schema().RootSchema(), dirtyRoot, req.Prefix)
	if err != nil {
		return nil, fmt.Errorf("gnmit: failed to GetOrCreate the prefix node: %v", err)
	}
	node, ok := nodeI.(ygot.GoStruct)
	if !ok {
		return nil, fmt.Errorf("gnmit: prefix path points to a non-GoStruct, this is not allowed: %T, %v", nodeI, nodeI)
	}
	nodeName := reflect.TypeOf(nodeI).Elem().Name()

	// TODO(wenbli): Reject paths that try to modify read-only values.
	// TODO(wenbli): Question: what to do if there are operational-state values in a container that is specified to be replaced or deleted?

	// Process deletes, then replace, then updates.
	for _, path := range req.Delete {
		if err := ytypes.DeleteNode(s.c.Schema().SchemaTree[nodeName], node, path); err != nil {
			return nil, fmt.Errorf("gnmit: DeleteNode error: %v", err)
		}
	}
	for _, update := range req.Replace {
		if err := ytypes.DeleteNode(s.c.Schema().SchemaTree[nodeName], node, update.Path); err != nil {
			return nil, fmt.Errorf("gnmit: DeleteNode error: %v", err)
		}
		if err := setNode(s.c.schema, node, update); err != nil {
			return nil, err
		}
	}
	for _, update := range req.Update {
		if err := setNode(s.c.schema, node, update); err != nil {
			return nil, err
		}
	}

	if err := dirtyRoot.Validate(); err != nil {
		return nil, fmt.Errorf("gnmit: invalid SetRequest: %v", err)
	}

	n, err := ygot.Diff(s.c.Schema().Root, dirtyRoot)
	if err != nil {
		return nil, fmt.Errorf("gnmit: error while creating update notification for Set: %v", err)
	}
	n.Timestamp = time.Now().UnixNano()
	n.Prefix = &gpb.Path{Origin: req.Prefix.Origin, Target: s.c.name}
	if n.Prefix.Origin == "" {
		n.Prefix.Origin = "openconfig"
	}

	// Update cache
	t := s.c.cache.GetTarget(s.c.name)
	if err := t.GnmiUpdate(n); err != nil {
		return nil, err
	}
	// TODO(wenbli): Should handle updates one at a time to avoid partial updates being reflected in the cache when an error occurs. AKA we should support transactional semantics.
	s.c.Schema().Root = dirtyRoot
	// TODO(wenbli): Currently the SetResponse is not filled.
	return &gpb.SetResponse{}, nil
}
<|endoftext|>