<commit_before>// Package protoparser contains a protobuf parser.
// nolint: govet, golint
package parser

import (
	"fmt"
	"io"
	"math/big"
	"strings"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

type Proto struct {
	Pos lexer.Position

	Entries []*Entry `{ @@ { ";" } }`
}

type Entry struct {
	Pos lexer.Position

	Syntax  string   `  "syntax" "=" @String`
	Package string   `| "package" @(Ident { "." Ident })`
	Import  *Import  `| @@`
	Message *Message `| @@`
	Service *Service `| @@`
	Enum    *Enum    `| @@`
	Option  *Option  `| "option" @@`
	Extend  *Extend  `| @@`
}

type Import struct {
	Public bool   `"import" @("public")?`
	Name   string `@String`
}

type Option struct {
	Pos lexer.Position

	Name  string  `( "(" @("."? Ident { "." Ident }) ")" | @("."? Ident { "." Ident }) )`
	Attr  *string `[ @("."? Ident { "." Ident }) ]`
	Value *Value  `"=" @@`
}

type Value struct {
	Pos lexer.Position

	String    *string    `  @String`
	Number    *big.Float `| ("-" | "+")? (@Float | @Int)`
	Bool      *bool      `| (@"true" | "false")`
	Reference *string    `| @("."? Ident { "." Ident })`
	ProtoText *ProtoText `| "{" @@ "}"`
	Array     *Array     `| @@`
}

type ProtoText struct {
	Pos lexer.Position

	Fields []ProtoTextField `( @@ ( "," | ";" )? )*`
}

type ProtoTextField struct {
	Pos lexer.Position

	Name  string `(@Ident | ( "[" @("."? Ident { "." Ident }) "]" ))`
	Value *Value `( ":"? @@ )`
}

type Array struct {
	Pos lexer.Position

	Elements []*Value `"[" [ @@ { [ "," ] @@ } ] "]"`
}

type Extensions struct {
	Pos lexer.Position

	Extensions []Range `"extensions" @@ { "," @@ }`
}

type Reserved struct {
	Pos lexer.Position

	Ranges     []Range  `@@ { "," @@ }`
	FieldNames []string `| @String { "," @String }`
}

type Range struct {
	Start int  `@Int`
	End   *int `  [ "to" ( @Int`
	Max   bool `           | @"max" ) ]`
}

type Extend struct {
	Pos lexer.Position

	Reference string   `"extend" @("."? Ident { "." Ident })`
	Fields    []*Field `"{" { @@ [ ";" ] } "}"`
}

type Service struct {
	Pos lexer.Position

	Name  string          `"service" @Ident`
	Entry []*ServiceEntry `"{" { @@ [ ";" ] } "}"`
}

type ServiceEntry struct {
	Pos lexer.Position

	Option *Option `  "option" @@`
	Method *Method `| @@`
}

type Method struct {
	Pos lexer.Position

	Name              string    `"rpc" @Ident`
	StreamingRequest  bool      `"(" [ @"stream" ]`
	Request           *Type     `    @@ ")"`
	StreamingResponse bool      `"returns" "(" [ @"stream" ]`
	Response          *Type     `              @@ ")"`
	Options           []*Option `[ "{" { "option" @@ ";" } "}" ]`
}

type Enum struct {
	Pos lexer.Position

	Name   string       `"enum" @Ident`
	Values []*EnumEntry `"{" { @@ { ";" } } "}"`
}

type EnumEntry struct {
	Pos lexer.Position

	Value    *EnumValue `  @@`
	Option   *Option    `| "option" @@`
	Reserved *Reserved  `| "reserved" @@`
}

type EnumValue struct {
	Pos lexer.Position

	Key   string `@Ident`
	Value int    `"=" @( [ "-" ] Int )`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Message struct {
	Pos lexer.Position

	Name    string          `"message" @Ident`
	Entries []*MessageEntry `"{" { @@ } "}"`
}

type MessageEntry struct {
	Pos lexer.Position

	Enum       *Enum       `( @@`
	Option     *Option     ` | "option" @@`
	Message    *Message    ` | @@`
	Oneof      *OneOf      ` | @@`
	Extend     *Extend     ` | @@`
	Reserved   *Reserved   ` | "reserved" @@`
	Extensions *Extensions ` | @@`
	Field      *Field      ` | @@ ) { ";" }`
}

type OneOf struct {
	Pos lexer.Position

	Name    string        `"oneof" @Ident`
	Entries []*OneOfEntry `"{" { @@ { ";" } } "}"`
}

type OneOfEntry struct {
	Pos lexer.Position

	Field  *Field  `@@`
	Option *Option `| "option" @@`
}

type Field struct {
	Pos lexer.Position

	Optional bool `[   @"optional"`
	Required bool `  | @"required"`
	Repeated bool `  | @"repeated" ]`

	Group  *Group  `( @@`
	Direct *Direct `| @@ )`
}

type Direct struct {
	Pos lexer.Position

	Type *Type  `@@`
	Name string `@Ident`
	Tag  int    `"=" @Int`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Group struct {
	Pos lexer.Position

	Name    string          `"group" @Ident`
	Tag     int             `"=" @Int`
	Entries []*MessageEntry `"{" { @@ [ ";" ] } "}"`
}

type Scalar int

const (
	None Scalar = iota
	Double
	Float
	Int32
	Int64
	Uint32
	Uint64
	Sint32
	Sint64
	Fixed32
	Fixed64
	SFixed32
	SFixed64
	Bool
	String
	Bytes
)

var scalarToString = map[Scalar]string{
	None: "None", Double: "Double", Float: "Float", Int32: "Int32", Int64: "Int64", Uint32: "Uint32",
	Uint64: "Uint64", Sint32: "Sint32", Sint64: "Sint64", Fixed32: "Fixed32", Fixed64: "Fixed64",
	SFixed32: "SFixed32", SFixed64: "SFixed64", Bool: "Bool", String: "String", Bytes: "Bytes",
}

func (s Scalar) GoString() string { return scalarToString[s] }

var stringToScalar = map[string]Scalar{
	"double": Double, "float": Float, "int32": Int32, "int64": Int64, "uint32": Uint32, "uint64": Uint64,
	"sint32": Sint32, "sint64": Sint64, "fixed32": Fixed32, "fixed64": Fixed64, "sfixed32": SFixed32,
	"sfixed64": SFixed64, "bool": Bool, "string": String, "bytes": Bytes,
}

func (s *Scalar) Parse(lex *lexer.PeekingLexer) error {
	token, err := lex.Peek(0)
	if err != nil {
		return fmt.Errorf("failed to peek next token: %w", err)
	}
	scalar, ok := stringToScalar[token.Value]
	if !ok {
		return participle.NextMatch
	}
	_, err = lex.Next()
	if err != nil {
		return fmt.Errorf("failed to read next token: %w", err)
	}
	*s = scalar
	return nil
}

type Type struct {
	Pos lexer.Position

	Scalar    Scalar   `  @@`
	Map       *MapType `| @@`
	Reference *string  `| @("."? Ident { "." Ident })`
}

type MapType struct {
	Pos lexer.Position

	Key   *Type `"map" "<" @@`
	Value *Type `"," @@ ">"`
}

// Parse protobuf.
func Parse(filename string, r io.Reader) (*Proto, error) {
	p := &Proto{}

	l := lexer.MustSimple([]lexer.Rule{
		{"String", `"(\\"|[^"])*"|'(\\'|[^'])*'`, nil},
		{"Ident", `[a-zA-Z_]([a-zA-Z_0-9])*`, nil},
		{"Float", `[-+]?(\d*\.\d+([eE]\d+)?|\d+[eE]\d+|inf)`, nil},
		{"Int", `(0[xX][0-9A-Fa-f]+)|([-+]?\d+)`, nil},
		{"Whitespace", `[ \t\n\r\s]+`, nil},
		{"BlockComment", `/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/`, nil},
		{"LineComment", `//(.*)[^\n]*\n`, nil},
		{"Symbols", `[={}\[\]()<>.,;:]`, nil},
	})

	parser := participle.MustBuild(
		&Proto{},
		participle.UseLookahead(2),
		participle.Unquote("String"),
		participle.Lexer(l),
		participle.Elide("Whitespace", "LineComment", "BlockComment"),
	)
	err := parser.Parse(filename, r, p)
	if err != nil {
		return p, err
	}
	return p, nil
}

func ParseString(filename string, source string) (*Proto, error) {
	return Parse(filename, strings.NewReader(source))
}
<commit_msg>Fix linter.<commit_after>// Package parser contains a protobuf parser.
// nolint: govet, golint
package parser

import (
	"fmt"
	"io"
	"math/big"
	"strings"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

type Proto struct {
	Pos lexer.Position

	Entries []*Entry `{ @@ { ";" } }`
}

type Entry struct {
	Pos lexer.Position

	Syntax  string   `  "syntax" "=" @String`
	Package string   `| "package" @(Ident { "." Ident })`
	Import  *Import  `| @@`
	Message *Message `| @@`
	Service *Service `| @@`
	Enum    *Enum    `| @@`
	Option  *Option  `| "option" @@`
	Extend  *Extend  `| @@`
}

type Import struct {
	Public bool   `"import" @("public")?`
	Name   string `@String`
}

type Option struct {
	Pos lexer.Position

	Name  string  `( "(" @("."? Ident { "." Ident }) ")" | @("."? Ident { "." Ident }) )`
	Attr  *string `[ @("."? Ident { "." Ident }) ]`
	Value *Value  `"=" @@`
}

type Value struct {
	Pos lexer.Position

	String    *string    `  @String`
	Number    *big.Float `| ("-" | "+")? (@Float | @Int)`
	Bool      *bool      `| (@"true" | "false")`
	Reference *string    `| @("."? Ident { "." Ident })`
	ProtoText *ProtoText `| "{" @@ "}"`
	Array     *Array     `| @@`
}

type ProtoText struct {
	Pos lexer.Position

	Fields []ProtoTextField `( @@ ( "," | ";" )? )*`
}

type ProtoTextField struct {
	Pos lexer.Position

	Name  string `(@Ident | ( "[" @("."? Ident { "." Ident }) "]" ))`
	Value *Value `( ":"? @@ )`
}

type Array struct {
	Pos lexer.Position

	Elements []*Value `"[" [ @@ { [ "," ] @@ } ] "]"`
}

type Extensions struct {
	Pos lexer.Position

	Extensions []Range `"extensions" @@ { "," @@ }`
}

type Reserved struct {
	Pos lexer.Position

	Ranges     []Range  `@@ { "," @@ }`
	FieldNames []string `| @String { "," @String }`
}

type Range struct {
	Start int  `@Int`
	End   *int `  [ "to" ( @Int`
	Max   bool `           | @"max" ) ]`
}

type Extend struct {
	Pos lexer.Position

	Reference string   `"extend" @("."? Ident { "." Ident })`
	Fields    []*Field `"{" { @@ [ ";" ] } "}"`
}

type Service struct {
	Pos lexer.Position

	Name  string          `"service" @Ident`
	Entry []*ServiceEntry `"{" { @@ [ ";" ] } "}"`
}

type ServiceEntry struct {
	Pos lexer.Position

	Option *Option `  "option" @@`
	Method *Method `| @@`
}

type Method struct {
	Pos lexer.Position

	Name              string    `"rpc" @Ident`
	StreamingRequest  bool      `"(" [ @"stream" ]`
	Request           *Type     `    @@ ")"`
	StreamingResponse bool      `"returns" "(" [ @"stream" ]`
	Response          *Type     `              @@ ")"`
	Options           []*Option `[ "{" { "option" @@ ";" } "}" ]`
}

type Enum struct {
	Pos lexer.Position

	Name   string       `"enum" @Ident`
	Values []*EnumEntry `"{" { @@ { ";" } } "}"`
}

type EnumEntry struct {
	Pos lexer.Position

	Value    *EnumValue `  @@`
	Option   *Option    `| "option" @@`
	Reserved *Reserved  `| "reserved" @@`
}

type EnumValue struct {
	Pos lexer.Position

	Key   string `@Ident`
	Value int    `"=" @( [ "-" ] Int )`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Message struct {
	Pos lexer.Position

	Name    string          `"message" @Ident`
	Entries []*MessageEntry `"{" { @@ } "}"`
}

type MessageEntry struct {
	Pos lexer.Position

	Enum       *Enum       `( @@`
	Option     *Option     ` | "option" @@`
	Message    *Message    ` | @@`
	Oneof      *OneOf      ` | @@`
	Extend     *Extend     ` | @@`
	Reserved   *Reserved   ` | "reserved" @@`
	Extensions *Extensions ` | @@`
	Field      *Field      ` | @@ ) { ";" }`
}

type OneOf struct {
	Pos lexer.Position

	Name    string        `"oneof" @Ident`
	Entries []*OneOfEntry `"{" { @@ { ";" } } "}"`
}

type OneOfEntry struct {
	Pos lexer.Position

	Field  *Field  `@@`
	Option *Option `| "option" @@`
}

type Field struct {
	Pos lexer.Position

	Optional bool `[   @"optional"`
	Required bool `  | @"required"`
	Repeated bool `  | @"repeated" ]`

	Group  *Group  `( @@`
	Direct *Direct `| @@ )`
}

type Direct struct {
	Pos lexer.Position

	Type *Type  `@@`
	Name string `@Ident`
	Tag  int    `"=" @Int`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Group struct {
	Pos lexer.Position

	Name    string          `"group" @Ident`
	Tag     int             `"=" @Int`
	Entries []*MessageEntry `"{" { @@ [ ";" ] } "}"`
}

type Scalar int

const (
	None Scalar = iota
	Double
	Float
	Int32
	Int64
	Uint32
	Uint64
	Sint32
	Sint64
	Fixed32
	Fixed64
	SFixed32
	SFixed64
	Bool
	String
	Bytes
)

var scalarToString = map[Scalar]string{
	None: "None", Double: "Double", Float: "Float", Int32: "Int32", Int64: "Int64", Uint32: "Uint32",
	Uint64: "Uint64", Sint32: "Sint32", Sint64: "Sint64", Fixed32: "Fixed32", Fixed64: "Fixed64",
	SFixed32: "SFixed32", SFixed64: "SFixed64", Bool: "Bool", String: "String", Bytes: "Bytes",
}

func (s Scalar) GoString() string { return scalarToString[s] }

var stringToScalar = map[string]Scalar{
	"double": Double, "float": Float, "int32": Int32, "int64": Int64, "uint32": Uint32, "uint64": Uint64,
	"sint32": Sint32, "sint64": Sint64, "fixed32": Fixed32, "fixed64": Fixed64, "sfixed32": SFixed32,
	"sfixed64": SFixed64, "bool": Bool, "string": String, "bytes": Bytes,
}

func (s *Scalar) Parse(lex *lexer.PeekingLexer) error {
	token, err := lex.Peek(0)
	if err != nil {
		return fmt.Errorf("failed to peek next token: %w", err)
	}
	scalar, ok := stringToScalar[token.Value]
	if !ok {
		return participle.NextMatch
	}
	_, err = lex.Next()
	if err != nil {
		return fmt.Errorf("failed to read next token: %w", err)
	}
	*s = scalar
	return nil
}

type Type struct {
	Pos lexer.Position

	Scalar    Scalar   `  @@`
	Map       *MapType `| @@`
	Reference *string  `| @("."? Ident { "." Ident })`
}

type MapType struct {
	Pos lexer.Position

	Key   *Type `"map" "<" @@`
	Value *Type `"," @@ ">"`
}

// Parse protobuf.
func Parse(filename string, r io.Reader) (*Proto, error) {
	p := &Proto{}

	l := lexer.MustSimple([]lexer.Rule{
		{"String", `"(\\"|[^"])*"|'(\\'|[^'])*'`, nil},
		{"Ident", `[a-zA-Z_]([a-zA-Z_0-9])*`, nil},
		{"Float", `[-+]?(\d*\.\d+([eE]\d+)?|\d+[eE]\d+|inf)`, nil},
		{"Int", `(0[xX][0-9A-Fa-f]+)|([-+]?\d+)`, nil},
		{"Whitespace", `[ \t\n\r\s]+`, nil},
		{"BlockComment", `/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/`, nil},
		{"LineComment", `//(.*)[^\n]*\n`, nil},
		{"Symbols", `[={}\[\]()<>.,;:]`, nil},
	})

	parser := participle.MustBuild(
		&Proto{},
		participle.UseLookahead(2),
		participle.Unquote("String"),
		participle.Lexer(l),
		participle.Elide("Whitespace", "LineComment", "BlockComment"),
	)
	err := parser.Parse(filename, r, p)
	if err != nil {
		return p, err
	}
	return p, nil
}

func ParseString(filename string, source string) (*Proto, error) {
	return Parse(filename, strings.NewReader(source))
}
<|endoftext|>


<commit_before>package parser

import (
  "io"
  "fmt"
  "github.com/bkidney/gofelex"
)

type Parser struct {
  s *gofelex.Scanner
  buf struct {
    tok gofelex.Token // Last read token
    lit string // Last read literal
    n int // Buffer Size (max = 1)
  }
}

func NewParser(r io.Reader) *Parser {
  return &Parser{s: gofelex.NewScanner(r)}
}

func (p *Parser) Parse() (string, error) {
  p.scanIgnoreWhitespace()
  return p.query()
}

func (p *Parser) query() (string, error) {
  var out string
  var err error

  if p.buf.tok == gofelex.IDENT {
    p.scanIgnoreWhitespace()
    out, err = p.action()
  } else {
    err = fmt.Errorf("found %q, expected IDENT", p.buf.lit)
    return "", err
  }

  return out, nil
}

func (p *Parser) action() (string, error) {
  return "", nil
}

func (p *Parser) join() (string, error) {
  return "", nil
}

func (p *Parser) logical() (string, error) {
  return "", nil
}

func (p *Parser) temporal() (string, error) {
  return "", nil
}

func (p *Parser) conditional() (string, error) {
  return "", nil
}

func (p *Parser) flow() (string, error) {
  return "", nil
}

// Utility Functions
func (p *Parser) scan() (tok gofelex.Token, lit string) {

  // Return any token in the buffer
  if p.buf.n != 0 {
    p.buf.n = 0
    return p.buf.tok, p.buf.lit
  }

  // Otherwise read a new one in
  tok, lit = p.s.Scan()

  // And save to buffer
  p.buf.tok, p.buf.lit = tok, lit

  return
}

func (p *Parser) scanIgnoreWhitespace() {
  var tok gofelex.Token

  tok, _ = p.scan()
  if tok == gofelex.WS {
    tok, _ = p.scan()
  }
}

func (p *Parser) unscan() { p.buf.n = 1 }
<commit_msg>Completes parsing, outputing TOKENS.<commit_after>package parser

import (
  "io"
  "fmt"
  "github.com/bkidney/gofelex"
)

type Parser struct {
  s *gofelex.Scanner
  buf struct {
    tok gofelex.Token // Last read token
    lit string // Last read literal
    n int // Buffer Size (max = 1)
  }
}

func NewParser(r io.Reader) *Parser {
  return &Parser{s: gofelex.NewScanner(r)}
}

func (p *Parser) Parse() (string, error) {
  p.scanIgnoreWhitespace()
  return p.query()
}

func (p *Parser) query() (string, error) {
  var out, ret string
  var err error

  err = nil

  if p.buf.tok == gofelex.IDENT {
    out, err = p.action()
  } else {
    err = fmt.Errorf("found %q, expected IDENT", p.buf.lit)
    return "", err
  }

  ret = out

  if p.buf.tok != gofelex.EOF {
    out, err = p.join()
    ret = ret + " " + out
  }

  return ret, err
}

func (p *Parser) action() (string, error) {
  var out string
  var err error

  out = "IDENT"
  err = nil

  p.scanIgnoreWhitespace()
  return out, err
}

func (p *Parser) join() (string, error) {
  var ret, out string
  var err error

  if p.buf.tok == gofelex.LOGICAL {
    out, err = p.logical()
  } else if p.buf.tok == gofelex.TEMPORAL {
    out, err = p.temporal()
  } else if p.buf.tok == gofelex.CONDITION {
    out, err = p.conditional()
  } else if p.buf.tok == gofelex.FLOW {
    out, err = p.flow()
  } else {
    err = fmt.Errorf("found %q, expected join type")
    return "", err
  }

  ret = out
  out, err = p.query()

  ret = ret + " " + out

  return ret, err
}

func (p *Parser) logical() (string, error) {
  var out string
  var err error

  out = "LOGICAL"
  err = nil

  p.scanIgnoreWhitespace()
  return out, err
}

func (p *Parser) temporal() (string, error) {
  var out string
  var err error

  out = "TEMPORAL"
  err = nil

  p.scanIgnoreWhitespace()
  return out, err
}

func (p *Parser) conditional() (string, error) {
  var out string
  var err error

  out = "CONDITION"
  err = nil

  p.scanIgnoreWhitespace()
  return out, err
}

func (p *Parser) flow() (string, error) {
  var out string
  var err error

  out = "FLOW"
  err = nil

  p.scanIgnoreWhitespace()
  return out, err
}

// Utility Functions
func (p *Parser) scan() (tok gofelex.Token, lit string) {

  // Return any token in the buffer
  if p.buf.n != 0 {
    p.buf.n = 0
    return p.buf.tok, p.buf.lit
  }

  // Otherwise read a new one in
  tok, lit = p.s.Scan()

  // And save to buffer
  p.buf.tok, p.buf.lit = tok, lit

  return
}

func (p *Parser) scanIgnoreWhitespace() {
  var tok gofelex.Token

  tok, _ = p.scan()
  if tok == gofelex.WS {
    tok, _ = p.scan()
  }
}

func (p *Parser) unscan() { p.buf.n = 1 }
<|endoftext|>


<commit_before>package vizzini_test

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"

	"github.com/onsi/say"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var bbsClient bbs.Client
var serviceClient bbs.ServiceClient
var domain string
var otherDomain string
var defaultRootFS string
var guid string
var startTime time.Time

var bbsAddress string
var bbsCA string
var bbsClientCert string
var bbsClientKey string
var consulAddress string
var routableDomainSuffix string
var hostAddress string
var logger lager.Logger

func init() {
	flag.StringVar(&bbsAddress, "bbs-address", "http://10.244.16.130:8889", "http address for the bbs (required)")
	flag.StringVar(&bbsCA, "bbs-ca", "", "bbs ca cert")
	flag.StringVar(&bbsClientCert, "bbs-client-cert", "", "bbs client ssl certificate")
	flag.StringVar(&bbsClientKey, "bbs-client-key", "", "bbs client ssl key")
	flag.StringVar(&consulAddress, "consul-address", "http://127.0.0.1:8500", "http address for the consul agent (required)")
	flag.StringVar(&routableDomainSuffix, "routable-domain-suffix", "bosh-lite.com", "suffix to use when constructing FQDN")
	flag.StringVar(&hostAddress, "host-address", "10.0.2.2", "address that a process running in a container on Diego can use to reach the machine running this test.  Typically the gateway on the vagrant VM.")
	flag.Parse()

	if bbsAddress == "" {
		log.Fatal("i need a bbs address to talk to Diego...")
	}

	if consulAddress == "" {
		log.Fatal("i need a consul address to talk to Diego...")
	}
}

func TestVizziniSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Vizzini Suite")
}

func NewGuid() string {
	u, err := uuid.NewV4()
	Ω(err).ShouldNot(HaveOccurred())
	return domain + "-" + u.String()[:8]
}

var _ = BeforeSuite(func() {
	timeout := os.Getenv("DEFAULT_EVENTUALLY_TIMEOUT")
	if timeout == "" {
		SetDefaultEventuallyTimeout(10 * time.Second)
	} else {
		duration, err := time.ParseDuration(timeout)
		Ω(err).ShouldNot(HaveOccurred(), "invalid timeout")
		fmt.Printf("Setting Default Eventually Timeout to %s\n", duration)
		SetDefaultEventuallyTimeout(duration)
	}
	SetDefaultEventuallyPollingInterval(500 * time.Millisecond)
	SetDefaultConsistentlyPollingInterval(200 * time.Millisecond)
	domain = fmt.Sprintf("vizzini-%d", GinkgoParallelNode())
	otherDomain = fmt.Sprintf("vizzini-other-%d", GinkgoParallelNode())
	defaultRootFS = models.PreloadedRootFS("cflinuxfs2")

	var err error
	bbsClient = initializeBBSClient()

	consulClient, err := consuladapter.NewClient(consulAddress)
	Ω(err).ShouldNot(HaveOccurred())

	sessionMgr := consuladapter.NewSessionManager(consulClient)
	consulSession, err := consuladapter.NewSession("vizzini", 10*time.Second, consulClient, sessionMgr)
	Ω(err).ShouldNot(HaveOccurred())

	logger = lagertest.NewTestLogger("vizzini")

	serviceClient = bbs.NewServiceClient(consulSession, clock.NewClock())
})

var _ = BeforeEach(func() {
	startTime = time.Now()
	guid = NewGuid()
})

var _ = AfterEach(func() {
	defer func() {
		endTime := time.Now()
		fmt.Fprint(GinkgoWriter, say.Cyan("\n%s\nThis test referenced GUID %s\nStart time: %s (%d)\nEnd time: %s (%d)\n", CurrentGinkgoTestDescription().FullTestText, guid, startTime, startTime.Unix(), endTime, endTime.Unix()))
	}()

	for _, domain := range []string{domain, otherDomain} {
		ClearOutTasksInDomain(domain)
		ClearOutDesiredLRPsInDomain(domain)
	}
})

var _ = AfterSuite(func() {
	for _, domain := range []string{domain, otherDomain} {
		bbsClient.UpsertDomain(domain, 5*time.Minute) //leave the domain around forever so that Diego cleans up if need be
	}

	for _, domain := range []string{domain, otherDomain} {
		ClearOutDesiredLRPsInDomain(domain)
		ClearOutTasksInDomain(domain)
	}
})

func initializeBBSClient() bbs.Client {
	bbsURL, err := url.Parse(bbsAddress)
	Ω(err).ShouldNot(HaveOccurred())

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(bbsAddress, bbsCA, bbsClientCert, bbsClientKey, 0, 0)
	Ω(err).ShouldNot(HaveOccurred())
	return bbsClient
}
<commit_msg>Use new bbs models<commit_after>package vizzini_test

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"

	"github.com/onsi/say"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/consuladapter"
)

var bbsClient bbs.Client
var serviceClient bbs.ServiceClient
var domain string
var otherDomain string
var defaultRootFS string
var guid string
var startTime time.Time

var bbsAddress string
var bbsCA string
var bbsClientCert string
var bbsClientKey string
var consulAddress string
var routableDomainSuffix string
var hostAddress string
var logger lager.Logger

func init() {
	flag.StringVar(&bbsAddress, "bbs-address", "http://10.244.16.130:8889", "http address for the bbs (required)")
	flag.StringVar(&bbsCA, "bbs-ca", "", "bbs ca cert")
	flag.StringVar(&bbsClientCert, "bbs-client-cert", "", "bbs client ssl certificate")
	flag.StringVar(&bbsClientKey, "bbs-client-key", "", "bbs client ssl key")
	flag.StringVar(&consulAddress, "consul-address", "http://127.0.0.1:8500", "http address for the consul agent (required)")
	flag.StringVar(&routableDomainSuffix, "routable-domain-suffix", "bosh-lite.com", "suffix to use when constructing FQDN")
	flag.StringVar(&hostAddress, "host-address", "10.0.2.2", "address that a process running in a container on Diego can use to reach the machine running this test.  Typically the gateway on the vagrant VM.")
	flag.Parse()

	if bbsAddress == "" {
		log.Fatal("i need a bbs address to talk to Diego...")
	}

	if consulAddress == "" {
		log.Fatal("i need a consul address to talk to Diego...")
	}
}

func TestVizziniSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Vizzini Suite")
}

func NewGuid() string {
	u, err := uuid.NewV4()
	Ω(err).ShouldNot(HaveOccurred())
	return domain + "-" + u.String()[:8]
}

var _ = BeforeSuite(func() {
	timeout := os.Getenv("DEFAULT_EVENTUALLY_TIMEOUT")
	if timeout == "" {
		SetDefaultEventuallyTimeout(10 * time.Second)
	} else {
		duration, err := time.ParseDuration(timeout)
		Ω(err).ShouldNot(HaveOccurred(), "invalid timeout")
		fmt.Printf("Setting Default Eventually Timeout to %s\n", duration)
		SetDefaultEventuallyTimeout(duration)
	}
	SetDefaultEventuallyPollingInterval(500 * time.Millisecond)
	SetDefaultConsistentlyPollingInterval(200 * time.Millisecond)
	domain = fmt.Sprintf("vizzini-%d", GinkgoParallelNode())
	otherDomain = fmt.Sprintf("vizzini-other-%d", GinkgoParallelNode())
	defaultRootFS = models.PreloadedRootFS("cflinuxfs2")

	var err error
	bbsClient = initializeBBSClient()

	consulClient, err := consuladapter.NewClient(consulAddress)
	Ω(err).ShouldNot(HaveOccurred())

	sessionMgr := consuladapter.NewSessionManager(consulClient)
	consulSession, err := consuladapter.NewSession("vizzini", 10*time.Second, consulClient, sessionMgr)
	Ω(err).ShouldNot(HaveOccurred())

	logger = lagertest.NewTestLogger("vizzini")

	serviceClient = bbs.NewServiceClient(consulSession, clock.NewClock())
})

var _ = BeforeEach(func() {
	startTime = time.Now()
	guid = NewGuid()
})

var _ = AfterEach(func() {
	defer func() {
		endTime := time.Now()
		fmt.Fprint(GinkgoWriter, say.Cyan("\n%s\nThis test referenced GUID %s\nStart time: %s (%d)\nEnd time: %s (%d)\n", CurrentGinkgoTestDescription().FullTestText, guid, startTime, startTime.Unix(), endTime, endTime.Unix()))
	}()

	for _, domain := range []string{domain, otherDomain} {
		ClearOutTasksInDomain(domain)
		ClearOutDesiredLRPsInDomain(domain)
	}
})

var _ = AfterSuite(func() {
	for _, domain := range []string{domain, otherDomain} {
		bbsClient.UpsertDomain(domain, 5*time.Minute) //leave the domain around forever so that Diego cleans up if need be
	}

	for _, domain := range []string{domain, otherDomain} {
		ClearOutDesiredLRPsInDomain(domain)
		ClearOutTasksInDomain(domain)
	}
})

func initializeBBSClient() bbs.Client {
	bbsURL, err := url.Parse(bbsAddress)
	Ω(err).ShouldNot(HaveOccurred())

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(bbsAddress, bbsCA, bbsClientCert, bbsClientKey, 0, 0)
	Ω(err).ShouldNot(HaveOccurred())
	return bbsClient
}
<|endoftext|>


<commit_before>package path

import (
	"fmt"
	"os"
)

// Exists checks if a file or directory exists.
func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// DirExists checks if a path exists and is a directory.
func DirExists(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err == nil && fi.IsDir() {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// IsEmpty checks if a given path is empty.
func IsEmpty(path string) (bool, error) {
	if b, _ := Exists(path); !b {
		return false, fmt.Errorf("%q path does not exist", path)
	}
	fi, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	if fi.IsDir() {
		f, err := os.Open(path)
		defer f.Close()
		if err != nil {
			return false, err
		}
		list, err := f.Readdir(-1)
		// f.Close() - see bug fix above
		return len(list) == 0, nil
	}
	return fi.Size() == 0, nil
}

// IsDir checks if a given path is a directory.
func isDir(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return fi.IsDir(), nil
}
<commit_msg>fix package name error<commit_after>package pathutil

import (
	"fmt"
	"os"
)

// Exists checks if a file or directory exists.
func Exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// DirExists checks if a path exists and is a directory.
func DirExists(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err == nil && fi.IsDir() {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// IsEmpty checks if a given path is empty.
func IsEmpty(path string) (bool, error) {
	if b, _ := Exists(path); !b {
		return false, fmt.Errorf("%q path does not exist", path)
	}
	fi, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	if fi.IsDir() {
		f, err := os.Open(path)
		defer f.Close()
		if err != nil {
			return false, err
		}
		list, err := f.Readdir(-1)
		// f.Close() - see bug fix above
		return len(list) == 0, nil
	}
	return fi.Size() == 0, nil
}

// IsDir checks if a given path is a directory.
func isDir(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return fi.IsDir(), nil
}
<|endoftext|>


<commit_before>package vsphere

// Properties describes know relation to properties to related objects and properties
var Properties = map[string]map[string][]string{
	"datastore": map[string][]string{
		"Datastore":      []string{"name"},
		"VirtualMachine": []string{"datastore"},
	},
	"host": map[string][]string{
		"HostSystem":     []string{"name", "parent"},
		"VirtualMachine": []string{"name", "runtime.host"},
	},
	"cluster": map[string][]string{
		"ClusterComputeResource": []string{"name"},
	},
	"network": map[string][]string{
		"DistributedVirtualPortgroup": []string{"name"},
		"Network":                     []string{"name"},
		"VirtualMachine":              []string{"network"},
	},
	"resourcepool": map[string][]string{
		"ResourcePool": []string{"name", "parent", "vm"},
	},
	"folder": map[string][]string{
		"Folder":         []string{"name", "parent"},
		"VirtualMachine": []string{"parent"},
	},
	"tags": map[string][]string{
		"VirtualMachine": []string{"tag"},
		"HostSystem":     []string{"tag"},
	},
	"numcpu": map[string][]string{
		"VirtualMachine": []string{"summary.config.numCpu"},
	},
	"memorysizemb": map[string][]string{
		"VirtualMachine": []string{"summary.config.memorySizeMB"},
	},
	"disks": map[string][]string{
		"VirtualMachine": []string{"guest.disk"},
	},
}
<commit_msg>simplify<commit_after>package vsphere

// Properties describes know relation to properties to related objects and properties
var Properties = map[string]map[string][]string{
	"datastore": {
		"Datastore":      {"name"},
		"VirtualMachine": {"datastore"},
	},
	"host": {
		"HostSystem":     {"name", "parent"},
		"VirtualMachine": {"name", "runtime.host"},
	},
	"cluster": {
		"ClusterComputeResource": {"name"},
	},
	"network": {
		"DistributedVirtualPortgroup": {"name"},
		"Network":                     {"name"},
		"VirtualMachine":              {"network"},
	},
	"resourcepool": {
		"ResourcePool": {"name", "parent", "vm"},
	},
	"folder": {
		"Folder":         {"name", "parent"},
		"VirtualMachine": {"parent"},
	},
	"tags": {
		"VirtualMachine": {"tag"},
		"HostSystem":     {"tag"},
	},
	"numcpu": {
		"VirtualMachine": {"summary.config.numCpu"},
	},
	"memorysizemb": {
		"VirtualMachine": {"summary.config.memorySizeMB"},
	},
	"disks": {
		"VirtualMachine": {"guest.disk"},
	},
}
<|endoftext|>


<commit_before>package order

import (
	"encoding/xml"
	"errors"
	"strconv"
	"strings"
	"time"

	"github.com/silenceper/wechat/v2/pay/config"
	"github.com/silenceper/wechat/v2/util"
)

// https://pay.weixin.qq.com/wiki/doc/api/jsapi.php?chapter=9_1
var payGateway = "https://api.mch.weixin.qq.com/pay/unifiedorder"

// SUCCESS 表示支付成功
const SUCCESS = "SUCCESS"

// Order struct extends context
type Order struct {
	*config.Config
}

// NewOrder return an instance of order package
func NewOrder(cfg *config.Config) *Order {
	order := Order{cfg}
	return &order
}

// Params was NEEDED when request Unified order
// 传入的参数，用于生成 prepay_id 的必需参数
type Params struct {
	TotalFee   string
	CreateIP   string
	Body       string
	OutTradeNo string
	TimeExpire string // 订单失效时间，格式为yyyyMMddHHmmss，如2009年12月27日9点10分10秒表示为20091227091010。
	OpenID     string
	TradeType  string
	SignType   string
	Detail     string
	Attach     string
	GoodsTag   string
	NotifyURL  string
}

// Config 是传出用于 js sdk 用的参数
type Config struct {
	Timestamp string `json:"timestamp"`
	NonceStr  string `json:"nonceStr"`
	PrePayID  string `json:"prePayId"`
	SignType  string `json:"signType"`
	Package   string `json:"package"`
	PaySign   string `json:"paySign"`
}

// ConfigForApp 是传出用于 app sdk 用的参数
type ConfigForApp struct {
	AppID     string `json:"appid"`
	MchID     string `json:"partnerid"` // 微信支付分配的商户号
	PrePayID  string `json:"prepayid"`
	Package   string `json:"package"`
	NonceStr  string `json:"nonceStr"`
	Timestamp string `json:"timestamp"`
	Sign      string `json:"sign"`
}

// PreOrder 是 Unified order 接口的返回
type PreOrder struct {
	ReturnCode string `xml:"return_code"`
	ReturnMsg  string `xml:"return_msg"`
	AppID      string `xml:"appid,omitempty"`
	MchID      string `xml:"mch_id,omitempty"`
	NonceStr   string `xml:"nonce_str,omitempty"`
	Sign       string `xml:"sign,omitempty"`
	ResultCode string `xml:"result_code,omitempty"`
	TradeType  string `xml:"trade_type,omitempty"`
	PrePayID   string `xml:"prepay_id,omitempty"`
	CodeURL    string `xml:"code_url,omitempty"`
	MWebURL    string `xml:"mweb_url,omitempty"`
	ErrCode    string `xml:"err_code,omitempty"`
	ErrCodeDes string `xml:"err_code_des,omitempty"`
}

// payRequest 接口请求参数
type payRequest struct {
	AppID          string `xml:"appid"`                 // 公众账号ID
	MchID          string `xml:"mch_id"`                // 商户号
	DeviceInfo     string `xml:"device_info,omitempty"` // 设备号
	NonceStr       string `xml:"nonce_str"`             // 随机字符串
	Sign           string `xml:"sign"`                  // 签名
	SignType       string `xml:"sign_type,omitempty"`   // 签名类型
	Body           string `xml:"body"`                  // 商品描述
	Detail         string `xml:"detail,omitempty"`      // 商品详情
	Attach         string `xml:"attach,omitempty"`      // 附加数据
	OutTradeNo     string `xml:"out_trade_no"`          // 商户订单号
	FeeType        string `xml:"fee_type,omitempty"`    // 标价币种
	TotalFee       string `xml:"total_fee"`             // 标价金额
	SpbillCreateIP string `xml:"spbill_create_ip"`      // 终端IP
	TimeStart      string `xml:"time_start,omitempty"`  // 交易起始时间
	TimeExpire     string `xml:"time_expire,omitempty"` // 交易结束时间
	GoodsTag       string `xml:"goods_tag,omitempty"`   // 订单优惠标记
	NotifyURL      string `xml:"notify_url"`            // 通知地址
	TradeType      string `xml:"trade_type"`            // 交易类型
	ProductID      string `xml:"product_id,omitempty"`  // 商品ID
	LimitPay       string `xml:"limit_pay,omitempty"`   // 指定支付方式
	OpenID         string `xml:"openid,omitempty"`      // 用户标识
	SceneInfo      string `xml:"scene_info,omitempty"`  // 场景信息

	XMLName struct{} `xml:"xml"`
}

func (req *payRequest) BridgePayRequest(p *Params, AppID, MchID, nonceStr, sign string) *payRequest {
	request := payRequest{
		AppID:          AppID,
		MchID:          MchID,
		NonceStr:       nonceStr,
		Sign:           sign,
		Body:           p.Body,
		OutTradeNo:     p.OutTradeNo,
		TotalFee:       p.TotalFee,
		SpbillCreateIP: p.CreateIP,
		NotifyURL:      p.NotifyURL,
		TradeType:      p.TradeType,
		OpenID:         p.OpenID,
		SignType:       p.SignType,
		Detail:         p.Detail,
		Attach:         p.Attach,
		GoodsTag:       p.GoodsTag,
	}
	return &request
}

// BridgeConfig get js bridge config
func (o *Order) BridgeConfig(p *Params) (cfg Config, err error) {
	var (
		buffer    strings.Builder
		timestamp = strconv.FormatInt(time.Now().Unix(), 10)
	)
	order, err := o.PrePayOrder(p)
	if err != nil {
		return
	}
	buffer.WriteString("appId=")
	buffer.WriteString(order.AppID)
	buffer.WriteString("&nonceStr=")
	buffer.WriteString(order.NonceStr)
	buffer.WriteString("&package=")
	buffer.WriteString("prepay_id=" + order.PrePayID)
	buffer.WriteString("&signType=")
	buffer.WriteString(p.SignType)
	buffer.WriteString("&timeStamp=")
	buffer.WriteString(timestamp)
	buffer.WriteString("&key=")
	buffer.WriteString(o.Key)

	sign, err := util.CalculateSign(buffer.String(), p.SignType, o.Key)
	if err != nil {
		return
	}
	// 签名
	cfg.PaySign = sign
	cfg.NonceStr = order.NonceStr
	cfg.Timestamp = timestamp
	cfg.PrePayID = order.PrePayID
	cfg.SignType = p.SignType
	cfg.Package = "prepay_id=" + order.PrePayID
	return
}

// BridgeAppConfig get app bridge config
func (o *Order) BridgeAppConfig(p *Params) (cfg ConfigForApp, err error) {
	var (
		timestamp string = strconv.FormatInt(time.Now().Unix(), 10)
		noncestr  string = util.RandomStr(32)
		_package  string = "Sign=WXPay"
	)
	order, err := o.PrePayOrder(p)
	if err != nil {
		return
	}

	result := map[string]string{
		"appid":     order.AppID,
		"partnerid": order.MchID,
		"prepayid":  order.PrePayID,
		"package":   _package,
		"noncestr":  noncestr,
		"timestamp": timestamp,
	}
	// 签名
	sign, err := util.ParamSign(result, o.Key)
	if err != nil {
		return
	}
	result["sign"] = sign
	cfg = ConfigForApp{
		AppID:     result["appid"],
		MchID:     result["partnerid"],
		PrePayID:  result["prepayid"],
		Package:   result["package"],
		NonceStr:  result["noncestr"],
		Timestamp: result["timestamp"],
		Sign:      result["sign"],
	}
	return
}

// PrePayOrder return data for invoke wechat payment
func (o *Order) PrePayOrder(p *Params) (payOrder PreOrder, err error) {
	nonceStr := util.RandomStr(32)
	param := map[string]string{
		"appid":            o.AppID,
		"body":             p.Body,
		"mch_id":           o.MchID,
		"nonce_str":        nonceStr,
		"out_trade_no":     p.OutTradeNo,
		"spbill_create_ip": p.CreateIP,
		"total_fee":        p.TotalFee,
		"trade_type":       p.TradeType,
		"openid":           p.OpenID,
		"sign_type":        p.SignType,
		"detail":           p.Detail,
		"attach":           p.Attach,
		"goods_tag":        p.GoodsTag,
	}
	// 签名类型
	if param["sign_type"] == "" {
		param["sign_type"] = util.SignTypeMD5
	}

	// 通知地址
	if p.NotifyURL != "" {
		param["notify_url"] = p.NotifyURL
	}

	if p.TimeExpire != "" {
		// 如果有传入交易结束时间
		param["time_expire"] = p.TimeExpire
	}

	sign, err := util.ParamSign(param, o.Key)
	if err != nil {
		return
	}
	request := new(payRequest).BridgePayRequest(p, o.AppID, o.MchID, nonceStr, sign)
	if len(p.TimeExpire) > 0 {
		// 如果有传入交易结束时间
		request.TimeExpire = p.TimeExpire
	}
	rawRet, err := util.PostXML(payGateway, request)
	if err != nil {
		return
	}
	err = xml.Unmarshal(rawRet, &payOrder)
	if err != nil {
		return
	}
	if payOrder.ReturnCode == SUCCESS {
		// pay success
		if payOrder.ResultCode == SUCCESS {
			err = nil
			return
		}
		err = errors.New(payOrder.ErrCode + payOrder.ErrCodeDes)
		return
	}
	err = errors.New("[msg : xmlUnmarshalError] [rawReturn : " + string(rawRet) + "] [sign : " + sign + "]")
	return
}

// PrePayID will request wechat merchant api and request for a pre payment order id
func (o *Order) PrePayID(p *Params) (prePayID string, err error) {
	order, err := o.PrePayOrder(p)
	if err != nil {
		return
	}
	if order.PrePayID == "" {
		err = errors.New("empty prepayid")
	}
	prePayID = order.PrePayID
	return
}
<commit_msg>修复微信支付缺少notify_url的bug (#472)<commit_after>package order

import (
	"encoding/xml"
	"errors"
	"strconv"
	"strings"
	"time"

	"github.com/silenceper/wechat/v2/pay/config"
	"github.com/silenceper/wechat/v2/util"
)

// https://pay.weixin.qq.com/wiki/doc/api/jsapi.php?chapter=9_1
var payGateway = "https://api.mch.weixin.qq.com/pay/unifiedorder"

// SUCCESS 表示支付成功
const SUCCESS = "SUCCESS"

// Order struct extends context
type Order struct {
	*config.Config
}

// NewOrder return an instance of order package
func NewOrder(cfg *config.Config) *Order {
	order := Order{cfg}
	return &order
}

// Params was NEEDED when request Unified order
// 传入的参数，用于生成 prepay_id 的必需参数
type Params struct {
	TotalFee   string
	CreateIP   string
	Body       string
	OutTradeNo string
	TimeExpire string // 订单失效时间，格式为yyyyMMddHHmmss，如2009年12月27日9点10分10秒表示为20091227091010。
	OpenID     string
	TradeType  string
	SignType   string
	Detail     string
	Attach     string
	GoodsTag   string
	NotifyURL  string
}

// Config 是传出用于 js sdk 用的参数
type Config struct {
	Timestamp string `json:"timestamp"`
	NonceStr  string `json:"nonceStr"`
	PrePayID  string `json:"prePayId"`
	SignType  string `json:"signType"`
	Package   string `json:"package"`
	PaySign   string `json:"paySign"`
}

// ConfigForApp 是传出用于 app sdk 用的参数
type ConfigForApp struct {
	AppID     string `json:"appid"`
	MchID     string `json:"partnerid"` // 微信支付分配的商户号
	PrePayID  string `json:"prepayid"`
	Package   string `json:"package"`
	NonceStr  string `json:"nonceStr"`
	Timestamp string `json:"timestamp"`
	Sign      string `json:"sign"`
}

// PreOrder 是 Unified order 接口的返回
type PreOrder struct {
	ReturnCode string `xml:"return_code"`
	ReturnMsg  string `xml:"return_msg"`
	AppID      string `xml:"appid,omitempty"`
	MchID      string `xml:"mch_id,omitempty"`
	NonceStr   string `xml:"nonce_str,omitempty"`
	Sign       string `xml:"sign,omitempty"`
	ResultCode string `xml:"result_code,omitempty"`
	TradeType  string `xml:"trade_type,omitempty"`
	PrePayID   string `xml:"prepay_id,omitempty"`
	CodeURL    string `xml:"code_url,omitempty"`
	MWebURL    string `xml:"mweb_url,omitempty"`
	ErrCode    string `xml:"err_code,omitempty"`
	ErrCodeDes string `xml:"err_code_des,omitempty"`
}

// payRequest 接口请求参数
type payRequest struct {
	AppID          string `xml:"appid"`                 // 公众账号ID
	MchID          string `xml:"mch_id"`                // 商户号
	DeviceInfo     string `xml:"device_info,omitempty"` // 设备号
	NonceStr       string `xml:"nonce_str"`             // 随机字符串
	Sign           string `xml:"sign"`                  // 签名
	SignType       string `xml:"sign_type,omitempty"`   // 签名类型
	Body           string `xml:"body"`                  // 商品描述
	Detail         string `xml:"detail,omitempty"`      // 商品详情
	Attach         string `xml:"attach,omitempty"`      // 附加数据
	OutTradeNo     string `xml:"out_trade_no"`          // 商户订单号
	FeeType        string `xml:"fee_type,omitempty"`    // 标价币种
	TotalFee       string `xml:"total_fee"`             // 标价金额
	SpbillCreateIP string `xml:"spbill_create_ip"`      // 终端IP
	TimeStart      string `xml:"time_start,omitempty"`  // 交易起始时间
	TimeExpire     string `xml:"time_expire,omitempty"` // 交易结束时间
	GoodsTag       string `xml:"goods_tag,omitempty"`   // 订单优惠标记
	NotifyURL      string `xml:"notify_url"`            // 通知地址
	TradeType      string `xml:"trade_type"`            // 交易类型
	ProductID      string `xml:"product_id,omitempty"`  // 商品ID
	LimitPay       string `xml:"limit_pay,omitempty"`   // 指定支付方式
	OpenID         string `xml:"openid,omitempty"`      // 用户标识
	SceneInfo      string `xml:"scene_info,omitempty"`  // 场景信息

	XMLName struct{} `xml:"xml"`
}

func (req *payRequest) BridgePayRequest(p *Params, AppID, MchID, nonceStr, sign string) *payRequest {
	request := payRequest{
		AppID:          AppID,
		MchID:          MchID,
		NonceStr:       nonceStr,
		Sign:           sign,
		Body:           p.Body,
		OutTradeNo:     p.OutTradeNo,
		TotalFee:       p.TotalFee,
		SpbillCreateIP: p.CreateIP,
		NotifyURL:      p.NotifyURL,
		TradeType:      p.TradeType,
		OpenID:         p.OpenID,
		SignType:       p.SignType,
		Detail:         p.Detail,
		Attach:         p.Attach,
		GoodsTag:       p.GoodsTag,
	}
	return &request
}

// BridgeConfig get js bridge config
func (o *Order) BridgeConfig(p *Params) (cfg Config, err error) {
	var (
		buffer    strings.Builder
		timestamp = strconv.FormatInt(time.Now().Unix(), 10)
	)
	order, err := o.PrePayOrder(p)
	if err != nil {
		return
	}
	buffer.WriteString("appId=")
	buffer.WriteString(order.AppID)
	buffer.WriteString("&nonceStr=")
	buffer.WriteString(order.NonceStr)
	buffer.WriteString("&package=")
	buffer.WriteString("prepay_id=" + order.PrePayID)
	buffer.WriteString("&signType=")
	buffer.WriteString(p.SignType)
	buffer.WriteString("&timeStamp=")
	buffer.WriteString(timestamp)
	buffer.WriteString("&key=")
	buffer.WriteString(o.Key)

	sign, err := util.CalculateSign(buffer.String(), p.SignType, o.Key)
	if err != nil {
		return
	}
	// 签名
	cfg.PaySign = sign
	cfg.NonceStr = order.NonceStr
	cfg.Timestamp = timestamp
	cfg.PrePayID = order.PrePayID
	cfg.SignType = p.SignType
	cfg.Package = "prepay_id=" + order.PrePayID
	return
}

// BridgeAppConfig get app bridge config
func (o *Order) BridgeAppConfig(p *Params) (cfg ConfigForApp, err error) {
	var (
		timestamp string = strconv.FormatInt(time.Now().Unix(), 10)
		noncestr  string = util.RandomStr(32)
		_package  string = "Sign=WXPay"
	)
	order, err := o.PrePayOrder(p)
	if err != nil {
		return
	}

	result := map[string]string{
		"appid":     order.AppID,
		"partnerid": order.MchID,
		"prepayid":  order.PrePayID,
		"package":   _package,
		"noncestr":  noncestr,
		"timestamp": timestamp,
	}
	// 签名
	sign, err := util.ParamSign(result, o.Key)
	if err != nil {
		return
	}
	result["sign"] = sign
	cfg = ConfigForApp{
		AppID:     result["appid"],
		MchID:     result["partnerid"],
		PrePayID:  result["prepayid"],
		Package:   result["package"],
		NonceStr:  result["noncestr"],
		Timestamp: result["timestamp"],
		Sign:      result["sign"],
	}
	return
}

// PrePayOrder return data for invoke wechat payment
func (o *Order) PrePayOrder(p *Params) (payOrder PreOrder, err error) {
	nonceStr := util.RandomStr(32)

	// 通知地址
	if len(p.NotifyURL) == 0 {
		p.NotifyURL = o.NotifyURL // 默认使用order.NotifyURL
	}

	param := map[string]string{
		"appid":            o.AppID,
		"body":             p.Body,
		"mch_id":           o.MchID,
		"nonce_str":        nonceStr,
		"out_trade_no":     p.OutTradeNo,
		"spbill_create_ip": p.CreateIP,
		"total_fee":        p.TotalFee,
		"trade_type":       p.TradeType,
		"openid":           p.OpenID,
		"sign_type":        p.SignType,
		"detail":           p.Detail,
		"attach":           p.Attach,
		"goods_tag":        p.GoodsTag,
		"notify_url":       p.NotifyURL,
	}
	// 签名类型
	if param["sign_type"] == "" {
		param["sign_type"] = util.SignTypeMD5
	}

	if p.TimeExpire != "" {
		// 如果有传入交易结束时间
		param["time_expire"] = p.TimeExpire
	}

	sign, err := util.ParamSign(param, o.Key)
	if err != nil {
		return
	}
	request := new(payRequest).BridgePayRequest(p, o.AppID, o.MchID, nonceStr, sign)
	if len(p.TimeExpire) > 0 {
		// 如果有传入交易结束时间
		request.TimeExpire = p.TimeExpire
	}
	rawRet, err := util.PostXML(payGateway, request)
	if err != nil {
		return
	}
	err = xml.Unmarshal(rawRet, &payOrder)
	if err != nil {
		return
	}
	if payOrder.ReturnCode == SUCCESS {
		// pay success
		if payOrder.ResultCode == SUCCESS {
			err = nil
			return
		}
		err = errors.New(payOrder.ErrCode + payOrder.ErrCodeDes)
		return
	}
	err = errors.New("[msg : xmlUnmarshalError] [rawReturn : " + string(rawRet) + "] [sign : " + sign + "]")
	return
}

// PrePayID will request wechat merchant api and request for a pre payment order id
func (o *Order) PrePayID(p *Params) (prePayID string, err error) {
	order, err := o.PrePayOrder(p)
	if err != nil {
		return
	}
	if order.PrePayID == "" {
		err = errors.New("empty prepayid")
	}
	prePayID = order.PrePayID
	return
}
<|endoftext|>


<commit_before>package main

// Test of initialization order of package-level vars.

var counter int

func next() int {
	c := counter
	counter++
	return c
}

func makeOrder1() [6]int {
	// The values of these vars are determined by the (arbitrary)
	// order in which we refer to them here. f=0, b=1, d=2, etc.
	return [6]int{f1, b1, d1, e1, c1, a1}
}

func makeOrder2() [6]int {
	// The values of these vars are independent of the order in
	// which we refer to them here.  a=6, b=7, c=8, etc.
	return [6]int{f2, b2, d2, e2, c2, a2}
}

var order1 = makeOrder1()

func main() {
	// order1 is a package-level variable:
	// [a-f]1 are initialized in reference order.
	if order1 != [6]int{0, 1, 2, 3, 4, 5} {
		panic(order1)
	}

	// order2 is a local variable:
	// [a-f]2 are initialized in lexical order.
	var order2 = makeOrder2()
	if order2 != [6]int{11, 7, 9, 10, 8, 6} {
		panic(order2)
	}
}

var a1, b1 = next(), next()
var c1, d1 = next(), next()
var e1, f1 = next(), next()

var a2, b2 = next(), next()
var c2, d2 = next(), next()
var e2, f2 = next(), next()
<commit_msg>go.tools/ssa/interp: fix init order test (partial build fix)<commit_after>package main

// Test of initialization order of package-level vars.

var counter int

func next() int {
	c := counter
	counter++
	return c
}

func makeOrder1() [6]int {
	return [6]int{f1, b1, d1, e1, c1, a1}
}

func makeOrder2() [6]int {
	return [6]int{f2, b2, d2, e2, c2, a2}
}

var order1 = makeOrder1()

func main() {
	// order1 is a package-level variable
	if order1 != [6]int{5, 1, 3, 4, 2, 0} {
		panic(order1)
	}

	// order2 is a local variable
	var order2 = makeOrder2()
	if order2 != [6]int{11, 7, 9, 10, 8, 6} {
		panic(order2)
	}
}

var a1, b1 = next(), next()
var c1, d1 = next(), next()
var e1, f1 = next(), next()

var a2, b2 = next(), next()
var c2, d2 = next(), next()
var e2, f2 = next(), next()
<|endoftext|>


<commit_before>package main

import (

	"code.google.com/p/gorilla/mux"
	"encoding/json"
	"fmt"
	"github.com/darkhelmet/env"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

func webRespond(resp http.ResponseWriter, status int, data interface{}) {
	b, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		panic(err)
	}
	resp.Header().Set("Content-Type", "application/json; charset=utf-8")
	resp.WriteHeader(status)
	resp.Write(b)
	resp.Write([]byte("\n"))
}

func webPinList(resp http.ResponseWriter, req *http.Request) {
	pins, err := dataPinList()
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pins)
}

func webPinCreate(resp http.ResponseWriter, req *http.Request) {
	pinReq := pin{}
	err := json.NewDecoder(req.Body).Decode(&pinReq)
	if err != nil {
		err = pgpinError{Id: "bad-request", Message: "malformed JSON body"}
		webErr(resp, err)
		return
	}
	pin, err := dataPinCreate(pinReq.DbId, pinReq.Name, pinReq.Query)
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pin)
}

func webPinGet(resp http.ResponseWriter, req *http.Request) {
	id := mux.Vars(req)["id"]
	pin, err := dataPinGet(id)
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pin)
}

func webPinDestroy(resp http.ResponseWriter, req *http.Request) {
	id := mux.Vars(req)["id"]
	pin, err := dataPinGet(id)
	if err != nil {
		webErr(resp, err)
		return
	}
	err = dataPinDelete(pin)
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pin)
}

func webStatus(resp http.ResponseWriter, req *http.Request) {
	err := dataTest()
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, &map[string]string{"message": "ok"})
}

func webNotFound(resp http.ResponseWriter, req *http.Request) {
	err := pgpinError{Id: "not-found", Message: "not found"}
	webErr(resp, err)
}

func webErr(resp http.ResponseWriter, err error) {
	switch err.(type) {
	case pgpinError:
		webRespond(resp, 500, err)
	default:
		webRespond(resp, 500, &map[string]string{"id": "internal-error", "message": "internal server error"})
	}
}

func webRouterHandler() http.HandlerFunc {
	router := mux.NewRouter()
	router.HandleFunc("/pins", webPinList).Methods("GET")
	router.HandleFunc("/pins", webPinCreate).Methods("POST")
	router.HandleFunc("/pins/{id}", webPinGet).Methods("GET")
	router.HandleFunc("/pins/{id}", webPinDestroy).Methods("DELETE")
	router.HandleFunc("/status", webStatus).Methods("GET")
	router.NotFoundHandler = http.HandlerFunc(webNotFound)
	return func(res http.ResponseWriter, req *http.Request) {
		router.ServeHTTP(res, req)
	}
}

type webStatusingResponseWriter struct {
	status int
	http.ResponseWriter
}

func (w *webStatusingResponseWriter) WriteHeader(s int) {
	w.status = s
	w.ResponseWriter.WriteHeader(s)
}

func webWrapLogging(f http.HandlerFunc) http.HandlerFunc {
	return func(res http.ResponseWriter, req *http.Request) {
		start := time.Now()
		method := req.Method
		path := req.URL.Path
		log("web.request.start", "method=%s path=%s", method, path)
		wres := webStatusingResponseWriter{-1, res}
		f(&wres, req)
		elapsed := float64(time.Since(start)) / 1000000.0
		log("web.request.finish", "method=%s path=%s status=%d elapsed=%f", method, path, wres.status, elapsed)
	}
}

func webTrap() {
	log("web.trap.set")
	trap := make(chan os.Signal)
	go func() {
		<- trap
		log("web.exit")
		os.Exit(0)
	}()
	signal.Notify(trap, syscall.SIGINT, syscall.SIGTERM)
}

func webStart() {
	log("web.start")
	dataStart()
	handler := webRouterHandler()
	handler = webWrapLogging(handler)
	webTrap()
	port := env.Int("PORT")
	log("web.serve", "port=%d", port)
	err := http.ListenAndServe(fmt.Sprintf(":%d", port), handler)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Normalize<commit_after>package main

import (

	"code.google.com/p/gorilla/mux"
	"encoding/json"
	"fmt"
	"github.com/darkhelmet/env"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

func webRespond(resp http.ResponseWriter, status int, data interface{}) {
	b, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		panic(err)
	}
	resp.Header().Set("Content-Type", "application/json; charset=utf-8")
	resp.WriteHeader(status)
	resp.Write(b)
	resp.Write([]byte("\n"))
}

func webPinList(resp http.ResponseWriter, req *http.Request) {
	pins, err := dataPinList()
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pins)
}

func webPinCreate(resp http.ResponseWriter, req *http.Request) {
	pinReq := pin{}
	err := json.NewDecoder(req.Body).Decode(&pinReq)
	if err != nil {
		err = pgpinError{Id: "bad-request", Message: "malformed JSON body"}
		webErr(resp, err)
		return
	}
	pin, err := dataPinCreate(pinReq.DbId, pinReq.Name, pinReq.Query)
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pin)
}

func webPinGet(resp http.ResponseWriter, req *http.Request) {
	id := mux.Vars(req)["id"]
	pin, err := dataPinGet(id)
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pin)
}

func webPinDestroy(resp http.ResponseWriter, req *http.Request) {
	id := mux.Vars(req)["id"]
	pin, err := dataPinGet(id)
	if err != nil {
		webErr(resp, err)
		return
	}
	err = dataPinDelete(pin)
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, pin)
}

func webStatus(resp http.ResponseWriter, req *http.Request) {
	err := dataTest()
	if err != nil {
		webErr(resp, err)
		return
	}
	webRespond(resp, 200, &map[string]string{"message": "ok"})
}

func webNotFound(resp http.ResponseWriter, req *http.Request) {
	err := pgpinError{Id: "not-found", Message: "not found"}
	webErr(resp, err)
}

func webErr(resp http.ResponseWriter, err error) {
	switch err.(type) {
	case pgpinError:
		webRespond(resp, 500, err)
	default:
		webRespond(resp, 500, &map[string]string{"id": "internal-error", "message": "internal server error"})
	}
}

func webRouterHandler() http.HandlerFunc {
	router := mux.NewRouter()
	router.HandleFunc("/pins", webPinList).Methods("GET")
	router.HandleFunc("/pins", webPinCreate).Methods("POST")
	router.HandleFunc("/pins/{id}", webPinGet).Methods("GET")
	router.HandleFunc("/pins/{id}", webPinDestroy).Methods("DELETE")
	router.HandleFunc("/status", webStatus).Methods("GET")
	router.NotFoundHandler = http.HandlerFunc(webNotFound)
	return func(w http.ResponseWriter, r *http.Request) {
		router.ServeHTTP(w, r)
	}
}

type webStatusingResponseWriter struct {
	status int
	http.ResponseWriter
}

func (w *webStatusingResponseWriter) WriteHeader(s int) {
	w.status = s
	w.ResponseWriter.WriteHeader(s)
}

func webWrapLogging(f http.HandlerFunc) http.HandlerFunc {
	return func(res http.ResponseWriter, req *http.Request) {
		start := time.Now()
		method := req.Method
		path := req.URL.Path
		log("web.request.start", "method=%s path=%s", method, path)
		wres := webStatusingResponseWriter{-1, res}
		f(&wres, req)
		elapsed := float64(time.Since(start)) / 1000000.0
		log("web.request.finish", "method=%s path=%s status=%d elapsed=%f", method, path, wres.status, elapsed)
	}
}

func webTrap() {
	log("web.trap.set")
	trap := make(chan os.Signal)
	go func() {
		<- trap
		log("web.exit")
		os.Exit(0)
	}()
	signal.Notify(trap, syscall.SIGINT, syscall.SIGTERM)
}

func webStart() {
	log("web.start")
	dataStart()
	handler := webRouterHandler()
	handler = webWrapLogging(handler)
	webTrap()
	port := env.Int("PORT")
	log("web.serve", "port=%d", port)
	err := http.ListenAndServe(fmt.Sprintf(":%d", port), handler)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package repo

import (
	"bytes"
	"compress/gzip"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kopia/kopia/blob"
)

const flushPackIndexTimeout = 10 * time.Second
const packObjectPrefix = "P"

type packInfo struct {
	currentPackData  bytes.Buffer
	currentPackIndex *packIndex
	currentPackID    string
}

type blockLocation struct {
	packIndex   int
	objectIndex int
}

type packManager struct {
	objectManager *ObjectManager
	storage       blob.Storage

	mu           sync.RWMutex
	blockToIndex map[string]*packIndex

	pendingPackIndexes    packIndexes
	flushPackIndexesAfter time.Time

	packGroups map[string]*packInfo
}

func (p *packManager) enabled() bool {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.pendingPackIndexes != nil
}

func (p *packManager) blockIDToPackSection(blockID string) (ObjectIDSection, bool, error) {
	if strings.HasPrefix(blockID, packObjectPrefix) {
		return ObjectIDSection{}, false, nil
	}

	pi, err := p.ensurePackIndexesLoaded()
	if err != nil {
		return ObjectIDSection{}, false, fmt.Errorf("can't load pack index: %v", err)
	}

	ndx := pi[blockID]
	if ndx == nil {
		return ObjectIDSection{}, false, nil
	}

	blk := ndx.Items[blockID]
	if blk == "" {
		return ObjectIDSection{}, false, nil
	}

	if plus := strings.IndexByte(blk, '+'); plus > 0 {
		if start, err := strconv.ParseInt(blk[0:plus], 10, 64); err == nil {
			if length, err := strconv.ParseInt(blk[plus+1:], 10, 64); err == nil {
				if base, err := ParseObjectID(ndx.PackObject); err == nil {
					return ObjectIDSection{
						Base:   base,
						Start:  start,
						Length: length,
					}, true, nil
				}
			}
		}
	}

	return ObjectIDSection{}, false, fmt.Errorf("invalid pack index for %q", blockID)
}

func (p *packManager) begin() error {
	p.ensurePackIndexesLoaded()
	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) AddToPack(packGroup string, blockID string, data []byte) (ObjectID, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	// See if we already have this block ID in some pack.
	if _, ok := p.blockToIndex[blockID]; ok {
		return ObjectID{StorageBlock: blockID}, nil
	}

	g := p.packGroups[packGroup]
	if g == nil {
		g = &packInfo{}
		p.packGroups[packGroup] = g
	}

	if g.currentPackIndex == nil {
		g.currentPackIndex = &packIndex{
			Items:      make(map[string]string),
			PackGroup:  packGroup,
			CreateTime: time.Now().UTC(),
		}
		g.currentPackID = p.newPackID()
		p.pendingPackIndexes[g.currentPackID] = g.currentPackIndex
		g.currentPackData.Reset()
	}

	offset := g.currentPackData.Len()
	g.currentPackData.Write(data)
	g.currentPackIndex.Items[blockID] = fmt.Sprintf("%v+%v", int64(offset), int64(len(data)))

	if g.currentPackData.Len() >= p.objectManager.format.MaxPackFileLength {
		if err := p.finishCurrentPackLocked(); err != nil {
			return NullObjectID, err
		}
	}

	if time.Now().After(p.flushPackIndexesAfter) {
		if err := p.finishCurrentPackLocked(); err != nil {
			return NullObjectID, err
		}
		if err := p.flushPackIndexesLocked(); err != nil {
			return NullObjectID, err
		}
	}

	p.blockToIndex[blockID] = g.currentPackIndex
	return ObjectID{StorageBlock: blockID}, nil
}

func (p *packManager) finishPacking() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	if err := p.finishCurrentPackLocked(); err != nil {
		return err
	}

	if err := p.flushPackIndexesLocked(); err != nil {
		return err
	}

	p.pendingPackIndexes = nil
	return nil
}

func (p *packManager) flushPackIndexesLocked() error {
	if len(p.pendingPackIndexes) > 0 {
		log.Printf("saving %v pack indexes", len(p.pendingPackIndexes))
		if err := p.writePackIndexes(p.pendingPackIndexes); err != nil {
			return err
		}
	}

	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) writePackIndexes(ndx packIndexes) error {
	w := p.objectManager.NewWriter(WriterOptions{
		isPackInternalObject: true,
		Description:          "pack index",
		BlockNamePrefix:      packObjectPrefix,
		splitter:             newNeverSplitter(),
	})
	defer w.Close()

	zw := gzip.NewWriter(w)
	if err := json.NewEncoder(zw).Encode(ndx); err != nil {
		return fmt.Errorf("can't encode pack index: %v", err)
	}
	zw.Close()

	if _, err := w.Result(); err != nil {
		return fmt.Errorf("can't save pack index object: %v", err)
	}

	return nil
}
func (p *packManager) finishCurrentPackLocked() error {
	for _, g := range p.packGroups {
		if err := p.finishPackLocked(g); err != nil {
			return err
		}
	}

	return nil
}

func (p *packManager) finishPackLocked(g *packInfo) error {
	if g.currentPackIndex == nil {
		return nil
	}
	w := p.objectManager.NewWriter(WriterOptions{
		Description:          fmt.Sprintf("pack:%v", g.currentPackID),
		splitter:             newNeverSplitter(),
		isPackInternalObject: true,
	})
	defer w.Close()

	if _, err := g.currentPackData.WriteTo(w); err != nil {
		return fmt.Errorf("unable to write pack: %v", err)
	}
	g.currentPackData.Reset()
	oid, err := w.Result()

	if err != nil {
		return fmt.Errorf("can't save pack data: %v", err)
	}

	g.currentPackIndex.PackObject = oid.String()
	g.currentPackIndex = nil

	return nil
}

func (p *packManager) loadMergedPackIndex(olderThan *time.Time) (map[string]*packIndex, []string, error) {
	ch, cancel := p.objectManager.storage.ListBlocks(packObjectPrefix)
	defer cancel()

	t0 := time.Now()

	var wg sync.WaitGroup

	errors := make(chan error, parallelFetches)
	var mu sync.Mutex

	packIndexData := map[string][]byte{}
	totalSize := 0
	var blockIDs []string
	for i := 0; i < parallelFetches; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for b := range ch {
				if b.Error != nil {
					errors <- b.Error
					return
				}

				if olderThan != nil && b.TimeStamp.After(*olderThan) {
					return
				}

				r, err := p.objectManager.Open(ObjectID{StorageBlock: b.BlockID})
				if err != nil {
					errors <- err
					return
				}

				data, err := ioutil.ReadAll(r)
				if err != nil {
					errors <- err
					return
				}

				mu.Lock()
				packIndexData[b.BlockID] = data
				blockIDs = append(blockIDs, b.BlockID)
				totalSize += len(data)
				mu.Unlock()
			}
		}()
	}

	wg.Wait()
	close(errors)

	// Propagate async errors, if any.
	for err := range errors {
		return nil, nil, err
	}

	if false {
		log.Printf("loaded %v pack indexes (%v bytes) in %v", len(packIndexData), totalSize, time.Since(t0))
	}

	merged := make(packIndexes)
	for blockID, content := range packIndexData {
		var r io.Reader = bytes.NewReader(content)
		zr, err := gzip.NewReader(r)
		if err != nil {
			return nil, nil, fmt.Errorf("unable to read pack index from %q: %v", blockID, err)
		}

		pi, err := loadPackIndexes(zr)
		if err != nil {
			return nil, nil, err
		}
		merged.merge(pi)
	}

	return merged, blockIDs, nil
}

func (p *packManager) ensurePackIndexesLoaded() (map[string]*packIndex, error) {
	p.mu.RLock()
	pi := p.blockToIndex
	p.mu.RUnlock()
	if pi != nil {
		return pi, nil
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	merged, _, err := p.loadMergedPackIndex(nil)
	if err != nil {
		return nil, err
	}

	pi = make(map[string]*packIndex)
	for _, pck := range merged {
		for blockID := range pck.Items {
			pi[blockID] = pck
		}
	}

	p.blockToIndex = pi
	// log.Printf("loaded pack index with %v entries", len(p.blockToIndex))

	return pi, nil
}

func (p *packManager) Compact(cutoffTime time.Time) error {
	merged, blockIDs, err := p.loadMergedPackIndex(&cutoffTime)
	if err != nil {
		return err
	}

	if len(blockIDs) < parallelFetches {
		log.Printf("skipping index compaction - the number of segments %v is too low", len(blockIDs))
		return nil
	}

	log.Printf("writing %v merged indexes", len(merged))

	if err := p.writePackIndexes(merged); err != nil {
		return err
	}

	ch := makeStringChannel(blockIDs)
	var wg sync.WaitGroup

	for i := 0; i < parallelDeletes; i++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()

			for blockID := range ch {
				if err := p.objectManager.storage.DeleteBlock(blockID); err != nil {
					log.Printf("warning: unable to delete %q: %v", blockID, err)
				}
			}
		}(i)
	}
	wg.Wait()
	return nil
}

func makeStringChannel(s []string) <-chan string {
	ch := make(chan string)
	go func() {
		defer close(ch)

		for _, v := range s {
			ch <- v
		}
	}()
	return ch
}

func (p *packManager) newPackID() string {
	id := make([]byte, 8)
	rand.Read(id)
	return hex.EncodeToString(id)
}

func (p *packManager) Flush() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.finishCurrentPackLocked()
}
<commit_msg>changed how pack indexes are flushed to avoid prematurely flushing directory packs to make bigger objects<commit_after>package repo

import (
	"bytes"
	"compress/gzip"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kopia/kopia/blob"
)

const flushPackIndexTimeout = 10 * time.Second
const packObjectPrefix = "P"

type packInfo struct {
	currentPackData  bytes.Buffer
	currentPackIndex *packIndex
	currentPackID    string
}

type blockLocation struct {
	packIndex   int
	objectIndex int
}

type packManager struct {
	objectManager *ObjectManager
	storage       blob.Storage

	mu           sync.RWMutex
	blockToIndex map[string]*packIndex

	pendingPackIndexes    packIndexes
	flushPackIndexesAfter time.Time

	packGroups map[string]*packInfo
}

func (p *packManager) enabled() bool {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.pendingPackIndexes != nil
}

func (p *packManager) blockIDToPackSection(blockID string) (ObjectIDSection, bool, error) {
	if strings.HasPrefix(blockID, packObjectPrefix) {
		return ObjectIDSection{}, false, nil
	}

	pi, err := p.ensurePackIndexesLoaded()
	if err != nil {
		return ObjectIDSection{}, false, fmt.Errorf("can't load pack index: %v", err)
	}

	ndx := pi[blockID]
	if ndx == nil {
		return ObjectIDSection{}, false, nil
	}

	blk := ndx.Items[blockID]
	if blk == "" {
		return ObjectIDSection{}, false, nil
	}

	if plus := strings.IndexByte(blk, '+'); plus > 0 {
		if start, err := strconv.ParseInt(blk[0:plus], 10, 64); err == nil {
			if length, err := strconv.ParseInt(blk[plus+1:], 10, 64); err == nil {
				if base, err := ParseObjectID(ndx.PackObject); err == nil {
					return ObjectIDSection{
						Base:   base,
						Start:  start,
						Length: length,
					}, true, nil
				}
			}
		}
	}

	return ObjectIDSection{}, false, fmt.Errorf("invalid pack index for %q", blockID)
}

func (p *packManager) begin() error {
	p.ensurePackIndexesLoaded()
	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) AddToPack(packGroup string, blockID string, data []byte) (ObjectID, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	// See if we already have this block ID in some pack.
	if _, ok := p.blockToIndex[blockID]; ok {
		return ObjectID{StorageBlock: blockID}, nil
	}

	g := p.packGroups[packGroup]
	if g == nil {
		g = &packInfo{}
		p.packGroups[packGroup] = g
	}

	if g.currentPackIndex == nil {
		g.currentPackIndex = &packIndex{
			Items:      make(map[string]string),
			PackGroup:  packGroup,
			CreateTime: time.Now().UTC(),
		}
		g.currentPackID = p.newPackID()
		g.currentPackData.Reset()
	}

	offset := g.currentPackData.Len()
	g.currentPackData.Write(data)
	g.currentPackIndex.Items[blockID] = fmt.Sprintf("%v+%v", int64(offset), int64(len(data)))

	if g.currentPackData.Len() >= p.objectManager.format.MaxPackFileLength {
		log.Printf("finishing pack %q", g.currentPackID)
		if err := p.finishPackLocked(g); err != nil {
			return NullObjectID, err
		}

		if time.Now().After(p.flushPackIndexesAfter) {
			if err := p.flushPackIndexesLocked(); err != nil {
				return NullObjectID, err
			}
		}
	}

	p.blockToIndex[blockID] = g.currentPackIndex
	return ObjectID{StorageBlock: blockID}, nil
}

func (p *packManager) finishPacking() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	if err := p.finishCurrentPackLocked(); err != nil {
		return err
	}

	if err := p.flushPackIndexesLocked(); err != nil {
		return err
	}

	p.pendingPackIndexes = nil
	return nil
}

func (p *packManager) flushPackIndexesLocked() error {
	if len(p.pendingPackIndexes) > 0 {
		log.Printf("saving %v pack indexes", len(p.pendingPackIndexes))
		if err := p.writePackIndexes(p.pendingPackIndexes); err != nil {
			return err
		}
	}

	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) writePackIndexes(ndx packIndexes) error {
	w := p.objectManager.NewWriter(WriterOptions{
		isPackInternalObject: true,
		Description:          "pack index",
		BlockNamePrefix:      packObjectPrefix,
		splitter:             newNeverSplitter(),
	})
	defer w.Close()

	zw := gzip.NewWriter(w)
	if err := json.NewEncoder(zw).Encode(ndx); err != nil {
		return fmt.Errorf("can't encode pack index: %v", err)
	}
	zw.Close()

	if _, err := w.Result(); err != nil {
		return fmt.Errorf("can't save pack index object: %v", err)
	}

	return nil
}
func (p *packManager) finishCurrentPackLocked() error {
	for _, g := range p.packGroups {
		if err := p.finishPackLocked(g); err != nil {
			return err
		}
	}

	return nil
}

func (p *packManager) finishPackLocked(g *packInfo) error {
	if g.currentPackIndex == nil {
		return nil
	}
	p.pendingPackIndexes[g.currentPackID] = g.currentPackIndex
	w := p.objectManager.NewWriter(WriterOptions{
		Description:          fmt.Sprintf("pack:%v", g.currentPackID),
		splitter:             newNeverSplitter(),
		isPackInternalObject: true,
	})
	defer w.Close()

	if _, err := g.currentPackData.WriteTo(w); err != nil {
		return fmt.Errorf("unable to write pack: %v", err)
	}
	g.currentPackData.Reset()
	oid, err := w.Result()

	if err != nil {
		return fmt.Errorf("can't save pack data: %v", err)
	}

	g.currentPackIndex.PackObject = oid.String()
	g.currentPackIndex = nil

	return nil
}

func (p *packManager) loadMergedPackIndex(olderThan *time.Time) (map[string]*packIndex, []string, error) {
	ch, cancel := p.objectManager.storage.ListBlocks(packObjectPrefix)
	defer cancel()

	t0 := time.Now()

	var wg sync.WaitGroup

	errors := make(chan error, parallelFetches)
	var mu sync.Mutex

	packIndexData := map[string][]byte{}
	totalSize := 0
	var blockIDs []string
	for i := 0; i < parallelFetches; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for b := range ch {
				if b.Error != nil {
					errors <- b.Error
					return
				}

				if olderThan != nil && b.TimeStamp.After(*olderThan) {
					return
				}

				r, err := p.objectManager.Open(ObjectID{StorageBlock: b.BlockID})
				if err != nil {
					errors <- err
					return
				}

				data, err := ioutil.ReadAll(r)
				if err != nil {
					errors <- err
					return
				}

				mu.Lock()
				packIndexData[b.BlockID] = data
				blockIDs = append(blockIDs, b.BlockID)
				totalSize += len(data)
				mu.Unlock()
			}
		}()
	}

	wg.Wait()
	close(errors)

	// Propagate async errors, if any.
	for err := range errors {
		return nil, nil, err
	}

	if false {
		log.Printf("loaded %v pack indexes (%v bytes) in %v", len(packIndexData), totalSize, time.Since(t0))
	}

	merged := make(packIndexes)
	for blockID, content := range packIndexData {
		var r io.Reader = bytes.NewReader(content)
		zr, err := gzip.NewReader(r)
		if err != nil {
			return nil, nil, fmt.Errorf("unable to read pack index from %q: %v", blockID, err)
		}

		pi, err := loadPackIndexes(zr)
		if err != nil {
			return nil, nil, err
		}
		merged.merge(pi)
	}

	return merged, blockIDs, nil
}

func (p *packManager) ensurePackIndexesLoaded() (map[string]*packIndex, error) {
	p.mu.RLock()
	pi := p.blockToIndex
	p.mu.RUnlock()
	if pi != nil {
		return pi, nil
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	merged, _, err := p.loadMergedPackIndex(nil)
	if err != nil {
		return nil, err
	}

	pi = make(map[string]*packIndex)
	for _, pck := range merged {
		for blockID := range pck.Items {
			pi[blockID] = pck
		}
	}

	p.blockToIndex = pi
	// log.Printf("loaded pack index with %v entries", len(p.blockToIndex))

	return pi, nil
}

func (p *packManager) Compact(cutoffTime time.Time) error {
	merged, blockIDs, err := p.loadMergedPackIndex(&cutoffTime)
	if err != nil {
		return err
	}

	if len(blockIDs) < parallelFetches {
		log.Printf("skipping index compaction - the number of segments %v is too low", len(blockIDs))
		return nil
	}

	log.Printf("writing %v merged indexes", len(merged))

	if err := p.writePackIndexes(merged); err != nil {
		return err
	}

	ch := makeStringChannel(blockIDs)
	var wg sync.WaitGroup

	for i := 0; i < parallelDeletes; i++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()

			for blockID := range ch {
				if err := p.objectManager.storage.DeleteBlock(blockID); err != nil {
					log.Printf("warning: unable to delete %q: %v", blockID, err)
				}
			}
		}(i)
	}
	wg.Wait()
	return nil
}

func makeStringChannel(s []string) <-chan string {
	ch := make(chan string)
	go func() {
		defer close(ch)

		for _, v := range s {
			ch <- v
		}
	}()
	return ch
}

func (p *packManager) newPackID() string {
	id := make([]byte, 8)
	rand.Read(id)
	return hex.EncodeToString(id)
}

func (p *packManager) Flush() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.finishCurrentPackLocked()
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdv3

import (
	"errors"
	"testing"

	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/mvcc/mvccpb"
	"github.com/ligato/cn-infra/db"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging/logroot"
	"github.com/onsi/gomega"
	"golang.org/x/net/context"
	"github.com/ligato/cn-infra/datasync"
)

var dataBroker *BytesConnectionEtcd
var dataBrokerErr *BytesConnectionEtcd
var pluginDataBroker *BytesBrokerWatcherEtcd
var pluginDataBrokerErr *BytesBrokerWatcherEtcd

// Mock data broker err
type MockKVErr struct {
	// NO-OP
}

func (mock *MockKVErr) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
	return nil, errors.New("test-error")
}

func (mock *MockKVErr) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
	return nil, errors.New("test-error")
}

func (mock *MockKVErr) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
	return nil, errors.New("test-error")
}

func (mock *MockKVErr) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
	return nil, nil
}

func (mock *MockKVErr) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
	return clientv3.OpResponse{}, nil
}

func (mock *MockKVErr) Txn(ctx context.Context) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockKVErr) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {
	return nil
}

func (mock *MockKVErr) Close() error {
	return nil
}

// Mock KV
type MockKV struct {
	// NO-OP
}

func (mock *MockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
	return nil, nil
}

func (mock *MockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
	response := *new(clientv3.GetResponse)
	kvs := new(mvccpb.KeyValue)
	kvs.Key = []byte{1}
	kvs.Value = []byte{73, 0x6f, 0x6d, 65, 0x2d, 0x6a, 73, 0x6f, 0x6e} //some-json
	response.Kvs = []*mvccpb.KeyValue{kvs}
	return &response, nil
}

func (mock *MockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
	response := *new(clientv3.DeleteResponse)
	response.PrevKvs = []*mvccpb.KeyValue{}
	return &response, nil
}

func (mock *MockKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
	return nil, nil
}

func (mock *MockKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
	return clientv3.OpResponse{}, nil
}

func (mock *MockKV) Txn(ctx context.Context) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {
	return nil
}

func (mock *MockKV) Close() error {
	return nil
}

// Mock Txn
type MockTxn struct {
}

func (mock *MockTxn) If(cs ...clientv3.Cmp) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockTxn) Then(ops ...clientv3.Op) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockTxn) Else(ops ...clientv3.Op) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockTxn) Commit() (*clientv3.TxnResponse, error) {
	return nil, nil
}

// Tests

func init() {
	mockKv := &MockKV{}
	mockKvErr := &MockKVErr{}
	dataBroker = &BytesConnectionEtcd{Logger: logroot.Logger(), etcdClient: &clientv3.Client{KV: mockKv, Watcher: mockKv}}
	dataBrokerErr = &BytesConnectionEtcd{Logger: logroot.Logger(), etcdClient: &clientv3.Client{KV: mockKvErr, Watcher: mockKvErr}}
	pluginDataBroker = &BytesBrokerWatcherEtcd{Logger: logroot.Logger(), kv: mockKv, watcher: mockKv}
	pluginDataBrokerErr = &BytesBrokerWatcherEtcd{Logger: logroot.Logger(), kv: mockKvErr, watcher: mockKvErr}
}

func TestNewTxn(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	gomega.Expect(newTxn).NotTo(gomega.BeNil())
}

func TestTxnPut(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	result := newTxn.Put("key", []byte("data"))
	gomega.Expect(result).NotTo(gomega.BeNil())
}

func TestTxnDelete(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	gomega.Expect(newTxn).NotTo(gomega.BeNil())
	result := newTxn.Delete("key")
	gomega.Expect(result).NotTo(gomega.BeNil())
}

func TestTxnCommit(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	result := newTxn.Commit()
	gomega.Expect(result).To(gomega.BeNil())
}

func TestPut(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	err := dataBroker.Put("key", []byte("data"))
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	// error case
	err = dataBrokerErr.Put("key", []byte("data"))
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestGetValue(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	result, found, _, err := dataBroker.GetValue("key")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(result).NotTo(gomega.BeNil())
	// error case
	result, found, _, err = dataBrokerErr.GetValue("key")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(found).To(gomega.BeFalse())
	gomega.Expect(result).To(gomega.BeNil())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestListValues(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	result, err := dataBroker.ListValues("key")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(result).ToNot(gomega.BeNil())

	// error case
	result, err = dataBrokerErr.ListValues("key")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(result).To(gomega.BeNil())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestListValuesRange(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	result, err := dataBroker.ListValuesRange("AKey", "ZKey")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(result).ToNot(gomega.BeNil())

	// error case
	result, err = dataBrokerErr.ListValuesRange("AKey", "ZKey")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(result).To(gomega.BeNil())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestDelete(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	response, err := dataBroker.Delete("vnf")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(response).To(gomega.BeFalse())
	// error case
	response, err = dataBrokerErr.Delete("vnf")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(response).To(gomega.BeFalse())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestNewBroker(t *testing.T) {
	gomega.RegisterTestingT(t)
	pdb := dataBroker.NewBroker("/pluginname")
	gomega.Expect(pdb).NotTo(gomega.BeNil())
}

func TestNewWatcher(t *testing.T) {
	gomega.RegisterTestingT(t)
	pdb := dataBroker.NewWatcher("/pluginname")
	gomega.Expect(pdb).NotTo(gomega.BeNil())
}

func TestWatch(t *testing.T) {
	gomega.RegisterTestingT(t)
	respChan := make(chan keyval.BytesWatchResp)
	err := pluginDataBroker.Watch(respChan, "key")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
}

func TestWatchPutResp(t *testing.T) {
	var rev int64 = 1
	value := []byte("data")
	key := "key"
	gomega.RegisterTestingT(t)
	createResp := NewBytesWatchPutResp(key, value, rev)
	gomega.Expect(createResp).NotTo(gomega.BeNil())
	gomega.Expect(createResp.GetChangeType()).To(gomega.BeEquivalentTo(db.Put))
	gomega.Expect(createResp.GetKey()).To(gomega.BeEquivalentTo(key))
	gomega.Expect(createResp.GetValue()).To(gomega.BeEquivalentTo(value))
	gomega.Expect(createResp.GetRevision()).To(gomega.BeEquivalentTo(rev))
}

func TestWatchDeleteResp(t *testing.T) {
	var rev int64 = 1
	key := "key"
	gomega.RegisterTestingT(t)
	createResp := NewBytesWatchDelResp(key, rev)
	gomega.Expect(createResp).NotTo(gomega.BeNil())
	gomega.Expect(createResp.GetChangeType()).To(gomega.BeEquivalentTo(datasync.Delete))
	gomega.Expect(createResp.GetKey()).To(gomega.BeEquivalentTo(key))
	gomega.Expect(createResp.GetValue()).To(gomega.BeNil())
	gomega.Expect(createResp.GetRevision()).To(gomega.BeEquivalentTo(rev))
}

func TestConfig(t *testing.T) {
	gomega.RegisterTestingT(t)
	cfg := &Config{DialTimeout: time.Second, OpTimeout: time.Second}
	etcdCfg, err := ConfigToClientv3(cfg)
	gomega.Expect(err).To(gomega.BeNil())
	gomega.Expect(etcdCfg).NotTo(gomega.BeNil())
	gomega.Expect(etcdCfg.OpTimeout).To(gomega.BeEquivalentTo(time.Second))
	gomega.Expect(etcdCfg.DialTimeout).To(gomega.BeEquivalentTo(time.Second))
	gomega.Expect(etcdCfg.TLS).To(gomega.BeNil())
}
<commit_msg> ODPM-361 fix datasync.Put<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdv3

import (
	"errors"
	"testing"

	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/mvcc/mvccpb"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging/logroot"
	"github.com/onsi/gomega"
	"golang.org/x/net/context"
)

var dataBroker *BytesConnectionEtcd
var dataBrokerErr *BytesConnectionEtcd
var pluginDataBroker *BytesBrokerWatcherEtcd
var pluginDataBrokerErr *BytesBrokerWatcherEtcd

// Mock data broker err
type MockKVErr struct {
	// NO-OP
}

func (mock *MockKVErr) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
	return nil, errors.New("test-error")
}

func (mock *MockKVErr) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
	return nil, errors.New("test-error")
}

func (mock *MockKVErr) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
	return nil, errors.New("test-error")
}

func (mock *MockKVErr) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
	return nil, nil
}

func (mock *MockKVErr) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
	return clientv3.OpResponse{}, nil
}

func (mock *MockKVErr) Txn(ctx context.Context) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockKVErr) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {
	return nil
}

func (mock *MockKVErr) Close() error {
	return nil
}

// Mock KV
type MockKV struct {
	// NO-OP
}

func (mock *MockKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
	return nil, nil
}

func (mock *MockKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
	response := *new(clientv3.GetResponse)
	kvs := new(mvccpb.KeyValue)
	kvs.Key = []byte{1}
	kvs.Value = []byte{73, 0x6f, 0x6d, 65, 0x2d, 0x6a, 73, 0x6f, 0x6e} //some-json
	response.Kvs = []*mvccpb.KeyValue{kvs}
	return &response, nil
}

func (mock *MockKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
	response := *new(clientv3.DeleteResponse)
	response.PrevKvs = []*mvccpb.KeyValue{}
	return &response, nil
}

func (mock *MockKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
	return nil, nil
}

func (mock *MockKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
	return clientv3.OpResponse{}, nil
}

func (mock *MockKV) Txn(ctx context.Context) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockKV) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan {
	return nil
}

func (mock *MockKV) Close() error {
	return nil
}

// Mock Txn
type MockTxn struct {
}

func (mock *MockTxn) If(cs ...clientv3.Cmp) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockTxn) Then(ops ...clientv3.Op) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockTxn) Else(ops ...clientv3.Op) clientv3.Txn {
	return &MockTxn{}
}

func (mock *MockTxn) Commit() (*clientv3.TxnResponse, error) {
	return nil, nil
}

// Tests

func init() {
	mockKv := &MockKV{}
	mockKvErr := &MockKVErr{}
	dataBroker = &BytesConnectionEtcd{Logger: logroot.Logger(), etcdClient: &clientv3.Client{KV: mockKv, Watcher: mockKv}}
	dataBrokerErr = &BytesConnectionEtcd{Logger: logroot.Logger(), etcdClient: &clientv3.Client{KV: mockKvErr, Watcher: mockKvErr}}
	pluginDataBroker = &BytesBrokerWatcherEtcd{Logger: logroot.Logger(), kv: mockKv, watcher: mockKv}
	pluginDataBrokerErr = &BytesBrokerWatcherEtcd{Logger: logroot.Logger(), kv: mockKvErr, watcher: mockKvErr}
}

func TestNewTxn(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	gomega.Expect(newTxn).NotTo(gomega.BeNil())
}

func TestTxnPut(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	result := newTxn.Put("key", []byte("data"))
	gomega.Expect(result).NotTo(gomega.BeNil())
}

func TestTxnDelete(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	gomega.Expect(newTxn).NotTo(gomega.BeNil())
	result := newTxn.Delete("key")
	gomega.Expect(result).NotTo(gomega.BeNil())
}

func TestTxnCommit(t *testing.T) {
	gomega.RegisterTestingT(t)
	newTxn := dataBroker.NewTxn()
	result := newTxn.Commit()
	gomega.Expect(result).To(gomega.BeNil())
}

func TestPut(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	err := dataBroker.Put("key", []byte("data"))
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	// error case
	err = dataBrokerErr.Put("key", []byte("data"))
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestGetValue(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	result, found, _, err := dataBroker.GetValue("key")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(result).NotTo(gomega.BeNil())
	// error case
	result, found, _, err = dataBrokerErr.GetValue("key")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(found).To(gomega.BeFalse())
	gomega.Expect(result).To(gomega.BeNil())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestListValues(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	result, err := dataBroker.ListValues("key")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(result).ToNot(gomega.BeNil())

	// error case
	result, err = dataBrokerErr.ListValues("key")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(result).To(gomega.BeNil())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestListValuesRange(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	result, err := dataBroker.ListValuesRange("AKey", "ZKey")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(result).ToNot(gomega.BeNil())

	// error case
	result, err = dataBrokerErr.ListValuesRange("AKey", "ZKey")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(result).To(gomega.BeNil())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestDelete(t *testing.T) {
	// regular case
	gomega.RegisterTestingT(t)
	response, err := dataBroker.Delete("vnf")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
	gomega.Expect(response).To(gomega.BeFalse())
	// error case
	response, err = dataBrokerErr.Delete("vnf")
	gomega.Expect(err).Should(gomega.HaveOccurred())
	gomega.Expect(response).To(gomega.BeFalse())
	gomega.Expect(err.Error()).To(gomega.BeEquivalentTo("test-error"))
}

func TestNewBroker(t *testing.T) {
	gomega.RegisterTestingT(t)
	pdb := dataBroker.NewBroker("/pluginname")
	gomega.Expect(pdb).NotTo(gomega.BeNil())
}

func TestNewWatcher(t *testing.T) {
	gomega.RegisterTestingT(t)
	pdb := dataBroker.NewWatcher("/pluginname")
	gomega.Expect(pdb).NotTo(gomega.BeNil())
}

func TestWatch(t *testing.T) {
	gomega.RegisterTestingT(t)
	respChan := make(chan keyval.BytesWatchResp)
	err := pluginDataBroker.Watch(respChan, "key")
	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
}

func TestWatchPutResp(t *testing.T) {
	var rev int64 = 1
	value := []byte("data")
	key := "key"
	gomega.RegisterTestingT(t)
	createResp := NewBytesWatchPutResp(key, value, rev)
	gomega.Expect(createResp).NotTo(gomega.BeNil())
	gomega.Expect(createResp.GetChangeType()).To(gomega.BeEquivalentTo(datasync.Put))
	gomega.Expect(createResp.GetKey()).To(gomega.BeEquivalentTo(key))
	gomega.Expect(createResp.GetValue()).To(gomega.BeEquivalentTo(value))
	gomega.Expect(createResp.GetRevision()).To(gomega.BeEquivalentTo(rev))
}

func TestWatchDeleteResp(t *testing.T) {
	var rev int64 = 1
	key := "key"
	gomega.RegisterTestingT(t)
	createResp := NewBytesWatchDelResp(key, rev)
	gomega.Expect(createResp).NotTo(gomega.BeNil())
	gomega.Expect(createResp.GetChangeType()).To(gomega.BeEquivalentTo(datasync.Delete))
	gomega.Expect(createResp.GetKey()).To(gomega.BeEquivalentTo(key))
	gomega.Expect(createResp.GetValue()).To(gomega.BeNil())
	gomega.Expect(createResp.GetRevision()).To(gomega.BeEquivalentTo(rev))
}

func TestConfig(t *testing.T) {
	gomega.RegisterTestingT(t)
	cfg := &Config{DialTimeout: time.Second, OpTimeout: time.Second}
	etcdCfg, err := ConfigToClientv3(cfg)
	gomega.Expect(err).To(gomega.BeNil())
	gomega.Expect(etcdCfg).NotTo(gomega.BeNil())
	gomega.Expect(etcdCfg.OpTimeout).To(gomega.BeEquivalentTo(time.Second))
	gomega.Expect(etcdCfg.DialTimeout).To(gomega.BeEquivalentTo(time.Second))
	gomega.Expect(etcdCfg.TLS).To(gomega.BeNil())
}
<|endoftext|>