<commit_before>package lhef

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
)

// Decoder represents an LHEF parser reading a particular input stream.
//
// A Decoder is initialized with an input io.Reader from which to read a version 1.0
// Les Houches Accord event file.
type Decoder struct {
	r       io.Reader
	dec     *xml.Decoder
	evt     xml.StartElement // the current xml.Token holding a HEPEUP
	Version int              // LHEF file version
	Run     HEPRUP           // User process run common block
}

func NewDecoder(r io.Reader) (*Decoder, error) {
	var err error
	dec := xml.NewDecoder(r)
	d := &Decoder{
		r:       r,
		dec:     dec,
		Version: -42,
		Run:     HEPRUP{},
	}
	// err := dec.dec.Decode(&dec.lhevt)
	// if err != nil {
	// 	return nil
	// }
	// fmt.Printf(">>> version=%v\n", dec.lhevt.Version)

	tok, err := dec.Token()
	if err != nil || tok == nil {
		return nil, err
	}

	// make sure we are reading a LHEF file
	start, ok := tok.(xml.StartElement)
	if !ok || start.Name.Local != "LesHouchesEvents" {
		fmt.Printf("err, not a LHEF file (%T: %v)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing LesHouchesEvent start-tag")
	}

	//fmt.Printf(">>> %v\n", start)
	version := start.Attr[0].Value
	fmt.Printf("    version=[%s]\n", version)
	switch version {
	default:
		d.Version = -42
	case "1.0":
		d.Version = 1
	case "2.0":
		d.Version = 2
	}

	var (
		init   xml.StartElement
		header xml.StartElement
	)

Loop:
	for {
		tok, err = dec.Token()
		if err != nil || tok == nil {
			return nil, err
		}
		switch tt := tok.(type) {
		case xml.Comment:
			// skip
		case xml.CharData:
			// skip
		case xml.StartElement:
			switch tt.Name.Local {
			case "init":
				init = tt
				break Loop
			case "header":
				header = tt
			}
		}
	}
	if init.Name.Local != "init" {
		fmt.Printf("err, not a valid LHEF file (%T: %v)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing init start-tag")
	}
	//fmt.Printf(">>> header: %v\n", header)
	//fmt.Printf(">>> init:   %v\n", init)

	// extract compulsory initialization information
	tok, err = dec.Token()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return nil, err
	}
	data, ok := tok.(xml.CharData)
	if !ok || len(data) == 0 {
		fmt.Printf("err, not a valid LHEF file (%T: %v)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing init payload")
	}
	buf := bytes.NewBuffer(data)
	_, err = fmt.Fscanf(
		buf,
		"\n%d %d %f %f %d %d %d %d %d %d\n",
		&d.Run.IDBMUP[0],
		&d.Run.IDBMUP[1],
		&d.Run.EBMUP[0],
		&d.Run.EBMUP[1],
		&d.Run.PDFGUP[0],
		&d.Run.PDFGUP[1],
		&d.Run.PDFSUP[0],
		&d.Run.PDFSUP[1],
		&d.Run.IDWTUP,
		&d.Run.NPRUP,
	)
	if err != nil {
		fmt.Printf("err:  %v\n", err)
		fmt.Printf("data: %v\n", string(data))
		return nil, err
	}

	d.Run.XSECUP = make([]float64, int(d.Run.NPRUP))
	d.Run.XERRUP = make([]float64, int(d.Run.NPRUP))
	d.Run.XMAXUP = make([]float64, int(d.Run.NPRUP))
	d.Run.LPRUP = make([]int32, int(d.Run.NPRUP))

	for i := 0; i < int(d.Run.NPRUP); i++ {
		_, err = fmt.Fscanf(
			buf,
			"%f %f %f %d\n",
			&d.Run.XSECUP[i],
			&d.Run.XERRUP[i],
			&d.Run.XMAXUP[i],
			&d.Run.LPRUP[i],
		)
	}
	//fmt.Printf(">>> data: %v\n", d.Run)

	if d.Version >= 2 {
		// do version-2 specific stuff
	}

	tok, err = dec.Token()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return nil, err
	}
	if end, ok := tok.(xml.EndElement); !ok || end.Name.Local != "init" {
		fmt.Printf("err: not a valid LHEF file (%T: %v) (expected </init>)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing init end-tag")
	}

	return d, nil
}

// advance to next event
func (d *Decoder) next() error {
LoopEvt:
	for {
		tok, err := d.dec.Token()
		if err != nil {
			return err
		}
		switch tt := tok.(type) {
		case xml.Comment:
			// skip
		case xml.CharData:
			// skip
		case xml.StartElement:
			switch tt.Name.Local {
			case "event":
				d.evt = tt
				break LoopEvt
			}
		}
	}

	return nil
}

// Read an event from the file
func (d *Decoder) Decode() (*HEPEUP, error) {

	// check whether the initialization was successful
	if d.Run.NPRUP < 0 {
		return nil, fmt.Errorf("lhef.Decode: initialization failed (no particle entries)")
	}

	err := d.next()
	if err != nil {
		return nil, err
	}

	// extract payload data
	tok, err := d.dec.Token()
	if err != nil {
		fmt.Printf("--> 1 err: %v\n", err)
		return nil, err
	}
	data, ok := tok.(xml.CharData)
	if !ok {
		return nil, fmt.Errorf("lhef.Decode: invalid token (%T: %v)", tok, tok)
	}
	if len(data) <= 0 {
		return nil, fmt.Errorf("lhef.Decode: empty event")
	}
	buf := bytes.NewBuffer(data)

	evt := &HEPEUP{
		NUP: 0,
	}

	_, err = fmt.Fscanf(
		buf,
		"\n%d %d %f %f %f %f\n",
		&evt.NUP,
		&evt.IDPRUP,
		&evt.XWGTUP,
		&evt.SCALUP,
		&evt.AQEDUP,
		&evt.AQCDUP,
	)
	if err != nil {
		fmt.Printf("--> 2 err: %v\n", err)
		fmt.Printf("  data:    %v\n", string(data))
		fmt.Printf("  token:   (%T: %v)\n", tok, tok)
		return nil, err
	}

	n := int(evt.NUP)
	evt.IDUP = make([]int64, n)
	evt.ISTUP = make([]int32, n)
	evt.MOTHUP = make([][2]int32, n)
	evt.ICOLUP = make([][2]int32, n)
	evt.PUP = make([][5]float64, n)
	evt.VTIMUP = make([]float64, n)
	evt.SPINUP = make([]float64, n)
	for i := 0; i < n; i++ {
		_, err = fmt.Fscanf(
			buf,
			"%d %d %d %d %d %d %f %f %f %f %f %f %f\n",
			&evt.IDUP[i],
			&evt.ISTUP[i],
			&evt.MOTHUP[i][0],
			&evt.MOTHUP[i][1],
			&evt.ICOLUP[i][0],
			&evt.ICOLUP[i][1],
			&evt.PUP[i][0], &evt.PUP[i][1], &evt.PUP[i][2],
			&evt.PUP[i][3], &evt.PUP[i][4],
			&evt.VTIMUP[i],
			&evt.SPINUP[i],
		)
		if err != nil {
			fmt.Printf("--> 3-%d err: %v\n", i, err)
			return nil, err
		}
	}

	// read any additional comments...
	/*evtComments*/ _ = buf.Bytes()

	// do

	// put "cursor" to next event...
	tok, err = d.dec.Token()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return nil, err
	}
	if end, ok := tok.(xml.EndElement); !ok || end.Name.Local != "event" {
		fmt.Printf("err: not a valid LHEF file (%T: %v) (expected </event>)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing event end-tag")
	}

	return evt, nil
}

// xml.Decoder.Token() (t Token, err error)

// EOF
<commit_msg>reader: make sure we detect header non-use<commit_after>package lhef

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
)

// Decoder represents an LHEF parser reading a particular input stream.
//
// A Decoder is initialized with an input io.Reader from which to read a version 1.0
// Les Houches Accord event file.
type Decoder struct {
	r       io.Reader
	dec     *xml.Decoder
	evt     xml.StartElement // the current xml.Token holding a HEPEUP
	Version int              // LHEF file version
	Run     HEPRUP           // User process run common block
}

func NewDecoder(r io.Reader) (*Decoder, error) {
	var err error
	dec := xml.NewDecoder(r)
	d := &Decoder{
		r:       r,
		dec:     dec,
		Version: -42,
		Run:     HEPRUP{},
	}
	// err := dec.dec.Decode(&dec.lhevt)
	// if err != nil {
	// 	return nil
	// }
	// fmt.Printf(">>> version=%v\n", dec.lhevt.Version)

	tok, err := dec.Token()
	if err != nil || tok == nil {
		return nil, err
	}

	// make sure we are reading a LHEF file
	start, ok := tok.(xml.StartElement)
	if !ok || start.Name.Local != "LesHouchesEvents" {
		fmt.Printf("err, not a LHEF file (%T: %v)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing LesHouchesEvent start-tag")
	}

	//fmt.Printf(">>> %v\n", start)
	version := start.Attr[0].Value
	fmt.Printf("    version=[%s]\n", version)
	switch version {
	default:
		d.Version = -42
	case "1.0":
		d.Version = 1
	case "2.0":
		d.Version = 2
	}

	var (
		init   xml.StartElement
		header xml.StartElement
	)

Loop:
	for {
		tok, err = dec.Token()
		if err != nil || tok == nil {
			return nil, err
		}
		switch tt := tok.(type) {
		case xml.Comment:
			// skip
		case xml.CharData:
			// skip
		case xml.StartElement:
			switch tt.Name.Local {
			case "init":
				init = tt
				break Loop
			case "header":
				header = tt //FIXME
				panic(fmt.Errorf("header not implemented: %v", header))
			}
		}
	}
	if init.Name.Local != "init" {
		fmt.Printf("err, not a valid LHEF file (%T: %v)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing init start-tag")
	}
	//fmt.Printf(">>> header: %v\n", header)
	//fmt.Printf(">>> init:   %v\n", init)

	// extract compulsory initialization information
	tok, err = dec.Token()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return nil, err
	}
	data, ok := tok.(xml.CharData)
	if !ok || len(data) == 0 {
		fmt.Printf("err, not a valid LHEF file (%T: %v)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing init payload")
	}
	buf := bytes.NewBuffer(data)
	_, err = fmt.Fscanf(
		buf,
		"\n%d %d %f %f %d %d %d %d %d %d\n",
		&d.Run.IDBMUP[0],
		&d.Run.IDBMUP[1],
		&d.Run.EBMUP[0],
		&d.Run.EBMUP[1],
		&d.Run.PDFGUP[0],
		&d.Run.PDFGUP[1],
		&d.Run.PDFSUP[0],
		&d.Run.PDFSUP[1],
		&d.Run.IDWTUP,
		&d.Run.NPRUP,
	)
	if err != nil {
		fmt.Printf("err:  %v\n", err)
		fmt.Printf("data: %v\n", string(data))
		return nil, err
	}

	d.Run.XSECUP = make([]float64, int(d.Run.NPRUP))
	d.Run.XERRUP = make([]float64, int(d.Run.NPRUP))
	d.Run.XMAXUP = make([]float64, int(d.Run.NPRUP))
	d.Run.LPRUP = make([]int32, int(d.Run.NPRUP))

	for i := 0; i < int(d.Run.NPRUP); i++ {
		_, err = fmt.Fscanf(
			buf,
			"%f %f %f %d\n",
			&d.Run.XSECUP[i],
			&d.Run.XERRUP[i],
			&d.Run.XMAXUP[i],
			&d.Run.LPRUP[i],
		)
	}
	//fmt.Printf(">>> data: %v\n", d.Run)

	if d.Version >= 2 {
		// do version-2 specific stuff
	}

	tok, err = dec.Token()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return nil, err
	}
	if end, ok := tok.(xml.EndElement); !ok || end.Name.Local != "init" {
		fmt.Printf("err: not a valid LHEF file (%T: %v) (expected </init>)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing init end-tag")
	}

	return d, nil
}

// advance to next event
func (d *Decoder) next() error {
LoopEvt:
	for {
		tok, err := d.dec.Token()
		if err != nil {
			return err
		}
		switch tt := tok.(type) {
		case xml.Comment:
			// skip
		case xml.CharData:
			// skip
		case xml.StartElement:
			switch tt.Name.Local {
			case "event":
				d.evt = tt
				break LoopEvt
			}
		}
	}

	return nil
}

// Read an event from the file
func (d *Decoder) Decode() (*HEPEUP, error) {

	// check whether the initialization was successful
	if d.Run.NPRUP < 0 {
		return nil, fmt.Errorf("lhef.Decode: initialization failed (no particle entries)")
	}

	err := d.next()
	if err != nil {
		return nil, err
	}

	// extract payload data
	tok, err := d.dec.Token()
	if err != nil {
		fmt.Printf("--> 1 err: %v\n", err)
		return nil, err
	}
	data, ok := tok.(xml.CharData)
	if !ok {
		return nil, fmt.Errorf("lhef.Decode: invalid token (%T: %v)", tok, tok)
	}
	if len(data) <= 0 {
		return nil, fmt.Errorf("lhef.Decode: empty event")
	}
	buf := bytes.NewBuffer(data)

	evt := &HEPEUP{
		NUP: 0,
	}

	_, err = fmt.Fscanf(
		buf,
		"\n%d %d %f %f %f %f\n",
		&evt.NUP,
		&evt.IDPRUP,
		&evt.XWGTUP,
		&evt.SCALUP,
		&evt.AQEDUP,
		&evt.AQCDUP,
	)
	if err != nil {
		fmt.Printf("--> 2 err: %v\n", err)
		fmt.Printf("  data:    %v\n", string(data))
		fmt.Printf("  token:   (%T: %v)\n", tok, tok)
		return nil, err
	}

	n := int(evt.NUP)
	evt.IDUP = make([]int64, n)
	evt.ISTUP = make([]int32, n)
	evt.MOTHUP = make([][2]int32, n)
	evt.ICOLUP = make([][2]int32, n)
	evt.PUP = make([][5]float64, n)
	evt.VTIMUP = make([]float64, n)
	evt.SPINUP = make([]float64, n)
	for i := 0; i < n; i++ {
		_, err = fmt.Fscanf(
			buf,
			"%d %d %d %d %d %d %f %f %f %f %f %f %f\n",
			&evt.IDUP[i],
			&evt.ISTUP[i],
			&evt.MOTHUP[i][0],
			&evt.MOTHUP[i][1],
			&evt.ICOLUP[i][0],
			&evt.ICOLUP[i][1],
			&evt.PUP[i][0], &evt.PUP[i][1], &evt.PUP[i][2],
			&evt.PUP[i][3], &evt.PUP[i][4],
			&evt.VTIMUP[i],
			&evt.SPINUP[i],
		)
		if err != nil {
			fmt.Printf("--> 3-%d err: %v\n", i, err)
			return nil, err
		}
	}

	// read any additional comments...
	/*evtComments*/ _ = buf.Bytes()

	// do

	// put "cursor" to next event...
	tok, err = d.dec.Token()
	if err != nil {
		fmt.Printf("err: %v\n", err)
		return nil, err
	}
	if end, ok := tok.(xml.EndElement); !ok || end.Name.Local != "event" {
		fmt.Printf("err: not a valid LHEF file (%T: %v) (expected </event>)\n", tok, tok)
		return nil, fmt.Errorf("lhef.Decoder: missing event end-tag")
	}

	return evt, nil
}

// xml.Decoder.Token() (t Token, err error)

// EOF
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"fmt"
	"sort"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"

	proxyconfig "istio.io/api/proxy/v1/config"
)

// Config is a configuration unit consisting of the type of configuration, the
// key identifier that is unique per type, and the content represented as a
// protobuf message.  The revision is optional, and if provided, identifies the
// last update operation on the object.
type Config struct {
	// Type is a short configuration name that matches the content message type
	Type string

	// Key is the type-dependent unique identifier for this config object, derived
	// from its content
	Key string

	// Revision is an opaque identifier for tracking updates to the config registry.
	// The implementation may use a change index or a commit log for the revision.
	// The config client should not make any assumptions about revisions and rely only on
	// exact equality to implement optimistic concurrency of read-write operations.
	//
	// The lifetime of an object of a particular revision depends on the underlying data store.
	// The data store may compactify old revisions in the interest of storage optimization.
	//
	// An empty revision carries a special meaning that the associated object has
	// not been stored and assigned a revision.
	Revision string

	// Content holds the configuration object as a protobuf message
	Content proto.Message
}

// ConfigStore describes a set of platform agnostic APIs that must be supported
// by the underlying platform to store and retrieve Istio configuration.
//
// The storage interface presented here assumes that the underlying storage
// layer supports _GET_ (list), _PUT_ (update), _POST_ (create) and _DELETE_
// semantics but does not guarantee any transactional semantics.
//
// Configuration objects possess a type property and a key property. The
// configuration key is derived from the content of the config object and
// uniquely identifies objects for a particular type. For example, if the
// object schema contains metadata fields _name_ and _namespace_, the
// configuration key may be defined as _namespace/name_.  Key definition is
// provided as part of the config type definition.
//
// _PUT_, _POST_, and _DELETE_ are mutator operations. These operations are
// asynchronous, and you might not see the effect immediately (e.g. _GET_ might
// not return the object by key immediately after you mutate the store.)
// Intermittent errors might occur even though the operation succeeds, so you
// should always check if the object store has been modified even if the
// mutating operation returns an error.  Objects should be created with _POST_
// operation and updated with _PUT_ operation.
//
// Revisions record the last mutation operation on each object. If a mutation
// is applied to a different revision of an object than what the underlying
// storage expects as defined by pure equality, the operation is blocked.  The
// client of this interface should not make assumptions about the structure or
// ordering of the revision identifier.
//
// Object references supplied and returned from this interface should be
// treated as read-only. Modifying them violates thread-safety.
type ConfigStore interface {
	// ConfigDescriptor exposes the configuration type schema known by the config store.
	// The type schema defines the bidrectional mapping between configuration
	// types and the protobuf encoding schema.
	ConfigDescriptor() ConfigDescriptor

	// Get retrieves a configuration element by a type and a key
	Get(typ, key string) (config proto.Message, exists bool, revision string)

	// List returns objects by type
	List(typ string) ([]Config, error)

	// Post creates a configuration object. If an object with the same key for
	// the type already exists, the operation fails with no side effects.
	Post(config proto.Message) (revision string, err error)

	// Put updates a configuration object in the store.  Put requires that the
	// object has been created.  Revision prevents overriding a value that has
	// been changed between prior _Get_ and _Put_ operation to achieve optimistic
	// concurrency. This method returns a new revision if the operation succeeds.
	Put(config proto.Message, oldRevision string) (newRevision string, err error)

	// Delete removes an object from the store by key
	Delete(typ, key string) error
}

// ConfigStoreCache is a local fully-replicated cache of the config store.  The
// cache actively synchronizes its local state with the remote store and
// provides a notification mechanism to receive update events. As such, the
// notification handlers must be registered prior to calling _Run_, and the
// cache requires initial synchronization grace period after calling  _Run_.
//
// Update notifications require the following consistency guarantee: the view
// in the cache must be AT LEAST as fresh as the moment notification arrives, but
// MAY BE more fresh (e.g. if _Delete_ cancels an _Add_ event).
//
// Handlers execute on the single worker queue in the order they are appended.
// Handlers receive the notification event and the associated object.  Note
// that all handlers must be registered before starting the cache controller.
type ConfigStoreCache interface {
	ConfigStore

	// RegisterEventHandler adds a handler to receive config update events for a
	// configuration type
	RegisterEventHandler(typ string, handler func(Config, Event))

	// Run until a signal is received
	Run(stop <-chan struct{})

	// HasSynced returns true after initial cache synchronization is complete
	HasSynced() bool
}

// ConfigDescriptor defines the bijection between the short type name and its
// fully qualified protobuf message name
type ConfigDescriptor []ProtoSchema

// ProtoSchema provides description of the configuration schema and its key function
type ProtoSchema struct {
	// Type refers to the short configuration type name
	Type string

	// MessageName refers to the protobuf message type name corresponding to the type
	MessageName string

	// Validate configuration as a protobuf message assuming the object is an
	// instance of the expected message type
	Validate func(config proto.Message) error

	// Key function derives the unique key from the configuration object metadata
	// properties. The object is not required to be complete or even valid to
	// derive its key, but must be an instance of the expected message type. For
	// example, if _name_ field is designated as the key, then the proto message
	// must only possess _name_ property.
	Key func(config proto.Message) string
}

// Types lists all known types in the config schema
func (descriptor ConfigDescriptor) Types() []string {
	types := make([]string, 0, len(descriptor))
	for _, t := range descriptor {
		types = append(types, t.Type)
	}
	return types
}

// GetByMessageName finds a schema by message name if it is available
func (descriptor ConfigDescriptor) GetByMessageName(name string) (ProtoSchema, bool) {
	for _, schema := range descriptor {
		if schema.MessageName == name {
			return schema, true
		}
	}
	return ProtoSchema{}, false
}

// GetByType finds a schema by type if it is available
func (descriptor ConfigDescriptor) GetByType(name string) (ProtoSchema, bool) {
	for _, schema := range descriptor {
		if schema.Type == name {
			return schema, true
		}
	}
	return ProtoSchema{}, false
}

// IstioConfigStore is a specialized interface to access config store using
// Istio configuration types
type IstioConfigStore interface {
	// RouteRules lists all routing rules
	RouteRules() map[string]*proxyconfig.RouteRule

	// IngressRules lists all ingress rules
	IngressRules() map[string]*proxyconfig.IngressRule

	// DestinationPolicies lists all destination rules
	DestinationPolicies() []*proxyconfig.DestinationPolicy

	// RouteRulesBySource selects routing rules by source service instances.
	// A rule must match at least one of the input service instances since the proxy
	// does not distinguish between source instances in the request.
	// The rules are sorted by precedence (high first) in a stable manner.
	RouteRulesBySource(instances []*ServiceInstance) []*proxyconfig.RouteRule

	// DestinationPolicy returns a policy for a service version.
	DestinationPolicy(destination string, tags Tags) *proxyconfig.DestinationVersionPolicy
}

const (
	// RouteRule defines the type for the route rule configuration
	RouteRule = "route-rule"
	// RouteRuleProto message name
	RouteRuleProto = "istio.proxy.v1.config.RouteRule"

	// IngressRule type
	IngressRule = "ingress-rule"
	// IngressRuleProto message name
	IngressRuleProto = "istio.proxy.v1.config.IngressRule"

	// DestinationPolicy defines the type for the destination policy configuration
	DestinationPolicy = "destination-policy"
	// DestinationPolicyProto message name
	DestinationPolicyProto = "istio.proxy.v1.config.DestinationPolicy"

	// HeaderURI is URI HTTP header
	HeaderURI = "uri"

	// HeaderAuthority is authority HTTP header
	HeaderAuthority = "authority"
)

var (
	// RouteRuleDescriptor describes route rules
	RouteRuleDescriptor = ProtoSchema{
		Type:        RouteRule,
		MessageName: RouteRuleProto,
		Validate:    ValidateRouteRule,
		Key: func(config proto.Message) string {
			rule := config.(*proxyconfig.RouteRule)
			return fmt.Sprintf("%s", rule.Name)
		},
	}

	// IngressRuleDescriptor describes ingress rules
	IngressRuleDescriptor = ProtoSchema{
		Type:        IngressRule,
		MessageName: IngressRuleProto,
		Validate:    ValidateIngressRule,
		Key: func(config proto.Message) string {
			rule := config.(*proxyconfig.IngressRule)
			return fmt.Sprintf("%s", rule.Name)
		},
	}

	// DestinationPolicyDescriptor describes destination rules
	DestinationPolicyDescriptor = ProtoSchema{
		Type:        DestinationPolicy,
		MessageName: DestinationPolicyProto,
		Validate:    ValidateDestinationPolicy,
		Key: func(config proto.Message) string {
			return config.(*proxyconfig.DestinationPolicy).Destination
		},
	}

	// IstioConfigTypes lists all Istio config types with schemas and validation
	IstioConfigTypes = ConfigDescriptor{
		RouteRuleDescriptor,
		IngressRuleDescriptor,
		DestinationPolicyDescriptor,
	}
)

// istioConfigStore provides a simple adapter for Istio configuration types
// from the generic config registry
type istioConfigStore struct {
	ConfigStore
}

// MakeIstioStore creates a wrapper around a store
func MakeIstioStore(store ConfigStore) IstioConfigStore {
	return &istioConfigStore{store}
}

func (i istioConfigStore) RouteRules() map[string]*proxyconfig.RouteRule {
	out := make(map[string]*proxyconfig.RouteRule)
	rs, err := i.List(RouteRule)
	if err != nil {
		glog.V(2).Infof("RouteRules => %v", err)
	}
	for _, r := range rs {
		if rule, ok := r.Content.(*proxyconfig.RouteRule); ok {
			out[r.Key] = rule
		}
	}
	return out
}

func (i *istioConfigStore) RouteRulesBySource(instances []*ServiceInstance) []*proxyconfig.RouteRule {
	rules := make([]Config, 0)
	for key, rule := range i.RouteRules() {
		// validate that rule match predicate applies to source service instances
		if rule.Match != nil {
			found := false
			for _, instance := range instances {
				// must match the source field if it is set
				if rule.Match.Source != "" && rule.Match.Source != instance.Service.Hostname {
					continue
				}
				// must match the tags field - the rule tags are a subset of the instance tags
				var tags Tags = rule.Match.SourceTags
				if tags.SubsetOf(instance.Tags) {
					found = true
					break
				}
			}
			if !found {
				continue
			}
		}
		rules = append(rules, Config{Key: key, Content: rule})
	}
	// sort by high precedence first, key string second (keys are unique)
	sort.Slice(rules, func(i, j int) bool {
		return rules[i].Content.(*proxyconfig.RouteRule).Precedence > rules[j].Content.(*proxyconfig.RouteRule).Precedence ||
			(rules[i].Content.(*proxyconfig.RouteRule).Precedence == rules[j].Content.(*proxyconfig.RouteRule).Precedence &&
				rules[i].Key < rules[j].Key)
	})

	// project to rules
	out := make([]*proxyconfig.RouteRule, len(rules))
	for i, rule := range rules {
		out[i] = rule.Content.(*proxyconfig.RouteRule)
	}
	return out
}

func (i *istioConfigStore) IngressRules() map[string]*proxyconfig.IngressRule {
	out := make(map[string]*proxyconfig.IngressRule)
	rs, err := i.List(IngressRule)
	if err != nil {
		glog.V(2).Infof("IngressRules => %v", err)
	}
	for _, r := range rs {
		if rule, ok := r.Content.(*proxyconfig.IngressRule); ok {
			out[r.Key] = rule
		}
	}
	return out
}

func (i *istioConfigStore) DestinationPolicies() []*proxyconfig.DestinationPolicy {
	out := make([]*proxyconfig.DestinationPolicy, 0)
	rs, err := i.List(DestinationPolicy)
	if err != nil {
		glog.V(2).Infof("DestinationPolicies => %v", err)
	}
	for _, r := range rs {
		if rule, ok := r.Content.(*proxyconfig.DestinationPolicy); ok {
			out = append(out, rule)
		}
	}
	return out
}

func (i *istioConfigStore) DestinationPolicy(destination string, tags Tags) *proxyconfig.DestinationVersionPolicy {
	value, exists, _ := i.Get(DestinationPolicy, destination)
	if exists {
		for _, policy := range value.(*proxyconfig.DestinationPolicy).Policy {
			if tags.Equals(policy.Tags) {
				return policy
			}
		}
	}
	return nil
}
<commit_msg>fix match rule discarded from ingress routes (#897) (#901)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"sort"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"

	proxyconfig "istio.io/api/proxy/v1/config"
)

// Config is a configuration unit consisting of the type of configuration, the
// key identifier that is unique per type, and the content represented as a
// protobuf message.  The revision is optional, and if provided, identifies the
// last update operation on the object.
type Config struct {
	// Type is a short configuration name that matches the content message type
	Type string

	// Key is the type-dependent unique identifier for this config object, derived
	// from its content
	Key string

	// Revision is an opaque identifier for tracking updates to the config registry.
	// The implementation may use a change index or a commit log for the revision.
	// The config client should not make any assumptions about revisions and rely only on
	// exact equality to implement optimistic concurrency of read-write operations.
	//
	// The lifetime of an object of a particular revision depends on the underlying data store.
	// The data store may compactify old revisions in the interest of storage optimization.
	//
	// An empty revision carries a special meaning that the associated object has
	// not been stored and assigned a revision.
	Revision string

	// Content holds the configuration object as a protobuf message
	Content proto.Message
}

// ConfigStore describes a set of platform agnostic APIs that must be supported
// by the underlying platform to store and retrieve Istio configuration.
//
// The storage interface presented here assumes that the underlying storage
// layer supports _GET_ (list), _PUT_ (update), _POST_ (create) and _DELETE_
// semantics but does not guarantee any transactional semantics.
//
// Configuration objects possess a type property and a key property. The
// configuration key is derived from the content of the config object and
// uniquely identifies objects for a particular type. For example, if the
// object schema contains metadata fields _name_ and _namespace_, the
// configuration key may be defined as _namespace/name_.  Key definition is
// provided as part of the config type definition.
//
// _PUT_, _POST_, and _DELETE_ are mutator operations. These operations are
// asynchronous, and you might not see the effect immediately (e.g. _GET_ might
// not return the object by key immediately after you mutate the store.)
// Intermittent errors might occur even though the operation succeeds, so you
// should always check if the object store has been modified even if the
// mutating operation returns an error.  Objects should be created with _POST_
// operation and updated with _PUT_ operation.
//
// Revisions record the last mutation operation on each object. If a mutation
// is applied to a different revision of an object than what the underlying
// storage expects as defined by pure equality, the operation is blocked.  The
// client of this interface should not make assumptions about the structure or
// ordering of the revision identifier.
//
// Object references supplied and returned from this interface should be
// treated as read-only. Modifying them violates thread-safety.
type ConfigStore interface {
	// ConfigDescriptor exposes the configuration type schema known by the config store.
	// The type schema defines the bidrectional mapping between configuration
	// types and the protobuf encoding schema.
	ConfigDescriptor() ConfigDescriptor

	// Get retrieves a configuration element by a type and a key
	Get(typ, key string) (config proto.Message, exists bool, revision string)

	// List returns objects by type
	List(typ string) ([]Config, error)

	// Post creates a configuration object. If an object with the same key for
	// the type already exists, the operation fails with no side effects.
	Post(config proto.Message) (revision string, err error)

	// Put updates a configuration object in the store.  Put requires that the
	// object has been created.  Revision prevents overriding a value that has
	// been changed between prior _Get_ and _Put_ operation to achieve optimistic
	// concurrency. This method returns a new revision if the operation succeeds.
	Put(config proto.Message, oldRevision string) (newRevision string, err error)

	// Delete removes an object from the store by key
	Delete(typ, key string) error
}

// ConfigStoreCache is a local fully-replicated cache of the config store.  The
// cache actively synchronizes its local state with the remote store and
// provides a notification mechanism to receive update events. As such, the
// notification handlers must be registered prior to calling _Run_, and the
// cache requires initial synchronization grace period after calling  _Run_.
//
// Update notifications require the following consistency guarantee: the view
// in the cache must be AT LEAST as fresh as the moment notification arrives, but
// MAY BE more fresh (e.g. if _Delete_ cancels an _Add_ event).
//
// Handlers execute on the single worker queue in the order they are appended.
// Handlers receive the notification event and the associated object.  Note
// that all handlers must be registered before starting the cache controller.
type ConfigStoreCache interface {
	ConfigStore

	// RegisterEventHandler adds a handler to receive config update events for a
	// configuration type
	RegisterEventHandler(typ string, handler func(Config, Event))

	// Run until a signal is received
	Run(stop <-chan struct{})

	// HasSynced returns true after initial cache synchronization is complete
	HasSynced() bool
}

// ConfigDescriptor defines the bijection between the short type name and its
// fully qualified protobuf message name
type ConfigDescriptor []ProtoSchema

// ProtoSchema provides description of the configuration schema and its key function
type ProtoSchema struct {
	// Type refers to the short configuration type name
	Type string

	// MessageName refers to the protobuf message type name corresponding to the type
	MessageName string

	// Validate configuration as a protobuf message assuming the object is an
	// instance of the expected message type
	Validate func(config proto.Message) error

	// Key function derives the unique key from the configuration object metadata
	// properties. The object is not required to be complete or even valid to
	// derive its key, but must be an instance of the expected message type. For
	// example, if _name_ field is designated as the key, then the proto message
	// must only possess _name_ property.
	Key func(config proto.Message) string
}

// Types lists all known types in the config schema
func (descriptor ConfigDescriptor) Types() []string {
	types := make([]string, 0, len(descriptor))
	for _, t := range descriptor {
		types = append(types, t.Type)
	}
	return types
}

// GetByMessageName finds a schema by message name if it is available
func (descriptor ConfigDescriptor) GetByMessageName(name string) (ProtoSchema, bool) {
	for _, schema := range descriptor {
		if schema.MessageName == name {
			return schema, true
		}
	}
	return ProtoSchema{}, false
}

// GetByType finds a schema by type if it is available
func (descriptor ConfigDescriptor) GetByType(name string) (ProtoSchema, bool) {
	for _, schema := range descriptor {
		if schema.Type == name {
			return schema, true
		}
	}
	return ProtoSchema{}, false
}

// IstioConfigStore is a specialized interface to access config store using
// Istio configuration types
type IstioConfigStore interface {
	// RouteRules lists all routing rules
	RouteRules() map[string]*proxyconfig.RouteRule

	// IngressRules lists all ingress rules
	IngressRules() map[string]*proxyconfig.IngressRule

	// DestinationPolicies lists all destination rules
	DestinationPolicies() []*proxyconfig.DestinationPolicy

	// RouteRulesBySource selects routing rules by source service instances.
	// A rule must match at least one of the input service instances since the proxy
	// does not distinguish between source instances in the request.
	// The rules are sorted by precedence (high first) in a stable manner.
	RouteRulesBySource(instances []*ServiceInstance) []*proxyconfig.RouteRule

	// DestinationPolicy returns a policy for a service version.
	DestinationPolicy(destination string, tags Tags) *proxyconfig.DestinationVersionPolicy
}

const (
	// RouteRule defines the type for the route rule configuration
	RouteRule = "route-rule"
	// RouteRuleProto message name
	RouteRuleProto = "istio.proxy.v1.config.RouteRule"

	// IngressRule type
	IngressRule = "ingress-rule"
	// IngressRuleProto message name
	IngressRuleProto = "istio.proxy.v1.config.IngressRule"

	// DestinationPolicy defines the type for the destination policy configuration
	DestinationPolicy = "destination-policy"
	// DestinationPolicyProto message name
	DestinationPolicyProto = "istio.proxy.v1.config.DestinationPolicy"

	// HeaderURI is URI HTTP header
	HeaderURI = "uri"

	// HeaderAuthority is authority HTTP header
	HeaderAuthority = "authority"
)

var (
	// RouteRuleDescriptor describes route rules
	RouteRuleDescriptor = ProtoSchema{
		Type:        RouteRule,
		MessageName: RouteRuleProto,
		Validate:    ValidateRouteRule,
		Key: func(config proto.Message) string {
			rule := config.(*proxyconfig.RouteRule)
			return rule.Name
		},
	}

	// IngressRuleDescriptor describes ingress rules
	IngressRuleDescriptor = ProtoSchema{
		Type:        IngressRule,
		MessageName: IngressRuleProto,
		Validate:    ValidateIngressRule,
		Key: func(config proto.Message) string {
			rule := config.(*proxyconfig.IngressRule)
			return rule.Name
		},
	}

	// DestinationPolicyDescriptor describes destination rules
	DestinationPolicyDescriptor = ProtoSchema{
		Type:        DestinationPolicy,
		MessageName: DestinationPolicyProto,
		Validate:    ValidateDestinationPolicy,
		Key: func(config proto.Message) string {
			return config.(*proxyconfig.DestinationPolicy).Destination
		},
	}

	// IstioConfigTypes lists all Istio config types with schemas and validation
	IstioConfigTypes = ConfigDescriptor{
		RouteRuleDescriptor,
		IngressRuleDescriptor,
		DestinationPolicyDescriptor,
	}
)

// istioConfigStore provides a simple adapter for Istio configuration types
// from the generic config registry
type istioConfigStore struct {
	ConfigStore
}

// MakeIstioStore creates a wrapper around a store
func MakeIstioStore(store ConfigStore) IstioConfigStore {
	return &istioConfigStore{store}
}

func (i istioConfigStore) RouteRules() map[string]*proxyconfig.RouteRule {
	out := make(map[string]*proxyconfig.RouteRule)
	rs, err := i.List(RouteRule)
	if err != nil {
		glog.V(2).Infof("RouteRules => %v", err)
	}
	for _, r := range rs {
		if rule, ok := r.Content.(*proxyconfig.RouteRule); ok {
			out[r.Key] = rule
		}
	}
	return out
}

func (i *istioConfigStore) RouteRulesBySource(instances []*ServiceInstance) []*proxyconfig.RouteRule {
	rules := make([]Config, 0)
	for key, rule := range i.RouteRules() {
		// validate that rule match predicate applies to source service instances
		if rule.Match != nil && rule.Match.Source != "" {
			found := false
			for _, instance := range instances {
				// must match the source field if it is set
				if rule.Match.Source != instance.Service.Hostname {
					continue
				}
				// must match the tags field - the rule tags are a subset of the instance tags
				var tags Tags = rule.Match.SourceTags
				if tags.SubsetOf(instance.Tags) {
					found = true
					break
				}
			}
			if !found {
				continue
			}
		}
		rules = append(rules, Config{Key: key, Content: rule})
	}
	// sort by high precedence first, key string second (keys are unique)
	sort.Slice(rules, func(i, j int) bool {
		return rules[i].Content.(*proxyconfig.RouteRule).Precedence > rules[j].Content.(*proxyconfig.RouteRule).Precedence ||
			(rules[i].Content.(*proxyconfig.RouteRule).Precedence == rules[j].Content.(*proxyconfig.RouteRule).Precedence &&
				rules[i].Key < rules[j].Key)
	})

	// project to rules
	out := make([]*proxyconfig.RouteRule, len(rules))
	for i, rule := range rules {
		out[i] = rule.Content.(*proxyconfig.RouteRule)
	}
	return out
}

func (i *istioConfigStore) IngressRules() map[string]*proxyconfig.IngressRule {
	out := make(map[string]*proxyconfig.IngressRule)
	rs, err := i.List(IngressRule)
	if err != nil {
		glog.V(2).Infof("IngressRules => %v", err)
	}
	for _, r := range rs {
		if rule, ok := r.Content.(*proxyconfig.IngressRule); ok {
			out[r.Key] = rule
		}
	}
	return out
}

func (i *istioConfigStore) DestinationPolicies() []*proxyconfig.DestinationPolicy {
	out := make([]*proxyconfig.DestinationPolicy, 0)
	rs, err := i.List(DestinationPolicy)
	if err != nil {
		glog.V(2).Infof("DestinationPolicies => %v", err)
	}
	for _, r := range rs {
		if rule, ok := r.Content.(*proxyconfig.DestinationPolicy); ok {
			out = append(out, rule)
		}
	}
	return out
}

func (i *istioConfigStore) DestinationPolicy(destination string, tags Tags) *proxyconfig.DestinationVersionPolicy {
	value, exists, _ := i.Get(DestinationPolicy, destination)
	if exists {
		for _, policy := range value.(*proxyconfig.DestinationPolicy).Policy {
			if tags.Equals(policy.Tags) {
				return policy
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package model

import (
	"github.com/garyburd/redigo/redis"
	"log"
	"time"
	"strconv"
"github.com/evolsnow/samaritan/conn"
)

//All model's redis actions

func getPassword(id int) string {
	c := conn.Pool.Get()
	defer c.Close()
	user := "user:" + strconv.Itoa(id)
	pwd, _ := redis.String(c.Do("HGET", user, "passwd"))
	return pwd
}

func updatePassword(id int, pwd string) {
	c := conn.Pool.Get()
	defer c.Close()
	user := "user:" + strconv.Itoa(id)
	_, err := c.Do("HSET", user, "passwd", pwd)
	if err != nil {
		log.Println("Failed to update password for user:%s", id)
	}
}

func readUser(id int) (reply []interface{}, err error) {
	c := conn.Pool.Get()
	defer c.Close()
	user := "user:" + strconv.Itoa(id)
	reply, err = redis.Values(c.Do("HGETALL", user))
	return
}

func readMissionRcv(mid int, uids []int) ([]interface{}, err error) {
	c := conn.Pool.Get()
	defer c.Close()
	replys := make([]interface{}, len(uids))
	receiverSet := "mission:receivers" + strconv.Itoa(mid)
	//todo luc

	return replys, nil
}

func readMission(mid int) (reply []interface{}, err error) {
	c := conn.Pool.Get()
	defer c.Close()
	mission := "mission:" + strconv.Itoa(mid)
	reply, err = redis.Values(c.Do("HGETALL", mission))
	return
}

func updateMissionRcv(mid, uid int) (err error) {
	c := conn.Pool.Get()
	defer c.Close()
	receiverSet := "mission:receivers" + strconv.Itoa(mid)
	_, err = c.Do("SADD", receiverSet, uid)
	return
}

func createMission(m *Mission){

}<commit_msg>split redis actions<commit_after>package model

import (
	"github.com/evolsnow/samaritan/conn"
	"github.com/garyburd/redigo/redis"
	"log"
	"strconv"
	"time"
)

//All model's redis actions

func getPassword(id int) string {
	c := conn.Pool.Get()
	defer c.Close()
	user := "user:" + strconv.Itoa(id)
	pwd, _ := redis.String(c.Do("HGET", user, "passwd"))
	return pwd
}

func updatePassword(id int, pwd string) {
	c := conn.Pool.Get()
	defer c.Close()
	user := "user:" + strconv.Itoa(id)
	_, err := c.Do("HSET", user, "passwd", pwd)
	if err != nil {
		log.Println("Failed to update password for user:%s", id)
	}
}

func readUser(id int) (reply []interface{}, err error) {
	c := conn.Pool.Get()
	defer c.Close()
	user := "user:" + strconv.Itoa(id)
	reply, err = redis.Values(c.Do("HGETALL", user))
	return
}

func readMissionRcv(mid int, uids []int) ([]interface{}, error) {
	c := conn.Pool.Get()
	defer c.Close()
	replys := make([]interface{}, len(uids))
	receiverSet := "mission:receivers" + strconv.Itoa(mid)
	//todo luc

	return replys, nil
}

func readMission(mid int) (reply []interface{}, err error) {
	c := conn.Pool.Get()
	defer c.Close()
	mission := "mission:" + strconv.Itoa(mid)
	reply, err = redis.Values(c.Do("HGETALL", mission))
	return
}

func updateMissionRcv(mid, uid int) (err error) {
	c := conn.Pool.Get()
	defer c.Close()
	receiverSet := "mission:receivers" + strconv.Itoa(mid)
	_, err = c.Do("SADD", receiverSet, uid)
	return
}

func createMission(m *Mission) {

}
<|endoftext|>


<commit_before>package ini

import "strconv"

type propertyConverter func(string) (interface{}, error)

// Returns true if the config contains a section with the given name, otherwise
// false.
func (c *Config) HasSection(section string) bool {
	_, exists := (*c)[section]
	return exists
}

// Returns true if a) the given section exists and b) the given property can be
// found within the section. Otherwise false is returned.
func (c *Config) HasProperty(section, property string) bool {
	items, err := c.GetItems(section)
	if err != nil {
		return false
	}
	for _, item := range items {
		if item.Property == property {
			return true
		}
	}
	return false
}

// Returns a list of all section names of the config.
// Note: the order is not deterministic!
func (c *Config) GetSections() (sections []string) {
	sections = []string{}
	for section, _ := range *c {
		sections = append(sections, section)
	}
	return sections
}

// Get a slice of *Item structs from the given section. The order of the
// elements in the returned slice is not determined. If the section does not
// exist, NoSectionError is returned.
func (c *Config) GetItems(section string) (items []*Item, err error) {
	items = []*Item{}
	if !c.HasSection(section) {
		return items, NoSectionError
	}
	for property, value := range (*c)[section] {
		items = append(items, &Item{property, value})
	}
	return items, nil
}

// Get the value of the passed property in the given section. If the section
// does not exist, NoSectionError is returned. If the property does not exist
// in the given section, NoPropertyError is returned.
func (c *Config) Get(section, property string) (value string, err error) {
	items, err := c.GetItems(section)
	if err != nil {
		return value, err
	}
	for _, item := range items {
		if (*item).Property == property {
			return (*item).Value, nil
		}
	}
	return value, NoPropertyError{property}
}

// Get the value of the passed property in the given section, apply the given
// function f to it and return the function's return values. The function must
// have the signature ``func(s string) (value interface{}, err error)``. If the
// passed section does not exist, the error NoSectionError will be returned.
// If the property does not exist within this section, NoPropertyError will be
// returned. If there was a different error returned, it came from the passed
// function.
func (c *Config) GetFormatted(section, property string, f propertyConverter) (value interface{}, err error) {
	strValue, err := c.Get(section, property)
	if err != nil {
		emptyValue, _ := f("")
		return emptyValue, err
	}
	value, err = f(strValue)
	if err != nil {
		emptyValue, _ := f("")
		return emptyValue, err
	}
	return value, nil
}

// Gets the value of the given property in the given section and returns it as
// a boolean value.
// The accepted values are: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False.
// All other values result in an error.
func (c *Config) GetBool(section, property string) (value bool, err error) {
	f := func(s string) (interface{}, error) {
		return strconv.ParseBool(s)
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(bool), err
}

// Gets the value of the given property in the given section and returns it as
// an integer. If the value cannot be converted to an int, an error is returned.
// For other possible error return values, see the documentation of the Get
// method.
func (c *Config) GetInt(section, property string) (value int, err error) {
	f := func(s string) (interface{}, error) {
		return strconv.Atoi(s)
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(int), err
}

// Gets the value of the given property in the given section and returns it as
// a float32.
func (c *Config) GetFloat32(section, property string) (value float32, err error) {
	f := func(s string) (interface{}, error) {
		value, err := strconv.ParseFloat(s, 32)
		if err != nil {
			return float32(0.0), err
		}
		return float32(value), nil
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(float32), err
}

// Gets the value of the given property in the given section and returns it as
// a float64.
func (c *Config) GetFloat64(section, property string) (value float64, err error) {
	f := func(s string) (interface{}, error) {
		return strconv.ParseFloat(s, 64)
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(float64), err
}
<commit_msg>added *Config.GetDefault method<commit_after>package ini

import "strconv"

type propertyConverter func(string) (interface{}, error)

// Returns true if the config contains a section with the given name, otherwise
// false.
func (c *Config) HasSection(section string) bool {
	_, exists := (*c)[section]
	return exists
}

// Returns true if a) the given section exists and b) the given property can be
// found within the section. Otherwise false is returned.
func (c *Config) HasProperty(section, property string) bool {
	items, err := c.GetItems(section)
	if err != nil {
		return false
	}
	for _, item := range items {
		if item.Property == property {
			return true
		}
	}
	return false
}

// Returns a list of all section names of the config.
// Note: the order is not deterministic!
func (c *Config) GetSections() (sections []string) {
	sections = []string{}
	for section, _ := range *c {
		sections = append(sections, section)
	}
	return sections
}

// Get a slice of *Item structs from the given section. The order of the
// elements in the returned slice is not determined. If the section does not
// exist, NoSectionError is returned.
func (c *Config) GetItems(section string) (items []*Item, err error) {
	items = []*Item{}
	if !c.HasSection(section) {
		return items, NoSectionError
	}
	for property, value := range (*c)[section] {
		items = append(items, &Item{property, value})
	}
	return items, nil
}

// Get the value of the passed property in the given section. If the section
// does not exist, NoSectionError is returned. If the property does not exist
// in the given section, NoPropertyError is returned.
func (c *Config) Get(section, property string) (value string, err error) {
	items, err := c.GetItems(section)
	if err != nil {
		return value, err
	}
	for _, item := range items {
		if (*item).Property == property {
			return (*item).Value, nil
		}
	}
	return value, NoPropertyError{property}
}

// Get the value of the passed property in the given section. If either the
// section doesn't exist or the property doesn't exist in the given section,
// the given default value `default_` is returned.
func (c *Config) GetDefault(section, property, default_ string) (value string) {
	value, err := c.Get(section, property)
	if err != nil {
		value = default_
	}
	return value
}

// Get the value of the passed property in the given section, apply the given
// function f to it and return the function's return values. The function must
// have the signature ``func(s string) (value interface{}, err error)``. If the
// passed section does not exist, the error NoSectionError will be returned.
// If the property does not exist within this section, NoPropertyError will be
// returned. If there was a different error returned, it came from the passed
// function.
func (c *Config) GetFormatted(section, property string, f propertyConverter) (value interface{}, err error) {
	strValue, err := c.Get(section, property)
	if err != nil {
		emptyValue, _ := f("")
		return emptyValue, err
	}
	value, err = f(strValue)
	if err != nil {
		emptyValue, _ := f("")
		return emptyValue, err
	}
	return value, nil
}

// Gets the value of the given property in the given section and returns it as
// a boolean value.
// The accepted values are: 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False.
// All other values result in an error.
func (c *Config) GetBool(section, property string) (value bool, err error) {
	f := func(s string) (interface{}, error) {
		return strconv.ParseBool(s)
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(bool), err
}

// Gets the value of the given property in the given section and returns it as
// an integer. If the value cannot be converted to an int, an error is returned.
// For other possible error return values, see the documentation of the Get
// method.
func (c *Config) GetInt(section, property string) (value int, err error) {
	f := func(s string) (interface{}, error) {
		return strconv.Atoi(s)
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(int), err
}

// Gets the value of the given property in the given section and returns it as
// a float32.
func (c *Config) GetFloat32(section, property string) (value float32, err error) {
	f := func(s string) (interface{}, error) {
		value, err := strconv.ParseFloat(s, 32)
		if err != nil {
			return float32(0.0), err
		}
		return float32(value), nil
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(float32), err
}

// Gets the value of the given property in the given section and returns it as
// a float64.
func (c *Config) GetFloat64(section, property string) (value float64, err error) {
	f := func(s string) (interface{}, error) {
		return strconv.ParseFloat(s, 64)
	}
	v, err := c.GetFormatted(section, property, f)
	return v.(float64), err
}
<|endoftext|>


<commit_before>package gedis

import (
	"fmt"
	"io"
)

// Reads an int64 from the Reader
func ReadNumber(r Reader) (n int64, err error) {
	b := make([]byte, 1)

	var sign int64 = 1

	_, err = r.Read(b)
	if err != nil {
		return
	}
	if b[0] == '-' {
		sign = -1
		b[0] = '0'
	}

	for {
		if b[0] >= '0' && b[0] <= '9' {
			n = n*10 + int64(b[0]-'0')
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if b[0] == '\n' {
				break
			} else {
				return 0, NewParseError("Invalid EOF")
			}
		} else {
			return 0, NewParseError("Invalid character")
		}

		_, err = r.Read(b)
		if err == io.EOF {
			break
		} else if err != nil {
			return
		}
	}

	return sign * n, nil
}

func readLine(r Reader) (line string, err error) {
	bs := make([]byte, 1024)
	l := 0

	b := make([]byte, 1)

	for {
		_, err = r.Read(b)

		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return "", err
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if err != nil {
				return "", err
			}
			if b[0] == '\n' {
				break
			} else {
				bs[l] = '\r'
				l++
			}
		}

		bs[l] = b[0]
		l++
	}

	line = string(bs[:l])

	return line, err
}

func readBulk(r Reader) (interface{}, error) {
	var bs []byte

	numBytes, err := ReadNumber(r)
	if err != nil {
		return nil, err
	}

	if numBytes == -1 {
		return nil, nil
	}

	bs = make([]byte, numBytes)

	bytesRead, err := r.Read(bs)

	if err != nil {
		return nil, err
	} else if int64(bytesRead) != numBytes {
		return nil, NewParseError("Invalid byte count read")
	}

	// Must read following two bytes for \r\n
	crlf := make([]byte, 2)
	r.Read(crlf)

	if crlf[0] != '\r' || crlf[1] != '\n' {
		return nil, NewParseError("Invalid EOF")
	}

	return string(bs), nil
}

func Read(r Reader) (ret interface{}, err error) {
	kind := make([]byte, 1)

	_, err = r.Read(kind)
	if err != nil {
		return
	}

	switch kind[0] {
	case '+':
		ret, err = readLine(r)
		if err == nil {
			if bs, ok := ret.(string); ok {
				return bs, nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '-':
		ret, err = readLine(r)

		if err == nil {
			if bs, ok := ret.([]byte); ok {
				err = fmt.Errorf(string(bs))
			} else {
				err = fmt.Errorf("Cannot convert to []byte: %#v", ret)
			}
			ret = nil
		}
	case ':':
		ret, err = ReadNumber(r)
	case '$':
		ret, err = readBulk(r)
		if err == nil {
			if ret == nil {
				return nil, nil
			}
			if bs, ok := ret.(string); ok {
				return bs, nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '*':
		n, err := ReadNumber(r)
		if err != nil {
			return nil, err
		}

		if n == -1 {
			return nil, nil
		}

		res := make([]interface{}, n)

		for i := int64(0); i < n; i++ {
			ret, err := Read(r)
			if err == nil {
				res[i] = ret
			} else {
				res[i] = err
			}
		}

		ret = res
	default:
		return nil, fmt.Errorf("Unexpected character %#v", kind[0])
	}

	return
}
<commit_msg>Fix parsing of errors on Read<commit_after>package gedis

import (
	"fmt"
	"io"
)

// Reads an int64 from the Reader
func ReadNumber(r Reader) (n int64, err error) {
	b := make([]byte, 1)

	var sign int64 = 1

	_, err = r.Read(b)
	if err != nil {
		return
	}
	if b[0] == '-' {
		sign = -1
		b[0] = '0'
	}

	for {
		if b[0] >= '0' && b[0] <= '9' {
			n = n*10 + int64(b[0]-'0')
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if b[0] == '\n' {
				break
			} else {
				return 0, NewParseError("Invalid EOF")
			}
		} else {
			return 0, NewParseError("Invalid character")
		}

		_, err = r.Read(b)
		if err == io.EOF {
			break
		} else if err != nil {
			return
		}
	}

	return sign * n, nil
}

func readLine(r Reader) (line string, err error) {
	bs := make([]byte, 1024)
	l := 0

	b := make([]byte, 1)

	for {
		_, err = r.Read(b)

		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return "", err
		} else if b[0] == '\r' {
			_, err = r.Read(b)
			if err != nil {
				return "", err
			}
			if b[0] == '\n' {
				break
			} else {
				bs[l] = '\r'
				l++
			}
		}

		bs[l] = b[0]
		l++
	}

	line = string(bs[:l])

	return line, err
}

func readBulk(r Reader) (interface{}, error) {
	var bs []byte

	numBytes, err := ReadNumber(r)
	if err != nil {
		return nil, err
	}

	if numBytes == -1 {
		return nil, nil
	}

	bs = make([]byte, numBytes)

	bytesRead, err := r.Read(bs)

	if err != nil {
		return nil, err
	} else if int64(bytesRead) != numBytes {
		return nil, NewParseError("Invalid byte count read")
	}

	// Must read following two bytes for \r\n
	crlf := make([]byte, 2)
	r.Read(crlf)

	if crlf[0] != '\r' || crlf[1] != '\n' {
		return nil, NewParseError("Invalid EOF")
	}

	return string(bs), nil
}

func Read(r Reader) (ret interface{}, err error) {
	kind := make([]byte, 1)

	_, err = r.Read(kind)
	if err != nil {
		return
	}

	switch kind[0] {
	case '+':
		ret, err = readLine(r)
		if err == nil {
			if bs, ok := ret.(string); ok {
				return bs, nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '-':
		ret, err = readLine(r)

		if err == nil {
			if bs, ok := ret.(string); ok {
				err = fmt.Errorf(string(bs))
			} else {
				err = fmt.Errorf("Cannot convert to []byte: %#v", ret)
			}
			ret = nil
		}
	case ':':
		ret, err = ReadNumber(r)
	case '$':
		ret, err = readBulk(r)
		if err == nil {
			if ret == nil {
				return nil, nil
			}
			if bs, ok := ret.(string); ok {
				return bs, nil
			} else {
				return nil, fmt.Errorf("Cannot convert to string: %#v", ret)
			}
		} else {
			return nil, err
		}
	case '*':
		n, err := ReadNumber(r)
		if err != nil {
			return nil, err
		}

		if n == -1 {
			return nil, nil
		}

		res := make([]interface{}, n)

		for i := int64(0); i < n; i++ {
			ret, err := Read(r)
			if err == nil {
				res[i] = ret
			} else {
				res[i] = err
			}
		}

		ret = res
	default:
		return nil, fmt.Errorf("Unexpected character %#v", kind[0])
	}

	return
}
<|endoftext|>


<commit_before>package model

import (
	"database/sql"
	"reflect"
	"strings"
	"sync"
	"time"
)

const (
	OrderByPK          = "ngorm:order_by_primary_key"
	QueryDestination   = "ngorm:query_destination"
	QueryOption        = "ngorm:query_option"
	Query              = "ngorm:query"
	HookQueryAfter     = "ngorm:query_after"
	HookQueryAfterFind = "ngorm:query_after_find"
	HookBeforeCreate   = "ngorm:before_create_hook"
	HookBeforeSave     = "ngorm:before_save_hook"
	BeforeCreate       = "ngorm:before_create"
	AfterCreate        = "ngorm:after_create"
	HookAfterCreate    = "ngorm:after_create"
	HookAfterSave      = "ngorm:after_save_hook"
	UpdateAttrs        = "ngorm:update_attrs"
)

//Model defines common fields that are used for defining SQL Tables. This is a
//helper that you can embed in your own struct definition.
//
// By embedding this, there is no need to define the supplied fields. For
// example.
//
//  type User struct {
//    Model
//    Name string
//  }
// Is the same as this
//  type User   struct {
//    ID        uint `gorm:"primary_key"`
//    CreatedAt time.Time
//    UpdatedAt time.Time
//    DeletedAt *time.Time `sql:"index"`
//    Name      string
//  }
type Model struct {
	ID        uint `gorm:"primary_key"`
	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt *time.Time `sql:"index"`
}

//Struct model definition
type Struct struct {
	PrimaryFields    []*StructField
	StructFields     []*StructField
	ModelType        reflect.Type
	DefaultTableName string
}

// StructField model field's struct definition
type StructField struct {

	// DBName is the name of the field as it is seen in the database, for
	// instance a field ID can be represented in the database as id.
	DBName          string
	Name            string
	Names           []string
	IsPrimaryKey    bool
	IsNormal        bool
	IsIgnored       bool
	IsScanner       bool
	HasDefaultValue bool
	Tag             reflect.StructTag
	TagSettings     map[string]string
	Struct          reflect.StructField
	IsForeignKey    bool
	Relationship    *Relationship
}

//Clone retruns a deep copy of the StructField
func (s *StructField) Clone() *StructField {
	clone := &StructField{
		DBName:          s.DBName,
		Name:            s.Name,
		Names:           s.Names,
		IsPrimaryKey:    s.IsPrimaryKey,
		IsNormal:        s.IsNormal,
		IsIgnored:       s.IsIgnored,
		IsScanner:       s.IsScanner,
		HasDefaultValue: s.HasDefaultValue,
		Tag:             s.Tag,
		TagSettings:     map[string]string{},
		Struct:          s.Struct,
		IsForeignKey:    s.IsForeignKey,
		Relationship:    s.Relationship,
	}

	for key, value := range s.TagSettings {
		clone.TagSettings[key] = value
	}

	return clone
}

// Relationship described the relationship between models
type Relationship struct {
	Kind                         string
	PolymorphicType              string
	PolymorphicDBName            string
	PolymorphicValue             string
	ForeignFieldNames            []string
	ForeignDBNames               []string
	AssociationForeignFieldNames []string
	AssociationForeignDBNames    []string
}

//ParseTagSetting returns a map[string]string for the tags that are set.
func ParseTagSetting(tags reflect.StructTag) map[string]string {
	setting := map[string]string{}
	for _, str := range []string{tags.Get("sql"), tags.Get("gorm")} {
		tags := strings.Split(str, ";")
		for _, value := range tags {
			v := strings.Split(value, ":")
			k := strings.TrimSpace(strings.ToUpper(v[0]))
			if len(v) >= 2 {
				setting[k] = strings.Join(v[1:], ":")
			} else {
				setting[k] = k
			}
		}
	}
	return setting
}

//SafeStructsMap provide safe storage and accessing of *Struct.
type SafeStructsMap struct {
	m map[reflect.Type]*Struct
	l *sync.RWMutex
}

//Set stores value witht the given key.
func (s *SafeStructsMap) Set(key reflect.Type, value *Struct) {
	s.l.Lock()
	defer s.l.Unlock()
	s.m[key] = value
}

//Get retrieves the value stored with the gived key.
func (s *SafeStructsMap) Get(key reflect.Type) *Struct {
	s.l.RLock()
	defer s.l.RUnlock()
	return s.m[key]
}

//NewStructsMap returns a safe map for storing *Struct objects.
func NewStructsMap() *SafeStructsMap {
	return &SafeStructsMap{l: new(sync.RWMutex), m: make(map[reflect.Type]*Struct)}
}

//Scope is the scope level of SQL building.
type Scope struct {
	Value           interface{}
	SQL             string
	SQLVars         []interface{}
	InstanceID      string
	PrimaryKeyField *Field
	SkipLeft        bool
	SelectAttrs     *[]string
	mu              sync.RWMutex
	data            map[string]interface{}
}

func (s *Scope) Set(key string, value interface{}) {
	s.mu.Lock()
	s.data[key] = value
	s.mu.Unlock()
}

func (s *Scope) Get(key string) (interface{}, bool) {
	s.mu.RLock()
	v, ok := s.data[key]
	s.mu.RUnlock()
	return v, ok
}

//Search is the search level of SQL buidling
type Search struct {
	WhereConditions  []map[string]interface{}
	OrConditions     []map[string]interface{}
	NotConditions    []map[string]interface{}
	HavingConditions []map[string]interface{}
	JoinConditions   []map[string]interface{}
	InitAttrs        []interface{}
	AssignAttrs      []interface{}
	Selects          map[string]interface{}
	Omits            []string
	Orders           []interface{}
	Preload          []SearchPreload
	Offset           interface{}
	Limit            interface{}
	Group            string
	TableName        string
	Raw              bool
	Unscoped         bool
	IgnoreOrderQuery bool
}

//SearchPreload is the preload search condition.
type SearchPreload struct {
	Schema     string
	Conditions []interface{}
}

//SQLCommon is the interface for SQL database interactions.
type SQLCommon interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
	Begin() (*sql.Tx, error)
}

// Expr is SQL expression
type Expr struct {
	Q    string
	Args []interface{}
}
<commit_msg>[model] Add Create hook key<commit_after>package model

import (
	"database/sql"
	"reflect"
	"strings"
	"sync"
	"time"
)

const (
	OrderByPK          = "ngorm:order_by_primary_key"
	QueryDestination   = "ngorm:query_destination"
	QueryOption        = "ngorm:query_option"
	Query              = "ngorm:query"
	HookQueryAfter     = "ngorm:query_after"
	HookQueryAfterFind = "ngorm:query_after_find"
	HookBeforeCreate   = "ngorm:before_create_hook"
	HookBeforeSave     = "ngorm:before_save_hook"
	Create             = "ngorm:create"
	BeforeCreate       = "ngorm:before_create"
	AfterCreate        = "ngorm:after_create"
	HookAfterCreate    = "ngorm:after_create"
	HookAfterSave      = "ngorm:after_save_hook"
	UpdateAttrs        = "ngorm:update_attrs"
)

//Model defines common fields that are used for defining SQL Tables. This is a
//helper that you can embed in your own struct definition.
//
// By embedding this, there is no need to define the supplied fields. For
// example.
//
//  type User struct {
//    Model
//    Name string
//  }
// Is the same as this
//  type User   struct {
//    ID        uint `gorm:"primary_key"`
//    CreatedAt time.Time
//    UpdatedAt time.Time
//    DeletedAt *time.Time `sql:"index"`
//    Name      string
//  }
type Model struct {
	ID        uint `gorm:"primary_key"`
	CreatedAt time.Time
	UpdatedAt time.Time
	DeletedAt *time.Time `sql:"index"`
}

//Struct model definition
type Struct struct {
	PrimaryFields    []*StructField
	StructFields     []*StructField
	ModelType        reflect.Type
	DefaultTableName string
}

// StructField model field's struct definition
type StructField struct {

	// DBName is the name of the field as it is seen in the database, for
	// instance a field ID can be represented in the database as id.
	DBName          string
	Name            string
	Names           []string
	IsPrimaryKey    bool
	IsNormal        bool
	IsIgnored       bool
	IsScanner       bool
	HasDefaultValue bool
	Tag             reflect.StructTag
	TagSettings     map[string]string
	Struct          reflect.StructField
	IsForeignKey    bool
	Relationship    *Relationship
}

//Clone retruns a deep copy of the StructField
func (s *StructField) Clone() *StructField {
	clone := &StructField{
		DBName:          s.DBName,
		Name:            s.Name,
		Names:           s.Names,
		IsPrimaryKey:    s.IsPrimaryKey,
		IsNormal:        s.IsNormal,
		IsIgnored:       s.IsIgnored,
		IsScanner:       s.IsScanner,
		HasDefaultValue: s.HasDefaultValue,
		Tag:             s.Tag,
		TagSettings:     map[string]string{},
		Struct:          s.Struct,
		IsForeignKey:    s.IsForeignKey,
		Relationship:    s.Relationship,
	}

	for key, value := range s.TagSettings {
		clone.TagSettings[key] = value
	}

	return clone
}

// Relationship described the relationship between models
type Relationship struct {
	Kind                         string
	PolymorphicType              string
	PolymorphicDBName            string
	PolymorphicValue             string
	ForeignFieldNames            []string
	ForeignDBNames               []string
	AssociationForeignFieldNames []string
	AssociationForeignDBNames    []string
}

//ParseTagSetting returns a map[string]string for the tags that are set.
func ParseTagSetting(tags reflect.StructTag) map[string]string {
	setting := map[string]string{}
	for _, str := range []string{tags.Get("sql"), tags.Get("gorm")} {
		tags := strings.Split(str, ";")
		for _, value := range tags {
			v := strings.Split(value, ":")
			k := strings.TrimSpace(strings.ToUpper(v[0]))
			if len(v) >= 2 {
				setting[k] = strings.Join(v[1:], ":")
			} else {
				setting[k] = k
			}
		}
	}
	return setting
}

//SafeStructsMap provide safe storage and accessing of *Struct.
type SafeStructsMap struct {
	m map[reflect.Type]*Struct
	l *sync.RWMutex
}

//Set stores value witht the given key.
func (s *SafeStructsMap) Set(key reflect.Type, value *Struct) {
	s.l.Lock()
	defer s.l.Unlock()
	s.m[key] = value
}

//Get retrieves the value stored with the gived key.
func (s *SafeStructsMap) Get(key reflect.Type) *Struct {
	s.l.RLock()
	defer s.l.RUnlock()
	return s.m[key]
}

//NewStructsMap returns a safe map for storing *Struct objects.
func NewStructsMap() *SafeStructsMap {
	return &SafeStructsMap{l: new(sync.RWMutex), m: make(map[reflect.Type]*Struct)}
}

//Scope is the scope level of SQL building.
type Scope struct {
	Value           interface{}
	SQL             string
	SQLVars         []interface{}
	InstanceID      string
	PrimaryKeyField *Field
	SkipLeft        bool
	SelectAttrs     *[]string
	mu              sync.RWMutex
	data            map[string]interface{}
}

func (s *Scope) Set(key string, value interface{}) {
	s.mu.Lock()
	s.data[key] = value
	s.mu.Unlock()
}

func (s *Scope) Get(key string) (interface{}, bool) {
	s.mu.RLock()
	v, ok := s.data[key]
	s.mu.RUnlock()
	return v, ok
}

//Search is the search level of SQL buidling
type Search struct {
	WhereConditions  []map[string]interface{}
	OrConditions     []map[string]interface{}
	NotConditions    []map[string]interface{}
	HavingConditions []map[string]interface{}
	JoinConditions   []map[string]interface{}
	InitAttrs        []interface{}
	AssignAttrs      []interface{}
	Selects          map[string]interface{}
	Omits            []string
	Orders           []interface{}
	Preload          []SearchPreload
	Offset           interface{}
	Limit            interface{}
	Group            string
	TableName        string
	Raw              bool
	Unscoped         bool
	IgnoreOrderQuery bool
}

//SearchPreload is the preload search condition.
type SearchPreload struct {
	Schema     string
	Conditions []interface{}
}

//SQLCommon is the interface for SQL database interactions.
type SQLCommon interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
	Begin() (*sql.Tx, error)
}

// Expr is SQL expression
type Expr struct {
	Q    string
	Args []interface{}
}
<|endoftext|>


<commit_before>package main

/*

Locking
=======

These methods are never called from the outside so don't follow the locking
policy in model.go.

TODO(jb): Refactor this into smaller and cleaner pieces.
TODO(jb): Increase performance by taking apparent peer bandwidth into account.

*/

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"sync"
	"time"

	"github.com/calmh/syncthing/buffers"
)

func (m *Model) pullFile(name string) error {
	m.RLock()
	var localFile = m.local[name]
	var globalFile = m.global[name]
	var nodeIDs = m.whoHas(name)
	m.RUnlock()

	if len(nodeIDs) == 0 {
		return fmt.Errorf("%s: no connected nodes with file available", name)
	}

	filename := path.Join(m.dir, name)
	sdir := path.Dir(filename)

	_, err := os.Stat(sdir)
	if err != nil && os.IsNotExist(err) {
		os.MkdirAll(sdir, 0777)
	}

	tmpFilename := tempName(filename, globalFile.Modified)
	tmpFile, err := os.Create(tmpFilename)
	if err != nil {
		return err
	}
	defer tmpFile.Close()

	contentChan := make(chan content, 32)
	var applyDone sync.WaitGroup
	applyDone.Add(1)
	go func() {
		applyContent(contentChan, tmpFile)
		applyDone.Done()
	}()

	local, remote := localFile.Blocks.To(globalFile.Blocks)
	var fetchDone sync.WaitGroup

	// One local copy routine

	fetchDone.Add(1)
	go func() {
		for _, block := range local {
			data, err := m.Request("<local>", name, block.Offset, block.Length, block.Hash)
			if err != nil {
				break
			}
			contentChan <- content{
				offset: int64(block.Offset),
				data:   data,
			}
		}
		fetchDone.Done()
	}()

	// N remote copy routines

	var remoteBlocks = blockIterator{blocks: remote}
	for i := 0; i < opts.Advanced.RequestsInFlight; i++ {
		curNode := nodeIDs[i%len(nodeIDs)]
		fetchDone.Add(1)

		go func(nodeID string) {
			for {
				block, ok := remoteBlocks.Next()
				if !ok {
					break
				}
				data, err := m.RequestGlobal(nodeID, name, block.Offset, block.Length, block.Hash)
				if err != nil {
					break
				}
				contentChan <- content{
					offset: int64(block.Offset),
					data:   data,
				}
			}
			fetchDone.Done()
		}(curNode)
	}

	fetchDone.Wait()
	close(contentChan)
	applyDone.Wait()

	err = hashCheck(tmpFilename, globalFile.Blocks)
	if err != nil {
		return fmt.Errorf("%s: %s (deleting)", path.Base(name), err.Error())
	}

	err = os.Chtimes(tmpFilename, time.Unix(globalFile.Modified, 0), time.Unix(globalFile.Modified, 0))
	if err != nil {
		return err
	}

	err = os.Rename(tmpFilename, filename)
	if err != nil {
		return err
	}

	return nil
}

func (m *Model) puller() {
	for {
		time.Sleep(time.Second)

		var ns []string
		m.RLock()
		for n := range m.need {
			ns = append(ns, n)
		}
		m.RUnlock()

		if len(ns) == 0 {
			continue
		}

		var limiter = make(chan bool, opts.Advanced.FilesInFlight)
		var allDone sync.WaitGroup

		for _, n := range ns {
			limiter <- true
			allDone.Add(1)

			go func(n string) {
				defer func() {
					allDone.Done()
					<-limiter
				}()

				f, ok := m.GlobalFile(n)
				if !ok {
					return
				}

				var err error
				if f.Flags&FlagDeleted == 0 {
					if opts.Debug.TraceFile {
						debugf("FILE: Pull %q", n)
					}
					err = m.pullFile(n)
				} else {
					if opts.Debug.TraceFile {
						debugf("FILE: Remove %q", n)
					}
					// Cheerfully ignore errors here
					_ = os.Remove(path.Join(m.dir, n))
				}
				if err == nil {
					m.UpdateLocal(f)
				} else {
					warnln(err)
				}
			}(n)
		}

		allDone.Wait()
	}
}

type content struct {
	offset int64
	data   []byte
}

func applyContent(cc <-chan content, dst io.WriterAt) error {
	var err error

	for c := range cc {
		_, err = dst.WriteAt(c.data, c.offset)
		if err != nil {
			return err
		}
		buffers.Put(c.data)
	}

	return nil
}

func hashCheck(name string, correct []Block) error {
	rf, err := os.Open(name)
	if err != nil {
		return err
	}
	defer rf.Close()

	current, err := Blocks(rf, BlockSize)
	if err != nil {
		return err
	}
	if len(current) != len(correct) {
		return errors.New("incorrect number of blocks")
	}
	for i := range current {
		if bytes.Compare(current[i].Hash, correct[i].Hash) != 0 {
			return fmt.Errorf("hash mismatch: %x != %x", current[i], correct[i])
		}
	}

	return nil
}

type blockIterator struct {
	sync.Mutex
	blocks []Block
}

func (i *blockIterator) Next() (b Block, ok bool) {
	i.Lock()
	defer i.Unlock()

	if len(i.blocks) == 0 {
		return
	}

	b, i.blocks = i.blocks[0], i.blocks[1:]
	ok = true

	return
}
<commit_msg>Close tmpfiles earlier (ref #2)<commit_after>package main

/*

Locking
=======

These methods are never called from the outside so don't follow the locking
policy in model.go.

TODO(jb): Refactor this into smaller and cleaner pieces.
TODO(jb): Increase performance by taking apparent peer bandwidth into account.

*/

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"sync"
	"time"

	"github.com/calmh/syncthing/buffers"
)

func (m *Model) pullFile(name string) error {
	m.RLock()
	var localFile = m.local[name]
	var globalFile = m.global[name]
	var nodeIDs = m.whoHas(name)
	m.RUnlock()

	if len(nodeIDs) == 0 {
		return fmt.Errorf("%s: no connected nodes with file available", name)
	}

	filename := path.Join(m.dir, name)
	sdir := path.Dir(filename)

	_, err := os.Stat(sdir)
	if err != nil && os.IsNotExist(err) {
		os.MkdirAll(sdir, 0777)
	}

	tmpFilename := tempName(filename, globalFile.Modified)
	tmpFile, err := os.Create(tmpFilename)
	if err != nil {
		return err
	}

	contentChan := make(chan content, 32)
	var applyDone sync.WaitGroup
	applyDone.Add(1)
	go func() {
		applyContent(contentChan, tmpFile)
		tmpFile.Close()
		applyDone.Done()
	}()

	local, remote := localFile.Blocks.To(globalFile.Blocks)
	var fetchDone sync.WaitGroup

	// One local copy routine

	fetchDone.Add(1)
	go func() {
		for _, block := range local {
			data, err := m.Request("<local>", name, block.Offset, block.Length, block.Hash)
			if err != nil {
				break
			}
			contentChan <- content{
				offset: int64(block.Offset),
				data:   data,
			}
		}
		fetchDone.Done()
	}()

	// N remote copy routines

	var remoteBlocks = blockIterator{blocks: remote}
	for i := 0; i < opts.Advanced.RequestsInFlight; i++ {
		curNode := nodeIDs[i%len(nodeIDs)]
		fetchDone.Add(1)

		go func(nodeID string) {
			for {
				block, ok := remoteBlocks.Next()
				if !ok {
					break
				}
				data, err := m.RequestGlobal(nodeID, name, block.Offset, block.Length, block.Hash)
				if err != nil {
					break
				}
				contentChan <- content{
					offset: int64(block.Offset),
					data:   data,
				}
			}
			fetchDone.Done()
		}(curNode)
	}

	fetchDone.Wait()
	close(contentChan)
	applyDone.Wait()

	err = hashCheck(tmpFilename, globalFile.Blocks)
	if err != nil {
		return fmt.Errorf("%s: %s (deleting)", path.Base(name), err.Error())
	}

	err = os.Chtimes(tmpFilename, time.Unix(globalFile.Modified, 0), time.Unix(globalFile.Modified, 0))
	if err != nil {
		return err
	}

	err = os.Rename(tmpFilename, filename)
	if err != nil {
		return err
	}

	return nil
}

func (m *Model) puller() {
	for {
		time.Sleep(time.Second)

		var ns []string
		m.RLock()
		for n := range m.need {
			ns = append(ns, n)
		}
		m.RUnlock()

		if len(ns) == 0 {
			continue
		}

		var limiter = make(chan bool, opts.Advanced.FilesInFlight)
		var allDone sync.WaitGroup

		for _, n := range ns {
			limiter <- true
			allDone.Add(1)

			go func(n string) {
				defer func() {
					allDone.Done()
					<-limiter
				}()

				f, ok := m.GlobalFile(n)
				if !ok {
					return
				}

				var err error
				if f.Flags&FlagDeleted == 0 {
					if opts.Debug.TraceFile {
						debugf("FILE: Pull %q", n)
					}
					err = m.pullFile(n)
				} else {
					if opts.Debug.TraceFile {
						debugf("FILE: Remove %q", n)
					}
					// Cheerfully ignore errors here
					_ = os.Remove(path.Join(m.dir, n))
				}
				if err == nil {
					m.UpdateLocal(f)
				} else {
					warnln(err)
				}
			}(n)
		}

		allDone.Wait()
	}
}

type content struct {
	offset int64
	data   []byte
}

func applyContent(cc <-chan content, dst io.WriterAt) error {
	var err error

	for c := range cc {
		_, err = dst.WriteAt(c.data, c.offset)
		buffers.Put(c.data)
		if err != nil {
			return err
		}
	}

	return nil
}

func hashCheck(name string, correct []Block) error {
	rf, err := os.Open(name)
	if err != nil {
		return err
	}
	defer rf.Close()

	current, err := Blocks(rf, BlockSize)
	if err != nil {
		return err
	}
	if len(current) != len(correct) {
		return errors.New("incorrect number of blocks")
	}
	for i := range current {
		if bytes.Compare(current[i].Hash, correct[i].Hash) != 0 {
			return fmt.Errorf("hash mismatch: %x != %x", current[i], correct[i])
		}
	}

	return nil
}

type blockIterator struct {
	sync.Mutex
	blocks []Block
}

func (i *blockIterator) Next() (b Block, ok bool) {
	i.Lock()
	defer i.Unlock()

	if len(i.blocks) == 0 {
		return
	}

	b, i.blocks = i.blocks[0], i.blocks[1:]
	ok = true

	return
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/url"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"
	"github.com/coreos/etcd-operator/test/e2e/e2eutil"
	"github.com/coreos/etcd-operator/test/e2e/framework"

	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/pkg/netutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/pkg/api/v1"
)

func TestSelfHosted(t *testing.T) {
	t.Run("create self hosted cluster from scratch", testCreateSelfHostedCluster)
	t.Run("migrate boot member to self hosted cluster", testCreateSelfHostedClusterWithBootMember)
	t.Run("backup for self hosted cluster", testSelfHostedClusterWithBackup)
	cleanupSelfHostedHostpath()
}

func testCreateSelfHostedCluster(t *testing.T) {
	if os.Getenv(framework.EnvCloudProvider) == "aws" {
		t.Skip("skipping test due to relying on PodIP reachability. TODO: Remove this skip later")
	}

	f := framework.Global
	c := e2eutil.NewCluster("test-etcd-", 3)
	c = e2eutil.ClusterWithSelfHosted(c, &spec.SelfHostedPolicy{})
	testEtcd, err := e2eutil.CreateCluster(t, f.KubeClient, f.Namespace, c)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := e2eutil.DeleteCluster(t, f.KubeClient, testEtcd); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := e2eutil.WaitUntilSizeReached(t, f.KubeClient, 3, 240*time.Second, testEtcd); err != nil {
		t.Fatalf("failed to create 3 members self-hosted etcd cluster: %v", err)
	}
}

func testCreateSelfHostedClusterWithBootMember(t *testing.T) {
	if os.Getenv(framework.EnvCloudProvider) == "aws" {
		t.Skip("skipping test due to relying on PodIP reachability. TODO: Remove this skip later")
	}

	dir, err := ioutil.TempDir("", "embed-etcd")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	host, _ := netutil.GetDefaultHost()

	rand.Seed(time.Now().UnixNano())
	port := rand.Intn(61000-32768-1) + 32768 // linux ephemeral ports

	embedCfg := embed.NewConfig()
	embedCfg.Dir = dir
	lpurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port+1))
	lcurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port))
	embedCfg.LCUrls = []url.URL{*lcurl}
	embedCfg.LPUrls = []url.URL{*lpurl}

	apurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port+1))
	acurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port))
	embedCfg.ACUrls = []url.URL{*acurl}
	embedCfg.APUrls = []url.URL{*apurl}
	embedCfg.InitialCluster = "default=" + apurl.String()

	e, err := embed.StartEtcd(embedCfg)
	if err != nil {
		t.Fatal(err)
	}
	defer e.Close()

	select {
	case <-e.Server.ReadyNotify():
	case <-time.After(30 * time.Second):
		t.Fatal("timeout: wait etcd server ready")
	}

	t.Log("etcdserver is ready")

	f := framework.Global

	c := e2eutil.NewCluster("test-etcd-", 3)
	c = e2eutil.ClusterWithSelfHosted(c, &spec.SelfHostedPolicy{
		BootMemberClientEndpoint: embedCfg.ACUrls[0].String(),
	})
	testEtcd, err := e2eutil.CreateCluster(t, f.KubeClient, f.Namespace, c)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := e2eutil.DeleteCluster(t, f.KubeClient, testEtcd); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := e2eutil.WaitUntilSizeReached(t, f.KubeClient, 3, 120*time.Second, testEtcd); err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}
}

func testSelfHostedClusterWithBackup(t *testing.T) {
	f := framework.Global

	cl := e2eutil.NewCluster("test-cluster-", 3)
	cl = e2eutil.ClusterWithBackup(cl, e2eutil.NewS3BackupPolicy(true))
	cl = e2eutil.ClusterWithSelfHosted(cl, &spec.SelfHostedPolicy{})

	testEtcd, err := e2eutil.CreateCluster(t, f.KubeClient, f.Namespace, cl)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		storageCheckerOptions := e2eutil.StorageCheckerOptions{
			S3Cli:    f.S3Cli,
			S3Bucket: f.S3Bucket,
		}
		err := e2eutil.DeleteClusterAndBackup(t, f.KubeClient, testEtcd, storageCheckerOptions)
		if err != nil {
			t.Fatal(err)
		}
	}()

	_, err = e2eutil.WaitUntilSizeReached(t, f.KubeClient, 3, 60*time.Second, testEtcd)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}
	fmt.Println("reached to 3 members cluster")
	err = e2eutil.WaitBackupPodUp(t, f.KubeClient, f.Namespace, testEtcd.Metadata.Name, 60*time.Second)
	if err != nil {
		t.Fatalf("failed to create backup pod: %v", err)
	}
	err = e2eutil.MakeBackup(f.KubeClient, f.Namespace, testEtcd.Metadata.Name)
	if err != nil {
		t.Fatalf("fail to make a latest backup: %v", err)
	}
}

func cleanupSelfHostedHostpath() {
	f := framework.Global
	nodes, err := f.KubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return
	}
	var wg sync.WaitGroup
	for i := range nodes.Items {
		wg.Add(1)
		go func(nodeName string) {
			defer wg.Done()

			name := fmt.Sprintf("cleanup-selfhosted-%s", nodeName)
			p := &v1.Pod{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
				},
				Spec: v1.PodSpec{
					NodeName:      nodeName,
					RestartPolicy: v1.RestartPolicyOnFailure,
					Containers: []v1.Container{{
						Name:  name,
						Image: "busybox",
						VolumeMounts: []v1.VolumeMount{
							// TODO: This is an assumption on etcd container volume mount.
							{Name: "etcd-data", MountPath: "/var/etcd"},
						},
						Command: []string{
							// TODO: this is an assumption on the format of etcd data dir.
							"/bin/sh", "-ec", fmt.Sprintf("rm -rf /var/etcd/%s-*", f.Namespace),
						},
					}},
					Volumes: []v1.Volume{{
						Name: "etcd-data",
						// TODO: This is an assumption on self hosted etcd volumes.
						VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/var/etcd"}},
					}},
				},
			}
			_, err := f.KubeClient.CoreV1().Pods(f.Namespace).Create(p)
			if err != nil {
				return
			}
			retryutil.Retry(5*time.Second, 5, func() (bool, error) {
				get, err := f.KubeClient.CoreV1().Pods(f.Namespace).Get(name, metav1.GetOptions{})
				if err != nil {
					return false, nil
				}
				ph := get.Status.Phase
				return ph == v1.PodSucceeded || ph == v1.PodFailed, nil
			})
			f.KubeClient.CoreV1().Pods(f.Namespace).Delete(name, nil)
		}(nodes.Items[i].Name)
	}
	wg.Wait()
}
<commit_msg>e2e: add missing AWS_TEST_ENABLED check<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/url"
	"os"
	"sync"
	"testing"
	"time"

	"github.com/coreos/etcd-operator/pkg/spec"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"
	"github.com/coreos/etcd-operator/test/e2e/e2eutil"
	"github.com/coreos/etcd-operator/test/e2e/framework"

	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/pkg/netutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/pkg/api/v1"
)

func TestSelfHosted(t *testing.T) {
	t.Run("create self hosted cluster from scratch", testCreateSelfHostedCluster)
	t.Run("migrate boot member to self hosted cluster", testCreateSelfHostedClusterWithBootMember)
	t.Run("backup for self hosted cluster", testSelfHostedClusterWithBackup)
	cleanupSelfHostedHostpath()
}

func testCreateSelfHostedCluster(t *testing.T) {
	if os.Getenv(framework.EnvCloudProvider) == "aws" {
		t.Skip("skipping test due to relying on PodIP reachability. TODO: Remove this skip later")
	}

	f := framework.Global
	c := e2eutil.NewCluster("test-etcd-", 3)
	c = e2eutil.ClusterWithSelfHosted(c, &spec.SelfHostedPolicy{})
	testEtcd, err := e2eutil.CreateCluster(t, f.KubeClient, f.Namespace, c)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := e2eutil.DeleteCluster(t, f.KubeClient, testEtcd); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := e2eutil.WaitUntilSizeReached(t, f.KubeClient, 3, 240*time.Second, testEtcd); err != nil {
		t.Fatalf("failed to create 3 members self-hosted etcd cluster: %v", err)
	}
}

func testCreateSelfHostedClusterWithBootMember(t *testing.T) {
	if os.Getenv(framework.EnvCloudProvider) == "aws" {
		t.Skip("skipping test due to relying on PodIP reachability. TODO: Remove this skip later")
	}

	dir, err := ioutil.TempDir("", "embed-etcd")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	host, _ := netutil.GetDefaultHost()

	rand.Seed(time.Now().UnixNano())
	port := rand.Intn(61000-32768-1) + 32768 // linux ephemeral ports

	embedCfg := embed.NewConfig()
	embedCfg.Dir = dir
	lpurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port+1))
	lcurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port))
	embedCfg.LCUrls = []url.URL{*lcurl}
	embedCfg.LPUrls = []url.URL{*lpurl}

	apurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port+1))
	acurl, _ := url.Parse(fmt.Sprintf("http://%s:%d", host, port))
	embedCfg.ACUrls = []url.URL{*acurl}
	embedCfg.APUrls = []url.URL{*apurl}
	embedCfg.InitialCluster = "default=" + apurl.String()

	e, err := embed.StartEtcd(embedCfg)
	if err != nil {
		t.Fatal(err)
	}
	defer e.Close()

	select {
	case <-e.Server.ReadyNotify():
	case <-time.After(30 * time.Second):
		t.Fatal("timeout: wait etcd server ready")
	}

	t.Log("etcdserver is ready")

	f := framework.Global

	c := e2eutil.NewCluster("test-etcd-", 3)
	c = e2eutil.ClusterWithSelfHosted(c, &spec.SelfHostedPolicy{
		BootMemberClientEndpoint: embedCfg.ACUrls[0].String(),
	})
	testEtcd, err := e2eutil.CreateCluster(t, f.KubeClient, f.Namespace, c)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := e2eutil.DeleteCluster(t, f.KubeClient, testEtcd); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := e2eutil.WaitUntilSizeReached(t, f.KubeClient, 3, 120*time.Second, testEtcd); err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}
}

func testSelfHostedClusterWithBackup(t *testing.T) {
	if os.Getenv("AWS_TEST_ENABLED") != "true" {
		t.Skip("skipping test since AWS_TEST_ENABLED is not set.")
	}

	f := framework.Global

	cl := e2eutil.NewCluster("test-cluster-", 3)
	cl = e2eutil.ClusterWithBackup(cl, e2eutil.NewS3BackupPolicy(true))
	cl = e2eutil.ClusterWithSelfHosted(cl, &spec.SelfHostedPolicy{})

	testEtcd, err := e2eutil.CreateCluster(t, f.KubeClient, f.Namespace, cl)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		storageCheckerOptions := e2eutil.StorageCheckerOptions{
			S3Cli:    f.S3Cli,
			S3Bucket: f.S3Bucket,
		}
		err := e2eutil.DeleteClusterAndBackup(t, f.KubeClient, testEtcd, storageCheckerOptions)
		if err != nil {
			t.Fatal(err)
		}
	}()

	_, err = e2eutil.WaitUntilSizeReached(t, f.KubeClient, 3, 60*time.Second, testEtcd)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}
	fmt.Println("reached to 3 members cluster")
	err = e2eutil.WaitBackupPodUp(t, f.KubeClient, f.Namespace, testEtcd.Metadata.Name, 60*time.Second)
	if err != nil {
		t.Fatalf("failed to create backup pod: %v", err)
	}
	err = e2eutil.MakeBackup(f.KubeClient, f.Namespace, testEtcd.Metadata.Name)
	if err != nil {
		t.Fatalf("fail to make a latest backup: %v", err)
	}
}

func cleanupSelfHostedHostpath() {
	f := framework.Global
	nodes, err := f.KubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return
	}
	var wg sync.WaitGroup
	for i := range nodes.Items {
		wg.Add(1)
		go func(nodeName string) {
			defer wg.Done()

			name := fmt.Sprintf("cleanup-selfhosted-%s", nodeName)
			p := &v1.Pod{
				ObjectMeta: metav1.ObjectMeta{
					Name: name,
				},
				Spec: v1.PodSpec{
					NodeName:      nodeName,
					RestartPolicy: v1.RestartPolicyOnFailure,
					Containers: []v1.Container{{
						Name:  name,
						Image: "busybox",
						VolumeMounts: []v1.VolumeMount{
							// TODO: This is an assumption on etcd container volume mount.
							{Name: "etcd-data", MountPath: "/var/etcd"},
						},
						Command: []string{
							// TODO: this is an assumption on the format of etcd data dir.
							"/bin/sh", "-ec", fmt.Sprintf("rm -rf /var/etcd/%s-*", f.Namespace),
						},
					}},
					Volumes: []v1.Volume{{
						Name: "etcd-data",
						// TODO: This is an assumption on self hosted etcd volumes.
						VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/var/etcd"}},
					}},
				},
			}
			_, err := f.KubeClient.CoreV1().Pods(f.Namespace).Create(p)
			if err != nil {
				return
			}
			retryutil.Retry(5*time.Second, 5, func() (bool, error) {
				get, err := f.KubeClient.CoreV1().Pods(f.Namespace).Get(name, metav1.GetOptions{})
				if err != nil {
					return false, nil
				}
				ph := get.Status.Phase
				return ph == v1.PodSucceeded || ph == v1.PodFailed, nil
			})
			f.KubeClient.CoreV1().Pods(f.Namespace).Delete(name, nil)
		}(nodes.Items[i].Name)
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>package models

type Album struct {
	id       int    `json:"id" db:"id"`
	title    string  `json:"title" db:"title"`
	artistId uint    `json:"artistId" db:"artist_id"`
}
<commit_msg>albums<commit_after>package models

type Album struct {
	Id       int    `json:"id" db:"id"`
	Title    string  `json:"title" db:"title"`
	ArtistId uint    `json:"artistId" db:"artist_id"`
}
<|endoftext|>