<commit_before>package cluster

import (
	"context"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

var listeners = map[string]*lxd.EventListener{}
var listenersLock sync.Mutex

// Events starts a task that continuously monitors the list of cluster nodes and
// maintains a pool of websocket connections against all of them, in order to
// get notified about events.
//
// Whenever an event is received the given callback is invoked.
func Events(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, f func(int64, api.Event)) (task.Func, task.Schedule) {
	// Update our pool of event listeners. Since database queries are
	// blocking, we spawn the actual logic in a goroutine, to abort
	// immediately when we receive the stop signal.
	update := func(ctx context.Context) {
		ch := make(chan struct{})
		go func() {
			eventsUpdateListeners(endpoints, cluster, serverCert, nil, f)
			ch <- struct{}{}
		}()
		select {
		case <-ch:
		case <-ctx.Done():
		}
	}

	schedule := task.Every(time.Second)

	return update, schedule
}

func eventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
			}
		}
	}

	address := endpoints.NetworkAddress()

	addresses := make([]string, len(members))
	for i, member := range members {
		addresses[i] = member.Address

		if member.Address == address {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// Don't bother trying to connect to offline nodes, or to ourselves.
		if !member.Online {
			if ok {
				listener.Disconnect()
			}

			listenersLock.Unlock()
			continue
		}

		// The node has already a listener associated to it.
		if ok {
			// Double check that the listener is still
			// connected. If it is, just move on, other
			// we'll try to connect again.
			if listeners[member.Address].IsActive() {
				listenersLock.Unlock()
				continue
			}

			delete(listeners, member.Address)
		}
		listenersLock.Unlock()

		listener, err := eventsConnect(member.Address, endpoints.NetworkCert(), serverCert())
		if err != nil {
			logger.Warn("Failed to get events from member", log.Ctx{"address": member.Address, "err": err})
			continue
		}
		logger.Debug("Listening for events on member", log.Ctx{"address": member.Address})
		listener.AddHandler(nil, func(event api.Event) { f(member.ID, event) })

		listenersLock.Lock()
		listeners[member.Address] = listener
		listenersLock.Unlock()
	}

	listenersLock.Lock()
	for address, listener := range listeners {
		if !shared.StringInSlice(address, addresses) {
			listener.Disconnect()
			delete(listeners, address)
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	// Set the project to the special wildcard in order to get notified
	// about all events across all projects.
	client = client.UseProject("*")

	return client.GetEvents()
}
<commit_msg>lxd/cluster/events: Updates eventsUpdateListeners member handling<commit_after>package cluster

import (
	"context"
	"sync"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

var listeners = map[string]*lxd.EventListener{}
var listenersLock sync.Mutex

// Events starts a task that continuously monitors the list of cluster nodes and
// maintains a pool of websocket connections against all of them, in order to
// get notified about events.
//
// Whenever an event is received the given callback is invoked.
func Events(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, f func(int64, api.Event)) (task.Func, task.Schedule) {
	// Update our pool of event listeners. Since database queries are
	// blocking, we spawn the actual logic in a goroutine, to abort
	// immediately when we receive the stop signal.
	update := func(ctx context.Context) {
		ch := make(chan struct{})
		go func() {
			eventsUpdateListeners(endpoints, cluster, serverCert, nil, f)
			ch <- struct{}{}
		}()
		select {
		case <-ch:
		case <-ctx.Done():
		}
	}

	schedule := task.Every(time.Second)

	return update, schedule
}

func eventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, members map[int64]APIHeartbeatMember, f func(int64, api.Event)) {
	// If no heartbeat members provided, populate from global database.
	if members == nil {
		var dbMembers []db.NodeInfo
		var offlineThreshold time.Duration

		err := cluster.Transaction(func(tx *db.ClusterTx) error {
			var err error

			dbMembers, err = tx.GetNodes()
			if err != nil {
				return err
			}

			offlineThreshold, err = tx.GetNodeOfflineThreshold()
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			logger.Warn("Failed to get current cluster members", log.Ctx{"err": err})
			return
		}

		members = make(map[int64]APIHeartbeatMember, len(dbMembers))
		for _, dbMember := range dbMembers {
			members[dbMember.ID] = APIHeartbeatMember{
				ID:            dbMember.ID,
				Name:          dbMember.Name,
				Address:       dbMember.Address,
				LastHeartbeat: dbMember.Heartbeat,
				Online:        !dbMember.IsOffline(offlineThreshold),
			}
		}
	}

	networkAddress := endpoints.NetworkAddress()

	keepListeners := make(map[string]struct{})
	wg := sync.WaitGroup{}
	for _, member := range members {
		// Don't bother trying to connect to ourselves or offline members.
		if member.Address == networkAddress || !member.Online {
			continue
		}

		listenersLock.Lock()
		listener, ok := listeners[member.Address]

		// If the member already has a listener associated to it, check that the listener is still active.
		// If it is, just move on to next member, but if not then we'll try to connect again.
		if ok {
			if listeners[member.Address].IsActive() {
				keepListeners[member.Address] = struct{}{} // Add to current listeners list.
				listenersLock.Unlock()
				continue
			}

			listener.Disconnect()
			delete(listeners, member.Address)
			logger.Info("Removed inactive member event listener", log.Ctx{"local": networkAddress, "remote": member.Address})
		}
		listenersLock.Unlock()

		keepListeners[member.Address] = struct{}{} // Add to current listeners list.

		// Connect to remote concurrently and add to active listeners if successful.
		wg.Add(1)
		go func(m APIHeartbeatMember) {
			defer wg.Done()
			listener, err := eventsConnect(m.Address, endpoints.NetworkCert(), serverCert())
			if err != nil {
				logger.Warn("Failed adding member event listener", log.Ctx{"local": networkAddress, "remote": m.Address, "err": err})
				return
			}

			listener.AddHandler(nil, func(event api.Event) { f(m.ID, event) })

			listenersLock.Lock()
			listeners[m.Address] = listener
			logger.Info("Added member event listener", log.Ctx{"local": networkAddress, "remote": m.Address})
			listenersLock.Unlock()
		}(member)
	}

	wg.Wait()

	// Disconnect and delete any out of date listeners.
	listenersLock.Lock()
	for address, listener := range listeners {
		if _, found := keepListeners[address]; !found {
			listener.Disconnect()
			delete(listeners, address)
			logger.Info("Removed old member event listener", log.Ctx{"local": networkAddress, "remote": address})
		}
	}
	listenersLock.Unlock()
}

// Establish a client connection to get events from the given node.
func eventsConnect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo) (*lxd.EventListener, error) {
	client, err := Connect(address, networkCert, serverCert, nil, true)
	if err != nil {
		return nil, err
	}

	// Set the project to the special wildcard in order to get notified
	// about all events across all projects.
	client = client.UseProject("*")

	return client.GetEvents()
}
<|endoftext|>


<commit_before>package service

import (
	"database/sql"
	"strings"
	"time"

	"github.com/builderscon/octav/octav/db"
	"github.com/builderscon/octav/octav/model"
	"github.com/builderscon/octav/octav/tools"
)

func (v *Conference) populateRowForCreate(vdb *db.Conference, payload model.CreateConferenceRequest) error {
	vdb.EID = tools.UUID()
	vdb.Slug = payload.Slug
	vdb.Title = payload.Title

	if payload.SubTitle.Valid() {
		vdb.SubTitle.Valid = true
		vdb.SubTitle.String = payload.SubTitle.String
	}
	return nil
}

func (v *Conference) populateRowForUpdate(vdb *db.Conference, payload model.UpdateConferenceRequest) error {
	if payload.Slug.Valid() {
		vdb.Slug = payload.Slug.String
	}

	if payload.Slug.Valid() {
		vdb.Title = payload.Title.String
	}

	if payload.SubTitle.Valid() {
		vdb.SubTitle.Valid = true
		vdb.SubTitle.String = payload.SubTitle.String
	}
	return nil
}

func (v *Conference) AddAdministrator(tx *db.Tx, cid, uid string) error {
	c := db.ConferenceAdministrator{
		ConferenceID: cid,
		UserID:       uid,
	}
	return c.Create(tx)
}

const datefmt = `2006-01-02`

func (v *Conference) LoadByRange(tx *db.Tx, vdbl *db.ConferenceList, since, lang, rangeStart, rangeEnd string, limit int) error {
	var rs time.Time
	var re time.Time
	var err error

	if rangeStart != "" {
		rs, err = time.Parse(datefmt, rangeStart)
		if err != nil {
			return err
		}
	}

	if rangeEnd != "" {
		re, err = time.Parse(datefmt, rangeEnd)
		if err != nil {
			return err
		}
	}

	if err := vdbl.LoadByRange(tx, since, rs, re, limit); err != nil {
		return err
	}

	return nil
}

func (v *Conference) AddDates(tx *db.Tx, cid string, dates ...model.ConferenceDate) error {
	for _, date := range dates {
		cd := db.ConferenceDate{
			ConferenceID: cid,
			Date:         date.Date.String(),
			Open:         sql.NullString{String: date.Open.String(), Valid: true},
			Close:        sql.NullString{String: date.Close.String(), Valid: true},
		}
		if err := cd.Create(tx, db.WithInsertIgnore(true)); err != nil {
			return err
		}
	}

	return nil
}

func (v *Conference) DeleteDates(tx *db.Tx, cid string, dates ...model.Date) error {
	vdb := db.ConferenceDate{}
	sdatelist := make([]string, len(dates))
	for i, dt := range dates {
		sdatelist[i] = dt.String()
	}
	return vdb.DeleteDates(tx, cid, sdatelist...)
}

func (v *Conference) LoadDates(tx *db.Tx, cdl *model.ConferenceDateList, cid string) error {
	vdbl := db.ConferenceDateList{}
	if err := vdbl.LoadByConferenceID(tx, cid); err != nil {
		return err
	}

	res := make(model.ConferenceDateList, len(vdbl))
	for i, vdb := range vdbl {
		dt := vdb.Date
		if i := strings.IndexByte(dt, 'T'); i > -1 { // Cheat. Loading from DB contains time....!!!!
			dt = dt[:i]
		}
		if err := res[i].Date.Parse(dt); err != nil {
			return err
		}

		if vdb.Open.Valid {
			t := vdb.Open.String
			if len(t) > 5 {
				t = t[:5]
			}
			if err := res[i].Open.Parse(t); err != nil {
				return err
			}
		}

		if vdb.Close.Valid {
			t := vdb.Close.String
			if len(t) > 5 {
				t = t[:5]
			}
			if err := res[i].Close.Parse(t); err != nil {
				return err
			}
		}
	}
	*cdl = res
	return nil
}

func (v *Conference) AddAdmin(tx *db.Tx, cid, uid string) error {
	cd := db.ConferenceAdministrator{
		ConferenceID: cid,
		UserID: uid,
	}
	if err := cd.Create(tx, db.WithInsertIgnore(true)); err != nil {
		return err
	}

	return nil
}

func (v *Conference) DeleteAdmin(tx *db.Tx, cid, uid string) error {
	return db.DeleteConferenceAdministrator(tx, cid, uid)
}

func (v *Conference) LoadAdmins(tx *db.Tx, cdl *model.UserList, cid string) error {
	var vdbl db.UserList
	if err := db.LoadConferenceAdministrators(tx, &vdbl, cid); err != nil {
		return err
	}

	res := make(model.UserList, len(vdbl))
	for i, vdb := range vdbl {
		var u model.User
		if err := u.FromRow(vdb); err != nil {
			return err
		}
		res[i] = u
	}
	*cdl = res
	return nil
}
<commit_msg>Arrrrgh, bad case of cut+paste<commit_after>package service

import (
	"database/sql"
	"strings"
	"time"

	"github.com/builderscon/octav/octav/db"
	"github.com/builderscon/octav/octav/model"
	"github.com/builderscon/octav/octav/tools"
)

func (v *Conference) populateRowForCreate(vdb *db.Conference, payload model.CreateConferenceRequest) error {
	vdb.EID = tools.UUID()
	vdb.Slug = payload.Slug
	vdb.Title = payload.Title

	if payload.SubTitle.Valid() {
		vdb.SubTitle.Valid = true
		vdb.SubTitle.String = payload.SubTitle.String
	}
	return nil
}

func (v *Conference) populateRowForUpdate(vdb *db.Conference, payload model.UpdateConferenceRequest) error {
	if payload.Slug.Valid() {
		vdb.Slug = payload.Slug.String
	}

	if payload.Title.Valid() {
		vdb.Title = payload.Title.String
	}

	if payload.SubTitle.Valid() {
		vdb.SubTitle.Valid = true
		vdb.SubTitle.String = payload.SubTitle.String
	}
	return nil
}

func (v *Conference) AddAdministrator(tx *db.Tx, cid, uid string) error {
	c := db.ConferenceAdministrator{
		ConferenceID: cid,
		UserID:       uid,
	}
	return c.Create(tx)
}

const datefmt = `2006-01-02`

func (v *Conference) LoadByRange(tx *db.Tx, vdbl *db.ConferenceList, since, lang, rangeStart, rangeEnd string, limit int) error {
	var rs time.Time
	var re time.Time
	var err error

	if rangeStart != "" {
		rs, err = time.Parse(datefmt, rangeStart)
		if err != nil {
			return err
		}
	}

	if rangeEnd != "" {
		re, err = time.Parse(datefmt, rangeEnd)
		if err != nil {
			return err
		}
	}

	if err := vdbl.LoadByRange(tx, since, rs, re, limit); err != nil {
		return err
	}

	return nil
}

func (v *Conference) AddDates(tx *db.Tx, cid string, dates ...model.ConferenceDate) error {
	for _, date := range dates {
		cd := db.ConferenceDate{
			ConferenceID: cid,
			Date:         date.Date.String(),
			Open:         sql.NullString{String: date.Open.String(), Valid: true},
			Close:        sql.NullString{String: date.Close.String(), Valid: true},
		}
		if err := cd.Create(tx, db.WithInsertIgnore(true)); err != nil {
			return err
		}
	}

	return nil
}

func (v *Conference) DeleteDates(tx *db.Tx, cid string, dates ...model.Date) error {
	vdb := db.ConferenceDate{}
	sdatelist := make([]string, len(dates))
	for i, dt := range dates {
		sdatelist[i] = dt.String()
	}
	return vdb.DeleteDates(tx, cid, sdatelist...)
}

func (v *Conference) LoadDates(tx *db.Tx, cdl *model.ConferenceDateList, cid string) error {
	vdbl := db.ConferenceDateList{}
	if err := vdbl.LoadByConferenceID(tx, cid); err != nil {
		return err
	}

	res := make(model.ConferenceDateList, len(vdbl))
	for i, vdb := range vdbl {
		dt := vdb.Date
		if i := strings.IndexByte(dt, 'T'); i > -1 { // Cheat. Loading from DB contains time....!!!!
			dt = dt[:i]
		}
		if err := res[i].Date.Parse(dt); err != nil {
			return err
		}

		if vdb.Open.Valid {
			t := vdb.Open.String
			if len(t) > 5 {
				t = t[:5]
			}
			if err := res[i].Open.Parse(t); err != nil {
				return err
			}
		}

		if vdb.Close.Valid {
			t := vdb.Close.String
			if len(t) > 5 {
				t = t[:5]
			}
			if err := res[i].Close.Parse(t); err != nil {
				return err
			}
		}
	}
	*cdl = res
	return nil
}

func (v *Conference) AddAdmin(tx *db.Tx, cid, uid string) error {
	cd := db.ConferenceAdministrator{
		ConferenceID: cid,
		UserID: uid,
	}
	if err := cd.Create(tx, db.WithInsertIgnore(true)); err != nil {
		return err
	}

	return nil
}

func (v *Conference) DeleteAdmin(tx *db.Tx, cid, uid string) error {
	return db.DeleteConferenceAdministrator(tx, cid, uid)
}

func (v *Conference) LoadAdmins(tx *db.Tx, cdl *model.UserList, cid string) error {
	var vdbl db.UserList
	if err := db.LoadConferenceAdministrators(tx, &vdbl, cid); err != nil {
		return err
	}

	res := make(model.UserList, len(vdbl))
	for i, vdb := range vdbl {
		var u model.User
		if err := u.FromRow(vdb); err != nil {
			return err
		}
		res[i] = u
	}
	*cdl = res
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/valyala/fasthttp"
	"net/url"
)

var bytesSlash = []byte("/") // heap optimization

// HTTPClient is a reusable HTTP Client.
type HTTPClient struct {
	client     fasthttp.Client
	Host       []byte
	HostString string
	debug      int
}

// HTTPClientDoOptions wraps options uses when calling `Do`.
type HTTPClientDoOptions struct {
	Debug                int
	PrettyPrintResponses bool
}

// NewHTTPClient creates a new HTTPClient.
func NewHTTPClient(host string, debug int) *HTTPClient {
	return &HTTPClient{
		client: fasthttp.Client{
			Name: "query_benchmarker",
		},
		Host:       []byte(host),
		HostString: host,
		debug:      debug,
	}
}

// Do performs the action specified by the given Query. It uses fasthttp, and
// tries to minimize heap allocations.
func (w *HTTPClient) Do(q *Query, opts *HTTPClientDoOptions) (lag float64, err error) {
	// populate uri from the reusable byte slice:
	uri := make([]byte, 0, 100)
	uri = append(uri, w.Host...)
	uri = append(uri, bytesSlash...)
	uri = append(uri, q.Path...)

	// populate a request with data from the Query:
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethodBytes(q.Method)
	req.Header.SetRequestURIBytes(uri)
	req.SetBody(q.Body)
	if opts.Debug > 0 {
		values, _ := url.ParseQuery(string(q.Path))
		fmt.Printf("debug: query - %s\n", values)
	}
	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	start := time.Now()
	err = w.client.Do(req, resp)
	lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds

	// Check that the status code was 200 OK:
	if err == nil {
		sc := resp.StatusCode()
		if sc != fasthttp.StatusOK {
			err = fmt.Errorf("Invalid write response (status %d): %s", sc, resp.Body())
			return
		}
	}

	if opts != nil {
		// Print debug messages, if applicable:
		switch opts.Debug {
		case 1:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag)
		case 2:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
		case 3:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
		case 4:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
			fmt.Fprintf(os.Stderr, "debug:   response: %s\n", string(resp.Body()))
		default:
		}

		// Pretty print JSON responses, if applicable:
		if opts.PrettyPrintResponses {
			// InfluxQL responses are in JSON and can be pretty-printed here.
			// Flux responses are just simple CSV.

			prefix := fmt.Sprintf("ID %d: ", q.ID)
			if json.Valid(resp.Body()) {
				var pretty bytes.Buffer
				err = json.Indent(&pretty, resp.Body(), prefix, "  ")
				if err != nil {
					return
				}

				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, pretty)
				if err != nil {
					return
				}
			} else {
				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, resp.Body())
				if err != nil {
					return
				}
			}
		}
	}

	return lag, err
}
<commit_msg>Better error debug output<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"time"

	"github.com/valyala/fasthttp"
)

var bytesSlash = []byte("/") // heap optimization
var byteZero = []byte{0}     // heap optimization

// HTTPClient is a reusable HTTP Client.
type HTTPClient struct {
	client     fasthttp.Client
	Host       []byte
	HostString string
	debug      int
}

// HTTPClientDoOptions wraps options uses when calling `Do`.
type HTTPClientDoOptions struct {
	Debug                int
	PrettyPrintResponses bool
}

// NewHTTPClient creates a new HTTPClient.
func NewHTTPClient(host string, debug int) *HTTPClient {
	return &HTTPClient{
		client: fasthttp.Client{
			Name: "query_benchmarker",
		},
		Host:       []byte(host),
		HostString: host,
		debug:      debug,
	}
}

// Do performs the action specified by the given Query. It uses fasthttp, and
// tries to minimize heap allocations.
func (w *HTTPClient) Do(q *Query, opts *HTTPClientDoOptions) (lag float64, err error) {
	// populate uri from the reusable byte slice:
	uri := make([]byte, 0, 100)
	uri = append(uri, w.Host...)
	uri = append(uri, bytesSlash...)
	uri = append(uri, q.Path...)

	// populate a request with data from the Query:
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethodBytes(q.Method)
	req.Header.SetRequestURIBytes(uri)
	req.SetBody(q.Body)
	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	start := time.Now()
	err = w.client.Do(req, resp)
	lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds

	if err != nil || resp.StatusCode() != fasthttp.StatusOK {
		values, _ := url.ParseQuery(string(uri))
		fmt.Printf("debug: url: %s, path %s, parsed url - %s\n", string(uri), q.Path, values)
	}

	// Check that the status code was 200 OK:
	if err == nil {
		sc := resp.StatusCode()
		if sc != fasthttp.StatusOK {
			err = fmt.Errorf("Invalid write response (status %d): %s", sc, resp.Body())
			return
		}
	}

	if opts != nil {
		// Print debug messages, if applicable:
		switch opts.Debug {
		case 1:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag)
		case 2:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
		case 3:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
		case 4:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
			fmt.Fprintf(os.Stderr, "debug:   response: %s\n", string(resp.Body()))
		default:
		}

		// Pretty print JSON responses, if applicable:
		if opts.PrettyPrintResponses {
			// InfluxQL responses are in JSON and can be pretty-printed here.
			// Flux responses are just simple CSV.

			prefix := fmt.Sprintf("ID %d: ", q.ID)
			if json.Valid(resp.Body()) {
				var pretty bytes.Buffer
				err = json.Indent(&pretty, resp.Body(), prefix, "  ")
				if err != nil {
					return
				}

				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, pretty)
				if err != nil {
					return
				}
			} else {
				_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, resp.Body())
				if err != nil {
					return
				}
			}
		}
	}

	return lag, err
}
<|endoftext|>


<commit_before>// This file was generated by counterfeiter
package eventfakes

import (
	"sync"

	"github.com/9corp/9volt/event"
	log "github.com/Sirupsen/logrus"
)

type FakeIClient struct {
	AddStub        func(string, string) error
	addMutex       sync.RWMutex
	addArgsForCall []struct {
		arg1 string
		arg2 string
	}
	addReturns struct {
		result1 error
	}
	addReturnsOnCall map[int]struct {
		result1 error
	}
	AddWithErrorLogStub        func(string, string, log.FieldLogger, log.Fields) error
	addWithErrorLogMutex       sync.RWMutex
	addWithErrorLogArgsForCall []struct {
		arg1 string
		arg2 string
		arg3 log.FieldLogger
		arg4 log.Fields
	}
	addWithErrorLogReturns struct {
		result1 error
	}
	addWithErrorLogReturnsOnCall map[int]struct {
		result1 error
	}
	invocations      map[string][][]interface{}
	invocationsMutex sync.RWMutex
}

func (fake *FakeIClient) Add(arg1 string, arg2 string) error {
	fake.addMutex.Lock()
	ret, specificReturn := fake.addReturnsOnCall[len(fake.addArgsForCall)]
	fake.addArgsForCall = append(fake.addArgsForCall, struct {
		arg1 string
		arg2 string
	}{arg1, arg2})
	fake.recordInvocation("Add", []interface{}{arg1, arg2})
	fake.addMutex.Unlock()
	if fake.AddStub != nil {
		return fake.AddStub(arg1, arg2)
	}
	if specificReturn {
		return ret.result1
	}
	return fake.addReturns.result1
}

func (fake *FakeIClient) AddCallCount() int {
	fake.addMutex.RLock()
	defer fake.addMutex.RUnlock()
	return len(fake.addArgsForCall)
}

func (fake *FakeIClient) AddArgsForCall(i int) (string, string) {
	fake.addMutex.RLock()
	defer fake.addMutex.RUnlock()
	return fake.addArgsForCall[i].arg1, fake.addArgsForCall[i].arg2
}

func (fake *FakeIClient) AddReturns(result1 error) {
	fake.AddStub = nil
	fake.addReturns = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddReturnsOnCall(i int, result1 error) {
	fake.AddStub = nil
	if fake.addReturnsOnCall == nil {
		fake.addReturnsOnCall = make(map[int]struct {
			result1 error
		})
	}
	fake.addReturnsOnCall[i] = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddWithErrorLog(arg1 string, arg2 string, arg3 log.FieldLogger, arg4 log.Fields) error {
	fake.addWithErrorLogMutex.Lock()
	ret, specificReturn := fake.addWithErrorLogReturnsOnCall[len(fake.addWithErrorLogArgsForCall)]
	fake.addWithErrorLogArgsForCall = append(fake.addWithErrorLogArgsForCall, struct {
		arg1 string
		arg2 string
		arg3 log.FieldLogger
		arg4 log.Fields
	}{arg1, arg2, arg3, arg4})
	fake.recordInvocation("AddWithErrorLog", []interface{}{arg1, arg2, arg3, arg4})
	fake.addWithErrorLogMutex.Unlock()
	if fake.AddWithErrorLogStub != nil {
		return fake.AddWithErrorLogStub(arg1, arg2, arg3, arg4)
	}
	if specificReturn {
		return ret.result1
	}
	return fake.addWithErrorLogReturns.result1
}

func (fake *FakeIClient) AddWithErrorLogCallCount() int {
	fake.addWithErrorLogMutex.RLock()
	defer fake.addWithErrorLogMutex.RUnlock()
	return len(fake.addWithErrorLogArgsForCall)
}

func (fake *FakeIClient) AddWithErrorLogArgsForCall(i int) (string, string, log.FieldLogger, log.Fields) {
	fake.addWithErrorLogMutex.RLock()
	defer fake.addWithErrorLogMutex.RUnlock()
	return fake.addWithErrorLogArgsForCall[i].arg1, fake.addWithErrorLogArgsForCall[i].arg2, fake.addWithErrorLogArgsForCall[i].arg3, fake.addWithErrorLogArgsForCall[i].arg4
}

func (fake *FakeIClient) AddWithErrorLogReturns(result1 error) {
	fake.AddWithErrorLogStub = nil
	fake.addWithErrorLogReturns = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddWithErrorLogReturnsOnCall(i int, result1 error) {
	fake.AddWithErrorLogStub = nil
	if fake.addWithErrorLogReturnsOnCall == nil {
		fake.addWithErrorLogReturnsOnCall = make(map[int]struct {
			result1 error
		})
	}
	fake.addWithErrorLogReturnsOnCall[i] = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) Invocations() map[string][][]interface{} {
	fake.invocationsMutex.RLock()
	defer fake.invocationsMutex.RUnlock()
	fake.addMutex.RLock()
	defer fake.addMutex.RUnlock()
	fake.addWithErrorLogMutex.RLock()
	defer fake.addWithErrorLogMutex.RUnlock()
	return fake.invocations
}

func (fake *FakeIClient) recordInvocation(key string, args []interface{}) {
	fake.invocationsMutex.Lock()
	defer fake.invocationsMutex.Unlock()
	if fake.invocations == nil {
		fake.invocations = map[string][][]interface{}{}
	}
	if fake.invocations[key] == nil {
		fake.invocations[key] = [][]interface{}{}
	}
	fake.invocations[key] = append(fake.invocations[key], args)
}

var _ event.IClient = new(FakeIClient)
<commit_msg>updated event queue fake<commit_after>// This file was generated by counterfeiter
package eventfakes

import (
	"sync"

	"github.com/9corp/9volt/event"
	log "github.com/Sirupsen/logrus"
)

type FakeIClient struct {
	AddStub        func(string, string) error
	addMutex       sync.RWMutex
	addArgsForCall []struct {
		arg1 string
		arg2 string
	}
	addReturns struct {
		result1 error
	}
	addReturnsOnCall map[int]struct {
		result1 error
	}
	AddWithErrorLogStub        func(string, log.FieldLogger, log.Fields) error
	addWithErrorLogMutex       sync.RWMutex
	addWithErrorLogArgsForCall []struct {
		arg1 string
		arg2 log.FieldLogger
		arg3 log.Fields
	}
	addWithErrorLogReturns struct {
		result1 error
	}
	addWithErrorLogReturnsOnCall map[int]struct {
		result1 error
	}
	AddWithLogStub        func(string, string, log.FieldLogger, log.Fields) error
	addWithLogMutex       sync.RWMutex
	addWithLogArgsForCall []struct {
		arg1 string
		arg2 string
		arg3 log.FieldLogger
		arg4 log.Fields
	}
	addWithLogReturns struct {
		result1 error
	}
	addWithLogReturnsOnCall map[int]struct {
		result1 error
	}
	invocations      map[string][][]interface{}
	invocationsMutex sync.RWMutex
}

func (fake *FakeIClient) Add(arg1 string, arg2 string) error {
	fake.addMutex.Lock()
	ret, specificReturn := fake.addReturnsOnCall[len(fake.addArgsForCall)]
	fake.addArgsForCall = append(fake.addArgsForCall, struct {
		arg1 string
		arg2 string
	}{arg1, arg2})
	fake.recordInvocation("Add", []interface{}{arg1, arg2})
	fake.addMutex.Unlock()
	if fake.AddStub != nil {
		return fake.AddStub(arg1, arg2)
	}
	if specificReturn {
		return ret.result1
	}
	return fake.addReturns.result1
}

func (fake *FakeIClient) AddCallCount() int {
	fake.addMutex.RLock()
	defer fake.addMutex.RUnlock()
	return len(fake.addArgsForCall)
}

func (fake *FakeIClient) AddArgsForCall(i int) (string, string) {
	fake.addMutex.RLock()
	defer fake.addMutex.RUnlock()
	return fake.addArgsForCall[i].arg1, fake.addArgsForCall[i].arg2
}

func (fake *FakeIClient) AddReturns(result1 error) {
	fake.AddStub = nil
	fake.addReturns = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddReturnsOnCall(i int, result1 error) {
	fake.AddStub = nil
	if fake.addReturnsOnCall == nil {
		fake.addReturnsOnCall = make(map[int]struct {
			result1 error
		})
	}
	fake.addReturnsOnCall[i] = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddWithErrorLog(arg1 string, arg2 log.FieldLogger, arg3 log.Fields) error {
	fake.addWithErrorLogMutex.Lock()
	ret, specificReturn := fake.addWithErrorLogReturnsOnCall[len(fake.addWithErrorLogArgsForCall)]
	fake.addWithErrorLogArgsForCall = append(fake.addWithErrorLogArgsForCall, struct {
		arg1 string
		arg2 log.FieldLogger
		arg3 log.Fields
	}{arg1, arg2, arg3})
	fake.recordInvocation("AddWithErrorLog", []interface{}{arg1, arg2, arg3})
	fake.addWithErrorLogMutex.Unlock()
	if fake.AddWithErrorLogStub != nil {
		return fake.AddWithErrorLogStub(arg1, arg2, arg3)
	}
	if specificReturn {
		return ret.result1
	}
	return fake.addWithErrorLogReturns.result1
}

func (fake *FakeIClient) AddWithErrorLogCallCount() int {
	fake.addWithErrorLogMutex.RLock()
	defer fake.addWithErrorLogMutex.RUnlock()
	return len(fake.addWithErrorLogArgsForCall)
}

func (fake *FakeIClient) AddWithErrorLogArgsForCall(i int) (string, log.FieldLogger, log.Fields) {
	fake.addWithErrorLogMutex.RLock()
	defer fake.addWithErrorLogMutex.RUnlock()
	return fake.addWithErrorLogArgsForCall[i].arg1, fake.addWithErrorLogArgsForCall[i].arg2, fake.addWithErrorLogArgsForCall[i].arg3
}

func (fake *FakeIClient) AddWithErrorLogReturns(result1 error) {
	fake.AddWithErrorLogStub = nil
	fake.addWithErrorLogReturns = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddWithErrorLogReturnsOnCall(i int, result1 error) {
	fake.AddWithErrorLogStub = nil
	if fake.addWithErrorLogReturnsOnCall == nil {
		fake.addWithErrorLogReturnsOnCall = make(map[int]struct {
			result1 error
		})
	}
	fake.addWithErrorLogReturnsOnCall[i] = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddWithLog(arg1 string, arg2 string, arg3 log.FieldLogger, arg4 log.Fields) error {
	fake.addWithLogMutex.Lock()
	ret, specificReturn := fake.addWithLogReturnsOnCall[len(fake.addWithLogArgsForCall)]
	fake.addWithLogArgsForCall = append(fake.addWithLogArgsForCall, struct {
		arg1 string
		arg2 string
		arg3 log.FieldLogger
		arg4 log.Fields
	}{arg1, arg2, arg3, arg4})
	fake.recordInvocation("AddWithLog", []interface{}{arg1, arg2, arg3, arg4})
	fake.addWithLogMutex.Unlock()
	if fake.AddWithLogStub != nil {
		return fake.AddWithLogStub(arg1, arg2, arg3, arg4)
	}
	if specificReturn {
		return ret.result1
	}
	return fake.addWithLogReturns.result1
}

func (fake *FakeIClient) AddWithLogCallCount() int {
	fake.addWithLogMutex.RLock()
	defer fake.addWithLogMutex.RUnlock()
	return len(fake.addWithLogArgsForCall)
}

func (fake *FakeIClient) AddWithLogArgsForCall(i int) (string, string, log.FieldLogger, log.Fields) {
	fake.addWithLogMutex.RLock()
	defer fake.addWithLogMutex.RUnlock()
	return fake.addWithLogArgsForCall[i].arg1, fake.addWithLogArgsForCall[i].arg2, fake.addWithLogArgsForCall[i].arg3, fake.addWithLogArgsForCall[i].arg4
}

func (fake *FakeIClient) AddWithLogReturns(result1 error) {
	fake.AddWithLogStub = nil
	fake.addWithLogReturns = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) AddWithLogReturnsOnCall(i int, result1 error) {
	fake.AddWithLogStub = nil
	if fake.addWithLogReturnsOnCall == nil {
		fake.addWithLogReturnsOnCall = make(map[int]struct {
			result1 error
		})
	}
	fake.addWithLogReturnsOnCall[i] = struct {
		result1 error
	}{result1}
}

func (fake *FakeIClient) Invocations() map[string][][]interface{} {
	fake.invocationsMutex.RLock()
	defer fake.invocationsMutex.RUnlock()
	fake.addMutex.RLock()
	defer fake.addMutex.RUnlock()
	fake.addWithErrorLogMutex.RLock()
	defer fake.addWithErrorLogMutex.RUnlock()
	fake.addWithLogMutex.RLock()
	defer fake.addWithLogMutex.RUnlock()
	return fake.invocations
}

func (fake *FakeIClient) recordInvocation(key string, args []interface{}) {
	fake.invocationsMutex.Lock()
	defer fake.invocationsMutex.Unlock()
	if fake.invocations == nil {
		fake.invocations = map[string][][]interface{}{}
	}
	if fake.invocations[key] == nil {
		fake.invocations[key] = [][]interface{}{}
	}
	fake.invocations[key] = append(fake.invocations[key], args)
}

var _ event.IClient = new(FakeIClient)
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package patch implements a simple patching mechanism for k8s resources.
Paths are specified in the form a.b.c.[key:value].d.[list_entry_value], where:
 -  [key:value] selects a list entry in list c which contains an entry with key:value
 -  [list_entry_value] selects a list entry in list d which is a regex match of list_entry_value.

Some examples are given below. Given a resource:

kind: Deployment
metadata:
  name: istio-citadel
  namespace: istio-system
a:
  b:
  - name: n1
    value: v1
  - name: n2
    list:
    - "vv1"
    - vv2=foo

values and list entries can be added, modifed or deleted.

MODIFY

1. set v1 to v1new

  path: a.b.[name:n1].value
  value: v1new

2. set vv1 to vv3

  // Note the lack of quotes around vv1 (see NOTES below).
  path: a.b.[name:n2].list.[vv1]
  value: vv3

3. set vv2=foo to vv2=bar (using regex match)

  path: a.b.[name:n2].list.[vv2]
  value: vv2=bar

DELETE

1. Delete container with name: n1

  path: a.b.[name:n1]

2. Delete list value vv1

  path: a.b.[name:n2].list.[vv1]

ADD

1. Add vv3 to list

  path: a.b.[name:n2].list
  value: vv3

2. Add new key:value to container name: n1

  path: a.b.[name:n1]
  value:
    new_attr: v3

*NOTES*
- Due to loss of string quoting during unmarshaling, keys and values should not be string quoted, even if they appear
that way in the object being patched.
- [key:value] treats ':' as a special separator character. Any ':' in the key or value string must be escaped as \:.
*/
package patch

import (
	"fmt"
	"strings"

	"gopkg.in/yaml.v2"

	"istio.io/api/operator/v1alpha1"
	"istio.io/istio/operator/pkg/helm"
	"istio.io/istio/operator/pkg/metrics"
	"istio.io/istio/operator/pkg/object"
	"istio.io/istio/operator/pkg/tpath"
	"istio.io/istio/operator/pkg/util"
	"istio.io/pkg/log"
)

var scope = log.RegisterScope("patch", "patch", 0)

// overlayMatches reports whether obj matches the overlay for either the default namespace or no namespace (cluster scope).
func overlayMatches(overlay *v1alpha1.K8SObjectOverlay, obj *object.K8sObject, defaultNamespace string) bool {
	oh := obj.Hash()
	if oh == object.Hash(overlay.Kind, defaultNamespace, overlay.Name) ||
		oh == object.Hash(overlay.Kind, "", overlay.Name) {
		return true
	}
	return false
}

// YAMLManifestPatch patches a base YAML in the given namespace with a list of overlays.
// Each overlay has the format described in the K8SObjectOverlay definition.
// It returns the patched manifest YAML.
func YAMLManifestPatch(baseYAML string, defaultNamespace string, overlays []*v1alpha1.K8SObjectOverlay) (string, error) {
	var ret strings.Builder
	var errs util.Errors
	objs, err := object.ParseK8sObjectsFromYAMLManifest(baseYAML)
	if err != nil {
		return "", err
	}

	matches := make(map[*v1alpha1.K8SObjectOverlay]object.K8sObjects)
	// Try to apply the defined overlays.
	for _, obj := range objs {
		oy, err := obj.YAML()
		if err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("object to YAML error (%s) for base object: \n%s", err, obj.YAMLDebugString()))
			continue
		}
		oys := string(oy)
		for _, overlay := range overlays {
			if overlayMatches(overlay, obj, defaultNamespace) {
				matches[overlay] = append(matches[overlay], obj)
				var errs2 util.Errors
				oys, errs2 = applyPatches(obj, overlay.Patches)
				errs = util.AppendErrs(errs, errs2)
			}
		}
		if _, err := ret.WriteString(oys + helm.YAMLSeparator); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("writeString: %s", err))
		}
	}

	for _, overlay := range overlays {
		// Each overlay should have exactly one match in the output manifest.
		switch {
		case len(matches[overlay]) == 0:
			errs = util.AppendErr(errs, fmt.Errorf("overlay for %s:%s does not match any object in output manifest. Available objects are:\n%s",
				overlay.Kind, overlay.Name, strings.Join(objs.Keys(), "\n")))
		case len(matches[overlay]) > 1:
			errs = util.AppendErr(errs, fmt.Errorf("overlay for %s:%s matches multiple objects in output manifest:\n%s",
				overlay.Kind, overlay.Name, strings.Join(objs.Keys(), "\n")))
		}
	}

	return ret.String(), errs.ToError()
}

// applyPatches applies the given patches against the given object. It returns the resulting patched YAML if successful,
// or a list of errors otherwise.
func applyPatches(base *object.K8sObject, patches []*v1alpha1.K8SObjectOverlay_PathValue) (outYAML string, errs util.Errors) {
	bo := make(map[interface{}]interface{})
	by, err := base.YAML()
	if err != nil {
		return "", util.NewErrs(err)
	}
	err = yaml.Unmarshal(by, bo)
	if err != nil {
		return "", util.NewErrs(err)
	}
	for _, p := range patches {
		if strings.TrimSpace(p.Path) == "" {
			scope.Warnf("value=%s has empty path, skip\n", p.Value)
			continue
		}
		scope.Debugf("applying path=%s, value=%s\n", p.Path, p.Value)
		inc, _, err := tpath.GetPathContext(bo, util.PathFromString(p.Path), true)
		if err != nil {
			errs = util.AppendErr(errs, err)
			metrics.ManifestPatchErrorTotal.Increment()
			continue
		}
		err = tpath.WritePathContext(inc, p.Value, false)
		if err != nil {
			errs = util.AppendErr(errs, err)
			metrics.ManifestPatchErrorTotal.Increment()
		}
	}
	oy, err := yaml.Marshal(bo)
	if err != nil {
		return "", util.AppendErr(errs, err)
	}
	return string(oy), errs
}
<commit_msg>Fixed Unable to add an item to a list using overlays in IstioOperator (#35044)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package patch implements a simple patching mechanism for k8s resources.
Paths are specified in the form a.b.c.[key:value].d.[list_entry_value], where:
 -  [key:value] selects a list entry in list c which contains an entry with key:value
 -  [list_entry_value] selects a list entry in list d which is a regex match of list_entry_value.

Some examples are given below. Given a resource:

kind: Deployment
metadata:
  name: istio-citadel
  namespace: istio-system
a:
  b:
  - name: n1
    value: v1
  - name: n2
    list:
    - "vv1"
    - vv2=foo

values and list entries can be added, modifed or deleted.

MODIFY

1. set v1 to v1new

  path: a.b.[name:n1].value
  value: v1new

2. set vv1 to vv3

  // Note the lack of quotes around vv1 (see NOTES below).
  path: a.b.[name:n2].list.[vv1]
  value: vv3

3. set vv2=foo to vv2=bar (using regex match)

  path: a.b.[name:n2].list.[vv2]
  value: vv2=bar

4. replace a port whose port was 15010

   - path: spec.ports.[port:15010]
     value:
       port: 15020
       name: grpc-xds
       protocol: TCP

DELETE

1. Delete container with name: n1

  path: a.b.[name:n1]

2. Delete list value vv1

  path: a.b.[name:n2].list.[vv1]

ADD

1. Add vv3 to list

  path: a.b.[name:n2].list.[1000]
  value: vv3

Note: the value 1000 is an example. That value used in the patch should
be a value greater than number of the items in the list. Choose 1000 is
just an example which normally is greater than the most of the lists used.

2. Add new key:value to container name: n1

  path: a.b.[name:n1]
  value:
    new_attr: v3

*NOTES*
- Due to loss of string quoting during unmarshaling, keys and values should not be string quoted, even if they appear
that way in the object being patched.
- [key:value] treats ':' as a special separator character. Any ':' in the key or value string must be escaped as \:.
*/
package patch

import (
	"fmt"
	"strings"

	"gopkg.in/yaml.v2"

	"istio.io/api/operator/v1alpha1"
	"istio.io/istio/operator/pkg/helm"
	"istio.io/istio/operator/pkg/metrics"
	"istio.io/istio/operator/pkg/object"
	"istio.io/istio/operator/pkg/tpath"
	"istio.io/istio/operator/pkg/util"
	"istio.io/pkg/log"
)

var scope = log.RegisterScope("patch", "patch", 0)

// overlayMatches reports whether obj matches the overlay for either the default namespace or no namespace (cluster scope).
func overlayMatches(overlay *v1alpha1.K8SObjectOverlay, obj *object.K8sObject, defaultNamespace string) bool {
	oh := obj.Hash()
	if oh == object.Hash(overlay.Kind, defaultNamespace, overlay.Name) ||
		oh == object.Hash(overlay.Kind, "", overlay.Name) {
		return true
	}
	return false
}

// YAMLManifestPatch patches a base YAML in the given namespace with a list of overlays.
// Each overlay has the format described in the K8SObjectOverlay definition.
// It returns the patched manifest YAML.
func YAMLManifestPatch(baseYAML string, defaultNamespace string, overlays []*v1alpha1.K8SObjectOverlay) (string, error) {
	var ret strings.Builder
	var errs util.Errors
	objs, err := object.ParseK8sObjectsFromYAMLManifest(baseYAML)
	if err != nil {
		return "", err
	}

	matches := make(map[*v1alpha1.K8SObjectOverlay]object.K8sObjects)
	// Try to apply the defined overlays.
	for _, obj := range objs {
		oy, err := obj.YAML()
		if err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("object to YAML error (%s) for base object: \n%s", err, obj.YAMLDebugString()))
			continue
		}
		oys := string(oy)
		for _, overlay := range overlays {
			if overlayMatches(overlay, obj, defaultNamespace) {
				matches[overlay] = append(matches[overlay], obj)
				var errs2 util.Errors
				oys, errs2 = applyPatches(obj, overlay.Patches)
				errs = util.AppendErrs(errs, errs2)
			}
		}
		if _, err := ret.WriteString(oys + helm.YAMLSeparator); err != nil {
			errs = util.AppendErr(errs, fmt.Errorf("writeString: %s", err))
		}
	}

	for _, overlay := range overlays {
		// Each overlay should have exactly one match in the output manifest.
		switch {
		case len(matches[overlay]) == 0:
			errs = util.AppendErr(errs, fmt.Errorf("overlay for %s:%s does not match any object in output manifest. Available objects are:\n%s",
				overlay.Kind, overlay.Name, strings.Join(objs.Keys(), "\n")))
		case len(matches[overlay]) > 1:
			errs = util.AppendErr(errs, fmt.Errorf("overlay for %s:%s matches multiple objects in output manifest:\n%s",
				overlay.Kind, overlay.Name, strings.Join(objs.Keys(), "\n")))
		}
	}

	return ret.String(), errs.ToError()
}

// applyPatches applies the given patches against the given object. It returns the resulting patched YAML if successful,
// or a list of errors otherwise.
func applyPatches(base *object.K8sObject, patches []*v1alpha1.K8SObjectOverlay_PathValue) (outYAML string, errs util.Errors) {
	bo := make(map[interface{}]interface{})
	by, err := base.YAML()
	if err != nil {
		return "", util.NewErrs(err)
	}
	err = yaml.Unmarshal(by, bo)
	if err != nil {
		return "", util.NewErrs(err)
	}
	for _, p := range patches {
		if strings.TrimSpace(p.Path) == "" {
			scope.Warnf("value=%s has empty path, skip\n", p.Value)
			continue
		}
		scope.Debugf("applying path=%s, value=%s\n", p.Path, p.Value)
		inc, _, err := tpath.GetPathContext(bo, util.PathFromString(p.Path), true)
		if err != nil {
			errs = util.AppendErr(errs, err)
			metrics.ManifestPatchErrorTotal.Increment()
			continue
		}
		err = tpath.WritePathContext(inc, p.Value, false)
		if err != nil {
			errs = util.AppendErr(errs, err)
			metrics.ManifestPatchErrorTotal.Increment()
		}
	}
	oy, err := yaml.Marshal(bo)
	if err != nil {
		return "", util.AppendErr(errs, err)
	}
	return string(oy), errs
}
<|endoftext|>


<commit_before>package imagecommands

import (
	"github.com/rackspace/rack/commandoptions"
	"github.com/rackspace/rack/handler"
	"github.com/rackspace/rack/internal/github.com/codegangsta/cli"
	"github.com/rackspace/rack/internal/github.com/fatih/structs"
	osImages "github.com/rackspace/rack/internal/github.com/rackspace/gophercloud/openstack/compute/v2/images"
	"github.com/rackspace/rack/internal/github.com/rackspace/gophercloud/rackspace/compute/v2/images"
	"github.com/rackspace/rack/util"
)

var get = cli.Command{
	Name:        "get",
	Usage:       util.Usage(commandPrefix, "get", "[--id <serverID> | --name <serverName> | --stdin id]"),
	Description: "Retreives an image",
	Action:      actionGet,
	Flags:       commandoptions.CommandFlags(flagsGet, keysGet),
	BashComplete: func(c *cli.Context) {
		commandoptions.CompleteFlags(commandoptions.CommandFlags(flagsGet, keysGet))
	},
}

func flagsGet() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:  "id",
			Usage: "[optional; required if `stdin` or `name` isn't provided] The ID of the image.",
		},
		cli.StringFlag{
			Name:  "name",
			Usage: "[optional; required if `id` or `stdin` isn't provided] The name of the image.",
		},
		cli.StringFlag{
			Name:  "stdin",
			Usage: "[optional; required if `id` or `name` isn't provided] The field being piped to STDIN. Valid values are: id",
		},
	}
}

var keysGet = []string{"ID", "Name", "Status", "Progress", "MinDisk", "MinRAM", "Created", "Updated", "Metadata"}

type paramsGet struct {
	image string
}

type commandGet handler.Command

func actionGet(c *cli.Context) {
	command := &commandGet{
		Ctx: &handler.Context{
			CLIContext: c,
		},
	}
	handler.Handle(command)
}

func (command *commandGet) Context() *handler.Context {
	return command.Ctx
}

func (command *commandGet) Keys() []string {
	return keysGet
}

func (command *commandGet) ServiceClientType() string {
	return serviceClientType
}

func (command *commandGet) HandleFlags(resource *handler.Resource) error {
	resource.Params = &paramsGet{}
	return nil
}

func (command *commandGet) HandlePipe(resource *handler.Resource, item string) error {
	resource.Params.(*paramsGet).image = item
	return nil
}

func (command *commandGet) HandleSingle(resource *handler.Resource) error {
	id, err := command.Ctx.IDOrName(osImages.IDFromName)
	resource.Params.(*paramsGet).image = id
	return err
}

func (command *commandGet) Execute(resource *handler.Resource) {
	imageID := resource.Params.(*paramsGet).image
	image, err := images.Get(command.Ctx.ServiceClient, imageID).Extract()
	if err != nil {
		resource.Err = err
		return
	}
	resource.Result = structs.Map(image)
}

func (command *commandGet) StdinField() string {
	return "id"
}

func (command *commandGet) PreCSV(resource *handler.Resource) {
	resource.FlattenMap("Metadata")
}

func (command *commandGet) PreTable(resource *handler.Resource) {
	command.PreCSV(resource)
}
<commit_msg>fix typo in 'image get' help text<commit_after>package imagecommands

import (
	"github.com/rackspace/rack/commandoptions"
	"github.com/rackspace/rack/handler"
	"github.com/rackspace/rack/internal/github.com/codegangsta/cli"
	"github.com/rackspace/rack/internal/github.com/fatih/structs"
	osImages "github.com/rackspace/rack/internal/github.com/rackspace/gophercloud/openstack/compute/v2/images"
	"github.com/rackspace/rack/internal/github.com/rackspace/gophercloud/rackspace/compute/v2/images"
	"github.com/rackspace/rack/util"
)

var get = cli.Command{
	Name:        "get",
	Usage:       util.Usage(commandPrefix, "get", "[--id <imageID> | --name <imageName> | --stdin id]"),
	Description: "Retreives information about the image.",
	Action:      actionGet,
	Flags:       commandoptions.CommandFlags(flagsGet, keysGet),
	BashComplete: func(c *cli.Context) {
		commandoptions.CompleteFlags(commandoptions.CommandFlags(flagsGet, keysGet))
	},
}

func flagsGet() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:  "id",
			Usage: "[optional; required if `stdin` or `name` isn't provided] The ID of the image.",
		},
		cli.StringFlag{
			Name:  "name",
			Usage: "[optional; required if `id` or `stdin` isn't provided] The name of the image.",
		},
		cli.StringFlag{
			Name:  "stdin",
			Usage: "[optional; required if `id` or `name` isn't provided] The field being piped to STDIN. Valid values are: id",
		},
	}
}

var keysGet = []string{"ID", "Name", "Status", "Progress", "MinDisk", "MinRAM", "Created", "Updated", "Metadata"}

type paramsGet struct {
	image string
}

type commandGet handler.Command

func actionGet(c *cli.Context) {
	command := &commandGet{
		Ctx: &handler.Context{
			CLIContext: c,
		},
	}
	handler.Handle(command)
}

func (command *commandGet) Context() *handler.Context {
	return command.Ctx
}

func (command *commandGet) Keys() []string {
	return keysGet
}

func (command *commandGet) ServiceClientType() string {
	return serviceClientType
}

func (command *commandGet) HandleFlags(resource *handler.Resource) error {
	resource.Params = &paramsGet{}
	return nil
}

func (command *commandGet) HandlePipe(resource *handler.Resource, item string) error {
	resource.Params.(*paramsGet).image = item
	return nil
}

func (command *commandGet) HandleSingle(resource *handler.Resource) error {
	id, err := command.Ctx.IDOrName(osImages.IDFromName)
	resource.Params.(*paramsGet).image = id
	return err
}

func (command *commandGet) Execute(resource *handler.Resource) {
	imageID := resource.Params.(*paramsGet).image
	image, err := images.Get(command.Ctx.ServiceClient, imageID).Extract()
	if err != nil {
		resource.Err = err
		return
	}
	resource.Result = structs.Map(image)
}

func (command *commandGet) StdinField() string {
	return "id"
}

func (command *commandGet) PreCSV(resource *handler.Resource) {
	resource.FlattenMap("Metadata")
}

func (command *commandGet) PreTable(resource *handler.Resource) {
	command.PreCSV(resource)
}
<|endoftext|>


<commit_before>// Copyright © 2017 Aqua Security Software Ltd. <info@aquasec.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package check

import (
	"testing"
)

func TestCheck_Run(t *testing.T) {
	type TestCase struct {
		check    Check
		Expected State
	}

	testCases := []TestCase{
		{check: Check{Type: "manual"}, Expected: WARN},
		{check: Check{Type: "skip"}, Expected: INFO},
		{check: Check{Type: "", Scored: false}, Expected: WARN}, // Not scored checks with no type should be marked warn
		{check: Check{Type: "", Scored: true}, Expected: WARN},  // If there are no tests in the check, warn
		{check: Check{Type: "manual", Scored: false}, Expected: WARN},
		{check: Check{Type: "skip", Scored: false}, Expected: INFO},
	}

	for _, testCase := range testCases {

		testCase.check.run()

		if testCase.check.State != testCase.Expected {
			t.Errorf("test failed, expected %s, actual %s\n", testCase.Expected, testCase.check.State)
		}
	}
}
<commit_msg>Update copyright date<commit_after>// Copyright © 2017-2019 Aqua Security Software Ltd. <info@aquasec.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package check

import (
	"testing"
)

func TestCheck_Run(t *testing.T) {
	type TestCase struct {
		check    Check
		Expected State
	}

	testCases := []TestCase{
		{check: Check{Type: "manual"}, Expected: WARN},
		{check: Check{Type: "skip"}, Expected: INFO},
		{check: Check{Type: "", Scored: false}, Expected: WARN}, // Not scored checks with no type should be marked warn
		{check: Check{Type: "", Scored: true}, Expected: WARN},  // If there are no tests in the check, warn
		{check: Check{Type: "manual", Scored: false}, Expected: WARN},
		{check: Check{Type: "skip", Scored: false}, Expected: INFO},
	}

	for _, testCase := range testCases {

		testCase.check.run()

		if testCase.check.State != testCase.Expected {
			t.Errorf("test failed, expected %s, actual %s\n", testCase.Expected, testCase.check.State)
		}
	}
}
<|endoftext|>


<commit_before>package fileSystem

import (
	"testing"
	"time"
)

func TestCreateAndGet(t *testing.T) {
	fs := New()

	// this should create successfully
	createAndGet(fs, "/foobar", t)
	createAndGet(fs, "/foo/bar", t)
	createAndGet(fs, "/foo/foo/bar", t)

	// already exist, create should fail
	_, err := fs.Create("/foobar", "bar", Permanent, 1, 1)

	if err == nil {
		t.Fatal("Create should fail")
	}

	// meet file, create should fail
	_, err = fs.Create("/foo/bar/bar", "bar", Permanent, 2, 1)

	if err == nil {
		t.Fatal("Create should fail")
	}

	// create a directory
	_, err = fs.Create("/fooDir", "", Permanent, 3, 1)

	if err != nil {
		t.Fatal("Cannot create /fooDir")
	}

	e, err := fs.Get("/fooDir", false, 3, 1)

	if err != nil || e.Dir != true {
		t.Fatal("Cannot create /fooDir ")
	}

	// create a file under directory
	_, err = fs.Create("/fooDir/bar", "bar", Permanent, 4, 1)

	if err != nil {
		t.Fatal("Cannot create /fooDir/bar = bar")
	}

}

func TestUpdateFile(t *testing.T) {
	fs := New()

	_, err := fs.Create("/foo/bar", "bar", Permanent, 1, 1)

	if err != nil {
		t.Fatalf("cannot create %s=bar [%s]", "/foo/bar", err.Error())
	}

	_, err = fs.Update("/foo/bar", "barbar", Permanent, 2, 1)

	if err != nil {
		t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error())
	}

	e, err := fs.Get("/foo/bar", false, 2, 1)

	if err != nil {
		t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error())
	}

	if e.Value != "barbar" {
		t.Fatalf("expect value of %s is barbar [%s]", "/foo/bar", e.Value)
	}

	// create a directory, update its ttl, to see if it will be deleted

	_, err = fs.Create("/foo/foo", "", Permanent, 3, 1)

	if err != nil {
		t.Fatalf("cannot create dir [%s] [%s]", "/foo/foo", err.Error())
	}

	_, err = fs.Create("/foo/foo/foo1", "bar1", Permanent, 4, 1)

	if err != nil {
		t.Fatal("cannot create [%s]", err.Error())
	}

	_, err = fs.Create("/foo/foo/foo2", "", Permanent, 5, 1)
	if err != nil {
		t.Fatal("cannot create [%s]", err.Error())
	}

	_, err = fs.Create("/foo/foo/foo2/boo", "boo1", Permanent, 6, 1)
	if err != nil {
		t.Fatal("cannot create [%s]", err.Error())
	}

	expire := time.Now().Add(time.Second*2)
	_, err = fs.Update("/foo/foo", "", expire, 7, 1)
	if err != nil {
		t.Fatalf("cannot update dir [%s] [%s]", "/foo/foo", err.Error())
	}

	// sleep 50ms, it should still reach the node
	time.Sleep(time.Microsecond*50)
	e, err = fs.Get("/foo/foo", true, 7, 1)

	if err != nil || e.Key != "/foo/foo" {
		t.Fatalf("cannot get dir before expiration [%s]", err.Error())
	}

	if e.KVPairs[0].Key != "/foo/foo/foo1" || e.KVPairs[0].Value != "bar1" {
		t.Fatalf("cannot get sub node before expiration [%s]", err.Error())
	}

	if e.KVPairs[1].Key != "/foo/foo/foo2" || e.KVPairs[1].Dir != true {
		t.Fatalf("cannot get sub dir before expiration [%s]", err.Error())
	}

	/*if e.KVPairs[2].Key != "/foo/foo/foo2/boo" || e.KVPairs[2].Value != "boo1" {
		t.Fatalf("cannot get sub node of sub dir before expiration [%s]", err.Error())
	}*/

	// wait for expiration
	time.Sleep(time.Second*3)
	e, err = fs.Get("/foo/foo", true, 7, 1)

	if err == nil {
		t.Fatal("still can get dir after expiration [%s]")
	}

	_, err = fs.Get("/foo/foo/foo1", true, 7, 1)
	if err == nil {
		t.Fatal("still can get sub node after expiration [%s]")
	}

	_, err = fs.Get("/foo/foo/foo2", true, 7, 1)
	if err == nil {
		t.Fatal("still can get sub dir after expiration [%s]")
	}

	_, err = fs.Get("/foo/foo/foo2/boo", true, 7, 1)
	if err == nil {
		t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error())
	}

}

func TestListDirectory(t *testing.T) {
	fs := New()

	// create dir /foo
	// set key-value /foo/foo=bar
	fs.Create("/foo/foo", "bar", Permanent, 1, 1)

	// create dir /foo/fooDir
	// set key-value /foo/fooDir/foo=bar
	fs.Create("/foo/fooDir/foo", "bar", Permanent, 2, 1)

	e, err := fs.Get("/foo", true, 2, 1)

	if err != nil {
		t.Fatalf("%v", err)
	}

	if len(e.KVPairs) != 2 {
		t.Fatalf("wrong number of kv pairs [%d/2]", len(e.KVPairs))
	}

	if e.KVPairs[0].Key != "/foo/foo" || e.KVPairs[0].Value != "bar" {
		t.Fatalf("wrong kv [/foo/foo/ / %s] -> [bar / %s]", e.KVPairs[0].Key, e.KVPairs[0].Value)
	}

	if e.KVPairs[1].Key != "/foo/fooDir" || e.KVPairs[1].Dir != true {
		t.Fatalf("wrong kv [/foo/fooDir/ / %s] -> [true / %v]", e.KVPairs[1].Key, e.KVPairs[1].Dir)
	}

	if e.KVPairs[1].KVPairs[0].Key != "/foo/fooDir/foo" || e.KVPairs[1].KVPairs[0].Value != "bar" {
		t.Fatalf("wrong kv [/foo/fooDir/foo / %s] -> [bar / %v]", e.KVPairs[1].KVPairs[0].Key, e.KVPairs[1].KVPairs[0].Value)
	}
	// test hidden node

	// create dir /foo/_hidden
	// set key-value /foo/_hidden/foo -> bar
	fs.Create("/foo/_hidden/foo", "bar", Permanent, 3, 1)

	e, _ = fs.Get("/foo", false, 2, 1)

	if len(e.KVPairs) != 2 {
		t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key)
	}
}

func TestRemove(t *testing.T) {
	fs := New()

	fs.Create("/foo", "bar", Permanent, 1, 1)
	_, err := fs.Delete("/foo", false, 1, 1)

	if err != nil {
		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
	}

	_, err = fs.Get("/foo", false, 1, 1)

	if err == nil || err.Error() != "Key Not Found" {
		t.Fatalf("can get the node after deletion")
	}

	fs.Create("/foo/bar", "bar", Permanent, 1, 1)
	fs.Create("/foo/car", "car", Permanent, 1, 1)
	fs.Create("/foo/dar/dar", "dar", Permanent, 1, 1)

	_, err = fs.Delete("/foo", false, 1, 1)

	if err == nil {
		t.Fatalf("should not be able to delete a directory without recursive")
	}

	_, err = fs.Delete("/foo", true, 1, 1)

	if err != nil {
		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
	}

	_, err = fs.Get("/foo", false, 1, 1)

	if err == nil || err.Error() != "Key Not Found" {
		t.Fatalf("can get the node after deletion ")
	}

}

func TestExpire(t *testing.T) {
	fs := New()

	expire := time.Now().Add(time.Second)

	fs.Create("/foo", "bar", expire, 1, 1)

	_, err := fs.InternalGet("/foo", 1, 1)

	if err != nil {
		t.Fatalf("can not get the node")
	}

	time.Sleep(time.Second * 2)

	_, err = fs.InternalGet("/foo", 1, 1)

	if err == nil {
		t.Fatalf("can get the node after expiration time")
	}

	// test if we can reach the node before expiration
	expire = time.Now().Add(time.Second)
	fs.Create("/foo", "bar", expire, 1, 1)

	time.Sleep(time.Millisecond * 50)
	_, err = fs.InternalGet("/foo", 1, 1)

	if err != nil {
		t.Fatalf("cannot get the node before expiration", err.Error())
	}

	expire = time.Now().Add(time.Second)

	fs.Create("/foo", "bar", expire, 1, 1)
	_, err = fs.Delete("/foo", false, 1, 1)

	if err != nil {
		t.Fatalf("cannot delete the node before expiration", err.Error())
	}

}

func TestTestAndSet(t *testing.T) { // TODO prevValue == nil ?
	fs := New()
	fs.Create("/foo", "bar", Permanent, 1, 1)

	// test on wrong previous value
	_, err := fs.TestAndSet("/foo", "barbar", 0, "car", Permanent, 2, 1)
	if err == nil {
		t.Fatal("test and set should fail barbar != bar")
	}

	// test on value
	e, err := fs.TestAndSet("/foo", "bar", 0, "car", Permanent, 3, 1)

	if err != nil {
		t.Fatal("test and set should succeed bar == bar")
	}

	if e.PrevValue != "bar" || e.Value != "car" {
		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "bar", e.Value, "car")
	}

	// test on index
	e, err = fs.TestAndSet("/foo", "", 3, "bar", Permanent, 4, 1)

	if err != nil {
		t.Fatal("test and set should succeed index 3 == 3")
	}

	if e.PrevValue != "car" || e.Value != "bar" {
		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "car", e.Value, "bar")
	}

	//e, err = fs.TestAndSet("/foo", )
}

func TestWatch(t *testing.T) {
	fs := New()
	// watch at a deeper path
	c, _ := fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.Create("/foo/foo/foo", "bar", Permanent, 1, 1)

	e := nonblockingRetrive(c)
	if e.Key != "/foo/foo/foo" {
		t.Fatal("watch for Create node fails")
	}

	c, _ = fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.Update("/foo/foo/foo", "car", Permanent, 2, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/foo" {
		t.Fatal("watch for Update node fails")
	}

	c, _ = fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.TestAndSet("/foo/foo/foo", "car", 0, "bar", Permanent, 3, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/foo" {
		t.Fatal("watch for TestAndSet node fails")
	}

	c, _ = fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.Delete("/foo", true, 4, 1) //recursively delete
	e = nonblockingRetrive(c)
	if e.Key != "/foo" {
		t.Fatal("watch for Delete node fails")
	}

	// watch at a prefix
	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.Create("/foo/foo/boo", "bar", Permanent, 5, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for Create subdirectory fails")
	}

	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.Update("/foo/foo/boo", "foo", Permanent, 6, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for Update subdirectory fails")
	}

	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.TestAndSet("/foo/foo/boo", "foo", 0, "bar", Permanent, 7, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for TestAndSet subdirectory fails")
	}

	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.Delete("/foo/foo/boo", false, 8, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for Delete subdirectory fails")
	}

}

func createAndGet(fs *FileSystem, path string, t *testing.T) {
	_, err := fs.Create(path, "bar", Permanent, 1, 1)

	if err != nil {
		t.Fatalf("cannot create %s=bar [%s]", path, err.Error())
	}

	e, err := fs.Get(path, false, 1, 1)

	if err != nil {
		t.Fatalf("cannot get %s [%s]", path, err.Error())
	}

	if e.Value != "bar" {
		t.Fatalf("expect value of %s is bar [%s]", path, e.Value)
	}

}

func nonblockingRetrive(c <-chan *Event) *Event{
	select {
	case e := <-c:
		return e
	default:
		return nil
	}
}<commit_msg>gofmt<commit_after>package fileSystem

import (
	"testing"
	"time"
)

func TestCreateAndGet(t *testing.T) {
	fs := New()

	// this should create successfully
	createAndGet(fs, "/foobar", t)
	createAndGet(fs, "/foo/bar", t)
	createAndGet(fs, "/foo/foo/bar", t)

	// already exist, create should fail
	_, err := fs.Create("/foobar", "bar", Permanent, 1, 1)

	if err == nil {
		t.Fatal("Create should fail")
	}

	// meet file, create should fail
	_, err = fs.Create("/foo/bar/bar", "bar", Permanent, 2, 1)

	if err == nil {
		t.Fatal("Create should fail")
	}

	// create a directory
	_, err = fs.Create("/fooDir", "", Permanent, 3, 1)

	if err != nil {
		t.Fatal("Cannot create /fooDir")
	}

	e, err := fs.Get("/fooDir", false, 3, 1)

	if err != nil || e.Dir != true {
		t.Fatal("Cannot create /fooDir ")
	}

	// create a file under directory
	_, err = fs.Create("/fooDir/bar", "bar", Permanent, 4, 1)

	if err != nil {
		t.Fatal("Cannot create /fooDir/bar = bar")
	}

}

func TestUpdateFile(t *testing.T) {
	fs := New()

	_, err := fs.Create("/foo/bar", "bar", Permanent, 1, 1)

	if err != nil {
		t.Fatalf("cannot create %s=bar [%s]", "/foo/bar", err.Error())
	}

	_, err = fs.Update("/foo/bar", "barbar", Permanent, 2, 1)

	if err != nil {
		t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error())
	}

	e, err := fs.Get("/foo/bar", false, 2, 1)

	if err != nil {
		t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error())
	}

	if e.Value != "barbar" {
		t.Fatalf("expect value of %s is barbar [%s]", "/foo/bar", e.Value)
	}

	// create a directory, update its ttl, to see if it will be deleted

	_, err = fs.Create("/foo/foo", "", Permanent, 3, 1)

	if err != nil {
		t.Fatalf("cannot create dir [%s] [%s]", "/foo/foo", err.Error())
	}

	_, err = fs.Create("/foo/foo/foo1", "bar1", Permanent, 4, 1)

	if err != nil {
		t.Fatal("cannot create [%s]", err.Error())
	}

	_, err = fs.Create("/foo/foo/foo2", "", Permanent, 5, 1)
	if err != nil {
		t.Fatal("cannot create [%s]", err.Error())
	}

	_, err = fs.Create("/foo/foo/foo2/boo", "boo1", Permanent, 6, 1)
	if err != nil {
		t.Fatal("cannot create [%s]", err.Error())
	}

	expire := time.Now().Add(time.Second * 2)
	_, err = fs.Update("/foo/foo", "", expire, 7, 1)
	if err != nil {
		t.Fatalf("cannot update dir [%s] [%s]", "/foo/foo", err.Error())
	}

	// sleep 50ms, it should still reach the node
	time.Sleep(time.Microsecond * 50)
	e, err = fs.Get("/foo/foo", true, 7, 1)

	if err != nil || e.Key != "/foo/foo" {
		t.Fatalf("cannot get dir before expiration [%s]", err.Error())
	}

	if e.KVPairs[0].Key != "/foo/foo/foo1" || e.KVPairs[0].Value != "bar1" {
		t.Fatalf("cannot get sub node before expiration [%s]", err.Error())
	}

	if e.KVPairs[1].Key != "/foo/foo/foo2" || e.KVPairs[1].Dir != true {
		t.Fatalf("cannot get sub dir before expiration [%s]", err.Error())
	}

	/*if e.KVPairs[2].Key != "/foo/foo/foo2/boo" || e.KVPairs[2].Value != "boo1" {
		t.Fatalf("cannot get sub node of sub dir before expiration [%s]", err.Error())
	}*/

	// wait for expiration
	time.Sleep(time.Second * 3)
	e, err = fs.Get("/foo/foo", true, 7, 1)

	if err == nil {
		t.Fatal("still can get dir after expiration [%s]")
	}

	_, err = fs.Get("/foo/foo/foo1", true, 7, 1)
	if err == nil {
		t.Fatal("still can get sub node after expiration [%s]")
	}

	_, err = fs.Get("/foo/foo/foo2", true, 7, 1)
	if err == nil {
		t.Fatal("still can get sub dir after expiration [%s]")
	}

	_, err = fs.Get("/foo/foo/foo2/boo", true, 7, 1)
	if err == nil {
		t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error())
	}

}

func TestListDirectory(t *testing.T) {
	fs := New()

	// create dir /foo
	// set key-value /foo/foo=bar
	fs.Create("/foo/foo", "bar", Permanent, 1, 1)

	// create dir /foo/fooDir
	// set key-value /foo/fooDir/foo=bar
	fs.Create("/foo/fooDir/foo", "bar", Permanent, 2, 1)

	e, err := fs.Get("/foo", true, 2, 1)

	if err != nil {
		t.Fatalf("%v", err)
	}

	if len(e.KVPairs) != 2 {
		t.Fatalf("wrong number of kv pairs [%d/2]", len(e.KVPairs))
	}

	if e.KVPairs[0].Key != "/foo/foo" || e.KVPairs[0].Value != "bar" {
		t.Fatalf("wrong kv [/foo/foo/ / %s] -> [bar / %s]", e.KVPairs[0].Key, e.KVPairs[0].Value)
	}

	if e.KVPairs[1].Key != "/foo/fooDir" || e.KVPairs[1].Dir != true {
		t.Fatalf("wrong kv [/foo/fooDir/ / %s] -> [true / %v]", e.KVPairs[1].Key, e.KVPairs[1].Dir)
	}

	if e.KVPairs[1].KVPairs[0].Key != "/foo/fooDir/foo" || e.KVPairs[1].KVPairs[0].Value != "bar" {
		t.Fatalf("wrong kv [/foo/fooDir/foo / %s] -> [bar / %v]", e.KVPairs[1].KVPairs[0].Key, e.KVPairs[1].KVPairs[0].Value)
	}
	// test hidden node

	// create dir /foo/_hidden
	// set key-value /foo/_hidden/foo -> bar
	fs.Create("/foo/_hidden/foo", "bar", Permanent, 3, 1)

	e, _ = fs.Get("/foo", false, 2, 1)

	if len(e.KVPairs) != 2 {
		t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key)
	}
}

func TestRemove(t *testing.T) {
	fs := New()

	fs.Create("/foo", "bar", Permanent, 1, 1)
	_, err := fs.Delete("/foo", false, 1, 1)

	if err != nil {
		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
	}

	_, err = fs.Get("/foo", false, 1, 1)

	if err == nil || err.Error() != "Key Not Found" {
		t.Fatalf("can get the node after deletion")
	}

	fs.Create("/foo/bar", "bar", Permanent, 1, 1)
	fs.Create("/foo/car", "car", Permanent, 1, 1)
	fs.Create("/foo/dar/dar", "dar", Permanent, 1, 1)

	_, err = fs.Delete("/foo", false, 1, 1)

	if err == nil {
		t.Fatalf("should not be able to delete a directory without recursive")
	}

	_, err = fs.Delete("/foo", true, 1, 1)

	if err != nil {
		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
	}

	_, err = fs.Get("/foo", false, 1, 1)

	if err == nil || err.Error() != "Key Not Found" {
		t.Fatalf("can get the node after deletion ")
	}

}

func TestExpire(t *testing.T) {
	fs := New()

	expire := time.Now().Add(time.Second)

	fs.Create("/foo", "bar", expire, 1, 1)

	_, err := fs.InternalGet("/foo", 1, 1)

	if err != nil {
		t.Fatalf("can not get the node")
	}

	time.Sleep(time.Second * 2)

	_, err = fs.InternalGet("/foo", 1, 1)

	if err == nil {
		t.Fatalf("can get the node after expiration time")
	}

	// test if we can reach the node before expiration
	expire = time.Now().Add(time.Second)
	fs.Create("/foo", "bar", expire, 1, 1)

	time.Sleep(time.Millisecond * 50)
	_, err = fs.InternalGet("/foo", 1, 1)

	if err != nil {
		t.Fatalf("cannot get the node before expiration", err.Error())
	}

	expire = time.Now().Add(time.Second)

	fs.Create("/foo", "bar", expire, 1, 1)
	_, err = fs.Delete("/foo", false, 1, 1)

	if err != nil {
		t.Fatalf("cannot delete the node before expiration", err.Error())
	}

}

func TestTestAndSet(t *testing.T) { // TODO prevValue == nil ?
	fs := New()
	fs.Create("/foo", "bar", Permanent, 1, 1)

	// test on wrong previous value
	_, err := fs.TestAndSet("/foo", "barbar", 0, "car", Permanent, 2, 1)
	if err == nil {
		t.Fatal("test and set should fail barbar != bar")
	}

	// test on value
	e, err := fs.TestAndSet("/foo", "bar", 0, "car", Permanent, 3, 1)

	if err != nil {
		t.Fatal("test and set should succeed bar == bar")
	}

	if e.PrevValue != "bar" || e.Value != "car" {
		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "bar", e.Value, "car")
	}

	// test on index
	e, err = fs.TestAndSet("/foo", "", 3, "bar", Permanent, 4, 1)

	if err != nil {
		t.Fatal("test and set should succeed index 3 == 3")
	}

	if e.PrevValue != "car" || e.Value != "bar" {
		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "car", e.Value, "bar")
	}

	//e, err = fs.TestAndSet("/foo", )
}

func TestWatch(t *testing.T) {
	fs := New()
	// watch at a deeper path
	c, _ := fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.Create("/foo/foo/foo", "bar", Permanent, 1, 1)

	e := nonblockingRetrive(c)
	if e.Key != "/foo/foo/foo" {
		t.Fatal("watch for Create node fails")
	}

	c, _ = fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.Update("/foo/foo/foo", "car", Permanent, 2, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/foo" {
		t.Fatal("watch for Update node fails")
	}

	c, _ = fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.TestAndSet("/foo/foo/foo", "car", 0, "bar", Permanent, 3, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/foo" {
		t.Fatal("watch for TestAndSet node fails")
	}

	c, _ = fs.WatcherHub.watch("/foo/foo/foo", false, 0)
	fs.Delete("/foo", true, 4, 1) //recursively delete
	e = nonblockingRetrive(c)
	if e.Key != "/foo" {
		t.Fatal("watch for Delete node fails")
	}

	// watch at a prefix
	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.Create("/foo/foo/boo", "bar", Permanent, 5, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for Create subdirectory fails")
	}

	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.Update("/foo/foo/boo", "foo", Permanent, 6, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for Update subdirectory fails")
	}

	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.TestAndSet("/foo/foo/boo", "foo", 0, "bar", Permanent, 7, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for TestAndSet subdirectory fails")
	}

	c, _ = fs.WatcherHub.watch("/foo", true, 0)
	fs.Delete("/foo/foo/boo", false, 8, 1)
	e = nonblockingRetrive(c)
	if e.Key != "/foo/foo/boo" {
		t.Fatal("watch for Delete subdirectory fails")
	}

}

func createAndGet(fs *FileSystem, path string, t *testing.T) {
	_, err := fs.Create(path, "bar", Permanent, 1, 1)

	if err != nil {
		t.Fatalf("cannot create %s=bar [%s]", path, err.Error())
	}

	e, err := fs.Get(path, false, 1, 1)

	if err != nil {
		t.Fatalf("cannot get %s [%s]", path, err.Error())
	}

	if e.Value != "bar" {
		t.Fatalf("expect value of %s is bar [%s]", path, e.Value)
	}

}

func nonblockingRetrive(c <-chan *Event) *Event {
	select {
	case e := <-c:
		return e
	default:
		return nil
	}
}
<|endoftext|>


<commit_before>package main

type Config struct {
	Chevalier struct {
		// In host:port format.
		ElasticsearchEndpoint string
		// ZMQ URI to listen on.
		ListenAddress string
	}
}

<commit_msg>elastigo only takes a hostname, not a port<commit_after>package main

type Config struct {
	Chevalier struct {
		// Just the hostname, not the port.
		ElasticsearchHost string
		// ZMQ URI to listen on.
		ListenAddress string
	}
}

<|endoftext|>


<commit_before>package tmysql

import (
	"fmt"
	"math/rand"
	"net/url"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/tolexo/aero/conf"
)

var (
	engines        map[string]bool
	connMySqlWrite string
	connMySqlRead  []string
)

func init() {
	engines = make(map[string]bool)
}

func initMaster() {
	lookup := "database.master"
	if conf.Exists(lookup) {
		connMySqlWrite = getMySqlConnString(lookup)
	}
}
func initSlaves() {
	lookup := "database.slaves"
	if conf.Exists(lookup) {
		slaves := conf.StringSlice(lookup, []string{})
		connMySqlRead = make([]string, len(slaves))
		for i, container := range slaves {
			connMySqlRead[i] = getMySqlConnString(container)
		}
	}
}
func getMySqlConnString(container string) string {
	if !conf.Exists(container) {
		panic("Container for mysql configuration not found")
	}

	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	timezone := conf.String(container+".timezone", "")

	return fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&loc=%s",
		username, password,
		host, port, db,
		url.QueryEscape(timezone),
	)
}

func getDefaultConn(write bool) string {
	if write {
		initMaster()
		return connMySqlWrite
	} else {
		initSlaves()
		if connMySqlRead == nil || len(connMySqlRead) == 0 {
			initMaster()
			return connMySqlWrite
		}
		return connMySqlRead[rand.Intn(len(connMySqlRead))]
	}
}

//Get MySql connection
func GetMySqlConn(writable bool) (dbConn gorm.DB, err error) {
	connStr := getDefaultConn(writable)
	if engines[connStr] == false {
		dbConn, err = newConn(connStr)
	}
	return
}

//newConn open mysql
func newConn(connStr string) (dbConn gorm.DB, err error) {
	if dbConn, err = gorm.Open("mysql", connStr); err == nil {
		engines[connStr] = true
		dbConn.DB().SetConnMaxLifetime(time.Minute * 5)
		dbConn.DB().SetMaxIdleConns(10)
		dbConn.DB().SetMaxOpenConns(200)
	}
	return
}
<commit_msg>PRA-TMYSQL-POOL: mysql setting added for first connection<commit_after>package tmysql

import (
	"fmt"
	"math/rand"
	"net/url"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/tolexo/aero/conf"
)

var (
	engines        map[string]bool
	connMySqlWrite string
	connMySqlRead  []string
)

func init() {
	engines = make(map[string]bool)
}

func initMaster() {
	lookup := "database.master"
	if conf.Exists(lookup) {
		connMySqlWrite = getMySqlConnString(lookup)
	}
}
func initSlaves() {
	lookup := "database.slaves"
	if conf.Exists(lookup) {
		slaves := conf.StringSlice(lookup, []string{})
		connMySqlRead = make([]string, len(slaves))
		for i, container := range slaves {
			connMySqlRead[i] = getMySqlConnString(container)
		}
	}
}
func getMySqlConnString(container string) string {
	if !conf.Exists(container) {
		panic("Container for mysql configuration not found")
	}

	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	timezone := conf.String(container+".timezone", "")

	return fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&loc=%s",
		username, password,
		host, port, db,
		url.QueryEscape(timezone),
	)
}

func getDefaultConn(write bool) string {
	if write {
		initMaster()
		return connMySqlWrite
	} else {
		initSlaves()
		if connMySqlRead == nil || len(connMySqlRead) == 0 {
			initMaster()
			return connMySqlWrite
		}
		return connMySqlRead[rand.Intn(len(connMySqlRead))]
	}
}

//Get MySql connection
func GetMySqlConn(writable bool) (dbConn gorm.DB, err error) {
	connStr := getDefaultConn(writable)
	if dbConn, err = gorm.Open("mysql", connStr); err == nil && engines[connStr] == false {
		engines[connStr] = true
		dbConn.DB().SetConnMaxLifetime(time.Minute * 5)
		dbConn.DB().SetMaxIdleConns(10)
		dbConn.DB().SetMaxOpenConns(200)
	}
	return
}
<|endoftext|>