<commit_before>package core

import (
	"context"
	"encoding/json"
	"expvar"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"chain/core/fetch"
	"chain/core/leader"
	"chain/core/mockhsm"
	"chain/core/txdb"
	"chain/crypto/ed25519"
	"chain/crypto/ed25519/chainkd"
	"chain/database/pg"
	"chain/database/sql"
	"chain/errors"
	"chain/net/http/httpjson"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

var (
	errAlreadyConfigured = errors.New("core is already configured; must reset first")
	errUnconfigured      = errors.New("core is not configured")
	errBadGenerator      = errors.New("generator returned an unsuccessful response")
	errBadBlockXPub      = errors.New("supplied block xpub is invalid")
	errNoClientTokens    = errors.New("cannot enable client auth without client access tokens")
	errBadSignerURL      = errors.New("block signer URL is invalid")
	errBadSignerPubkey   = errors.New("block signer pubkey is invalid")
	errBadQuorum         = errors.New("quorum must be greater than 0 if there are signers")
	// errProdReset is returned when reset is called on a
	// production system.
	errProdReset = errors.New("reset called on production system")
)

// reserved mockhsm key alias
const (
	networkRPCVersion = 1
	autoBlockKeyAlias = "_CHAIN_CORE_AUTO_BLOCK_KEY"
)

func isProduction() bool {
	bt := expvar.Get("buildtag")
	return bt != nil && bt.String() != `"dev"`
}

// Schema returns DDL for the entire Chain Core Postgres schema.
func Schema() string {
	return files["schema.sql"]
}

func (a *api) reset(ctx context.Context) error {
	if isProduction() {
		return errors.Wrap(errProdReset)
	}

	w := httpjson.ResponseWriter(ctx)
	closeConnOK(w)
	execSelf("RESET=true")
	panic("unreached")
}

func (a *api) info(ctx context.Context) (map[string]interface{}, error) {
	if a.config == nil {
		// never configured
		return map[string]interface{}{
			"is_configured": false,
		}, nil
	}
	if leader.IsLeading() {
		return a.leaderInfo(ctx)
	} else {
		var resp map[string]interface{}
		err := callLeader(ctx, "/info", nil, &resp)
		return resp, err
	}
}

func (a *api) leaderInfo(ctx context.Context) (map[string]interface{}, error) {
	localHeight := a.c.Height()
	var (
		generatorHeight  uint64
		generatorFetched time.Time
	)
	if a.config.IsGenerator {
		generatorHeight = localHeight
		generatorFetched = time.Now()
	} else {
		generatorHeight, generatorFetched = fetch.GeneratorHeight()
	}

	buildCommit := json.RawMessage(expvar.Get("buildcommit").String())
	buildDate := json.RawMessage(expvar.Get("builddate").String())

	// Because everything is asynchronous, it's possible for the localHeight to
	// be higher than our cached generator height. In that case, display the
	// generatorHeight as our height.
	if localHeight > generatorHeight {
		generatorHeight = localHeight
	}

	return map[string]interface{}{
		"is_configured":                     true,
		"configured_at":                     a.config.ConfiguredAt,
		"is_signer":                         a.config.IsSigner,
		"is_generator":                      a.config.IsGenerator,
		"generator_url":                     a.config.GeneratorURL,
		"generator_access_token":            obfuscateTokenSecret(a.config.GeneratorAccessToken),
		"blockchain_id":                     a.config.BlockchainID,
		"block_height":                      localHeight,
		"generator_block_height":            generatorHeight,
		"generator_block_height_fetched_at": generatorFetched,
		"is_production":                     isProduction(),
		"network_rpc_version":               networkRPCVersion,
		"build_commit":                      &buildCommit,
		"build_date":                        &buildDate,
	}, nil
}

// Configure configures the core by writing to the database.
// If running in a cored process,
// the caller must ensure that the new configuration is properly reloaded,
// for example by restarting the process.
//
// If c.IsSigner is true, Configure generates a new mockhsm keypair
// for signing blocks, and assigns it to c.BlockXPub.
//
// If c.IsGenerator is true, Configure creates an initial block,
// saves it, and assigns its hash to c.BlockchainID.
// Otherwise, c.IsGenerator is false, and Configure makes a test request
// to GeneratorURL to detect simple configuration mistakes.
func Configure(ctx context.Context, db pg.DB, c *Config) error {
	var err error
	if !c.IsGenerator {
		err = tryGenerator(
			ctx,
			c.GeneratorURL,
			c.GeneratorAccessToken,
			c.BlockchainID.String(),
		)
		if err != nil {
			return err
		}
	}

	var signingKeys []ed25519.PublicKey
	if c.IsSigner {
		var blockXPub chainkd.XPub
		if c.BlockXPub == "" {
			hsm := mockhsm.New(db)
			coreXPub, created, err := hsm.GetOrCreateKey(ctx, autoBlockKeyAlias)
			if err != nil {
				return err
			}
			blockXPub = coreXPub.XPub
			if created {
				log.Printf("Generated new block-signing key %s\n", blockXPub.String())
			} else {
				log.Printf("Using block-signing key %s\n", blockXPub.String())
			}
			c.BlockXPub = blockXPub.String()
		} else {
			var blockXPub chainkd.XPub
			err = blockXPub.UnmarshalText([]byte(c.BlockXPub))
			if err != nil {
				return errors.Wrap(errBadBlockXPub, err.Error())
			}
		}
		signingKeys = append(signingKeys, blockXPub.PublicKey())
	}

	if c.IsGenerator {
		for _, signer := range c.Signers {
			_, err = url.Parse(signer.URL)
			if err != nil {
				return errors.Wrap(errBadSignerURL, err.Error())
			}
			signingKey, err := chainkd.NewEd25519PublicKey(signer.Pubkey)
			if err != nil {
				return errors.Wrap(errBadSignerPubkey, err.Error())
			}
			signingKeys = append(signingKeys, signingKey)
		}

		if c.Quorum == 0 && len(signingKeys) > 0 {
			return errors.Wrap(errBadQuorum)
		}

		block, err := protocol.NewInitialBlock(signingKeys, c.Quorum, time.Now())
		if err != nil {
			return err
		}
		store, pool := txdb.New(db.(*sql.DB))
		chain, err := protocol.NewChain(ctx, store, pool, nil)
		if err != nil {
			return err
		}

		err = chain.CommitBlock(ctx, block, state.Empty())
		if err != nil {
			return err
		}

		c.BlockchainID = block.Hash()
		chain.MaxIssuanceWindow = c.MaxIssuanceWindow
	}

	var blockSignerData []byte
	if len(c.Signers) > 0 {
		blockSignerData, err = json.Marshal(c.Signers)
		if err != nil {
			return errors.Wrap(err)
		}
	}

	const q = `
		INSERT INTO config (is_signer, block_xpub, is_generator,
			blockchain_id, generator_url, generator_access_token,
			remote_block_signers, max_issuance_window_ms, configured_at)
		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
	`
	_, err = db.Exec(
		ctx,
		q,
		c.IsSigner,
		c.BlockXPub,
		c.IsGenerator,
		c.BlockchainID,
		c.GeneratorURL,
		c.GeneratorAccessToken,
		blockSignerData,
		bc.DurationMillis(c.MaxIssuanceWindow),
	)
	return err
}

func (a *api) configure(ctx context.Context, x *Config) error {
	if a.config != nil {
		return errAlreadyConfigured
	}

	err := Configure(ctx, pg.FromContext(ctx), x)
	if err != nil {
		return err
	}

	w := httpjson.ResponseWriter(ctx)
	closeConnOK(w)
	execSelf()
	panic("unreached")
}

// LoadConfig loads the stored configuration, if any, from the database.
func LoadConfig(ctx context.Context, db pg.DB) (*Config, error) {
	const q = `
			SELECT is_signer, is_generator,
			blockchain_id, generator_url, generator_access_token, block_xpub,
			remote_block_signers, max_issuance_window_ms, configured_at
			FROM config
		`

	c := new(Config)
	var (
		blockSignerData []byte
		miw             int64
	)
	err := db.QueryRow(ctx, q).Scan(
		&c.IsSigner,
		&c.IsGenerator,
		&c.BlockchainID,
		&c.GeneratorURL,
		&c.GeneratorAccessToken,
		&c.BlockXPub,
		&blockSignerData,
		&miw,
		&c.ConfiguredAt,
	)
	if err == sql.ErrNoRows {
		return nil, nil
	} else if err != nil {
		return nil, errors.Wrap(err, "fetching Core config")
	}

	if len(blockSignerData) > 0 {
		err = json.Unmarshal(blockSignerData, &c.Signers)
		if err != nil {
			return nil, errors.Wrap(err)
		}
	}

	c.MaxIssuanceWindow = time.Duration(miw) * time.Millisecond
	return c, nil
}

func tryGenerator(ctx context.Context, url, accessToken, blockchainID string) error {
	client := &rpc.Client{
		BaseURL:      url,
		AccessToken:  accessToken,
		BlockchainID: blockchainID,
	}
	var x struct {
		BlockHeight uint64 `json:"block_height"`
	}
	err := client.Call(ctx, "/rpc/block-height", nil, &x)
	if err != nil {
		return errors.Wrap(errBadGenerator, err.Error())
	}

	if x.BlockHeight < 1 {
		return errBadGenerator
	}

	return nil
}

func closeConnOK(w http.ResponseWriter) {
	w.Header().Add("Connection", "close")
	w.WriteHeader(http.StatusNoContent)

	hijacker, ok := w.(http.Hijacker)
	if !ok {
		log.Printf("no hijacker")
		return
	}
	conn, buf, err := hijacker.Hijack()
	if err != nil {
		log.Printf("could not hijack connection: %s\n", err)
		return
	}
	err = buf.Flush()
	if err != nil {
		log.Printf("could not flush connection buffer: %s\n", err)
	}
	err = conn.Close()
	if err != nil {
		log.Printf("could not close connection: %s\n", err)
	}
}

// execSelf execs Args with environment values replaced
// by the ones in env.
func execSelf(env ...string) {
	binpath, err := exec.LookPath(os.Args[0])
	if err != nil {
		panic(err)
	}

	env = mergeEnvLists(env, os.Environ())
	err = syscall.Exec(binpath, os.Args, env)
	if err != nil {
		panic(err)
	}
}

// mergeEnvLists merges the two environment lists such that
// variables with the same name in "in" replace those in "out".
// This always returns a newly allocated slice.
func mergeEnvLists(in, out []string) []string {
	out = append([]string(nil), out...)
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}

func obfuscateTokenSecret(token string) string {
	toks := strings.SplitN(token, ":", 2)
	var res string
	if len(toks) > 0 {
		res += toks[0]
	}
	if len(toks) > 1 {
		res += ":********"
	}
	return res
}
<commit_msg>core: use chain/log instead of log<commit_after>package core

import (
	"context"
	"encoding/json"
	"expvar"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"chain/core/fetch"
	"chain/core/leader"
	"chain/core/mockhsm"
	"chain/core/txdb"
	"chain/crypto/ed25519"
	"chain/crypto/ed25519/chainkd"
	"chain/database/pg"
	"chain/database/sql"
	"chain/errors"
	"chain/log"
	"chain/net/http/httpjson"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

var (
	errAlreadyConfigured = errors.New("core is already configured; must reset first")
	errUnconfigured      = errors.New("core is not configured")
	errBadGenerator      = errors.New("generator returned an unsuccessful response")
	errBadBlockXPub      = errors.New("supplied block xpub is invalid")
	errNoClientTokens    = errors.New("cannot enable client auth without client access tokens")
	errBadSignerURL      = errors.New("block signer URL is invalid")
	errBadSignerPubkey   = errors.New("block signer pubkey is invalid")
	errBadQuorum         = errors.New("quorum must be greater than 0 if there are signers")
	// errProdReset is returned when reset is called on a
	// production system.
	errProdReset = errors.New("reset called on production system")
)

// reserved mockhsm key alias
const (
	networkRPCVersion = 1
	autoBlockKeyAlias = "_CHAIN_CORE_AUTO_BLOCK_KEY"
)

func isProduction() bool {
	bt := expvar.Get("buildtag")
	return bt != nil && bt.String() != `"dev"`
}

// Schema returns DDL for the entire Chain Core Postgres schema.
func Schema() string {
	return files["schema.sql"]
}

func (a *api) reset(ctx context.Context) error {
	if isProduction() {
		return errors.Wrap(errProdReset)
	}

	closeConnOK(httpjson.ResponseWriter(ctx), httpjson.Request(ctx))
	execSelf("RESET=true")
	panic("unreached")
}

func (a *api) info(ctx context.Context) (map[string]interface{}, error) {
	if a.config == nil {
		// never configured
		return map[string]interface{}{
			"is_configured": false,
		}, nil
	}
	if leader.IsLeading() {
		return a.leaderInfo(ctx)
	} else {
		var resp map[string]interface{}
		err := callLeader(ctx, "/info", nil, &resp)
		return resp, err
	}
}

func (a *api) leaderInfo(ctx context.Context) (map[string]interface{}, error) {
	localHeight := a.c.Height()
	var (
		generatorHeight  uint64
		generatorFetched time.Time
	)
	if a.config.IsGenerator {
		generatorHeight = localHeight
		generatorFetched = time.Now()
	} else {
		generatorHeight, generatorFetched = fetch.GeneratorHeight()
	}

	buildCommit := json.RawMessage(expvar.Get("buildcommit").String())
	buildDate := json.RawMessage(expvar.Get("builddate").String())

	// Because everything is asynchronous, it's possible for the localHeight to
	// be higher than our cached generator height. In that case, display the
	// generatorHeight as our height.
	if localHeight > generatorHeight {
		generatorHeight = localHeight
	}

	return map[string]interface{}{
		"is_configured":                     true,
		"configured_at":                     a.config.ConfiguredAt,
		"is_signer":                         a.config.IsSigner,
		"is_generator":                      a.config.IsGenerator,
		"generator_url":                     a.config.GeneratorURL,
		"generator_access_token":            obfuscateTokenSecret(a.config.GeneratorAccessToken),
		"blockchain_id":                     a.config.BlockchainID,
		"block_height":                      localHeight,
		"generator_block_height":            generatorHeight,
		"generator_block_height_fetched_at": generatorFetched,
		"is_production":                     isProduction(),
		"network_rpc_version":               networkRPCVersion,
		"build_commit":                      &buildCommit,
		"build_date":                        &buildDate,
	}, nil
}

// Configure configures the core by writing to the database.
// If running in a cored process,
// the caller must ensure that the new configuration is properly reloaded,
// for example by restarting the process.
//
// If c.IsSigner is true, Configure generates a new mockhsm keypair
// for signing blocks, and assigns it to c.BlockXPub.
//
// If c.IsGenerator is true, Configure creates an initial block,
// saves it, and assigns its hash to c.BlockchainID.
// Otherwise, c.IsGenerator is false, and Configure makes a test request
// to GeneratorURL to detect simple configuration mistakes.
func Configure(ctx context.Context, db pg.DB, c *Config) error {
	var err error
	if !c.IsGenerator {
		err = tryGenerator(
			ctx,
			c.GeneratorURL,
			c.GeneratorAccessToken,
			c.BlockchainID.String(),
		)
		if err != nil {
			return err
		}
	}

	var signingKeys []ed25519.PublicKey
	if c.IsSigner {
		var blockXPub chainkd.XPub
		if c.BlockXPub == "" {
			hsm := mockhsm.New(db)
			coreXPub, created, err := hsm.GetOrCreateKey(ctx, autoBlockKeyAlias)
			if err != nil {
				return err
			}
			blockXPub = coreXPub.XPub
			if created {
				log.Messagef(ctx, "Generated new block-signing key %s\n", blockXPub.String())
			} else {
				log.Messagef(ctx, "Using block-signing key %s\n", blockXPub.String())
			}
			c.BlockXPub = blockXPub.String()
		} else {
			var blockXPub chainkd.XPub
			err = blockXPub.UnmarshalText([]byte(c.BlockXPub))
			if err != nil {
				return errors.Wrap(errBadBlockXPub, err.Error())
			}
		}
		signingKeys = append(signingKeys, blockXPub.PublicKey())
	}

	if c.IsGenerator {
		for _, signer := range c.Signers {
			_, err = url.Parse(signer.URL)
			if err != nil {
				return errors.Wrap(errBadSignerURL, err.Error())
			}
			signingKey, err := chainkd.NewEd25519PublicKey(signer.Pubkey)
			if err != nil {
				return errors.Wrap(errBadSignerPubkey, err.Error())
			}
			signingKeys = append(signingKeys, signingKey)
		}

		if c.Quorum == 0 && len(signingKeys) > 0 {
			return errors.Wrap(errBadQuorum)
		}

		block, err := protocol.NewInitialBlock(signingKeys, c.Quorum, time.Now())
		if err != nil {
			return err
		}
		store, pool := txdb.New(db.(*sql.DB))
		chain, err := protocol.NewChain(ctx, store, pool, nil)
		if err != nil {
			return err
		}

		err = chain.CommitBlock(ctx, block, state.Empty())
		if err != nil {
			return err
		}

		c.BlockchainID = block.Hash()
		chain.MaxIssuanceWindow = c.MaxIssuanceWindow
	}

	var blockSignerData []byte
	if len(c.Signers) > 0 {
		blockSignerData, err = json.Marshal(c.Signers)
		if err != nil {
			return errors.Wrap(err)
		}
	}

	const q = `
		INSERT INTO config (is_signer, block_xpub, is_generator,
			blockchain_id, generator_url, generator_access_token,
			remote_block_signers, max_issuance_window_ms, configured_at)
		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
	`
	_, err = db.Exec(
		ctx,
		q,
		c.IsSigner,
		c.BlockXPub,
		c.IsGenerator,
		c.BlockchainID,
		c.GeneratorURL,
		c.GeneratorAccessToken,
		blockSignerData,
		bc.DurationMillis(c.MaxIssuanceWindow),
	)
	return err
}

func (a *api) configure(ctx context.Context, x *Config) error {
	if a.config != nil {
		return errAlreadyConfigured
	}

	err := Configure(ctx, pg.FromContext(ctx), x)
	if err != nil {
		return err
	}

	closeConnOK(httpjson.ResponseWriter(ctx), httpjson.Request(ctx))
	execSelf()
	panic("unreached")
}

// LoadConfig loads the stored configuration, if any, from the database.
func LoadConfig(ctx context.Context, db pg.DB) (*Config, error) {
	const q = `
			SELECT is_signer, is_generator,
			blockchain_id, generator_url, generator_access_token, block_xpub,
			remote_block_signers, max_issuance_window_ms, configured_at
			FROM config
		`

	c := new(Config)
	var (
		blockSignerData []byte
		miw             int64
	)
	err := db.QueryRow(ctx, q).Scan(
		&c.IsSigner,
		&c.IsGenerator,
		&c.BlockchainID,
		&c.GeneratorURL,
		&c.GeneratorAccessToken,
		&c.BlockXPub,
		&blockSignerData,
		&miw,
		&c.ConfiguredAt,
	)
	if err == sql.ErrNoRows {
		return nil, nil
	} else if err != nil {
		return nil, errors.Wrap(err, "fetching Core config")
	}

	if len(blockSignerData) > 0 {
		err = json.Unmarshal(blockSignerData, &c.Signers)
		if err != nil {
			return nil, errors.Wrap(err)
		}
	}

	c.MaxIssuanceWindow = time.Duration(miw) * time.Millisecond
	return c, nil
}

func tryGenerator(ctx context.Context, url, accessToken, blockchainID string) error {
	client := &rpc.Client{
		BaseURL:      url,
		AccessToken:  accessToken,
		BlockchainID: blockchainID,
	}
	var x struct {
		BlockHeight uint64 `json:"block_height"`
	}
	err := client.Call(ctx, "/rpc/block-height", nil, &x)
	if err != nil {
		return errors.Wrap(errBadGenerator, err.Error())
	}

	if x.BlockHeight < 1 {
		return errBadGenerator
	}

	return nil
}

func closeConnOK(w http.ResponseWriter, req *http.Request) {
	w.Header().Add("Connection", "close")
	w.WriteHeader(http.StatusNoContent)

	hijacker, ok := w.(http.Hijacker)
	if !ok {
		log.Messagef(req.Context(), "no hijacker")
		return
	}
	conn, buf, err := hijacker.Hijack()
	if err != nil {
		log.Messagef(req.Context(), "could not hijack connection: %s\n", err)
		return
	}
	err = buf.Flush()
	if err != nil {
		log.Messagef(req.Context(), "could not flush connection buffer: %s\n", err)
	}
	err = conn.Close()
	if err != nil {
		log.Messagef(req.Context(), "could not close connection: %s\n", err)
	}
}

// execSelf execs Args with environment values replaced
// by the ones in env.
func execSelf(env ...string) {
	binpath, err := exec.LookPath(os.Args[0])
	if err != nil {
		panic(err)
	}

	env = mergeEnvLists(env, os.Environ())
	err = syscall.Exec(binpath, os.Args, env)
	if err != nil {
		panic(err)
	}
}

// mergeEnvLists merges the two environment lists such that
// variables with the same name in "in" replace those in "out".
// This always returns a newly allocated slice.
func mergeEnvLists(in, out []string) []string {
	out = append([]string(nil), out...)
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}

func obfuscateTokenSecret(token string) string {
	toks := strings.SplitN(token, ":", 2)
	var res string
	if len(toks) > 0 {
		res += toks[0]
	}
	if len(toks) > 1 {
		res += ":********"
	}
	return res
}
<|endoftext|>


<commit_before>package core

import (
	"strconv"
	"time"

	m "github.com/AntoineAugusti/fake-google/models"
)

// Find the first result coming from a pool of search servers
func First(query string, replicas ...m.Search) m.Result {
	c := make(chan m.Result)
	// Cancellation channel. See https://blog.golang.org/pipelines
	done := make(chan struct{})
	// Close that channel when this pipeline exits, as a signal
	// for all the goroutines we started to exit.
	defer close(done)

	searchReplica := func(i int) {
		select {
		case c <- replicas[i].Search(query):
		case <-done:
		}
	}

	for i := range replicas {
		go searchReplica(i)
	}

	return <-c
}

// Create nb numbers of servers of a given type
func CreateServers(serverType string, nb, latency int) []m.Search {
	var servers []m.Search

	for i := 1; i <= nb; i++ {
		id := strconv.Itoa(i)
		servers = append(servers, m.NewSearchServer(serverType+id, latency))
	}

	return servers
}

func Google(query string, nbReplicas, timeoutValue, latency int) (results m.Results) {

	services := []string{"web", "image", "video"}

	// We expect a response from each service we have
	resultChannel := make(chan m.Result, len(services))

	// Run the search query on multiple instances of each service
	for _, serviceName := range services {
		servers := CreateServers(serviceName, nbReplicas, latency)
		go func() {
			resultChannel <- First(query, servers...)
		}()
	}

	// Define the timeout for a search query
	timeout := time.After(time.Duration(timeoutValue) * time.Millisecond)

	// Go find a result for each service
	for i := 0; i < 3; i++ {
		select {
		case result := <-resultChannel:
			results = append(results, result)
		// Exit if we've been waiting for too long to have a search result
		case <-timeout:
			return
		}
	}

	return
}
<commit_msg>Kill magic number and add comment<commit_after>package core

import (
	"strconv"
	"time"

	m "github.com/AntoineAugusti/fake-google/models"
)

// Find the first result coming from a pool of search servers
func First(query string, replicas ...m.Search) m.Result {
	c := make(chan m.Result)
	// Cancellation channel. See https://blog.golang.org/pipelines
	done := make(chan struct{})
	// Close that channel when this pipeline exits, as a signal
	// for all the goroutines we started to exit.
	defer close(done)

	searchReplica := func(i int) {
		select {
		case c <- replicas[i].Search(query):
		case <-done:
		}
	}

	for i := range replicas {
		go searchReplica(i)
	}

	return <-c
}

// Create nb numbers of servers of a given type
func CreateServers(serverType string, nb, latency int) []m.Search {
	var servers []m.Search

	for i := 1; i <= nb; i++ {
		id := strconv.Itoa(i)
		servers = append(servers, m.NewSearchServer(serverType+id, latency))
	}

	return servers
}

func Google(query string, nbReplicas, timeoutValue, latency int) (results m.Results) {

	services := []string{"web", "image", "video"}

	// We expect a response from each service we have
	resultChannel := make(chan m.Result, len(services))

	// Run the search query on multiple instances of each service
	for _, serviceName := range services {
		// This is plain stupid: we don't want to "create" servers
		// everytime we receive a search query. Instead we would
		// likely fetch here the available servers for the given
		// service
		servers := CreateServers(serviceName, nbReplicas, latency)
		go func() {
			resultChannel <- First(query, servers...)
		}()
	}

	// Define the timeout for a search query
	timeout := time.After(time.Duration(timeoutValue) * time.Millisecond)

	// Go find a result for each service
	for i := 0; i < len(services); i++ {
		select {
		case result := <-resultChannel:
			results = append(results, result)
		// Exit if we've been waiting for too long to have a search result
		case <-timeout:
			return
		}
	}

	return
}
<|endoftext|>


<commit_before>package GoSDK

const (
	_PORTALS_DEV_PREAMBLE  = "/admin/portals/"
	_PORTALS_USER_PREAMBLE = "/api/v/2/portals/"
)

func (d *DevClient) GetPortals(systemKey string) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _PORTALS_DEV_PREAMBLE+systemKey, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) GetPortal(systemKey, name string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) CreatePortal(systemKey, name string, dash map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}

	resp, err := post(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, make(map[string]interface{}), creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	// An empty portal has now been created.. populate with provided data
	return d.UpdatePortal(systemKey, name, dash)
}

func (d *DevClient) UpdatePortal(systemKey, name string, dash map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := put(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, dash, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) DeletePortal(systemKey, name string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	resp, err := delete(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>create portal with data passed in rather than create empty portal than try to update it with data<commit_after>package GoSDK

const (
	_PORTALS_DEV_PREAMBLE  = "/admin/portals/"
	_PORTALS_USER_PREAMBLE = "/api/v/2/portals/"
)

func (d *DevClient) GetPortals(systemKey string) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _PORTALS_DEV_PREAMBLE+systemKey, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) GetPortal(systemKey, name string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) CreatePortal(systemKey, name string, dash map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}

	resp, err := post(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, dash, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
}

func (d *DevClient) UpdatePortal(systemKey, name string, dash map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := put(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, dash, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) DeletePortal(systemKey, name string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	resp, err := delete(d, _PORTALS_USER_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/go-contrib/uuid"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func PreReceive(dir, oldrev, newrev, ref string) error {
	// We only care about master
	if ref != "refs/heads/master" {
		return nil
	}

	temp := filepath.Join(os.TempDir(), uuid.NewV4().String())
	err := os.Mkdir(temp, 0x777)
	if err != nil {
		return fmt.Errorf("- failed to create directory: %v", err)
	}
	//defer os.RemoveAll(temp)

	// Export to temp
	log.Println("exporting", dir, newrev, "to", temp)
	os.Chdir(dir)
	bs, err := exec.Command("bash", "-c", "git archive --format=tar master | tar -C "+temp+" -x ").CombinedOutput()
	if err != nil {
		return fmt.Errorf("- failed to export: %s", bs)
	}
	log.Println("-", string(bs))

	// Build
	log.Println("building")
	os.Chdir(temp)
	bs, err = exec.Command("go", "build", "-v").CombinedOutput()
	if err != nil {
		return fmt.Errorf("- failed to build: %s", bs)
	}
	log.Println("-", string(bs))

	// Clean
	log.Println("cleaning")
	filepath.Walk(temp, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if strings.HasPrefix(filepath.Base(path), ".") {
				os.RemoveAll(path)
				return filepath.SkipDir
			}
		} else {
			if strings.HasSuffix(path, ".go") {
				os.Remove(path)
			}
		}

		return nil
	})
	// Sync to endpoints
	// Disable app in load balancer
	// Swap to new version
	// Start the app
	// Enable app in load balancer
	// Cleanup old versions on the endpoints

	return fmt.Errorf("Not Implemented")
}
<commit_msg>fix permissions<commit_after>package main

import (
	"fmt"
	"github.com/go-contrib/uuid"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func PreReceive(dir, oldrev, newrev, ref string) error {
	// We only care about master
	if ref != "refs/heads/master" {
		return nil
	}

	temp := filepath.Join(os.TempDir(), uuid.NewV4().String())
	err := os.Mkdir(temp, 0777)
	if err != nil {
		return fmt.Errorf("- failed to create directory: %v", err)
	}
	//defer os.RemoveAll(temp)

	// Export to temp
	log.Println("exporting", dir, newrev, "to", temp)
	os.Chdir(dir)
	bs, err := exec.Command("bash", "-c", "git archive --format=tar master | tar -C "+temp+" -x ").CombinedOutput()
	if err != nil {
		return fmt.Errorf("- failed to export: %s", bs)
	}
	log.Println("-", string(bs))

	// Build
	log.Println("building")
	os.Chdir(temp)
	bs, err = exec.Command("go", "build", "-v").CombinedOutput()
	if err != nil {
		return fmt.Errorf("- failed to build: %s", bs)
	}
	log.Println("-", string(bs))

	// Clean
	log.Println("cleaning")
	filepath.Walk(temp, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			if strings.HasPrefix(filepath.Base(path), ".") {
				os.RemoveAll(path)
				return filepath.SkipDir
			}
		} else {
			if strings.HasSuffix(path, ".go") {
				os.Remove(path)
			}
		}

		return nil
	})
	// Sync to endpoints
	// Disable app in load balancer
	// Swap to new version
	// Start the app
	// Enable app in load balancer
	// Cleanup old versions on the endpoints

	return fmt.Errorf("Not Implemented")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"net/http"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

const numGoroutines = 10
const maxSleepTimeSecs = 1

var urls = []string{
	"https://groupon.com",
	"https://groupon.com/getaways",
	"https://groupon.com/goods",
	"http://google.com",
	"http://amazon.com",
	"http://yahoo.com",
	"http://microsoft.com",
	"http://example.com",
	"http://unreachable.com/noexist",
}

func main() {
	demo1()
	// demo2()
	// demo3()
}

func demo1() {
	ch := make(chan string)

	for _, url := range urls {
		go func(url string) {
			ch <- httpGetCheck(url)
		}(url)
	}

	for i := 0; i < len(urls); i++ {
		fmt.Println(<-ch)
	}

	fmt.Println("Done!")
}

func httpGetCheck(url string) string {
	response, err := http.Get(url)
	if err != nil || response.StatusCode != http.StatusOK {
		return fmt.Sprintf("*** (%v) is NOT 200 OK", url)
	}

	return fmt.Sprintf("    (%v) is 200 OK", url)
}

func demo2() {
	ch := make(chan string)

	for _, url := range urls {
		go func(url string) {
			ch <- httpGetCheck(url)
		}(url)
	}

	secondMarks := ticker(1 * time.Second)
	timeout := time.After(6 * time.Second)

	done := false
	for messagesReceived := 0; messagesReceived < len(urls) && !done; {
		select {
		case message := <-ch:
			fmt.Println(message)
			messagesReceived++
		case tick := <-secondMarks:
			fmt.Printf("======== %v =======\n", tick)
		case <-timeout:
			fmt.Println("******** Bored ********")
			done = true
		}
	}

	fmt.Println("Done!")
}

func ticker(period time.Duration) chan int {
	ch := make(chan int)

	go func() {
		i := 0
		for {
			ch <- i
			time.Sleep(period)
			i++
		}
	}()

	return ch
}

func demo3() {
	jeff := steadySpeaker(400*time.Millisecond, "jeff")
	sara := steadySpeaker(600*time.Millisecond, "\tsara")
	pickles := steadySpeaker(1000*time.Millisecond, "\t\tpickles")
	random := randomSpeaker()
	timeout := time.After(5 * time.Second)

	done := false
	for !done {
		select {
		case phrase := <-jeff:
			fmt.Printf("phrase: %v\n", phrase)
		case phrase := <-sara:
			fmt.Printf("phrase: %v\n", phrase)
		case phrase := <-pickles:
			fmt.Printf("phrase: %v\n", phrase)
		case phrase := <-random:
			fmt.Printf("phrase: \t\t\t\t%v\n", phrase)
		case <-timeout:
			done = true
		}
	}

	fmt.Println("Enough!")
}

func steadySpeaker(period time.Duration, phrase string) chan string {
	ch := make(chan string)

	go func() {
		for {
			time.Sleep(period)
			ch <- phrase
		}
	}()

	return ch
}

func randomSpeaker() chan int {
	ch := make(chan int)

	go func() {
		for i := 0; i < 1000; i++ {
			time.Sleep(time.Duration(rand.Intn(2000)) * time.Millisecond)
			ch <- i
		}
	}()

	return ch
}
<commit_msg>change some variable names<commit_after>package main

import (
	"fmt"
	"math/rand"
	"net/http"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

const numGoroutines = 10
const maxSleepTimeSecs = 1

var urls = []string{
	"https://groupon.com",
	"https://groupon.com/getaways",
	"https://groupon.com/goods",
	"http://google.com",
	"http://amazon.com",
	"http://yahoo.com",
	"http://microsoft.com",
	"http://example.com",
	"http://unreachable.com/noexist",
}

func main() {
	demo1()
	// demo2()
	// demo3()
}

func demo1() {
	messages := make(chan string)

	for _, url := range urls {
		go func(url string) {
			messages <- httpGetCheck(url)
		}(url)
	}

	for i := 0; i < len(urls); i++ {
		fmt.Println(<-messages)
	}

	fmt.Println("Done!")
}

func httpGetCheck(url string) string {
	response, err := http.Get(url)
	if err != nil || response.StatusCode != http.StatusOK {
		return fmt.Sprintf("*** (%v) is NOT 200 OK", url)
	}

	return fmt.Sprintf("    (%v) is 200 OK", url)
}

func demo2() {
	messages := make(chan string)

	for _, url := range urls {
		go func(url string) {
			messages <- httpGetCheck(url)
		}(url)
	}

	secondMarks := ticker(1 * time.Second)
	timeout := time.After(6 * time.Second)

	done := false
	for messagesReceived := 0; messagesReceived < len(urls) && !done; {
		select {
		case message := <-messages:
			fmt.Println(message)
			messagesReceived++
		case tick := <-secondMarks:
			fmt.Printf("======== %v =======\n", tick)
		case <-timeout:
			fmt.Println("******** Bored ********")
			done = true
		}
	}

	fmt.Println("Done!")
}

func ticker(period time.Duration) chan int {
	ch := make(chan int)

	go func() {
		i := 0
		for {
			ch <- i
			time.Sleep(period)
			i++
		}
	}()

	return ch
}

func demo3() {
	jeff := steadySpeaker(400*time.Millisecond, "jeff")
	sara := steadySpeaker(600*time.Millisecond, "\tsara")
	pickles := steadySpeaker(1000*time.Millisecond, "\t\tpickles")
	random := randomSpeaker()
	timeout := time.After(5 * time.Second)

	done := false
	for !done {
		select {
		case phrase := <-jeff:
			fmt.Printf("phrase: %v\n", phrase)
		case phrase := <-sara:
			fmt.Printf("phrase: %v\n", phrase)
		case phrase := <-pickles:
			fmt.Printf("phrase: %v\n", phrase)
		case phrase := <-random:
			fmt.Printf("phrase: \t\t\t\t%v\n", phrase)
		case <-timeout:
			done = true
		}
	}

	fmt.Println("Enough!")
}

func steadySpeaker(period time.Duration, phrase string) chan string {
	ch := make(chan string)

	go func() {
		for {
			time.Sleep(period)
			ch <- phrase
		}
	}()

	return ch
}

func randomSpeaker() chan int {
	ch := make(chan int)

	go func() {
		for i := 0; i < 1000; i++ {
			time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
			ch <- i
		}
	}()

	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"

	"github.com/Financial-Times/base-ft-rw-app-go"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/http-handlers-go"
	"github.com/Financial-Times/public-organisations-api/organisations"

	"fmt"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
)

func main() {
	log.Infof("Application starting with args %s", os.Args)
	app := cli.App("public-organisations-api-neo4j", "A public RESTful API for accessing organisations in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	//neoURL := app.StringOpt("neo-url", "http://ftper60304-law1a-eu-t:8080/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	env := app.StringOpt("env", "local", "environment this app is running in")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.public.organisations.api.ftaps59382-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")
	cacheDuration := app.StringOpt("cache-duration", "1h", "Duration Get requests should be cached for. e.g. 2h45m would set the max-age value to '7440' seconds")

	app.Action = func() {
		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)
		if *env != "local" {
			f, err := os.OpenFile("/var/log/apps/public-organisations-api-go-app.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
			if err == nil {
				log.SetOutput(f)

			} else {
				log.Fatalf("Failed to initialise log file, %v", err)
			}
			defer f.Close()
		}

		log.Infof("public-organisations-api will listen on port: %s, connecting to: %s", *port, *neoURL)

		runServer(*neoURL, *port, *cacheDuration, *env)

	}
	log.SetFormatter(&log.TextFormatter{})
	log.SetLevel(log.InfoLevel)
	log.Infof("Application started with args %s", os.Args)
	app.Run(os.Args)
}

func runServer(neoURL string, port string, cacheDuration string, env string) {

	if duration, durationErr := time.ParseDuration(cacheDuration); durationErr != nil {
		log.Fatalf("Failed to parse cache duration string, %v", durationErr)
	} else {
		organisations.CacheControlHeader = fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(duration.Seconds(), 'f', 0, 64))
	}

	db, err := neoism.Connect(neoURL)
	db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
	if err != nil {
		log.Fatalf("Error connecting to neo4j %s", err)
	}

	organisations.OrganisationDriver = organisations.NewCypherDriver(db, env)

	servicesRouter := mux.NewRouter()

	// Healthchecks and standards first
	servicesRouter.HandleFunc("/__health", v1a.Handler("PublicOrganisationsRead Healthchecks",
		"Checks for accessing neo4j", organisations.HealthCheck()))
	servicesRouter.HandleFunc("/ping", organisations.Ping)
	servicesRouter.HandleFunc("/__ping", organisations.Ping)
	servicesRouter.HandleFunc("/__gtg", organisations.GoodToGo)

	// Then API specific ones:
	servicesRouter.HandleFunc("/organisations/{uuid}", organisations.GetOrganisation).Methods("GET")

	servicesRouter.HandleFunc("/organisations/{uuid}", organisations.MethodNotAllowedHandler)

	var monitoringRouter http.Handler = servicesRouter
	monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
	monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

	// The top one of these feels more correct, but the lower one matches what we have in Dropwizard,
	// so it's what apps expect currently same as ping, the content of build-info needs more definition
	//using http router here to be able to catch "/"
	http.HandleFunc("/__build-info", organisations.BuildInfoHandler)
	http.HandleFunc("/build-info", organisations.BuildInfoHandler)
	http.Handle("/", monitoringRouter)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

}
<commit_msg>Fixed breaking change from base-rw package re-work<commit_after>package main

import (
	"net/http"
	"os"

	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/http-handlers-go"
	"github.com/Financial-Times/public-organisations-api/organisations"

	"fmt"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
)

func main() {
	log.Infof("Application starting with args %s", os.Args)
	app := cli.App("public-organisations-api-neo4j", "A public RESTful API for accessing organisations in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	//neoURL := app.StringOpt("neo-url", "http://ftper60304-law1a-eu-t:8080/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	env := app.StringOpt("env", "local", "environment this app is running in")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.public.organisations.api.ftaps59382-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")
	cacheDuration := app.StringOpt("cache-duration", "1h", "Duration Get requests should be cached for. e.g. 2h45m would set the max-age value to '7440' seconds")

	app.Action = func() {
		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)
		if *env != "local" {
			f, err := os.OpenFile("/var/log/apps/public-organisations-api-go-app.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
			if err == nil {
				log.SetOutput(f)

			} else {
				log.Fatalf("Failed to initialise log file, %v", err)
			}
			defer f.Close()
		}

		log.Infof("public-organisations-api will listen on port: %s, connecting to: %s", *port, *neoURL)

		runServer(*neoURL, *port, *cacheDuration, *env)

	}
	log.SetFormatter(&log.TextFormatter{})
	log.SetLevel(log.InfoLevel)
	log.Infof("Application started with args %s", os.Args)
	app.Run(os.Args)
}

func runServer(neoURL string, port string, cacheDuration string, env string) {

	if duration, durationErr := time.ParseDuration(cacheDuration); durationErr != nil {
		log.Fatalf("Failed to parse cache duration string, %v", durationErr)
	} else {
		organisations.CacheControlHeader = fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(duration.Seconds(), 'f', 0, 64))
	}

	db, err := neoism.Connect(neoURL)
	db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
	if err != nil {
		log.Fatalf("Error connecting to neo4j %s", err)
	}

	organisations.OrganisationDriver = organisations.NewCypherDriver(db, env)

	servicesRouter := mux.NewRouter()

	// Healthchecks and standards first
	servicesRouter.HandleFunc("/__health", v1a.Handler("PublicOrganisationsRead Healthchecks",
		"Checks for accessing neo4j", organisations.HealthCheck()))
	servicesRouter.HandleFunc("/ping", organisations.Ping)
	servicesRouter.HandleFunc("/__ping", organisations.Ping)
	servicesRouter.HandleFunc("/__gtg", organisations.GoodToGo)

	// Then API specific ones:
	servicesRouter.HandleFunc("/organisations/{uuid}", organisations.GetOrganisation).Methods("GET")

	servicesRouter.HandleFunc("/organisations/{uuid}", organisations.MethodNotAllowedHandler)

	var monitoringRouter http.Handler = servicesRouter
	monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
	monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

	// The top one of these feels more correct, but the lower one matches what we have in Dropwizard,
	// so it's what apps expect currently same as ping, the content of build-info needs more definition
	//using http router here to be able to catch "/"
	http.HandleFunc("/__build-info", organisations.BuildInfoHandler)
	http.HandleFunc("/build-info", organisations.BuildInfoHandler)
	http.Handle("/", monitoringRouter)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

}
<|endoftext|>


<commit_before>package hcl2template

import (
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/hashicorp/packer/builder/null"
	. "github.com/hashicorp/packer/hcl2template/internal"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/zclconf/go-cty/cty"
)

func getBasicParser() *Parser {
	return &Parser{
		Parser: hclparse.NewParser(),
		BuilderSchemas: packer.MapOfBuilder{
			"amazon-ebs":     func() (packer.Builder, error) { return &MockBuilder{}, nil },
			"virtualbox-iso": func() (packer.Builder, error) { return &MockBuilder{}, nil },
			"null":           func() (packer.Builder, error) { return &null.Builder{}, nil },
		},
		ProvisionersSchemas: packer.MapOfProvisioner{
			"shell": func() (packer.Provisioner, error) { return &MockProvisioner{}, nil },
			"file":  func() (packer.Provisioner, error) { return &MockProvisioner{}, nil },
		},
		PostProcessorsSchemas: packer.MapOfPostProcessor{
			"amazon-import": func() (packer.PostProcessor, error) { return &MockPostProcessor{}, nil },
			"manifest":      func() (packer.PostProcessor, error) { return &MockPostProcessor{}, nil },
		},
	}
}

type parseTestArgs struct {
	filename string
	vars     map[string]string
	varFiles []string
}

type parseTest struct {
	name   string
	parser *Parser
	args   parseTestArgs

	parseWantCfg           *PackerConfig
	parseWantDiags         bool
	parseWantDiagHasErrors bool

	getBuildsWantBuilds []packer.Build
	getBuildsWantDiags  bool
	// getBuildsWantDiagHasErrors bool
}

func testParse(t *testing.T, tests []parseTest) {
	t.Helper()

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotCfg, gotDiags := tt.parser.Parse(tt.args.filename, tt.args.varFiles, tt.args.vars)
			moreDiags := gotCfg.Initialize()
			gotDiags = append(gotDiags, moreDiags...)
			if tt.parseWantDiags == (gotDiags == nil) {
				t.Fatalf("Parser.parse() unexpected %q diagnostics.", gotDiags)
			}
			if tt.parseWantDiagHasErrors != gotDiags.HasErrors() {
				t.Fatalf("Parser.parse() unexpected diagnostics HasErrors. %s", gotDiags)
			}
			if diff := cmp.Diff(tt.parseWantCfg, gotCfg, cmpOpts...); diff != "" {
				t.Fatalf("Parser.parse() wrong packer config. %s", diff)
			}

			if gotCfg != nil && !tt.parseWantDiagHasErrors {
				gotInputVar := gotCfg.InputVariables
				for name, value := range tt.parseWantCfg.InputVariables {
					if variable, ok := gotInputVar[name]; ok {
						if diff := cmp.Diff(variable, value, cmpOpts...); diff != "" {
							t.Fatalf("Parser.parse(): unexpected variable values %s: %s", name, diff)
						}
					} else {
						t.Fatalf("Parser.parse() missing input variable. %s", name)
					}
				}

				gotLocalVar := gotCfg.LocalVariables
				for name, value := range tt.parseWantCfg.LocalVariables {
					if variable, ok := gotLocalVar[name]; ok {
						if diff := cmp.Diff(variable, value, cmpOpts...); diff != "" {
							t.Fatalf("Parser.parse(): unexpected variable values %s: %s", name, diff)
						}
					} else {
						t.Fatalf("Parser.parse() missing local variable. %s", name)
					}
				}
			}

			if gotDiags.HasErrors() {
				return
			}

			gotBuilds, gotDiags := gotCfg.GetBuilds(packer.GetBuildsOptions{})
			if tt.getBuildsWantDiags == (gotDiags == nil) {
				t.Fatalf("Parser.getBuilds() unexpected diagnostics. %s", gotDiags)
			}
			if diff := cmp.Diff(tt.getBuildsWantBuilds, gotBuilds, cmpOpts...); diff != "" {
				t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff)
			}
		})
	}
}

var (
	// everything in the tests is a basicNestedMockConfig this allow to test
	// each known type to packer ( and embedding ) in one go.
	basicNestedMockConfig = NestedMockConfig{
		String:   "string",
		Int:      42,
		Int64:    43,
		Bool:     true,
		Trilean:  config.TriTrue,
		Duration: 10 * time.Second,
		MapStringString: map[string]string{
			"a": "b",
			"c": "d",
		},
		SliceString: []string{
			"a",
			"b",
			"c",
		},
		SliceSliceString: [][]string{
			{"a", "b"},
			{"c", "d"},
		},
		Tags: []MockTag{},
	}

	basicMockBuilder = &MockBuilder{
		Config: MockConfig{
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				basicNestedMockConfig,
				basicNestedMockConfig,
			},
		},
	}

	basicMockProvisioner = &MockProvisioner{
		Config: MockConfig{
			NotSquashed:      "value <UNKNOWN>",
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				{
					Tags: dynamicTagList,
				},
			},
		},
	}
	basicMockPostProcessor = &MockPostProcessor{
		Config: MockConfig{
			NotSquashed:      "value <UNKNOWN>",
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				{
					Tags: []MockTag{},
				},
			},
		},
	}
	basicMockCommunicator = &MockCommunicator{
		Config: MockConfig{
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				{
					Tags: []MockTag{},
				},
			},
		},
	}

	emptyMockBuilder = &MockBuilder{
		Config: MockConfig{
			NestedMockConfig: NestedMockConfig{
				Tags: []MockTag{},
			},
			Nested:      NestedMockConfig{},
			NestedSlice: []NestedMockConfig{},
		},
	}

	emptyMockProvisioner = &MockProvisioner{
		Config: MockConfig{
			NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
			NestedSlice:      []NestedMockConfig{},
		},
	}

	dynamicTagList = []MockTag{
		{
			Key:   "first_tag_key",
			Value: "first_tag_value",
		},
		{
			Key:   "Component",
			Value: "user-service",
		},
		{
			Key:   "Environment",
			Value: "production",
		},
	}
)

var ctyValueComparer = cmp.Comparer(func(x, y cty.Value) bool {
	return x.RawEquals(y)
})

var ctyTypeComparer = cmp.Comparer(func(x, y cty.Type) bool {
	if x == cty.NilType && y == cty.NilType {
		return true
	}
	if x == cty.NilType || y == cty.NilType {
		return false
	}
	return x.Equals(y)
})

var cmpOpts = []cmp.Option{
	ctyValueComparer,
	ctyTypeComparer,
	cmpopts.IgnoreUnexported(
		PackerConfig{},
		Variable{},
		SourceBlock{},
		ProvisionerBlock{},
		PostProcessorBlock{},
		packer.CoreBuild{},
		HCL2Provisioner{},
		HCL2PostProcessor{},
		packer.CoreBuildPostProcessor{},
		packer.CoreBuildProvisioner{},
		packer.CoreBuildPostProcessor{},
		null.Builder{},
	),
	cmpopts.IgnoreFields(PackerConfig{},
		"Cwd", // Cwd will change for every os type
	),
	cmpopts.IgnoreFields(VariableAssignment{},
		"Expr", // its an interface
	),
	cmpopts.IgnoreTypes(HCL2Ref{}),
	cmpopts.IgnoreTypes([]*LocalBlock{}),
	cmpopts.IgnoreTypes([]hcl.Range{}),
	cmpopts.IgnoreTypes(hcl.Range{}),
	cmpopts.IgnoreInterfaces(struct{ hcl.Expression }{}),
	cmpopts.IgnoreInterfaces(struct{ hcl.Body }{}),
}
<commit_msg>simplify tests<commit_after>package hcl2template

import (
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/hashicorp/packer/builder/null"
	. "github.com/hashicorp/packer/hcl2template/internal"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/zclconf/go-cty/cty"
)

func getBasicParser() *Parser {
	return &Parser{
		Parser: hclparse.NewParser(),
		BuilderSchemas: packer.MapOfBuilder{
			"amazon-ebs":     func() (packer.Builder, error) { return &MockBuilder{}, nil },
			"virtualbox-iso": func() (packer.Builder, error) { return &MockBuilder{}, nil },
			"null":           func() (packer.Builder, error) { return &null.Builder{}, nil },
		},
		ProvisionersSchemas: packer.MapOfProvisioner{
			"shell": func() (packer.Provisioner, error) { return &MockProvisioner{}, nil },
			"file":  func() (packer.Provisioner, error) { return &MockProvisioner{}, nil },
		},
		PostProcessorsSchemas: packer.MapOfPostProcessor{
			"amazon-import": func() (packer.PostProcessor, error) { return &MockPostProcessor{}, nil },
			"manifest":      func() (packer.PostProcessor, error) { return &MockPostProcessor{}, nil },
		},
	}
}

type parseTestArgs struct {
	filename string
	vars     map[string]string
	varFiles []string
}

type parseTest struct {
	name   string
	parser *Parser
	args   parseTestArgs

	parseWantCfg           *PackerConfig
	parseWantDiags         bool
	parseWantDiagHasErrors bool

	getBuildsWantBuilds []packer.Build
	getBuildsWantDiags  bool
	// getBuildsWantDiagHasErrors bool
}

func testParse(t *testing.T, tests []parseTest) {
	t.Helper()

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			gotCfg, gotDiags := tt.parser.Parse(tt.args.filename, tt.args.varFiles, tt.args.vars)
			moreDiags := gotCfg.Initialize()
			gotDiags = append(gotDiags, moreDiags...)
			if tt.parseWantDiags == (gotDiags == nil) {
				t.Fatalf("Parser.parse() unexpected %q diagnostics.", gotDiags)
			}
			if tt.parseWantDiagHasErrors != gotDiags.HasErrors() {
				t.Fatalf("Parser.parse() unexpected diagnostics HasErrors. %s", gotDiags)
			}
			if diff := cmp.Diff(tt.parseWantCfg, gotCfg, cmpOpts...); diff != "" {
				t.Fatalf("Parser.parse() wrong packer config. %s", diff)
			}

			if gotCfg != nil && !tt.parseWantDiagHasErrors {
				if diff := cmp.Diff(tt.parseWantCfg.InputVariables, gotCfg.InputVariables, cmpOpts...); diff != "" {
					t.Fatalf("Parser.parse() unexpected input vars. %s", diff)
				}

				if diff := cmp.Diff(tt.parseWantCfg.LocalVariables, gotCfg.LocalVariables, cmpOpts...); diff != "" {
					t.Fatalf("Parser.parse() unexpected local vars. %s", diff)
				}
			}

			if gotDiags.HasErrors() {
				return
			}

			gotBuilds, gotDiags := gotCfg.GetBuilds(packer.GetBuildsOptions{})
			if tt.getBuildsWantDiags == (gotDiags == nil) {
				t.Fatalf("Parser.getBuilds() unexpected diagnostics. %s", gotDiags)
			}
			if diff := cmp.Diff(tt.getBuildsWantBuilds, gotBuilds, cmpOpts...); diff != "" {
				t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff)
			}
		})
	}
}

var (
	// everything in the tests is a basicNestedMockConfig this allow to test
	// each known type to packer ( and embedding ) in one go.
	basicNestedMockConfig = NestedMockConfig{
		String:   "string",
		Int:      42,
		Int64:    43,
		Bool:     true,
		Trilean:  config.TriTrue,
		Duration: 10 * time.Second,
		MapStringString: map[string]string{
			"a": "b",
			"c": "d",
		},
		SliceString: []string{
			"a",
			"b",
			"c",
		},
		SliceSliceString: [][]string{
			{"a", "b"},
			{"c", "d"},
		},
		Tags: []MockTag{},
	}

	basicMockBuilder = &MockBuilder{
		Config: MockConfig{
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				basicNestedMockConfig,
				basicNestedMockConfig,
			},
		},
	}

	basicMockProvisioner = &MockProvisioner{
		Config: MockConfig{
			NotSquashed:      "value <UNKNOWN>",
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				{
					Tags: dynamicTagList,
				},
			},
		},
	}
	basicMockPostProcessor = &MockPostProcessor{
		Config: MockConfig{
			NotSquashed:      "value <UNKNOWN>",
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				{
					Tags: []MockTag{},
				},
			},
		},
	}
	basicMockCommunicator = &MockCommunicator{
		Config: MockConfig{
			NestedMockConfig: basicNestedMockConfig,
			Nested:           basicNestedMockConfig,
			NestedSlice: []NestedMockConfig{
				{
					Tags: []MockTag{},
				},
			},
		},
	}

	emptyMockBuilder = &MockBuilder{
		Config: MockConfig{
			NestedMockConfig: NestedMockConfig{
				Tags: []MockTag{},
			},
			Nested:      NestedMockConfig{},
			NestedSlice: []NestedMockConfig{},
		},
	}

	emptyMockProvisioner = &MockProvisioner{
		Config: MockConfig{
			NestedMockConfig: NestedMockConfig{Tags: []MockTag{}},
			NestedSlice:      []NestedMockConfig{},
		},
	}

	dynamicTagList = []MockTag{
		{
			Key:   "first_tag_key",
			Value: "first_tag_value",
		},
		{
			Key:   "Component",
			Value: "user-service",
		},
		{
			Key:   "Environment",
			Value: "production",
		},
	}
)

var ctyValueComparer = cmp.Comparer(func(x, y cty.Value) bool {
	return x.RawEquals(y)
})

var ctyTypeComparer = cmp.Comparer(func(x, y cty.Type) bool {
	if x == cty.NilType && y == cty.NilType {
		return true
	}
	if x == cty.NilType || y == cty.NilType {
		return false
	}
	return x.Equals(y)
})

var cmpOpts = []cmp.Option{
	ctyValueComparer,
	ctyTypeComparer,
	cmpopts.IgnoreUnexported(
		PackerConfig{},
		Variable{},
		SourceBlock{},
		ProvisionerBlock{},
		PostProcessorBlock{},
		packer.CoreBuild{},
		HCL2Provisioner{},
		HCL2PostProcessor{},
		packer.CoreBuildPostProcessor{},
		packer.CoreBuildProvisioner{},
		packer.CoreBuildPostProcessor{},
		null.Builder{},
	),
	cmpopts.IgnoreFields(PackerConfig{},
		"Cwd", // Cwd will change for every os type
	),
	cmpopts.IgnoreFields(VariableAssignment{},
		"Expr", // its an interface
	),
	cmpopts.IgnoreTypes(HCL2Ref{}),
	cmpopts.IgnoreTypes([]*LocalBlock{}),
	cmpopts.IgnoreTypes([]hcl.Range{}),
	cmpopts.IgnoreTypes(hcl.Range{}),
	cmpopts.IgnoreInterfaces(struct{ hcl.Expression }{}),
	cmpopts.IgnoreInterfaces(struct{ hcl.Body }{}),
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"context"
	"fmt"
	"path"

	"github.com/apache/beam/sdks/go/pkg/beam/core/funcx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/mtime"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
	"github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
	"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/go/pkg/beam/util/errorx"
)

// ParDo is a DoFn executor.
type ParDo struct {
	UID     UnitID
	Fn      *graph.DoFn
	Inbound []*graph.Inbound
	Side    []SideInputAdapter
	Out     []Node

	PID      string
	emitters []ReusableEmitter
	ctx      context.Context
	inv      *invoker

	side  SideInputReader
	cache *cacheElm

	status Status
	err    errorx.GuardedError
}

// GetPID returns the PTransformID for this ParDo.
func (n *ParDo) GetPID() string {
	return n.PID
}

// cacheElm holds per-window cached information about side input.
type cacheElm struct {
	key       typex.Window
	sideinput []ReusableInput
	extra     []interface{}
}

// ID returns the UnitID for this ParDo.
func (n *ParDo) ID() UnitID {
	return n.UID
}

// Up initializes this
func (n *ParDo) Up(ctx context.Context) error {
	if n.status != Initializing {
		return fmt.Errorf("invalid status for pardo %v: %v, want Initializing", n.UID, n.status)
	}
	n.status = Up
	n.inv = newInvoker(n.Fn.ProcessElementFn())

	if _, err := InvokeWithoutEventTime(ctx, n.Fn.SetupFn(), nil); err != nil {
		return n.fail(err)
	}

	emitters, err := makeEmitters(n.Fn.ProcessElementFn(), n.Out)
	if err != nil {
		return n.fail(err)
	}
	n.emitters = emitters
	return nil
}

// StartBundle starts processing for this bundle and does before bundle processing operations.
func (n *ParDo) StartBundle(ctx context.Context, id string, data DataContext) error {
	if n.status != Up {
		return fmt.Errorf("invalid status for pardo %v: %v, want Up", n.UID, n.status)
	}
	n.status = Active
	n.side = data.SideInput
	// Allocating contexts all the time is expensive, but we seldom re-write them,
	// and never accept modified contexts from users, so we will cache them per-bundle
	// per-unit, to avoid the constant allocation overhead.
	n.ctx = metrics.SetPTransformID(ctx, n.PID)

	if err := MultiStartBundle(n.ctx, id, data, n.Out...); err != nil {
		return n.fail(err)
	}

	// TODO(BEAM-3303): what to set for StartBundle/FinishBundle window and emitter timestamp?

	if _, err := n.invokeDataFn(n.ctx, window.SingleGlobalWindow, mtime.ZeroTimestamp, n.Fn.StartBundleFn(), nil); err != nil {
		return n.fail(err)
	}
	return nil
}

// ProcessElement processes each parallel element with the DoFn.
func (n *ParDo) ProcessElement(ctx context.Context, elm *FullValue, values ...ReStream) error {
	if n.status != Active {
		return fmt.Errorf("invalid status for pardo %v: %v, want Active", n.UID, n.status)
	}
	// If the function observes windows, we must invoke it for each window. The expected fast path
	// is that either there is a single window or the function doesn't observes windows.

	if !mustExplodeWindows(n.inv.fn, elm, len(n.Side) > 0) {
		val, err := n.invokeProcessFn(n.ctx, elm.Windows, elm.Timestamp, &MainInput{Key: *elm, Values: values})
		if err != nil {
			return n.fail(err)
		}

		// Forward direct output, if any. It is always a main output.
		if val != nil {
			return n.Out[0].ProcessElement(n.ctx, val)
		}
	} else {
		for _, w := range elm.Windows {
			wElm := FullValue{Elm: elm.Elm, Elm2: elm.Elm2, Timestamp: elm.Timestamp, Windows: []typex.Window{w}}

			val, err := n.invokeProcessFn(n.ctx, wElm.Windows, wElm.Timestamp, &MainInput{Key: wElm, Values: values})
			if err != nil {
				return n.fail(err)
			}

			// Forward direct output, if any. It is always a main output.
			if val != nil {
				return n.Out[0].ProcessElement(n.ctx, val)
			}
		}
	}
	return nil
}

// mustExplodeWindows returns true iif we need to call the function
// for each window. It is needed if the function either observes the
// window, either directly or indirectly via (windowed) side inputs.
func mustExplodeWindows(fn *funcx.Fn, elm *FullValue, usesSideInput bool) bool {
	if len(elm.Windows) < 2 {
		return false
	}
	_, explode := fn.Window()
	return explode || usesSideInput
}

// FinishBundle finishes processing for this bundle and does post bundle processing operations.
func (n *ParDo) FinishBundle(ctx context.Context) error {
	if n.status != Active {
		return fmt.Errorf("invalid status for pardo %v: %v, want Active", n.UID, n.status)
	}
	n.status = Up
	n.inv.Reset()

	if _, err := n.invokeDataFn(n.ctx, window.SingleGlobalWindow, mtime.ZeroTimestamp, n.Fn.FinishBundleFn(), nil); err != nil {
		return n.fail(err)
	}
	n.side = nil
	n.cache = nil

	if err := MultiFinishBundle(n.ctx, n.Out...); err != nil {
		return n.fail(err)
	}
	return nil
}

// Down tearsdown DoFn resources.
func (n *ParDo) Down(ctx context.Context) error {
	if n.status == Down {
		return n.err.Error()
	}
	n.status = Down
	n.side = nil
	n.cache = nil

	if _, err := InvokeWithoutEventTime(ctx, n.Fn.TeardownFn(), nil); err != nil {
		n.err.TrySetError(err)
	}
	return n.err.Error()
}

func (n *ParDo) initSideInput(ctx context.Context, w typex.Window) error {
	if n.cache == nil {
		// First time: init single-element cache. We know that side input
		// must come before emitters in the signature.

		sideCount := len(n.Side)
		emitCount := len(n.emitters)

		n.cache = &cacheElm{
			key:   w,
			extra: make([]interface{}, sideCount+emitCount, sideCount+emitCount),
		}
		for i, emit := range n.emitters {
			n.cache.extra[i+sideCount] = emit.Value()
		}
	} else if w.Equals(n.cache.key) {
		// Fast path: same window. Just unwind the side inputs.

		for _, s := range n.cache.sideinput {
			if err := s.Init(); err != nil {
				return err
			}
		}
		return nil
	}

	// Slow path: init side input for the given window

	streams := make([]ReStream, len(n.Side), len(n.Side))
	for i, adapter := range n.Side {
		s, err := adapter.NewIterable(ctx, n.side, w)
		if err != nil {
			return err
		}
		streams[i] = s
	}

	sideinput, err := makeSideInputs(n.Fn.ProcessElementFn(), n.Inbound, streams)
	if err != nil {
		return err
	}
	n.cache.sideinput = sideinput
	for i := 0; i < len(n.Side); i++ {
		n.cache.extra[i] = sideinput[i].Value()
	}

	for _, s := range n.cache.sideinput {
		if err := s.Init(); err != nil {
			return err
		}
	}
	return nil
}

// invokeDataFn handle non-per element invocations.
func (n *ParDo) invokeDataFn(ctx context.Context, ws []typex.Window, ts typex.EventTime, fn *funcx.Fn, opt *MainInput) (*FullValue, error) {
	if fn == nil {
		return nil, nil
	}
	if err := n.preInvoke(ctx, ws, ts); err != nil {
		return nil, err
	}
	val, err := Invoke(ctx, ws, ts, fn, opt, n.cache.extra...)
	if err != nil {
		return nil, err
	}
	if err := n.postInvoke(); err != nil {
		return nil, err
	}
	return val, nil
}

// invokeProcessFn handles the per element invocations
func (n *ParDo) invokeProcessFn(ctx context.Context, ws []typex.Window, ts typex.EventTime, opt *MainInput) (*FullValue, error) {
	if err := n.preInvoke(ctx, ws, ts); err != nil {
		return nil, err
	}
	val, err := n.inv.Invoke(ctx, ws, ts, opt, n.cache.extra...)
	if err != nil {
		return nil, err
	}
	if err := n.postInvoke(); err != nil {
		return nil, err
	}
	return val, nil
}

func (n *ParDo) preInvoke(ctx context.Context, ws []typex.Window, ts typex.EventTime) error {
	for _, e := range n.emitters {
		if err := e.Init(ctx, ws, ts); err != nil {
			return err
		}
	}
	return n.initSideInput(ctx, ws[0])
}

func (n *ParDo) postInvoke() error {
	for _, s := range n.cache.sideinput {
		if err := s.Reset(); err != nil {
			return err
		}
	}
	return nil
}

func (n *ParDo) fail(err error) error {
	n.status = Broken
	n.err.TrySetError(err)
	return err
}

func (n *ParDo) String() string {
	return fmt.Sprintf("ParDo[%v] Out:%v", path.Base(n.Fn.Name()), IDs(n.Out...))
}
<commit_msg>Address comments on comment wording.<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"context"
	"fmt"
	"path"

	"github.com/apache/beam/sdks/go/pkg/beam/core/funcx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/mtime"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
	"github.com/apache/beam/sdks/go/pkg/beam/core/metrics"
	"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/go/pkg/beam/util/errorx"
)

// ParDo is a DoFn executor.
type ParDo struct {
	UID     UnitID
	Fn      *graph.DoFn
	Inbound []*graph.Inbound
	Side    []SideInputAdapter
	Out     []Node

	PID      string
	emitters []ReusableEmitter
	ctx      context.Context
	inv      *invoker

	side  SideInputReader
	cache *cacheElm

	status Status
	err    errorx.GuardedError
}

// GetPID returns the PTransformID for this ParDo.
func (n *ParDo) GetPID() string {
	return n.PID
}

// cacheElm holds per-window cached information about side input.
type cacheElm struct {
	key       typex.Window
	sideinput []ReusableInput
	extra     []interface{}
}

// ID returns the UnitID for this ParDo.
func (n *ParDo) ID() UnitID {
	return n.UID
}

// Up initializes this ParDo and does one-time DoFn setup.
func (n *ParDo) Up(ctx context.Context) error {
	if n.status != Initializing {
		return fmt.Errorf("invalid status for pardo %v: %v, want Initializing", n.UID, n.status)
	}
	n.status = Up
	n.inv = newInvoker(n.Fn.ProcessElementFn())

	if _, err := InvokeWithoutEventTime(ctx, n.Fn.SetupFn(), nil); err != nil {
		return n.fail(err)
	}

	emitters, err := makeEmitters(n.Fn.ProcessElementFn(), n.Out)
	if err != nil {
		return n.fail(err)
	}
	n.emitters = emitters
	return nil
}

// StartBundle does pre-bundle processing operation for the DoFn.
func (n *ParDo) StartBundle(ctx context.Context, id string, data DataContext) error {
	if n.status != Up {
		return fmt.Errorf("invalid status for pardo %v: %v, want Up", n.UID, n.status)
	}
	n.status = Active
	n.side = data.SideInput
	// Allocating contexts all the time is expensive, but we seldom re-write them,
	// and never accept modified contexts from users, so we will cache them per-bundle
	// per-unit, to avoid the constant allocation overhead.
	n.ctx = metrics.SetPTransformID(ctx, n.PID)

	if err := MultiStartBundle(n.ctx, id, data, n.Out...); err != nil {
		return n.fail(err)
	}

	// TODO(BEAM-3303): what to set for StartBundle/FinishBundle window and emitter timestamp?

	if _, err := n.invokeDataFn(n.ctx, window.SingleGlobalWindow, mtime.ZeroTimestamp, n.Fn.StartBundleFn(), nil); err != nil {
		return n.fail(err)
	}
	return nil
}

// ProcessElement processes each parallel element with the DoFn.
func (n *ParDo) ProcessElement(ctx context.Context, elm *FullValue, values ...ReStream) error {
	if n.status != Active {
		return fmt.Errorf("invalid status for pardo %v: %v, want Active", n.UID, n.status)
	}
	// If the function observes windows, we must invoke it for each window. The expected fast path
	// is that either there is a single window or the function doesn't observes windows.

	if !mustExplodeWindows(n.inv.fn, elm, len(n.Side) > 0) {
		val, err := n.invokeProcessFn(n.ctx, elm.Windows, elm.Timestamp, &MainInput{Key: *elm, Values: values})
		if err != nil {
			return n.fail(err)
		}

		// Forward direct output, if any. It is always a main output.
		if val != nil {
			return n.Out[0].ProcessElement(n.ctx, val)
		}
	} else {
		for _, w := range elm.Windows {
			wElm := FullValue{Elm: elm.Elm, Elm2: elm.Elm2, Timestamp: elm.Timestamp, Windows: []typex.Window{w}}

			val, err := n.invokeProcessFn(n.ctx, wElm.Windows, wElm.Timestamp, &MainInput{Key: wElm, Values: values})
			if err != nil {
				return n.fail(err)
			}

			// Forward direct output, if any. It is always a main output.
			if val != nil {
				return n.Out[0].ProcessElement(n.ctx, val)
			}
		}
	}
	return nil
}

// mustExplodeWindows returns true iif we need to call the function
// for each window. It is needed if the function either observes the
// window, either directly or indirectly via (windowed) side inputs.
func mustExplodeWindows(fn *funcx.Fn, elm *FullValue, usesSideInput bool) bool {
	if len(elm.Windows) < 2 {
		return false
	}
	_, explode := fn.Window()
	return explode || usesSideInput
}

// FinishBundle does post-bundle processing operations for the DoFn.
// Note: This is not a "FinalizeBundle" operation. Data is not yet durably
// persisted at this point.
func (n *ParDo) FinishBundle(ctx context.Context) error {
	if n.status != Active {
		return fmt.Errorf("invalid status for pardo %v: %v, want Active", n.UID, n.status)
	}
	n.status = Up
	n.inv.Reset()

	if _, err := n.invokeDataFn(n.ctx, window.SingleGlobalWindow, mtime.ZeroTimestamp, n.Fn.FinishBundleFn(), nil); err != nil {
		return n.fail(err)
	}
	n.side = nil
	n.cache = nil

	if err := MultiFinishBundle(n.ctx, n.Out...); err != nil {
		return n.fail(err)
	}
	return nil
}

// Down performs best-effort teardown of DoFn resources. (May not run.)
func (n *ParDo) Down(ctx context.Context) error {
	if n.status == Down {
		return n.err.Error()
	}
	n.status = Down
	n.side = nil
	n.cache = nil

	if _, err := InvokeWithoutEventTime(ctx, n.Fn.TeardownFn(), nil); err != nil {
		n.err.TrySetError(err)
	}
	return n.err.Error()
}

func (n *ParDo) initSideInput(ctx context.Context, w typex.Window) error {
	if n.cache == nil {
		// First time: init single-element cache. We know that side input
		// must come before emitters in the signature.

		sideCount := len(n.Side)
		emitCount := len(n.emitters)

		n.cache = &cacheElm{
			key:   w,
			extra: make([]interface{}, sideCount+emitCount, sideCount+emitCount),
		}
		for i, emit := range n.emitters {
			n.cache.extra[i+sideCount] = emit.Value()
		}
	} else if w.Equals(n.cache.key) {
		// Fast path: same window. Just unwind the side inputs.

		for _, s := range n.cache.sideinput {
			if err := s.Init(); err != nil {
				return err
			}
		}
		return nil
	}

	// Slow path: init side input for the given window

	streams := make([]ReStream, len(n.Side), len(n.Side))
	for i, adapter := range n.Side {
		s, err := adapter.NewIterable(ctx, n.side, w)
		if err != nil {
			return err
		}
		streams[i] = s
	}

	sideinput, err := makeSideInputs(n.Fn.ProcessElementFn(), n.Inbound, streams)
	if err != nil {
		return err
	}
	n.cache.sideinput = sideinput
	for i := 0; i < len(n.Side); i++ {
		n.cache.extra[i] = sideinput[i].Value()
	}

	for _, s := range n.cache.sideinput {
		if err := s.Init(); err != nil {
			return err
		}
	}
	return nil
}

// invokeDataFn handle non-per element invocations.
func (n *ParDo) invokeDataFn(ctx context.Context, ws []typex.Window, ts typex.EventTime, fn *funcx.Fn, opt *MainInput) (*FullValue, error) {
	if fn == nil {
		return nil, nil
	}
	if err := n.preInvoke(ctx, ws, ts); err != nil {
		return nil, err
	}
	val, err := Invoke(ctx, ws, ts, fn, opt, n.cache.extra...)
	if err != nil {
		return nil, err
	}
	if err := n.postInvoke(); err != nil {
		return nil, err
	}
	return val, nil
}

// invokeProcessFn handles the per element invocations
func (n *ParDo) invokeProcessFn(ctx context.Context, ws []typex.Window, ts typex.EventTime, opt *MainInput) (*FullValue, error) {
	if err := n.preInvoke(ctx, ws, ts); err != nil {
		return nil, err
	}
	val, err := n.inv.Invoke(ctx, ws, ts, opt, n.cache.extra...)
	if err != nil {
		return nil, err
	}
	if err := n.postInvoke(); err != nil {
		return nil, err
	}
	return val, nil
}

func (n *ParDo) preInvoke(ctx context.Context, ws []typex.Window, ts typex.EventTime) error {
	for _, e := range n.emitters {
		if err := e.Init(ctx, ws, ts); err != nil {
			return err
		}
	}
	return n.initSideInput(ctx, ws[0])
}

func (n *ParDo) postInvoke() error {
	for _, s := range n.cache.sideinput {
		if err := s.Reset(); err != nil {
			return err
		}
	}
	return nil
}

func (n *ParDo) fail(err error) error {
	n.status = Broken
	n.err.TrySetError(err)
	return err
}

func (n *ParDo) String() string {
	return fmt.Sprintf("ParDo[%v] Out:%v", path.Base(n.Fn.Name()), IDs(n.Out...))
}
<|endoftext|>


<commit_before>package probe

import (
	"context"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	log "github.com/sirupsen/logrus"
	"github.com/weaveworks/common/mtime"
	"golang.org/x/time/rate"

	"github.com/weaveworks/scope/report"
)

const (
	spiedReportBufferSize    = 16
	shortcutReportBufferSize = 1024
)

// ReportPublisher publishes reports, probably to a remote collector.
type ReportPublisher interface {
	Publish(r report.Report) error
}

// Probe sits there, generating and publishing reports.
type Probe struct {
	spyInterval, publishInterval time.Duration
	publisher                    ReportPublisher
	rateLimiter                  *rate.Limiter
	ticksPerFullReport           int
	noControls                   bool

	tickers   []Ticker
	reporters []Reporter
	taggers   []Tagger

	quit chan struct{}
	done sync.WaitGroup

	spiedReports    chan report.Report
	shortcutReports chan report.Report
}

// Tagger tags nodes with value-add node metadata.
type Tagger interface {
	Name() string
	Tag(r report.Report) (report.Report, error)
}

// Reporter generates Reports.
type Reporter interface {
	Name() string
	Report() (report.Report, error)
}

// ReporterFunc uses a function to implement a Reporter
func ReporterFunc(name string, f func() (report.Report, error)) Reporter {
	return reporterFunc{name, f}
}

type reporterFunc struct {
	name string
	f    func() (report.Report, error)
}

func (r reporterFunc) Name() string                   { return r.name }
func (r reporterFunc) Report() (report.Report, error) { return r.f() }

// Ticker is something which will be invoked every spyDuration.
// It's useful for things that should be updated on that interval.
// For example, cached shared state between Taggers and Reporters.
type Ticker interface {
	Name() string
	Tick() error
}

// New makes a new Probe.
func New(
	spyInterval, publishInterval time.Duration,
	publisher ReportPublisher,
	ticksPerFullReport int,
	noControls bool,
) *Probe {
	result := &Probe{
		spyInterval:        spyInterval,
		publishInterval:    publishInterval,
		publisher:          publisher,
		rateLimiter:        rate.NewLimiter(rate.Every(publishInterval/100), 1),
		ticksPerFullReport: ticksPerFullReport,
		noControls:         noControls,
		quit:               make(chan struct{}),
		spiedReports:       make(chan report.Report, spiedReportBufferSize),
		shortcutReports:    make(chan report.Report, shortcutReportBufferSize),
	}
	return result
}

// AddTagger adds a new Tagger to the Probe
func (p *Probe) AddTagger(ts ...Tagger) {
	p.taggers = append(p.taggers, ts...)
}

// AddReporter adds a new Reported to the Probe
func (p *Probe) AddReporter(rs ...Reporter) {
	p.reporters = append(p.reporters, rs...)
}

// AddTicker adds a new Ticker to the Probe
func (p *Probe) AddTicker(ts ...Ticker) {
	p.tickers = append(p.tickers, ts...)
}

// Start starts the probe
func (p *Probe) Start() {
	p.done.Add(2)
	go p.spyLoop()
	go p.publishLoop()
}

// Stop stops the probe
func (p *Probe) Stop() error {
	close(p.quit)
	p.done.Wait()
	return nil
}

// Publish will queue a report for immediate publication,
// bypassing the spy tick
func (p *Probe) Publish(rpt report.Report) {
	rpt = p.tag(rpt)
	p.shortcutReports <- rpt
}

func (p *Probe) spyLoop() {
	defer p.done.Done()
	spyTick := time.Tick(p.spyInterval)

	for {
		select {
		case <-spyTick:
			p.tick()
			rpt := p.report()
			rpt = p.tag(rpt)
			p.spiedReports <- rpt
		case <-p.quit:
			return
		}
	}
}

func (p *Probe) tick() {
	for _, ticker := range p.tickers {
		t := time.Now()
		err := ticker.Tick()
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "ticker"},
			{Name: "module", Value: ticker.Name()},
		})
		if err != nil {
			log.Errorf("Error doing ticker: %v", err)
		}
	}
}

func (p *Probe) report() report.Report {
	reports := make(chan report.Report, len(p.reporters))
	for _, rep := range p.reporters {
		go func(rep Reporter) {
			t := time.Now()
			timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v reporter took longer than %v", rep.Name(), p.spyInterval) })
			newReport, err := rep.Report()
			if !timer.Stop() {
				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
			}
			metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
				{Name: "operation", Value: "reporter"},
				{Name: "module", Value: rep.Name()},
			})
			if err != nil {
				log.Errorf("Error generating %s report: %v", rep.Name(), err)
				newReport = report.MakeReport() // empty is OK to merge
			}
			reports <- newReport
		}(rep)
	}

	result := report.MakeReport()
	result.TS = mtime.Now()
	for i := 0; i < cap(reports); i++ {
		result.UnsafeMerge(<-reports)
	}
	return result
}

func (p *Probe) tag(r report.Report) report.Report {
	var err error
	for _, tagger := range p.taggers {
		t := time.Now()
		timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v tagger took longer than %v", tagger.Name(), p.spyInterval) })
		r, err = tagger.Tag(r)
		if !timer.Stop() {
			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
		}
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "tagger"},
			{Name: "module", Value: tagger.Name()},
		})
		if err != nil {
			log.Errorf("Error applying tagger: %v", err)
		}
	}
	return r
}

func (p *Probe) drainAndSanitise(rpt report.Report, rs chan report.Report) report.Report {
	p.rateLimiter.Wait(context.Background())
	rpt = rpt.Copy()
ForLoop:
	for {
		select {
		case r := <-rs:
			rpt.UnsafeMerge(r)
		default:
			break ForLoop
		}
	}

	if p.noControls {
		rpt.WalkTopologies(func(t *report.Topology) {
			t.Controls = report.Controls{}
		})
	}
	return rpt
}

func (p *Probe) publishLoop() {
	defer p.done.Done()
	pubTick := time.Tick(p.publishInterval)
	publishCount := 0
	var lastFullReport report.Report

	for {
		var err error
		select {
		case <-pubTick:
			rpt := p.drainAndSanitise(report.MakeReport(), p.spiedReports)
			fullReport := (publishCount % p.ticksPerFullReport) == 0
			if !fullReport {
				rpt.UnsafeUnMerge(lastFullReport)
			}
			err = p.publisher.Publish(rpt)
			if err == nil {
				if fullReport {
					lastFullReport = rpt
				}
				publishCount++
			} else {
				// If we failed to send then drop back to full report next time
				publishCount = 0
			}

		case rpt := <-p.shortcutReports:
			rpt = p.drainAndSanitise(rpt, p.shortcutReports)
			err = p.publisher.Publish(rpt)

		case <-p.quit:
			return
		}
		if err != nil {
			log.Infof("Publish: %v", err)
		}
	}
}
<commit_msg>Set the Window duration which each report covers<commit_after>package probe

import (
	"context"
	"sync"
	"time"

	"github.com/armon/go-metrics"
	log "github.com/sirupsen/logrus"
	"github.com/weaveworks/common/mtime"
	"golang.org/x/time/rate"

	"github.com/weaveworks/scope/report"
)

const (
	spiedReportBufferSize    = 16
	shortcutReportBufferSize = 1024
)

// ReportPublisher publishes reports, probably to a remote collector.
type ReportPublisher interface {
	Publish(r report.Report) error
}

// Probe sits there, generating and publishing reports.
type Probe struct {
	spyInterval, publishInterval time.Duration
	publisher                    ReportPublisher
	rateLimiter                  *rate.Limiter
	ticksPerFullReport           int
	noControls                   bool

	tickers   []Ticker
	reporters []Reporter
	taggers   []Tagger

	quit chan struct{}
	done sync.WaitGroup

	spiedReports    chan report.Report
	shortcutReports chan report.Report
}

// Tagger tags nodes with value-add node metadata.
type Tagger interface {
	Name() string
	Tag(r report.Report) (report.Report, error)
}

// Reporter generates Reports.
type Reporter interface {
	Name() string
	Report() (report.Report, error)
}

// ReporterFunc uses a function to implement a Reporter
func ReporterFunc(name string, f func() (report.Report, error)) Reporter {
	return reporterFunc{name, f}
}

type reporterFunc struct {
	name string
	f    func() (report.Report, error)
}

func (r reporterFunc) Name() string                   { return r.name }
func (r reporterFunc) Report() (report.Report, error) { return r.f() }

// Ticker is something which will be invoked every spyDuration.
// It's useful for things that should be updated on that interval.
// For example, cached shared state between Taggers and Reporters.
type Ticker interface {
	Name() string
	Tick() error
}

// New makes a new Probe.
func New(
	spyInterval, publishInterval time.Duration,
	publisher ReportPublisher,
	ticksPerFullReport int,
	noControls bool,
) *Probe {
	result := &Probe{
		spyInterval:        spyInterval,
		publishInterval:    publishInterval,
		publisher:          publisher,
		rateLimiter:        rate.NewLimiter(rate.Every(publishInterval/100), 1),
		ticksPerFullReport: ticksPerFullReport,
		noControls:         noControls,
		quit:               make(chan struct{}),
		spiedReports:       make(chan report.Report, spiedReportBufferSize),
		shortcutReports:    make(chan report.Report, shortcutReportBufferSize),
	}
	return result
}

// AddTagger adds a new Tagger to the Probe
func (p *Probe) AddTagger(ts ...Tagger) {
	p.taggers = append(p.taggers, ts...)
}

// AddReporter adds a new Reported to the Probe
func (p *Probe) AddReporter(rs ...Reporter) {
	p.reporters = append(p.reporters, rs...)
}

// AddTicker adds a new Ticker to the Probe
func (p *Probe) AddTicker(ts ...Ticker) {
	p.tickers = append(p.tickers, ts...)
}

// Start starts the probe
func (p *Probe) Start() {
	p.done.Add(2)
	go p.spyLoop()
	go p.publishLoop()
}

// Stop stops the probe
func (p *Probe) Stop() error {
	close(p.quit)
	p.done.Wait()
	return nil
}

// Publish will queue a report for immediate publication,
// bypassing the spy tick
func (p *Probe) Publish(rpt report.Report) {
	rpt = p.tag(rpt)
	p.shortcutReports <- rpt
}

func (p *Probe) spyLoop() {
	defer p.done.Done()
	spyTick := time.Tick(p.spyInterval)

	for {
		select {
		case <-spyTick:
			p.tick()
			rpt := p.report()
			rpt = p.tag(rpt)
			p.spiedReports <- rpt
		case <-p.quit:
			return
		}
	}
}

func (p *Probe) tick() {
	for _, ticker := range p.tickers {
		t := time.Now()
		err := ticker.Tick()
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "ticker"},
			{Name: "module", Value: ticker.Name()},
		})
		if err != nil {
			log.Errorf("Error doing ticker: %v", err)
		}
	}
}

func (p *Probe) report() report.Report {
	reports := make(chan report.Report, len(p.reporters))
	for _, rep := range p.reporters {
		go func(rep Reporter) {
			t := time.Now()
			timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v reporter took longer than %v", rep.Name(), p.spyInterval) })
			newReport, err := rep.Report()
			if !timer.Stop() {
				log.Warningf("%v reporter took %v (longer than %v)", rep.Name(), time.Now().Sub(t), p.spyInterval)
			}
			metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
				{Name: "operation", Value: "reporter"},
				{Name: "module", Value: rep.Name()},
			})
			if err != nil {
				log.Errorf("Error generating %s report: %v", rep.Name(), err)
				newReport = report.MakeReport() // empty is OK to merge
			}
			reports <- newReport
		}(rep)
	}

	result := report.MakeReport()
	result.TS = mtime.Now()
	for i := 0; i < cap(reports); i++ {
		result.UnsafeMerge(<-reports)
	}
	return result
}

func (p *Probe) tag(r report.Report) report.Report {
	var err error
	for _, tagger := range p.taggers {
		t := time.Now()
		timer := time.AfterFunc(p.spyInterval, func() { log.Warningf("%v tagger took longer than %v", tagger.Name(), p.spyInterval) })
		r, err = tagger.Tag(r)
		if !timer.Stop() {
			log.Warningf("%v tagger took %v (longer than %v)", tagger.Name(), time.Now().Sub(t), p.spyInterval)
		}
		metrics.MeasureSinceWithLabels([]string{"duration", "seconds"}, t, []metrics.Label{
			{Name: "operation", Value: "tagger"},
			{Name: "module", Value: tagger.Name()},
		})
		if err != nil {
			log.Errorf("Error applying tagger: %v", err)
		}
	}
	return r
}

func (p *Probe) drainAndSanitise(rpt report.Report, rs chan report.Report) report.Report {
	p.rateLimiter.Wait(context.Background())
	rpt = rpt.Copy()
ForLoop:
	for {
		select {
		case r := <-rs:
			rpt.UnsafeMerge(r)
		default:
			break ForLoop
		}
	}

	if p.noControls {
		rpt.WalkTopologies(func(t *report.Topology) {
			t.Controls = report.Controls{}
		})
	}
	return rpt
}

func (p *Probe) publishLoop() {
	defer p.done.Done()
	startTime := mtime.Now()
	pubTick := time.Tick(p.publishInterval)
	publishCount := 0
	var lastFullReport report.Report

	for {
		var err error
		select {
		case <-pubTick:
			rpt := p.drainAndSanitise(report.MakeReport(), p.spiedReports)
			fullReport := (publishCount % p.ticksPerFullReport) == 0
			if !fullReport {
				rpt.UnsafeUnMerge(lastFullReport)
			}
			rpt.Window = mtime.Now().Sub(startTime)
			startTime = mtime.Now()
			err = p.publisher.Publish(rpt)
			if err == nil {
				if fullReport {
					lastFullReport = rpt
				}
				publishCount++
			} else {
				// If we failed to send then drop back to full report next time
				publishCount = 0
			}

		case rpt := <-p.shortcutReports:
			rpt = p.drainAndSanitise(rpt, p.shortcutReports)
			err = p.publisher.Publish(rpt)

		case <-p.quit:
			return
		}
		if err != nil {
			log.Infof("Publish: %v", err)
		}
	}
}
<|endoftext|>