<commit_before>package slack

import (
	"fmt"
	"net/http"
	"strings"

	slcontracts "github.com/estafette/estafette-ci-api/slack/contracts"

	"github.com/estafette/estafette-ci-contracts"

	"github.com/estafette/estafette-ci-api/cockroach"
	"github.com/estafette/estafette-ci-api/config"
	"github.com/estafette/estafette-ci-api/estafette"
	crypt "github.com/estafette/estafette-ci-crypt"
	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// EventHandler handles http events for Slack integration
type EventHandler interface {
	Handle(*gin.Context)
	HasValidVerificationToken(slcontracts.SlashCommand) bool
}

type eventHandlerImpl struct {
	secretHelper                 crypt.SecretHelper
	config                       config.SlackConfig
	slackAPIClient               APIClient
	cockroachDBClient            cockroach.DBClient
	apiConfig                    config.APIServerConfig
	buildService                 estafette.BuildService
	githubJobVarsFunc            func(string, string, string) (string, string, error)
	bitbucketJobVarsFunc         func(string, string, string) (string, string, error)
	prometheusInboundEventTotals *prometheus.CounterVec
}

// NewSlackEventHandler returns a new slack.EventHandler
func NewSlackEventHandler(secretHelper crypt.SecretHelper, config config.SlackConfig, slackAPIClient APIClient, cockroachDBClient cockroach.DBClient, apiConfig config.APIServerConfig, buildService estafette.BuildService, githubJobVarsFunc func(string, string, string) (string, string, error), bitbucketJobVarsFunc func(string, string, string) (string, string, error), prometheusInboundEventTotals *prometheus.CounterVec) EventHandler {
	return &eventHandlerImpl{
		secretHelper:                 secretHelper,
		config:                       config,
		slackAPIClient:               slackAPIClient,
		cockroachDBClient:            cockroachDBClient,
		apiConfig:                    apiConfig,
		buildService:                 buildService,
		githubJobVarsFunc:            githubJobVarsFunc,
		bitbucketJobVarsFunc:         bitbucketJobVarsFunc,
		prometheusInboundEventTotals: prometheusInboundEventTotals,
	}
}

func (h *eventHandlerImpl) Handle(c *gin.Context) {

	// https://api.slack.com/slash-commands

	h.prometheusInboundEventTotals.With(prometheus.Labels{"event": "", "source": "slack"}).Inc()

	var slashCommand slcontracts.SlashCommand
	// This will infer what binder to use depending on the content-type header.
	err := c.Bind(&slashCommand)
	if err != nil {
		log.Error().Err(err).Msg("Binding form data from Slack command webhook failed")
		c.String(http.StatusInternalServerError, "Binding form data from Slack command webhook failed")
		return
	}

	hasValidVerificationToken := h.HasValidVerificationToken(slashCommand)
	if !hasValidVerificationToken {
		log.Warn().Str("expectedToken", h.config.AppVerificationToken).Str("actualToken", slashCommand.Token).Msg("Verification token for Slack command is invalid")
		c.String(http.StatusBadRequest, "Verification token for Slack command is invalid")
		return
	}

	if slashCommand.Command == "/estafette" {
		if slashCommand.Text != "" {
			splittedText := strings.Split(slashCommand.Text, " ")
			if splittedText != nil && len(splittedText) > 0 {
				command := splittedText[0]
				arguments := splittedText[1:len(splittedText)]
				switch command {
				case "encrypt":

					encryptedString, err := h.secretHelper.Encrypt(strings.Join(arguments, " "))
					if err != nil {
						log.Error().Err(err).Interface("slashCommand", slashCommand).Msg("Failed to encrypt secret")
						c.String(http.StatusOK, "Incorrect usage of /estafette encrypt")
						return
					}

					c.String(http.StatusOK, fmt.Sprintf("estafette.secret(%v)", encryptedString))
					return

				case "release":

					// # release latest green version (for release branch?)
					// /estafette release github.com/estafette/estafette-ci-builder beta
					// /estafette release github.com/estafette/estafette-ci-api tooling

					// # release latest green version for particular branch
					// /estafette release github.com/estafette/estafette-ci-builder beta master

					// # release specific version
					// /estafette release github.com/estafette/estafette-ci-builder beta 0.0.47
					// /estafette release github.com/estafette/estafette-ci-api beta 0.0.130

					if len(arguments) < 3 {
						c.String(http.StatusOK, "You have to few arguments, the command has to be of type /estafette release <repo> <release> <version>")
						return
					}

					// retrieve pipeline with first argument
					fullRepoName := arguments[0]
					releaseName := arguments[1]
					buildVersion := arguments[2]

					fullRepoNameArray := strings.Split(fullRepoName, "/")
					if len(fullRepoNameArray) != 1 && len(fullRepoNameArray) != 3 {
						c.String(http.StatusOK, "Your repository needs to be of the form <repo name> or <repo source>/<repo owner>/<repo name>")
						return
					}

					var pipeline *contracts.Pipeline
					if len(fullRepoNameArray) == 1 {
						pipelines, err := h.cockroachDBClient.GetPipelinesByRepoName(fullRepoName, false)
						if err != nil {
							log.Error().Err(err).Msgf("Failed retrieving pipelines for repo name %v by name", fullRepoName)
							c.String(http.StatusOK, fmt.Sprintf("Retrieving the pipeline for repository %v from the database failed: %v", fullRepoName, err))
							return
						}
						if len(pipelines) <= 0 {
							c.String(http.StatusOK, fmt.Sprintf("The repo %v in your command does not have any estafette builds", fullRepoName))
							return
						}
						if len(pipelines) > 1 {
							commandsExample := ""
							for _, p := range pipelines {
								commandsExample += fmt.Sprintf("/estafette release %v/%v/%v %v %v\n", p.RepoSource, p.RepoOwner, p.RepoName, releaseName, buildVersion)
							}
							c.String(http.StatusOK, fmt.Sprintf("There are multiple pipelines with name %v, use the full name instead:\n%v", fullRepoName, commandsExample))
							return
						}
						pipeline = pipelines[0]
					} else {
						pipeline, err := h.cockroachDBClient.GetPipeline(fullRepoNameArray[0], fullRepoNameArray[1], fullRepoNameArray[2], false)
						if err != nil {
							c.String(http.StatusOK, fmt.Sprintf("Retrieving the pipeline for repository %v from the database failed: %v", fullRepoName, err))
							return
						}
						if pipeline == nil {
							c.String(http.StatusOK, fmt.Sprintf("The repo %v in your command does not have any estafette builds", fullRepoName))
							return
						}
					}

					// check if version exists
					builds, err := h.cockroachDBClient.GetPipelineBuildsByVersion(pipeline.RepoSource, pipeline.RepoOwner, pipeline.RepoName, buildVersion, false)

					if err != nil {
						c.String(http.StatusOK, fmt.Sprintf("Retrieving the build for repository %v and version %v from the database failed: %v", fullRepoName, buildVersion, err))
						return
					}

					var build *contracts.Build
					// get succeeded build
					for _, b := range builds {
						if b.BuildStatus == "succeeded" {
							build = b
							break
						}
					}

					if build == nil {
						c.String(http.StatusOK, fmt.Sprintf("The version %v in your command does not exist", buildVersion))
						return
					}
					if build.BuildStatus != "succeeded" {
						c.String(http.StatusOK, fmt.Sprintf("The build for version %v is not successful and cannot be used", buildVersion))
						return
					}

					// check if release target exists
					releaseExists := false
					for _, releaseTarget := range build.ReleaseTargets {
						if releaseTarget.Name == releaseName {
							// todo support release action

							releaseExists = true
							break
						}
					}

					if !releaseExists {
						c.String(http.StatusOK, fmt.Sprintf("The release %v in your command is not defined in the manifest", releaseName))
						return
					}

					// get user profile from api to set email address for TriggeredBy
					profile, err := h.slackAPIClient.GetUserProfile(slashCommand.UserID)
					if err != nil {
						c.String(http.StatusOK, fmt.Sprintf("Failed retrieving Slack user profile for user id %v: %v", slashCommand.UserID, err))
						return
					}

					// create release object and hand off to build service
					createdRelease, err := h.buildService.CreateRelease(contracts.Release{
						Name:           releaseName,
						Action:         "", // no support for releas action yet
						RepoSource:     build.RepoSource,
						RepoOwner:      build.RepoOwner,
						RepoName:       build.RepoName,
						ReleaseVersion: buildVersion,
						TriggeredBy:    profile.Email,
					}, *build.ManifestObject, build.RepoBranch, build.RepoRevision, false)

					if err != nil {
						errorMessage := fmt.Sprintf("Failed creating release %v for pipeline %v/%v/%v version %v for release command issued by %v", releaseName, build.RepoSource, build.RepoOwner, build.RepoName, buildVersion, profile.Email)
						log.Error().Err(err).Msg(errorMessage)
						c.String(http.StatusOK, fmt.Sprintf("Inserting starting the release: %v", err))
						return
					}

					c.String(http.StatusOK, fmt.Sprintf("Started releasing version %v to %v: %vpipelines/%v/%v/%v/releases/%v/logs", buildVersion, releaseName, h.apiConfig.BaseURL, build.RepoSource, build.RepoOwner, build.RepoName, createdRelease.ID))
					return
				}
			}
		}
	}

	c.String(http.StatusOK, "Aye aye!")
}

func (h *eventHandlerImpl) HasValidVerificationToken(slashCommand slcontracts.SlashCommand) bool {
	return slashCommand.Token == h.config.AppVerificationToken
}
<commit_msg>debug logging for slash commands<commit_after>package slack

import (
	"fmt"
	"net/http"
	"strings"

	slcontracts "github.com/estafette/estafette-ci-api/slack/contracts"

	"github.com/estafette/estafette-ci-contracts"

	"github.com/estafette/estafette-ci-api/cockroach"
	"github.com/estafette/estafette-ci-api/config"
	"github.com/estafette/estafette-ci-api/estafette"
	crypt "github.com/estafette/estafette-ci-crypt"
	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// EventHandler handles http events for Slack integration
type EventHandler interface {
	Handle(*gin.Context)
	HasValidVerificationToken(slcontracts.SlashCommand) bool
}

type eventHandlerImpl struct {
	secretHelper                 crypt.SecretHelper
	config                       config.SlackConfig
	slackAPIClient               APIClient
	cockroachDBClient            cockroach.DBClient
	apiConfig                    config.APIServerConfig
	buildService                 estafette.BuildService
	githubJobVarsFunc            func(string, string, string) (string, string, error)
	bitbucketJobVarsFunc         func(string, string, string) (string, string, error)
	prometheusInboundEventTotals *prometheus.CounterVec
}

// NewSlackEventHandler returns a new slack.EventHandler
func NewSlackEventHandler(secretHelper crypt.SecretHelper, config config.SlackConfig, slackAPIClient APIClient, cockroachDBClient cockroach.DBClient, apiConfig config.APIServerConfig, buildService estafette.BuildService, githubJobVarsFunc func(string, string, string) (string, string, error), bitbucketJobVarsFunc func(string, string, string) (string, string, error), prometheusInboundEventTotals *prometheus.CounterVec) EventHandler {
	return &eventHandlerImpl{
		secretHelper:                 secretHelper,
		config:                       config,
		slackAPIClient:               slackAPIClient,
		cockroachDBClient:            cockroachDBClient,
		apiConfig:                    apiConfig,
		buildService:                 buildService,
		githubJobVarsFunc:            githubJobVarsFunc,
		bitbucketJobVarsFunc:         bitbucketJobVarsFunc,
		prometheusInboundEventTotals: prometheusInboundEventTotals,
	}
}

func (h *eventHandlerImpl) Handle(c *gin.Context) {

	// https://api.slack.com/slash-commands

	h.prometheusInboundEventTotals.With(prometheus.Labels{"event": "", "source": "slack"}).Inc()

	var slashCommand slcontracts.SlashCommand
	// This will infer what binder to use depending on the content-type header.
	err := c.Bind(&slashCommand)
	if err != nil {
		log.Error().Err(err).Msg("Binding form data from Slack command webhook failed")
		c.String(http.StatusInternalServerError, "Binding form data from Slack command webhook failed")
		return
	}

	hasValidVerificationToken := h.HasValidVerificationToken(slashCommand)
	if !hasValidVerificationToken {
		log.Warn().Str("expectedToken", h.config.AppVerificationToken).Str("actualToken", slashCommand.Token).Msg("Verification token for Slack command is invalid")
		c.String(http.StatusBadRequest, "Verification token for Slack command is invalid")
		return
	}

	if slashCommand.Command == "/estafette" {
		if slashCommand.Text != "" {
			splittedText := strings.Split(slashCommand.Text, " ")
			if splittedText != nil && len(splittedText) > 0 {
				command := splittedText[0]
				arguments := splittedText[1:len(splittedText)]
				switch command {
				case "encrypt":

					log.Debug().Msg("Handling slash command /estafette encrypt")

					encryptedString, err := h.secretHelper.Encrypt(strings.Join(arguments, " "))
					if err != nil {
						log.Error().Err(err).Interface("slashCommand", slashCommand).Msg("Failed to encrypt secret")
						c.String(http.StatusOK, "Incorrect usage of /estafette encrypt")
						return
					}

					c.String(http.StatusOK, fmt.Sprintf("estafette.secret(%v)", encryptedString))
					return

				case "release":

					log.Debug().Msg("Handling slash command /estafette release")

					// # release latest green version (for release branch?)
					// /estafette release github.com/estafette/estafette-ci-builder beta
					// /estafette release github.com/estafette/estafette-ci-api tooling

					// # release latest green version for particular branch
					// /estafette release github.com/estafette/estafette-ci-builder beta master

					// # release specific version
					// /estafette release github.com/estafette/estafette-ci-builder beta 0.0.47
					// /estafette release github.com/estafette/estafette-ci-api beta 0.0.130

					if len(arguments) < 3 {
						c.String(http.StatusOK, "You have to few arguments, the command has to be of type /estafette release <repo> <release> <version>")
						return
					}

					// retrieve pipeline with first argument
					fullRepoName := arguments[0]
					releaseName := arguments[1]
					buildVersion := arguments[2]

					fullRepoNameArray := strings.Split(fullRepoName, "/")
					if len(fullRepoNameArray) != 1 && len(fullRepoNameArray) != 3 {
						c.String(http.StatusOK, "Your repository needs to be of the form <repo name> or <repo source>/<repo owner>/<repo name>")
						return
					}

					var pipeline *contracts.Pipeline
					if len(fullRepoNameArray) == 1 {
						pipelines, err := h.cockroachDBClient.GetPipelinesByRepoName(fullRepoName, false)
						if err != nil {
							log.Error().Err(err).Msgf("Failed retrieving pipelines for repo name %v by name", fullRepoName)
							c.String(http.StatusOK, fmt.Sprintf("Retrieving the pipeline for repository %v from the database failed: %v", fullRepoName, err))
							return
						}
						if len(pipelines) <= 0 {
							c.String(http.StatusOK, fmt.Sprintf("The repo %v in your command does not have any estafette builds", fullRepoName))
							return
						}
						if len(pipelines) > 1 {
							commandsExample := ""
							for _, p := range pipelines {
								commandsExample += fmt.Sprintf("/estafette release %v/%v/%v %v %v\n", p.RepoSource, p.RepoOwner, p.RepoName, releaseName, buildVersion)
							}
							c.String(http.StatusOK, fmt.Sprintf("There are multiple pipelines with name %v, use the full name instead:\n%v", fullRepoName, commandsExample))
							return
						}
						pipeline = pipelines[0]
					} else {
						pipeline, err := h.cockroachDBClient.GetPipeline(fullRepoNameArray[0], fullRepoNameArray[1], fullRepoNameArray[2], false)
						if err != nil {
							c.String(http.StatusOK, fmt.Sprintf("Retrieving the pipeline for repository %v from the database failed: %v", fullRepoName, err))
							return
						}
						if pipeline == nil {
							c.String(http.StatusOK, fmt.Sprintf("The repo %v in your command does not have any estafette builds", fullRepoName))
							return
						}
					}

					// check if version exists
					builds, err := h.cockroachDBClient.GetPipelineBuildsByVersion(pipeline.RepoSource, pipeline.RepoOwner, pipeline.RepoName, buildVersion, false)

					if err != nil {
						c.String(http.StatusOK, fmt.Sprintf("Retrieving the build for repository %v and version %v from the database failed: %v", fullRepoName, buildVersion, err))
						return
					}

					var build *contracts.Build
					// get succeeded build
					for _, b := range builds {
						if b.BuildStatus == "succeeded" {
							build = b
							break
						}
					}

					if build == nil {
						c.String(http.StatusOK, fmt.Sprintf("The version %v in your command does not exist", buildVersion))
						return
					}
					if build.BuildStatus != "succeeded" {
						c.String(http.StatusOK, fmt.Sprintf("The build for version %v is not successful and cannot be used", buildVersion))
						return
					}

					// check if release target exists
					releaseExists := false
					for _, releaseTarget := range build.ReleaseTargets {
						if releaseTarget.Name == releaseName {
							// todo support release action

							releaseExists = true
							break
						}
					}

					if !releaseExists {
						c.String(http.StatusOK, fmt.Sprintf("The release %v in your command is not defined in the manifest", releaseName))
						return
					}

					// get user profile from api to set email address for TriggeredBy
					profile, err := h.slackAPIClient.GetUserProfile(slashCommand.UserID)
					if err != nil {
						c.String(http.StatusOK, fmt.Sprintf("Failed retrieving Slack user profile for user id %v: %v", slashCommand.UserID, err))
						return
					}

					// create release object and hand off to build service
					createdRelease, err := h.buildService.CreateRelease(contracts.Release{
						Name:           releaseName,
						Action:         "", // no support for releas action yet
						RepoSource:     build.RepoSource,
						RepoOwner:      build.RepoOwner,
						RepoName:       build.RepoName,
						ReleaseVersion: buildVersion,
						TriggeredBy:    profile.Email,
					}, *build.ManifestObject, build.RepoBranch, build.RepoRevision, false)

					if err != nil {
						errorMessage := fmt.Sprintf("Failed creating release %v for pipeline %v/%v/%v version %v for release command issued by %v", releaseName, build.RepoSource, build.RepoOwner, build.RepoName, buildVersion, profile.Email)
						log.Error().Err(err).Msg(errorMessage)
						c.String(http.StatusOK, fmt.Sprintf("Inserting starting the release: %v", err))
						return
					}

					c.String(http.StatusOK, fmt.Sprintf("Started releasing version %v to %v: %vpipelines/%v/%v/%v/releases/%v/logs", buildVersion, releaseName, h.apiConfig.BaseURL, build.RepoSource, build.RepoOwner, build.RepoName, createdRelease.ID))
					return
				}
			}
		}
	}

	c.String(http.StatusOK, "Aye aye!")
}

func (h *eventHandlerImpl) HasValidVerificationToken(slashCommand slcontracts.SlashCommand) bool {
	return slashCommand.Token == h.config.AppVerificationToken
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"os"
)

const (
	CONTENT_MD5        = "Content-Md5"
	CONTENT_TYPE       = "Content-Type"
	CONTENT_EXPIRES    = "Expires"
	CONTENT_ACL        = "x-amz-acl"
	ENV_AWS_ACCESS_KEY = "AWS_ACCESS_KEY"
	ENV_AWS_SECRET_KEY = "AWS_SECRET_KEY"
)

func abortWith(message string) {
	fmt.Println("ERROR: " + message)
	os.Exit(1)
}

// copied from https://launchpad.net/goamz
var unreserved = make([]bool, 128)
var hex = "0123456789ABCDEF"

func init() {
	// RFC3986
	u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~"
	for _, c := range u {
		unreserved[c] = true
	}
}

func Encode(s string) string {
	encode := false
	for i := 0; i != len(s); i++ {
		c := s[i]
		if c > 127 || !unreserved[c] {
			encode = true
			break
		}
	}
	if !encode {
		return s
	}
	e := make([]byte, len(s)*3)
	ei := 0
	for i := 0; i != len(s); i++ {
		c := s[i]
		if c > 127 || !unreserved[c] {
			e[ei] = '%'
			e[ei+1] = hex[c>>4]
			e[ei+2] = hex[c&0xF]
			ei += 3
		} else {
			e[ei] = c
			ei += 1
		}
	}
	return string(e[:ei])
}
<commit_msg>use same env variables as all other aws tools<commit_after>package aws

import (
	"fmt"
	"os"
)

const (
	CONTENT_MD5            = "Content-Md5"
	CONTENT_TYPE           = "Content-Type"
	CONTENT_EXPIRES        = "Expires"
	CONTENT_ACL            = "x-amz-acl"
	ENV_AWS_ACCESS_KEY     = "AWS_ACCESS_KEY_ID"
	ENV_AWS_SECRET_KEY     = "AWS_SECRET_ACCESS_KEY"
	ENV_AWS_DEFAULT_REGION = "AWS_DEFAULT_REGION"
)

func abortWith(message string) {
	fmt.Println("ERROR: " + message)
	os.Exit(1)
}

// copied from https://launchpad.net/goamz
var unreserved = make([]bool, 128)
var hex = "0123456789ABCDEF"

func init() {
	// RFC3986
	u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~"
	for _, c := range u {
		unreserved[c] = true
	}
}

func Encode(s string) string {
	encode := false
	for i := 0; i != len(s); i++ {
		c := s[i]
		if c > 127 || !unreserved[c] {
			encode = true
			break
		}
	}
	if !encode {
		return s
	}
	e := make([]byte, len(s)*3)
	ei := 0
	for i := 0; i != len(s); i++ {
		c := s[i]
		if c > 127 || !unreserved[c] {
			e[ei] = '%'
			e[ei+1] = hex[c>>4]
			e[ei+2] = hex[c&0xF]
			ei += 3
		} else {
			e[ei] = c
			ei += 1
		}
	}
	return string(e[:ei])
}
<|endoftext|>


<commit_before>package registry

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/registry/v2"
	"github.com/docker/docker/utils"
)

func getV2Builder(e *Endpoint) *v2.URLBuilder {
	if e.URLBuilder == nil {
		e.URLBuilder = v2.NewURLBuilder(e.URL)
	}
	return e.URLBuilder
}

func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) {
	// TODO check if should use Mirror
	if index.Official {
		ep, err = newEndpoint(REGISTRYSERVER, true)
		if err != nil {
			return
		}
		err = validateEndpoint(ep)
		if err != nil {
			return
		}
	} else if r.indexEndpoint.String() == index.GetAuthConfigKey() {
		ep = r.indexEndpoint
	} else {
		ep, err = NewEndpoint(index)
		if err != nil {
			return
		}
	}

	ep.URLBuilder = v2.NewURLBuilder(ep.URL)
	return
}

// GetV2Authorization gets the authorization needed to the given image
// If readonly access is requested, then only the authorization may
// only be used for Get operations.
func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) {
	scopes := []string{"pull"}
	if !readOnly {
		scopes = append(scopes, "push")
	}

	log.Debugf("Getting authorization for %s %s", imageName, scopes)
	return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil
}

//
// 1) Check if TarSum of each layer exists /v2/
//  1.a) if 200, continue
//  1.b) if 300, then push the
//  1.c) if anything else, err
// 2) PUT the created/signed manifest
//
func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) {
	routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)

	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return nil, err
	}
	if err := auth.Authorize(req); err != nil {
		return nil, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res)
	}

	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Error while reading the http response: %s", err)
	}
	return buf, nil
}

// - Succeeded to head image blob (already exists)
// - Failed with no error (continue to Push the Blob)
// - Failed with error
func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) {
	routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum)
	if err != nil {
		return false, err
	}

	method := "HEAD"
	log.Debugf("[registry] Calling %q %s", method, routeURL)

	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return false, err
	}
	if err := auth.Authorize(req); err != nil {
		return false, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return false, err
	}
	res.Body.Close() // close early, since we're not needing a body on this call .. yet?
	switch res.StatusCode {
	case 200:
		// return something indicating no push needed
		return true, nil
	case 404:
		// return something indicating blob push needed
		return false, nil
	}
	return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s:%s", res.StatusCode, imageName, sumType, sum), res)
}

func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error {
	routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum)
	if err != nil {
		return err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)
	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return err
	}
	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
	}

	_, err = io.Copy(blobWrtr, res.Body)
	return err
}

func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) {
	routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum)
	if err != nil {
		return nil, 0, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)
	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return nil, 0, err
	}
	if err := auth.Authorize(req); err != nil {
		return nil, 0, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, 0, err
	}
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, 0, errLoginRequired
		}
		return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s:%s", res.StatusCode, imageName, sumType, sum), res)
	}
	lenStr := res.Header.Get("Content-Length")
	l, err := strconv.ParseInt(lenStr, 10, 64)
	if err != nil {
		return nil, 0, err
	}

	return res.Body, l, err
}

// Push the image to the server for storage.
// 'layer' is an uncompressed reader of the blob to be pushed.
// The server will generate it's own checksum calculation.
func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error {
	routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName)
	if err != nil {
		return err
	}

	log.Debugf("[registry] Calling %q %s", "POST", routeURL)
	req, err := r.reqFactory.NewRequest("POST", routeURL, nil)
	if err != nil {
		return err
	}

	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	location := res.Header.Get("Location")

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, location)
	req, err = r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr))
	if err != nil {
		return err
	}
	queryParams := req.URL.Query()
	queryParams.Add("digest", sumType+":"+sumStr)
	req.URL.RawQuery = queryParams.Encode()
	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err = r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode != 201 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		errBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}
		log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header)
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s:%s", res.StatusCode, imageName, sumType, sumStr), res)
	}

	return nil
}

// Finally Push the (signed) manifest of the blobs we've just pushed
func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error {
	routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName)
	if err != nil {
		return err
	}

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, routeURL)
	req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr)
	if err != nil {
		return err
	}
	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		errBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}
		log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header)
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res)
	}

	return nil
}

type remoteTags struct {
	name string
	tags []string
}

// Given a repository name, returns a json array of string tags
func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) {
	routeURL, err := getV2Builder(ep).BuildTagsURL(imageName)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)

	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return nil, err
	}
	if err := auth.Authorize(req); err != nil {
		return nil, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res)
	}

	decoder := json.NewDecoder(res.Body)
	var remote remoteTags
	err = decoder.Decode(&remote)
	if err != nil {
		return nil, fmt.Errorf("Error while decoding the http response: %s", err)
	}
	return remote.tags, nil
}
<commit_msg>Open up v2 http status code checks for put and head checks<commit_after>package registry

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/registry/v2"
	"github.com/docker/docker/utils"
)

func getV2Builder(e *Endpoint) *v2.URLBuilder {
	if e.URLBuilder == nil {
		e.URLBuilder = v2.NewURLBuilder(e.URL)
	}
	return e.URLBuilder
}

func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) {
	// TODO check if should use Mirror
	if index.Official {
		ep, err = newEndpoint(REGISTRYSERVER, true)
		if err != nil {
			return
		}
		err = validateEndpoint(ep)
		if err != nil {
			return
		}
	} else if r.indexEndpoint.String() == index.GetAuthConfigKey() {
		ep = r.indexEndpoint
	} else {
		ep, err = NewEndpoint(index)
		if err != nil {
			return
		}
	}

	ep.URLBuilder = v2.NewURLBuilder(ep.URL)
	return
}

// GetV2Authorization gets the authorization needed to the given image
// If readonly access is requested, then only the authorization may
// only be used for Get operations.
func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) {
	scopes := []string{"pull"}
	if !readOnly {
		scopes = append(scopes, "push")
	}

	log.Debugf("Getting authorization for %s %s", imageName, scopes)
	return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil
}

//
// 1) Check if TarSum of each layer exists /v2/
//  1.a) if 200, continue
//  1.b) if 300, then push the
//  1.c) if anything else, err
// 2) PUT the created/signed manifest
//
func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) {
	routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)

	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return nil, err
	}
	if err := auth.Authorize(req); err != nil {
		return nil, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res)
	}

	buf, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("Error while reading the http response: %s", err)
	}
	return buf, nil
}

// - Succeeded to head image blob (already exists)
// - Failed with no error (continue to Push the Blob)
// - Failed with error
func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) {
	routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum)
	if err != nil {
		return false, err
	}

	method := "HEAD"
	log.Debugf("[registry] Calling %q %s", method, routeURL)

	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return false, err
	}
	if err := auth.Authorize(req); err != nil {
		return false, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return false, err
	}
	res.Body.Close() // close early, since we're not needing a body on this call .. yet?
	switch {
	case res.StatusCode >= 200 && res.StatusCode < 400:
		// return something indicating no push needed
		return true, nil
	case res.StatusCode == 404:
		// return something indicating blob push needed
		return false, nil
	}

	return false, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying head request for %s - %s:%s", res.StatusCode, imageName, sumType, sum), res)
}

func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error {
	routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum)
	if err != nil {
		return err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)
	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return err
	}
	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
	}

	_, err = io.Copy(blobWrtr, res.Body)
	return err
}

func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) {
	routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum)
	if err != nil {
		return nil, 0, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)
	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return nil, 0, err
	}
	if err := auth.Authorize(req); err != nil {
		return nil, 0, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, 0, err
	}
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, 0, errLoginRequired
		}
		return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob - %s:%s", res.StatusCode, imageName, sumType, sum), res)
	}
	lenStr := res.Header.Get("Content-Length")
	l, err := strconv.ParseInt(lenStr, 10, 64)
	if err != nil {
		return nil, 0, err
	}

	return res.Body, l, err
}

// Push the image to the server for storage.
// 'layer' is an uncompressed reader of the blob to be pushed.
// The server will generate it's own checksum calculation.
func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error {
	routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName)
	if err != nil {
		return err
	}

	log.Debugf("[registry] Calling %q %s", "POST", routeURL)
	req, err := r.reqFactory.NewRequest("POST", routeURL, nil)
	if err != nil {
		return err
	}

	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	location := res.Header.Get("Location")

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, location)
	req, err = r.reqFactory.NewRequest(method, location, ioutil.NopCloser(blobRdr))
	if err != nil {
		return err
	}
	queryParams := req.URL.Query()
	queryParams.Add("digest", sumType+":"+sumStr)
	req.URL.RawQuery = queryParams.Encode()
	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err = r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode != 201 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		errBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}
		log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header)
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob - %s:%s", res.StatusCode, imageName, sumType, sumStr), res)
	}

	return nil
}

// Finally Push the (signed) manifest of the blobs we've just pushed
func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error {
	routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName)
	if err != nil {
		return err
	}

	method := "PUT"
	log.Debugf("[registry] Calling %q %s", method, routeURL)
	req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr)
	if err != nil {
		return err
	}
	if err := auth.Authorize(req); err != nil {
		return err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	// All 2xx and 3xx responses can be accepted for a put.
	if res.StatusCode >= 400 {
		if res.StatusCode == 401 {
			return errLoginRequired
		}
		errBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}
		log.Debugf("Unexpected response from server: %q %#v", errBody, res.Header)
		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res)
	}

	return nil
}

type remoteTags struct {
	name string
	tags []string
}

// Given a repository name, returns a json array of string tags
func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) {
	routeURL, err := getV2Builder(ep).BuildTagsURL(imageName)
	if err != nil {
		return nil, err
	}

	method := "GET"
	log.Debugf("[registry] Calling %q %s", method, routeURL)

	req, err := r.reqFactory.NewRequest(method, routeURL, nil)
	if err != nil {
		return nil, err
	}
	if err := auth.Authorize(req); err != nil {
		return nil, err
	}
	res, _, err := r.doRequest(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 401 {
			return nil, errLoginRequired
		} else if res.StatusCode == 404 {
			return nil, ErrDoesNotExist
		}
		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res)
	}

	decoder := json.NewDecoder(res.Body)
	var remote remoteTags
	err = decoder.Decode(&remote)
	if err != nil {
		return nil, fmt.Errorf("Error while decoding the http response: %s", err)
	}
	return remote.tags, nil
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheusreceiver

import (
	"context"
	"time"

	"github.com/prometheus/prometheus/discovery"
	"github.com/prometheus/prometheus/scrape"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/receiver/prometheusreceiver/internal"
)

// pReceiver is the type that provides Prometheus scraper/receiver functionality.
type pReceiver struct {
	cfg        *Config
	consumer   consumer.MetricsConsumer
	cancelFunc context.CancelFunc

	logger *zap.Logger
}

// New creates a new prometheus.Receiver reference.
func newPrometheusReceiver(logger *zap.Logger, cfg *Config, next consumer.MetricsConsumer) *pReceiver {
	pr := &pReceiver{
		cfg:      cfg,
		consumer: next,
		logger:   logger,
	}
	return pr
}

// Start is the method that starts Prometheus scraping and it
// is controlled by having previously defined a Configuration using perhaps New.
func (r *pReceiver) Start(ctx context.Context, host component.Host) error {
	discoveryCtx, cancel := context.WithCancel(context.Background())
	r.cancelFunc = cancel

	logger := internal.NewZapToGokitLogAdapter(r.logger)

	discoveryManager := discovery.NewManager(discoveryCtx, logger)
	discoveryCfg := make(map[string]discovery.Configs)
	for _, scrapeConfig := range r.cfg.PrometheusConfig.ScrapeConfigs {
		discoveryCfg[scrapeConfig.JobName] = scrapeConfig.ServiceDiscoveryConfigs
	}
	if err := discoveryManager.ApplyConfig(discoveryCfg); err != nil {
		return err
	}
	go func() {
		if err := discoveryManager.Run(); err != nil {
			r.logger.Error("Discovery manager failed", zap.Error(err))
			host.ReportFatalError(err)
		}
	}()

	var jobsMap *internal.JobsMap
	if !r.cfg.UseStartTimeMetric {
		jobsMap = internal.NewJobsMap(2 * time.Minute)
	}
	ocaStore := internal.NewOcaStore(ctx, r.consumer, r.logger, jobsMap, r.cfg.UseStartTimeMetric, r.cfg.StartTimeMetricRegex, r.cfg.Name())

	scrapeManager := scrape.NewManager(logger, ocaStore)
	ocaStore.SetScrapeManager(scrapeManager)
	if err := scrapeManager.ApplyConfig(r.cfg.PrometheusConfig); err != nil {
		return err
	}
	go func() {
		if err := scrapeManager.Run(discoveryManager.SyncCh()); err != nil {
			r.logger.Error("Scrape manager failed", zap.Error(err))
			host.ReportFatalError(err)
		}
	}()
	return nil
}

// Shutdown stops and cancels the underlying Prometheus scrapers.
func (r *pReceiver) Shutdown(context.Context) error {
	r.cancelFunc()
	return nil
}
<commit_msg>propogate name and transport for prometheus receiver and exporter (#2680)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheusreceiver

import (
	"context"
	"time"

	"github.com/prometheus/prometheus/discovery"
	"github.com/prometheus/prometheus/scrape"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/obsreport"
	"go.opentelemetry.io/collector/receiver/prometheusreceiver/internal"
)

const transport = "http"

// pReceiver is the type that provides Prometheus scraper/receiver functionality.
type pReceiver struct {
	cfg        *Config
	consumer   consumer.MetricsConsumer
	cancelFunc context.CancelFunc

	logger *zap.Logger
}

// New creates a new prometheus.Receiver reference.
func newPrometheusReceiver(logger *zap.Logger, cfg *Config, next consumer.MetricsConsumer) *pReceiver {
	pr := &pReceiver{
		cfg:      cfg,
		consumer: next,
		logger:   logger,
	}
	return pr
}

// Start is the method that starts Prometheus scraping and it
// is controlled by having previously defined a Configuration using perhaps New.
func (r *pReceiver) Start(ctx context.Context, host component.Host) error {
	discoveryCtx, cancel := context.WithCancel(context.Background())
	r.cancelFunc = cancel

	logger := internal.NewZapToGokitLogAdapter(r.logger)

	discoveryManager := discovery.NewManager(discoveryCtx, logger)
	discoveryCfg := make(map[string]discovery.Configs)
	for _, scrapeConfig := range r.cfg.PrometheusConfig.ScrapeConfigs {
		discoveryCfg[scrapeConfig.JobName] = scrapeConfig.ServiceDiscoveryConfigs
	}
	if err := discoveryManager.ApplyConfig(discoveryCfg); err != nil {
		return err
	}
	go func() {
		if err := discoveryManager.Run(); err != nil {
			r.logger.Error("Discovery manager failed", zap.Error(err))
			host.ReportFatalError(err)
		}
	}()

	var jobsMap *internal.JobsMap
	if !r.cfg.UseStartTimeMetric {
		jobsMap = internal.NewJobsMap(2 * time.Minute)
	}
	receiverCtx := obsreport.ReceiverContext(ctx, r.cfg.Name(), transport)
	ocaStore := internal.NewOcaStore(receiverCtx, r.consumer, r.logger, jobsMap, r.cfg.UseStartTimeMetric, r.cfg.StartTimeMetricRegex, r.cfg.Name())

	scrapeManager := scrape.NewManager(logger, ocaStore)
	ocaStore.SetScrapeManager(scrapeManager)
	if err := scrapeManager.ApplyConfig(r.cfg.PrometheusConfig); err != nil {
		return err
	}
	go func() {
		if err := scrapeManager.Run(discoveryManager.SyncCh()); err != nil {
			r.logger.Error("Scrape manager failed", zap.Error(err))
			host.ReportFatalError(err)
		}
	}()
	return nil
}

// Shutdown stops and cancels the underlying Prometheus scrapers.
func (r *pReceiver) Shutdown(context.Context) error {
	r.cancelFunc()
	return nil
}
<|endoftext|>


<commit_before>package bashcomp

import (
	"flag"
	"os"
	"path"
	"strings"
	"text/template"
)

const (
	completionFlagName = "bash-completion"
)

var (
	bashCompleteEnabled = flag.Bool(completionFlagName, false, "dump bash completion source")
)

type values struct {
	Command    string
	Flags      string
	AllowFiles bool
	RawArgs    string
}

// Generate bash completion source file from command line flags for a command.
// This is for commands that accept file names as arguments.
func HandleBashCompletion() {
	handleBashCompletionWithOptions(true)
}

// Generate bash completion source file from command line flags for a command.
//
// This is for commands that take fixed set of strings as arguments, not file names.
func HandleBashCompletionNoFiles(rawArgs ...string) {
	handleBashCompletionWithOptions(false, rawArgs...)
}

func handleBashCompletionWithOptions(allowFiles bool, rawArgs ...string) {
	if !flag.Parsed() {
		flag.Parse()
	}
	if !*bashCompleteEnabled {
		return
	}
	templ, err := template.New("template").Parse(source)
	if err != nil {
		panic(err)
	}
	v := values{}
	v.Command = path.Base(os.Args[0])
	v.AllowFiles = allowFiles
	v.RawArgs = strings.Join(rawArgs, " ")

	v.Flags = ""
	flag.VisitAll(func(flag *flag.Flag) {
		if flag.Name != completionFlagName {
			v.Flags = v.Flags + " -" + flag.Name
		}
	})

	err = templ.Execute(os.Stdout, v)
	if err != nil {
		panic(err)
	}
	os.Exit(0)
}

var source = `
# Bash autocomplete script for the {{.Command}} command.
# Source it with the following command:
# . <({{.Command}} -bash-completion)
_{{ .Command }}_complete() {
  local cur="${COMP_WORDS[COMP_CWORD]}"

  local cand=""
  case "$cur" in
    -*)
      cand="{{.Flags}}"
      ;;
  esac
  if [ "x$cand" = "x" ] ; then
    COMPREPLY=(
        {{if.AllowFiles}}
        $(compgen -f -- ${cur})
        {{else}}
        $(compgen -W "{{.RawArgs}}" -- ${cur})
        {{end}}
        )
  else
    COMPREPLY=($(compgen -W "$cand" -- ${cur}))
  fi
}

complete -o filenames -o bashdefault -F _{{.Command}}_complete {{.Command}}
`
<commit_msg>Show help on empty completion<commit_after>package bashcomp

import (
	"flag"
	"os"
	"path"
	"strings"
	"text/template"
)

const (
	completionFlagName = "bash-completion"
)

var (
	bashCompleteEnabled = flag.Bool(completionFlagName, false, "dump bash completion source")
)

type values struct {
	Command    string
	Flags      string
	AllowFiles bool
	RawArgs    string
}

// Generate bash completion source file from command line flags for a command.
// This is for commands that accept file names as arguments.
func HandleBashCompletion() {
	handleBashCompletionWithOptions(true)
}

// Generate bash completion source file from command line flags for a command.
//
// This is for commands that take fixed set of strings as arguments, not file names.
func HandleBashCompletionNoFiles(rawArgs ...string) {
	handleBashCompletionWithOptions(false, rawArgs...)
}

func handleBashCompletionWithOptions(allowFiles bool, rawArgs ...string) {
	if !flag.Parsed() {
		flag.Parse()
	}
	if !*bashCompleteEnabled {
		return
	}
	templ, err := template.New("template").Parse(source)
	if err != nil {
		panic(err)
	}
	v := values{}
	v.Command = path.Base(os.Args[0])
	v.AllowFiles = allowFiles
	v.RawArgs = strings.Join(rawArgs, " ")

	v.Flags = ""
	flag.VisitAll(func(flag *flag.Flag) {
		if flag.Name != completionFlagName {
			v.Flags = v.Flags + " -" + flag.Name
		}
	})

	err = templ.Execute(os.Stdout, v)
	if err != nil {
		panic(err)
	}
	os.Exit(0)
}

var source = `
# Bash autocomplete script for the {{.Command}} command.
# Source it with the following command:
# . <({{.Command}} -bash-completion)
_{{ .Command }}_complete() {
  local cur="${COMP_WORDS[COMP_CWORD]}"

  local cand=""
  case "$cur" in
    "")
      {{ .Command }} -h >/dev/tty
      ;;
    -*)
      cand="{{.Flags}}"
      ;;
  esac
  if [ "x$cand" = "x" ] ; then
    COMPREPLY=(
        {{if.AllowFiles}}
        $(compgen -f -- ${cur})
        {{else}}
        $(compgen -W "{{.RawArgs}}" -- ${cur})
        {{end}}
        )
  else
    COMPREPLY=($(compgen -W "$cand" -- ${cur}))
  fi
}

complete -o filenames -o bashdefault -F _{{.Command}}_complete {{.Command}}
`
<|endoftext|>


<commit_before>package datastore

import (
	"log"

	dsq "github.com/jbenet/go-datastore/query"
)

// Here are some basic datastore implementations.

type keyMap map[Key]interface{}

// MapDatastore uses a standard Go map for internal storage.
type MapDatastore struct {
	values keyMap
}

// NewMapDatastore constructs a MapDatastore
func NewMapDatastore() (d *MapDatastore) {
	return &MapDatastore{
		values: keyMap{},
	}
}

// Put implements Datastore.Put
func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
	d.values[key] = value
	return nil
}

// Get implements Datastore.Get
func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
	val, found := d.values[key]
	if !found {
		return nil, ErrNotFound
	}
	return val, nil
}

// Has implements Datastore.Has
func (d *MapDatastore) Has(key Key) (exists bool, err error) {
	_, found := d.values[key]
	return found, nil
}

// Delete implements Datastore.Delete
func (d *MapDatastore) Delete(key Key) (err error) {
	delete(d.values, key)
	return nil
}

// Query implements Datastore.Query
func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) {
	re := make([]dsq.Entry, 0, len(d.values))
	for k, v := range d.values {
		re = append(re, dsq.Entry{Key: k.String(), Value: v})
	}
	r := dsq.ResultsWithEntries(q, re)
	r = dsq.NaiveQueryApply(q, r)
	return r, nil
}

// NullDatastore stores nothing, but conforms to the API.
// Useful to test with.
type NullDatastore struct {
}

// NewNullDatastore constructs a null datastoe
func NewNullDatastore() *NullDatastore {
	return &NullDatastore{}
}

// Put implements Datastore.Put
func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
	return nil
}

// Get implements Datastore.Get
func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
	return nil, nil
}

// Has implements Datastore.Has
func (d *NullDatastore) Has(key Key) (exists bool, err error) {
	return false, nil
}

// Delete implements Datastore.Delete
func (d *NullDatastore) Delete(key Key) (err error) {
	return nil
}

// Query implements Datastore.Query
func (d *NullDatastore) Query(q dsq.Query) (dsq.Results, error) {
	return dsq.ResultsWithEntries(q, nil), nil
}

// LogDatastore logs all accesses through the datastore.
type LogDatastore struct {
	Name  string
	child Datastore
}

// Shim is a datastore which has a child.
type Shim interface {
	Datastore

	Children() []Datastore
}

// NewLogDatastore constructs a log datastore.
func NewLogDatastore(ds Datastore, name string) Shim {
	if len(name) < 1 {
		name = "LogDatastore"
	}
	return &LogDatastore{Name: name, child: ds}
}

// Children implements Shim
func (d *LogDatastore) Children() []Datastore {
	return []Datastore{d.child}
}

// Put implements Datastore.Put
func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
	log.Printf("%s: Put %s\n", d.Name, key)
	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
	return d.child.Put(key, value)
}

// Get implements Datastore.Get
func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
	log.Printf("%s: Get %s\n", d.Name, key)
	return d.child.Get(key)
}

// Has implements Datastore.Has
func (d *LogDatastore) Has(key Key) (exists bool, err error) {
	log.Printf("%s: Has %s\n", d.Name, key)
	return d.child.Has(key)
}

// Delete implements Datastore.Delete
func (d *LogDatastore) Delete(key Key) (err error) {
	log.Printf("%s: Delete %s\n", d.Name, key)
	return d.child.Delete(key)
}

// Query implements Datastore.Query
func (d *LogDatastore) Query(q dsq.Query) (dsq.Results, error) {
	log.Printf("%s: Query\n", d.Name)
	return d.child.Query(q)
}
<commit_msg>MapDatastore.Delete now returns ErrNotFound<commit_after>package datastore

import (
	"log"

	dsq "github.com/jbenet/go-datastore/query"
)

// Here are some basic datastore implementations.

type keyMap map[Key]interface{}

// MapDatastore uses a standard Go map for internal storage.
type MapDatastore struct {
	values keyMap
}

// NewMapDatastore constructs a MapDatastore
func NewMapDatastore() (d *MapDatastore) {
	return &MapDatastore{
		values: keyMap{},
	}
}

// Put implements Datastore.Put
func (d *MapDatastore) Put(key Key, value interface{}) (err error) {
	d.values[key] = value
	return nil
}

// Get implements Datastore.Get
func (d *MapDatastore) Get(key Key) (value interface{}, err error) {
	val, found := d.values[key]
	if !found {
		return nil, ErrNotFound
	}
	return val, nil
}

// Has implements Datastore.Has
func (d *MapDatastore) Has(key Key) (exists bool, err error) {
	_, found := d.values[key]
	return found, nil
}

// Delete implements Datastore.Delete
func (d *MapDatastore) Delete(key Key) (err error) {
	if _, found := d.values[key]; !found {
		return ErrNotFound
	}
	delete(d.values, key)
	return nil
}

// Query implements Datastore.Query
func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) {
	re := make([]dsq.Entry, 0, len(d.values))
	for k, v := range d.values {
		re = append(re, dsq.Entry{Key: k.String(), Value: v})
	}
	r := dsq.ResultsWithEntries(q, re)
	r = dsq.NaiveQueryApply(q, r)
	return r, nil
}

// NullDatastore stores nothing, but conforms to the API.
// Useful to test with.
type NullDatastore struct {
}

// NewNullDatastore constructs a null datastoe
func NewNullDatastore() *NullDatastore {
	return &NullDatastore{}
}

// Put implements Datastore.Put
func (d *NullDatastore) Put(key Key, value interface{}) (err error) {
	return nil
}

// Get implements Datastore.Get
func (d *NullDatastore) Get(key Key) (value interface{}, err error) {
	return nil, nil
}

// Has implements Datastore.Has
func (d *NullDatastore) Has(key Key) (exists bool, err error) {
	return false, nil
}

// Delete implements Datastore.Delete
func (d *NullDatastore) Delete(key Key) (err error) {
	return nil
}

// Query implements Datastore.Query
func (d *NullDatastore) Query(q dsq.Query) (dsq.Results, error) {
	return dsq.ResultsWithEntries(q, nil), nil
}

// LogDatastore logs all accesses through the datastore.
type LogDatastore struct {
	Name  string
	child Datastore
}

// Shim is a datastore which has a child.
type Shim interface {
	Datastore

	Children() []Datastore
}

// NewLogDatastore constructs a log datastore.
func NewLogDatastore(ds Datastore, name string) Shim {
	if len(name) < 1 {
		name = "LogDatastore"
	}
	return &LogDatastore{Name: name, child: ds}
}

// Children implements Shim
func (d *LogDatastore) Children() []Datastore {
	return []Datastore{d.child}
}

// Put implements Datastore.Put
func (d *LogDatastore) Put(key Key, value interface{}) (err error) {
	log.Printf("%s: Put %s\n", d.Name, key)
	// log.Printf("%s: Put %s ```%s```", d.Name, key, value)
	return d.child.Put(key, value)
}

// Get implements Datastore.Get
func (d *LogDatastore) Get(key Key) (value interface{}, err error) {
	log.Printf("%s: Get %s\n", d.Name, key)
	return d.child.Get(key)
}

// Has implements Datastore.Has
func (d *LogDatastore) Has(key Key) (exists bool, err error) {
	log.Printf("%s: Has %s\n", d.Name, key)
	return d.child.Has(key)
}

// Delete implements Datastore.Delete
func (d *LogDatastore) Delete(key Key) (err error) {
	log.Printf("%s: Delete %s\n", d.Name, key)
	return d.child.Delete(key)
}

// Query implements Datastore.Query
func (d *LogDatastore) Query(q dsq.Query) (dsq.Results, error) {
	log.Printf("%s: Query\n", d.Name)
	return d.child.Query(q)
}
<|endoftext|>


<commit_before>package graph

import (
	"os"
	"time"
)

// A user profile.
// http://developers.facebook.com/docs/reference/api/user
type User struct {
	// Identification
	ID string
	// First name
	FirstName string
	// Last name
	LastName string
	// Full name
	Name string
	// A link to the profile
	Link string
	// The Blurb that appears under the profile picture
	About string
	// Birthday
	Birthday *time.Time
	// Work history list
	Work []Workplace
	// Education history list
	Educations []Education
	// The contact email adress
	Email string
	// Link to the personal website
	Website string
	// Hometown
	Hometown Object
	// Current location
	Location Object
	// Biography
	Bio string
	// Favorite quotes
	Quotes string
	// Gender
	Gender string
	// Genders the user is interested in
	InterestedIn string
	// Types of relationships the user is seeking for
	MeetingFor string
	// Relationship status
	RelationshipStatus string
	// Religion
	Religion string
	// Political view
	Political string
	// Verification status
	Verified string
	// The user's significant other
	SignificantOther string
	// Timezone
	Timezone string
	// An anonymous, but unique identifier for the user. Available to everyone on Facebook.
	ThirdPartyID string
	// The last time the user's profile was updated. Available to everyone on Facebook.
	LastUpdated *time.Time
	// The user's locale. Publicly available. A JSON string containing the ISO Language Code and ISO Country Code.
	Locale string
	/*
		// ##### Connections #####
		// TODO: Replace all strings with actual Connection structs
		// The News Feed. Requires read_stream permission
		Home []Post
		// Wall. Requires read_stream permission to see non-public posts.
		Feed []Post
		// Photos, videos and posts in which the user has been tagged. Requires read_stream permission.
		//Tagged TODO
		// Own posts. Requires read_stream permission to see non-public posts.
		Posts string
		// Profile picture
		Picture *Picture
		// Friends of the user
		Friends string
		// Activities listed on the profile page
		Activities string
		// Interests listed on the profile page
		Interests string
		// Music listed on the profile page
		Music string
		// Books listed on the profile page
		Books string
		// Movies listed on the profile page
		Movies string
		// Television listed on the profile pages
		Television string
		// Pages this user has liked. Requires user_likes or friend_likes permission
		Likes string
		// Photos this user is tagged in. Requires user_photo_video_tags, friend_photo_video_tags and user_photos or friend_photos permissions
		Photos string
		// Photo albums this user has created. Requires user_photos or friend_photos permission
		Albums string
		// Videos this user has been tagged in. Requires user_videos or friend_videos permission
		Videos string
		// Groups this user is a member of. Requires user_groups or friend_groups permission
		Groups string
		// Status updates. Requires read_stream permission
		Statuses string
		// Posted links. Requires read_stream permission
		Links string
		// Notes. Requires read_stream permission
		Notes string
		// Events this user is attending. Requires user_events or friend_events permission
		Events string
		// Threads in this user's inbox. Requires read_mailbox permission
		InBox string
		// Messages in this user's outbox. Requires read_mailbox permission
		OutBox string
		// Updates in this user's inbox. Requires read_mailbox permission
		Updates string
	*/
	/* The Facebook pages owned by the current user. If the manage_pages permission has been granted,
	 * this connection also yields access_tokens that can be used to query the Graph API on behalf of the page.
	 */
	Accounts string
	// Places the current user has checked-into.
	Checkins string
	/* The user's outstanding requests for the app associated with the access token.
	 * The access token should be app secret signed and not user session signed. See more info here.
	 */
	PlatformRequests string

	// Not documented in the API but streamed
	UpdatedTime     *time.Time
	FanCount        float64
	Mission         string
	Category        string
	Username        string
	Products        string
	Founded         *time.Time
	CompanyOverview string
}

func (u *User) String() string {
	return "ID: " + u.ID + "\tName: " + u.Name + "\tFirst name: " + u.FirstName +
		"\tLast name: " + u.LastName + "\tLink: " + u.Link + "\tGender: " +
		u.Gender + "\tLocale: " + u.Locale + "\tUpdated time: " + u.UpdatedTime.String() +
		"\n"
}

func parseUser(data map[string]interface{}) (user User, err os.Error) {
	for key, value := range data {
		switch key {
		case "id":
			user.ID = value.(string)
		case "first_name":
			user.FirstName = value.(string)
		case "last_name":
			user.LastName = value.(string)
		case "name":
			user.Name = value.(string)
		case "link":
			user.Link = value.(string)
		case "about":
			user.About = value.(string)
		case "birthday":
			user.Birthday, err = parseTime(value.(string))
		case "work":
			user.Work, err = parseWork(value.([]interface{}))
		case "education":
			user.Educations = parseEducations(value.([]interface{}))
		case "email":
			user.Email = value.(string)
		case "website":
			user.Website = value.(string)
		case "hometown":
			user.Hometown = parseObject(value.(map[string]interface{}))
		case "location":
			user.Location = parseObject(value.(map[string]interface{}))
		case "bio":
			user.Bio = value.(string)
		case "quotes":
			user.Quotes = value.(string)
		case "gender":
			user.Gender = value.(string)
		case "interested_in":
			user.InterestedIn = value.(string)
		case "meeting_for":
			user.MeetingFor = value.(string)
		case "relationship_status":
			user.RelationshipStatus = value.(string)
		case "religion":
			user.Religion = value.(string)
		case "political":
			user.Political = value.(string)
		case "verified":
			user.Verified = value.(string)
		case "significant_other":
			user.SignificantOther = value.(string)
		case "timezone":
			user.Timezone = value.(string)
		case "third_party_id":
			user.ThirdPartyID = value.(string)
		case "last_updated":
			user.LastUpdated, err = parseTime(value.(string))
		case "locale":
			user.Locale = value.(string)

		// Not documented in the API but streamed
		case "mission":
			user.Mission = value.(string)
		case "category":
			user.Category = value.(string)
		case "username":
			user.Username = value.(string)
		case "products":
			user.Products = value.(string)
		case "founded":
			user.Founded, err = parseTime(value.(string))
		case "company_overview":
			user.CompanyOverview = value.(string)
		case "fan_count":
			user.FanCount = value.(float64)
		case "type":
			// TODO: Look into type

			// Parse metadata if requested
			/*
				case "metadata":
					// TODO: get and parse connections
					metadata := value.(map[string]interface{})
					for k, v := range metadata["connections"].(map[string]interface{}) {
						switch k {
						case "home":
							// TODO: You can only access the "home" connection for the current user.
							//user.Home, err = GetPosts(v.(string)) // Pass URL
						case "feed":
							//err = FetchData(user.Feed, v.(string)) // Pass URL
						case "tagged":
							//err = FetchData(user.Tagged, v.(string)) // Pass URL
						case "picture":
							user.Picture = NewPicture(v.(string)) // Pass URL
						}
					}
			*/
		}
	}
	return
}
<commit_msg>Add parsing and saving of connections fields to User.<commit_after>package graph

import (
	"os"
	"time"
)

// A user profile.
// http://developers.facebook.com/docs/reference/api/user
type User struct {
	// Identification
	ID string
	// First name
	FirstName string
	// Last name
	LastName string
	// Full name
	Name string
	// A link to the profile
	Link string
	// The Blurb that appears under the profile picture
	About string
	// Birthday
	Birthday *time.Time
	// Work history list
	Work []Workplace
	// Education history list
	Educations []Education
	// The contact email adress
	Email string
	// Link to the personal website
	Website string
	// Hometown
	Hometown Object
	// Current location
	Location Object
	// Biography
	Bio string
	// Favorite quotes
	Quotes string
	// Gender
	Gender string
	// Genders the user is interested in
	InterestedIn string
	// Types of relationships the user is seeking for
	MeetingFor string
	// Relationship status
	RelationshipStatus string
	// Religion
	Religion string
	// Political view
	Political string
	// Verification status
	Verified string
	// The user's significant other
	SignificantOther string
	// Timezone
	Timezone string
	// An anonymous, but unique identifier for the user. Available to everyone on Facebook.
	ThirdPartyID string
	// The last time the user's profile was updated. Available to everyone on Facebook.
	LastUpdated *time.Time
	// The user's locale. Publicly available. A JSON string containing the ISO Language Code and ISO Country Code.
	Locale string
	/* The Facebook pages owned by the current user. If the manage_pages permission has been granted,
	 * this connection also yields access_tokens that can be used to query the Graph API on behalf of the page.
	 */
	Accounts string
	// Places the current user has checked-into.
	Checkins string
	/* The user's outstanding requests for the app associated with the access token.
	 * The access token should be app secret signed and not user session signed. See more info here.
	 */
	PlatformRequests string

	// Not documented in the API but streamed
	UpdatedTime     *time.Time
	FanCount        float64
	Mission         string
	Category        string
	Username        string
	Products        string
	Founded         *time.Time
	CompanyOverview string

	// ##### Connections #####
	home             string
	feed             string
	tagged           string
	posts            string
	picture          string
	friends          string
	activities       string
	interests        string
	music            string
	books            string
	television       string
	likes            string
	photos           string
	albums           string
	videos           string
	groups           string
	statuses         string
	links            string
	notes            string
	events           string
	inbox            string
	outbox           string
	updates          string
	accounts         string
	checkins         string
	platformrequests string
	friendlists      string
}

func (u *User) String() string {
	return "ID: " + u.ID + "\tName: " + u.Name + "\tFirst name: " + u.FirstName +
		"\tLast name: " + u.LastName + "\tLink: " + u.Link + "\tGender: " +
		u.Gender + "\tLocale: " + u.Locale + "\tUpdated time: " + u.UpdatedTime.String() +
		"\n"
}

func parseUser(data map[string]interface{}) (user User, err os.Error) {
	for key, value := range data {
		switch key {
		case "id":
			user.ID = value.(string)
		case "first_name":
			user.FirstName = value.(string)
		case "last_name":
			user.LastName = value.(string)
		case "name":
			user.Name = value.(string)
		case "link":
			user.Link = value.(string)
		case "about":
			user.About = value.(string)
		case "birthday":
			user.Birthday, err = parseTime(value.(string))
		case "work":
			user.Work, err = parseWork(value.([]interface{}))
		case "education":
			user.Educations = parseEducations(value.([]interface{}))
		case "email":
			user.Email = value.(string)
		case "website":
			user.Website = value.(string)
		case "hometown":
			user.Hometown = parseObject(value.(map[string]interface{}))
		case "location":
			user.Location = parseObject(value.(map[string]interface{}))
		case "bio":
			user.Bio = value.(string)
		case "quotes":
			user.Quotes = value.(string)
		case "gender":
			user.Gender = value.(string)
		case "interested_in":
			user.InterestedIn = value.(string)
		case "meeting_for":
			user.MeetingFor = value.(string)
		case "relationship_status":
			user.RelationshipStatus = value.(string)
		case "religion":
			user.Religion = value.(string)
		case "political":
			user.Political = value.(string)
		case "verified":
			user.Verified = value.(string)
		case "significant_other":
			user.SignificantOther = value.(string)
		case "timezone":
			user.Timezone = value.(string)
		case "third_party_id":
			user.ThirdPartyID = value.(string)
		case "last_updated":
			user.LastUpdated, err = parseTime(value.(string))
		case "locale":
			user.Locale = value.(string)

		// Not documented in the API but streamed
		case "mission":
			user.Mission = value.(string)
		case "category":
			user.Category = value.(string)
		case "username":
			user.Username = value.(string)
		case "products":
			user.Products = value.(string)
		case "founded":
			user.Founded, err = parseTime(value.(string))
		case "company_overview":
			user.CompanyOverview = value.(string)
		case "fan_count":
			user.FanCount = value.(float64)
		case "type":
			// Ignore type here
			// Connections
		case "metadata":
			metadata := value.(map[string]interface{})
			for k, va := range metadata["connections"].(map[string]interface{}) {
				switch k {
				case "home":
					user.home = va.(string)
				case "feed":
					user.feed = va.(string)
				case "tagged":
					user.tagged = va.(string)
				case "posts":
					user.posts = va.(string)
				case "picture":
					user.picture = va.(string)
				case "friends":
					user.friends = va.(string)
				case "activities":
					user.activities = va.(string)
				case "interests":
					user.interests = va.(string)
				case "music":
					user.music = va.(string)
				case "books":
					user.books = va.(string)
				case "television":
					user.television = va.(string)
				case "likes":
					user.likes = va.(string)
				case "photos":
					user.photos = va.(string)
				case "albums":
					user.albums = va.(string)
				case "videos":
					user.videos = va.(string)
				case "groups":
					user.groups = va.(string)
				case "statuses":
					user.statuses = va.(string)
				case "links":
					user.links = va.(string)
				case "notes":
					user.notes = va.(string)
				case "events":
					user.events = va.(string)
				case "inbox":
					user.inbox = va.(string)
				case "outbox":
					user.outbox = va.(string)
				case "updates":
					user.updates = va.(string)
				case "accounts":
					user.accounts = va.(string)
				case "checkins":
					user.checkins = va.(string)
				case "platformrequests":
					user.platformrequests = va.(string)
				case "friendlists":
					user.friendlists = va.(string)
				default:
					// TODO: Print/log unsupported field
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

// calcTargetTopLevel() heuristically computes a new top level that
// the segmentStack should be merged to.
func (ss *segmentStack) calcTargetTopLevel() int {
	var minMergePercentage float64
	if ss.options != nil {
		minMergePercentage = ss.options.MinMergePercentage
	}
	if minMergePercentage <= 0 {
		minMergePercentage = DefaultCollectionOptions.MinMergePercentage
	}

	newTopLevel := 0
	maxTopLevel := len(ss.a) - 2

	for newTopLevel < maxTopLevel {
		numX0 := ss.a[newTopLevel].Len()
		numX1 := ss.a[newTopLevel+1].Len()
		if (float64(numX1) / float64(numX0)) > minMergePercentage {
			break
		}

		newTopLevel++
	}

	return newTopLevel
}

// ------------------------------------------------------

// merge() returns a new segmentStack, merging all the segments that
// are at the given newTopLevel and higher.
func (ss *segmentStack) merge(newTopLevel int, base *segmentStack) (
	*segmentStack, error) {
	// ----------------------------------------------------
	// First, rough estimate the bytes neeeded.

	totOps := ss.a[newTopLevel].Len()
	totBytes := ss.a[newTopLevel].NumKeyValBytes()

	iterPrealloc, err := ss.StartIterator(nil, nil, IteratorOptions{
		IncludeDeletions: true,
		SkipLowerLevel:   true,
		MinSegmentLevel:  newTopLevel + 1,
		MaxSegmentHeight: len(ss.a),
		base:             base,
	})
	if err != nil {
		return nil, err
	}

	defer iterPrealloc.Close()

	for {
		_, key, val, err := iterPrealloc.CurrentEx()
		if err == ErrIteratorDone {
			break
		}
		if err != nil {
			return nil, err
		}

		totOps++
		totBytes += len(key) + len(val)

		err = iterPrealloc.Next()
		if err == ErrIteratorDone {
			break
		}
		if err != nil {
			return nil, err
		}
	}

	// ----------------------------------------------------
	// Next, use an iterator for the actual merge.

	mergedSegment, err := newSegment(totOps, totBytes)
	if err != nil {
		return nil, err
	}

	err = ss.mergeInto(newTopLevel, mergedSegment, base, true)
	if err != nil {
		return nil, err
	}

	a := make([]Segment, 0, newTopLevel+1)
	a = append(a, ss.a[0:newTopLevel]...)
	a = append(a, mergedSegment)

	return &segmentStack{
		options:            ss.options,
		a:                  a,
		refs:               1,
		lowerLevelSnapshot: ss.lowerLevelSnapshot.addRef(),
	}, nil
}

func (ss *segmentStack) mergeInto(minSegmentLevel int, dest SegmentMutator,
	base *segmentStack, optimizeTail bool) error {
	iter, err := ss.startIterator(nil, nil, IteratorOptions{
		IncludeDeletions: true,
		SkipLowerLevel:   true,
		MinSegmentLevel:  minSegmentLevel,
		MaxSegmentHeight: len(ss.a),
		base:             base,
	})
	if err != nil {
		return err
	}

	defer iter.Close()

OUTER:
	for {
		entryEx, key, val, err := iter.CurrentEx()
		if err == ErrIteratorDone {
			break
		}
		if err != nil {
			return err
		}

		if optimizeTail && len(iter.cursors) == 1 {
			// When only 1 cursor remains, copy the remains of the
			// last segment more directly instead of Next()'ing
			// through the iterator.
			cursor := &iter.cursors[0]

			segment := iter.ss.a[cursor.ssIndex]
			segmentOps := segment.Len()

			for pos := cursor.pos; pos < segmentOps; pos++ {
				op, k, v := segment.GetOperationKeyVal(pos)

				err = dest.Mutate(op, k, v)
				if err != nil {
					return err
				}
			}

			break OUTER
		}

		op := entryEx.Operation
		if op == OperationMerge {
			// TODO: the merge operator implementation is currently
			// inefficient and not lazy enough right now.
			val, err = ss.get(key, len(ss.a)-1, base)
			if err != nil {
				return err
			}

			if val == nil {
				op = OperationDel
			} else {
				op = OperationSet
			}
		}

		err = dest.Mutate(op, key, val)
		if err != nil {
			return err
		}

		err = iter.Next()
		if err == ErrIteratorDone {
			break
		}
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>faster merge through rougher estimation of memory needed<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

// calcTargetTopLevel() heuristically computes a new top level that
// the segmentStack should be merged to.
func (ss *segmentStack) calcTargetTopLevel() int {
	var minMergePercentage float64
	if ss.options != nil {
		minMergePercentage = ss.options.MinMergePercentage
	}
	if minMergePercentage <= 0 {
		minMergePercentage = DefaultCollectionOptions.MinMergePercentage
	}

	newTopLevel := 0
	maxTopLevel := len(ss.a) - 2

	for newTopLevel < maxTopLevel {
		numX0 := ss.a[newTopLevel].Len()
		numX1 := ss.a[newTopLevel+1].Len()
		if (float64(numX1) / float64(numX0)) > minMergePercentage {
			break
		}

		newTopLevel++
	}

	return newTopLevel
}

// ------------------------------------------------------

// merge() returns a new segmentStack, merging all the segments that
// are at the given newTopLevel and higher.
func (ss *segmentStack) merge(newTopLevel int, base *segmentStack) (
	*segmentStack, error) {
	// ----------------------------------------------------
	// First, rough estimate the bytes neeeded.

	var totOps, totBytes int
	for i := newTopLevel; i < len(ss.a); i++ {
		totOps += ss.a[i].Len()
		totBytes += ss.a[i].NumKeyValBytes()
	}

	// ----------------------------------------------------
	// Next, use an iterator for the actual merge.

	mergedSegment, err := newSegment(totOps, totBytes)
	if err != nil {
		return nil, err
	}

	err = ss.mergeInto(newTopLevel, mergedSegment, base, true)
	if err != nil {
		return nil, err
	}

	a := make([]Segment, 0, newTopLevel+1)
	a = append(a, ss.a[0:newTopLevel]...)
	a = append(a, mergedSegment)

	return &segmentStack{
		options:            ss.options,
		a:                  a,
		refs:               1,
		lowerLevelSnapshot: ss.lowerLevelSnapshot.addRef(),
	}, nil
}

func (ss *segmentStack) mergeInto(minSegmentLevel int, dest SegmentMutator,
	base *segmentStack, optimizeTail bool) error {
	iter, err := ss.startIterator(nil, nil, IteratorOptions{
		IncludeDeletions: true,
		SkipLowerLevel:   true,
		MinSegmentLevel:  minSegmentLevel,
		MaxSegmentHeight: len(ss.a),
		base:             base,
	})
	if err != nil {
		return err
	}

	defer iter.Close()

OUTER:
	for {
		entryEx, key, val, err := iter.CurrentEx()
		if err == ErrIteratorDone {
			break
		}
		if err != nil {
			return err
		}

		if optimizeTail && len(iter.cursors) == 1 {
			// When only 1 cursor remains, copy the remains of the
			// last segment more directly instead of Next()'ing
			// through the iterator.
			cursor := &iter.cursors[0]

			segment := iter.ss.a[cursor.ssIndex]
			segmentOps := segment.Len()

			for pos := cursor.pos; pos < segmentOps; pos++ {
				op, k, v := segment.GetOperationKeyVal(pos)

				err = dest.Mutate(op, k, v)
				if err != nil {
					return err
				}
			}

			break OUTER
		}

		op := entryEx.Operation
		if op == OperationMerge {
			// TODO: the merge operator implementation is currently
			// inefficient and not lazy enough right now.
			val, err = ss.get(key, len(ss.a)-1, base)
			if err != nil {
				return err
			}

			if val == nil {
				op = OperationDel
			} else {
				op = OperationSet
			}
		}

		err = dest.Mutate(op, key, val)
		if err != nil {
			return err
		}

		err = iter.Next()
		if err == ErrIteratorDone {
			break
		}
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package lib

import (
	// "errors"
	"fmt"
	"github.com/namsral/flag"
	"os"
	"strings"
)

const RESERVED_SPACE = 450000000 // 450Mb

type Config struct {
	Folders    []string `json:"folders"`
	DryRun     bool     `json:"dryRun"`
	NotifyCalc int      `json:"notifyCalc"`
	NotifyMove int      `json:"notifyMove"`
	Version    string   `json:"version"`
}

// NotifyCalc/NotifyMove possible values
// 0 - no notification
// 1 - simple notification
// 2 - detailed notification

type Settings struct {
	Config

	Conf          string
	Log           string
	ReservedSpace int64
}

func NewSettings(version string) (*Settings, error) {
	var config, log, folders string
	var dryRun bool
	var notifyCalc, notifyMove int

	// location := SearchFile(name, locations)
	// if location != "" {
	// 	flag.Set("config", filepath.Join(location, name))
	// }

	// /boot/config/plugins/unbalance/
	flag.StringVar(&config, "config", "", "config location")
	flag.StringVar(&log, "log", "", "folder where log file will be written to")
	flag.StringVar(&folders, "folders", "", "folders that will be scanned for media")
	flag.BoolVar(&dryRun, "dryRun", true, "perform a dry-run rather than actual work")
	flag.IntVar(&notifyCalc, "notifyCalc", 0, "notify via email after calculation operation has completed (unraid notifications must be set up first): 0 - No notifications; 1 - Simple notifications; 2 - Detailed notifications")
	flag.IntVar(&notifyMove, "notifyMove", 0, "notify via email after move operation has completed (unraid notifications must be set up first): 0 - No notifications; 1 - Simple notifications; 2 - Detailed notifications")

	flag.Set("config", "/boot/config/plugins/unbalance/unbalance.conf")
	flag.Parse()

	// fmt.Printf("folders: %s\nconfig: %s\n", folders, config)

	s := &Settings{}
	if folders == "" {
		s.Folders = make([]string, 0)
	} else {
		s.Folders = strings.Split(folders, "|")
	}
	s.DryRun = dryRun
	s.NotifyCalc = notifyCalc
	s.NotifyMove = notifyMove
	s.Version = version

	s.Conf = config
	s.Log = log
	s.ReservedSpace = RESERVED_SPACE

	return s, nil
}

func (s *Settings) AddFolder(folder string) {
	s.Folders = append(s.Folders, folder)
}

func (s *Settings) DeleteFolder(folder string) {

	index := -1
	for p, v := range s.Folders {
		if v == folder {
			index = p
			break
		}
	}

	if index == -1 {
		return
	}

	s.Folders = append(s.Folders[:index], s.Folders[index+1:]...)
}

func (s *Settings) ToggleDryRun() {
	s.DryRun = !s.DryRun
}

func (s *Settings) Save() (err error) {
	tmpFile := s.Conf + ".tmp"

	folders := strings.Join(s.Folders, "|")
	if err = WriteLine(tmpFile, fmt.Sprintf("folders=%s", folders)); err != nil {
		return err
	}

	if err = WriteLine(tmpFile, fmt.Sprintf("dryRun=%t", s.DryRun)); err != nil {
		return err
	}

	if err = WriteLine(tmpFile, fmt.Sprintf("notifyCalc=%d", s.NotifyCalc)); err != nil {
		return err
	}

	if err = WriteLine(tmpFile, fmt.Sprintf("notifyMove=%d", s.NotifyCalc)); err != nil {
		return err
	}

	os.Rename(tmpFile, s.Conf)

	return
}
<commit_msg>(back) Set a default logging location in the settings struct<commit_after>package lib

import (
	// "errors"
	"fmt"
	"github.com/namsral/flag"
	"os"
	"strings"
)

const RESERVED_SPACE = 450000000 // 450Mb

type Config struct {
	Folders    []string `json:"folders"`
	DryRun     bool     `json:"dryRun"`
	NotifyCalc int      `json:"notifyCalc"`
	NotifyMove int      `json:"notifyMove"`
	Version    string   `json:"version"`
}

// NotifyCalc/NotifyMove possible values
// 0 - no notification
// 1 - simple notification
// 2 - detailed notification

type Settings struct {
	Config

	Conf          string
	Log           string
	ReservedSpace int64
}

func NewSettings(version string) (*Settings, error) {
	var config, log, folders string
	var dryRun bool
	var notifyCalc, notifyMove int

	// location := SearchFile(name, locations)
	// if location != "" {
	// 	flag.Set("config", filepath.Join(location, name))
	// }

	// /boot/config/plugins/unbalance/
	flag.StringVar(&config, "config", "", "config location")
	flag.StringVar(&log, "log", "/boot/logs", "folder where log file will be written to")
	flag.StringVar(&folders, "folders", "", "folders that will be scanned for media")
	flag.BoolVar(&dryRun, "dryRun", true, "perform a dry-run rather than actual work")
	flag.IntVar(&notifyCalc, "notifyCalc", 0, "notify via email after calculation operation has completed (unraid notifications must be set up first): 0 - No notifications; 1 - Simple notifications; 2 - Detailed notifications")
	flag.IntVar(&notifyMove, "notifyMove", 0, "notify via email after move operation has completed (unraid notifications must be set up first): 0 - No notifications; 1 - Simple notifications; 2 - Detailed notifications")

	flag.Set("config", "/boot/config/plugins/unBALANCE/unbalance.conf")
	flag.Parse()

	// fmt.Printf("folders: %s\nconfig: %s\n", folders, config)

	s := &Settings{}
	if folders == "" {
		s.Folders = make([]string, 0)
	} else {
		s.Folders = strings.Split(folders, "|")
	}
	s.DryRun = dryRun
	s.NotifyCalc = notifyCalc
	s.NotifyMove = notifyMove
	s.Version = version

	s.Conf = config
	s.Log = log
	s.ReservedSpace = RESERVED_SPACE

	return s, nil
}

func (s *Settings) AddFolder(folder string) {
	s.Folders = append(s.Folders, folder)
}

func (s *Settings) DeleteFolder(folder string) {

	index := -1
	for p, v := range s.Folders {
		if v == folder {
			index = p
			break
		}
	}

	if index == -1 {
		return
	}

	s.Folders = append(s.Folders[:index], s.Folders[index+1:]...)
}

func (s *Settings) ToggleDryRun() {
	s.DryRun = !s.DryRun
}

func (s *Settings) Save() (err error) {
	tmpFile := s.Conf + ".tmp"

	folders := strings.Join(s.Folders, "|")
	if err = WriteLine(tmpFile, fmt.Sprintf("folders=%s", folders)); err != nil {
		return err
	}

	if err = WriteLine(tmpFile, fmt.Sprintf("dryRun=%t", s.DryRun)); err != nil {
		return err
	}

	if err = WriteLine(tmpFile, fmt.Sprintf("notifyCalc=%d", s.NotifyCalc)); err != nil {
		return err
	}

	if err = WriteLine(tmpFile, fmt.Sprintf("notifyMove=%d", s.NotifyCalc)); err != nil {
		return err
	}

	os.Rename(tmpFile, s.Conf)

	return
}
<|endoftext|>


<commit_before>// This is to play with basic concurrency concepts in golang

package main

import (
	"fmt"
	"math/rand"
	"time"
)

func dataSource(dataChannel chan float64, sourceNum int) {

	const outputMin, outputMax float64 = -100, 100
	const delayMaxMs int32 = 1000;

	for {
		// simulate the data generation delay
		delay := time.Millisecond * time.Duration(rand.Int31n(delayMaxMs))
		time.Sleep(delay)
		// generate the data
		data := outputMin + (outputMax - outputMin) * rand.Float64()
		fmt.Printf("Source %v puts data %v into the source channel after a delay of %v\n", sourceNum, data, delay)
		dataChannel <- data
	}
}

func dataProcessor(inputChannel chan float64, outputChannel chan float64, processorNumber int) {

	const delayMaxMs int32 = 1000;

	for {
		inputData := <-inputChannel
		fmt.Printf("Processor %v received input data %v\n", processorNumber, inputData)
		// simulate a processing delay
		delay := time.Millisecond * time.Duration(rand.Int31n(delayMaxMs))
		time.Sleep(delay)
		// process the data
		outputData := inputData// TODO: Do the actual processing
		outputChannel <- outputData
		fmt.Printf("Processor %v turned input data %v into output data %v with processing delay of %v\n",
			processorNumber, inputData, outputData, delay)
	}

}

var channelStats []int64

func balancerStats(channelToIncrement int) {
	for len(channelStats) < channelToIncrement + 1 { channelStats = append(channelStats, 0); }
	channelStats[channelToIncrement]++
	sum := channelStats[0]
	for i := 1; i < len(channelStats); i++ { sum += channelStats[i]; }
	fmt.Printf("Load balancer stats: ");
	for _, v := range channelStats {
		fmt.Printf("%d%% ", (v * 100) / sum)
	}
	fmt.Println();
}

func loadBalancer(inputChannel chan float64, processingChannels []chan float64) {
	outputsNumber := len(processingChannels)
	for {
		inputData := <-inputChannel
		fmt.Printf("Load balancer received %v data\n", inputData)
		// choose one of the free input channels
		freeChannelIndices := make([]int, 0, outputsNumber)
		for i :=0; i < outputsNumber; i++ {
			if len(processingChannels[i]) < cap(processingChannels[i]) {
				freeChannelIndices = append(freeChannelIndices, i)
			}
		}
		freeChannels := len(freeChannelIndices)
		fmt.Printf("Load balancer found %v free channels\n", freeChannels)
		if freeChannels > 0 {
			selectedChannel := freeChannelIndices[rand.Intn(freeChannels)]
			fmt.Printf("Load balancer is dispatching the data %v to the processing channel %v\n",
				inputData, selectedChannel)
			processingChannels[selectedChannel] <- inputData
			balancerStats(selectedChannel); // update and print balancer stats
		} else {
			// no free channel found. drop the data, report an error
			fmt.Printf("Load balancer found no free processing channel. Dropping the data %v\n", inputData)
		}
	}
}

func main() {

	const inputChannelsNumber int = 6
	const processingChannelsNumber int = 5
	const processingChannelCapacity int = 3

	fmt.Println("Let's the concurrency experiment begin...")

	inputData := make(chan float64);
	balancerData := make(chan float64);
	outputData := make(chan float64);

	processingChannels := make([]chan float64, processingChannelsNumber)
	for i := 0; i < processingChannelsNumber; i++ {
		processingChannels[i] = make(chan float64, processingChannelCapacity)
	}

	// start data processors
	for i := 0; i < processingChannelsNumber; i++ {
		go dataProcessor(processingChannels[i], outputData, i)
	}

	// start the load balancer
	go loadBalancer(balancerData, processingChannels)

	// start input data sources
	for i := 0; i < inputChannelsNumber; i++ {
		go dataSource(inputData, i)
	}

	// receive input data and pass them to load balancer
	go func() {
		for {
			data := <-inputData
			fmt.Printf("Main routine received the data %v from sources and passed it to the load balancer.\n",
				data)
			balancerData <- data
		}
	}()

	// receive output data (this is the main thread's main loop)
	for {
		data := <-outputData;
		fmt.Printf("Main routine received the output data %v\n", data)
	}

}
<commit_msg>Added the processing pipeline<commit_after>// This is to play with basic concurrency concepts in golang

package main

import (
	"fmt"
	"math"
	"math/rand"
	"time"
)

func dataSource(dataChannel chan float64, sourceNum int) {

	const outputMin, outputMax float64 = -100, 100
	const delayMaxMs int32 = 1000;

	for {
		// simulate the data generation delay
		delay := time.Millisecond * time.Duration(rand.Int31n(delayMaxMs))
		time.Sleep(delay)
		// generate the data
		data := outputMin + (outputMax - outputMin) * rand.Float64()
		fmt.Printf("Source %v puts data %v into the source channel after a delay of %v\n", sourceNum, data, delay)
		dataChannel <- data
	}
}

type Task struct {
  source, intermediate, result float64
  error string
}

func processingStage(inputChannel chan Task, outputChannel chan Task, stageNumber int) {
	for {
		task := <-inputChannel;
		if task.error == "" {
			switch stageNumber {
			case 0:
				if task.source >= 0.0 {
					task.intermediate = math.Sqrt( task.source )
				} else {
					task.error = "Negative Sqrt argument"
				}
			case 1:
				task.result = task.intermediate * task.intermediate;
			case 2:
				if task.result != task.source {
					task.error = "Source and result mismatch"
				}
			}
		}
		outputChannel <- task;
	}
}

func dataProcessor(inputChannel chan float64, outputChannel chan Task, processorNumber int) {

	const delayMaxMs int32 = 1000;

	const numberOfPipelineStages = 3

	// create the processing pipeline
	pipelineInput := make(chan Task)
	pipelineOutput := pipelineInput
	for stage := 0; stage < numberOfPipelineStages; stage++ {
		pipelineIntermediate := make(chan Task)
		go processingStage(pipelineOutput, pipelineIntermediate, stage)
		pipelineOutput = pipelineIntermediate
	}

	// forwart pipeline output to processor output
	go func() {
		for {
			data := <-pipelineOutput
			fmt.Printf("Processor %v obtained output data %v from the pipeline\n",
				processorNumber, data)
			outputChannel <- data
		}
	}()

	for {
		inputData := <-inputChannel
		fmt.Printf("Processor %v received input data %v\n", processorNumber, inputData)
		// simulate a processing delay
		delay := time.Millisecond * time.Duration(rand.Int31n(delayMaxMs))
		time.Sleep(delay)
		fmt.Printf("Processor %v puts input data %v into the pipeline after processing delay of %v\n",
			processorNumber, inputData, delay)
		pipelineInput <- Task{source: inputData}
	}

}

var channelStats []int64

func balancerStats(channelToIncrement int) {
	for len(channelStats) < channelToIncrement + 1 { channelStats = append(channelStats, 0); }
	channelStats[channelToIncrement]++
	sum := channelStats[0]
	for i := 1; i < len(channelStats); i++ { sum += channelStats[i]; }
	fmt.Printf("Load balancer stats: ");
	for _, v := range channelStats {
		fmt.Printf("%d%% ", (v * 100) / sum)
	}
	fmt.Println();
}

func loadBalancer(inputChannel chan float64, processingChannels []chan float64) {
	outputsNumber := len(processingChannels)
	for {
		inputData := <-inputChannel
		fmt.Printf("Load balancer received %v data\n", inputData)
		// choose one of the free input channels
		freeChannelIndices := make([]int, 0, outputsNumber)
		for i :=0; i < outputsNumber; i++ {
			if len(processingChannels[i]) < cap(processingChannels[i]) {
				freeChannelIndices = append(freeChannelIndices, i)
			}
		}
		freeChannels := len(freeChannelIndices)
		fmt.Printf("Load balancer found %v free channels\n", freeChannels)
		if freeChannels > 0 {
			selectedChannel := freeChannelIndices[rand.Intn(freeChannels)]
			fmt.Printf("Load balancer is dispatching the data %v to the processing channel %v\n",
				inputData, selectedChannel)
			processingChannels[selectedChannel] <- inputData
			balancerStats(selectedChannel); // update and print balancer stats
		} else {
			// no free channel found. drop the data, report an error
			fmt.Printf("Load balancer found no free processing channel. Dropping the data %v\n", inputData)
		}
	}
}

func main() {

	const inputChannelsNumber int = 6
	const processingChannelsNumber int = 5
	const processingChannelCapacity int = 3

	fmt.Println("Let's the concurrency experiment begin...")

	inputData := make(chan float64);
	balancerData := make(chan float64);
	outputData := make(chan Task);

	processingChannels := make([]chan float64, processingChannelsNumber)
	for i := 0; i < processingChannelsNumber; i++ {
		processingChannels[i] = make(chan float64, processingChannelCapacity)
	}

	// start data processors
	for i := 0; i < processingChannelsNumber; i++ {
		go dataProcessor(processingChannels[i], outputData, i)
	}

	// start the load balancer
	go loadBalancer(balancerData, processingChannels)

	// start input data sources
	for i := 0; i < inputChannelsNumber; i++ {
		go dataSource(inputData, i)
	}

	// receive input data and pass them to load balancer
	go func() {
		for {
			data := <-inputData
			fmt.Printf("Main routine received the data %v from sources and passed it to the load balancer.\n",
				data)
			balancerData <- data
		}
	}()

	// receive output data (this is the main thread's main loop)
	for {
		data := <-outputData;
		fmt.Printf("Main routine received the output data %v\n", data)
	}

}
<|endoftext|>