<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package engine

import (
	"fmt"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	jsonw "github.com/keybase/go-jsonw"
)

// TrackToken is an engine.
type TrackToken struct {
	libkb.Contextified
	arg                 *TrackTokenArg
	them                *libkb.User
	signingKeyPub       libkb.GenericKey
	signingKeyPriv      libkb.GenericKey
	trackStatementBytes []byte
	trackStatement      *jsonw.Wrapper
	lockedKey           *libkb.SKB
	lockedWhich         string
}

type TrackTokenArg struct {
	Token   keybase1.TrackToken
	Me      *libkb.User
	Options keybase1.TrackOptions
}

// NewTrackToken creates a TrackToken engine.
func NewTrackToken(arg *TrackTokenArg, g *libkb.GlobalContext) *TrackToken {
	return &TrackToken{
		arg:          arg,
		Contextified: libkb.NewContextified(g),
	}
}

// Name is the unique engine name.
func (e *TrackToken) Name() string {
	return "TrackToken"
}

// GetPrereqs returns the engine prereqs.
func (e *TrackToken) Prereqs() Prereqs {
	return Prereqs{
		Session: true,
	}
}

// RequiredUIs returns the required UIs.
func (e *TrackToken) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{}
}

// SubConsumers returns the other UI consumers for this engine.
func (e *TrackToken) SubConsumers() []libkb.UIConsumer {
	return nil
}

// Run starts the engine.
func (e *TrackToken) Run(ctx *Context) (err error) {
	e.G().Trace("TrackToken", func() error { return err })()
	if len(e.arg.Token) == 0 {
		err = fmt.Errorf("missing TrackToken argument")
		return err
	}
	if err = e.loadMe(); err != nil {
		e.G().Log.Info("loadme err: %s", err)
		return err
	}

	var outcome *libkb.IdentifyOutcome
	outcome, err = e.G().TrackCache.Get(e.arg.Token)
	if err != nil {
		return err
	}

	if outcome.TrackStatus() == keybase1.TrackStatus_UPDATE_OK && !e.arg.Options.ForceRetrack {
		e.G().Log.Debug("tracking statement up-to-date.")
		return nil
	}

	if err = e.loadThem(outcome.Username); err != nil {
		return err
	}

	if e.arg.Me.Equal(e.them) {
		err = libkb.SelfTrackError{}
		return err
	}

	if err = e.isTrackTokenStale(outcome); err != nil {
		e.G().Log.Debug("Track statement is stale")
		return err
	}

	ska := libkb.SecretKeyArg{
		Me:      e.arg.Me,
		KeyType: libkb.DeviceSigningKeyType,
	}
	e.lockedKey, e.lockedWhich, err = e.G().Keyrings.GetSecretKeyLocked(ctx.LoginContext, ska)
	if err != nil {
		e.G().Log.Debug("secretkey err: %s", err)
		return err
	}
	e.lockedKey.SetUID(e.arg.Me.GetUID())
	e.signingKeyPub, err = e.lockedKey.GetPubKey()
	if err != nil {
		e.G().Log.Debug("getpubkey err: %s", err)
		return err
	}

	if e.trackStatement, err = e.arg.Me.TrackingProofFor(e.signingKeyPub, e.them, outcome); err != nil {
		e.G().Log.Debug("tracking proof err: %s", err)
		return err
	}

	if e.trackStatementBytes, err = e.trackStatement.Marshal(); err != nil {
		return err
	}

	e.G().Log.Debug("| Tracking statement: %s", string(e.trackStatementBytes))

	if e.arg.Options.LocalOnly || e.arg.Options.ExpiringLocal {
		e.G().Log.Debug("| Local")
		err = e.storeLocalTrack()
	} else {
		err = e.storeRemoteTrack(ctx)
	}

	if err == nil {
		e.G().NotifyRouter.HandleUserChanged(e.arg.Me.GetUID())
		e.G().NotifyRouter.HandleUserChanged(e.them.GetUID())
	}

	return err
}

func (e *TrackToken) isTrackTokenStale(o *libkb.IdentifyOutcome) (err error) {
	if idt := e.arg.Me.IDTable(); idt == nil {
		return nil
	} else if tm := idt.GetTrackMap(); tm == nil {
		return nil
	} else if v := tm[o.Username]; len(v) == 0 {
		return nil
	} else if lastTrack := v[len(v)-1]; lastTrack != nil && !lastTrack.IsRevoked() && o.TrackUsed == nil {
		// If we had a valid track that we didn't use in the identification, then
		// someone must have slipped in before us. Distinguish this case from the
		// other case below for the purposes of testing, to make sure we hit
		// both error cases in our tests.
		return libkb.TrackStaleError{FirstTrack: true}
	} else if o.TrackUsed == nil || lastTrack == nil {
		return nil
	} else if o.TrackUsed.GetTrackerSeqno() != lastTrack.GetSeqno() {
		// Similarly, if there was a last track for this user that wasn't the
		// one we were expecting, someone also must have intervened.
		return libkb.TrackStaleError{FirstTrack: false}
	}
	return nil
}

func (e *TrackToken) loadMe() error {
	if e.arg.Me != nil {
		return nil
	}

	me, err := libkb.LoadMe(libkb.NewLoadUserArg(e.G()))
	if err != nil {
		return err
	}
	e.arg.Me = me
	return nil
}

func (e *TrackToken) loadThem(username string) error {
	them, err := libkb.LoadUser(libkb.NewLoadUserByNameArg(e.G(), username))
	if err != nil {
		return err
	}
	e.them = them
	return nil
}

func (e *TrackToken) storeLocalTrack() error {
	return libkb.StoreLocalTrack(e.arg.Me.GetUID(), e.them.GetUID(), e.arg.Options.ExpiringLocal, e.trackStatement, e.G())
}

func (e *TrackToken) storeRemoteTrack(ctx *Context) (err error) {
	e.G().Log.Debug("+ StoreRemoteTrack")

	defer func() {
		e.G().Log.Debug("- StoreRemoteTrack -> %s", libkb.ErrToOk(err))
	}()

	var secretStore libkb.SecretStore
	if e.arg.Me != nil {
		e.lockedKey.SetUID(e.arg.Me.GetUID())
		secretStore = libkb.NewSecretStore(e.G(), e.arg.Me.GetNormalizedName())
	}
	// need to unlock private key
	parg := ctx.SecretKeyPromptArg(libkb.SecretKeyArg{}, "tracking signature")
	e.signingKeyPriv, err = e.lockedKey.PromptAndUnlock(parg, e.lockedWhich, secretStore, nil, e.arg.Me)
	if err != nil {
		return err
	}
	if e.signingKeyPriv == nil {
		return libkb.NoSecretKeyError{}
	}

	sig, sigid, err := e.signingKeyPriv.SignToString(e.trackStatementBytes)
	if err != nil {
		return err
	}

	_, err = e.G().API.Post(libkb.APIArg{
		Endpoint:    "follow",
		NeedSession: true,
		Args: libkb.HTTPArgs{
			"sig_id_base":  libkb.S{Val: sigid.ToString(false)},
			"sig_id_short": libkb.S{Val: sigid.ToShortID()},
			"sig":          libkb.S{Val: sig},
			"uid":          libkb.UIDArg(e.them.GetUID()),
			"type":         libkb.S{Val: "track"},
			"signing_kid":  e.signingKeyPub.GetKID(),
		},
	})

	if err != nil {
		e.G().Log.Info("api error: %s", err)
		return err
	}

	linkid := libkb.ComputeLinkID(e.trackStatementBytes)
	e.arg.Me.SigChainBump(linkid, sigid)

	return err
}
<commit_msg>this check can't be !=, since it doesn't take untracks into account<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package engine

import (
	"fmt"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	jsonw "github.com/keybase/go-jsonw"
)

// TrackToken is an engine.
type TrackToken struct {
	libkb.Contextified
	arg                 *TrackTokenArg
	them                *libkb.User
	signingKeyPub       libkb.GenericKey
	signingKeyPriv      libkb.GenericKey
	trackStatementBytes []byte
	trackStatement      *jsonw.Wrapper
	lockedKey           *libkb.SKB
	lockedWhich         string
}

type TrackTokenArg struct {
	Token   keybase1.TrackToken
	Me      *libkb.User
	Options keybase1.TrackOptions
}

// NewTrackToken creates a TrackToken engine.
func NewTrackToken(arg *TrackTokenArg, g *libkb.GlobalContext) *TrackToken {
	return &TrackToken{
		arg:          arg,
		Contextified: libkb.NewContextified(g),
	}
}

// Name is the unique engine name.
func (e *TrackToken) Name() string {
	return "TrackToken"
}

// GetPrereqs returns the engine prereqs.
func (e *TrackToken) Prereqs() Prereqs {
	return Prereqs{
		Session: true,
	}
}

// RequiredUIs returns the required UIs.
func (e *TrackToken) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{}
}

// SubConsumers returns the other UI consumers for this engine.
func (e *TrackToken) SubConsumers() []libkb.UIConsumer {
	return nil
}

// Run starts the engine.
func (e *TrackToken) Run(ctx *Context) (err error) {
	e.G().Trace("TrackToken", func() error { return err })()
	if len(e.arg.Token) == 0 {
		err = fmt.Errorf("missing TrackToken argument")
		return err
	}
	if err = e.loadMe(); err != nil {
		e.G().Log.Info("loadme err: %s", err)
		return err
	}

	var outcome *libkb.IdentifyOutcome
	outcome, err = e.G().TrackCache.Get(e.arg.Token)
	if err != nil {
		return err
	}

	if outcome.TrackStatus() == keybase1.TrackStatus_UPDATE_OK && !e.arg.Options.ForceRetrack {
		e.G().Log.Debug("tracking statement up-to-date.")
		return nil
	}

	if err = e.loadThem(outcome.Username); err != nil {
		return err
	}

	if e.arg.Me.Equal(e.them) {
		err = libkb.SelfTrackError{}
		return err
	}

	if err = e.isTrackTokenStale(outcome); err != nil {
		e.G().Log.Debug("Track statement is stale")
		return err
	}

	ska := libkb.SecretKeyArg{
		Me:      e.arg.Me,
		KeyType: libkb.DeviceSigningKeyType,
	}
	e.lockedKey, e.lockedWhich, err = e.G().Keyrings.GetSecretKeyLocked(ctx.LoginContext, ska)
	if err != nil {
		e.G().Log.Debug("secretkey err: %s", err)
		return err
	}
	e.lockedKey.SetUID(e.arg.Me.GetUID())
	e.signingKeyPub, err = e.lockedKey.GetPubKey()
	if err != nil {
		e.G().Log.Debug("getpubkey err: %s", err)
		return err
	}

	if e.trackStatement, err = e.arg.Me.TrackingProofFor(e.signingKeyPub, e.them, outcome); err != nil {
		e.G().Log.Debug("tracking proof err: %s", err)
		return err
	}

	if e.trackStatementBytes, err = e.trackStatement.Marshal(); err != nil {
		return err
	}

	e.G().Log.Debug("| Tracking statement: %s", string(e.trackStatementBytes))

	if e.arg.Options.LocalOnly || e.arg.Options.ExpiringLocal {
		e.G().Log.Debug("| Local")
		err = e.storeLocalTrack()
	} else {
		err = e.storeRemoteTrack(ctx)
	}

	if err == nil {
		e.G().NotifyRouter.HandleUserChanged(e.arg.Me.GetUID())
		e.G().NotifyRouter.HandleUserChanged(e.them.GetUID())
	}

	return err
}

func (e *TrackToken) isTrackTokenStale(o *libkb.IdentifyOutcome) (err error) {
	if idt := e.arg.Me.IDTable(); idt == nil {
		return nil
	} else if tm := idt.GetTrackMap(); tm == nil {
		return nil
	} else if v := tm[o.Username]; len(v) == 0 {
		return nil
	} else if lastTrack := v[len(v)-1]; lastTrack != nil && !lastTrack.IsRevoked() && o.TrackUsed == nil {
		// If we had a valid track that we didn't use in the identification, then
		// someone must have slipped in before us. Distinguish this case from the
		// other case below for the purposes of testing, to make sure we hit
		// both error cases in our tests.
		return libkb.TrackStaleError{FirstTrack: true}
	} else if o.TrackUsed == nil || lastTrack == nil {
		return nil
	} else if o.TrackUsed.GetTrackerSeqno() < lastTrack.GetSeqno() {
		// Similarly, if there was a last track for this user that wasn't the
		// one we were expecting, someone also must have intervened.
		e.G().Log.Debug("Stale track! We were at seqno %d, but %d is already in chain", o.TrackUsed.GetTrackerSeqno(), lastTrack.GetSeqno())
		return libkb.TrackStaleError{FirstTrack: false}
	}
	return nil
}

func (e *TrackToken) loadMe() error {
	if e.arg.Me != nil {
		return nil
	}

	me, err := libkb.LoadMe(libkb.NewLoadUserArg(e.G()))
	if err != nil {
		return err
	}
	e.arg.Me = me
	return nil
}

func (e *TrackToken) loadThem(username string) error {
	them, err := libkb.LoadUser(libkb.NewLoadUserByNameArg(e.G(), username))
	if err != nil {
		return err
	}
	e.them = them
	return nil
}

func (e *TrackToken) storeLocalTrack() error {
	return libkb.StoreLocalTrack(e.arg.Me.GetUID(), e.them.GetUID(), e.arg.Options.ExpiringLocal, e.trackStatement, e.G())
}

func (e *TrackToken) storeRemoteTrack(ctx *Context) (err error) {
	e.G().Log.Debug("+ StoreRemoteTrack")

	defer func() {
		e.G().Log.Debug("- StoreRemoteTrack -> %s", libkb.ErrToOk(err))
	}()

	var secretStore libkb.SecretStore
	if e.arg.Me != nil {
		e.lockedKey.SetUID(e.arg.Me.GetUID())
		secretStore = libkb.NewSecretStore(e.G(), e.arg.Me.GetNormalizedName())
	}
	// need to unlock private key
	parg := ctx.SecretKeyPromptArg(libkb.SecretKeyArg{}, "tracking signature")
	e.signingKeyPriv, err = e.lockedKey.PromptAndUnlock(parg, e.lockedWhich, secretStore, nil, e.arg.Me)
	if err != nil {
		return err
	}
	if e.signingKeyPriv == nil {
		return libkb.NoSecretKeyError{}
	}

	sig, sigid, err := e.signingKeyPriv.SignToString(e.trackStatementBytes)
	if err != nil {
		return err
	}

	_, err = e.G().API.Post(libkb.APIArg{
		Endpoint:    "follow",
		NeedSession: true,
		Args: libkb.HTTPArgs{
			"sig_id_base":  libkb.S{Val: sigid.ToString(false)},
			"sig_id_short": libkb.S{Val: sigid.ToShortID()},
			"sig":          libkb.S{Val: sig},
			"uid":          libkb.UIDArg(e.them.GetUID()),
			"type":         libkb.S{Val: "track"},
			"signing_kid":  e.signingKeyPub.GetKID(),
		},
	})

	if err != nil {
		e.G().Log.Info("api error: %s", err)
		return err
	}

	linkid := libkb.ComputeLinkID(e.trackStatementBytes)
	e.arg.Me.SigChainBump(linkid, sigid)

	return err
}
<|endoftext|>


<commit_before>package books

import (
	//"database/sql"
	"github.com/moshee/gas"
	//"github.com/argusdusty/Ferret"
	"sort"
	"time"
)

func Index(g *gas.Gas) {
	releases := make([]*Release, 0, 20)
	rows, err := gas.DB.Query("SELECT * FROM books.recent_releases LIMIT 20")
	if err != nil {
		g.Render("books", "index-error", err)
		return
	}

	for rows.Next() {
		r := new(Release)
		s := new(BookSeries)
		var cs Chapters
		var gs TranslationGroups

		err = rows.Scan(&r.Id, &r.Language, &r.ReleaseDate, &r.IsLastRelease, &r.Extra, &cs.Volumes, &cs.Nums, &s.Id, &s.Title, &gs.Ids, &gs.Names)
		if err != nil {
			g.Render("books", "index-error", err)
			return
		}

		r.BookSeries = s
		r.Chapters = cs
		sort.Sort(gs)
		r.TranslationGroups = gs

		releases = append(releases, r)
	}

	series := make([]*BookSeries, 0, 10)
	rows, err = gas.DB.Query("SELECT * FROM books.latest_series LIMIT 10")
	if err != nil {
		g.Render("books", "index-error", err)
		return
	}

	for rows.Next() {
		s := new(BookSeries)
		err := rows.Scan(&s.Id, &s.Title, &s.SeriesKind, &s.Vintage, &s.DateAdded, &s.NSFW, &s.AvgRating, &s.Demographic, &s.Tags)
		if err != nil {
			g.Render("books", "index-error", err)
			return
		}

		series = append(series, s)
	}

	n := new(NewsPost)
	u := new(User)

	row := gas.DB.QueryRow("SELECT * FROM books.latest_news LIMIT 1")

	if err := row.Scan(&n.Id, &u.Id, &u.Name, &n.Category, &n.DatePosted, &n.Title, &n.Body); err != nil {
		g.Render("books", "index-error", err)
		return
	}

	n.User = u

	g.Render("books", "index", &struct {
		Releases []*Release
		Series   []*BookSeries
		News     *NewsPost
		Now      time.Time
	}{
		releases,
		series,
		n,
		time.Now(),
	})
}
<commit_msg>eliminate some copying<commit_after>package books

import (
	//"database/sql"
	"github.com/moshee/gas"
	//"github.com/argusdusty/Ferret"
	"sort"
	"time"
)

func Index(g *gas.Gas) {
	const releaseCount = 20
	releases := make([]Release, releaseCount)
	rows, err := gas.DB.Query("SELECT * FROM books.recent_releases LIMIT 20")
	if err != nil {
		g.Render("books", "index-error", err)
		return
	}
	defer rows.Close()

	n := 0

	for ; rows.Next(); n++ {
		r := &releases[n]
		s := new(BookSeries)
		var cs Chapters
		var gs TranslationGroups

		err = rows.Scan(&r.Id, &r.Language, &r.ReleaseDate, &r.IsLastRelease, &r.Extra, &cs.Volumes, &cs.Nums, &s.Id, &s.Title, &gs.Ids, &gs.Names)
		if err != nil {
			g.Render("books", "index-error", err)
			return
		}

		r.BookSeries = s
		r.Chapters = cs
		sort.Sort(gs)
		r.TranslationGroups = gs
	}

	if n < releaseCount {
		releases = releases[:n]
	}

	series := make([]BookSeries, 10)
	rows, err = gas.DB.Query("SELECT * FROM books.latest_series LIMIT 10")
	if err != nil {
		g.Render("books", "index-error", err)
		return
	}
	defer rows.Close()

	for n = 0; rows.Next(); n++ {
		s := &series[n]
		err := rows.Scan(&s.Id, &s.Title, &s.SeriesKind, &s.Vintage, &s.DateAdded, &s.NSFW, &s.AvgRating, &s.Demographic, &s.Tags)
		if err != nil {
			g.Render("books", "index-error", err)
			return
		}
	}

	if n < 10 {
		series = series[:n]
	}

	news := new(NewsPost)
	u := new(User)

	row := gas.DB.QueryRow("SELECT * FROM books.latest_news LIMIT 1")

	if err := row.Scan(&news.Id, &u.Id, &u.Name, &news.Category, &news.DatePosted, &news.Title, &news.Body); err != nil {
		g.Render("books", "index-error", err)
		return
	}

	news.User = u

	g.Render("books", "index", &struct {
		Releases []Release
		Series   []BookSeries
		News     *NewsPost
		Now      time.Time
	}{
		releases,
		series,
		news,
		time.Now(),
	})
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package resourcekey contains well known type and label keys for resources.
package resourcekey // import "go.opentelemetry.io/otel/sdk/resource/resourcekey"

// Constants for Service resources.
const (
	// A uniquely identifying name for a Service.
	ServiceKeyName       = "service.name"
	ServiceKeyNamespace  = "service.namespace"
	ServiceKeyInstanceID = "service.instance.id"
	ServiceKeyVersion    = "service.version"
)

// Constants for Library resources.
const (
	// A uniquely identifying name for a Library.
	LibraryKeyName     = "library.name"
	LibraryKeyLanguage = "library.language"
	LibraryKeyVersion  = "library.version"
)

// Constants for Kubernetes resources.
const (
	// A uniquely identifying name for the Kubernetes cluster. Kubernetes
	// does not have cluster names as an internal concept so this may be
	// set to any meaningful value within the environment. For example,
	// GKE clusters have a name which can be used for this label.
	K8SKeyClusterName    = "k8s.cluster.name"
	K8SKeyNamespaceName  = "k8s.namespace.name"
	K8SKeyPodName        = "k8s.pod.name"
	K8SKeyDeploymentName = "k8s.deployment.name"
)

// Constants for Container resources.
const (
	// A uniquely identifying name for the Container.
	ContainerKeyName      = "container.name"
	ContainerKeyImageName = "container.image.name"
	ContainerKeyImageTag  = "container.image.tag"
)

// Constants for Cloud resources.
const (
	CloudKeyProvider  = "cloud.provider"
	CloudKeyAccountID = "cloud.account.id"
	CloudKeyRegion    = "cloud.region"
	CloudKeyZone      = "cloud.zone"

	// Cloud Providers
	CloudProviderAWS   = "aws"
	CloudProviderGCP   = "gcp"
	CloudProviderAZURE = "azure"
)

// Constants for Host resources.
const (
	// A uniquely identifying name for the host.
	HostKeyName = "host.name"

	// A hostname as returned by the 'hostname' command on host machine.
	HostKeyHostName     = "host.hostname"
	HostKeyID           = "host.id"
	HostKeyType         = "host.type"
	HostKeyImageName    = "host.image.name"
	HostKeyImageID      = "host.image.id"
	HostKeyImageVersion = "host.image.version"
)
<commit_msg>Update resourcekey package doc<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package resourcekey contains standard resource attribute keys as defined
// by the OpenTelemetry specification
// (https://github.com/open-telemetry/opentelemetry-specification/tree/v0.4.0/specification/resource/semantic_conventions).
package resourcekey // import "go.opentelemetry.io/otel/sdk/resource/resourcekey"

// Constants for Service resources.
const (
	// A uniquely identifying name for a Service.
	ServiceKeyName       = "service.name"
	ServiceKeyNamespace  = "service.namespace"
	ServiceKeyInstanceID = "service.instance.id"
	ServiceKeyVersion    = "service.version"
)

// Constants for Library resources.
const (
	// A uniquely identifying name for a Library.
	LibraryKeyName     = "library.name"
	LibraryKeyLanguage = "library.language"
	LibraryKeyVersion  = "library.version"
)

// Constants for Kubernetes resources.
const (
	// A uniquely identifying name for the Kubernetes cluster. Kubernetes
	// does not have cluster names as an internal concept so this may be
	// set to any meaningful value within the environment. For example,
	// GKE clusters have a name which can be used for this label.
	K8SKeyClusterName    = "k8s.cluster.name"
	K8SKeyNamespaceName  = "k8s.namespace.name"
	K8SKeyPodName        = "k8s.pod.name"
	K8SKeyDeploymentName = "k8s.deployment.name"
)

// Constants for Container resources.
const (
	// A uniquely identifying name for the Container.
	ContainerKeyName      = "container.name"
	ContainerKeyImageName = "container.image.name"
	ContainerKeyImageTag  = "container.image.tag"
)

// Constants for Cloud resources.
const (
	CloudKeyProvider  = "cloud.provider"
	CloudKeyAccountID = "cloud.account.id"
	CloudKeyRegion    = "cloud.region"
	CloudKeyZone      = "cloud.zone"

	// Cloud Providers
	CloudProviderAWS   = "aws"
	CloudProviderGCP   = "gcp"
	CloudProviderAZURE = "azure"
)

// Constants for Host resources.
const (
	// A uniquely identifying name for the host.
	HostKeyName = "host.name"

	// A hostname as returned by the 'hostname' command on host machine.
	HostKeyHostName     = "host.hostname"
	HostKeyID           = "host.id"
	HostKeyType         = "host.type"
	HostKeyImageName    = "host.image.name"
	HostKeyImageID      = "host.image.id"
	HostKeyImageVersion = "host.image.version"
)
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/couchbase/cbauth"
	"io"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

func handleError(err error) {
	if err != nil {
		fmt.Printf("Error occured: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	help := flag.Bool("help", false, "Help")
	config := flag.String("configfile", "config.json", "Scan load config file")
	outfile := flag.String("resultfile", "results.json", "Result report file")
	cpus := flag.Int("cpus", runtime.NumCPU(), "Number of CPUs")
	cluster := flag.String("cluster", "127.0.0.1:9000", "Cluster server address")
	auth := flag.String("auth", "Administrator:asdasd", "Auth")
	statsfile := flag.String("statsfile", "", "Periodic statistics report file")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile := flag.String("memprofile", "", "write mem profile to file")

	flag.Parse()

	if *help {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *cpuprofile != "" {
		fd, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Println("Failed create cpu profile file")
			os.Exit(1)
		}
		pprof.StartCPUProfile(fd)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		fd, err := os.Create(*memprofile)
		if err != nil {
			fmt.Println("Failed create mem profile file")
			os.Exit(1)
		}
		defer pprof.WriteHeapProfile(fd)
	}

	runtime.GOMAXPROCS(*cpus)
	up := strings.Split(*auth, ":")
	_, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1])
	if err != nil {
		fmt.Println("Failed to initialize cbauth: %s\n", err)
		os.Exit(1)
	}

	cfg, err := parseConfig(*config)
	handleError(err)

	var statsW io.Writer
	if *statsfile != "" {
		if f, err := os.Create(*statsfile); err != nil {
			handleError(err)
		} else {
			statsW = f
			defer f.Close()
		}
	}

	t0 := time.Now()
	res, err := RunCommands(*cluster, cfg, statsW)
	handleError(err)
	dur := time.Now().Sub(t0)

	totalRows := uint64(0)
	for _, result := range res.ScanResults {
		totalRows += result.Rows
	}
	res.Rows = totalRows
	res.Duration = dur.Seconds() - 4 // account for warmup

	rate := int(float64(totalRows) / res.Duration)

	fmt.Printf("Throughput = %d rows/sec\n", rate)

	os.Remove(*outfile)
	err = writeResults(res, *outfile)
	handleError(err)

}
<commit_msg>MB-15882: Support setting log level in cbindexperf<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/couchbase/cbauth"
	"github.com/couchbase/indexing/secondary/logging"
	"io"
	"os"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

func handleError(err error) {
	if err != nil {
		fmt.Printf("Error occured: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	help := flag.Bool("help", false, "Help")
	config := flag.String("configfile", "config.json", "Scan load config file")
	outfile := flag.String("resultfile", "results.json", "Result report file")
	cpus := flag.Int("cpus", runtime.NumCPU(), "Number of CPUs")
	cluster := flag.String("cluster", "127.0.0.1:9000", "Cluster server address")
	auth := flag.String("auth", "Administrator:asdasd", "Auth")
	statsfile := flag.String("statsfile", "", "Periodic statistics report file")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile := flag.String("memprofile", "", "write mem profile to file")
	logLevel := flag.String("logLevel", "error", "Log Level")

	flag.Parse()

	logging.SetLogLevel(logging.Level(*logLevel))
	fmt.Println("Log Level = %v", *logLevel)

	if *help {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *cpuprofile != "" {
		fd, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Println("Failed create cpu profile file")
			os.Exit(1)
		}
		pprof.StartCPUProfile(fd)
		defer pprof.StopCPUProfile()
	}
	if *memprofile != "" {
		fd, err := os.Create(*memprofile)
		if err != nil {
			fmt.Println("Failed create mem profile file")
			os.Exit(1)
		}
		defer pprof.WriteHeapProfile(fd)
	}

	runtime.GOMAXPROCS(*cpus)
	up := strings.Split(*auth, ":")
	_, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1])
	if err != nil {
		fmt.Println("Failed to initialize cbauth: %s\n", err)
		os.Exit(1)
	}

	cfg, err := parseConfig(*config)
	handleError(err)

	var statsW io.Writer
	if *statsfile != "" {
		if f, err := os.Create(*statsfile); err != nil {
			handleError(err)
		} else {
			statsW = f
			defer f.Close()
		}
	}

	t0 := time.Now()
	res, err := RunCommands(*cluster, cfg, statsW)
	handleError(err)
	dur := time.Now().Sub(t0)

	totalRows := uint64(0)
	for _, result := range res.ScanResults {
		totalRows += result.Rows
	}
	res.Rows = totalRows
	res.Duration = dur.Seconds() - 4 // account for warmup

	rate := int(float64(totalRows) / res.Duration)

	fmt.Printf("Throughput = %d rows/sec\n", rate)

	os.Remove(*outfile)
	err = writeResults(res, *outfile)
	handleError(err)

}
<|endoftext|>


<commit_before>package testauth

import (
	"testing"

	"github.com/Nivl/go-rest-tools/primitives/models/lifecycle"
	"github.com/Nivl/go-rest-tools/security/auth"
	"github.com/Nivl/go-rest-tools/storage/db"
)

// NewAuth creates a new user and their session
func NewAuth(t *testing.T, q db.DB) (*auth.User, *auth.Session) {
	user := NewUser(t, q, nil)
	session := &auth.Session{
		UserID: user.ID,
	}

	if err := session.Create(q); err != nil {
		t.Fatal(err)
	}

	lifecycle.SaveModels(t, session)
	return user, session
}

// NewAdminAuth creates a new admin and their session
func NewAdminAuth(t *testing.T, q db.DB) (*auth.User, *auth.Session) {
	user := NewUser(t, q, &auth.User{IsAdmin: true})
	session := &auth.Session{
		UserID: user.ID,
	}

	if err := session.Create(q); err != nil {
		t.Fatal(err)
	}

	lifecycle.SaveModels(t, session)
	return user, session
}
<commit_msg>tests(auth): add testauth.NewSession()<commit_after>package testauth

import (
	"testing"

	"github.com/Nivl/go-rest-tools/primitives/models/lifecycle"
	"github.com/Nivl/go-rest-tools/security/auth"
	"github.com/Nivl/go-rest-tools/storage/db"
)

// NewAuth creates a new user and their session
func NewAuth(t *testing.T, q db.DB) (*auth.User, *auth.Session) {
	user := NewUser(t, q, nil)
	session := &auth.Session{
		UserID: user.ID,
	}

	if err := session.Create(q); err != nil {
		t.Fatal(err)
	}

	lifecycle.SaveModels(t, session)
	return user, session
}

// NewAdminAuth creates a new admin and their session
func NewAdminAuth(t *testing.T, q db.DB) (*auth.User, *auth.Session) {
	user := NewUser(t, q, &auth.User{IsAdmin: true})
	session := &auth.Session{
		UserID: user.ID,
	}

	if err := session.Create(q); err != nil {
		t.Fatal(err)
	}

	lifecycle.SaveModels(t, session)
	return user, session
}

// NewSession creates a new session for the given user
func NewSession(t *testing.T, q db.DB, user *auth.User) *auth.Session {
	session := &auth.Session{
		UserID: user.ID,
	}

	if err := session.Create(q); err != nil {
		t.Fatal(err)
	}

	lifecycle.SaveModels(t, session)
	return session
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"github.com/jrxfive/consulchecks/boron/Godeps/_workspace/src/github.com/satori/go.uuid"
	"io/ioutil"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

const usageText = `Usage: boron [options]
Options:
  -ip=""                   IP of service, default localhost
  -port=                   Port of Service, default 8000
  -protocol                Protocol of Service, default empty ie tcp:// or http://
  -service                 To enable Service parameters in template generation, default false
  -tags                    Tags to determine correct measurement
  -measurement             Single Telegraf timemeasurement to check against
  -plugin                  Telegraf Plugin name
  -plugin-parameters       Parameters for Telegraf Plugin, separator '|'
  -telegraf-location       Absolute path of telegraf binary
  -working-location        Working location to generate temporary plugin files
  -lessthan                Warning and Critical values will notify if less than
  -warning=                Exits with code 1 if exceeded, Optional
  -critical=               Exits with code 2 if exceeded, Required
Examples:
	./boron -plugin mem --working-location . -telegraf-location ./telegraf -measurement 'mem_used' -critical 0
	./boron -plugin cpu --working-location . -telegraf-location ./telegraf -plugin-parameters 'percpu = true|totalcpu = true|drop = ["cpu_time"]' -measurement 'cpu_usage_idle' -critical 0 -tags 'cpu="cpu0"'
	./boron -warning=20971520 -critical=104857600 -measurement 'some.example.com.host.10.0.0.1.port.6379.redis_used_memory' -plugin redis -telegraf-location './telegraf' -service -ip "0.0.0.0" -port 6379 -protocol "tcp://" -plugin-parameters "key = value|key2 = value2"
`

func main() {
	os.Exit(boronMain())
}

func boronMain() int {

	var (
		IP               string
		Port             int
		Protocol         string
		Service          bool
		Measurement      string
		Plugin           string
		PluginParameters string
		Tags             string
		TelegrafLocation string
		WorkingLocation  string
		LessThan         bool
		Warning          float64
		Critical         float64
	)

	randomFloat := rand.Float64()

	boronFlags := flag.NewFlagSet("boron", flag.ExitOnError)
	boronFlags.Usage = func() { printUsage() }

	//
	boronFlags.StringVar(&IP, "ip", "localhost", "IP of Service")
	boronFlags.IntVar(&Port, "port", 8000, "Port of Service")
	boronFlags.StringVar(&Protocol, "protocol", "", "Protocol of Service i.e http:// or tcp://")
	boronFlags.BoolVar(&Service, "service", false, "Query a Service which requires an array of Protocol,IP,Port")

	//
	boronFlags.StringVar(&Tags, "tags", "", "tags to determine correct measurement")
	boronFlags.StringVar(&Measurement, "measurement", "", "Single Telegraf timemeasurement to check against")
	boronFlags.StringVar(&Plugin, "plugin", "", "Telegraf Plugin name")
	boronFlags.StringVar(&PluginParameters, "plugin-parameters", "", "Parameters for Telegraf Plugin")
	boronFlags.StringVar(&TelegrafLocation, "telegraf-location", "/usr/bin/telegraf", "Absolute path of telegraf binary")
	boronFlags.StringVar(&WorkingLocation, "working-location", "/tmp/", "Working location to generate temporary plugin files")

	boronFlags.BoolVar(&LessThan, "lessthan", false, "Warning and Critical values will notify if less than")
	boronFlags.Float64Var(&Warning, "warning", randomFloat, "Exits with code 1")
	boronFlags.Float64Var(&Critical, "critical", randomFloat, "Exits with code 2")

	if err := boronFlags.Parse(os.Args[1:]); err != nil {
		boronFlags.Usage()
		return 1
	}

	temporaryConfig, err := writeTemplate(Plugin, PluginParameters, IP, Port, Service, Protocol, WorkingLocation, TelegrafTemplate)
	if err != nil {
		return 2
	}

	defer os.Remove(temporaryConfig)

	measurementResult, err := executeTeleGraphite(TelegrafLocation, temporaryConfig, Plugin, Measurement, Tags)
	if err != nil || measurementResult == math.MaxFloat64 {
		fmt.Fprintf(os.Stderr, "Error could not find measurement: %s for specified plugin: %s\n", Measurement, Plugin)
		return 1
	}

	exitCode := thresholdChecker(measurementResult, Measurement, LessThan, Warning, Critical, randomFloat)
	return exitCode
}

//Splits Plugin parameters
func splitParameters(PluginParameters string) []string {
	return strings.Split(PluginParameters, "|")
}

//Generates template  file for given plugin
func writeTemplate(Plugin string, PluginParameters string, IP string, Port int, Service bool, Protocol string, WorkingLocation, Template string) (string, error) {

	genUUID := uuid.NewV4()

	t, err := template.New("plugin").Parse(Template)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return "", err
	}

	temporaryFileLocation := fmt.Sprintf("%s%s-%s.conf", WorkingLocation, Plugin, genUUID)
	fh, err := os.Create(temporaryFileLocation)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return "", err
	}
	defer fh.Close()

	wr := bufio.NewWriter(fh)
	defer wr.Flush()

	tp := TelegrafPlugin{
		Plugin:   Plugin,
		KV:       splitParameters(PluginParameters),
		IP:       IP,
		Port:     Port,
		Protocol: Protocol,
		Service:  Service,
	}

	err = t.Execute(wr, tp)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return "", err
	}

	return temporaryFileLocation, nil
}

//Executes Telegraf in STDOUT mode with the generated config file.
//Parses STDOUT and abstracts value
func executeTeleGraphite(TelegrafLocation, Config, Plugin, Measurement, Tags string) (float64, error) {
	cmd := exec.Command(TelegrafLocation, "-config", Config, "-filter", Plugin, "-test", "|", "grep", Measurement)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return math.MaxFloat64, err
	}

	if err := cmd.Start(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return math.MaxFloat64, err
	}

	r, err := ioutil.ReadAll(stdout)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return math.MaxFloat64, err
	}

	re := regexp.MustCompile(fmt.Sprintf(`.*(\[\]|\[.*\])\s+(%s)\s+value=(\w+)`, Measurement))
	regexResult := re.FindAllStringSubmatch(string(r), -1)

	if len(regexResult) >= 1 {
		for _, group := range regexResult {
			tags := group[1]
			value := group[3]

			if tags == "[]" {

				f, err := strconv.ParseFloat(value, 64)
				if err != nil {
					fmt.Fprintln(os.Stderr, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement)))
					return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement))
				}
				return f, nil

			} else if strings.Contains(tags, Tags) {

				f, err := strconv.ParseFloat(value, 64)
				if err != nil {
					fmt.Fprintln(os.Stderr, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement)))
					return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement))
				}
				return f, nil
			}
			continue
		}
	} else {
		return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not found", Measurement))
	}
	return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not found", Measurement))
}

func messageGenerator(thresholdType string, thresholdValue, curerntValue float64, measurement string) {
	fmt.Fprintf(os.Stdout, "Threshold Event:%s ThresholdValue:%f CurrentValue:%f Measurement:%s\n", thresholdType, thresholdValue, curerntValue, measurement)
}

func exceeded(thresholdValue, currentValue float64, lessthan bool) bool {
	switch lessthan {
	case true:
		if thresholdValue <= currentValue {
			return false
		}
		return true
	case false:
		if thresholdValue >= currentValue {
			return false
		}
		return true
	default:
		return false
	}

}

func thresholdChecker(currentValue float64, measurement string, lessthan bool, warningValue, criticalValue, random float64) int {
	if warningValue == random && criticalValue == random {
		fmt.Fprintln(os.Stderr, "Critical threshold must be specified")
		return 1
	}

	if exceeded(criticalValue, currentValue, lessthan) {
		//ALERT CRITICAL
		messageGenerator("CRITICAL", criticalValue, currentValue, measurement)
		return 2
	} else if warningValue != random && exceeded(criticalValue, currentValue, lessthan) == false && exceeded(warningValue, currentValue, lessthan) == true {
		//ALERT WARNING
		messageGenerator("WARNING", warningValue, currentValue, measurement)
		return 1
	} else {
		messageGenerator("PASSING", warningValue, currentValue, measurement)
		return 0
	}

}

func printUsage() {
	fmt.Fprintln(os.Stderr, usageText)
}
<commit_msg>functions renamed<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"github.com/jrxfive/consulchecks/boron/Godeps/_workspace/src/github.com/satori/go.uuid"
	"io/ioutil"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

const usageText = `Usage: boron [options]
Options:
  -ip=""                   IP of service, default localhost
  -port=                   Port of Service, default 8000
  -protocol                Protocol of Service, default empty ie tcp:// or http://
  -service                 To enable Service parameters in template generation, default false
  -tags                    Tags to determine correct measurement
  -measurement             Single Telegraf timemeasurement to check against
  -plugin                  Telegraf Plugin name
  -plugin-parameters       Parameters for Telegraf Plugin, separator '|'
  -telegraf-location       Absolute path of telegraf binary
  -working-location        Working location to generate temporary plugin files
  -lessthan                Warning and Critical values will notify if less than
  -warning=                Exits with code 1 if exceeded, Optional
  -critical=               Exits with code 2 if exceeded, Required
Examples:
	./boron -plugin mem --working-location . -telegraf-location ./telegraf -measurement 'mem_used' -critical 0
	./boron -plugin cpu --working-location . -telegraf-location ./telegraf -plugin-parameters 'percpu = true|totalcpu = true|drop = ["cpu_time"]' -measurement 'cpu_usage_idle' -critical 0 -tags 'cpu="cpu0"'
	./boron -warning=20971520 -critical=104857600 -measurement 'some.example.com.host.10.0.0.1.port.6379.redis_used_memory' -plugin redis -telegraf-location './telegraf' -service -ip "0.0.0.0" -port 6379 -protocol "tcp://" -plugin-parameters "key = value|key2 = value2"
`

func main() {
	os.Exit(boronMain())
}

func boronMain() int {

	var (
		IP               string
		Port             int
		Protocol         string
		Service          bool
		Measurement      string
		Plugin           string
		PluginParameters string
		Tags             string
		TelegrafLocation string
		WorkingLocation  string
		LessThan         bool
		Warning          float64
		Critical         float64
	)

	randomFloat := rand.Float64()

	boronFlags := flag.NewFlagSet("boron", flag.ExitOnError)
	boronFlags.Usage = func() { printUsage() }

	//
	boronFlags.StringVar(&IP, "ip", "localhost", "IP of Service")
	boronFlags.IntVar(&Port, "port", 8000, "Port of Service")
	boronFlags.StringVar(&Protocol, "protocol", "", "Protocol of Service i.e http:// or tcp://")
	boronFlags.BoolVar(&Service, "service", false, "Query a Service which requires an array of Protocol,IP,Port")

	//
	boronFlags.StringVar(&Tags, "tags", "", "tags to determine correct measurement")
	boronFlags.StringVar(&Measurement, "measurement", "", "Single Telegraf timemeasurement to check against")
	boronFlags.StringVar(&Plugin, "plugin", "", "Telegraf Plugin name")
	boronFlags.StringVar(&PluginParameters, "plugin-parameters", "", "Parameters for Telegraf Plugin")
	boronFlags.StringVar(&TelegrafLocation, "telegraf-location", "/usr/bin/telegraf", "Absolute path of telegraf binary")
	boronFlags.StringVar(&WorkingLocation, "working-location", "/tmp/", "Working location to generate temporary plugin files")

	boronFlags.BoolVar(&LessThan, "lessthan", false, "Warning and Critical values will notify if less than")
	boronFlags.Float64Var(&Warning, "warning", randomFloat, "Exits with code 1")
	boronFlags.Float64Var(&Critical, "critical", randomFloat, "Exits with code 2")

	if err := boronFlags.Parse(os.Args[1:]); err != nil {
		boronFlags.Usage()
		return 1
	}

	temporaryConfig, err := writeTemplate(Plugin, PluginParameters, IP, Port, Service, Protocol, WorkingLocation, TelegrafTemplate)
	if err != nil {
		return 2
	}

	defer os.Remove(temporaryConfig)

	measurementResult, err := executeTelegraf(TelegrafLocation, temporaryConfig, Plugin, Measurement, Tags)
	if err != nil || measurementResult == math.MaxFloat64 {
		fmt.Fprintf(os.Stderr, "Error could not find measurement: %s for specified plugin: %s\n err: ", Measurement, Plugin, err)
		return 1
	}

	exitCode := thresholdChecker(measurementResult, Measurement, LessThan, Warning, Critical, randomFloat)
	return exitCode
}

//Splits Plugin parameters
func splitParameters(PluginParameters string) []string {
	return strings.Split(PluginParameters, "|")
}

//Generates template  file for given plugin
func writeTemplate(Plugin string, PluginParameters string, IP string, Port int, Service bool, Protocol string, WorkingLocation, Template string) (string, error) {

	genUUID := uuid.NewV4()

	t, err := template.New("plugin").Parse(Template)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return "", err
	}

	temporaryFileLocation := fmt.Sprintf("%s%s-%s.conf", WorkingLocation, Plugin, genUUID)
	fh, err := os.Create(temporaryFileLocation)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return "", err
	}
	defer fh.Close()

	wr := bufio.NewWriter(fh)
	defer wr.Flush()

	tp := TelegrafPlugin{
		Plugin:   Plugin,
		KV:       splitParameters(PluginParameters),
		IP:       IP,
		Port:     Port,
		Protocol: Protocol,
		Service:  Service,
	}

	err = t.Execute(wr, tp)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return "", err
	}

	return temporaryFileLocation, nil
}

//Executes Telegraf in STDOUT mode with the generated config file.
//Parses STDOUT and abstracts value
func executeTelegraf(TelegrafLocation, Config, Plugin, Measurement, Tags string) (float64, error) {
	cmd := exec.Command(TelegrafLocation, "-config", Config, "-filter", Plugin, "-test", "|", "grep", Measurement)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return math.MaxFloat64, err
	}

	if err := cmd.Start(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return math.MaxFloat64, err
	}

	r, err := ioutil.ReadAll(stdout)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return math.MaxFloat64, err
	}

	re := regexp.MustCompile(fmt.Sprintf(`.*(\[\]|\[.*\])\s+(%s)\s+value=(\w+)`, Measurement))
	regexResult := re.FindAllStringSubmatch(string(r), -1)

	if len(regexResult) >= 1 {
		for _, group := range regexResult {
			tags := group[1]
			value := group[3]

			if tags == "[]" {

				f, err := strconv.ParseFloat(value, 64)
				if err != nil {
					fmt.Fprintln(os.Stderr, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement)))
					return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement))
				}
				return f, nil

			} else if strings.Contains(tags, Tags) {

				f, err := strconv.ParseFloat(value, 64)
				if err != nil {
					fmt.Fprintln(os.Stderr, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement)))
					return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not a float64 value", Measurement))
				}
				return f, nil
			}
			continue
		}
	} else {
		return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not found", Measurement))
	}
	return math.MaxFloat64, errors.New(fmt.Sprintf("Measurement: %s not found", Measurement))
}

func messageGenerator(thresholdType string, thresholdValue, curerntValue float64, measurement string) {
	fmt.Fprintf(os.Stdout, "Threshold Event:%s ThresholdValue:%f CurrentValue:%f Measurement:%s\n", thresholdType, thresholdValue, curerntValue, measurement)
}

func exceeded(thresholdValue, currentValue float64, lessthan bool) bool {
	switch lessthan {
	case true:
		if thresholdValue <= currentValue {
			return false
		}
		return true
	case false:
		if thresholdValue >= currentValue {
			return false
		}
		return true
	default:
		return false
	}

}

func thresholdChecker(currentValue float64, measurement string, lessthan bool, warningValue, criticalValue, random float64) int {
	if warningValue == random && criticalValue == random {
		fmt.Fprintln(os.Stderr, "Critical threshold must be specified")
		return 1
	}

	if exceeded(criticalValue, currentValue, lessthan) {
		//ALERT CRITICAL
		messageGenerator("CRITICAL", criticalValue, currentValue, measurement)
		return 2
	} else if warningValue != random && exceeded(criticalValue, currentValue, lessthan) == false && exceeded(warningValue, currentValue, lessthan) == true {
		//ALERT WARNING
		messageGenerator("WARNING", warningValue, currentValue, measurement)
		return 1
	} else {
		messageGenerator("PASSING", warningValue, currentValue, measurement)
		return 0
	}

}

func printUsage() {
	fmt.Fprintln(os.Stderr, usageText)
}
<|endoftext|>


<commit_before>package bosh

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

var (
	client     = http.DefaultClient
	transport  = http.DefaultTransport
	bodyReader = ioutil.ReadAll
)

type Config struct {
	URL                 string
	Host                string
	Username            string
	Password            string
	TaskPollingInterval time.Duration
	AllowInsecureSSL    bool
}

type Client struct {
	config Config
}

type Task struct {
	Id     int
	State  string
	Result string
}

func NewClient(config Config) Client {
	if config.TaskPollingInterval == time.Duration(0) {
		config.TaskPollingInterval = 5 * time.Second
	}

	if config.AllowInsecureSSL {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}

		client = &http.Client{
			Transport: transport,
		}
	}

	return Client{
		config: config,
	}
}

func (c Client) GetConfig() Config {
	return c.config
}

func (c Client) rewriteURL(uri string) (string, error) {
	parsedURL, err := url.Parse(uri)
	if err != nil {
		return "", err
	}

	parsedURL.Scheme = ""
	parsedURL.Host = ""

	return c.config.URL + parsedURL.String(), nil
}

func (c Client) checkTask(location string) (Task, error) {
	location, err := c.rewriteURL(location)
	if err != nil {
		return Task{}, err
	}

	var task Task
	request, err := http.NewRequest("GET", location, nil)
	if err != nil {
		return task, err
	}
	request.SetBasicAuth(c.config.Username, c.config.Password)

	response, err := transport.RoundTrip(request)
	if err != nil {
		return task, err
	}

	err = json.NewDecoder(response.Body).Decode(&task)
	if err != nil {
		return task, err
	}

	return task, nil
}

func (c Client) checkTaskStatus(location string) (int, error) {
	for {
		task, err := c.checkTask(location)
		if err != nil {
			return 0, err
		}

		switch task.State {
		case "done":
			return task.Id, nil
		case "error":
			taskOutputs, err := c.GetTaskOutput(task.Id)
			if err != nil {
				return task.Id, fmt.Errorf("failed to get full bosh task event log, bosh task failed with an error status %q", task.Result)
			}
			errorMessage := taskOutputs[len(taskOutputs)-1].Error.Message
			return task.Id, fmt.Errorf("bosh task failed with an error status %q", errorMessage)
		case "errored":
			taskOutputs, err := c.GetTaskOutput(task.Id)
			if err != nil {
				return task.Id, fmt.Errorf("failed to get full bosh task event log, bosh task failed with an errored status %q", task.Result)
			}
			errorMessage := taskOutputs[len(taskOutputs)-1].Error.Message
			return task.Id, fmt.Errorf("bosh task failed with an errored status %q", errorMessage)
		case "cancelled":
			return task.Id, errors.New("bosh task was cancelled")
		default:
			time.Sleep(c.config.TaskPollingInterval)
		}
	}
}
<commit_msg>Add DirectorCACert to the bosh client config.<commit_after>package bosh

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

var (
	client     = http.DefaultClient
	transport  = http.DefaultTransport
	bodyReader = ioutil.ReadAll
)

type Config struct {
	URL                 string
	Host                string
	DirectorCACert      string
	Username            string
	Password            string
	TaskPollingInterval time.Duration
	AllowInsecureSSL    bool
}

type Client struct {
	config Config
}

type Task struct {
	Id     int
	State  string
	Result string
}

func NewClient(config Config) Client {
	if config.TaskPollingInterval == time.Duration(0) {
		config.TaskPollingInterval = 5 * time.Second
	}

	if config.AllowInsecureSSL {
		transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}

		client = &http.Client{
			Transport: transport,
		}
	}

	return Client{
		config: config,
	}
}

func (c Client) GetConfig() Config {
	return c.config
}

func (c Client) rewriteURL(uri string) (string, error) {
	parsedURL, err := url.Parse(uri)
	if err != nil {
		return "", err
	}

	parsedURL.Scheme = ""
	parsedURL.Host = ""

	return c.config.URL + parsedURL.String(), nil
}

func (c Client) checkTask(location string) (Task, error) {
	location, err := c.rewriteURL(location)
	if err != nil {
		return Task{}, err
	}

	var task Task
	request, err := http.NewRequest("GET", location, nil)
	if err != nil {
		return task, err
	}
	request.SetBasicAuth(c.config.Username, c.config.Password)

	response, err := transport.RoundTrip(request)
	if err != nil {
		return task, err
	}

	err = json.NewDecoder(response.Body).Decode(&task)
	if err != nil {
		return task, err
	}

	return task, nil
}

func (c Client) checkTaskStatus(location string) (int, error) {
	for {
		task, err := c.checkTask(location)
		if err != nil {
			return 0, err
		}

		switch task.State {
		case "done":
			return task.Id, nil
		case "error":
			taskOutputs, err := c.GetTaskOutput(task.Id)
			if err != nil {
				return task.Id, fmt.Errorf("failed to get full bosh task event log, bosh task failed with an error status %q", task.Result)
			}
			errorMessage := taskOutputs[len(taskOutputs)-1].Error.Message
			return task.Id, fmt.Errorf("bosh task failed with an error status %q", errorMessage)
		case "errored":
			taskOutputs, err := c.GetTaskOutput(task.Id)
			if err != nil {
				return task.Id, fmt.Errorf("failed to get full bosh task event log, bosh task failed with an errored status %q", task.Result)
			}
			errorMessage := taskOutputs[len(taskOutputs)-1].Error.Message
			return task.Id, fmt.Errorf("bosh task failed with an errored status %q", errorMessage)
		case "cancelled":
			return task.Id, errors.New("bosh task was cancelled")
		default:
			time.Sleep(c.config.TaskPollingInterval)
		}
	}
}
<|endoftext|>


<commit_before>package bql

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"gopkg.in/sensorbee/sensorbee.v0/core"
	"gopkg.in/sensorbee/sensorbee.v0/data"
)

// TODO: create bql/builtin directory and move components in this file to there

func createSharedStateSink(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Sink, error) {
	// Get only name parameter from params
	name, ok := params["name"]
	if !ok {
		return nil, fmt.Errorf("cannot find 'name' parameter")
	}
	nameStr, err := data.AsString(name)
	if err != nil {
		return nil, err
	}

	// TODO: Support cascading delete. Because it isn't supported yet,
	// the sink will be running even after the target state is dropped.
	// Moreover, creating a state having the same name after dropping
	// the previous state might result in a confusing behavior.
	return core.NewSharedStateSink(ctx, nameStr)
}

func init() {
	MustRegisterGlobalSinkCreator("uds", SinkCreatorFunc(createSharedStateSink))
}

type readerSource struct {
	filename string
	tsField  data.Path
	ioParams *IOParams

	// repeat is the number of times that the input data is read. When its value
	// is less than 0, the source will read the input again and again until it's
	// stopped. When the value is 0, the source only read the input once. When
	// the value is k (> 0), the input is read k+1 times including the first run.
	repeat int64

	// interval is the interval between emissions of two consecutive tuples.
	// When its value is less than or equal to 0, the source tries to emit
	// tuples as fast as possible.
	interval time.Duration
	stopCh   chan struct{}
}

func (s *readerSource) GenerateStream(ctx *core.Context, w core.Writer) error {
	for r := int64(0); s.repeat < 0 || r <= s.repeat; r++ {
		if err := s.generateStream(ctx, w); err != nil {
			return err
		}
	}
	return nil
}

func (s *readerSource) generateStream(ctx *core.Context, w core.Writer) error {
	f, err := os.Open(s.filename)
	if err != nil {
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			ctx.ErrLog(err).WithField("node_name", s.ioParams.Name).
				Warning("Cannot close the file")
		}
	}()

	r := bufio.NewReader(f)
	next := time.Now()
	for lineNumber := 0; ; lineNumber++ {
		line, err := r.ReadBytes('\n')
		if err != nil && err != io.EOF {
			return err
		}

		line = bytes.TrimSpace(line)
		if len(line) == 0 {
			if err == io.EOF {
				break
			}
			continue
		}

		m := data.Map{}
		if err := json.Unmarshal(line, &m); err != nil {
			ctx.ErrLog(err).WithField("node_name", s.ioParams.Name).
				WithField("jsonl_line_number", lineNumber).
				WithField("body", string(line)).Warning("Ignoring the line due to a json parse error")
			continue
		}

		t := core.NewTuple(m)
		if s.interval > 0 {
			// When the interval parameter is given, a proper application
			// timestamp should be assigned to each tuple.
			t.Timestamp = next
		}
		if s.tsField != nil {
			if v, err := t.Data.Get(s.tsField); err == nil {
				if ts, err := data.ToTimestamp(v); err != nil {
					ctx.ErrLog(err).WithField("node_name", s.ioParams.Name).
						WithField("jsonl_line_number", lineNumber).
						WithField("timestamp_field", s.tsField).
						WithField("timestamp_field_value", v).
						Warning("Cannot convert a value in timestamp_field to a timestamp")
				} else {
					t.Timestamp = ts
				}
			}
		}

		if err := w.Write(ctx, t); err != nil {
			return err
		}

		if s.interval > 0 {
			// wait as accurate as possible
			now := time.Now()
			next = next.Add(s.interval)
			if next.Before(now) {
				// delayed too much and should be rescheduled.
				next = now.Add(s.interval)
			}

			select {
			case <-s.stopCh:
				// This works as long as createFileSource returns a source
				// wrapped with core.NewRewindableSource or core.ImplementSourceStop.
				return core.ErrSourceStopped
			case <-time.After(next.Sub(now)):
			}
		}
	}
	return nil
}

func (s *readerSource) Stop(ctx *core.Context) error {
	close(s.stopCh)
	return nil
}

func createFileSource(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
	// TODO: add format parameter

	v := &struct {
		Path           string `bql:",required"`
		Rewindable     bool
		TimestampField string
		Repeat         int64
		Interval       time.Duration
	}{
		Rewindable:     false,
		TimestampField: "",
		Repeat:         0,
	}
	dec := data.NewDecoder(nil)
	if err := dec.Decode(params, v); err != nil {
		return nil, err
	}

	var tsField data.Path
	if v.TimestampField != "" {
		var err error
		if tsField, err = data.CompilePath(v.TimestampField); err != nil {
			return nil, fmt.Errorf("'timestamp_field' parameter doesn't have a valid path: %v", err)
		}
	}

	s := &readerSource{
		filename: v.Path,
		tsField:  tsField,
		ioParams: ioParams,
		repeat:   v.Repeat,
		interval: v.Interval,
		stopCh:   make(chan struct{}),
	}
	if v.Rewindable {
		return core.NewRewindableSource(s), nil
	}
	return core.ImplementSourceStop(s), nil
}

func init() {
	MustRegisterGlobalSourceCreator("file", SourceCreatorFunc(createFileSource))
}

type writerSink struct {
	m           sync.Mutex
	w           io.Writer
	shouldClose bool
}

func (s *writerSink) Write(ctx *core.Context, t *core.Tuple) error {
	// TODO: support custom formatting. There're several things that need to
	// be considered such as concurrent formatting, zero-copy write, and so on.
	// While encoding tuples outside the lock supports concurrent formatting,
	// it makes it difficult to support zero-copy write.

	js := t.Data.String() // Format this outside the lock

	// This lock is required to avoid interleaving JSONs.
	s.m.Lock()
	defer s.m.Unlock()
	if s.w == nil {
		return errors.New("the sink is already closed")
	}
	_, err := fmt.Fprintln(s.w, js)
	return err
}

func (s *writerSink) Close(ctx *core.Context) error {
	s.m.Lock()
	defer s.m.Unlock()
	if s.w == nil {
		return nil
	}
	if s.shouldClose {
		if c, ok := s.w.(io.Closer); ok {
			return c.Close()
		}
	}
	return nil
}

func createStdoutSink(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Sink, error) {
	return &writerSink{
		w: os.Stdout,
	}, nil
}

func createFileSink(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Sink, error) {
	// TODO: currently this sink isn't secure because it accepts any path.
	// TODO: support buffering
	// TODO: provide "format" parameter to support output formats other than "jsonl".
	//       "jsonl" should be the default value.
	// TODO: support "compression" parameter with values like "gz".

	v := &struct {
		Path     string `bql:",required"`
		Truncate bool
	}{
		Truncate: false,
	}
	dec := data.NewDecoder(nil)
	if err := dec.Decode(params, v); err != nil {
		return nil, err
	}

	flags := os.O_WRONLY | os.O_APPEND | os.O_CREATE
	if v.Truncate {
		flags |= os.O_TRUNC
	}

	file, err := os.OpenFile(v.Path, flags, 0644)
	if err != nil {
		return nil, err
	}
	return &writerSink{
		w:           file,
		shouldClose: true,
	}, nil
}

func init() {
	MustRegisterGlobalSinkCreator("stdout", SinkCreatorFunc(createStdoutSink))
	MustRegisterGlobalSinkCreator("file", SinkCreatorFunc(createFileSink))
}

func createDroppedTupleCollectorSource(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
	return core.NewDroppedTupleCollectorSource(), nil
}

func init() {
	MustRegisterGlobalSourceCreator("dropped_tuples", SourceCreatorFunc(createDroppedTupleCollectorSource))
}

type nodeStatusSource struct {
	topology core.Topology
	interval time.Duration
	stopCh   chan struct{}
}

func (s *nodeStatusSource) GenerateStream(ctx *core.Context, w core.Writer) error {
	next := time.Now().Add(s.interval)
	for {
		select {
		case <-s.stopCh:
			return nil
		case <-time.After(next.Sub(time.Now())):
		}
		now := time.Now()

		for name, n := range s.topology.Nodes() {
			t := &core.Tuple{
				Timestamp:     now,
				ProcTimestamp: now,
				Data:          n.Status(),
			}
			t.Data["node_name"] = data.String(name)
			t.Data["node_type"] = data.String(n.Type().String())
			w.Write(ctx, t)
		}

		next = next.Add(s.interval)
		if next.Before(now) {
			// delayed too much and should be rescheduled.
			next = now.Add(s.interval)
		}
	}
}

func (s *nodeStatusSource) Stop(ctx *core.Context) error {
	close(s.stopCh)
	return nil
}

// createNodeStatusSourceCreator creates a SourceCreator which creates
// nodeStatusSource. Because it requires core.Topology, it cannot be registered
// statically. It'll be registered in a function like NewTopologyBuilder.
func createNodeStatusSourceCreator(t core.Topology) SourceCreator {
	return SourceCreatorFunc(func(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
		interval := 1 * time.Second
		if v, ok := params["interval"]; !ok {
		} else if d, err := data.ToDuration(v); err != nil {
			return nil, err
		} else {
			interval = d
		}

		return &nodeStatusSource{
			topology: t,
			interval: interval,
			stopCh:   make(chan struct{}),
		}, nil
	})
}

type edgeStatusSource struct {
	topology core.Topology
	interval time.Duration
	stopCh   chan struct{}
}

func (s *edgeStatusSource) GenerateStream(ctx *core.Context, w core.Writer) error {
	next := time.Now().Add(s.interval)

	inputPath := data.MustCompilePath("input_stats.inputs")

	for {
		select {
		case <-s.stopCh:
			return nil
		case <-time.After(next.Sub(time.Now())):
		}
		now := time.Now()

		// collect all nodes that can receive data
		receivers := map[string]core.Node{}
		for name, b := range s.topology.Boxes() {
			receivers[name] = b
		}
		for name, s := range s.topology.Sinks() {
			receivers[name] = s
		}

		// loop over those receiver nodes and consider all of
		// their incoming edges
		for name, n := range receivers {
			nodeStatus := n.Status()
			// get the input status
			inputs, err := nodeStatus.Get(inputPath)
			if err != nil {
				ctx.ErrLog(err).WithField("node_status", nodeStatus).
					WithField("node_name", name).
					Error("No input_stats present in node status")
				continue
			}
			inputMap, err := data.AsMap(inputs)
			if err != nil {
				ctx.ErrLog(err).WithField("inputs", inputs).
					WithField("node_name", name).
					Error("input_stats.inputs is not a Map")
				continue
			}

			// loop over the input nodes to get an edge-centric view
			for inputName, inputStats := range inputMap {
				inputNode, err := s.topology.Node(inputName)
				if err != nil {
					ctx.ErrLog(err).WithField("sender", inputName).
						WithField("receiver", name).
						Error("Node listens to non-existing node")
					continue
				}
				edgeData := data.Map{
					"sender": data.Map{
						"node_name": data.String(inputName),
						"node_type": data.String(inputNode.Type().String()),
					},
					"receiver": data.Map{
						"node_name": data.String(name),
						"node_type": data.String(n.Type().String()),
					},
					// use the input statistics for that edge from the
					// receiver as edge statistics. the data is correct,
					// but the wording may be a bit weird, e.g. "num_received"
					// should maybe rather be "num_transferred"
					"stats": inputStats,
				}
				// write tuple
				t := &core.Tuple{
					Timestamp:     now,
					ProcTimestamp: now,
					Data:          edgeData,
				}
				w.Write(ctx, t)
			}

		}

		next = next.Add(s.interval)
		if next.Before(now) {
			// delayed too much and should be rescheduled.
			next = now.Add(s.interval)
		}
	}
}

func (s *edgeStatusSource) Stop(ctx *core.Context) error {
	close(s.stopCh)
	return nil
}

func createEdgeStatusSourceCreator(t core.Topology) SourceCreator {
	return SourceCreatorFunc(func(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
		interval := 1 * time.Second
		if v, ok := params["interval"]; !ok {
		} else if d, err := data.ToDuration(v); err != nil {
			return nil, err
		} else {
			interval = d
		}

		return &edgeStatusSource{
			topology: t,
			interval: interval,
			stopCh:   make(chan struct{}),
		}, nil
	})
}
<commit_msg>support file rotate<commit_after>package bql

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"gopkg.in/natefinch/lumberjack.v2"
	"gopkg.in/sensorbee/sensorbee.v0/core"
	"gopkg.in/sensorbee/sensorbee.v0/data"
)

// TODO: create bql/builtin directory and move components in this file to there

func createSharedStateSink(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Sink, error) {
	// Get only name parameter from params
	name, ok := params["name"]
	if !ok {
		return nil, fmt.Errorf("cannot find 'name' parameter")
	}
	nameStr, err := data.AsString(name)
	if err != nil {
		return nil, err
	}

	// TODO: Support cascading delete. Because it isn't supported yet,
	// the sink will be running even after the target state is dropped.
	// Moreover, creating a state having the same name after dropping
	// the previous state might result in a confusing behavior.
	return core.NewSharedStateSink(ctx, nameStr)
}

func init() {
	MustRegisterGlobalSinkCreator("uds", SinkCreatorFunc(createSharedStateSink))
}

type readerSource struct {
	filename string
	tsField  data.Path
	ioParams *IOParams

	// repeat is the number of times that the input data is read. When its value
	// is less than 0, the source will read the input again and again until it's
	// stopped. When the value is 0, the source only read the input once. When
	// the value is k (> 0), the input is read k+1 times including the first run.
	repeat int64

	// interval is the interval between emissions of two consecutive tuples.
	// When its value is less than or equal to 0, the source tries to emit
	// tuples as fast as possible.
	interval time.Duration
	stopCh   chan struct{}
}

func (s *readerSource) GenerateStream(ctx *core.Context, w core.Writer) error {
	for r := int64(0); s.repeat < 0 || r <= s.repeat; r++ {
		if err := s.generateStream(ctx, w); err != nil {
			return err
		}
	}
	return nil
}

func (s *readerSource) generateStream(ctx *core.Context, w core.Writer) error {
	f, err := os.Open(s.filename)
	if err != nil {
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			ctx.ErrLog(err).WithField("node_name", s.ioParams.Name).
				Warning("Cannot close the file")
		}
	}()

	r := bufio.NewReader(f)
	next := time.Now()
	for lineNumber := 0; ; lineNumber++ {
		line, err := r.ReadBytes('\n')
		if err != nil && err != io.EOF {
			return err
		}

		line = bytes.TrimSpace(line)
		if len(line) == 0 {
			if err == io.EOF {
				break
			}
			continue
		}

		m := data.Map{}
		if err := json.Unmarshal(line, &m); err != nil {
			ctx.ErrLog(err).WithField("node_name", s.ioParams.Name).
				WithField("jsonl_line_number", lineNumber).
				WithField("body", string(line)).Warning("Ignoring the line due to a json parse error")
			continue
		}

		t := core.NewTuple(m)
		if s.interval > 0 {
			// When the interval parameter is given, a proper application
			// timestamp should be assigned to each tuple.
			t.Timestamp = next
		}
		if s.tsField != nil {
			if v, err := t.Data.Get(s.tsField); err == nil {
				if ts, err := data.ToTimestamp(v); err != nil {
					ctx.ErrLog(err).WithField("node_name", s.ioParams.Name).
						WithField("jsonl_line_number", lineNumber).
						WithField("timestamp_field", s.tsField).
						WithField("timestamp_field_value", v).
						Warning("Cannot convert a value in timestamp_field to a timestamp")
				} else {
					t.Timestamp = ts
				}
			}
		}

		if err := w.Write(ctx, t); err != nil {
			return err
		}

		if s.interval > 0 {
			// wait as accurate as possible
			now := time.Now()
			next = next.Add(s.interval)
			if next.Before(now) {
				// delayed too much and should be rescheduled.
				next = now.Add(s.interval)
			}

			select {
			case <-s.stopCh:
				// This works as long as createFileSource returns a source
				// wrapped with core.NewRewindableSource or core.ImplementSourceStop.
				return core.ErrSourceStopped
			case <-time.After(next.Sub(now)):
			}
		}
	}
	return nil
}

func (s *readerSource) Stop(ctx *core.Context) error {
	close(s.stopCh)
	return nil
}

func createFileSource(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
	// TODO: add format parameter

	v := &struct {
		Path           string `bql:",required"`
		Rewindable     bool
		TimestampField string
		Repeat         int64
		Interval       time.Duration
	}{
		Rewindable:     false,
		TimestampField: "",
		Repeat:         0,
	}
	dec := data.NewDecoder(nil)
	if err := dec.Decode(params, v); err != nil {
		return nil, err
	}

	var tsField data.Path
	if v.TimestampField != "" {
		var err error
		if tsField, err = data.CompilePath(v.TimestampField); err != nil {
			return nil, fmt.Errorf("'timestamp_field' parameter doesn't have a valid path: %v", err)
		}
	}

	s := &readerSource{
		filename: v.Path,
		tsField:  tsField,
		ioParams: ioParams,
		repeat:   v.Repeat,
		interval: v.Interval,
		stopCh:   make(chan struct{}),
	}
	if v.Rewindable {
		return core.NewRewindableSource(s), nil
	}
	return core.ImplementSourceStop(s), nil
}

func init() {
	MustRegisterGlobalSourceCreator("file", SourceCreatorFunc(createFileSource))
}

type writerSink struct {
	m           sync.Mutex
	w           io.Writer
	shouldClose bool
}

func (s *writerSink) Write(ctx *core.Context, t *core.Tuple) error {
	// TODO: support custom formatting. There're several things that need to
	// be considered such as concurrent formatting, zero-copy write, and so on.
	// While encoding tuples outside the lock supports concurrent formatting,
	// it makes it difficult to support zero-copy write.

	js := t.Data.String() // Format this outside the lock

	// This lock is required to avoid interleaving JSONs.
	s.m.Lock()
	defer s.m.Unlock()
	if s.w == nil {
		return errors.New("the sink is already closed")
	}
	_, err := fmt.Fprintln(s.w, js)
	return err
}

func (s *writerSink) Close(ctx *core.Context) error {
	s.m.Lock()
	defer s.m.Unlock()
	if s.w == nil {
		return nil
	}
	if s.shouldClose {
		if c, ok := s.w.(io.Closer); ok {
			return c.Close()
		}
	}
	return nil
}

func createStdoutSink(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Sink, error) {
	return &writerSink{
		w: os.Stdout,
	}, nil
}

func createFileSink(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Sink, error) {
	// TODO: currently this sink isn't secure because it accepts any path.
	// TODO: support buffering
	// TODO: provide "format" parameter to support output formats other than "jsonl".
	//       "jsonl" should be the default value.
	// TODO: support "compression" parameter with values like "gz".

	v := &struct {
		Path     string `bql:",required"`
		Truncate bool
		// rotate information
		MaxSize    int `bql:"maxsize"`
		MaxAge     int `bql:"maxage"`
		MaxBackups int `bql:"maxbackups"`
	}{
		Truncate: false,
		MaxSize:  0,
	}
	dec := data.NewDecoder(nil)
	if err := dec.Decode(params, v); err != nil {
		return nil, err
	}

	var w io.Writer
	if v.MaxSize > 0 {
		l := lumberjack.Logger{
			Filename: v.Path,
		}
		if v.MaxAge > 0 {
			l.MaxAge = v.MaxAge
		}
		if v.MaxBackups > 0 {
			l.MaxBackups = v.MaxBackups
		}
		w = &l
	} else {
		flags := os.O_WRONLY | os.O_APPEND | os.O_CREATE
		if v.Truncate {
			flags |= os.O_TRUNC
		}

		file, err := os.OpenFile(v.Path, flags, 0644)
		if err != nil {
			return nil, err
		}
		w = file
	}
	return &writerSink{
		w:           w,
		shouldClose: true,
	}, nil
}

func init() {
	MustRegisterGlobalSinkCreator("stdout", SinkCreatorFunc(createStdoutSink))
	MustRegisterGlobalSinkCreator("file", SinkCreatorFunc(createFileSink))
}

func createDroppedTupleCollectorSource(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
	return core.NewDroppedTupleCollectorSource(), nil
}

func init() {
	MustRegisterGlobalSourceCreator("dropped_tuples", SourceCreatorFunc(createDroppedTupleCollectorSource))
}

type nodeStatusSource struct {
	topology core.Topology
	interval time.Duration
	stopCh   chan struct{}
}

func (s *nodeStatusSource) GenerateStream(ctx *core.Context, w core.Writer) error {
	next := time.Now().Add(s.interval)
	for {
		select {
		case <-s.stopCh:
			return nil
		case <-time.After(next.Sub(time.Now())):
		}
		now := time.Now()

		for name, n := range s.topology.Nodes() {
			t := &core.Tuple{
				Timestamp:     now,
				ProcTimestamp: now,
				Data:          n.Status(),
			}
			t.Data["node_name"] = data.String(name)
			t.Data["node_type"] = data.String(n.Type().String())
			w.Write(ctx, t)
		}

		next = next.Add(s.interval)
		if next.Before(now) {
			// delayed too much and should be rescheduled.
			next = now.Add(s.interval)
		}
	}
}

func (s *nodeStatusSource) Stop(ctx *core.Context) error {
	close(s.stopCh)
	return nil
}

// createNodeStatusSourceCreator creates a SourceCreator which creates
// nodeStatusSource. Because it requires core.Topology, it cannot be registered
// statically. It'll be registered in a function like NewTopologyBuilder.
func createNodeStatusSourceCreator(t core.Topology) SourceCreator {
	return SourceCreatorFunc(func(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
		interval := 1 * time.Second
		if v, ok := params["interval"]; !ok {
		} else if d, err := data.ToDuration(v); err != nil {
			return nil, err
		} else {
			interval = d
		}

		return &nodeStatusSource{
			topology: t,
			interval: interval,
			stopCh:   make(chan struct{}),
		}, nil
	})
}

type edgeStatusSource struct {
	topology core.Topology
	interval time.Duration
	stopCh   chan struct{}
}

func (s *edgeStatusSource) GenerateStream(ctx *core.Context, w core.Writer) error {
	next := time.Now().Add(s.interval)

	inputPath := data.MustCompilePath("input_stats.inputs")

	for {
		select {
		case <-s.stopCh:
			return nil
		case <-time.After(next.Sub(time.Now())):
		}
		now := time.Now()

		// collect all nodes that can receive data
		receivers := map[string]core.Node{}
		for name, b := range s.topology.Boxes() {
			receivers[name] = b
		}
		for name, s := range s.topology.Sinks() {
			receivers[name] = s
		}

		// loop over those receiver nodes and consider all of
		// their incoming edges
		for name, n := range receivers {
			nodeStatus := n.Status()
			// get the input status
			inputs, err := nodeStatus.Get(inputPath)
			if err != nil {
				ctx.ErrLog(err).WithField("node_status", nodeStatus).
					WithField("node_name", name).
					Error("No input_stats present in node status")
				continue
			}
			inputMap, err := data.AsMap(inputs)
			if err != nil {
				ctx.ErrLog(err).WithField("inputs", inputs).
					WithField("node_name", name).
					Error("input_stats.inputs is not a Map")
				continue
			}

			// loop over the input nodes to get an edge-centric view
			for inputName, inputStats := range inputMap {
				inputNode, err := s.topology.Node(inputName)
				if err != nil {
					ctx.ErrLog(err).WithField("sender", inputName).
						WithField("receiver", name).
						Error("Node listens to non-existing node")
					continue
				}
				edgeData := data.Map{
					"sender": data.Map{
						"node_name": data.String(inputName),
						"node_type": data.String(inputNode.Type().String()),
					},
					"receiver": data.Map{
						"node_name": data.String(name),
						"node_type": data.String(n.Type().String()),
					},
					// use the input statistics for that edge from the
					// receiver as edge statistics. the data is correct,
					// but the wording may be a bit weird, e.g. "num_received"
					// should maybe rather be "num_transferred"
					"stats": inputStats,
				}
				// write tuple
				t := &core.Tuple{
					Timestamp:     now,
					ProcTimestamp: now,
					Data:          edgeData,
				}
				w.Write(ctx, t)
			}

		}

		next = next.Add(s.interval)
		if next.Before(now) {
			// delayed too much and should be rescheduled.
			next = now.Add(s.interval)
		}
	}
}

func (s *edgeStatusSource) Stop(ctx *core.Context) error {
	close(s.stopCh)
	return nil
}

func createEdgeStatusSourceCreator(t core.Topology) SourceCreator {
	return SourceCreatorFunc(func(ctx *core.Context, ioParams *IOParams, params data.Map) (core.Source, error) {
		interval := 1 * time.Second
		if v, ok := params["interval"]; !ok {
		} else if d, err := data.ToDuration(v); err != nil {
			return nil, err
		} else {
			interval = d
		}

		return &edgeStatusSource{
			topology: t,
			interval: interval,
			stopCh:   make(chan struct{}),
		}, nil
	})
}
<|endoftext|>


<commit_before>// Package bridgeproxy relays with a remote TLS host via a HTTP proxy bridge.
package bridgeproxy

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"os"
	"strings"
)

// Peer is a server we are connecting to. This can either be an
// intermediate http(s) proxy server or the final server we want
// to connect to.
type Peer struct {
	TLSConfig    *tls.Config // nil if unencrypted, valid config otherwise
	HostName     string      // The hostname to connect to
	Port         int         // The port to connect to on the hostname
	ConnectExtra string      // Extra headers to send after the CONNECT line
}

func forward(src io.ReadCloser, dst io.WriteCloser) {
	if _, err := io.Copy(dst, src); err != nil {
		fmt.Println("Could not forward:", err)
	}
	src.Close()
	dst.Close()
}

// readLine reads a network line one byte at a time. We need to read unbuffered
// as we might later turn the connection after a 200 response for a CONNECT
// into a tls connection, for which we need a net.Conn.
func readLine(src io.Reader) (string, error) {
	line := make([]byte, 0, 64)
	length := 0
	for length < 2 || line[length-2] != '\r' || line[length-1] != '\n' {
		line = append(line, 0)
		if _, err := io.ReadFull(src, line[length:]); err != nil {
			return "", err
		}
		length++
	}
	return string(line[:length]), nil
}

func handleRequest(client net.Conn, peers []Peer) {
	var connection net.Conn
	var err error
	var connectionClosureHandled bool

	defer func() {
		if !connectionClosureHandled {
			client.Close()
			if connection != nil {
				connection.Close()
			}
		}
	}()

	for i, peer := range peers {
		// The first peer has to be dialed, others happen via connect
		if i == 0 {
			connection, err = net.Dial("tcp", fmt.Sprintf("%s:%d", peer.HostName, peer.Port))
			if err != nil {
				fmt.Println("ERROR: Could not connect", err)
				return
			}
		} else {
			fmt.Fprintf(connection, "CONNECT %s:%d HTTP/1.0\r\n%s\r\n\r\n", peer.HostName, peer.Port, peers[i-1].ConnectExtra)

			line, err := readLine(io.LimitReader(connection, 1024))
			if err != nil {
				fmt.Println("Could not read:", err)
				return
			}
			if !strings.HasPrefix(line, "HTTP/1.0 200") && !strings.HasPrefix(line, "HTTP/1.1 200") {
				prefix := "Could not CONNECT to a proxy: "
				fmt.Fprintf(client, "HTTP/1.0 502 Bad Gateway\r\n")
				fmt.Fprintf(client, "Content-Type: text/plain\r\n")
				fmt.Fprintf(client, "Content-Length: %d\r\n", len(prefix)+len(line))
				fmt.Fprintf(client, "\r\n")
				fmt.Fprintf(client, prefix+line)
				return
			}
			if line, err = readLine(connection); err != nil {
				fmt.Println("Invalid second response line:", line)
				return
			}
		}

		if peer.TLSConfig != nil {
			connection = tls.Client(connection, peer.TLSConfig)
		}
	}

	connectionClosureHandled = true
	// Forward traffic between the client connected to us and the remote proxy
	go forward(client, connection)
	go forward(connection, client)
}

// Serve serves the specified configuration, forwarding any packets from the
// local address given in listenAdress to the last peer specified in peers via
// any peers before specified before it.
func Serve(listenAdress string, peers []Peer) {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", listenAdress)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on", listenAdress)
	fmt.Println("- Forwarding requests to", peers[len(peers)-1], "via", peers[0:len(peers)-1])
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go handleRequest(conn, peers)
	}
}
<commit_msg>Implement nicer (HTTP-based) error handling<commit_after>// Package bridgeproxy relays with a remote TLS host via a HTTP proxy bridge.
package bridgeproxy

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"strings"
)

// Peer is a server we are connecting to. This can either be an
// intermediate http(s) proxy server or the final server we want
// to connect to.
type Peer struct {
	TLSConfig    *tls.Config // nil if unencrypted, valid config otherwise
	HostName     string      // The hostname to connect to
	Port         int         // The port to connect to on the hostname
	ConnectExtra string      // Extra headers to send after the CONNECT line
}

func forward(src io.ReadCloser, dst io.WriteCloser) {
	if _, err := io.Copy(dst, src); err != nil {
		fmt.Println("Could not forward:", err)
	}
	src.Close()
	dst.Close()
}

// readLine reads a network line one byte at a time. We need to read unbuffered
// as we might later turn the connection after a 200 response for a CONNECT
// into a tls connection, for which we need a net.Conn.
func readLine(src io.Reader) (string, error) {
	line := make([]byte, 0, 64)
	length := 0
	for length < 2 || line[length-2] != '\r' || line[length-1] != '\n' {
		line = append(line, 0)
		if _, err := io.ReadFull(src, line[length:]); err != nil {
			return "", err
		}
		length++
	}
	return string(line[:length]), nil
}

func writeResponse(w io.Writer, code int, format string, printf ...interface{}) {
	msg := fmt.Sprintf(format, printf...)
	fmt.Fprintf(w, "HTTP/1.0 %d %s\r\n", code, http.StatusText(code))
	fmt.Fprintf(w, "Content-Type: text/plain\r\n")
	fmt.Fprintf(w, "Content-Length: %d\r\n", len(msg))
	fmt.Fprintf(w, "\r\n")
	fmt.Fprintf(w, "%s", msg)
}

func handleRequest(client net.Conn, peers []Peer) {
	var connection net.Conn
	var err error
	var connectionClosureHandled bool

	defer func() {
		if !connectionClosureHandled {
			client.Close()
			if connection != nil {
				connection.Close()
			}
		}
	}()

	for i, peer := range peers {
		// The first peer has to be dialed, others happen via connect
		if i == 0 {
			connection, err = net.Dial("tcp", fmt.Sprintf("%s:%d", peer.HostName, peer.Port))
			if err != nil {
				fmt.Println("ERROR: Could not connect", err)
				writeResponse(client, 502, "Could not dial proxy: %s\r\n", err)
				return
			}
		} else {
			fmt.Fprintf(connection, "CONNECT %s:%d HTTP/1.0\r\n%s\r\n\r\n", peer.HostName, peer.Port, peers[i-1].ConnectExtra)

			line, err := readLine(io.LimitReader(connection, 1024))
			if err != nil {
				fmt.Println("Could not read:", err)
				writeResponse(client, 502, "Could not CONNECT to %s: %s\r\n", err.Error())
				return
			}
			if !strings.HasPrefix(line, "HTTP/1.0 200") && !strings.HasPrefix(line, "HTTP/1.1 200") {
				writeResponse(client, 502, "Could not CONNECT to %s: %s", peer.HostName, line)
				return
			}
			if line, err = readLine(connection); err != nil {
				fmt.Println("Invalid second response line:", line)
				writeResponse(client, 502, "Could not CONNECT to %s: Missing second line", peer)
				return
			}
		}

		if peer.TLSConfig != nil {
			connection = tls.Client(connection, peer.TLSConfig)
		}
	}

	connectionClosureHandled = true
	// Forward traffic between the client connected to us and the remote proxy
	go forward(client, connection)
	go forward(connection, client)
}

// Serve serves the specified configuration, forwarding any packets from the
// local address given in listenAdress to the last peer specified in peers via
// any peers before specified before it.
func Serve(listenAdress string, peers []Peer) {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", listenAdress)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on", listenAdress)
	fmt.Println("- Forwarding requests to", peers[len(peers)-1], "via", peers[0:len(peers)-1])
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go handleRequest(conn, peers)
	}
}
<|endoftext|>


<commit_before>package bsw

import (
	"errors"
	"github.com/miekg/dns"
	"strings"
)

func LookupIP(ip, serverAddr string) (string, error) {
	fqdn, err := dns.ReverseAddr(ip)
	if err != nil {
		return "", err
	}
	m := &dns.Msg{}
	m.SetQuestion(fqdn, dns.TypePTR)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.PTR); ok {
		return strings.TrimRight(a.Ptr, "."), nil
	} else {
		return "", errors.New("No PTR record returned")
	}
}

func LookupName(fqdn, serverAddr string) (string, error) {
	m := &dns.Msg{}
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.A); ok {
		ip := a.A.String()
		return ip, nil
	} else {
		return "", errors.New("No A record returned")
	}
}

func LookupCname(fqdn, serverAddr string) (string, error) {
	m := &dns.Msg{}
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeCNAME)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.CNAME); ok {
		name := a.Target
		return strings.TrimRight(name, "."), nil
	} else {
		return "", errors.New("No CNAME record returned")
	}
}

func LookupName6(fqdn, serverAddr string) (string, error) {
	m := &dns.Msg{}
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeAAAA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.AAAA); ok {
		ip := a.AAAA.String()
		return ip, nil
	} else {
		return "", errors.New("No AAAA record returned")
	}
}
<commit_msg>Comments generic functions<commit_after>package bsw

import (
	"errors"
	"github.com/miekg/dns"
	"strings"
)

// Return hostname from PTR record or error.
func LookupIP(ip, serverAddr string) (string, error) {
	fqdn, err := dns.ReverseAddr(ip)
	if err != nil {
		return "", err
	}
	m := &dns.Msg{}
	m.SetQuestion(fqdn, dns.TypePTR)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.PTR); ok {
		return strings.TrimRight(a.Ptr, "."), nil
	} else {
		return "", errors.New("No PTR record returned")
	}
}

// Return IPv4 address from A record or error.
func LookupName(fqdn, serverAddr string) (string, error) {
	m := &dns.Msg{}
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.A); ok {
		ip := a.A.String()
		return ip, nil
	} else {
		return "", errors.New("No A record returned")
	}
}

// Return IPv4 address from CNAME record or error.
func LookupCname(fqdn, serverAddr string) (string, error) {
	m := &dns.Msg{}
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeCNAME)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.CNAME); ok {
		name := a.Target
		return strings.TrimRight(name, "."), nil
	} else {
		return "", errors.New("No CNAME record returned")
	}
}

// Return IPv6 address from AAAA record or error
func LookupName6(fqdn, serverAddr string) (string, error) {
	m := &dns.Msg{}
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeAAAA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.AAAA); ok {
		ip := a.AAAA.String()
		return ip, nil
	} else {
		return "", errors.New("No AAAA record returned")
	}
}
<|endoftext|>


<commit_before>package bsw

// Version of blacksheepwall
const VERSION = "2.2.1"
<commit_msg>2.3.0<commit_after>package bsw

// Version of blacksheepwall
const VERSION = "2.3.0"
<|endoftext|>