<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"encoding/json"
	"fmt"
	"mig"
	"os"
	"time"
)

// spoolInspection walks through the local directories and performs the following
// 1. load actions and commandsthat are sitting in the directories and waiting for processing
// 2. evaluate actions and commands that are inflight (todo)
// 3. remove finished and invalid actions and commands once the DeleteAfter period is passed
func spoolInspection(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("spoolInspection() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving spoolInspection()"}.Debug()
	}()
	ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "initiating spool inspection"}.Debug()

	err = loadNewActions(ctx)
	if err != nil {
		panic(err)
	}
	err = loadReturnedCommands(ctx)
	if err != nil {
		panic(err)
	}
	err = expireCommands(ctx)
	if err != nil {
		panic(err)
	}
	err = cleanDir(ctx, ctx.Directories.Action.Done)
	if err != nil {
		panic(err)
	}
	err = cleanDir(ctx, ctx.Directories.Action.Invalid)
	if err != nil {
		panic(err)
	}
	err = timeoutAgents(ctx)
	if err != nil {
		panic(err)
	}
	return
}

// loadNewActions walks through the new actions directories and load the actions
// that are passed their scheduled date. It also delete expired actions.
func loadNewActions(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("loadNewActions() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadNewActions()"}.Debug()
	}()
	dir, err := os.Open(ctx.Directories.Action.New)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		filename := ctx.Directories.Action.New + "/" + DirEntry.Name()
		a, err := mig.ActionFromFile(filename)
		if err != nil {
			// failing to load this file, log and skip it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to load new action file %s", filename)}.Err()
			continue
		}
		if time.Now().After(a.ExpireAfter) {
			// delete expired
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: fmt.Sprintf("removing expired action '%s'", a.Name)}
			os.Remove(filename)
		} else if time.Now().After(a.ValidFrom) {
			// queue it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: fmt.Sprintf("scheduling action '%s'", a.Name)}
			ctx.Channels.NewAction <- filename
		}
	}
	dir.Close()
	return
}

// loadReturnedCommands walks through the returned commands directory and loads
// the commands into the scheduler
func loadReturnedCommands(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("loadReturnedCommands() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadReturnedCommands()"}.Debug()
	}()
	dir, err := os.Open(ctx.Directories.Command.Returned)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		filename := ctx.Directories.Command.Returned + "/" + DirEntry.Name()
		cmd, err := mig.CmdFromFile(filename)
		if err != nil {
			// failing to load this file, log and skip it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to load returned command file %s", filename)}.Err()
			continue
		}
		// queue it
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: fmt.Sprintf("loading returned command '%s'", cmd.Action.Name)}
		ctx.Channels.CommandReturned <- filename
	}
	dir.Close()
	return
}

// expireCommands loads commands in the inflight directory
// and terminate the expired ones
func expireCommands(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("expireCommands() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving expireCommands()"}.Debug()
	}()
	dir, err := os.Open(ctx.Directories.Command.InFlight)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		filename := ctx.Directories.Command.InFlight + "/" + DirEntry.Name()
		cmd, err := mig.CmdFromFile(filename)
		if err != nil {
			// failing to load this file, log and skip it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to inflight command file %s", filename)}.Err()
			continue
		}

		if time.Now().After(cmd.Action.ExpireAfter) {
			desc := fmt.Sprintf("expiring command '%s' on agent '%s'", cmd.Action.Name, cmd.Agent.Name)
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: desc}
			cmd.Status = "expired"
			cmd.FinishTime = time.Now().UTC()
			// write it into the returned command directory
			data, err := json.Marshal(cmd)
			if err != nil {
				panic(err)
			}
			dest := fmt.Sprintf("%s/%.0f-%.0f.json", ctx.Directories.Command.Returned, cmd.Action.ID, cmd.ID)
			err = safeWrite(ctx, dest, data)
			if err != nil {
				panic(err)
			}
			//ctx.Directories.Command.Returned
			os.Remove(filename)
		}
	}
	dir.Close()
	return
}

// cleanDir walks through a directory and delete the files that
// are older than the configured DeleteAfter parameter
func cleanDir(ctx Context, targetDir string) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("cleanDir() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving cleanDir()"}.Debug()
	}()
	deletionPoint, err := time.ParseDuration(ctx.Collector.DeleteAfter)
	dir, err := os.Open(targetDir)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		// if the DeleteAfter value is after the time of last modification,
		// the file is due for deletion
		if time.Now().Add(-deletionPoint).After(DirEntry.ModTime()) {
			filename := targetDir + "/" + DirEntry.Name()
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("removing '%s'", filename)}
			os.Remove(filename)
		}
	}
	dir.Close()
	return
}

// timeoutAgents updates the status of agents that are no longer heartbeating to "offline"
func timeoutAgents(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("timeoutAgents() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving timeoutAgents()"}.Debug()
	}()
	timeOutPeriod, err := time.ParseDuration(ctx.Agent.TimeOut)
	if err != nil {
		panic(err)
	}
	pointInTime := time.Now().Add(-timeOutPeriod)
	err = ctx.DB.MarkOfflineAgents(pointInTime)
	if err != nil {
		panic(err)
	}
	return
}
<commit_msg>[minor] debug logging of active agents in collector<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"encoding/json"
	"fmt"
	"mig"
	"os"
	"time"
)

// spoolInspection walks through the local directories and performs the following
// 1. load actions and commandsthat are sitting in the directories and waiting for processing
// 2. evaluate actions and commands that are inflight (todo)
// 3. remove finished and invalid actions and commands once the DeleteAfter period is passed
func spoolInspection(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("spoolInspection() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving spoolInspection()"}.Debug()
	}()
	ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "initiating spool inspection"}.Debug()

	err = loadNewActions(ctx)
	if err != nil {
		panic(err)
	}
	err = loadReturnedCommands(ctx)
	if err != nil {
		panic(err)
	}
	err = expireCommands(ctx)
	if err != nil {
		panic(err)
	}
	err = cleanDir(ctx, ctx.Directories.Action.Done)
	if err != nil {
		panic(err)
	}
	err = cleanDir(ctx, ctx.Directories.Action.Invalid)
	if err != nil {
		panic(err)
	}
	err = timeoutAgents(ctx)
	if err != nil {
		panic(err)
	}
	ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("%d agent listeners are currently active", len(activeAgentsList))}.Debug()

	return
}

// loadNewActions walks through the new actions directories and load the actions
// that are passed their scheduled date. It also delete expired actions.
func loadNewActions(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("loadNewActions() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadNewActions()"}.Debug()
	}()
	dir, err := os.Open(ctx.Directories.Action.New)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		filename := ctx.Directories.Action.New + "/" + DirEntry.Name()
		a, err := mig.ActionFromFile(filename)
		if err != nil {
			// failing to load this file, log and skip it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to load new action file %s", filename)}.Err()
			continue
		}
		if time.Now().After(a.ExpireAfter) {
			// delete expired
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: fmt.Sprintf("removing expired action '%s'", a.Name)}
			os.Remove(filename)
		} else if time.Now().After(a.ValidFrom) {
			// queue it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: fmt.Sprintf("scheduling action '%s'", a.Name)}
			ctx.Channels.NewAction <- filename
		}
	}
	dir.Close()
	return
}

// loadReturnedCommands walks through the returned commands directory and loads
// the commands into the scheduler
func loadReturnedCommands(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("loadReturnedCommands() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadReturnedCommands()"}.Debug()
	}()
	dir, err := os.Open(ctx.Directories.Command.Returned)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		filename := ctx.Directories.Command.Returned + "/" + DirEntry.Name()
		cmd, err := mig.CmdFromFile(filename)
		if err != nil {
			// failing to load this file, log and skip it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to load returned command file %s", filename)}.Err()
			continue
		}
		// queue it
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: fmt.Sprintf("loading returned command '%s'", cmd.Action.Name)}
		ctx.Channels.CommandReturned <- filename
	}
	dir.Close()
	return
}

// expireCommands loads commands in the inflight directory
// and terminate the expired ones
func expireCommands(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("expireCommands() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving expireCommands()"}.Debug()
	}()
	dir, err := os.Open(ctx.Directories.Command.InFlight)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		filename := ctx.Directories.Command.InFlight + "/" + DirEntry.Name()
		cmd, err := mig.CmdFromFile(filename)
		if err != nil {
			// failing to load this file, log and skip it
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to inflight command file %s", filename)}.Err()
			continue
		}

		if time.Now().After(cmd.Action.ExpireAfter) {
			desc := fmt.Sprintf("expiring command '%s' on agent '%s'", cmd.Action.Name, cmd.Agent.Name)
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: desc}
			cmd.Status = "expired"
			cmd.FinishTime = time.Now().UTC()
			// write it into the returned command directory
			data, err := json.Marshal(cmd)
			if err != nil {
				panic(err)
			}
			dest := fmt.Sprintf("%s/%.0f-%.0f.json", ctx.Directories.Command.Returned, cmd.Action.ID, cmd.ID)
			err = safeWrite(ctx, dest, data)
			if err != nil {
				panic(err)
			}
			//ctx.Directories.Command.Returned
			os.Remove(filename)
		}
	}
	dir.Close()
	return
}

// cleanDir walks through a directory and delete the files that
// are older than the configured DeleteAfter parameter
func cleanDir(ctx Context, targetDir string) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("cleanDir() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving cleanDir()"}.Debug()
	}()
	deletionPoint, err := time.ParseDuration(ctx.Collector.DeleteAfter)
	dir, err := os.Open(targetDir)
	dirContent, err := dir.Readdir(-1)
	if err != nil {
		panic(err)
	}
	// loop over the content of the directory
	for _, DirEntry := range dirContent {
		if !DirEntry.Mode().IsRegular() {
			// ignore non file
			continue
		}
		// if the DeleteAfter value is after the time of last modification,
		// the file is due for deletion
		if time.Now().Add(-deletionPoint).After(DirEntry.ModTime()) {
			filename := targetDir + "/" + DirEntry.Name()
			ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("removing '%s'", filename)}
			os.Remove(filename)
		}
	}
	dir.Close()
	return
}

// timeoutAgents updates the status of agents that are no longer heartbeating to "offline"
func timeoutAgents(ctx Context) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("timeoutAgents() -> %v", e)
		}
		ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving timeoutAgents()"}.Debug()
	}()
	timeOutPeriod, err := time.ParseDuration(ctx.Agent.TimeOut)
	if err != nil {
		panic(err)
	}
	pointInTime := time.Now().Add(-timeOutPeriod)
	err = ctx.DB.MarkOfflineAgents(pointInTime)
	if err != nil {
		panic(err)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package version

import (
	"fmt"

	inaming "v.io/x/ref/profiles/internal/naming"

	"v.io/v23/naming"
	"v.io/v23/rpc/version"
)

// Range represents a range of RPC versions.
type Range struct {
	Min, Max version.RPCVersion
}

var (
	// SupportedRange represents the range of protocol verions supported by this
	// implementation.
	// Max should be incremented whenever we make a protocol
	// change that's not both forward and backward compatible.
	// Min should be incremented whenever we want to remove
	// support for old protocol versions.
	SupportedRange = &Range{Min: version.RPCVersion5, Max: version.RPCVersion8}

	// Export the methods on supportedRange.
	Endpoint           = SupportedRange.Endpoint
	ProxiedEndpoint    = SupportedRange.ProxiedEndpoint
	CommonVersion      = SupportedRange.CommonVersion
	CheckCompatibility = SupportedRange.CheckCompatibility
)

var (
	NoCompatibleVersionErr = fmt.Errorf("No compatible RPC version available")
	UnknownVersionErr      = fmt.Errorf("There was not enough information to determine a version.")
)

// IsVersionError returns true if err is a versioning related error.
func IsVersionError(err error) bool {
	return err == NoCompatibleVersionErr || err == UnknownVersionErr
}

// Endpoint returns an endpoint with the Min/MaxRPCVersion properly filled in
// to match this implementations supported protocol versions.
func (r *Range) Endpoint(protocol, address string, rid naming.RoutingID) *inaming.Endpoint {
	return &inaming.Endpoint{
		Protocol:      protocol,
		Address:       address,
		RID:           rid,
		MinRPCVersion: r.Min,
		MaxRPCVersion: r.Max,
	}
}

// intersectRanges finds the intersection between ranges
// supported by two endpoints.  We make an assumption here that if one
// of the endpoints has an UnknownVersion we assume it has the same
// extent as the other endpoint. If both endpoints have Unknown for a
// version number, an error is produced.
// For example:
//   a == (2, 4) and b == (Unknown, Unknown), intersect(a,b) == (2, 4)
//   a == (2, Unknown) and b == (3, 4), intersect(a,b) == (3, 4)
func intersectRanges(amin, amax, bmin, bmax version.RPCVersion) (min, max version.RPCVersion, err error) {
	u := version.UnknownRPCVersion

	min = amin
	if min == u || (bmin != u && bmin > min) {
		min = bmin
	}
	max = amax
	if max == u || (bmax != u && bmax < max) {
		max = bmax
	}

	if min == u || max == u {
		err = UnknownVersionErr
	} else if min > max {
		err = NoCompatibleVersionErr
	}
	return
}

func intersectEndpoints(a, b *inaming.Endpoint) (min, max version.RPCVersion, err error) {
	return intersectRanges(a.MinRPCVersion, a.MaxRPCVersion, b.MinRPCVersion, b.MaxRPCVersion)
}

func (r1 *Range) Intersect(r2 *Range) (*Range, error) {
	min, max, err := intersectRanges(r1.Min, r1.Max, r2.Min, r2.Max)
	if err != nil {
		return nil, err
	}
	r := &Range{Min: min, Max: max}
	return r, nil
}

// ProxiedEndpoint returns an endpoint with the Min/MaxRPCVersion properly filled in
// to match the intersection of capabilities of this process and the proxy.
func (r *Range) ProxiedEndpoint(rid naming.RoutingID, proxy naming.Endpoint) (*inaming.Endpoint, error) {
	proxyEP, ok := proxy.(*inaming.Endpoint)
	if !ok {
		return nil, fmt.Errorf("unrecognized naming.Endpoint type %T", proxy)
	}

	ep := &inaming.Endpoint{
		Protocol:      proxyEP.Protocol,
		Address:       proxyEP.Address,
		RID:           rid,
		MinRPCVersion: r.Min,
		MaxRPCVersion: r.Max,
	}

	// This is the endpoint we are going to advertise.  It should only claim to support versions in
	// the intersection of those we support and those the proxy supports.
	var err error
	ep.MinRPCVersion, ep.MaxRPCVersion, err = intersectEndpoints(ep, proxyEP)
	if err != nil {
		return nil, fmt.Errorf("attempting to register with incompatible proxy: %s", proxy)
	}
	return ep, nil
}

// CommonVersion determines which version of the RPC protocol should be used
// between two endpoints.  Returns an error if the resulting version is incompatible
// with this RPC implementation.
func (r *Range) CommonVersion(a, b naming.Endpoint) (version.RPCVersion, error) {
	aEP, ok := a.(*inaming.Endpoint)
	if !ok {
		return 0, fmt.Errorf("Unrecognized naming.Endpoint type: %T", a)
	}
	bEP, ok := b.(*inaming.Endpoint)
	if !ok {
		return 0, fmt.Errorf("Unrecognized naming.Endpoint type: %T", b)
	}

	_, max, err := intersectEndpoints(aEP, bEP)
	if err != nil {
		return 0, err
	}

	// We want to use the maximum common version of the protocol.  We just
	// need to make sure that it is supported by this RPC implementation.
	if max < r.Min || max > r.Max {
		return version.UnknownRPCVersion, NoCompatibleVersionErr
	}
	return max, nil
}

// CheckCompatibility returns an error if the given endpoint is incompatible
// with this RPC implementation.  It returns nil otherwise.
func (r *Range) CheckCompatibility(remote naming.Endpoint) error {
	remoteEP, ok := remote.(*inaming.Endpoint)
	if !ok {
		return fmt.Errorf("Unrecognized naming.Endpoint type: %T", remote)
	}

	_, _, err := intersectRanges(r.Min, r.Max,
		remoteEP.MinRPCVersion, remoteEP.MaxRPCVersion)

	return err
}
<commit_msg>TBR rpc version: temporarily rollback supported version from 8 to 7.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package version

import (
	"fmt"

	inaming "v.io/x/ref/profiles/internal/naming"

	"v.io/v23/naming"
	"v.io/v23/rpc/version"
)

// Range represents a range of RPC versions.
type Range struct {
	Min, Max version.RPCVersion
}

var (
	// SupportedRange represents the range of protocol verions supported by this
	// implementation.
	// Max should be incremented whenever we make a protocol
	// change that's not both forward and backward compatible.
	// Min should be incremented whenever we want to remove
	// support for old protocol versions.
	SupportedRange = &Range{Min: version.RPCVersion5, Max: version.RPCVersion7}

	// Export the methods on supportedRange.
	Endpoint           = SupportedRange.Endpoint
	ProxiedEndpoint    = SupportedRange.ProxiedEndpoint
	CommonVersion      = SupportedRange.CommonVersion
	CheckCompatibility = SupportedRange.CheckCompatibility
)

var (
	NoCompatibleVersionErr = fmt.Errorf("No compatible RPC version available")
	UnknownVersionErr      = fmt.Errorf("There was not enough information to determine a version.")
)

// IsVersionError returns true if err is a versioning related error.
func IsVersionError(err error) bool {
	return err == NoCompatibleVersionErr || err == UnknownVersionErr
}

// Endpoint returns an endpoint with the Min/MaxRPCVersion properly filled in
// to match this implementations supported protocol versions.
func (r *Range) Endpoint(protocol, address string, rid naming.RoutingID) *inaming.Endpoint {
	return &inaming.Endpoint{
		Protocol:      protocol,
		Address:       address,
		RID:           rid,
		MinRPCVersion: r.Min,
		MaxRPCVersion: r.Max,
	}
}

// intersectRanges finds the intersection between ranges
// supported by two endpoints.  We make an assumption here that if one
// of the endpoints has an UnknownVersion we assume it has the same
// extent as the other endpoint. If both endpoints have Unknown for a
// version number, an error is produced.
// For example:
//   a == (2, 4) and b == (Unknown, Unknown), intersect(a,b) == (2, 4)
//   a == (2, Unknown) and b == (3, 4), intersect(a,b) == (3, 4)
func intersectRanges(amin, amax, bmin, bmax version.RPCVersion) (min, max version.RPCVersion, err error) {
	u := version.UnknownRPCVersion

	min = amin
	if min == u || (bmin != u && bmin > min) {
		min = bmin
	}
	max = amax
	if max == u || (bmax != u && bmax < max) {
		max = bmax
	}

	if min == u || max == u {
		err = UnknownVersionErr
	} else if min > max {
		err = NoCompatibleVersionErr
	}
	return
}

func intersectEndpoints(a, b *inaming.Endpoint) (min, max version.RPCVersion, err error) {
	return intersectRanges(a.MinRPCVersion, a.MaxRPCVersion, b.MinRPCVersion, b.MaxRPCVersion)
}

func (r1 *Range) Intersect(r2 *Range) (*Range, error) {
	min, max, err := intersectRanges(r1.Min, r1.Max, r2.Min, r2.Max)
	if err != nil {
		return nil, err
	}
	r := &Range{Min: min, Max: max}
	return r, nil
}

// ProxiedEndpoint returns an endpoint with the Min/MaxRPCVersion properly filled in
// to match the intersection of capabilities of this process and the proxy.
func (r *Range) ProxiedEndpoint(rid naming.RoutingID, proxy naming.Endpoint) (*inaming.Endpoint, error) {
	proxyEP, ok := proxy.(*inaming.Endpoint)
	if !ok {
		return nil, fmt.Errorf("unrecognized naming.Endpoint type %T", proxy)
	}

	ep := &inaming.Endpoint{
		Protocol:      proxyEP.Protocol,
		Address:       proxyEP.Address,
		RID:           rid,
		MinRPCVersion: r.Min,
		MaxRPCVersion: r.Max,
	}

	// This is the endpoint we are going to advertise.  It should only claim to support versions in
	// the intersection of those we support and those the proxy supports.
	var err error
	ep.MinRPCVersion, ep.MaxRPCVersion, err = intersectEndpoints(ep, proxyEP)
	if err != nil {
		return nil, fmt.Errorf("attempting to register with incompatible proxy: %s", proxy)
	}
	return ep, nil
}

// CommonVersion determines which version of the RPC protocol should be used
// between two endpoints.  Returns an error if the resulting version is incompatible
// with this RPC implementation.
func (r *Range) CommonVersion(a, b naming.Endpoint) (version.RPCVersion, error) {
	aEP, ok := a.(*inaming.Endpoint)
	if !ok {
		return 0, fmt.Errorf("Unrecognized naming.Endpoint type: %T", a)
	}
	bEP, ok := b.(*inaming.Endpoint)
	if !ok {
		return 0, fmt.Errorf("Unrecognized naming.Endpoint type: %T", b)
	}

	_, max, err := intersectEndpoints(aEP, bEP)
	if err != nil {
		return 0, err
	}

	// We want to use the maximum common version of the protocol.  We just
	// need to make sure that it is supported by this RPC implementation.
	if max < r.Min || max > r.Max {
		return version.UnknownRPCVersion, NoCompatibleVersionErr
	}
	return max, nil
}

// CheckCompatibility returns an error if the given endpoint is incompatible
// with this RPC implementation.  It returns nil otherwise.
func (r *Range) CheckCompatibility(remote naming.Endpoint) error {
	remoteEP, ok := remote.(*inaming.Endpoint)
	if !ok {
		return fmt.Errorf("Unrecognized naming.Endpoint type: %T", remote)
	}

	_, _, err := intersectRanges(r.Min, r.Max,
		remoteEP.MinRPCVersion, remoteEP.MaxRPCVersion)

	return err
}
<|endoftext|>


<commit_before>package raftgorums

import (
	"errors"

	"github.com/relab/raft/commonpb"
)

// Keys for indexing term and who was voted for.
const (
	KeyTerm uint64 = iota
	KeyVotedFor
	KeyFirstIndex
	KeyNextIndex
	KeySnapshot
)

// Storage provides an interface for storing and retrieving Raft state.
type Storage interface {
	Set(key uint64, value uint64) error
	Get(key uint64) (uint64, error)

	// Entries must be stored such that Entry.Index can be used to retrieve
	// that entry in the future.
	StoreEntries([]*commonpb.Entry) error
	// Retrieves entry with Entry.Index == index.
	GetEntry(index uint64) (*commonpb.Entry, error)
	// Get the inclusive range of entries from first to last.
	GetEntries(first, last uint64) ([]*commonpb.Entry, error)
	// Remove the inclusive range of entries from first to last.
	RemoveEntries(first, last uint64) error

	// Should return 1 if not set.
	FirstIndex() (uint64, error)
	// Should return 1 if not set.
	NextIndex() (uint64, error)

	SetSnapshot(*commonpb.Snapshot) error
	GetSnapshot() (*commonpb.Snapshot, error)
}

// TODO Create LogStore wrapper.

// Memory implements the Storage interface as an in-memory storage.
type Memory struct {
	kvstore map[uint64]uint64
	log     map[uint64]*commonpb.Entry
}

// NewMemory returns a memory backed storage.
func NewMemory(kvstore map[uint64]uint64, log map[uint64]*commonpb.Entry) *Memory {
	return &Memory{
		kvstore: kvstore,
		log:     log,
	}
}

// Set implements the Storage interface.
func (m *Memory) Set(key, value uint64) error {
	m.kvstore[key] = value
	return nil
}

// Get implements the Storage interface.
func (m *Memory) Get(key uint64) (uint64, error) {
	return m.kvstore[key], nil
}

// StoreEntries implements the Storage interface.
func (m *Memory) StoreEntries(entries []*commonpb.Entry) error {
	i, _ := m.NextIndex()
	for _, entry := range entries {
		m.log[i] = entry
		i++
	}
	return m.Set(KeyNextIndex, i)
}

// GetEntry implements the Storage interface.
func (m *Memory) GetEntry(index uint64) (*commonpb.Entry, error) {
	entry, ok := m.log[index]

	if !ok {
		return nil, ErrKeyNotFound
	}

	return entry, nil
}

// GetEntries implements the Storage interface.
func (m *Memory) GetEntries(first, last uint64) ([]*commonpb.Entry, error) {
	entries := make([]*commonpb.Entry, last-first+1)

	i := first
	for j := range entries {
		entries[j] = m.log[i]
		i++
	}

	return entries, nil
}

// RemoveEntries implements the Storage interface.
func (m *Memory) RemoveEntries(first, last uint64) error {
	for i := first; i <= last; i++ {
		delete(m.log, i)
	}

	return m.Set(KeyNextIndex, first)
}

// FirstIndex implements the Storage interface.
func (m *Memory) FirstIndex() (uint64, error) {
	first, _ := m.Get(KeyFirstIndex)
	return first, nil
}

// NextIndex implements the Storage interface.
func (m *Memory) NextIndex() (uint64, error) {
	next, _ := m.Get(KeyNextIndex)
	return next, nil
}

// SetSnapshot implements the Storage interface.
func (m *Memory) SetSnapshot(*commonpb.Snapshot) error {
	return nil
}

// GetSnapshot implements the Storage interface.
func (m *Memory) GetSnapshot() (*commonpb.Snapshot, error) {
	return nil, errors.New("not implemented")
}
<commit_msg>raftgorums/storage.go: Fix ignored errors<commit_after>package raftgorums

import (
	"errors"

	"github.com/relab/raft/commonpb"
)

// Keys for indexing term and who was voted for.
const (
	KeyTerm uint64 = iota
	KeyVotedFor
	KeyFirstIndex
	KeyNextIndex
	KeySnapshot
)

// Storage provides an interface for storing and retrieving Raft state.
type Storage interface {
	Set(key uint64, value uint64) error
	Get(key uint64) (uint64, error)

	// Entries must be stored such that Entry.Index can be used to retrieve
	// that entry in the future.
	StoreEntries([]*commonpb.Entry) error
	// Retrieves entry with Entry.Index == index.
	GetEntry(index uint64) (*commonpb.Entry, error)
	// Get the inclusive range of entries from first to last.
	GetEntries(first, last uint64) ([]*commonpb.Entry, error)
	// Remove the inclusive range of entries from first to last.
	RemoveEntries(first, last uint64) error

	// Should return 1 if not set.
	FirstIndex() (uint64, error)
	// Should return 1 if not set.
	NextIndex() (uint64, error)

	SetSnapshot(*commonpb.Snapshot) error
	GetSnapshot() (*commonpb.Snapshot, error)
}

// TODO Create LogStore wrapper.

// Memory implements the Storage interface as an in-memory storage.
type Memory struct {
	kvstore map[uint64]uint64
	log     map[uint64]*commonpb.Entry
}

// NewMemory returns a memory backed storage.
func NewMemory(kvstore map[uint64]uint64, log map[uint64]*commonpb.Entry) *Memory {
	return &Memory{
		kvstore: kvstore,
		log:     log,
	}
}

// Set implements the Storage interface.
func (m *Memory) Set(key, value uint64) error {
	m.kvstore[key] = value
	return nil
}

// Get implements the Storage interface.
func (m *Memory) Get(key uint64) (uint64, error) {
	return m.kvstore[key], nil
}

// StoreEntries implements the Storage interface.
func (m *Memory) StoreEntries(entries []*commonpb.Entry) error {
	i := m.kvstore[KeyNextIndex]
	for _, entry := range entries {
		m.log[i] = entry
		i++
	}
	return m.Set(KeyNextIndex, i)
}

// GetEntry implements the Storage interface.
func (m *Memory) GetEntry(index uint64) (*commonpb.Entry, error) {
	entry, ok := m.log[index]

	if !ok {
		return nil, ErrKeyNotFound
	}

	return entry, nil
}

// GetEntries implements the Storage interface.
func (m *Memory) GetEntries(first, last uint64) ([]*commonpb.Entry, error) {
	entries := make([]*commonpb.Entry, last-first+1)

	i := first
	for j := range entries {
		entries[j] = m.log[i]
		i++
	}

	return entries, nil
}

// RemoveEntries implements the Storage interface.
func (m *Memory) RemoveEntries(first, last uint64) error {
	for i := first; i <= last; i++ {
		delete(m.log, i)
	}

	return m.Set(KeyNextIndex, first)
}

// FirstIndex implements the Storage interface.
func (m *Memory) FirstIndex() (uint64, error) {
	first := m.kvstore[KeyFirstIndex]
	return first, nil
}

// NextIndex implements the Storage interface.
func (m *Memory) NextIndex() (uint64, error) {
	next := m.kvstore[KeyNextIndex]
	return next, nil
}

// SetSnapshot implements the Storage interface.
func (m *Memory) SetSnapshot(*commonpb.Snapshot) error {
	return nil
}

// GetSnapshot implements the Storage interface.
func (m *Memory) GetSnapshot() (*commonpb.Snapshot, error) {
	return nil, errors.New("not implemented")
}
<|endoftext|>


<commit_before>package http

import (
	"context"
	"encoding/json"
	"fmt"
	nethttp "net/http"
	"strings"

	idpctx "github.com/influxdata/platform/context"
	"github.com/prometheus/client_golang/prometheus"
)

// PlatformHandler is a collection of all the service handlers.
type PlatformHandler struct {
	BucketHandler        *BucketHandler
	UserHandler          *UserHandler
	OrgHandler           *OrgHandler
	AuthorizationHandler *AuthorizationHandler
	DashboardHandler     *DashboardHandler
	AssetHandler         *AssetHandler
	ChronografHandler    *ChronografHandler
	SourceHandler        *SourceHandler
	TaskHandler          *TaskHandler
	FluxLangHandler      *FluxLangHandler
}

func setCORSResponseHeaders(w nethttp.ResponseWriter, r *nethttp.Request) {
	if origin := r.Header.Get("Origin"); origin != "" {
		w.Header().Set("Access-Control-Allow-Origin", origin)
		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
		w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization")
	}
}

var platformLinks = map[string]interface{}{
	"sources": "/v2/sources",
	"flux": map[string]string{
		"self":        "/v2/flux",
		"ast":         "/v2/flux/ast",
		"suggestions": "/v2/flux/suggestions",
	},
}

func (h *PlatformHandler) serveLinks(w nethttp.ResponseWriter, r *nethttp.Request) {
	ctx := r.Context()
	if err := encodeResponse(ctx, w, nethttp.StatusOK, platformLinks); err != nil {
		EncodeError(ctx, err, w)
		return
	}
}

// ServeHTTP delegates a request to the appropriate subhandler.
func (h *PlatformHandler) ServeHTTP(w nethttp.ResponseWriter, r *nethttp.Request) {

	setCORSResponseHeaders(w, r)
	if r.Method == "OPTIONS" {
		return
	}

	// Server the chronograf assets for any basepath that does not start with addressable parts
	// of the platform API.
	if !strings.HasPrefix(r.URL.Path, "/v1") &&
		!strings.HasPrefix(r.URL.Path, "/v2") &&
		!strings.HasPrefix(r.URL.Path, "/chronograf/") {
		h.AssetHandler.ServeHTTP(w, r)
		return
	}

	// Serve the links base links for the API.
	if r.URL.Path == "/v2/" || r.URL.Path == "/v2" {
		h.serveLinks(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v2/flux") {
		h.FluxLangHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/chronograf/") {
		h.ChronografHandler.ServeHTTP(w, r)
		return
	}

	ctx := r.Context()
	var err error
	if ctx, err = extractAuthorization(ctx, r); err != nil {
		// TODO(desa): add back eventually when things have settled
		//nethttp.Error(w, err.Error(), nethttp.StatusBadRequest)
	}
	r = r.WithContext(ctx)

	if strings.HasPrefix(r.URL.Path, "/v1/buckets") {
		h.BucketHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/users") {
		h.UserHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/orgs") {
		h.OrgHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/authorizations") {
		h.AuthorizationHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/dashboards") {
		h.DashboardHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v2/sources") {
		h.SourceHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/tasks") {
		h.TaskHandler.ServeHTTP(w, r)
	}

	nethttp.NotFound(w, r)
}

// PrometheusCollectors satisfies the prom.PrometheusCollector interface.
func (h *PlatformHandler) PrometheusCollectors() []prometheus.Collector {
	// TODO: collect and return relevant metrics.
	return nil
}

func extractAuthorization(ctx context.Context, r *nethttp.Request) (context.Context, error) {
	t, err := ParseAuthHeaderToken(r)
	if err != nil {
		return ctx, err
	}
	return idpctx.SetToken(ctx, t), nil
}

func mustMarshalJSON(i interface{}) []byte {
	b, err := json.Marshal(i)
	if err != nil {
		panic(fmt.Sprintf("failed to marshal json: %v", err))
	}

	return b
}
<commit_msg>feat(http): add status feed to platform links<commit_after>package http

import (
	"context"
	"encoding/json"
	"fmt"
	nethttp "net/http"
	"strings"

	idpctx "github.com/influxdata/platform/context"
	"github.com/prometheus/client_golang/prometheus"
)

// PlatformHandler is a collection of all the service handlers.
type PlatformHandler struct {
	BucketHandler        *BucketHandler
	UserHandler          *UserHandler
	OrgHandler           *OrgHandler
	AuthorizationHandler *AuthorizationHandler
	DashboardHandler     *DashboardHandler
	AssetHandler         *AssetHandler
	ChronografHandler    *ChronografHandler
	SourceHandler        *SourceHandler
	TaskHandler          *TaskHandler
	FluxLangHandler      *FluxLangHandler
}

func setCORSResponseHeaders(w nethttp.ResponseWriter, r *nethttp.Request) {
	if origin := r.Header.Get("Origin"); origin != "" {
		w.Header().Set("Access-Control-Allow-Origin", origin)
		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
		w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization")
	}
}

var platformLinks = map[string]interface{}{
	"sources": "/v2/sources",
	"flux": map[string]string{
		"self":        "/v2/flux",
		"ast":         "/v2/flux/ast",
		"suggestions": "/v2/flux/suggestions",
	},
	"external": map[string]string{
		"statusFeed": "https://www.influxdata.com/feed/json",
	},
}

func (h *PlatformHandler) serveLinks(w nethttp.ResponseWriter, r *nethttp.Request) {
	ctx := r.Context()
	if err := encodeResponse(ctx, w, nethttp.StatusOK, platformLinks); err != nil {
		EncodeError(ctx, err, w)
		return
	}
}

// ServeHTTP delegates a request to the appropriate subhandler.
func (h *PlatformHandler) ServeHTTP(w nethttp.ResponseWriter, r *nethttp.Request) {

	setCORSResponseHeaders(w, r)
	if r.Method == "OPTIONS" {
		return
	}

	// Server the chronograf assets for any basepath that does not start with addressable parts
	// of the platform API.
	if !strings.HasPrefix(r.URL.Path, "/v1") &&
		!strings.HasPrefix(r.URL.Path, "/v2") &&
		!strings.HasPrefix(r.URL.Path, "/chronograf/") {
		h.AssetHandler.ServeHTTP(w, r)
		return
	}

	// Serve the links base links for the API.
	if r.URL.Path == "/v2/" || r.URL.Path == "/v2" {
		h.serveLinks(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v2/flux") {
		h.FluxLangHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/chronograf/") {
		h.ChronografHandler.ServeHTTP(w, r)
		return
	}

	ctx := r.Context()
	var err error
	if ctx, err = extractAuthorization(ctx, r); err != nil {
		// TODO(desa): add back eventually when things have settled
		//nethttp.Error(w, err.Error(), nethttp.StatusBadRequest)
	}
	r = r.WithContext(ctx)

	if strings.HasPrefix(r.URL.Path, "/v1/buckets") {
		h.BucketHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/users") {
		h.UserHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/orgs") {
		h.OrgHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/authorizations") {
		h.AuthorizationHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/dashboards") {
		h.DashboardHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v2/sources") {
		h.SourceHandler.ServeHTTP(w, r)
		return
	}

	if strings.HasPrefix(r.URL.Path, "/v1/tasks") {
		h.TaskHandler.ServeHTTP(w, r)
	}

	nethttp.NotFound(w, r)
}

// PrometheusCollectors satisfies the prom.PrometheusCollector interface.
func (h *PlatformHandler) PrometheusCollectors() []prometheus.Collector {
	// TODO: collect and return relevant metrics.
	return nil
}

func extractAuthorization(ctx context.Context, r *nethttp.Request) (context.Context, error) {
	t, err := ParseAuthHeaderToken(r)
	if err != nil {
		return ctx, err
	}
	return idpctx.SetToken(ctx, t), nil
}

func mustMarshalJSON(i interface{}) []byte {
	b, err := json.Marshal(i)
	if err != nil {
		panic(fmt.Sprintf("failed to marshal json: %v", err))
	}

	return b
}
<|endoftext|>


<commit_before>package jsonapie;

import(
    . ".."
);

func init() {
    // safety check to make sure RelationshipBehaviorFromFieldToField is a RelationshipBehavior and a RelationshipBehaviorIder
    var t RelationshipBehavior = &RelationshipBehaviorFromFieldToField{};
    _ = t;
    var t2 IderRelationshipBehavior = &RelationshipBehaviorFromFieldToField{};
    _ = t2;
}

type RelationshipBehaviorFromFieldToField struct {
    SrcFieldName string
    DstFieldName string
    FromFieldToId *RelationshipBehaviorFromFieldToId
}

func NewRelationshipBehaviorFromFieldToField(srcFieldName, dstFieldName string, required RelationshipRequirement) *RelationshipBehaviorFromFieldToField {
    return &RelationshipBehaviorFromFieldToField{
        SrcFieldName: srcFieldName,
        DstFieldName: dstFieldName,
        FromFieldToId: NewRelationshipBehaviorFromFieldToId(srcFieldName, required),
    }
}

func(l *RelationshipBehaviorFromFieldToField) IsSingle() (bool) { return false; }

func(l *RelationshipBehaviorFromFieldToField) LinkIder(s Session, srcR, dstR *ResourceManagerResource, src Ider) (dst []Ider) {
    a := s.GetData().API;
    ids := l.FromFieldToId.LinkId(s,srcR, dstR, src);
    //dstrmr := rmr.RM.GetResource(rmr.DstR);
    dst = []Ider{}
    for _, id := range ids {
        newdst, err := dstR.R.FindManyByField(s, RequestParams{}, l.DstFieldName, id);
        if(err != nil) {
            a.Logger.Printf("RelationshipBehaviorFromFieldToField got an error from FindManyByField for %s: %s", dstR.Name, err);
        }
        dst = append(dst, newdst...);
    }
    return dst;
}

func(l *RelationshipBehaviorFromFieldToField) VerifyLinks(s Session, ider Ider, linkages *OutputLinkage) error {
    panic("TODO");
    return l.FromFieldToId.VerifyLinks(s,ider,linkages);
}
func(l *RelationshipBehaviorFromFieldToField) PreSave(s Session, ider Ider, linkages *OutputLinkage) error {
    panic("TODO");
    return l.FromFieldToId.PreSave(s,ider,linkages);
}
func(l *RelationshipBehaviorFromFieldToField) PostSave(s Session, ider Ider, linkages *OutputLinkage) error {
    panic("TODO");
    return l.FromFieldToId.PostSave(s,ider,linkages);
}
<commit_msg>Small pseudocode change for RelationshipBehaviorFromFieldToField<commit_after>package jsonapie;

import(
    . ".."
    "errors"
);

func init() {
    // safety check to make sure RelationshipBehaviorFromFieldToField is a RelationshipBehavior and a RelationshipBehaviorIder
    var t RelationshipBehavior = &RelationshipBehaviorFromFieldToField{};
    _ = t;
    var t2 IderRelationshipBehavior = &RelationshipBehaviorFromFieldToField{};
    _ = t2;
}

type RelationshipBehaviorFromFieldToField struct {
    SrcFieldName string
    DstFieldName string
    Required RelationshipRequirement
    FromFieldToId *RelationshipBehaviorFromFieldToId
}

func NewRelationshipBehaviorFromFieldToField(srcFieldName, dstFieldName string, required RelationshipRequirement) *RelationshipBehaviorFromFieldToField {
    return &RelationshipBehaviorFromFieldToField{
        SrcFieldName: srcFieldName,
        DstFieldName: dstFieldName,
        Required: required,
        FromFieldToId: NewRelationshipBehaviorFromFieldToId(srcFieldName, required),
    }
}

func(l *RelationshipBehaviorFromFieldToField) IsSingle() (bool) { return false; }

func(l *RelationshipBehaviorFromFieldToField) LinkIder(s Session, srcR, dstR *ResourceManagerResource, src Ider) (dst []Ider) {
    a := s.GetData().API;
    ids := l.FromFieldToId.LinkId(s,srcR, dstR, src);
    //dstrmr := rmr.RM.GetResource(rmr.DstR);
    dst = []Ider{}
    for _, id := range ids {
        newdst, err := dstR.R.FindManyByField(s, RequestParams{}, l.DstFieldName, id);
        if(err != nil) {
            a.Logger.Printf("RelationshipBehaviorFromFieldToField got an error from FindManyByField for %s: %s", dstR.Name, err);
        }
        dst = append(dst, newdst...);
    }
    return dst;
}

func(l *RelationshipBehaviorFromFieldToField) VerifyLinks(s Session, ider Ider, linkages *OutputLinkage) error {
    isEmpty := linkages == nil || linkages.Links == nil || len(linkages.Links) == 0;
    if(isEmpty && l.Required == Required) {
        return errors.New("Linkage is empty but is required");
    }
    //return l.FromFieldToId.VerifyLinks(s,ider,linkages);
}
func(l *RelationshipBehaviorFromFieldToField) PreSave(s Session, ider Ider, linkages *OutputLinkage) error {
    return nil; // no PreSave as we need Ider to be flushed to DB before we can use its ID
}
func(l *RelationshipBehaviorFromFieldToField) PostSave(s Session, ider Ider, linkages *OutputLinkage) error {
    id := GetId(ider);
    a := s.GetData().API;
    resource := a.RM.GetResource(l.DstFieldName);
    //resource.B.FindMany(s, nil,
    // retrieve current list of links
    // calculate differences
    // for -- remove ones that shouldn't be there anymore
    // for -- add ones that should be there now
}
<|endoftext|>


<commit_before>package loadbalancer

import (
	"fmt"
	"time"
)

// State is the state of the load balancer with a string representation.
type State interface {
	fmt.Stringer

	// GetName returns the name of this load balancer
	GetName() string

	// HashListener returns the backendPort and true if the listener exists.
	HasListener(extPort uint32, protocol Protocol) (uint32, bool)

	// VisitListeners provides a mechanism for caller to iterate through all the listeners
	VisitListeners(v func(lbPort, instancePort uint32, protocol Protocol))
}

// Result is the result of an operation
type Result interface {
	fmt.Stringer
}

// TODO(chungers) -- Update the interface to support Vhosts for L7 routing.

// Driver is the generic driver for a signle L4 load balancer instance
type Driver interface {

	// Name is the name of the load balancer
	Name() string

	// State returns the current state of the load balancer
	State() (State, error)

	// PublishService publishes a service in the LB by adding a load balancing rule
	PublishService(ext Protocol, extPort uint32, backend Protocol, backendPort uint32) (Result, error)

	// UnpublishService dissociates the load balancer from the backend service at the given port.
	UnpublishService(extPort uint32) (Result, error)

	// ConfigureHealthCheck configures the health checks for instance removal and reconfiguration
	// The parameters healthy and unhealthy indicate the number of consecutive success or fail pings required to
	// mark a backend instance as healthy or unhealthy.   The ping occurs on the backendPort parameter and
	// at the interval specified.
	ConfigureHealthCheck(backendPort uint32, healthy, unhealthy int, interval, timeout time.Duration) (Result, error)

	// RegisterBackend registers instances identified by the IDs to the LB's backend pool
	RegisterBackend(id string, more ...string) (Result, error)

	// DeregisterBackend removes the specified instances from the backend pool
	DeregisterBackend(id string, more ...string) (Result, error)
}
<commit_msg>fix comment typo (#103)<commit_after>package loadbalancer

import (
	"fmt"
	"time"
)

// State is the state of the load balancer with a string representation.
type State interface {
	fmt.Stringer

	// GetName returns the name of this load balancer
	GetName() string

	// HashListener returns the backendPort and true if the listener exists.
	HasListener(extPort uint32, protocol Protocol) (uint32, bool)

	// VisitListeners provides a mechanism for caller to iterate through all the listeners
	VisitListeners(v func(lbPort, instancePort uint32, protocol Protocol))
}

// Result is the result of an operation
type Result interface {
	fmt.Stringer
}

// TODO(chungers) -- Update the interface to support Vhosts for L7 routing.

// Driver is the generic driver for a single L4 load balancer instance
type Driver interface {

	// Name is the name of the load balancer
	Name() string

	// State returns the current state of the load balancer
	State() (State, error)

	// PublishService publishes a service in the LB by adding a load balancing rule
	PublishService(ext Protocol, extPort uint32, backend Protocol, backendPort uint32) (Result, error)

	// UnpublishService dissociates the load balancer from the backend service at the given port.
	UnpublishService(extPort uint32) (Result, error)

	// ConfigureHealthCheck configures the health checks for instance removal and reconfiguration
	// The parameters healthy and unhealthy indicate the number of consecutive success or fail pings required to
	// mark a backend instance as healthy or unhealthy.   The ping occurs on the backendPort parameter and
	// at the interval specified.
	ConfigureHealthCheck(backendPort uint32, healthy, unhealthy int, interval, timeout time.Duration) (Result, error)

	// RegisterBackend registers instances identified by the IDs to the LB's backend pool
	RegisterBackend(id string, more ...string) (Result, error)

	// DeregisterBackend removes the specified instances from the backend pool
	DeregisterBackend(id string, more ...string) (Result, error)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/hpcloud/tail"
)

var Commands = []cli.Command{
	commandWatch,
	commandReview,
	commandList,
}

var commandWatch = cli.Command{
	Name:  "watch",
	Usage: "",
	Description: `
`,
	Action: doWatch,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "output,o",
			Usage: "",
		},
	},
}

var commandReview = cli.Command{
	Name:  "review",
	Usage: "",
	Description: `
`,
	Action: doReview,
}

var commandList = cli.Command{
	Name:  "list",
	Usage: "",
	Description: `
`,
	Action: doList,
}

func debug(v ...interface{}) {
	if os.Getenv("DEBUG") != "" {
		log.Println(v...)
	}
}

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func doWatch(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	tty := c.Args()[0]
	output := c.String("output")

	if output == "" {
		fp, err := ioutil.TempFile("/tmp", "informer")
		assert(err)
		defer fp.Close()
		output = fp.Name()
	}

	if !strings.HasPrefix(tty, "pts/") {
		fmt.Errorf("Unrecognized psuedo terminal [%s]", tty)
		os.Exit(2)
	}

	if _, err := os.Stat("/dev/" + tty); os.IsNotExist(err) {
		fmt.Errorf("Psuedo terminal [%s] currently does NOT exist.")
		os.Exit(2)
	}

	debug("DEBUG: Scanning for psuedo terminal ", tty)

	out, err := exec.Command("ps", "fauwwx").Output()
	assert(err)
	psreg := regexp.MustCompile(
		`\n(\S+)\s+(\d+)\s+\S+\s+\S+\s+\S+\s+\S+\s+\?\s+\S+\s+\S+\s+\S+\s+\S+[\|\\_ ]+\S*\bsshd\b.*\n\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+` + tty + `\s`,
	)

	if !psreg.Match(out) {
		fmt.Errorf("Unable to locate corresponding ssh session for [%s]", tty)
		os.Exit(2)
	}

	pid := string(psreg.FindSubmatch(out)[2])

	cmd := exec.Command("strace", "-e", "read", "-s16384", "-q", "-x", "-p", pid, "-o", output)
	cmd.Start()
	defer cmd.Process.Kill()

	tmp, err := tail.TailFile(output, tail.Config{Follow: true})
	assert(err)

	fds := make(map[int]string, 2)
	keys := make([]int, 2)

	tmpreg := regexp.MustCompile(`(read)\((\d+), "(.*)"`)
	for line := range tmp.Lines {
		if tmpreg.Match([]byte(line.Text)) {
			group := tmpreg.FindSubmatch([]byte(line.Text))

			key, err := strconv.Atoi(string(group[2]))
			assert(err)
			fds[key] = string(group[1])
			if len(fds) >= 2 {
				for i := range fds {
					keys = append(keys, i)
				}
				sort.Ints(keys)
				break
			}
		}
	}
	tmp.Kill(nil)

	out, err = exec.Command("clear").Output()
	assert(err)
	fmt.Print(string(out))

	t, err := tail.TailFile(output, tail.Config{Follow: true})
	assert(err)
	defer t.Kill(nil)

	outreg := regexp.MustCompile(
		fmt.Sprintf(`read\(%d, "(.*)"`, keys[len(keys)-1]),
	)

	asciireg := regexp.MustCompile(`\\x(..)`)
	for line := range t.Lines {
		if outreg.Match([]byte(line.Text)) {
			s := string(outreg.FindSubmatch([]byte(line.Text))[1])
			s = asciireg.ReplaceAllStringFunc(s, func(ss string) string {
				ascii, err := strconv.ParseInt(strings.Replace(ss, `\x`, "", -1), 16, 64)
				assert(err)
				return string(ascii)
			})
			s = strings.Replace(s, `\n`, string(0x0a), -1)
			s = strings.Replace(s, `\r`, string(0x0d), -1)

			fmt.Print(s)
		}
	}
}

func doReview(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	fp, err := os.Open(c.Args()[0])
	assert(err)

	fds := make(map[int]string, 2)
	keys := make([]int, 2)

	scanner := bufio.NewScanner(fp)

	tmpreg := regexp.MustCompile(`(read)\((\d+), "(.*)"`)
	for scanner.Scan() {
		text := []byte(scanner.Text())
		if tmpreg.Match(text) {
			group := tmpreg.FindSubmatch(text)

			key, err := strconv.Atoi(string(group[2]))
			assert(err)
			fds[key] = string(group[1])
			if len(fds) >= 2 {
				for i := range fds {
					keys = append(keys, i)
				}
				sort.Ints(keys)
				break
			}
		}
	}

	fp.Close()

	out, err := exec.Command("clear").Output()
	assert(err)
	fmt.Print(string(out))

	fp, err = os.Open(c.Args()[0])
	assert(err)
	defer fp.Close()

	outreg := regexp.MustCompile(
		fmt.Sprintf(`read\(%d, "(.*)"`, keys[len(keys)-1]),
	)
	asciireg := regexp.MustCompile(`\\x(..)`)

	scanner = bufio.NewScanner(fp)
	for scanner.Scan() {
		text := []byte(scanner.Text())
		if outreg.Match(text) {
			s := string(outreg.FindSubmatch(text)[1])
			s = asciireg.ReplaceAllStringFunc(s, func(ss string) string {
				ascii, err := strconv.ParseInt(strings.Replace(ss, `\x`, "", -1), 16, 64)
				assert(err)
				return string(ascii)
			})
			s = strings.Replace(s, `\n`, string(0x0a), -1)
			s = strings.Replace(s, `\r`, string(0x0d), -1)

			fmt.Print(s)
		}
	}

	fmt.Println()
	assert(scanner.Err())
}

func doList(c *cli.Context) {
	out, err := exec.Command("w", "-hs").Output()
	assert(err)

	fmt.Println(string(out))
}
<commit_msg>support multi-byte text<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/hpcloud/tail"
)

var Commands = []cli.Command{
	commandWatch,
	commandReview,
	commandList,
}

var commandWatch = cli.Command{
	Name:  "watch",
	Usage: "",
	Description: `
`,
	Action: doWatch,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "output,o",
			Usage: "",
		},
	},
}

var commandReview = cli.Command{
	Name:  "review",
	Usage: "",
	Description: `
`,
	Action: doReview,
}

var commandList = cli.Command{
	Name:  "list",
	Usage: "",
	Description: `
`,
	Action: doList,
}

func debug(v ...interface{}) {
	if os.Getenv("DEBUG") != "" {
		log.Println(v...)
	}
}

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func doWatch(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	tty := c.Args()[0]
	output := c.String("output")

	if output == "" {
		fp, err := ioutil.TempFile("/tmp", "informer")
		assert(err)
		defer fp.Close()
		output = fp.Name()
	}

	if !strings.HasPrefix(tty, "pts/") {
		fmt.Errorf("Unrecognized psuedo terminal [%s]", tty)
		os.Exit(2)
	}

	if _, err := os.Stat("/dev/" + tty); os.IsNotExist(err) {
		fmt.Errorf("Psuedo terminal [%s] currently does NOT exist.")
		os.Exit(2)
	}

	debug("DEBUG: Scanning for psuedo terminal ", tty)

	out, err := exec.Command("ps", "fauwwx").Output()
	assert(err)
	psreg := regexp.MustCompile(
		`\n(\S+)\s+(\d+)\s+\S+\s+\S+\s+\S+\s+\S+\s+\?\s+\S+\s+\S+\s+\S+\s+\S+[\|\\_ ]+\S*\bsshd\b.*\n\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+` + tty + `\s`,
	)

	if !psreg.Match(out) {
		fmt.Errorf("Unable to locate corresponding ssh session for [%s]", tty)
		os.Exit(2)
	}

	pid := string(psreg.FindSubmatch(out)[2])

	cmd := exec.Command("strace", "-e", "read", "-s16384", "-q", "-x", "-p", pid, "-o", output)
	cmd.Start()
	defer cmd.Process.Kill()

	tmp, err := tail.TailFile(output, tail.Config{Follow: true})
	assert(err)

	fds := make(map[int]string, 2)
	keys := make([]int, 2)

	tmpreg := regexp.MustCompile(`(read)\((\d+), "(.*)"`)
	for line := range tmp.Lines {
		if tmpreg.Match([]byte(line.Text)) {
			group := tmpreg.FindSubmatch([]byte(line.Text))

			key, err := strconv.Atoi(string(group[2]))
			assert(err)
			fds[key] = string(group[1])
			if len(fds) >= 2 {
				for i := range fds {
					keys = append(keys, i)
				}
				sort.Ints(keys)
				break
			}
		}
	}
	tmp.Kill(nil)

	out, err = exec.Command("clear").Output()
	assert(err)
	fmt.Print(string(out))

	t, err := tail.TailFile(output, tail.Config{Follow: true})
	assert(err)
	defer t.Kill(nil)

	outreg := regexp.MustCompile(
		fmt.Sprintf(`read\(%d, "(.*)"`, keys[len(keys)-1]),
	)

	for line := range t.Lines {
		if outreg.Match([]byte(line.Text)) {
			s := string(outreg.FindSubmatch([]byte(line.Text))[1])

			s = strings.Replace(s, `\x`, `%`, -1)
			s = strings.Replace(s, `\n`, `%0a`, -1)
			s = strings.Replace(s, `\r`, `%0d`, -1)

			s, err = url.QueryUnescape(s)
			assert(err)

			fmt.Print(s)
		}
	}
}

func doReview(c *cli.Context) {
	if len(c.Args()) < 1 {
		cli.ShowAppHelp(c)
		os.Exit(1)
	}

	fp, err := os.Open(c.Args()[0])
	assert(err)

	fds := make(map[int]string, 2)
	keys := make([]int, 2)

	scanner := bufio.NewScanner(fp)

	tmpreg := regexp.MustCompile(`(read)\((\d+), "(.*)"`)
	for scanner.Scan() {
		text := []byte(scanner.Text())
		if tmpreg.Match(text) {
			group := tmpreg.FindSubmatch(text)

			key, err := strconv.Atoi(string(group[2]))
			assert(err)
			fds[key] = string(group[1])
			if len(fds) >= 2 {
				for i := range fds {
					keys = append(keys, i)
				}
				sort.Ints(keys)
				break
			}
		}
	}

	fp.Close()

	out, err := exec.Command("clear").Output()
	assert(err)
	fmt.Print(string(out))

	fp, err = os.Open(c.Args()[0])
	assert(err)
	defer fp.Close()

	outreg := regexp.MustCompile(
		fmt.Sprintf(`read\(%d, "(.*)"`, keys[len(keys)-1]),
	)

	scanner = bufio.NewScanner(fp)
	for scanner.Scan() {
		text := []byte(scanner.Text())
		if outreg.Match(text) {
			s := string(outreg.FindSubmatch(text)[1])

			s = strings.Replace(s, `\x`, `%`, -1)
			s = strings.Replace(s, `\n`, `%0a`, -1)
			s = strings.Replace(s, `\r`, `%0d`, -1)

			s, err = url.QueryUnescape(s)
			assert(err)

			fmt.Print(s)
		}
	}

	fmt.Println()
	assert(scanner.Err())
}

func doList(c *cli.Context) {
	out, err := exec.Command("w", "-hs").Output()
	assert(err)

	fmt.Println(string(out))
}
<|endoftext|>


<commit_before>package memory

import (
	"strings"
	"sync"
	"time"

	"github.com/grafana/metrictank/stats"
	lru "github.com/hashicorp/golang-lru"
	"github.com/raintank/schema"
	log "github.com/sirupsen/logrus"
)

var (
	// metric idx.memory.find-cache.hit is a counter of findCache hits
	findCacheHit = stats.NewCounterRate32("idx.memory.find-cache.hit")
	// metric idx.memory.find-cache.miss is a counter of findCache misses
	findCacheMiss = stats.NewCounterRate32("idx.memory.find-cache.miss")
)

// FindCache is a caching layer for the in-memory index. The cache provides
// per org LRU caches of patterns and the resulting []*Nodes from searches
// on the index.  Users should call `InvalidateFor(orgId, path)` when new
// entries are added to the cache to invalidate any cached patterns that match
// the path. `invalidateQueueSize` sets the maximum number of invalidations for
// a specific orgId that can be running at any time. If this number is exceeded
// then the cache for that orgId will be immediately purged and disabled for
// `backoffTime`.  This mechanism protects the instance from excessive resource
// usage when a large number of new series are added at once.
type FindCache struct {
	sync.RWMutex
	cache               map[uint32]*lru.Cache
	size                int
	invalidateQueueSize int
	backoffTime         time.Duration
	newSeries           map[uint32]chan struct{}
	backoff             map[uint32]time.Time
}

func NewFindCache(size, invalidateQueueSize int, backoffTime time.Duration) *FindCache {
	fc := &FindCache{
		cache:               make(map[uint32]*lru.Cache),
		size:                size,
		invalidateQueueSize: invalidateQueueSize,
		backoffTime:         backoffTime,
		newSeries:           make(map[uint32]chan struct{}),
		backoff:             make(map[uint32]time.Time),
	}
	return fc
}

func (c *FindCache) Get(orgId uint32, pattern string) ([]*Node, bool) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		findCacheMiss.Inc()
		return nil, ok
	}
	nodes, ok := cache.Get(pattern)
	if !ok {
		findCacheMiss.Inc()
		return nil, ok
	}
	findCacheHit.Inc()
	return nodes.([]*Node), ok
}

func (c *FindCache) Add(orgId uint32, pattern string, nodes []*Node) {
	c.RLock()
	cache, ok := c.cache[orgId]
	t := c.backoff[orgId]
	c.RUnlock()
	var err error
	if !ok {
		// dont init the cache if we are in backoff mode.
		if time.Until(t) > 0 {
			return
		}
		cache, err = lru.New(c.size)
		if err != nil {
			log.Errorf("memory-idx: findCache failed to create lru. err=%s", err)
			return
		}
		c.Lock()
		c.cache[orgId] = cache
		c.newSeries[orgId] = make(chan struct{}, c.invalidateQueueSize)
		c.Unlock()
	}
	cache.Add(pattern, nodes)
}

// Purge clears the cache for the specified orgId
func (c *FindCache) Purge(orgId uint32) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		return
	}
	cache.Purge()
}

// PurgeAll clears the caches for all orgIds
func (c *FindCache) PurgeAll() {
	c.RLock()
	orgs := make([]uint32, len(c.cache))
	i := 0
	for k := range c.cache {
		orgs[i] = k
		i++
	}
	c.RUnlock()
	for _, org := range orgs {
		c.Purge(org)
	}
}

// InvalidateFor removes entries from the cache for 'orgId'
// that match the provided path. If lots of InvalidateFor calls
// are made at once and we end up with `invalidateQueueSize` concurrent
// goroutines processing the invalidations, we purge the cache and
// disable it for `backoffTime`. Future InvalidateFor calls made during
// the backoff time will then return immediately.
func (c *FindCache) InvalidateFor(orgId uint32, path string) {
	c.RLock()
	ch := c.newSeries[orgId]
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok || cache.Len() < 1 {
		return
	}

	select {
	case ch <- struct{}{}:
	default:
		c.Lock()
		c.backoff[orgId] = time.Now().Add(c.backoffTime)
		delete(c.cache, orgId)
		c.Unlock()
		for i := 0; i < len(ch); i++ {
			select {
			case <-ch:
			default:
			}
		}
		log.Infof("memory-idx: findCache invalidate-queue full. Disabling cache for %s. num-cached-entries=%d", c.backoffTime.String(), cache.Len())
		return
	}

	// convert our path to a tree so that we can call `find(tree, pattern)`
	// for each pattern in the cache.
	tree := treeFromPath(path)

	for _, k := range cache.Keys() {
		matches, err := find(tree, k.(string))
		if err != nil {
			log.Errorf("memory-idx: checking if new series matches expressions in findCache. series=%s expr=%s err=%s", path, k, err)
			continue
		}
		if len(matches) > 0 {
			cache.Remove(k)
		}
	}
	select {
	case <-ch:
	default:
	}
}

// PurgeFindCache purges the findCaches for all orgIds
func (m *UnpartitionedMemoryIdx) PurgeFindCache() {
	m.findCache.PurgeAll()
}

// PurgeFindCache purges the findCaches for all orgIds
// across all partitions
func (p *PartitionedMemoryIdx) PurgeFindCache() {
	for _, m := range p.Partition {
		m.findCache.PurgeAll()
	}
}

// treeFromPath creates a index tree from a series path.
// The tree will have a single leaf node and nodes for
// each branch.
func treeFromPath(path string) *Tree {
	tree := &Tree{
		Items: map[string]*Node{
			"": {
				Path:     "",
				Children: make([]string, 0),
				Defs:     make([]schema.MKey, 0),
			},
		},
	}
	pos := strings.Index(path, ".")
	prevPos := 0
	for {
		branch := path[:pos]
		// add as child of parent branch
		thisNode := branch[prevPos+1:]
		if prevPos == 0 {
			thisNode = branch[prevPos:]
		}
		tree.Items[path[:prevPos]].Children = []string{thisNode}

		// create this branch/leaf
		tree.Items[branch] = &Node{
			Path: branch,
		}
		if branch == path {
			tree.Items[branch].Defs = []schema.MKey{{}}
			break
		}
		prevPos = pos
		nextPos := strings.Index(path[pos+1:], ".")
		if nextPos < 0 {
			pos = len(path)
		} else {
			pos = pos + nextPos + 1
		}
	}

	return tree
}
<commit_msg>simplify and better comments<commit_after>package memory

import (
	"strings"
	"sync"
	"time"

	"github.com/grafana/metrictank/stats"
	lru "github.com/hashicorp/golang-lru"
	"github.com/raintank/schema"
	log "github.com/sirupsen/logrus"
)

var (
	// metric idx.memory.find-cache.hit is a counter of findCache hits
	findCacheHit = stats.NewCounterRate32("idx.memory.find-cache.hit")
	// metric idx.memory.find-cache.miss is a counter of findCache misses
	findCacheMiss = stats.NewCounterRate32("idx.memory.find-cache.miss")
)

// FindCache is a caching layer for the in-memory index. The cache provides
// per org LRU caches of patterns and the resulting []*Nodes from searches
// on the index.  Users should call `InvalidateFor(orgId, path)` when new
// entries are added to the cache to invalidate any cached patterns that match
// the path. `invalidateQueueSize` sets the maximum number of invalidations for
// a specific orgId that can be running at any time. If this number is exceeded
// then the cache for that orgId will be immediately purged and disabled for
// `backoffTime`.  This mechanism protects the instance from excessive resource
// usage when a large number of new series are added at once.
type FindCache struct {
	sync.RWMutex
	cache               map[uint32]*lru.Cache
	size                int
	invalidateQueueSize int
	backoffTime         time.Duration
	newSeries           map[uint32]chan struct{}
	backoff             map[uint32]time.Time
}

func NewFindCache(size, invalidateQueueSize int, backoffTime time.Duration) *FindCache {
	fc := &FindCache{
		cache:               make(map[uint32]*lru.Cache),
		size:                size,
		invalidateQueueSize: invalidateQueueSize,
		backoffTime:         backoffTime,
		newSeries:           make(map[uint32]chan struct{}),
		backoff:             make(map[uint32]time.Time),
	}
	return fc
}

func (c *FindCache) Get(orgId uint32, pattern string) ([]*Node, bool) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		findCacheMiss.Inc()
		return nil, ok
	}
	nodes, ok := cache.Get(pattern)
	if !ok {
		findCacheMiss.Inc()
		return nil, ok
	}
	findCacheHit.Inc()
	return nodes.([]*Node), ok
}

func (c *FindCache) Add(orgId uint32, pattern string, nodes []*Node) {
	c.RLock()
	cache, ok := c.cache[orgId]
	t := c.backoff[orgId]
	c.RUnlock()
	var err error
	if !ok {
		// dont init the cache if we are in backoff mode.
		if time.Until(t) > 0 {
			return
		}
		cache, err = lru.New(c.size)
		if err != nil {
			log.Errorf("memory-idx: findCache failed to create lru. err=%s", err)
			return
		}
		c.Lock()
		c.cache[orgId] = cache
		c.newSeries[orgId] = make(chan struct{}, c.invalidateQueueSize)
		c.Unlock()
	}
	cache.Add(pattern, nodes)
}

// Purge clears the cache for the specified orgId
func (c *FindCache) Purge(orgId uint32) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		return
	}
	cache.Purge()
}

// PurgeAll clears the caches for all orgIds
func (c *FindCache) PurgeAll() {
	c.RLock()
	orgs := make([]uint32, len(c.cache))
	i := 0
	for k := range c.cache {
		orgs[i] = k
		i++
	}
	c.RUnlock()
	for _, org := range orgs {
		c.Purge(org)
	}
}

// InvalidateFor removes entries from the cache for 'orgId'
// that match the provided path. If lots of InvalidateFor calls
// are made at once and we end up with `invalidateQueueSize` concurrent
// goroutines processing the invalidations, we purge the cache and
// disable it for `backoffTime`. Future InvalidateFor calls made during
// the backoff time will then return immediately.
func (c *FindCache) InvalidateFor(orgId uint32, path string) {
	c.RLock()
	ch := c.newSeries[orgId]
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok || cache.Len() < 1 {
		return
	}

	select {
	case ch <- struct{}{}:
	default:
		c.Lock()
		c.backoff[orgId] = time.Now().Add(c.backoffTime)
		delete(c.cache, orgId)
		c.Unlock()
		for i := 0; i < len(ch); i++ {
			select {
			case <-ch:
			default:
			}
		}
		log.Infof("memory-idx: findCache invalidate-queue full. Disabling cache for %s. num-cached-entries=%d", c.backoffTime.String(), cache.Len())
		return
	}

	// convert our path to a tree so that we can call `find(tree, pattern)`
	// for each pattern in the cache and purge it if it matches the path or a subtree of it.
	// we can't simply prune all cache keys that equal path or a subtree of it, because
	// what's cached are search patterns which may contain wildcards and other expressions
	tree := treeFromPath(path)

	for _, k := range cache.Keys() {
		matches, err := find(tree, k.(string))
		if err != nil {
			log.Errorf("memory-idx: checking if new series matches expressions in findCache. series=%s expr=%s err=%s", path, k, err)
			continue
		}
		if len(matches) > 0 {
			cache.Remove(k)
		}
	}
	select {
	case <-ch:
	default:
	}
}

// PurgeFindCache purges the findCaches for all orgIds
func (m *UnpartitionedMemoryIdx) PurgeFindCache() {
	m.findCache.PurgeAll()
}

// PurgeFindCache purges the findCaches for all orgIds
// across all partitions
func (p *PartitionedMemoryIdx) PurgeFindCache() {
	for _, m := range p.Partition {
		m.findCache.PurgeAll()
	}
}

// treeFromPath creates an index tree from a series path.
// The tree will have a single leaf node and nodes for
// each branch.
func treeFromPath(path string) *Tree {
	tree := Tree{
		Items: map[string]*Node{
			"": {},
		},
	}
	pos := strings.Index(path, ".")
	var parentBranch string
	prevPos := -1
	for {
		branch := path[:pos]
		thisNode := branch[prevPos+1:]

		tree.Items[parentBranch].Children = []string{thisNode}

		// create this branch/leaf
		tree.Items[branch] = &Node{
			Path: branch,
		}
		if branch == path {
			tree.Items[branch].Defs = []schema.MKey{{}}
			break
		}
		prevPos = pos
		nextPos := strings.Index(path[pos+1:], ".")
		if nextPos < 0 {
			pos = len(path)
		} else {
			pos = pos + nextPos + 1
		}
		parentBranch = branch
	}

	return &tree
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
	}

	for _, container := range containers {
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if infos, err := inspectContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while inspecting service %s for %s: %v`, user.Username, service, appName, err)
		} else if infos.State.Health != nil {
			logs := make([]string, 0)
			for _, log := range infos.State.Health.Log {
				logs = append(logs, log.Output)
			}

			log.Printf(`[%s] Healthcheck output for %s: %s`, user.Username, service, logs)
		}

		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while stopping service %s for %s: %v`, user.Username, service, appName, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, DeployTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			log.Printf(`[%s] Container %s for %s is healthy`, user.Username, appName, message.From)

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]deployedService, oldContainers []types.Container) {
	defer cancel()
	defer func() {
		backgroundMutex.Lock()
		defer backgroundMutex.Unlock()

		backgroundTasks[string(appName)] = false
	}()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
		log.Printf(`[%s] Succeeded to deploy %s`, user.Username, appName)
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)
	log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

	createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(service, user, string(appName)), getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
		return nil, err
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func composeFailed(w http.ResponseWriter, user *auth.User, appName []byte, err error) {
	errorHandler(w, err)
	if err != nil {
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
	} else {
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func composeHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	backgroundMutex.Lock()

	if value, ok := backgroundTasks[appNameStr]; ok && value {
		backgroundMutex.Unlock()
		composeFailed(w, user, appName, fmt.Errorf(`Application already in deployment`))
		return
	}

	backgroundTasks[appNameStr] = true
	backgroundMutex.Unlock()

	log.Printf(`[%s] Deploying %s`, user.Username, appName)

	oldContainers, err := listContainers(user, appNameStr)
	if err != nil {
		composeFailed(w, user, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		composeFailed(w, user, appName, fmt.Errorf(`Application not owned`))
		forbidden(w)
	}

	newServices := make(map[string]deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = *deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		cancel()
		composeFailed(w, user, appName, err)
	} else {
		jsonHttp.ResponseJSON(w, results{newServices})
	}
}
<commit_msg>Some pointers fix<commit_after>package docker

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/ViBiOh/dashboard/auth"
	"github.com/ViBiOh/dashboard/jsonHttp"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/network"
	"gopkg.in/yaml.v2"
)

const minMemory = 16777216
const maxMemory = 805306368
const defaultTag = `:latest`
const deploySuffix = `_deploy`
const networkMode = `traefik`
const linkSeparator = `:`

var imageTag = regexp.MustCompile(`^\S*?:\S+$`)

type dockerComposeService struct {
	Image       string
	Command     []string
	Environment map[string]string
	Labels      map[string]string
	Links       []string
	Ports       []string
	ReadOnly    bool  `yaml:"read_only"`
	CPUShares   int64 `yaml:"cpu_shares"`
	MemoryLimit int64 `yaml:"mem_limit"`
}

type dockerCompose struct {
	Version  string
	Services map[string]dockerComposeService
}

type deployedService struct {
	ID   string
	Name string
}

func getConfig(service *dockerComposeService, user *auth.User, appName string) *container.Config {
	environments := make([]string, len(service.Environment))
	for key, value := range service.Environment {
		environments = append(environments, key+`=`+value)
	}

	if service.Labels == nil {
		service.Labels = make(map[string]string)
	}

	service.Labels[ownerLabel] = user.Username
	service.Labels[appLabel] = appName

	config := container.Config{
		Image:  service.Image,
		Labels: service.Labels,
		Env:    environments,
	}

	if len(service.Command) != 0 {
		config.Cmd = service.Command
	}

	return &config
}

func getHostConfig(service *dockerComposeService) *container.HostConfig {
	hostConfig := container.HostConfig{
		LogConfig: container.LogConfig{Type: `json-file`, Config: map[string]string{
			`max-size`: `50m`,
		}},
		NetworkMode:   networkMode,
		RestartPolicy: container.RestartPolicy{Name: `on-failure`, MaximumRetryCount: 5},
		Resources: container.Resources{
			CPUShares: 128,
			Memory:    minMemory,
		},
		SecurityOpt: []string{`no-new-privileges`},
	}

	if service.ReadOnly {
		hostConfig.ReadonlyRootfs = service.ReadOnly
	}

	if service.CPUShares != 0 {
		hostConfig.Resources.CPUShares = service.CPUShares
	}

	if service.MemoryLimit != 0 {
		if service.MemoryLimit <= maxMemory {
			hostConfig.Resources.Memory = service.MemoryLimit
		} else {
			hostConfig.Resources.Memory = maxMemory
		}
	}

	return &hostConfig
}

func getNetworkConfig(service *dockerComposeService, deployedServices map[string]*deployedService) *network.NetworkingConfig {
	traefikConfig := network.EndpointSettings{}

	for _, link := range service.Links {
		linkParts := strings.Split(link, linkSeparator)

		target := linkParts[0]
		if linkedService, ok := (deployedServices)[target]; ok {
			target = getFinalName(linkedService.Name)
		}

		alias := linkParts[0]
		if len(linkParts) > 1 {
			alias = linkParts[1]
		}

		traefikConfig.Links = append(traefikConfig.Links, target+linkSeparator+alias)
	}

	return &network.NetworkingConfig{
		EndpointsConfig: map[string]*network.EndpointSettings{
			networkMode: &traefikConfig,
		},
	}
}

func pullImage(image string, user *auth.User) error {
	if !imageTag.MatchString(image) {
		image = image + defaultTag
	}

	log.Printf(`[%s] Starting pull of image %s`, user.Username, image)
	pull, err := docker.ImagePull(context.Background(), image, types.ImagePullOptions{})
	if err != nil {
		return fmt.Errorf(`[%s] Error while pulling image: %v`, user.Username, err)
	}

	readBody(pull)
	log.Printf(`[%s] Ending pull of image %s`, user.Username, image)
	return nil
}

func cleanContainers(containers []types.Container, user *auth.User) {
	for _, container := range containers {
		log.Printf(`[%s] Stopping containers %s`, user.Username, strings.Join(container.Names, `, `))
		stopContainer(container.ID)
	}

	for _, container := range containers {
		log.Printf(`[%s] Deleting containers %s`, user.Username, strings.Join(container.Names, `, `))
		rmContainer(container.ID)
	}
}

func renameDeployedContainers(containers map[string]*deployedService, user *auth.User) error {
	for service, container := range containers {
		if err := docker.ContainerRename(context.Background(), container.ID, getFinalName(container.Name)); err != nil {
			return fmt.Errorf(`[%s] Error while renaming container %s: %v`, user.Username, service, err)
		}
	}

	return nil
}

func getServiceFullName(app string, service string) string {
	return app + `_` + service + deploySuffix
}

func getFinalName(serviceFullName string) string {
	return strings.TrimSuffix(serviceFullName, deploySuffix)
}

func deleteServices(appName []byte, services map[string]*deployedService, user *auth.User) {
	log.Printf(`[%s] Deleting services for %s`, user.Username, appName)
	for service, container := range services {
		if infos, err := inspectContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while inspecting service %s for %s: %v`, user.Username, service, appName, err)
		} else if infos.State.Health != nil {
			logs := make([]string, 0)
			for _, log := range infos.State.Health.Log {
				logs = append(logs, log.Output)
			}

			log.Printf(`[%s] Healthcheck output for %s: %s`, user.Username, service, logs)
		}

		if err := stopContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while stopping service %s for %s: %v`, user.Username, service, appName, err)
		}

		if err := rmContainer(container.ID); err != nil {
			log.Printf(`[%s] Error while deleting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}
}

func startServices(appName []byte, services map[string]*deployedService, user *auth.User) error {
	log.Printf(`[%s] Starting services for %s`, user.Username, appName)
	for service, container := range services {
		if err := startContainer(container.ID); err != nil {
			return fmt.Errorf(`[%s] Error while starting service %s for %s: %v`, user.Username, service, appName, err)
		}
	}

	return nil
}

func inspectServices(services map[string]*deployedService, user *auth.User) []*types.ContainerJSON {
	containers := make([]*types.ContainerJSON, 0, len(services))

	for service, container := range services {
		infos, err := inspectContainer(container.ID)
		if err != nil {
			log.Printf(`[%s] Error while inspecting container %s: %v`, user.Username, service, err)
		}

		containers = append(containers, &infos)
	}

	return containers
}

func areContainersHealthy(ctx context.Context, user *auth.User, appName []byte, containers []*types.ContainerJSON) bool {
	containersIdsWithHealthcheck := make([]string, 0, len(containers))
	for _, container := range containers {
		if container.Config.Healthcheck != nil && len(container.Config.Healthcheck.Test) != 0 {
			containersIdsWithHealthcheck = append(containersIdsWithHealthcheck, container.ID)
		}
	}

	if len(containersIdsWithHealthcheck) == 0 {
		return true
	}

	filtersArgs := filters.NewArgs()
	if err := healthyStatusFilters(user, &filtersArgs, containersIdsWithHealthcheck); err != nil {
		log.Printf(`[%s] Error while defining healthy filters: %v`, user.Username, err)
		return true
	}

	timeoutCtx, cancel := context.WithTimeout(ctx, DeployTimeout)
	defer cancel()

	messages, errors := docker.Events(timeoutCtx, types.EventsOptions{Filters: filtersArgs})
	healthyContainers := make(map[string]bool, len(containersIdsWithHealthcheck))

	for {
		select {
		case <-ctx.Done():
			return false
		case message := <-messages:
			healthyContainers[message.ID] = true
			log.Printf(`[%s] Container %s for %s is healthy`, user.Username, appName, message.From)

			if len(healthyContainers) == len(containersIdsWithHealthcheck) {
				return true
			}
		case err := <-errors:
			log.Printf(`[%s] Error while reading healthy events: %v`, user.Username, err)
			return false
		}
	}
}

func finishDeploy(ctx context.Context, cancel context.CancelFunc, user *auth.User, appName []byte, services map[string]*deployedService, oldContainers []types.Container) {
	defer cancel()
	defer func() {
		backgroundMutex.Lock()
		defer backgroundMutex.Unlock()

		backgroundTasks[string(appName)] = false
	}()

	log.Printf(`[%s] Waiting for %s to start...`, user.Username, appName)

	if areContainersHealthy(ctx, user, appName, inspectServices(services, user)) {
		log.Printf(`[%s] Health check succeeded for %s`, user.Username, appName)
		cleanContainers(oldContainers, user)

		if err := renameDeployedContainers(services, user); err != nil {
			log.Print(err)
		}
		log.Printf(`[%s] Succeeded to deploy %s`, user.Username, appName)
	} else {
		log.Printf(`[%s] Health check failed for %s`, user.Username, appName)
		deleteServices(appName, services, user)
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func createContainer(user *auth.User, appName []byte, serviceName string, services map[string]*deployedService, service *dockerComposeService) (*deployedService, error) {
	if err := pullImage(service.Image, user); err != nil {
		return nil, err
	}

	serviceFullName := getServiceFullName(string(appName), serviceName)
	log.Printf(`[%s] Creating service %s for %s`, user.Username, serviceName, appName)

	createdContainer, err := docker.ContainerCreate(context.Background(), getConfig(service, user, string(appName)), getHostConfig(service), getNetworkConfig(service, services), serviceFullName)
	if err != nil {
		err = fmt.Errorf(`[%s] Error while creating service %s for %s: %v`, user.Username, serviceName, appName, err)
		return nil, err
	}

	return &deployedService{ID: createdContainer.ID, Name: serviceFullName}, nil
}

func composeFailed(w http.ResponseWriter, user *auth.User, appName []byte, err error) {
	errorHandler(w, err)
	if err != nil {
		log.Printf(`[%s] Failed to deploy %s: %v`, user.Username, appName, err)
	} else {
		log.Printf(`[%s] Failed to deploy %s`, user.Username, appName)
	}
}

func composeHandler(w http.ResponseWriter, user *auth.User, appName []byte, composeFile []byte) {
	if len(appName) == 0 || len(composeFile) == 0 {
		badRequest(w, fmt.Errorf(`[%s] An application name and a compose file are required`, user.Username))
		return
	}

	compose := dockerCompose{}
	if err := yaml.Unmarshal(composeFile, &compose); err != nil {
		errorHandler(w, fmt.Errorf(`[%s] Error while unmarshalling compose file: %v`, user.Username, err))
		return
	}

	appNameStr := string(appName)
	backgroundMutex.Lock()

	if value, ok := backgroundTasks[appNameStr]; ok && value {
		backgroundMutex.Unlock()
		composeFailed(w, user, appName, fmt.Errorf(`Application already in deployment`))
		return
	}

	backgroundTasks[appNameStr] = true
	backgroundMutex.Unlock()

	log.Printf(`[%s] Deploying %s`, user.Username, appName)

	oldContainers, err := listContainers(user, appNameStr)
	if err != nil {
		composeFailed(w, user, appName, err)
		return
	}

	if len(oldContainers) > 0 && oldContainers[0].Labels[ownerLabel] != user.Username {
		composeFailed(w, user, appName, fmt.Errorf(`Application not owned`))
		forbidden(w)
	}

	newServices := make(map[string]*deployedService)
	for serviceName, service := range compose.Services {
		if deployedService, err := createContainer(user, appName, serviceName, newServices, &service); err != nil {
			break
		} else {
			newServices[serviceName] = deployedService
		}
	}

	ctx, cancel := context.WithCancel(context.Background())
	go finishDeploy(ctx, cancel, user, appName, newServices, oldContainers)

	if err == nil {
		err = startServices(appName, newServices, user)
	}

	if err != nil {
		cancel()
		composeFailed(w, user, appName, err)
	} else {
		jsonHttp.ResponseJSON(w, results{newServices})
	}
}
<|endoftext|>