<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/copystructure"
)

// newShadowContext creates a new context that will shadow the given context
// when walking the graph. The resulting context should be used _only once_
// for a graph walk.
//
// The returned Shadow should be closed after the graph walk with the
// real context is complete. Errors from the shadow can be retrieved there.
//
// Most importantly, any operations done on the shadow context (the returned
// context) will NEVER affect the real context. All structures are deep
// copied, no real providers or resources are used, etc.
func newShadowContext(c *Context) (*Context, *Context, Shadow) {
	// Copy the targets
	targetRaw, err := copystructure.Copy(c.targets)
	if err != nil {
		panic(err)
	}

	// Copy the variables
	varRaw, err := copystructure.Copy(c.variables)
	if err != nil {
		panic(err)
	}

	// Copy the provider inputs
	providerInputRaw, err := copystructure.Copy(c.providerInputConfig)
	if err != nil {
		panic(err)
	}

	// The factories
	componentsReal, componentsShadow := newShadowComponentFactory(c.components)

	// Create the shadow
	shadow := &Context{
		components: componentsShadow,
		destroy:    c.destroy,
		diff:       c.diff.DeepCopy(),
		hooks:      nil,
		module:     c.module,
		state:      c.state.DeepCopy(),
		targets:    targetRaw.([]string),
		variables:  varRaw.(map[string]interface{}),

		// NOTE(mitchellh): This is not going to work for shadows that are
		// testing that input results in the proper end state. At the time
		// of writing, input is not used in any state-changing graph
		// walks anyways, so this checks nothing. We set it to this to avoid
		// any panics but even a "nil" value worked here.
		uiInput: new(MockUIInput),

		// Hardcoded to 4 since parallelism in the shadow doesn't matter
		// a ton since we're doing far less compared to the real side
		// and our operations are MUCH faster.
		parallelSem:         NewSemaphore(4),
		providerInputConfig: providerInputRaw.(map[string]map[string]interface{}),
	}

	// Create the real context. This is effectively just a copy of
	// the context given except we need to modify some of the values
	// to point to the real side of a shadow so the shadow can compare values.
	real := &Context{
		// The fields below are changed.
		components: componentsReal,

		// The fields below are direct copies
		destroy: c.destroy,
		diff:    c.diff,
		// diffLock - no copy
		hooks:  c.hooks,
		module: c.module,
		sh:     c.sh,
		state:  c.state,
		// stateLock - no copy
		targets:   c.targets,
		uiInput:   c.uiInput,
		variables: c.variables,

		// l - no copy
		parallelSem:         c.parallelSem,
		providerInputConfig: c.providerInputConfig,
		runCh:               c.runCh,
		shadowErr:           c.shadowErr,
	}

	return real, shadow, &shadowContextCloser{
		Components: componentsShadow,
	}
}

// shadowContextVerify takes the real and shadow context and verifies they
// have equal diffs and states.
func shadowContextVerify(real, shadow *Context) error {
	var result error

	// Compare the states
	if !real.state.Equal(shadow.state) {
		result = multierror.Append(result, fmt.Errorf(
			"Real and shadow states do not match! "+
				"Real state:\n\n%s\n\n"+
				"Shadow state:\n\n%s\n\n",
			real.state, shadow.state))
	}

	// Compare the diffs
	if !real.diff.Equal(shadow.diff) {
		result = multierror.Append(result, fmt.Errorf(
			"Real and shadow diffs do not match! "+
				"Real diff:\n\n%s\n\n"+
				"Shadow diff:\n\n%s\n\n",
			real.diff, shadow.diff))
	}

	return result
}

// shadowContextCloser is the io.Closer returned by newShadowContext that
// closes all the shadows and returns the results.
type shadowContextCloser struct {
	Components *shadowComponentFactory
}

// Close closes the shadow context.
func (c *shadowContextCloser) CloseShadow() error {
	return c.Components.CloseShadow()
}

func (c *shadowContextCloser) ShadowError() error {
	return c.Components.ShadowError()
}
<commit_msg>terraform: compared states from shadow graph must be pruned<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/copystructure"
)

// newShadowContext creates a new context that will shadow the given context
// when walking the graph. The resulting context should be used _only once_
// for a graph walk.
//
// The returned Shadow should be closed after the graph walk with the
// real context is complete. Errors from the shadow can be retrieved there.
//
// Most importantly, any operations done on the shadow context (the returned
// context) will NEVER affect the real context. All structures are deep
// copied, no real providers or resources are used, etc.
func newShadowContext(c *Context) (*Context, *Context, Shadow) {
	// Copy the targets
	targetRaw, err := copystructure.Copy(c.targets)
	if err != nil {
		panic(err)
	}

	// Copy the variables
	varRaw, err := copystructure.Copy(c.variables)
	if err != nil {
		panic(err)
	}

	// Copy the provider inputs
	providerInputRaw, err := copystructure.Copy(c.providerInputConfig)
	if err != nil {
		panic(err)
	}

	// The factories
	componentsReal, componentsShadow := newShadowComponentFactory(c.components)

	// Create the shadow
	shadow := &Context{
		components: componentsShadow,
		destroy:    c.destroy,
		diff:       c.diff.DeepCopy(),
		hooks:      nil,
		module:     c.module,
		state:      c.state.DeepCopy(),
		targets:    targetRaw.([]string),
		variables:  varRaw.(map[string]interface{}),

		// NOTE(mitchellh): This is not going to work for shadows that are
		// testing that input results in the proper end state. At the time
		// of writing, input is not used in any state-changing graph
		// walks anyways, so this checks nothing. We set it to this to avoid
		// any panics but even a "nil" value worked here.
		uiInput: new(MockUIInput),

		// Hardcoded to 4 since parallelism in the shadow doesn't matter
		// a ton since we're doing far less compared to the real side
		// and our operations are MUCH faster.
		parallelSem:         NewSemaphore(4),
		providerInputConfig: providerInputRaw.(map[string]map[string]interface{}),
	}

	// Create the real context. This is effectively just a copy of
	// the context given except we need to modify some of the values
	// to point to the real side of a shadow so the shadow can compare values.
	real := &Context{
		// The fields below are changed.
		components: componentsReal,

		// The fields below are direct copies
		destroy: c.destroy,
		diff:    c.diff,
		// diffLock - no copy
		hooks:  c.hooks,
		module: c.module,
		sh:     c.sh,
		state:  c.state,
		// stateLock - no copy
		targets:   c.targets,
		uiInput:   c.uiInput,
		variables: c.variables,

		// l - no copy
		parallelSem:         c.parallelSem,
		providerInputConfig: c.providerInputConfig,
		runCh:               c.runCh,
		shadowErr:           c.shadowErr,
	}

	return real, shadow, &shadowContextCloser{
		Components: componentsShadow,
	}
}

// shadowContextVerify takes the real and shadow context and verifies they
// have equal diffs and states.
func shadowContextVerify(real, shadow *Context) error {
	var result error

	// The states compared must be pruned so they're minimal/clean
	real.state.prune()
	shadow.state.prune()

	// Compare the states
	if !real.state.Equal(shadow.state) {
		result = multierror.Append(result, fmt.Errorf(
			"Real and shadow states do not match! "+
				"Real state:\n\n%s\n\n"+
				"Shadow state:\n\n%s\n\n",
			real.state, shadow.state))
	}

	// Compare the diffs
	if !real.diff.Equal(shadow.diff) {
		result = multierror.Append(result, fmt.Errorf(
			"Real and shadow diffs do not match! "+
				"Real diff:\n\n%s\n\n"+
				"Shadow diff:\n\n%s\n\n",
			real.diff, shadow.diff))
	}

	return result
}

// shadowContextCloser is the io.Closer returned by newShadowContext that
// closes all the shadows and returns the results.
type shadowContextCloser struct {
	Components *shadowComponentFactory
}

// Close closes the shadow context.
func (c *shadowContextCloser) CloseShadow() error {
	return c.Components.CloseShadow()
}

func (c *shadowContextCloser) ShadowError() error {
	return c.Components.ShadowError()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/operationtype"
	"github.com/lxc/lxd/lxd/db/warningtype"
	"github.com/lxc/lxd/lxd/filter"
	"github.com/lxc/lxd/lxd/lifecycle"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/request"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
)

var warningsCmd = APIEndpoint{
	Path: "warnings",

	Get: APIEndpointAction{Handler: warningsGet},
}

var warningCmd = APIEndpoint{
	Path: "warnings/{id}",

	Get:    APIEndpointAction{Handler: warningGet},
	Patch:  APIEndpointAction{Handler: warningPatch},
	Put:    APIEndpointAction{Handler: warningPut},
	Delete: APIEndpointAction{Handler: warningDelete},
}

func filterWarnings(warnings []api.Warning, clauses []filter.Clause) []api.Warning {
	filtered := []api.Warning{}

	for _, warning := range warnings {
		if !filter.Match(warning, clauses) {
			continue
		}

		filtered = append(filtered, warning)
	}

	return filtered
}

// swagger:operation GET /1.0/warnings warnings warnings_get
//
// List the warnings
//
// Returns a list of warnings.
//
// ---
// produces:
//   - application/json
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: Sync response
//     schema:
//       type: object
//       description: Sync response
//       properties:
//         type:
//           type: string
//           description: Response type
//           example: sync
//         status:
//           type: string
//           description: Status description
//           example: Success
//         status_code:
//           type: integer
//           description: Status code
//           example: 200
//         metadata:
//           type: array
//           description: List of endpoints
//           items:
//             type: string
//           example: |-
//             [
//               "/1.0/warnings/39c61a48-cc17-40ae-8248-4f7b4cadedf4",
//               "/1.0/warnings/951779a5-2820-4d96-b01e-88fe820e5310"
//             ]
//   "500":
//     $ref: "#/responses/InternalServerError"

// swagger:operation GET /1.0/warnings?recursion=1 warnings warnings_get_recursion1
//
// Get the warnings
//
// Returns a list of warnings (structs).
//
// ---
// produces:
//   - application/json
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: API endpoints
//     schema:
//       type: object
//       description: Sync response
//       properties:
//         type:
//           type: string
//           description: Response type
//           example: sync
//         status:
//           type: string
//           description: Status description
//           example: Success
//         status_code:
//           type: integer
//           description: Status code
//           example: 200
//         metadata:
//           type: array
//           description: List of warnings
//           items:
//             $ref: "#/definitions/Warning"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningsGet(d *Daemon, r *http.Request) response.Response {
	// Parse the recursion field
	recursionStr := r.FormValue("recursion")

	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	// Parse filter value
	var clauses []filter.Clause

	filterStr := r.FormValue("filter")
	if filterStr != "" {
		clauses, err = filter.Parse(filterStr)
		if err != nil {
			return response.SmartError(fmt.Errorf("Failed to filter warnings: %w", err))
		}
	}

	// Parse the project field
	projectName := queryParam(r, "project")

	var dbWarnings []db.Warning

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		filter := db.WarningFilter{}

		if projectName != "" {
			filter.Project = &projectName
		}

		dbWarnings, err = tx.GetWarnings(filter)
		if err != nil {
			return err
		}

		if err != nil {
			return fmt.Errorf("Failed to get warnings: %w", err)
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	warnings := make([]api.Warning, len(dbWarnings))

	for i, w := range dbWarnings {
		warning, err := w.ToAPI(d.db.Cluster)
		if err != nil {
			return response.SmartError(err)
		}

		warnings[i] = warning
	}

	if recursion == 0 {
		var resultList []string

		for _, w := range filterWarnings(warnings, clauses) {
			url := fmt.Sprintf("/%s/warnings/%s", version.APIVersion, w.UUID)
			resultList = append(resultList, url)
		}

		return response.SyncResponse(true, resultList)
	}

	// Return detailed list of warning
	return response.SyncResponse(true, filterWarnings(warnings, clauses))
}

// swagger:operation GET /1.0/warnings/{uuid} warnings warning_get
//
// Get the warning
//
// Gets a specific warning.
//
// ---
// produces:
//   - application/json
// responses:
//   "200":
//     description: Warning
//     schema:
//       type: object
//       description: Sync response
//       properties:
//         type:
//           type: string
//           description: Response type
//           example: sync
//         status:
//           type: string
//           description: Status description
//           example: Success
//         status_code:
//           type: integer
//           description: Status code
//           example: 200
//         metadata:
//           $ref: "#/definitions/Warning"
//   "404":
//     $ref: "#/responses/NotFound"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningGet(d *Daemon, r *http.Request) response.Response {
	id, err := url.PathUnescape(mux.Vars(r)["id"])
	if err != nil {
		return response.SmartError(err)
	}

	var dbWarning *db.Warning

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		dbWarning, err = tx.GetWarning(id)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	resp, err := dbWarning.ToAPI(d.db.Cluster)
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponse(true, resp)
}

// swagger:operation PATCH /1.0/warnings/{uuid} warnings warning_patch
//
// Partially update the warning
//
// Updates a subset of the warning status.
//
// ---
// consumes:
//   - application/json
// produces:
//   - application/json
// parameters:
//   - in: body
//     name: warning
//     description: Warning status
//     required: true
//     schema:
//       $ref: "#/definitions/WarningPut"
// responses:
//   "200":
//     $ref: "#/responses/EmptySyncResponse"
//   "400":
//     $ref: "#/responses/BadRequest"
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningPatch(d *Daemon, r *http.Request) response.Response {
	return warningPut(d, r)
}

// swagger:operation PUT /1.0/warnings/{uuid} warnings warning_put
//
// Update the warning
//
// Updates the warning status.
//
// ---
// consumes:
//   - application/json
// produces:
//   - application/json
// parameters:
//   - in: body
//     name: warning
//     description: Warning status
//     required: true
//     schema:
//       $ref: "#/definitions/WarningPut"
// responses:
//   "200":
//     $ref: "#/responses/EmptySyncResponse"
//   "400":
//     $ref: "#/responses/BadRequest"
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningPut(d *Daemon, r *http.Request) response.Response {
	id, err := url.PathUnescape(mux.Vars(r)["id"])
	if err != nil {
		return response.SmartError(err)
	}

	req := api.WarningPut{}

	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return response.BadRequest(err)
	}

	// Currently, we only allow changing the status to acknowledged or new.
	status, ok := warningtype.StatusTypes[req.Status]
	if !ok {
		// Invalid status
		return response.BadRequest(fmt.Errorf("Invalid warning type %q", req.Status))
	}

	if status != warningtype.StatusAcknowledged && status != warningtype.StatusNew {
		return response.Forbidden(fmt.Errorf(`Status may only be set to "acknowledge" or "new"`))
	}

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		err := tx.UpdateWarningStatus(id, status)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	if status == warningtype.StatusAcknowledged {
		d.State().Events.SendLifecycle(project.Default, lifecycle.WarningAcknowledged.Event(id, request.CreateRequestor(r), nil))
	} else {
		d.State().Events.SendLifecycle(project.Default, lifecycle.WarningReset.Event(id, request.CreateRequestor(r), nil))
	}

	return response.EmptySyncResponse
}

// swagger:operation DELETE /1.0/warnings/{uuid} warnings warning_delete
//
// Delete the warning
//
// Removes the warning.
//
// ---
// produces:
//   - application/json
// responses:
//   "200":
//     $ref: "#/responses/EmptySyncResponse"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningDelete(d *Daemon, r *http.Request) response.Response {
	id, err := url.PathUnescape(mux.Vars(r)["id"])
	if err != nil {
		return response.SmartError(err)
	}

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		err := tx.DeleteWarning(id)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	d.State().Events.SendLifecycle(project.Default, lifecycle.WarningDeleted.Event(id, request.CreateRequestor(r), nil))

	return response.EmptySyncResponse
}

func pruneResolvedWarningsTask(d *Daemon) (task.Func, task.Schedule) {
	f := func(ctx context.Context) {
		opRun := func(op *operations.Operation) error {
			return pruneResolvedWarnings(ctx, d)
		}

		op, err := operations.OperationCreate(d.State(), "", operations.OperationClassTask, operationtype.WarningsPruneResolved, nil, nil, opRun, nil, nil, nil)
		if err != nil {
			logger.Error("Failed to start prune resolved warnings operation", logger.Ctx{"err": err})
			return
		}

		logger.Info("Pruning resolved warnings")
		err = op.Start()
		if err != nil {
			logger.Error("Failed to prune resolved warnings", logger.Ctx{"err": err})
		}

		_, _ = op.Wait(ctx)
		logger.Info("Done pruning resolved warnings")
	}

	return f, task.Daily()
}

func pruneResolvedWarnings(ctx context.Context, d *Daemon) error {
	err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		// Retrieve warnings by resolved status.
		statusResolved := warningtype.StatusResolved
		filter := db.WarningFilter{
			Status: &statusResolved,
		}

		warnings, err := tx.GetWarnings(filter)
		if err != nil {
			return fmt.Errorf("Failed to get resolved warnings: %w", err)
		}

		for _, w := range warnings {
			// Delete the warning if it has been resolved for at least 24 hours
			if time.Since(w.UpdatedDate) >= 24*time.Hour {
				err = tx.DeleteWarning(w.UUID)
				if err != nil {
					return err
				}
			}
		}

		return nil
	})
	if err != nil {
		return fmt.Errorf("Failed to delete warnings: %w", err)
	}

	return nil
}
<commit_msg>lxd/warnings: Add getWarningEntityURL<commit_after>package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/db/operationtype"
	"github.com/lxc/lxd/lxd/db/warningtype"
	"github.com/lxc/lxd/lxd/filter"
	"github.com/lxc/lxd/lxd/lifecycle"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/request"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
)

var warningsCmd = APIEndpoint{
	Path: "warnings",

	Get: APIEndpointAction{Handler: warningsGet},
}

var warningCmd = APIEndpoint{
	Path: "warnings/{id}",

	Get:    APIEndpointAction{Handler: warningGet},
	Patch:  APIEndpointAction{Handler: warningPatch},
	Put:    APIEndpointAction{Handler: warningPut},
	Delete: APIEndpointAction{Handler: warningDelete},
}

func filterWarnings(warnings []api.Warning, clauses []filter.Clause) []api.Warning {
	filtered := []api.Warning{}

	for _, warning := range warnings {
		if !filter.Match(warning, clauses) {
			continue
		}

		filtered = append(filtered, warning)
	}

	return filtered
}

// swagger:operation GET /1.0/warnings warnings warnings_get
//
// List the warnings
//
// Returns a list of warnings.
//
// ---
// produces:
//   - application/json
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: Sync response
//     schema:
//       type: object
//       description: Sync response
//       properties:
//         type:
//           type: string
//           description: Response type
//           example: sync
//         status:
//           type: string
//           description: Status description
//           example: Success
//         status_code:
//           type: integer
//           description: Status code
//           example: 200
//         metadata:
//           type: array
//           description: List of endpoints
//           items:
//             type: string
//           example: |-
//             [
//               "/1.0/warnings/39c61a48-cc17-40ae-8248-4f7b4cadedf4",
//               "/1.0/warnings/951779a5-2820-4d96-b01e-88fe820e5310"
//             ]
//   "500":
//     $ref: "#/responses/InternalServerError"

// swagger:operation GET /1.0/warnings?recursion=1 warnings warnings_get_recursion1
//
// Get the warnings
//
// Returns a list of warnings (structs).
//
// ---
// produces:
//   - application/json
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: API endpoints
//     schema:
//       type: object
//       description: Sync response
//       properties:
//         type:
//           type: string
//           description: Response type
//           example: sync
//         status:
//           type: string
//           description: Status description
//           example: Success
//         status_code:
//           type: integer
//           description: Status code
//           example: 200
//         metadata:
//           type: array
//           description: List of warnings
//           items:
//             $ref: "#/definitions/Warning"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningsGet(d *Daemon, r *http.Request) response.Response {
	// Parse the recursion field
	recursionStr := r.FormValue("recursion")

	recursion, err := strconv.Atoi(recursionStr)
	if err != nil {
		recursion = 0
	}

	// Parse filter value
	var clauses []filter.Clause

	filterStr := r.FormValue("filter")
	if filterStr != "" {
		clauses, err = filter.Parse(filterStr)
		if err != nil {
			return response.SmartError(fmt.Errorf("Failed to filter warnings: %w", err))
		}
	}

	// Parse the project field
	projectName := queryParam(r, "project")

	var dbWarnings []db.Warning

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		filter := db.WarningFilter{}

		if projectName != "" {
			filter.Project = &projectName
		}

		dbWarnings, err = tx.GetWarnings(filter)
		if err != nil {
			return err
		}

		if err != nil {
			return fmt.Errorf("Failed to get warnings: %w", err)
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	warnings := make([]api.Warning, len(dbWarnings))

	for i, w := range dbWarnings {
		warning, err := w.ToAPI(d.db.Cluster)
		if err != nil {
			return response.SmartError(err)
		}

		warnings[i] = warning
	}

	if recursion == 0 {
		var resultList []string

		for _, w := range filterWarnings(warnings, clauses) {
			url := fmt.Sprintf("/%s/warnings/%s", version.APIVersion, w.UUID)
			resultList = append(resultList, url)
		}

		return response.SyncResponse(true, resultList)
	}

	// Return detailed list of warning
	return response.SyncResponse(true, filterWarnings(warnings, clauses))
}

// swagger:operation GET /1.0/warnings/{uuid} warnings warning_get
//
// Get the warning
//
// Gets a specific warning.
//
// ---
// produces:
//   - application/json
// responses:
//   "200":
//     description: Warning
//     schema:
//       type: object
//       description: Sync response
//       properties:
//         type:
//           type: string
//           description: Response type
//           example: sync
//         status:
//           type: string
//           description: Status description
//           example: Success
//         status_code:
//           type: integer
//           description: Status code
//           example: 200
//         metadata:
//           $ref: "#/definitions/Warning"
//   "404":
//     $ref: "#/responses/NotFound"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningGet(d *Daemon, r *http.Request) response.Response {
	id, err := url.PathUnescape(mux.Vars(r)["id"])
	if err != nil {
		return response.SmartError(err)
	}

	var dbWarning *db.Warning

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		dbWarning, err = tx.GetWarning(id)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	resp, err := dbWarning.ToAPI(d.db.Cluster)
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponse(true, resp)
}

// swagger:operation PATCH /1.0/warnings/{uuid} warnings warning_patch
//
// Partially update the warning
//
// Updates a subset of the warning status.
//
// ---
// consumes:
//   - application/json
// produces:
//   - application/json
// parameters:
//   - in: body
//     name: warning
//     description: Warning status
//     required: true
//     schema:
//       $ref: "#/definitions/WarningPut"
// responses:
//   "200":
//     $ref: "#/responses/EmptySyncResponse"
//   "400":
//     $ref: "#/responses/BadRequest"
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningPatch(d *Daemon, r *http.Request) response.Response {
	return warningPut(d, r)
}

// swagger:operation PUT /1.0/warnings/{uuid} warnings warning_put
//
// Update the warning
//
// Updates the warning status.
//
// ---
// consumes:
//   - application/json
// produces:
//   - application/json
// parameters:
//   - in: body
//     name: warning
//     description: Warning status
//     required: true
//     schema:
//       $ref: "#/definitions/WarningPut"
// responses:
//   "200":
//     $ref: "#/responses/EmptySyncResponse"
//   "400":
//     $ref: "#/responses/BadRequest"
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningPut(d *Daemon, r *http.Request) response.Response {
	id, err := url.PathUnescape(mux.Vars(r)["id"])
	if err != nil {
		return response.SmartError(err)
	}

	req := api.WarningPut{}

	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return response.BadRequest(err)
	}

	// Currently, we only allow changing the status to acknowledged or new.
	status, ok := warningtype.StatusTypes[req.Status]
	if !ok {
		// Invalid status
		return response.BadRequest(fmt.Errorf("Invalid warning type %q", req.Status))
	}

	if status != warningtype.StatusAcknowledged && status != warningtype.StatusNew {
		return response.Forbidden(fmt.Errorf(`Status may only be set to "acknowledge" or "new"`))
	}

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		err := tx.UpdateWarningStatus(id, status)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	if status == warningtype.StatusAcknowledged {
		d.State().Events.SendLifecycle(project.Default, lifecycle.WarningAcknowledged.Event(id, request.CreateRequestor(r), nil))
	} else {
		d.State().Events.SendLifecycle(project.Default, lifecycle.WarningReset.Event(id, request.CreateRequestor(r), nil))
	}

	return response.EmptySyncResponse
}

// swagger:operation DELETE /1.0/warnings/{uuid} warnings warning_delete
//
// Delete the warning
//
// Removes the warning.
//
// ---
// produces:
//   - application/json
// responses:
//   "200":
//     $ref: "#/responses/EmptySyncResponse"
//   "500":
//     $ref: "#/responses/InternalServerError"
func warningDelete(d *Daemon, r *http.Request) response.Response {
	id, err := url.PathUnescape(mux.Vars(r)["id"])
	if err != nil {
		return response.SmartError(err)
	}

	err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		err := tx.DeleteWarning(id)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return response.SmartError(err)
	}

	d.State().Events.SendLifecycle(project.Default, lifecycle.WarningDeleted.Event(id, request.CreateRequestor(r), nil))

	return response.EmptySyncResponse
}

func pruneResolvedWarningsTask(d *Daemon) (task.Func, task.Schedule) {
	f := func(ctx context.Context) {
		opRun := func(op *operations.Operation) error {
			return pruneResolvedWarnings(ctx, d)
		}

		op, err := operations.OperationCreate(d.State(), "", operations.OperationClassTask, operationtype.WarningsPruneResolved, nil, nil, opRun, nil, nil, nil)
		if err != nil {
			logger.Error("Failed to start prune resolved warnings operation", logger.Ctx{"err": err})
			return
		}

		logger.Info("Pruning resolved warnings")
		err = op.Start()
		if err != nil {
			logger.Error("Failed to prune resolved warnings", logger.Ctx{"err": err})
		}

		_, _ = op.Wait(ctx)
		logger.Info("Done pruning resolved warnings")
	}

	return f, task.Daily()
}

func pruneResolvedWarnings(ctx context.Context, d *Daemon) error {
	err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
		// Retrieve warnings by resolved status.
		statusResolved := warningtype.StatusResolved
		filter := db.WarningFilter{
			Status: &statusResolved,
		}

		warnings, err := tx.GetWarnings(filter)
		if err != nil {
			return fmt.Errorf("Failed to get resolved warnings: %w", err)
		}

		for _, w := range warnings {
			// Delete the warning if it has been resolved for at least 24 hours
			if time.Since(w.UpdatedDate) >= 24*time.Hour {
				err = tx.DeleteWarning(w.UUID)
				if err != nil {
					return err
				}
			}
		}

		return nil
	})
	if err != nil {
		return fmt.Errorf("Failed to delete warnings: %w", err)
	}

	return nil
}

// getWarningEntityURL fetches the entity corresponding to the warning from the database, and generates a URL.
func getWarningEntityURL(ctx context.Context, tx *sql.Tx, warning *cluster.Warning) (string, error) {
	if warning.EntityID == -1 || warning.EntityTypeCode == -1 {
		return "", nil
	}

	_, ok := cluster.EntityNames[warning.EntityTypeCode]
	if !ok {
		return "", fmt.Errorf("Unknown entity type")
	}

	var url string
	switch warning.EntityTypeCode {
	case cluster.TypeImage:
		entities, err := cluster.GetImages(ctx, tx, cluster.ImageFilter{ID: &warning.EntityID})
		if err != nil {
			return "", err
		}

		if len(entities) == 0 {
			return "", db.ErrUnknownEntityID
		}

		apiImage := api.Image{Fingerprint: entities[0].Fingerprint}
		url = apiImage.URL(version.APIVersion, entities[0].Project).String()
	case cluster.TypeProfile:
		entities, err := cluster.GetProfiles(ctx, tx, cluster.ProfileFilter{ID: &warning.EntityID})
		if err != nil {
			return "", err
		}

		if len(entities) == 0 {
			return "", db.ErrUnknownEntityID
		}

		apiProfile := api.Profile{Name: entities[0].Name}
		url = apiProfile.URL(version.APIVersion, entities[0].Project).String()
	case cluster.TypeProject:
		entities, err := cluster.GetProjects(ctx, tx, cluster.ProjectFilter{ID: &warning.EntityID})
		if err != nil {
			return "", err
		}

		if len(entities) == 0 {
			return "", db.ErrUnknownEntityID
		}

		apiProject := api.Project{Name: entities[0].Name}
		url = apiProject.URL(version.APIVersion).String()
	case cluster.TypeCertificate:
		entities, err := cluster.GetCertificates(ctx, tx, cluster.CertificateFilter{ID: &warning.EntityID})
		if err != nil {
			return "", err
		}

		if len(entities) == 0 {
			return "", db.ErrUnknownEntityID
		}

		apiCertificate := api.Certificate{Fingerprint: entities[0].Fingerprint}
		url = apiCertificate.URL(version.APIVersion).String()
	case cluster.TypeContainer:
		fallthrough
	case cluster.TypeInstance:
		entities, err := cluster.GetInstances(ctx, tx, cluster.InstanceFilter{ID: &warning.EntityID})
		if err != nil {
			return "", err
		}

		if len(entities) == 0 {
			return "", db.ErrUnknownEntityID
		}

		apiInstance := api.Instance{Name: entities[0].Name}
		url = apiInstance.URL(version.APIVersion, entities[0].Project).String()
	}

	return url, nil
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/davecgh/go-spew/spew"
	pkgTest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logstream"
	"github.com/knative/serving/pkg/apis/serving/v1alpha1"
	rnames "github.com/knative/serving/pkg/reconciler/revision/resources/names"
	"github.com/knative/serving/test"
	v1a1test "github.com/knative/serving/test/v1alpha1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	timeoutExpectedOutput  = "Slept for 0 milliseconds"
	revisionTimeoutSeconds = 45
	timeoutRequestDuration = 43 * time.Second
)

func TestDestroyPodInflight(t *testing.T) {
	t.Parallel()
	cancel := logstream.Start(t)
	defer cancel()

	clients := Setup(t)

	t.Log("Creating a new Route and Configuration")
	names, err := CreateRouteAndConfig(t, clients, "timeout", &v1a1test.Options{RevisionTimeoutSeconds: revisionTimeoutSeconds})
	if err != nil {
		t.Fatalf("Failed to create Route and Configuration: %v", err)
	}
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	t.Log("When the Revision can have traffic routed to it, the Route is marked as Ready")
	if err := v1a1test.WaitForRouteState(clients.ServingAlphaClient, names.Route, v1a1test.IsRouteReady, "RouteIsReady"); err != nil {
		t.Fatalf("The Route %s was not marked as Ready to serve traffic: %v", names.Route, err)
	}

	route, err := clients.ServingAlphaClient.Routes.Get(names.Route, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Error fetching Route %s: %v", names.Route, err)
	}
	domain := route.Status.URL.Host

	err = v1a1test.WaitForConfigurationState(clients.ServingAlphaClient, names.Config, func(c *v1alpha1.Configuration) (bool, error) {
		if c.Status.LatestCreatedRevisionName != names.Revision {
			names.Revision = c.Status.LatestCreatedRevisionName
			return true, nil
		}
		return false, nil
	}, "ConfigurationUpdatedWithRevision")
	if err != nil {
		t.Fatalf("Error obtaining Revision's name %v", err)
	}

	_, err = pkgTest.WaitForEndpointState(
		clients.KubeClient,
		t.Logf,
		domain,
		v1a1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesBody(timeoutExpectedOutput))),
		"TimeoutAppServesText",
		test.ServingFlags.ResolvableDomain)
	if err != nil {
		t.Fatalf("The endpoint for Route %s at domain %s didn't serve the expected text \"%s\": %v", names.Route, domain, helloWorldExpectedOutput, err)
	}

	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, domain, test.ServingFlags.ResolvableDomain)
	if err != nil {
		t.Fatalf("Error creating spoofing client: %v", err)
	}

	// The timeout app sleeps for the time passed via the timeout query parameter in milliseconds
	timeoutRequestDurationInMillis := timeoutRequestDuration / time.Millisecond
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/?timeout=%d", domain, timeoutRequestDurationInMillis), nil)
	if err != nil {
		t.Fatalf("Error creating http request: %v", err)
	}

	g, _ := errgroup.WithContext(context.Background())

	g.Go(func() error {
		t.Log("Sending in a long running request")
		res, err := client.Do(req)
		if err != nil {
			return err
		}

		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("Expected response to have status 200, had %d", res.StatusCode)
		}
		expectedBody := fmt.Sprintf("Slept for %d milliseconds", timeoutRequestDurationInMillis)
		gotBody := string(res.Body)
		if gotBody != expectedBody {
			return fmt.Errorf("Unexpected body, expected: %q got: %q", expectedBody, gotBody)
		}
		return nil
	})

	g.Go(func() error {
		// Give the request a bit of time to be established and reach the pod.
		time.Sleep(timeoutRequestDuration / 2)

		t.Log("Destroying the configuration (also destroys the pods)")
		return clients.ServingAlphaClient.Configs.Delete(names.Config, nil)
	})

	if err := g.Wait(); err != nil {
		t.Errorf("Something went wrong with the request: %v", err)
	}
}

const (
	// Give the pods plenty of time to disappear. It will take them at least 20 seconds to vanish
	// because we have a hard-coded sleep of 20 seconds before initiating the shutdown process.
	// This is still well below the 5 minutes it might take them to disappear max.
	maxTimeToDelete = 180 * time.Second
)

func TestDestroyPodTimely(t *testing.T) {
	t.Parallel()
	cancel := logstream.Start(t)
	defer cancel()

	clients := Setup(t)

	names := test.ResourceNames{
		Service: test.ObjectNameForTest(t),
		Image:   "helloworld",
	}

	defer test.TearDown(clients, names)
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })

	objects, err := v1a1test.CreateRunLatestServiceReady(t, clients, &names, &v1a1test.Options{RevisionTimeoutSeconds: 5 * 60})
	if err != nil {
		t.Fatalf("Failed to create a service: %v", err)
	}

	start := time.Now()

	// Deleting the service will also delete all pods.
	clients.ServingAlphaClient.Services.Delete(names.Service, nil)

	// Wait until the pod is shutdown. We don't wait for the pod itself to vanish but rather until all
	// of the containers of that pod are no longer running. It can take an arbitrarily long time to
	// actually remove the pod itself while we only care about containers being stopped.
	deploymentName := rnames.Deployment(objects.Revision)
	var podList *v1.PodList
	pkgTest.WaitForPodListState(
		clients.KubeClient,
		func(p *v1.PodList) (bool, error) {
			podList = p
			for _, pod := range p.Items {
				if !strings.Contains(pod.Name, deploymentName) {
					continue
				}
				for _, status := range pod.Status.ContainerStatuses {
					// There are still containers running, keep retrying.
					if status.State.Running != nil {
						return false, nil
					}
				}
			}
			return true, nil
		},
		"WaitForPodsToDisappear", test.ServingNamespace)

	timeToDelete := time.Since(start)
	if timeToDelete > maxTimeToDelete {
		t.Logf("Pod list: %s", spew.Sprint(podList))
		t.Errorf("Time to delete pods = %v, want < %v", timeToDelete, maxTimeToDelete)
	}
}
<commit_msg>Reduce exhaustive logging of a failed 'TestDestroyPodTimely' test. (#4460)<commit_after>// +build e2e

/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"

	"golang.org/x/sync/errgroup"

	"github.com/davecgh/go-spew/spew"
	pkgTest "github.com/knative/pkg/test"
	"github.com/knative/pkg/test/logstream"
	"github.com/knative/serving/pkg/apis/serving/v1alpha1"
	rnames "github.com/knative/serving/pkg/reconciler/revision/resources/names"
	"github.com/knative/serving/test"
	v1a1test "github.com/knative/serving/test/v1alpha1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	timeoutExpectedOutput  = "Slept for 0 milliseconds"
	revisionTimeoutSeconds = 45
	timeoutRequestDuration = 43 * time.Second
)

func TestDestroyPodInflight(t *testing.T) {
	t.Parallel()
	cancel := logstream.Start(t)
	defer cancel()

	clients := Setup(t)

	t.Log("Creating a new Route and Configuration")
	names, err := CreateRouteAndConfig(t, clients, "timeout", &v1a1test.Options{RevisionTimeoutSeconds: revisionTimeoutSeconds})
	if err != nil {
		t.Fatalf("Failed to create Route and Configuration: %v", err)
	}
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
	defer test.TearDown(clients, names)

	t.Log("When the Revision can have traffic routed to it, the Route is marked as Ready")
	if err := v1a1test.WaitForRouteState(clients.ServingAlphaClient, names.Route, v1a1test.IsRouteReady, "RouteIsReady"); err != nil {
		t.Fatalf("The Route %s was not marked as Ready to serve traffic: %v", names.Route, err)
	}

	route, err := clients.ServingAlphaClient.Routes.Get(names.Route, metav1.GetOptions{})
	if err != nil {
		t.Fatalf("Error fetching Route %s: %v", names.Route, err)
	}
	domain := route.Status.URL.Host

	err = v1a1test.WaitForConfigurationState(clients.ServingAlphaClient, names.Config, func(c *v1alpha1.Configuration) (bool, error) {
		if c.Status.LatestCreatedRevisionName != names.Revision {
			names.Revision = c.Status.LatestCreatedRevisionName
			return true, nil
		}
		return false, nil
	}, "ConfigurationUpdatedWithRevision")
	if err != nil {
		t.Fatalf("Error obtaining Revision's name %v", err)
	}

	_, err = pkgTest.WaitForEndpointState(
		clients.KubeClient,
		t.Logf,
		domain,
		v1a1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesBody(timeoutExpectedOutput))),
		"TimeoutAppServesText",
		test.ServingFlags.ResolvableDomain)
	if err != nil {
		t.Fatalf("The endpoint for Route %s at domain %s didn't serve the expected text \"%s\": %v", names.Route, domain, helloWorldExpectedOutput, err)
	}

	client, err := pkgTest.NewSpoofingClient(clients.KubeClient, t.Logf, domain, test.ServingFlags.ResolvableDomain)
	if err != nil {
		t.Fatalf("Error creating spoofing client: %v", err)
	}

	// The timeout app sleeps for the time passed via the timeout query parameter in milliseconds
	timeoutRequestDurationInMillis := timeoutRequestDuration / time.Millisecond
	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/?timeout=%d", domain, timeoutRequestDurationInMillis), nil)
	if err != nil {
		t.Fatalf("Error creating http request: %v", err)
	}

	g, _ := errgroup.WithContext(context.Background())

	g.Go(func() error {
		t.Log("Sending in a long running request")
		res, err := client.Do(req)
		if err != nil {
			return err
		}

		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("Expected response to have status 200, had %d", res.StatusCode)
		}
		expectedBody := fmt.Sprintf("Slept for %d milliseconds", timeoutRequestDurationInMillis)
		gotBody := string(res.Body)
		if gotBody != expectedBody {
			return fmt.Errorf("Unexpected body, expected: %q got: %q", expectedBody, gotBody)
		}
		return nil
	})

	g.Go(func() error {
		// Give the request a bit of time to be established and reach the pod.
		time.Sleep(timeoutRequestDuration / 2)

		t.Log("Destroying the configuration (also destroys the pods)")
		return clients.ServingAlphaClient.Configs.Delete(names.Config, nil)
	})

	if err := g.Wait(); err != nil {
		t.Errorf("Something went wrong with the request: %v", err)
	}
}

const (
	// Give the pods plenty of time to disappear. It will take them at least 20 seconds to vanish
	// because we have a hard-coded sleep of 20 seconds before initiating the shutdown process.
	// This is still well below the 5 minutes it might take them to disappear max.
	maxTimeToDelete = 180 * time.Second
)

func TestDestroyPodTimely(t *testing.T) {
	t.Parallel()
	cancel := logstream.Start(t)
	defer cancel()

	clients := Setup(t)

	names := test.ResourceNames{
		Service: test.ObjectNameForTest(t),
		Image:   "helloworld",
	}

	defer test.TearDown(clients, names)
	test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })

	objects, err := v1a1test.CreateRunLatestServiceReady(t, clients, &names, &v1a1test.Options{RevisionTimeoutSeconds: 5 * 60})
	if err != nil {
		t.Fatalf("Failed to create a service: %v", err)
	}

	start := time.Now()

	// Deleting the service will also delete all pods.
	clients.ServingAlphaClient.Services.Delete(names.Service, nil)

	// Wait until the pod is shutdown. We don't wait for the pod itself to vanish but rather until all
	// of the containers of that pod are no longer running. It can take an arbitrarily long time to
	// actually remove the pod itself while we only care about containers being stopped.
	deploymentName := rnames.Deployment(objects.Revision)
	var latestPodState v1.Pod
	pkgTest.WaitForPodListState(
		clients.KubeClient,
		func(p *v1.PodList) (bool, error) {
			for _, pod := range p.Items {
				if !strings.Contains(pod.Name, deploymentName) {
					continue
				}
				latestPodState = pod
				for _, status := range pod.Status.ContainerStatuses {
					// There are still containers running, keep retrying.
					if status.State.Running != nil {
						return false, nil
					}
				}
			}
			return true, nil
		},
		"WaitForPodsToDisappear", test.ServingNamespace)

	timeToDelete := time.Since(start)
	if timeToDelete > maxTimeToDelete {
		t.Logf("State: %s", spew.Sprint(latestPodState))
		t.Errorf("Time to delete pods = %v, want < %v", timeToDelete, maxTimeToDelete)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	log "github.com/funkygao/log4go"
)

type argsRule struct {
	cmd           cli.Command
	ui            cli.Ui
	requires      []string
	adminRequires map[string]struct{} // need admin rights: prompt password
	conditions    map[string][]string
}

func validateArgs(cmd cli.Command, ui cli.Ui) *argsRule {
	return &argsRule{
		cmd:           cmd,
		ui:            ui,
		requires:      make([]string, 0),
		adminRequires: make(map[string]struct{}),
		conditions:    make(map[string][]string),
	}
}

func ipaddr(hostname string) string {
	ips, _ := net.LookupIP(host)
	return ips[0].String()
}

func (this *argsRule) require(option ...string) *argsRule {
	this.requires = append(this.requires, option...)
	return this
}

func (this *argsRule) on(whenOption string, requiredOption ...string) *argsRule {
	if _, present := this.conditions[whenOption]; !present {
		this.conditions[whenOption] = make([]string, 0)
	}
	this.conditions[whenOption] = append(this.conditions[whenOption],
		requiredOption...)
	return this
}

func (this *argsRule) requireAdminRights(option ...string) *argsRule {
	for _, opt := range option {
		this.adminRequires[opt] = struct{}{}
	}
	return this
}

func (this *argsRule) invalid(args []string) bool {
	argSet := make(map[string]struct{}, len(args))
	for _, arg := range args {
		argSet[arg] = struct{}{}
	}

	// required
	for _, req := range this.requires {
		if _, present := argSet[req]; !present {
			this.ui.Error(color.Red("%s required", req))
			this.ui.Output(this.cmd.Help())
			return true
		}
	}

	// conditions
	for when, requires := range this.conditions {
		if _, present := argSet[when]; present {
			for _, req := range requires {
				if _, found := argSet[req]; !found {
					this.ui.Error(color.Red("%s required when %s present",
						req, when))
					this.ui.Output(this.cmd.Help())
					return true
				}
			}
		}
	}

	// admin required
	adminAuthRequired := false
	for _, arg := range args {
		if _, present := this.adminRequires[arg]; present {
			adminAuthRequired = true
			break
		}
	}
	if adminAuthRequired {
		if pass := os.Getenv("GK_PASS"); Authenticator("", pass) {
			return false
		}

		pass, err := this.ui.AskSecret("password for admin(or GK_PASS): ")
		this.ui.Output("")
		if err != nil {
			this.ui.Error(err.Error())
			return true
		}
		if !Authenticator("", pass) {
			this.ui.Error("invalid admin password, bye!")
			return true
		}
	}

	return false
}

func patternMatched(s, pattern string) bool {
	if pattern != "" {
		if pattern[0] == '~' {
			// NOT
			return !strings.Contains(s, pattern[1:])
		}

		if pattern[0] == '=' {
			return s == pattern[1:]
		}

		if !strings.Contains(s, pattern) {
			return false
		}
	}

	return true
}

func refreshScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}

func ensureZoneValid(zone string) {
	ctx.ZoneZkAddrs(zone) // will panic if zone not found
}

func forSortedZones(fn func(zkzone *zk.ZkZone)) {
	for _, zone := range ctx.SortedZones() {
		if strings.HasPrefix(zone, "z_") {
			// zk only
			continue
		}

		zkAddrs := ctx.ZoneZkAddrs(zone)
		if strings.TrimSpace(zkAddrs) == "" {
			continue
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, zkAddrs))
		fn(zkzone)
	}
}

func forAllSortedZones(fn func(zkzone *zk.ZkZone)) {
	for _, zone := range ctx.SortedZones() {
		zkAddrs := ctx.ZoneZkAddrs(zone)
		if strings.TrimSpace(zkAddrs) == "" {
			continue
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, zkAddrs))
		fn(zkzone)
	}
}

func swallow(err error) {
	if err != nil {
		panic(err)
	}
}

type sortedStrMap struct {
	keys []string
	vals []interface{}
}

// TODO map[string]interface{}
func sortStrMap(m map[string]int) sortedStrMap {
	sortedKeys := make([]string, 0, len(m))
	for key, _ := range m {
		sortedKeys = append(sortedKeys, key)
	}
	sort.Strings(sortedKeys)

	r := sortedStrMap{
		keys: sortedKeys,
		vals: make([]interface{}, len(m)),
	}
	for idx, key := range sortedKeys {
		r.vals[idx] = m[key]
	}

	return r
}

func printSwallowedErrors(ui cli.Ui, zkzone *zk.ZkZone) {
	errs := zkzone.Errors()
	if len(errs) == 0 {
		return
	}

	for _, e := range errs {
		ui.Error(color.Red("%v", e))
	}
}

func shortIp(host string) string {
	parts := strings.SplitN(host, ".", 4)
	return strings.Join(parts[2:], ".")
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

func toLogLevel(levelStr string) log.Level {
	level := log.TRACE
	switch levelStr {
	case "info":
		level = log.INFO

	case "warn":
		level = log.WARNING

	case "error":
		level = log.ERROR

	case "debug":
		level = log.DEBUG

	case "trace":
		level = log.TRACE

	case "alarm":
		level = log.ALARM
	}

	return level
}

func setupLogging(logFile, level, crashLogFile string) {
	logLevel := toLogLevel(level)

	for _, filter := range log.Global {
		filter.Level = logLevel
	}

	log.LogBufferLength = 32 // default 32, chan cap

	if logFile == "stdout" {
		log.AddFilter("stdout", logLevel, log.NewConsoleLogWriter())
	} else {
		log.DeleteFilter("stdout")

		filer := log.NewFileLogWriter(logFile, true, false, 0)
		filer.SetFormat("[%d %T] [%L] (%S) %M")
		filer.SetRotateSize(0)
		filer.SetRotateLines(0)
		filer.SetRotateDaily(true)
		log.AddFilter("file", logLevel, filer)
	}

	if crashLogFile != "" {
		f, err := os.OpenFile(crashLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			panic(err)
		}

		syscall.Dup2(int(f.Fd()), int(os.Stdout.Fd()))
		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
		fmt.Fprintf(os.Stderr, "\n%s %s (build: %s)\n===================\n",
			time.Now().String(),
			gafka.Version, gafka.BuildId)
	}

}
<commit_msg>fix compile err<commit_after>package command

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	log "github.com/funkygao/log4go"
)

type argsRule struct {
	cmd           cli.Command
	ui            cli.Ui
	requires      []string
	adminRequires map[string]struct{} // need admin rights: prompt password
	conditions    map[string][]string
}

func validateArgs(cmd cli.Command, ui cli.Ui) *argsRule {
	return &argsRule{
		cmd:           cmd,
		ui:            ui,
		requires:      make([]string, 0),
		adminRequires: make(map[string]struct{}),
		conditions:    make(map[string][]string),
	}
}

func ipaddr(hostname string) string {
	ips, _ := net.LookupIP(hostname)
	return ips[0].String()
}

func (this *argsRule) require(option ...string) *argsRule {
	this.requires = append(this.requires, option...)
	return this
}

func (this *argsRule) on(whenOption string, requiredOption ...string) *argsRule {
	if _, present := this.conditions[whenOption]; !present {
		this.conditions[whenOption] = make([]string, 0)
	}
	this.conditions[whenOption] = append(this.conditions[whenOption],
		requiredOption...)
	return this
}

func (this *argsRule) requireAdminRights(option ...string) *argsRule {
	for _, opt := range option {
		this.adminRequires[opt] = struct{}{}
	}
	return this
}

func (this *argsRule) invalid(args []string) bool {
	argSet := make(map[string]struct{}, len(args))
	for _, arg := range args {
		argSet[arg] = struct{}{}
	}

	// required
	for _, req := range this.requires {
		if _, present := argSet[req]; !present {
			this.ui.Error(color.Red("%s required", req))
			this.ui.Output(this.cmd.Help())
			return true
		}
	}

	// conditions
	for when, requires := range this.conditions {
		if _, present := argSet[when]; present {
			for _, req := range requires {
				if _, found := argSet[req]; !found {
					this.ui.Error(color.Red("%s required when %s present",
						req, when))
					this.ui.Output(this.cmd.Help())
					return true
				}
			}
		}
	}

	// admin required
	adminAuthRequired := false
	for _, arg := range args {
		if _, present := this.adminRequires[arg]; present {
			adminAuthRequired = true
			break
		}
	}
	if adminAuthRequired {
		if pass := os.Getenv("GK_PASS"); Authenticator("", pass) {
			return false
		}

		pass, err := this.ui.AskSecret("password for admin(or GK_PASS): ")
		this.ui.Output("")
		if err != nil {
			this.ui.Error(err.Error())
			return true
		}
		if !Authenticator("", pass) {
			this.ui.Error("invalid admin password, bye!")
			return true
		}
	}

	return false
}

func patternMatched(s, pattern string) bool {
	if pattern != "" {
		if pattern[0] == '~' {
			// NOT
			return !strings.Contains(s, pattern[1:])
		}

		if pattern[0] == '=' {
			return s == pattern[1:]
		}

		if !strings.Contains(s, pattern) {
			return false
		}
	}

	return true
}

func refreshScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}

func ensureZoneValid(zone string) {
	ctx.ZoneZkAddrs(zone) // will panic if zone not found
}

func forSortedZones(fn func(zkzone *zk.ZkZone)) {
	for _, zone := range ctx.SortedZones() {
		if strings.HasPrefix(zone, "z_") {
			// zk only
			continue
		}

		zkAddrs := ctx.ZoneZkAddrs(zone)
		if strings.TrimSpace(zkAddrs) == "" {
			continue
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, zkAddrs))
		fn(zkzone)
	}
}

func forAllSortedZones(fn func(zkzone *zk.ZkZone)) {
	for _, zone := range ctx.SortedZones() {
		zkAddrs := ctx.ZoneZkAddrs(zone)
		if strings.TrimSpace(zkAddrs) == "" {
			continue
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, zkAddrs))
		fn(zkzone)
	}
}

func swallow(err error) {
	if err != nil {
		panic(err)
	}
}

type sortedStrMap struct {
	keys []string
	vals []interface{}
}

// TODO map[string]interface{}
func sortStrMap(m map[string]int) sortedStrMap {
	sortedKeys := make([]string, 0, len(m))
	for key, _ := range m {
		sortedKeys = append(sortedKeys, key)
	}
	sort.Strings(sortedKeys)

	r := sortedStrMap{
		keys: sortedKeys,
		vals: make([]interface{}, len(m)),
	}
	for idx, key := range sortedKeys {
		r.vals[idx] = m[key]
	}

	return r
}

func printSwallowedErrors(ui cli.Ui, zkzone *zk.ZkZone) {
	errs := zkzone.Errors()
	if len(errs) == 0 {
		return
	}

	for _, e := range errs {
		ui.Error(color.Red("%v", e))
	}
}

func shortIp(host string) string {
	parts := strings.SplitN(host, ".", 4)
	return strings.Join(parts[2:], ".")
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}

func toLogLevel(levelStr string) log.Level {
	level := log.TRACE
	switch levelStr {
	case "info":
		level = log.INFO

	case "warn":
		level = log.WARNING

	case "error":
		level = log.ERROR

	case "debug":
		level = log.DEBUG

	case "trace":
		level = log.TRACE

	case "alarm":
		level = log.ALARM
	}

	return level
}

func setupLogging(logFile, level, crashLogFile string) {
	logLevel := toLogLevel(level)

	for _, filter := range log.Global {
		filter.Level = logLevel
	}

	log.LogBufferLength = 32 // default 32, chan cap

	if logFile == "stdout" {
		log.AddFilter("stdout", logLevel, log.NewConsoleLogWriter())
	} else {
		log.DeleteFilter("stdout")

		filer := log.NewFileLogWriter(logFile, true, false, 0)
		filer.SetFormat("[%d %T] [%L] (%S) %M")
		filer.SetRotateSize(0)
		filer.SetRotateLines(0)
		filer.SetRotateDaily(true)
		log.AddFilter("file", logLevel, filer)
	}

	if crashLogFile != "" {
		f, err := os.OpenFile(crashLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			panic(err)
		}

		syscall.Dup2(int(f.Fd()), int(os.Stdout.Fd()))
		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
		fmt.Fprintf(os.Stderr, "\n%s %s (build: %s)\n===================\n",
			time.Now().String(),
			gafka.Version, gafka.BuildId)
	}

}
<|endoftext|>


<commit_before>package tsc

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
	"time"
	"unicode"
	"unicode/utf8"
)

func nonil(err ...error) error {
	for _, err := range err {
		if err != nil {
			return err
		}
	}
	return nil
}

type Event struct {
	Name    string      // https://developer.github.com/webhooks/#events
	Payload interface{} // https://developer.github.com/v3/activity/events/types/
	Args    map[string]string
}

type Script struct {
	// ErrorLog specifies an optional logger for errors serving requests.
	// If nil, logging goes to os.Stderr via the log package's standard logger.
	ErrorLog *log.Logger

	OutputFunc func() io.Writer

	tmpl *template.Template
	args map[string]string
}

func New(file string, args []string) (*Script, error) {
	if len(args)&1 == 1 {
		return nil, errors.New("number of arguments for template script must be even")
	}
	sc := &Script{}
	if len(args) != 0 {
		sc.args = make(map[string]string)
		for i := 0; i < len(args); i += 2 {
			if len(args[i]) < 2 || args[i][0] != '-' {
				return nil, errors.New("invalid flag name: " + args[i])
			}
			r, n := utf8.DecodeRuneInString(args[i][1:])
			if r == utf8.RuneError {
				return nil, errors.New("invalid flag name: " + args[i])
			}
			sc.args[string(unicode.ToUpper(r))+args[i][1+n:]] = args[i+1]
		}
	}
	tmpl, err := template.New(filepath.Base(file)).Funcs(sc.funcs()).ParseFiles(file)
	if err != nil {
		return nil, err
	}
	sc.tmpl = tmpl
	return sc, nil
}

func (s *Script) Webhook(event string, payload interface{}) {
	w := s.output()
	err := s.tmpl.Execute(w, Event{Name: event, Payload: payload, Args: s.args})
	if c, ok := w.(io.Closer); ok {
		err = nonil(err, c.Close())
	}
	if err != nil {
		s.logf("ERROR template script error: %v", err)
	}
}

func (s *Script) funcs() template.FuncMap {
	return template.FuncMap{
		"env": func(s string) string {
			return os.Getenv(s)
		},
		"exec": func(cmd string, args ...string) (string, error) {
			out, err := exec.Command(cmd, args...).Output()
			return string(bytes.TrimSpace(out)), err
		},
		"sleep": func(s string) (string, error) {
			d, err := time.ParseDuration(s)
			if err != nil {
				return "", err
			}
			time.Sleep(d)
			return "", nil
		},
		"log": func(v ...interface{}) string {
			if len(v) != 0 {
				s.log(v...)
			}
			return ""
		},
		"logf": func(format string, v ...interface{}) string {
			if format == "" {
				return ""
			}
			if len(v) == 0 {
				s.logf("%s", format)
			} else {
				s.logf(format, v...)
			}
			return ""
		},
	}
}

func (s *Script) output() io.Writer {
	if s.OutputFunc != nil {
		return s.OutputFunc()
	} else {
		return ioutil.Discard
	}
}

func (s *Script) logf(format string, v ...interface{}) {
	if s.ErrorLog != nil {
		s.ErrorLog.Printf(format, v...)
	} else {
		log.Printf(format, v...)
	}
}

func (s *Script) log(v ...interface{}) {
	if s.ErrorLog != nil {
		s.ErrorLog.Println(v...)
	} else {
		log.Println(v...)
	}
}
<commit_msg>cmd/webhook: support executing bash scripts<commit_after>package tsc

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
	"time"
	"unicode"
	"unicode/utf8"
)

func nonil(err ...error) error {
	for _, err := range err {
		if err != nil {
			return err
		}
	}
	return nil
}

type Event struct {
	Name    string      // https://developer.github.com/webhooks/#events
	Payload interface{} // https://developer.github.com/v3/activity/events/types/
	Args    map[string]string
}

type Script struct {
	// ErrorLog specifies an optional logger for errors serving requests.
	// If nil, logging goes to os.Stderr via the log package's standard logger.
	ErrorLog *log.Logger

	OutputFunc func() io.Writer

	bash bool
	tmpl *template.Template
	args map[string]string
}

func New(file string, args []string) (*Script, error) {
	if len(args)&1 == 1 {
		return nil, errors.New("number of arguments for template script must be even")
	}
	s := &Script{}
	if len(args) != 0 {
		s.args = make(map[string]string, len(args)/2)
		for i := 0; i < len(args); i += 2 {
			if len(args[i]) < 2 || args[i][0] != '-' {
				return nil, errors.New("invalid flag name: " + args[i])
			}
			r, n := utf8.DecodeRuneInString(args[i][1:])
			if r == utf8.RuneError {
				return nil, errors.New("invalid flag name: " + args[i])
			}
			s.args[string(unicode.ToUpper(r))+args[i][1+n:]] = args[i+1]
		}
	}
	s.bash = strings.HasSuffix(file, ".sh") || strings.HasSuffix(file, ".bash")
	var err error
	s.tmpl, err = template.New(filepath.Base(file)).Funcs(s.funcs()).ParseFiles(file)
	if err != nil {
		return nil, err
	}
	return s, nil
}

func (s *Script) Webhook(event string, payload interface{}) {
	e := &Event{
		Name:    event,
		Payload: payload,
		Args:    s.args,
	}
	var err error
	if s.bash {
		err = s.runBash(e)
	} else {
		err = s.execute(s.output(), e)
	}
	if err != nil {
		s.logf("ERROR template script error: %v", err)
	}
}

func (s *Script) runBash(e *Event) (err error) {
	var buf bytes.Buffer
	if err = s.execute(&buf, e); err != nil {
		return err
	}

	cmd := exec.Command("bash")
	cmd.Stdin = bytes.NewReader(buf.Bytes())
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stderr
	cmd.Env = append(os.Environ(), "WEBHOOK=1") // cache?

	if err = cmd.Run(); err != nil {
		// dump script for later troubleshooting
		f, e := ioutil.TempFile("webhook", "debug")
		if e == nil {
			_, e = io.Copy(f, bytes.NewReader(buf.Bytes()))
			e = nonil(e, f.Close())
		}
		if e != nil {
			return fmt.Errorf("%s (failed to dump script file: %s)", err, e)
		}
		return fmt.Errorf("%s: failed with: %s", f.Name(), err)
	}

	return nil
}

func (s *Script) execute(w io.Writer, e *Event) error {
	err := s.tmpl.Execute(w, e)
	if c, ok := w.(io.Closer); ok {
		err = nonil(err, c.Close())
	}
	return err
}

func (s *Script) funcs() template.FuncMap {
	return template.FuncMap{
		"env": func(s string) string {
			return os.Getenv(s)
		},
		"exec": func(cmd string, args ...string) (string, error) {
			out, err := exec.Command(cmd, args...).Output()
			return string(bytes.TrimSpace(out)), err
		},
		"sleep": func(s string) (string, error) {
			d, err := time.ParseDuration(s)
			if err != nil {
				return "", err
			}
			time.Sleep(d)
			return "", nil
		},
		"log": func(v ...interface{}) string {
			if len(v) != 0 {
				s.log(v...)
			}
			return ""
		},
		"logf": func(format string, v ...interface{}) string {
			if format == "" {
				return ""
			}
			if len(v) == 0 {
				s.logf("%s", format)
			} else {
				s.logf(format, v...)
			}
			return ""
		},
	}
}

func (s *Script) output() io.Writer {
	if s.OutputFunc != nil {
		return s.OutputFunc()
	} else {
		return ioutil.Discard
	}
}

func (s *Script) logf(format string, v ...interface{}) {
	if s.ErrorLog != nil {
		s.ErrorLog.Printf(format, v...)
	} else {
		log.Printf(format, v...)
	}
}

func (s *Script) log(v ...interface{}) {
	if s.ErrorLog != nil {
		s.ErrorLog.Println(v...)
	} else {
		log.Println(v...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

type networkMessage struct {
	msg       []byte
	msgType   uint8
	queueable bool
	ts        time.Time
}

type networkConnection struct {
	Conn       *net.UDPConn
	Ip         string
	Port       uint32
	Capability uint8
	sleepMode  bool     // Device is not able to receive messages currently.
	sleepQueue [][]byte // Device message queue.
}

var messageQueue chan networkMessage
var outSockets map[string]networkConnection
var dhcpLeases map[string]string
var netMutex *sync.Mutex

const (
	NETWORK_GDL90_STANDARD = 1
	NETWORK_AHRS_FFSIM     = 2
	NETWORK_AHRS_GDL90     = 4
	dhcp_lease_file        = "/var/lib/dhcp/dhcpd.leases"
)

// Read the "dhcpd.leases" file and parse out IP/hostname.
func getDHCPLeases() (map[string]string, error) {
	dat, err := ioutil.ReadFile(dhcp_lease_file)
	ret := make(map[string]string)
	if err != nil {
		return ret, err
	}
	lines := strings.Split(string(dat), "\n")
	open_block := false
	block_ip := ""
	for _, line := range lines {
		spaced := strings.Split(line, " ")
		if len(spaced) > 2 && spaced[0] == "lease" {
			open_block = true
			block_ip = spaced[1]
		} else if open_block && len(spaced) >= 4 && spaced[2] == "client-hostname" {
			hostname := strings.TrimRight(strings.TrimLeft(strings.Join(spaced[3:], " "), "\""), "\";")
			ret[block_ip] = hostname
			open_block = false
		}
	}
	return ret, nil
}

func sendToAllConnectedClients(msg networkMessage) {
	netMutex.Lock()
	defer netMutex.Unlock()
	for k, netconn := range outSockets {
		if (netconn.Capability & msg.msgType) != 0 { // Check if this port is able to accept the type of message we're sending.
			// Check if the client is in sleep mode.
			if !netconn.sleepMode { // Write immediately.
				netconn.Conn.Write(msg.msg)
			} else if msg.queueable { // Queue the message if the message is "queueable". Discard otherwise.
				if len(netconn.sleepQueue) >= maxUserMsgQueueSize { // Too many messages queued? Client has been asleep for too long. Drop the oldest.
					log.Printf("%s:%d - message queue overflow.\n", netconn.Ip, netconn.Port)
					netconn.sleepQueue = netconn.sleepQueue[1:maxUserMsgQueueSize-1]
				}
				netconn.sleepQueue = append(netconn.sleepQueue, msg.msg)
				outSockets[k] = netconn
			}
		}
	}
}

// Just returns the number of DHCP leases for now.
func getNetworkStats() uint {
	ret := uint(len(dhcpLeases))
	globalStatus.Connected_Users = ret
	return ret
}

// See who has a DHCP lease and make a UDP connection to each of them.
func refreshConnectedClients() {
	netMutex.Lock()
	defer netMutex.Unlock()
	validConnections := make(map[string]bool)
	t, err := getDHCPLeases()
	if err != nil {
		log.Printf("getDHCPLeases(): %s\n", err.Error())
		return
	}
	dhcpLeases = t
	// Client connected that wasn't before.
	for ip, hostname := range dhcpLeases {
		for _, networkOutput := range globalSettings.NetworkOutputs {
			ipAndPort := ip + ":" + strconv.Itoa(int(networkOutput.Port))
			if _, ok := outSockets[ipAndPort]; !ok {
				log.Printf("client connected: %s:%d (%s).\n", ip, networkOutput.Port, hostname)
				addr, err := net.ResolveUDPAddr("udp", ipAndPort)
				if err != nil {
					log.Printf("ResolveUDPAddr(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				outConn, err := net.DialUDP("udp", nil, addr)
				if err != nil {
					log.Printf("DialUDP(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				newq := make([][]byte, 0)
				sleepMode := false
				if networkOutput.Port == 4000 { // Start off FF in sleep mode until something is received.
					sleepMode = true
				}
				outSockets[ipAndPort] = networkConnection{Conn: outConn, Ip: ip, Port: networkOutput.Port, Capability: networkOutput.Capability, sleepMode: sleepMode, sleepQueue: newq}
			}
			validConnections[ipAndPort] = true
		}
	}
	// Client that was connected before that isn't.
	for ipAndPort, conn := range outSockets {
		if _, ok := validConnections[ipAndPort]; !ok {
			log.Printf("removed connection %s.\n", ipAndPort)
			conn.Conn.Close()
			delete(outSockets, ipAndPort)
		}
	}
}

func checkMessageQueues() {
	netMutex.Lock()
	defer netMutex.Unlock()
	for k, netconn := range outSockets {
		if len(netconn.sleepQueue) > 0 && !netconn.sleepMode {
			// Empty the sleep queue.
			tmpQueue := netconn.sleepQueue
			tmpConn := netconn.Conn
			go func() {
				time.Sleep(5 * time.Second) // Give it time to start listening, some apps send the "woke up" message too quickly.
				for _, msg := range tmpQueue {
					tmpConn.Write(msg)
					time.Sleep(50 * time.Millisecond) // Slow down the sending, 20/sec.
				}
			}()
			netconn.sleepQueue = make([][]byte, 0)
			outSockets[k] = netconn
			log.Printf("%s - emptied %d in queue.\n", k, len(tmpQueue))
		}
	}
}

func messageQueueSender() {
	secondTimer := time.NewTicker(5 * time.Second)
	for {
		select {
		case msg := <-messageQueue:
			sendToAllConnectedClients(msg)
		case <-secondTimer.C:
			getNetworkStats()
			checkMessageQueues()
		}
	}
}

func sendMsg(msg []byte, msgType uint8, queueable bool) {
	messageQueue <- networkMessage{msg: msg, msgType: msgType, queueable: queueable, ts: time.Now()}
}

func sendGDL90(msg []byte, queueable bool) {
	sendMsg(msg, NETWORK_GDL90_STANDARD, queueable)
}

func monitorDHCPLeases() {
	//TODO: inotify or dhcp event hook.
	timer := time.NewTicker(30 * time.Second)
	for {
		select {
		case <-timer.C:
			refreshConnectedClients()
		}
	}
}

// Monitor clients going in/out of sleep mode. This will be different for different apps.
func sleepMonitor() {
	// FF sleep mode.
	addr := net.UDPAddr{Port: 50113, IP: net.ParseIP("0.0.0.0")}
	conn, err := net.ListenUDP("udp", &addr)
	defer conn.Close()
	if err != nil {
		log.Printf("err: %s\n", err.Error())
		log.Printf("error listening on port 50113 (FF comm) - assuming FF is always awake (if connected).\n")
		return
	}
	for {
		buf := make([]byte, 1024)
		n, addr, err := conn.ReadFrom(buf)
		ipAndPort := strings.Split(addr.String(), ":")
		ip := ipAndPort[0]
		if err != nil {
			log.Printf("err: %s\n", err.Error())
			return
		}
		// Got message, check if it's in the correct format.
		if n < 3 || buf[0] != 0xFF || buf[1] != 0xFE {
			continue
		}
		s := string(buf[2:n])
		s = strings.Replace(s, "\x00", "", -1)
		ffIpAndPort := ip + ":4000"
		netMutex.Lock()
		p, ok := outSockets[ffIpAndPort]
		if !ok {
			// Can't do anything, the client isn't even technically connected.
			netMutex.Unlock()
			continue
		}
		if strings.HasPrefix(s, "i-want-to-play-ffm-udp") || strings.HasPrefix(s, "i-can-play-ffm-udp") {
			if p.sleepMode {
				log.Printf("%s - woke up\n", ffIpAndPort)
				p.sleepMode = false
			}
		} else if strings.HasPrefix(s, "i-cannot-play-ffm-udp") {
			if !p.sleepMode {
				log.Printf("%s - went to sleep\n", ffIpAndPort)
				p.sleepMode = true
			}
		}
		outSockets[ffIpAndPort] = p
		netMutex.Unlock()
	}
}

func initNetwork() {
	messageQueue = make(chan networkMessage, 1024) // Buffered channel, 1024 messages.
	outSockets = make(map[string]networkConnection)
	netMutex = &sync.Mutex{}
	refreshConnectedClients()
	go monitorDHCPLeases()
	go messageQueueSender()
	go sleepMonitor()
}
<commit_msg>Fixed bug in network code.<commit_after>package main

import (
	"io/ioutil"
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

type networkMessage struct {
	msg       []byte
	msgType   uint8
	queueable bool
	ts        time.Time
}

type networkConnection struct {
	Conn       *net.UDPConn
	Ip         string
	Port       uint32
	Capability uint8
	sleepMode  bool     // Device is not able to receive messages currently.
	sleepQueue [][]byte // Device message queue.
}

var messageQueue chan networkMessage
var outSockets map[string]networkConnection
var dhcpLeases map[string]string
var netMutex *sync.Mutex

const (
	NETWORK_GDL90_STANDARD = 1
	NETWORK_AHRS_FFSIM     = 2
	NETWORK_AHRS_GDL90     = 4
	dhcp_lease_file        = "/var/lib/dhcp/dhcpd.leases"
)

// Read the "dhcpd.leases" file and parse out IP/hostname.
func getDHCPLeases() (map[string]string, error) {
	dat, err := ioutil.ReadFile(dhcp_lease_file)
	ret := make(map[string]string)
	if err != nil {
		return ret, err
	}
	lines := strings.Split(string(dat), "\n")
	open_block := false
	block_ip := ""
	for _, line := range lines {
		spaced := strings.Split(line, " ")
		if len(spaced) > 2 && spaced[0] == "lease" {
			open_block = true
			block_ip = spaced[1]
		} else if open_block && len(spaced) >= 4 && spaced[2] == "client-hostname" {
			hostname := strings.TrimRight(strings.TrimLeft(strings.Join(spaced[3:], " "), "\""), "\";")
			ret[block_ip] = hostname
			open_block = false
		} else if open_block && strings.HasPrefix(spaced[0], "}") { // No hostname.
			open_block = false
			ret[block_ip] = ""
		}
	}
	return ret, nil
}

func sendToAllConnectedClients(msg networkMessage) {
	netMutex.Lock()
	defer netMutex.Unlock()
	for k, netconn := range outSockets {
		if (netconn.Capability & msg.msgType) != 0 { // Check if this port is able to accept the type of message we're sending.
			// Check if the client is in sleep mode.
			if !netconn.sleepMode { // Write immediately.
				netconn.Conn.Write(msg.msg)
			} else if msg.queueable { // Queue the message if the message is "queueable". Discard otherwise.
				if len(netconn.sleepQueue) >= maxUserMsgQueueSize { // Too many messages queued? Client has been asleep for too long. Drop the oldest.
					log.Printf("%s:%d - message queue overflow.\n", netconn.Ip, netconn.Port)
					netconn.sleepQueue = netconn.sleepQueue[1:maxUserMsgQueueSize-1]
				}
				netconn.sleepQueue = append(netconn.sleepQueue, msg.msg)
				outSockets[k] = netconn
			}
		}
	}
}

// Just returns the number of DHCP leases for now.
func getNetworkStats() uint {
	ret := uint(len(dhcpLeases))
	globalStatus.Connected_Users = ret
	return ret
}

// See who has a DHCP lease and make a UDP connection to each of them.
func refreshConnectedClients() {
	netMutex.Lock()
	defer netMutex.Unlock()
	validConnections := make(map[string]bool)
	t, err := getDHCPLeases()
	if err != nil {
		log.Printf("getDHCPLeases(): %s\n", err.Error())
		return
	}
	dhcpLeases = t
	// Client connected that wasn't before.
	for ip, hostname := range dhcpLeases {
		for _, networkOutput := range globalSettings.NetworkOutputs {
			ipAndPort := ip + ":" + strconv.Itoa(int(networkOutput.Port))
			if _, ok := outSockets[ipAndPort]; !ok {
				log.Printf("client connected: %s:%d (%s).\n", ip, networkOutput.Port, hostname)
				addr, err := net.ResolveUDPAddr("udp", ipAndPort)
				if err != nil {
					log.Printf("ResolveUDPAddr(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				outConn, err := net.DialUDP("udp", nil, addr)
				if err != nil {
					log.Printf("DialUDP(%s): %s\n", ipAndPort, err.Error())
					continue
				}
				newq := make([][]byte, 0)
				sleepMode := false
				if networkOutput.Port == 4000 { // Start off FF in sleep mode until something is received.
					sleepMode = true
				}
				outSockets[ipAndPort] = networkConnection{Conn: outConn, Ip: ip, Port: networkOutput.Port, Capability: networkOutput.Capability, sleepMode: sleepMode, sleepQueue: newq}
			}
			validConnections[ipAndPort] = true
		}
	}
	// Client that was connected before that isn't.
	for ipAndPort, conn := range outSockets {
		if _, ok := validConnections[ipAndPort]; !ok {
			log.Printf("removed connection %s.\n", ipAndPort)
			conn.Conn.Close()
			delete(outSockets, ipAndPort)
		}
	}
}

func checkMessageQueues() {
	netMutex.Lock()
	defer netMutex.Unlock()
	for k, netconn := range outSockets {
		if len(netconn.sleepQueue) > 0 && !netconn.sleepMode {
			// Empty the sleep queue.
			tmpQueue := netconn.sleepQueue
			tmpConn := netconn.Conn
			go func() {
				time.Sleep(5 * time.Second) // Give it time to start listening, some apps send the "woke up" message too quickly.
				for _, msg := range tmpQueue {
					tmpConn.Write(msg)
					time.Sleep(50 * time.Millisecond) // Slow down the sending, 20/sec.
				}
			}()
			netconn.sleepQueue = make([][]byte, 0)
			outSockets[k] = netconn
			log.Printf("%s - emptied %d in queue.\n", k, len(tmpQueue))
		}
	}
}

func messageQueueSender() {
	secondTimer := time.NewTicker(5 * time.Second)
	for {
		select {
		case msg := <-messageQueue:
			sendToAllConnectedClients(msg)
		case <-secondTimer.C:
			getNetworkStats()
			checkMessageQueues()
		}
	}
}

func sendMsg(msg []byte, msgType uint8, queueable bool) {
	messageQueue <- networkMessage{msg: msg, msgType: msgType, queueable: queueable, ts: time.Now()}
}

func sendGDL90(msg []byte, queueable bool) {
	sendMsg(msg, NETWORK_GDL90_STANDARD, queueable)
}

func monitorDHCPLeases() {
	//TODO: inotify or dhcp event hook.
	timer := time.NewTicker(30 * time.Second)
	for {
		select {
		case <-timer.C:
			refreshConnectedClients()
		}
	}
}

// Monitor clients going in/out of sleep mode. This will be different for different apps.
func sleepMonitor() {
	// FF sleep mode.
	addr := net.UDPAddr{Port: 50113, IP: net.ParseIP("0.0.0.0")}
	conn, err := net.ListenUDP("udp", &addr)
	defer conn.Close()
	if err != nil {
		log.Printf("err: %s\n", err.Error())
		log.Printf("error listening on port 50113 (FF comm) - assuming FF is always awake (if connected).\n")
		return
	}
	for {
		buf := make([]byte, 1024)
		n, addr, err := conn.ReadFrom(buf)
		ipAndPort := strings.Split(addr.String(), ":")
		ip := ipAndPort[0]
		if err != nil {
			log.Printf("err: %s\n", err.Error())
			return
		}
		// Got message, check if it's in the correct format.
		if n < 3 || buf[0] != 0xFF || buf[1] != 0xFE {
			continue
		}
		s := string(buf[2:n])
		s = strings.Replace(s, "\x00", "", -1)
		ffIpAndPort := ip + ":4000"
		netMutex.Lock()
		p, ok := outSockets[ffIpAndPort]
		if !ok {
			// Can't do anything, the client isn't even technically connected.
			netMutex.Unlock()
			continue
		}
		if strings.HasPrefix(s, "i-want-to-play-ffm-udp") || strings.HasPrefix(s, "i-can-play-ffm-udp") {
			if p.sleepMode {
				log.Printf("%s - woke up\n", ffIpAndPort)
				p.sleepMode = false
			}
		} else if strings.HasPrefix(s, "i-cannot-play-ffm-udp") {
			if !p.sleepMode {
				log.Printf("%s - went to sleep\n", ffIpAndPort)
				p.sleepMode = true
			}
		}
		outSockets[ffIpAndPort] = p
		netMutex.Unlock()
	}
}

func initNetwork() {
	messageQueue = make(chan networkMessage, 1024) // Buffered channel, 1024 messages.
	outSockets = make(map[string]networkConnection)
	netMutex = &sync.Mutex{}
	refreshConnectedClients()
	go monitorDHCPLeases()
	go messageQueueSender()
	go sleepMonitor()
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"sync"

	log "github.com/funkygao/log4go"
)

type fastListener struct {
	net.Listener
	gw *Gateway
}

func FastListener(gw *Gateway, l net.Listener) net.Listener {
	return &fastListener{
		Listener: l,
		gw:       gw,
	}
}

func (l *fastListener) Accept() (net.Conn, error) {
	c, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}

	log.Trace("%s new conn from %s", l.Listener.Addr(), c.RemoteAddr())
	if l.gw != nil && !options.DisableMetrics {
		l.gw.svrMetrics.TotalConns.Inc(1)
		l.gw.svrMetrics.ConcurrentConns.Inc(1)
	}

	return &fastListenerConn{c, l.gw}, nil
}

type fastListenerConn struct {
	net.Conn
	gw *Gateway
}

func (c *fastListenerConn) Close() error {
	log.Trace("%s closing conn from %s", c.Conn.LocalAddr(), c.Conn.RemoteAddr())

	err := c.Conn.Close()
	if c.gw != nil && !options.DisableMetrics {
		c.gw.svrMetrics.ConcurrentConns.Dec(1)
	}
	return err
}

type limitListener struct {
	net.Listener
	throttler chan struct{}
	gw        *Gateway
}

// LimitListener returns a Listener that accepts at most n simultaneous
// connections from the provided Listener.
func LimitListener(gw *Gateway, l net.Listener, n int) net.Listener {
	return &limitListener{l, make(chan struct{}, n), gw}
}

func (l *limitListener) acquire() { l.throttler <- struct{}{} }
func (l *limitListener) release() { <-l.throttler }

func (l *limitListener) Accept() (net.Conn, error) {
	l.acquire()
	c, err := l.Listener.Accept()
	if err != nil {
		l.release()
		return nil, err
	}

	log.Trace("%s new conn from %s", l.Listener.Addr(), c.RemoteAddr())
	if l.gw != nil && !options.DisableMetrics {
		l.gw.svrMetrics.TotalConns.Inc(1)
		l.gw.svrMetrics.ConcurrentConns.Inc(1)
	}

	return &limitListenerConn{Conn: c, release: l.release, gw: l.gw}, nil
}

type limitListenerConn struct {
	net.Conn
	releaseOnce sync.Once
	release     func()
	gw          *Gateway
}

func (c *limitListenerConn) Close() error {
	log.Trace("%s closing conn from %s", c.Conn.LocalAddr(), c.Conn.RemoteAddr())

	err := c.Conn.Close()
	if c.gw != nil && !options.DisableMetrics {
		c.gw.svrMetrics.ConcurrentConns.Dec(1)
	}
	c.releaseOnce.Do(c.release)
	return err
}
<commit_msg>cange log level from Trace to Debug<commit_after>package main

import (
	"net"
	"sync"

	log "github.com/funkygao/log4go"
)

type fastListener struct {
	net.Listener
	gw *Gateway
}

func FastListener(gw *Gateway, l net.Listener) net.Listener {
	return &fastListener{
		Listener: l,
		gw:       gw,
	}
}

func (l *fastListener) Accept() (net.Conn, error) {
	c, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}

	log.Debug("%s new conn from %s", l.Listener.Addr(), c.RemoteAddr())
	if l.gw != nil && !options.DisableMetrics {
		l.gw.svrMetrics.TotalConns.Inc(1)
		l.gw.svrMetrics.ConcurrentConns.Inc(1)
	}

	return &fastListenerConn{c, l.gw}, nil
}

type fastListenerConn struct {
	net.Conn
	gw *Gateway
}

func (c *fastListenerConn) Close() error {
	log.Debug("%s closing conn from %s", c.Conn.LocalAddr(), c.Conn.RemoteAddr())

	err := c.Conn.Close()
	if c.gw != nil && !options.DisableMetrics {
		c.gw.svrMetrics.ConcurrentConns.Dec(1)
	}
	return err
}

type limitListener struct {
	net.Listener
	throttler chan struct{}
	gw        *Gateway
}

// LimitListener returns a Listener that accepts at most n simultaneous
// connections from the provided Listener.
func LimitListener(gw *Gateway, l net.Listener, n int) net.Listener {
	return &limitListener{l, make(chan struct{}, n), gw}
}

func (l *limitListener) acquire() { l.throttler <- struct{}{} }
func (l *limitListener) release() { <-l.throttler }

func (l *limitListener) Accept() (net.Conn, error) {
	l.acquire()
	c, err := l.Listener.Accept()
	if err != nil {
		l.release()
		return nil, err
	}

	log.Debug("%s new conn from %s", l.Listener.Addr(), c.RemoteAddr())
	if l.gw != nil && !options.DisableMetrics {
		l.gw.svrMetrics.TotalConns.Inc(1)
		l.gw.svrMetrics.ConcurrentConns.Inc(1)
	}

	return &limitListenerConn{Conn: c, release: l.release, gw: l.gw}, nil
}

type limitListenerConn struct {
	net.Conn
	releaseOnce sync.Once
	release     func()
	gw          *Gateway
}

func (c *limitListenerConn) Close() error {
	log.Debug("%s closing conn from %s", c.Conn.LocalAddr(), c.Conn.RemoteAddr())

	err := c.Conn.Close()
	if c.gw != nil && !options.DisableMetrics {
		c.gw.svrMetrics.ConcurrentConns.Dec(1)
	}
	c.releaseOnce.Do(c.release)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/mashiike/didumean"

	ktail "github.com/fujiwara/kinesis-tailf"
)

func main() {
	if err := _main(); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func _main() error {
	var region, streamName, partitionKey string
	var appendLF bool

	flag.BoolVar(&appendLF, "lf", false, "append LF(\\n) to each record")
	flag.StringVar(&streamName, "stream", "", "stream name")
	flag.StringVar(&partitionKey, "partition-key", "", "partition key")
	flag.StringVar(&region, "region", os.Getenv("AWS_REGION"), "region")
	didumean.Parse()

	if streamName == "" {
		fmt.Fprintln(os.Stderr, "Usage of kinesis-cat:")
		flag.PrintDefaults()
		return nil
	}

	var sess *session.Session
	if region != "" {
		sess = session.New(
			&aws.Config{Region: aws.String(region)},
		)
	} else {
		sess = session.New()
	}

	ctx := context.Background()
	app := ktail.New(sess, streamName)
	app.AppendLF = appendLF

	if len(flag.Args()) == 0 {
		return app.Cat(ctx, partitionKey, os.Stdin)
	}
	for _, f := range flag.Args() {
		src, err := os.Open(f)
		if err != nil {
			return err
		}
		if err := app.Cat(ctx, partitionKey, src); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>args - as stdin<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/mashiike/didumean"

	ktail "github.com/fujiwara/kinesis-tailf"
)

func main() {
	if err := _main(); err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func _main() error {
	var region, streamName, partitionKey string
	var appendLF bool

	flag.BoolVar(&appendLF, "lf", false, "append LF(\\n) to each record")
	flag.StringVar(&streamName, "stream", "", "stream name")
	flag.StringVar(&partitionKey, "partition-key", "", "partition key")
	flag.StringVar(&region, "region", os.Getenv("AWS_REGION"), "region")
	didumean.Parse()

	if streamName == "" {
		fmt.Fprintln(os.Stderr, "Usage of kinesis-cat:")
		flag.PrintDefaults()
		return nil
	}

	var sess *session.Session
	if region != "" {
		sess = session.New(
			&aws.Config{Region: aws.String(region)},
		)
	} else {
		sess = session.New()
	}

	ctx := context.Background()
	app := ktail.New(sess, streamName)
	app.AppendLF = appendLF

	if len(flag.Args()) == 0 {
		return app.Cat(ctx, partitionKey, os.Stdin)
	}
	for _, f := range flag.Args() {
		var src io.ReadCloser
		var err error
		if f == "-" {
			src = os.Stdin
		} else if src, err = os.Open(f); err != nil {
			return err
		}
		if err := app.Cat(ctx, partitionKey, src); err != nil {
			return err
		}
		src.Close()
	}
	return nil
}
<|endoftext|>