<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/http/httputil"
	"os"
	"path"
	"strings"

	"github.com/go-swagger/go-swagger/client"
	"github.com/go-swagger/go-swagger/httpkit"
	"github.com/go-swagger/go-swagger/spec"
	"github.com/go-swagger/go-swagger/strfmt"
	"github.com/go-swagger/go-swagger/swag"
)

// Runtime represents an API client that uses the transport
// to make http requests based on a swagger specification.
type Runtime struct {
	DefaultMediaType      string
	DefaultAuthentication client.AuthInfoWriter
	Consumers             map[string]httpkit.Consumer
	Producers             map[string]httpkit.Producer

	Transport http.RoundTripper
	Spec      *spec.Document
	Host      string
	BasePath  string
	Formats   strfmt.Registry
	Debug     bool

	client          *http.Client
	methodsAndPaths map[string]methodAndPath
}

// New creates a new default runtime for a swagger api client.
func New(swaggerSpec *spec.Document) *Runtime {
	var rt Runtime
	rt.DefaultMediaType = httpkit.JSONMime

	// TODO: actually infer this stuff from the spec
	rt.Consumers = map[string]httpkit.Consumer{
		httpkit.JSONMime: httpkit.JSONConsumer(),
	}
	rt.Producers = map[string]httpkit.Producer{
		httpkit.JSONMime: httpkit.JSONProducer(),
	}
	rt.Spec = swaggerSpec
	rt.Transport = http.DefaultTransport
	rt.client = http.DefaultClient
	rt.client.Transport = rt.Transport
	rt.Host = swaggerSpec.Host()
	rt.BasePath = swaggerSpec.BasePath()
	if !strings.HasPrefix(rt.BasePath, "/") {
		rt.BasePath = "/" + rt.BasePath
	}
	rt.Debug = os.Getenv("DEBUG") == "1"
	schemes := swaggerSpec.Spec().Schemes
	if len(schemes) == 0 {
		schemes = append(schemes, "http")
	}
	rt.methodsAndPaths = make(map[string]methodAndPath)
	for mth, pathItem := range rt.Spec.Operations() {
		for pth, op := range pathItem {
			nm := ensureUniqueName(op.ID, mth, pth, rt.methodsAndPaths)
			op.ID = nm
			if len(op.Schemes) > 0 {
				rt.methodsAndPaths[nm] = methodAndPath{mth, pth, op.Schemes}
			} else {
				rt.methodsAndPaths[nm] = methodAndPath{mth, pth, schemes}
			}
		}
	}
	return &rt
}

var namesCounter int64

func ensureUniqueName(key, method, path string, operations map[string]methodAndPath) string {
	nm := key
	if nm == "" {
		nm = swag.ToGoName(strings.ToLower(method) + " " + path)
	}
	_, found := operations[nm]
	if found {
		namesCounter++
		return fmt.Sprintf("%s%d", nm, namesCounter)
	}
	return nm
}

// Submit a request and when there is a body on success it will turn that into the result
// all other things are turned into an api error for swagger which retains the status code
func (r *Runtime) Submit(context *client.Operation) (interface{}, error) {
	operationID, params, readResponse, auth := context.ID, context.Params, context.Reader, context.AuthInfo
	mthPth, ok := r.methodsAndPaths[operationID]
	if !ok {
		return nil, fmt.Errorf("unknown operation: %q", operationID)
	}
	request, err := newRequest(mthPth.Method, mthPth.PathPattern, params)
	if err != nil {
		return nil, err
	}

	// TODO: infer most appropriate content type
	request.SetHeaderParam(httpkit.HeaderContentType, r.DefaultMediaType)
	var accept []string
	for k := range r.Consumers {
		accept = append(accept, k)
	}

	request.SetHeaderParam(httpkit.HeaderAccept, accept...)

	if auth == nil && r.DefaultAuthentication != nil {
		auth = r.DefaultAuthentication
	}
	if auth != nil {
		if err := auth.AuthenticateRequest(request, r.Formats); err != nil {
			return nil, err
		}
	}

	req, err := request.BuildHTTP(r.Producers[r.DefaultMediaType], r.Formats)

	// set the scheme
	if req.URL.Scheme == "" {
		req.URL.Scheme = "http"
	}
	schLen := len(mthPth.Schemes)
	if schLen > 0 {
		scheme := mthPth.Schemes[0]
		// prefer https, but skip when not possible
		if scheme != "https" && schLen > 1 {
			for _, sch := range mthPth.Schemes {
				if sch == "https" {
					scheme = sch
					break
				}
			}
		}
		req.URL.Scheme = scheme
	}

	req.URL.Host = r.Host
	req.URL.Path = path.Join(r.BasePath, req.URL.Path)
	if err != nil {
		return nil, err
	}

	r.client.Transport = r.Transport
	if r.Debug {
		b, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	res, err := r.client.Do(req) // make requests, by default follows 10 redirects before failing
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if r.Debug {
		b, err := httputil.DumpResponse(res, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	ct := res.Header.Get(httpkit.HeaderContentType)
	if ct == "" { // this should really really never occur
		ct = r.DefaultMediaType
	}

	mt, _, err := mime.ParseMediaType(ct)
	if err != nil && err != io.EOF {
		return nil, fmt.Errorf("parse content type: %s", err)
	}
	cons, ok := r.Consumers[mt]
	if !ok {
		// scream about not knowing what to do
		return nil, fmt.Errorf("no consumer: %q", ct)
	}
	return readResponse.ReadResponse(response{res}, cons)
}
<commit_msg>stop the crashing in #197<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"mime"
	"net/http"
	"net/http/httputil"
	"os"
	"path"
	"strings"

	"github.com/go-swagger/go-swagger/client"
	"github.com/go-swagger/go-swagger/httpkit"
	"github.com/go-swagger/go-swagger/spec"
	"github.com/go-swagger/go-swagger/strfmt"
	"github.com/go-swagger/go-swagger/swag"
)

// Runtime represents an API client that uses the transport
// to make http requests based on a swagger specification.
type Runtime struct {
	DefaultMediaType      string
	DefaultAuthentication client.AuthInfoWriter
	Consumers             map[string]httpkit.Consumer
	Producers             map[string]httpkit.Producer

	Transport http.RoundTripper
	Spec      *spec.Document
	Host      string
	BasePath  string
	Formats   strfmt.Registry
	Debug     bool

	client          *http.Client
	methodsAndPaths map[string]methodAndPath
}

// New creates a new default runtime for a swagger api client.
func New(swaggerSpec *spec.Document) *Runtime {
	var rt Runtime
	rt.DefaultMediaType = httpkit.JSONMime

	// TODO: actually infer this stuff from the spec
	rt.Consumers = map[string]httpkit.Consumer{
		httpkit.JSONMime: httpkit.JSONConsumer(),
	}
	rt.Producers = map[string]httpkit.Producer{
		httpkit.JSONMime: httpkit.JSONProducer(),
	}
	rt.Spec = swaggerSpec
	rt.Transport = http.DefaultTransport
	rt.client = http.DefaultClient
	rt.client.Transport = rt.Transport
	rt.Host = swaggerSpec.Host()
	rt.BasePath = swaggerSpec.BasePath()
	if !strings.HasPrefix(rt.BasePath, "/") {
		rt.BasePath = "/" + rt.BasePath
	}
	rt.Debug = os.Getenv("DEBUG") == "1"
	schemes := swaggerSpec.Spec().Schemes
	if len(schemes) == 0 {
		schemes = append(schemes, "http")
	}
	rt.methodsAndPaths = make(map[string]methodAndPath)
	for mth, pathItem := range rt.Spec.Operations() {
		for pth, op := range pathItem {
			nm := ensureUniqueName(op.ID, mth, pth, rt.methodsAndPaths)
			op.ID = nm
			if len(op.Schemes) > 0 {
				rt.methodsAndPaths[nm] = methodAndPath{mth, pth, op.Schemes}
			} else {
				rt.methodsAndPaths[nm] = methodAndPath{mth, pth, schemes}
			}
		}
	}
	return &rt
}

var namesCounter int64

func ensureUniqueName(key, method, path string, operations map[string]methodAndPath) string {
	nm := key
	if nm == "" {
		nm = swag.ToGoName(strings.ToLower(method) + " " + path)
	}
	_, found := operations[nm]
	if found {
		namesCounter++
		return fmt.Sprintf("%s%d", nm, namesCounter)
	}
	return nm
}

// Submit a request and when there is a body on success it will turn that into the result
// all other things are turned into an api error for swagger which retains the status code
func (r *Runtime) Submit(context *client.Operation) (interface{}, error) {
	operationID, params, readResponse, auth := context.ID, context.Params, context.Reader, context.AuthInfo
	mthPth, ok := r.methodsAndPaths[operationID]
	if !ok {
		return nil, fmt.Errorf("unknown operation: %q", operationID)
	}
	request, err := newRequest(mthPth.Method, mthPth.PathPattern, params)
	if err != nil {
		return nil, err
	}

	// TODO: infer most appropriate content type
	request.SetHeaderParam(httpkit.HeaderContentType, r.DefaultMediaType)
	var accept []string
	for k := range r.Consumers {
		accept = append(accept, k)
	}

	request.SetHeaderParam(httpkit.HeaderAccept, accept...)

	if auth == nil && r.DefaultAuthentication != nil {
		auth = r.DefaultAuthentication
	}
	if auth != nil {
		if err := auth.AuthenticateRequest(request, r.Formats); err != nil {
			return nil, err
		}
	}

	req, err := request.BuildHTTP(r.Producers[r.DefaultMediaType], r.Formats)

	// set the scheme
	if req.URL.Scheme == "" {
		req.URL.Scheme = "http"
	}
	schLen := len(mthPth.Schemes)
	if schLen > 0 {
		scheme := mthPth.Schemes[0]
		// prefer https, but skip when not possible
		if scheme != "https" && schLen > 1 {
			for _, sch := range mthPth.Schemes {
				if sch == "https" {
					scheme = sch
					break
				}
			}
		}
		req.URL.Scheme = scheme
	}

	req.URL.Host = r.Host
	req.URL.Path = path.Join(r.BasePath, req.URL.Path)
	if err != nil {
		return nil, err
	}

	r.client.Transport = r.Transport
	if r.Debug {
		b, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	res, err := r.client.Do(req) // make requests, by default follows 10 redirects before failing
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if r.Debug {
		b, err := httputil.DumpResponse(res, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	ct := res.Header.Get(httpkit.HeaderContentType)
	if ct == "" { // this should really really never occur
		ct = r.DefaultMediaType
	}

	mt, _, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("parse content type: %s", err)
	}
	cons, ok := r.Consumers[mt]
	if !ok {
		// scream about not knowing what to do
		return nil, fmt.Errorf("no consumer: %q", ct)
	}
	return readResponse.ReadResponse(response{res}, cons)
}
<|endoftext|>


<commit_before>package messages

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/opentable/sous/util/logging"
	"github.com/opentable/sous/util/logging/constants"
)

// HTTPLogEntry struct to hold log entry messages
type HTTPLogEntry struct {
	logging.CallerInfo
	logging.Level
	message        string
	serverSide     bool
	isResponse     bool
	resourceFamily string
	method         string
	url            string
	server         string
	path           string
	parms          string
	status         int
	requestSize    int64
	responseSize   int64
	dur            time.Duration
}

var httpLogFile = ioutil.Discard

func init() {
	if test, debug := flag.CommandLine.Lookup("test.count"), os.Getenv("SOUS_DEBUG_SERVER"); test != nil || debug != "" {
		f, err := ioutil.TempFile("", "http-log")
		if err != nil {
			panic(err)
		}
		fmt.Fprintf(os.Stderr, "Logging HTTP bodies to %q\n", f.Name())
		httpLogFile = f
	}
}

// ReportClientHTTPRequest reports a response recieved by Sous as a client.
// n.b. this interface subject to change
func ReportClientHTTPRequest(logger logging.LogSink, message string, rq *http.Request, resName string) {
	m := buildHTTPLogMessage(false, false, message, rq, 0, 0, resName, 0)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportClientHTTPResponse reports a response recieved by Sous as a client.
func ReportClientHTTPResponse(logger logging.LogSink, message string, rz *http.Response, resName string, dur time.Duration) {
	fmt.Fprintf(httpLogFile, "\n\n=== %s %s -> %s ===\n\n", rz.Request.Method, rz.Request.URL.String(), rz.Status)
	if rz.Body != nil {
		b := &bytes.Buffer{}
		tee := io.TeeReader(rz.Body, b)
		io.Copy(httpLogFile, tee)
		rz.Body = ioutil.NopCloser(b)
	}

	m := buildHTTPLogMessage(false, true, message, rz.Request, rz.StatusCode, rz.ContentLength, resName, dur)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportServerHTTPRequest reports a response recieved by Sous as a client.
// n.b. this interface subject to change
func ReportServerHTTPRequest(logger logging.LogSink, message string, rq *http.Request, resName string) {
	m := buildHTTPLogMessage(true, false, message, rq, 0, 0, resName, 0)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportServerHTTPResponse reports a response recieved by Sous as a client.
func ReportServerHTTPResponse(logger logging.LogSink, message string, rz *http.Response, resName string, dur time.Duration) {
	m := buildHTTPLogMessage(true, true, message, rz.Request, rz.StatusCode, rz.ContentLength, resName, dur)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportServerHTTPResponding reports a response to a request - this is useful in cases where a ResponseWriter is encapsulating the actual response.
func ReportServerHTTPResponding(logger logging.LogSink, message string, req *http.Request, status int, responseContentLength int64, resName string, dur time.Duration) {
	m := buildHTTPLogMessage(true, true, message, req, status, responseContentLength, resName, dur)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

func buildHTTPLogMessage(
	server, response bool,
	message string,
	rq *http.Request,
	statusCode int,
	responseContentLength int64,
	resName string,
	dur time.Duration,
) *HTTPLogEntry {
	url := rq.URL

	qps := map[string]string{}
	for k, v := range url.Query() {
		qps[k] = strings.Join(v, ",")
	}

	m := newHTTPLogEntry(
		message,
		server,
		response,
		resName,
		rq.Method,
		url.String(),
		statusCode,
		rq.ContentLength,
		responseContentLength,
		dur,
	)
	m.ExcludeMe()
	m.ExcludePathPattern("github.com/opentable/swaggering")     // XXX should be more local
	m.ExcludePathPattern("github.com/opentable/go-singularity") // XXX should be more local
	return m
}

func newHTTPLogEntry(
	message string,
	server, response bool,
	resName, method, urlstring string,
	status int,
	rqSize, rzSize int64,
	dur time.Duration,
) *HTTPLogEntry {
	u, err := url.Parse(urlstring)
	if err != nil {
		u = &url.URL{}
	}

	lvl := logging.InformationLevel
	if status < 400 {
		lvl = logging.ExtraDebug1Level
	}

	return &HTTPLogEntry{
		Level:      lvl,
		CallerInfo: logging.GetCallerInfo(logging.NotHere()),

		message:        message,
		serverSide:     server,
		isResponse:     response,
		resourceFamily: resName,
		method:         method,
		url:            urlstring,
		server:         u.Host,
		path:           u.Path,
		parms:          u.RawQuery,
		status:         status,
		requestSize:    rqSize,
		responseSize:   rzSize,
		dur:            dur,
	}
}

// MetricsTo function to send metrics to graphite
func (msg *HTTPLogEntry) MetricsTo(metrics logging.MetricsSink) {
	side := "client"
	if msg.serverSide {
		side = "server"
	}
	host := strings.Replace(msg.server, ".", "_", -1)

	nameGen := func(pat string, fields ...interface{}) []string {
		base := fmt.Sprintf("%s-%s-%s", side, msg.method, pat)
		if len(fields) > 0 {
			base = fmt.Sprintf(base, fields...)
		}

		// put the metrics into 3 buckets:
		// by "resource family" - a way of aggregating URLs
		// by hostname (with s/./_/g for graphite's comfort)
		// by the pair of (resource, host)
		return []string{
			fmt.Sprintf("%s.%s", base, msg.resourceFamily),
			fmt.Sprintf("%s.%s", base, host),
			fmt.Sprintf("%s.%s.%s", base, host, msg.resourceFamily),
		}
	}

	for _, name := range nameGen("http-request-duration") {
		metrics.UpdateTimer(name, msg.dur)
	}
	for _, name := range nameGen("http-status.%d", msg.status) {
		metrics.IncCounter(name, 1)
	}

	for _, name := range nameGen("http-request-size") {
		metrics.UpdateSample(name, msg.requestSize)
	}

	for _, name := range nameGen("http-response-size") {
		metrics.UpdateSample(name, msg.responseSize)
	}
}

// EachField to populate the proper fields from message
func (msg *HTTPLogEntry) EachField(f logging.FieldReportFn) {
	msg.EachFieldWithoutCallerInfo(f)
	msg.CallerInfo.EachField(f)
}

// EachFieldWithoutCallerInfo allows sub messages to populate
// logging.FieldReportFn without out having to call CallerInfo
func (msg *HTTPLogEntry) EachFieldWithoutCallerInfo(f logging.FieldReportFn) {
	// Could be simpler, but this is a precursor to logging the "isResponse" field
	incoming := (msg.serverSide && !msg.isResponse) || (!msg.serverSide && msg.isResponse)
	f("@loglov3-otl", constants.SousHttpV1)
	f("resource-family", msg.resourceFamily)
	f("incoming", incoming)
	f("method", msg.method)
	f("status", msg.status)
	f("duration", int64(msg.dur/time.Microsecond))

	f("body-size", msg.requestSize)
	// body?
	f("response-size", msg.responseSize)
	// response-body?

	f("url", msg.url)
	f("url-hostname", msg.server)
	f("url-pathname", msg.path)
	f("url-querystring", msg.parms)
}

// Status method to retrieve status from message
func (msg *HTTPLogEntry) Status() int {
	return msg.status
}

// Message retrieve message from log message
func (msg *HTTPLogEntry) Message() string {
	var channelName string
	if msg.serverSide {
		if msg.isResponse {
			channelName = "<- Server "
		} else {
			channelName = "-> Server "
		}
	} else {
		if msg.isResponse {
			channelName = "Client <- "
		} else {
			channelName = "Client -> "
		}
	}
	return channelName + msg.message
}
<commit_msg>Cleaning up HTTP tests<commit_after>package messages

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/opentable/sous/util/logging"
	"github.com/opentable/sous/util/logging/constants"
)

// HTTPLogEntry struct to hold log entry messages
type HTTPLogEntry struct {
	logging.CallerInfo
	logging.Level
	message        string
	serverSide     bool
	isResponse     bool
	resourceFamily string
	method         string
	url            string
	server         string
	path           string
	parms          string
	status         int
	requestSize    int64
	responseSize   int64
	dur            time.Duration
}

var httpLogFile = ioutil.Discard

func init() {
	if test, debug := flag.CommandLine.Lookup("test.count"), os.Getenv("SOUS_DEBUG_SERVER"); test != nil || debug != "" {
		f, err := ioutil.TempFile("", "http-log")
		if err != nil {
			panic(err)
		}
		fmt.Fprintf(os.Stderr, "Logging HTTP bodies to %q\n", f.Name())
		httpLogFile = f
	}
}

// ReportClientHTTPRequest reports a response recieved by Sous as a client.
// n.b. this interface subject to change
func ReportClientHTTPRequest(logger logging.LogSink, message string, rq *http.Request, resName string) {
	m := buildHTTPLogMessage(false, false, message, rq, 0, 0, resName, 0)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportClientHTTPResponse reports a response recieved by Sous as a client.
func ReportClientHTTPResponse(logger logging.LogSink, message string, rz *http.Response, resName string, dur time.Duration) {
	fmt.Fprintf(httpLogFile, "\n\n=== %s %s -> %s ===\n\n", rz.Request.Method, rz.Request.URL.String(), rz.Status)
	if rz.Body != nil {
		b := &bytes.Buffer{}
		tee := io.TeeReader(rz.Body, b)
		io.Copy(httpLogFile, tee)
		rz.Body = ioutil.NopCloser(b)
	}

	m := buildHTTPLogMessage(false, true, message, rz.Request, rz.StatusCode, rz.ContentLength, resName, dur)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportServerHTTPRequest reports a response recieved by Sous as a client.
// n.b. this interface subject to change
func ReportServerHTTPRequest(logger logging.LogSink, message string, rq *http.Request, resName string) {
	m := buildHTTPLogMessage(true, false, message, rq, 0, 0, resName, 0)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportServerHTTPResponse reports a response recieved by Sous as a client.
func ReportServerHTTPResponse(logger logging.LogSink, message string, rz *http.Response, resName string, dur time.Duration) {
	m := buildHTTPLogMessage(true, true, message, rz.Request, rz.StatusCode, rz.ContentLength, resName, dur)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

// ReportServerHTTPResponding reports a response to a request - this is useful in cases where a ResponseWriter is encapsulating the actual response.
func ReportServerHTTPResponding(logger logging.LogSink, message string, req *http.Request, status int, responseContentLength int64, resName string, dur time.Duration) {
	m := buildHTTPLogMessage(true, true, message, req, status, responseContentLength, resName, dur)
	m.ExcludeMe()
	logging.Deliver(m, logger)
}

func buildHTTPLogMessage(
	server, response bool,
	message string,
	rq *http.Request,
	statusCode int,
	responseContentLength int64,
	resName string,
	dur time.Duration,
) *HTTPLogEntry {
	url := rq.URL

	qps := map[string]string{}
	for k, v := range url.Query() {
		qps[k] = strings.Join(v, ",")
	}

	method := rq.Method
	urlstring := url.String()
	rqSize := rq.ContentLength

	lvl := logging.InformationLevel
	if statusCode < 300 {
		lvl = logging.ExtraDebug1Level
	}

	ci := logging.GetCallerInfo(logging.NotHere())
	ci.ExcludePathPattern("github.com/opentable/swaggering")     // XXX should be more local
	ci.ExcludePathPattern("github.com/opentable/go-singularity") // XXX should be more local

	return &HTTPLogEntry{
		Level:      lvl,
		CallerInfo: ci,

		message:        message,
		serverSide:     server,
		isResponse:     response,
		resourceFamily: resName,
		method:         method,
		url:            urlstring,
		server:         url.Host,
		path:           url.Path,
		parms:          url.RawQuery,
		status:         statusCode,
		requestSize:    rqSize,
		responseSize:   responseContentLength,
		dur:            dur,
	}
}

// MetricsTo function to send metrics to graphite
func (msg *HTTPLogEntry) MetricsTo(metrics logging.MetricsSink) {
	side := "client"
	if msg.serverSide {
		side = "server"
	}
	host := strings.Replace(msg.server, ".", "_", -1)

	nameGen := func(pat string, fields ...interface{}) []string {
		base := fmt.Sprintf("%s-%s-%s", side, msg.method, pat)
		if len(fields) > 0 {
			base = fmt.Sprintf(base, fields...)
		}

		// put the metrics into 3 buckets:
		// by "resource family" - a way of aggregating URLs
		// by hostname (with s/./_/g for graphite's comfort)
		// by the pair of (resource, host)
		return []string{
			fmt.Sprintf("%s.%s", base, msg.resourceFamily),
			fmt.Sprintf("%s.%s", base, host),
			fmt.Sprintf("%s.%s.%s", base, host, msg.resourceFamily),
		}
	}

	for _, name := range nameGen("http-request-duration") {
		metrics.UpdateTimer(name, msg.dur)
	}
	for _, name := range nameGen("http-status.%d", msg.status) {
		metrics.IncCounter(name, 1)
	}

	for _, name := range nameGen("http-request-size") {
		metrics.UpdateSample(name, msg.requestSize)
	}

	for _, name := range nameGen("http-response-size") {
		metrics.UpdateSample(name, msg.responseSize)
	}
}

// EachField to populate the proper fields from message
func (msg *HTTPLogEntry) EachField(f logging.FieldReportFn) {
	msg.EachFieldWithoutCallerInfo(f)
	msg.CallerInfo.EachField(f)
}

// EachFieldWithoutCallerInfo allows sub messages to populate
// logging.FieldReportFn without out having to call CallerInfo
func (msg *HTTPLogEntry) EachFieldWithoutCallerInfo(f logging.FieldReportFn) {
	// Could be simpler, but this is a precursor to logging the "isResponse" field
	incoming := (msg.serverSide && !msg.isResponse) || (!msg.serverSide && msg.isResponse)
	f("@loglov3-otl", constants.SousHttpV1)
	f("resource-family", msg.resourceFamily)
	f("incoming", incoming)
	f("method", msg.method)
	f("status", msg.status)
	f("duration", int64(msg.dur/time.Microsecond))

	f("body-size", msg.requestSize)
	// body?
	f("response-size", msg.responseSize)
	// response-body?

	f("url", msg.url)
	f("url-hostname", msg.server)
	f("url-pathname", msg.path)
	f("url-querystring", msg.parms)
}

// Status method to retrieve status from message
func (msg *HTTPLogEntry) Status() int {
	return msg.status
}

// Message retrieve message from log message
func (msg *HTTPLogEntry) Message() string {
	var channelName string
	if msg.serverSide {
		if msg.isResponse {
			channelName = "<- Server "
		} else {
			channelName = "-> Server "
		}
	} else {
		if msg.isResponse {
			channelName = "Client <- "
		} else {
			channelName = "Client -> "
		}
	}
	return channelName + msg.message
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/infinityworks/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

// Data is used to store data from all the relevant endpoints in the API
type Data struct {
	Data []struct {
		HealthState string `json:"healthState"`
		Name        string `json:"name"`
		State       string `json:"state"`
		System      bool   `json:"system"`
		Scale       int    `json:"scale"`
		HostName    string `json:"hostname"`
		ID          string `json:"id"`
		StackID     string `json:"stackId"`
		EnvID       string `json:"environmentId"`
		BaseType    string `json:"basetype"`
		Type        string `json:"type"`
		AgentState  string `json:"agentState"`
	} `json:"data"`
}

// processMetrics - Collects the data from the API, returns data object
func (e *Exporter) processMetrics(data *Data, endpoint string, hideSys bool, ch chan<- prometheus.Metric) error {

	// Metrics - range through the data object
	for _, x := range data.Data {

		// If system services have been ignored, the loop simply skips them
		if hideSys == true && x.System == true {
			continue
		}

		// Checks the metric is of the expected type
		dataType := x.BaseType
		if dataType == "" {
			dataType = x.Type
		}
		if checkMetric(endpoint, dataType) == false {
			continue
		}

		log.Debug("Processing metrics for %s", endpoint)

		if endpoint == "hosts" {
			var s = x.HostName
			if x.Name != "" {
				s = x.Name
			}
			if err := e.setHostMetrics(s, x.State, x.AgentState); err != nil {
				log.Errorf("Error processing host metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, [agent] %s ", x.HostName, x.State, x.AgentState)

				continue
			}

		} else if endpoint == "stacks" {

			// Used to create a map of stackID and stackName
			// Later used as a dimension in service metrics
			stackRef = storeStackRef(x.ID, x.Name)

			if err := e.setStackMetrics(x.Name, x.State, x.HealthState, strconv.FormatBool(x.System)); err != nil {
				log.Errorf("Error processing stack metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %t", x.Name, x.State, x.HealthState, x.System)
				continue
			}

		} else if endpoint == "services" {

			// Retrieves the stack Name from the previous values stored.
			var stackName = retrieveStackRef(x.StackID)

			if stackName == "unknown" {
				log.Warnf("Failed to obtain stack_name for %s from the API", x.Name)
			}

			if err := e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale); err != nil {
				log.Errorf("Error processing service metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %s, %d", x.Name, stackName, x.State, x.HealthState, x.Scale)
				continue
			}

			e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale)
		}

	}

	return nil
}

// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics
func (e *Exporter) gatherData(rancherURL string, accessKey string, secretKey string, endpoint string, ch chan<- prometheus.Metric) (*Data, error) {

	// Return the correct URL path
	url := setEndpoint(rancherURL, endpoint)

	// Create new data slice from Struct
	var data = new(Data)

	// Scrape EndPoint for JSON Data
	err := getJSON(url, accessKey, secretKey, &data)
	if err != nil {
		log.Error("Error getting JSON from endpoint ", endpoint)
		return nil, err
	}
	log.Debugf("JSON Fetched for: "+endpoint+": ", data)

	return data, err
}

// getJSON return json from server, return the formatted JSON
func getJSON(url string, accessKey string, secretKey string, target interface{}) error {

	start := time.Now()

	// Counter for internal exporter metrics
	measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "main", "fnc": "getJSON"}).Inc()

	log.Info("Scraping: ", url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	req.SetBasicAuth(accessKey, secretKey)
	resp, err := client.Do(req)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	respFormatted := json.NewDecoder(resp.Body).Decode(target)

	// Timings recorded as part of internal metrics
	elapsed := float64((time.Since(start)) / time.Microsecond)
	measure.FunctionDurations.WithLabelValues("main", "getJSON").Observe(elapsed)

	// Close the response body, the underlying Transport should then close the connection.
	resp.Body.Close()

	// return formatted JSON
	return respFormatted
}

// setEndpoint - Determines the correct URL endpoint to use, gives us backwards compatibility
func setEndpoint(rancherURL string, component string) string {

	var endpoint string

	endpoint = (rancherURL + "/" + component + "/")
    endpoint = strings.Replace(endpoint, "v1", "v2-beta", 1)

	return endpoint
}

// storeStackRef stores the stackID and stack name for use as a label elsewhere
func storeStackRef(stackID string, stackName string) map[string]string {

	stackRef[stackID] = stackName

	return stackRef
}

// retrieveStackRef returns the stack name, when sending the stackID
func retrieveStackRef(stackID string) string {

	for key, value := range stackRef {
		if stackID == "" {
			return "unknown"
		} else if stackID == key {
			log.Debugf("StackRef - Key is %s, Value is %s StackID is %s", key, value, stackID)
			return value
		}
	}
	// returns unknown if no match was found
	return "unknown"
}
<commit_msg>Return error when key or secret for Rancher API is wrong.<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/infinityworks/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

// Data is used to store data from all the relevant endpoints in the API
type Data struct {
	Data []struct {
		HealthState string `json:"healthState"`
		Name        string `json:"name"`
		State       string `json:"state"`
		System      bool   `json:"system"`
		Scale       int    `json:"scale"`
		HostName    string `json:"hostname"`
		ID          string `json:"id"`
		StackID     string `json:"stackId"`
		EnvID       string `json:"environmentId"`
		BaseType    string `json:"basetype"`
		Type        string `json:"type"`
		AgentState  string `json:"agentState"`
	} `json:"data"`
}

// processMetrics - Collects the data from the API, returns data object
func (e *Exporter) processMetrics(data *Data, endpoint string, hideSys bool, ch chan<- prometheus.Metric) error {

	// Metrics - range through the data object
	for _, x := range data.Data {

		// If system services have been ignored, the loop simply skips them
		if hideSys == true && x.System == true {
			continue
		}

		// Checks the metric is of the expected type
		dataType := x.BaseType
		if dataType == "" {
			dataType = x.Type
		}
		if checkMetric(endpoint, dataType) == false {
			continue
		}

		log.Debug("Processing metrics for %s", endpoint)

		if endpoint == "hosts" {
			var s = x.HostName
			if x.Name != "" {
				s = x.Name
			}
			if err := e.setHostMetrics(s, x.State, x.AgentState); err != nil {
				log.Errorf("Error processing host metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, [agent] %s ", x.HostName, x.State, x.AgentState)

				continue
			}

		} else if endpoint == "stacks" {

			// Used to create a map of stackID and stackName
			// Later used as a dimension in service metrics
			stackRef = storeStackRef(x.ID, x.Name)

			if err := e.setStackMetrics(x.Name, x.State, x.HealthState, strconv.FormatBool(x.System)); err != nil {
				log.Errorf("Error processing stack metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %t", x.Name, x.State, x.HealthState, x.System)
				continue
			}

		} else if endpoint == "services" {

			// Retrieves the stack Name from the previous values stored.
			var stackName = retrieveStackRef(x.StackID)

			if stackName == "unknown" {
				log.Warnf("Failed to obtain stack_name for %s from the API", x.Name)
			}

			if err := e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale); err != nil {
				log.Errorf("Error processing service metrics: %s", err)
				log.Errorf("Attempt Failed to set %s, %s, %s, %s, %d", x.Name, stackName, x.State, x.HealthState, x.Scale)
				continue
			}

			e.setServiceMetrics(x.Name, stackName, x.State, x.HealthState, x.Scale)
		}

	}

	return nil
}

// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics
func (e *Exporter) gatherData(rancherURL string, accessKey string, secretKey string, endpoint string, ch chan<- prometheus.Metric) (*Data, error) {

	// Return the correct URL path
	url := setEndpoint(rancherURL, endpoint)

	// Create new data slice from Struct
	var data = new(Data)

	// Scrape EndPoint for JSON Data
	err := getJSON(url, accessKey, secretKey, &data)
	if err != nil {
		log.Error("Error getting JSON from endpoint ", endpoint)
		return nil, err
	}
	log.Debugf("JSON Fetched for: "+endpoint+": ", data)

	return data, err
}

// getJSON return json from server, return the formatted JSON
func getJSON(url string, accessKey string, secretKey string, target interface{}) error {

	start := time.Now()

	// Counter for internal exporter metrics
	measure.FunctionCountTotal.With(prometheus.Labels{"pkg": "main", "fnc": "getJSON"}).Inc()

	log.Info("Scraping: ", url)

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	req.SetBasicAuth(accessKey, secretKey)
	resp, err := client.Do(req)

	if err != nil {
		log.Error("Error Collecting JSON from API: ", err)
	}

	if resp.StatusCode != 200 {
		log.Error("Error Collecting JSON from API: ", resp.Status)
	}

	respFormatted := json.NewDecoder(resp.Body).Decode(target)

	// Timings recorded as part of internal metrics
	elapsed := float64((time.Since(start)) / time.Microsecond)
	measure.FunctionDurations.WithLabelValues("main", "getJSON").Observe(elapsed)

	// Close the response body, the underlying Transport should then close the connection.
	resp.Body.Close()

	// return formatted JSON
	return respFormatted
}

// setEndpoint - Determines the correct URL endpoint to use, gives us backwards compatibility
func setEndpoint(rancherURL string, component string) string {

	var endpoint string

	endpoint = (rancherURL + "/" + component + "/")
	endpoint = strings.Replace(endpoint, "v1", "v2-beta", 1)

	return endpoint
}

// storeStackRef stores the stackID and stack name for use as a label elsewhere
func storeStackRef(stackID string, stackName string) map[string]string {

	stackRef[stackID] = stackName

	return stackRef
}

// retrieveStackRef returns the stack name, when sending the stackID
func retrieveStackRef(stackID string) string {

	for key, value := range stackRef {
		if stackID == "" {
			return "unknown"
		} else if stackID == key {
			log.Debugf("StackRef - Key is %s, Value is %s StackID is %s", key, value, stackID)
			return value
		}
	}
	// returns unknown if no match was found
	return "unknown"
}
<|endoftext|>


<commit_before>package junos

// To View the entire configuration, use the keyword "full" for the first
// argument. If anything else outside of "full" is specified, it will return
// the configuration of the specified top-level stanza only. So "security"
// would return everything under the "security" stanza.
func ExampleJunos_viewConfiguration() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// Output format can be "text" or "xml".
	config, err := jnpr.GetConfig("full", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(config)
}

// Comparing and working with rollback configurations.
func ExampleJunos_rollbackConfigurations() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// If you want to view the difference between the current configuration and a rollback
	// one, then you can use the ConfigDiff() function to specify a previous config:
	diff, err := jnpr.ConfigDiff(3)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(diff)

	// You can rollback to a previous state, or the rescue configuration by using
	// the RollbackConfig() function:
	err := jnpr.RollbackConfig(3)
	if err != nil {
		fmt.Println(err)
	}

	// Create a rescue config from the active configuration.
	jnpr.Rescue("save")

	// You can also delete a rescue config.
	jnpr.Rescue("delete")

	// Rollback to the "rescue" configuration.
	err := jnpr.RollbackConfig("rescue")
	if err != nil {
		fmt.Println(err)
	}
}

// Configuring devices.
func ExampleJunos_configuringDevices() {
	// Use the Config() function to configure a Junos device.

	// When configuring a device, it is good practice to lock the configuration database,
	// load the config, commit the configuration, and then unlock the configuration database.
	// You can do this with the following functions: Lock(), Commit(), Unlock().

	// Multiple ways to commit a configuration.

	// Commit the configuration as normal.
	Commit()

	// Check the configuration for any syntax errors (NOTE: you must still issue a
	// Commit() afterwards).
	CommitCheck()

	// Commit at a later time, i.e. 4:30 PM.
	CommitAt("16:30:00")

	// Rollback configuration if a Commit() is not issued within the given <minutes>.
	CommitConfirm(15)

	// You can configure the Junos device by uploading a local file, or pulling from an
	// FTP/HTTP server. The LoadConfig() function takes three arguments:

	// filename or URL, format, and a boolean (true/false) "commit-on-load".

	// If you specify a URL, it must be in the following format:

	// ftp://<username>:<password>@hostname/pathname/file-name
	// http://<username>:<password>@hostname/pathname/file-name

	// Note: The default value for the FTP path variable is the user’s home directory. Thus,
	// by default the file path to the configuration file is relative to the user directory.
	// To specify an absolute path when using FTP, start the path with the characters %2F;
	// for example: ftp://username:password@hostname/%2Fpath/filename.

	// The format of the commands within the file must be one of the following types:

	// set
	// system name-server 1.1.1.1

	// text
	// system {
	//     name-server 1.1.1.1;
	// }

	// xml
	// <system>
	//     <name-server>
	//         <name>1.1.1.1</name>
	//     </name-server>
	// </system>

	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// Configure a device from a file:

	// If the third option is "true" then after the configuration is loaded, a commit
	// will be issued. If set to "false," you will have to commit the configuration
	// using one of the Commit() functions.
	jnpr.Lock()
	err = jnpr.Config("path-to-file.txt", "set", true)
	if err != nil {
		fmt.Println(err)
	}
	jnpr.Unlock()

	// Configure a device using commands specified in a []string:

	// You can also use the following format also:
	//
	// var configCommands = `
	//     set interfaces ge-0/0/5.0 family inet address 192.168.0.1/24
	//     set security zones security-zone trust interfaces ge-0/0/5.0
	// `
	//
	configCommands := []string{
		"set interfaces ge-0/0/5.0 family inet address 192.168.0.1/24",
		"set security zones security-zone trust interfaces ge-0/0/5.0",
	}
	err = jnpr.Config(configCommands, "set", true)
	if err != nil {
		fmt.Println(err)
	}
}

// Running operational mode commands on a device.
func ExampleJunos_runningCommands() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// You can run operational mode commands such as "show" and "request" by using the
	// Command() function. Output formats can be "text" or "xml".

	// Results returned in text format.
	txtOutput, err := jnpr.RunCommand("show chassis hardware", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(txtOutput)

	// Results returned in XML format.
	xmlOutput, err := jnpr.RunCommand("show chassis hardware", "xml")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(xmlOutput)

	// Reboot the device
	jnpr.Reboot()
}

// Viewing basic information about the device.
func ExampleJunos_deviceInformation() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// When you call the PrintFacts() function, it just prints out the platform
	// and software information to the console.
	jnpr.PrintFacts()

	// You can also loop over the struct field that contains this information yourself:
	fmt.Printf("Hostname: %s", jnpr.Hostname)
	for _, data := range jnpr.Platform {
		fmt.Printf("Model: %s, Version: %s", data.Model, data.Version)
	}
	// Output: Model: SRX240H2, Version: 12.1X47-D10.4
}

// Establishing a connection to Junos Space and working with devices.
func ExampleJunosSpace_devices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Get the list of devices.
	devices, err := space.Devices()
	if err != nil {
		fmt.Println(err)
	}

	// Iterate over our device list and display some information about them.
	for _, device := range devices.Devices {
		fmt.Printf("Name: %s, IP Address: %s, Platform: %s\n", device.Name, device.IP, device.Platform)
	}

	// Add a device to Junos Space.
	jobID, err = space.AddDevice("sdubs-fw", "admin", "juniper123")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(jobID)
	// Output: 1345283

	// Remove a device from Junos Space.
	err = space.RemoveDevice("sdubs-fw")
	if err != nil {
		fmt.Println(err)
	}

	// Resynchronize a device. A good option if you do a lot of configuration to a device
	// outside of Junos Space.
	job, err := space.Resync("firewall-A")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}

// Software upgrades using Junos Space.
func ExampleJunosSpace_softwareUpgrade() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Staging software on a device. The last parameter is whether or not to remove any
	// existing images from the device; boolean.
	//
	// This will not upgrade the device, but only place the image there to be used at a later
	// time.
	jobID, err := space.StageSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", false)
	if err != nil {
		fmt.Println(err)
	}

	// If you want to issue a software upgrade to the device, here's how:

	// Configure our options, such as whether or not to reboot the device, etc.
	options := &junos.SoftwareUpgrade{
		UseDownloaded: true,
		Validate:      false,
		Reboot:        false,
		RebootAfter:   0,
		Cleanup:       false,
		RemoveAfter:   false,
	}

	jobID, err := space.DeploySoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", options)
	if err != nil {
		fmt.Println(err)
	}

	// Remove a staged image from the device.
	jobID, err := space.RemoveStagedSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz")
	if err != nil {
		fmt.Println(err)
	}
}

// Viewing information about Security Director devices (SRX, J-series, etc.).
func ExampleJunosSpace_securityDirectorDevices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security devices:
	devices, err := space.SecurityDevices()
	if err != nil {
		fmt.Println(err)
	}

	for _, device := range devices.Devices {
		fmt.Printf("%+v\n", device)
	}
}

// Working with address and service objects.
func ExampleJunosSpace_addressObjects() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// To view the address and service objects, you use the Addresses() and Services() functions. Both of them
	// take a "filter" parameter, which lets you search for objects matching your filter.

	//If you leave the parameter blank (e.g. ""), or specify "all", then every object is returned.

	// Address objects
	addresses, err := space.Addresses("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, address := range addresses.Addresses {
		fmt.Printf("%+v\n", address)
	}

	// Service objects
	services, err := space.Services("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, service := range services.Services {
		fmt.Printf("%+v\n", service)
	}

	// Add an address group. "true" as the first parameter means that we assume the
	// group is going to be an address group.
	space.AddGroup(true, "Blacklist-IPs", "Blacklisted IP addresses")

	// Add a service group. We do this by specifying "false" as the first parameter.
	space.AddGroup(false, "Web-Protocols", "All web-based protocols and ports")

	// Add an address object
	space.AddAddress("my-laptop", "2.2.2.2", "My personal laptop")

	// Add a network
	space.AddAddress("corporate-users", "192.168.1.0/24", "People on campus")

	// Add a service object with an 1800 second inactivity timeout (using "0" disables this feature)
	space.AddService("udp", "udp-5000", 5000, 5000, "UDP port 5000", 1800)

	// Add a service object with a port range
	space.AddService("tcp", "high-port-range", 40000, 65000, "TCP high ports", 0)

	// If you want to modify an existing object group, you do this with the ModifyObject() function. The
	// first parameter is whether the object is an address group (true) or a service group (false).

	// Add a service to a group
	space.ModifyObject(false, "add", "service-group", "service-name")

	// Remove an address object from a group
	space.ModifyObject(true, "remove", "Whitelisted-Addresses", "bad-ip")

	// Rename an object
	space.ModifyObject(false, "rename", "Web-Services", "Web-Ports")

	// Delete an object
	space.ModifyObject(true, "delete", "my-laptop")
}

// Working with polymorphic (variable) objects.
func ExampleJunosSpace_variables() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Add a variable
	// The parameters are as follows: variable-name, description, default-value
	space.AddVariable("test-variable", "Our test variable", "default-object")

	// Create our session state for modifying variables
	v, err := space.ModifyVariable()
	if err != nil {
		log.Fatal(err)
	}

	// Adding objects to the variable
	v.Add("test-variable", "srx-1", "user-pc")
	v.Add("test-variable", "corp-firewall", "db-server")

	// Delete a variable
	space.DeleteVariable("test-variable")
}

// Working with policies.
func ExampleJunosSpace_policies() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security policies Junos Space manages:
	policies, err := space.Policies()
	if err != nil {
		fmt.Println(err)
	}

	for _, policy := range policies.Policies {
		fmt.Printf("%s\n", policy.Name)
	}

	// For example, say we have been adding and removing objects in a group, and that group
	// is referenced in a firewall policy. Here's how to update the policy:

	// Update the policy. If "false" is specified, then the policy is only published, and the
	// device is not updated.
	job, err := space.PublishPolicy("Internet-Firewall-Policy", true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)

	// Let's update a device knowing that we have some previously published services.
	job, err := space.UpdateDevice("firewall-1.company.com")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}
<commit_msg>Updated examples<commit_after>package junos

import (
	"fmt"
	"github.com/scottdware/go-junos"
	"log"
)

// To View the entire configuration, use the keyword "full" for the first
// argument. If anything else outside of "full" is specified, it will return
// the configuration of the specified top-level stanza only. So "security"
// would return everything under the "security" stanza.
func ExampleJunos_viewConfiguration() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// Output format can be "text" or "xml".
	config, err := jnpr.GetConfig("full", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(config)
}

// Comparing and working with rollback configurations.
func ExampleJunos_rollbackConfigurations() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// If you want to view the difference between the current configuration and a rollback
	// one, then you can use the ConfigDiff() function to specify a previous config:
	diff, err := jnpr.ConfigDiff(3)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(diff)

	// You can rollback to a previous state, or the rescue configuration by using
	// the RollbackConfig() function:
	err := jnpr.RollbackConfig(3)
	if err != nil {
		fmt.Println(err)
	}

	// Create a rescue config from the active configuration.
	jnpr.Rescue("save")

	// You can also delete a rescue config.
	jnpr.Rescue("delete")

	// Rollback to the "rescue" configuration.
	err := jnpr.RollbackConfig("rescue")
	if err != nil {
		fmt.Println(err)
	}
}

// Configuring devices.
func ExampleJunos_configuringDevices() {
	// Use the Config() function to configure a Junos device.

	// When configuring a device, it is good practice to lock the configuration database,
	// load the config, commit the configuration, and then unlock the configuration database.
	// You can do this with the following functions: Lock(), Commit(), Unlock().

	// Multiple ways to commit a configuration.

	// Commit the configuration as normal.
	Commit()

	// Check the configuration for any syntax errors (NOTE: you must still issue a
	// Commit() afterwards).
	CommitCheck()

	// Commit at a later time, i.e. 4:30 PM.
	CommitAt("16:30:00")

	// Rollback configuration if a Commit() is not issued within the given <minutes>.
	CommitConfirm(15)

	// You can configure the Junos device by uploading a local file, or pulling from an
	// FTP/HTTP server. The LoadConfig() function takes three arguments:

	// filename or URL, format, and a boolean (true/false) "commit-on-load".

	// If you specify a URL, it must be in the following format:

	// ftp://<username>:<password>@hostname/pathname/file-name
	// http://<username>:<password>@hostname/pathname/file-name

	// Note: The default value for the FTP path variable is the user’s home directory. Thus,
	// by default the file path to the configuration file is relative to the user directory.
	// To specify an absolute path when using FTP, start the path with the characters %2F;
	// for example: ftp://username:password@hostname/%2Fpath/filename.

	// The format of the commands within the file must be one of the following types:

	// set
	// system name-server 1.1.1.1

	// text
	// system {
	//     name-server 1.1.1.1;
	// }

	// xml
	// <system>
	//     <name-server>
	//         <name>1.1.1.1</name>
	//     </name-server>
	// </system>

	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// Configure a device from a file:

	// If the third option is "true" then after the configuration is loaded, a commit
	// will be issued. If set to "false," you will have to commit the configuration
	// using one of the Commit() functions.
	jnpr.Lock()
	err = jnpr.Config("path-to-file.txt", "set", true)
	if err != nil {
		fmt.Println(err)
	}
	jnpr.Unlock()

	// Configure a device using commands specified in a []string:

	// You can also use the following format also:
	//
	// var configCommands = `
	//     set interfaces ge-0/0/5.0 family inet address 192.168.0.1/24
	//     set security zones security-zone trust interfaces ge-0/0/5.0
	// `
	//
	configCommands := []string{
		"set interfaces ge-0/0/5.0 family inet address 192.168.0.1/24",
		"set security zones security-zone trust interfaces ge-0/0/5.0",
	}
	err = jnpr.Config(configCommands, "set", true)
	if err != nil {
		fmt.Println(err)
	}
}

// Running operational mode commands on a device.
func ExampleJunos_runningCommands() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// You can run operational mode commands such as "show" and "request" by using the
	// Command() function. Output formats can be "text" or "xml".

	// Results returned in text format.
	txtOutput, err := jnpr.RunCommand("show chassis hardware", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(txtOutput)

	// Results returned in XML format.
	xmlOutput, err := jnpr.RunCommand("show chassis hardware", "xml")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(xmlOutput)

	// Reboot the device
	jnpr.Reboot()
}

// Viewing basic information about the device.
func ExampleJunos_deviceInformation() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// When you call the PrintFacts() function, it just prints out the platform
	// and software information to the console.
	jnpr.PrintFacts()

	// You can also loop over the struct field that contains this information yourself:
	fmt.Printf("Hostname: %s", jnpr.Hostname)
	for _, data := range jnpr.Platform {
		fmt.Printf("Model: %s, Version: %s", data.Model, data.Version)
	}
	// Output: Model: SRX240H2, Version: 12.1X47-D10.4
}

// Establishing a connection to Junos Space and working with devices.
func ExampleJunosSpace_devices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Get the list of devices.
	devices, err := space.Devices()
	if err != nil {
		fmt.Println(err)
	}

	// Iterate over our device list and display some information about them.
	for _, device := range devices.Devices {
		fmt.Printf("Name: %s, IP Address: %s, Platform: %s\n", device.Name, device.IP, device.Platform)
	}

	// Add a device to Junos Space.
	jobID, err = space.AddDevice("sdubs-fw", "admin", "juniper123")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(jobID)
	// Output: 1345283

	// Remove a device from Junos Space.
	err = space.RemoveDevice("sdubs-fw")
	if err != nil {
		fmt.Println(err)
	}

	// Resynchronize a device. A good option if you do a lot of configuration to a device
	// outside of Junos Space.
	job, err := space.Resync("firewall-A")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}

// Software upgrades using Junos Space.
func ExampleJunosSpace_softwareUpgrade() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Staging software on a device. The last parameter is whether or not to remove any
	// existing images from the device; boolean.
	//
	// This will not upgrade the device, but only place the image there to be used at a later
	// time.
	jobID, err := space.StageSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", false)
	if err != nil {
		fmt.Println(err)
	}

	// If you want to issue a software upgrade to the device, here's how:

	// Configure our options, such as whether or not to reboot the device, etc.
	options := &junos.SoftwareUpgrade{
		UseDownloaded: true,
		Validate:      false,
		Reboot:        false,
		RebootAfter:   0,
		Cleanup:       false,
		RemoveAfter:   false,
	}

	jobID, err := space.DeploySoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", options)
	if err != nil {
		fmt.Println(err)
	}

	// Remove a staged image from the device.
	jobID, err := space.RemoveStagedSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz")
	if err != nil {
		fmt.Println(err)
	}
}

// Viewing information about Security Director devices (SRX, J-series, etc.).
func ExampleJunosSpace_securityDirectorDevices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security devices:
	devices, err := space.SecurityDevices()
	if err != nil {
		fmt.Println(err)
	}

	for _, device := range devices.Devices {
		fmt.Printf("%+v\n", device)
	}
}

// Working with address and service objects.
func ExampleJunosSpace_addressObjects() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// To view the address and service objects, you use the Addresses() and Services() functions. Both of them
	// take a "filter" parameter, which lets you search for objects matching your filter.

	//If you leave the parameter blank (e.g. ""), or specify "all", then every object is returned.

	// Address objects
	addresses, err := space.Addresses("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, address := range addresses.Addresses {
		fmt.Printf("%+v\n", address)
	}

	// Service objects
	services, err := space.Services("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, service := range services.Services {
		fmt.Printf("%+v\n", service)
	}

	// Add an address group. "true" as the first parameter means that we assume the
	// group is going to be an address group.
	space.AddGroup(true, "Blacklist-IPs", "Blacklisted IP addresses")

	// Add a service group. We do this by specifying "false" as the first parameter.
	space.AddGroup(false, "Web-Protocols", "All web-based protocols and ports")

	// Add an address object
	space.AddAddress("my-laptop", "2.2.2.2", "My personal laptop")

	// Add a network
	space.AddAddress("corporate-users", "192.168.1.0/24", "People on campus")

	// Add a service object with an 1800 second inactivity timeout (using "0" disables this feature)
	space.AddService("udp", "udp-5000", 5000, 5000, "UDP port 5000", 1800)

	// Add a service object with a port range
	space.AddService("tcp", "high-port-range", 40000, 65000, "TCP high ports", 0)

	// If you want to modify an existing object group, you do this with the ModifyObject() function. The
	// first parameter is whether the object is an address group (true) or a service group (false).

	// Add a service to a group
	space.ModifyObject(false, "add", "service-group", "service-name")

	// Remove an address object from a group
	space.ModifyObject(true, "remove", "Whitelisted-Addresses", "bad-ip")

	// Rename an object
	space.ModifyObject(false, "rename", "Web-Services", "Web-Ports")

	// Delete an object
	space.ModifyObject(true, "delete", "my-laptop")
}

// Working with polymorphic (variable) objects.
func ExampleJunosSpace_variables() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Add a variable
	// The parameters are as follows: variable-name, description, default-value
	space.AddVariable("test-variable", "Our test variable", "default-object")

	// Create our session state for modifying variables
	v, err := space.ModifyVariable()
	if err != nil {
		log.Fatal(err)
	}

	// Adding objects to the variable
	v.Add("test-variable", "srx-1", "user-pc")
	v.Add("test-variable", "corp-firewall", "db-server")

	// Delete a variable
	space.DeleteVariable("test-variable")
}

// Working with policies.
func ExampleJunosSpace_policies() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security policies Junos Space manages:
	policies, err := space.Policies()
	if err != nil {
		fmt.Println(err)
	}

	for _, policy := range policies.Policies {
		fmt.Printf("%s\n", policy.Name)
	}

	// For example, say we have been adding and removing objects in a group, and that group
	// is referenced in a firewall policy. Here's how to update the policy:

	// Update the policy. If "false" is specified, then the policy is only published, and the
	// device is not updated.
	job, err := space.PublishPolicy("Internet-Firewall-Policy", true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)

	// Let's update a device knowing that we have some previously published services.
	job, err := space.UpdateDevice("firewall-1.company.com")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}
<|endoftext|>


<commit_before>package application

import (
	"encoding/json"
	"fmt"
	"path"
	"reflect"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/resin-io/edge-node-manager/board"
	"github.com/resin-io/edge-node-manager/config"
	"github.com/resin-io/edge-node-manager/database"
	"github.com/resin-io/edge-node-manager/device"
	"github.com/resin-io/edge-node-manager/device/status"
	"github.com/resin-io/edge-node-manager/supervisor"
	tarinator "github.com/verybluebot/tarinator-go"
)

// List holds all the applications assigned to the edge-node-manager
// Key is the applicationUUID
var List map[int]*Application

type Application struct {
	ResinUUID     int                       `json:"id"`
	Name          string                    `json:"name"`
	BoardType     board.Type                `json:"-"`
	Commit        string                    `json:"-"`      // Ignore this when unmarshalling from the supervisor as we want to set the target commit
	TargetCommit  string                    `json:"commit"` // Set json tag to commit as the supervisor has no concept of target commit
	Config        map[string]interface{}    `json:"config"`
	FilePath      string                    `json:"-"`
	Devices       map[string]*device.Device `json:"-"` // Key is the device's localUUID
	OnlineDevices map[string]bool           `json:"-"` // Key is the device's localUUID
	deleteFlag    bool                      //Mark an application for deletion
}

func (a Application) String() string {
	return fmt.Sprintf(
		"UUID: %d, "+
			"Name: %s, "+
			"Board type: %s, "+
			"Commit: %s, "+
			"Target commit: %s, "+
			"Config: %v, "+
			"File path: %s",
		a.ResinUUID,
		a.Name,
		a.BoardType,
		a.Commit,
		a.TargetCommit,
		a.Config,
		a.FilePath)
}

func Load() []error {
	bytes, errs := supervisor.DependentApplicationsList()
	if errs != nil {
		return errs
	}

	var buffer []Application
	if err := json.Unmarshal(bytes, &buffer); err != nil {
		return []error{err}
	}

	for _, application := range List {
		application.deleteFlag = true
	}

	for key, value := range buffer {
		resinUUID := value.ResinUUID

		application, exists := List[resinUUID]

		if exists {
			application.deleteFlag = false
			application.Config = value.Config
		} else {
			List[resinUUID] = &buffer[key]
			application := List[resinUUID]

			if _, exists := application.Config["RESIN_HOST_TYPE"]; exists {
				application.BoardType = (board.Type)(application.Config["RESIN_HOST_TYPE"].(string))
			}
		}

		application = List[resinUUID]
		if errs := application.GetDevices(); errs != nil {
			return errs
		}
	}

	for key, application := range List {
		if application.deleteFlag {
			delete(List, key)
		}
	}

	return nil
}

func (a *Application) GetOnlineDevices() error {
	board, err := board.Create(a.BoardType, "", nil)
	if err != nil {
		return err
	}

	if a.OnlineDevices, err = board.Scan(a.ResinUUID); err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"Number of online devices": len(a.OnlineDevices),
	}).Info("Processing application")

	if log.GetLevel() != log.DebugLevel {
		return nil
	}

	for localUUID := range a.OnlineDevices {
		log.WithFields(log.Fields{
			"Local UUID": localUUID,
		}).Debug("Online device")
	}

	return nil
}

func (a *Application) ProvisionDevices() []error {
	for localUUID := range a.OnlineDevices {
		if _, exists := a.Devices[localUUID]; exists {
			log.WithFields(log.Fields{
				"Local UUID": localUUID,
			}).Debug("Device already provisioned")
			continue
		}

		log.WithFields(log.Fields{
			"Local UUID": localUUID,
		}).Info("Provisioning device")

		resinUUID, name, errs := supervisor.DependentDeviceProvision(a.ResinUUID)
		if errs != nil {
			return errs
		}

		d, err := device.Create(a.BoardType, name, localUUID, resinUUID, a.ResinUUID, a.Name, a.Commit)
		if err != nil {
			return []error{err}
		}

		a.Devices[d.LocalUUID] = d

		log.WithFields(log.Fields{
			"Name":       d.Name,
			"Local UUID": localUUID,
		}).Info("Provisioned device")
	}

	return nil
}

func (a *Application) SetOfflineDeviceStatus() []error {
	for _, d := range a.Devices {
		if _, exists := a.OnlineDevices[d.LocalUUID]; !exists {
			if errs := d.SetStatus(status.OFFLINE); errs != nil {
				return errs
			}
		}
	}

	return nil
}

func (a *Application) UpdateOnlineDevices() []error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		online, err := d.Board.Online()
		if err != nil {
			return []error{err}
		}

		if !online {
			d.SetStatus(status.OFFLINE)
			return nil
		}

		d.SetStatus(status.IDLE)

		if d.Commit == d.TargetCommit {
			log.WithFields(log.Fields{
				"Device": d,
			}).Debug("Device up to date")
			continue
		}

		log.WithFields(log.Fields{
			"Device": d,
		}).Debug("Device not up to date")

		if err := a.checkCommit(); err != nil {
			return []error{err}
		}

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Starting update")

		d.SetStatus(status.INSTALLING)

		if err := d.Board.Update(a.FilePath); err != nil {
			log.WithFields(log.Fields{
				"Name": d.Name,
			}).Error("Update failed")
			d.SetStatus(status.IDLE)
			return []error{err}
		}

		d.Commit = d.TargetCommit
		d.SetStatus(status.IDLE)

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Finished update")
	}

	return nil
}

func (a *Application) UpdateConfigOnlineDevices() []error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		online, err := d.Board.Online()
		if err != nil {
			return []error{err}
		}

		if !online {
			return nil
		}

		if reflect.DeepEqual(d.Config, d.TargetConfig) {
			log.WithFields(log.Fields{
				"Device": d,
			}).Debug("Device config up to date")
			continue
		}

		log.WithFields(log.Fields{
			"Device": d,
		}).Debug("Device config not up to date")

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Starting config update")

		if err := d.Board.UpdateConfig(d.TargetConfig); err != nil {
			if err.Error() != "Update config not implemented" {
				log.WithFields(log.Fields{
					"Name": d.Name,
				}).Error("Update config failed")
				return []error{err}
			}
		}

		d.Config = d.TargetConfig

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Finished config update")
	}

	return nil
}

func (a *Application) UpdateEnvironmentOnlineDevices() []error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		online, err := d.Board.Online()
		if err != nil {
			return []error{err}
		}

		if !online {
			return nil
		}

		if reflect.DeepEqual(d.Environment, d.TargetEnvironment) {
			log.WithFields(log.Fields{
				"Device": d,
			}).Debug("Device environment up to date")
			continue
		}

		log.WithFields(log.Fields{
			"Device": d,
		}).Debug("Device environment not up to date")

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Starting environment update")

		if err := d.Board.UpdateEnvironment(d.TargetEnvironment); err != nil {
			if err.Error() != "Update environment not implemented" {
				log.WithFields(log.Fields{
					"Name": d.Name,
				}).Error("Update environment failed")
				return []error{err}
			}
		}

		d.Environment = d.TargetEnvironment

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Finished environment update")
	}

	return nil
}

func (a *Application) HandleDeleteFlag() error {
	for key, d := range a.Devices {
		if !d.DeleteFlag {
			continue
		}

		delete(a.Devices, key)

		if err := database.DeleteDevice(a.ResinUUID, d.ResinUUID); err != nil {
			log.WithFields(log.Fields{
				"Error": err,
			}).Error("Unable to delete device")
			return err
		}

		log.WithFields(log.Fields{
			"name": d.Name,
		}).Info("Device deleted")
	}

	return nil
}

func (a *Application) HandleFlags() error {
	if err := a.handleRestartFlag(); err != nil {
		return err
	}

	return nil
}

func (a *Application) PutDevices() error {
	buffer := make(map[string][]byte)
	for _, d := range a.Devices {
		bytes, err := d.Marshall()
		if err != nil {
			return err
		}
		buffer[d.ResinUUID] = bytes
	}

	return database.PutDevices(a.ResinUUID, buffer)
}

func (a *Application) GetDevices() []error {
	if a.Devices == nil {
		a.Devices = make(map[string]*device.Device)
	}

	buffer, err := database.GetDevices(a.ResinUUID)
	if err != nil {
		return []error{err}
	}

	for _, bytes := range buffer {
		d, err := device.Unmarshall(bytes)
		if err != nil {
			return []error{err}
		}

		// Only add the device from the DB if it does not already exist
		if _, exists := a.Devices[d.LocalUUID]; !exists {
			a.Devices[d.LocalUUID] = d
		}

		// Sync device with resin
		if errs := a.Devices[d.LocalUUID].Sync(); errs != nil {
			return errs
		}
	}

	return nil
}

func init() {
	log.SetLevel(config.GetLogLevel())

	List = make(map[int]*Application)

	log.Debug("Initialised applications")
}

func (a *Application) checkCommit() error {
	if a.Commit == a.TargetCommit {
		return nil
	}

	if err := supervisor.DependentApplicationUpdate(a.ResinUUID, a.TargetCommit); err != nil {
		return err
	}

	a.FilePath = config.GetAssetsDir()
	a.FilePath = path.Join(a.FilePath, strconv.Itoa(a.ResinUUID))
	a.FilePath = path.Join(a.FilePath, a.TargetCommit)
	tarPath := path.Join(a.FilePath, "binary.tar")

	if err := tarinator.UnTarinate(a.FilePath, tarPath); err != nil {
		return err
	}

	a.Commit = a.TargetCommit

	log.Info("Application firmware extracted")

	return nil
}

func (a *Application) handleRestartFlag() error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		if !d.RestartFlag {
			continue
		}

		online, err := d.Board.Online()
		if err != nil {
			return err
		}

		if !online {
			d.SetStatus(status.OFFLINE)
			return nil
		}

		d.SetStatus(status.IDLE)

		if err = d.Board.Restart(); err != nil {
			return err
		}

		d.RestartFlag = false

		log.WithFields(log.Fields{
			"name": d.Name,
		}).Info("Device restarted")
	}

	return nil
}
<commit_msg>Retry updates<commit_after>package application

import (
	"encoding/json"
	"fmt"
	"path"
	"reflect"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/resin-io/edge-node-manager/board"
	"github.com/resin-io/edge-node-manager/config"
	"github.com/resin-io/edge-node-manager/database"
	"github.com/resin-io/edge-node-manager/device"
	"github.com/resin-io/edge-node-manager/device/status"
	"github.com/resin-io/edge-node-manager/radio/bluetooth"
	"github.com/resin-io/edge-node-manager/supervisor"
	tarinator "github.com/verybluebot/tarinator-go"
)

// List holds all the applications assigned to the edge-node-manager
// Key is the applicationUUID
var List map[int]*Application

type Application struct {
	ResinUUID     int                       `json:"id"`
	Name          string                    `json:"name"`
	BoardType     board.Type                `json:"-"`
	Commit        string                    `json:"-"`      // Ignore this when unmarshalling from the supervisor as we want to set the target commit
	TargetCommit  string                    `json:"commit"` // Set json tag to commit as the supervisor has no concept of target commit
	Config        map[string]interface{}    `json:"config"`
	FilePath      string                    `json:"-"`
	Devices       map[string]*device.Device `json:"-"` // Key is the device's localUUID
	OnlineDevices map[string]bool           `json:"-"` // Key is the device's localUUID
	deleteFlag    bool                      //Mark an application for deletion
}

func (a Application) String() string {
	return fmt.Sprintf(
		"UUID: %d, "+
			"Name: %s, "+
			"Board type: %s, "+
			"Commit: %s, "+
			"Target commit: %s, "+
			"Config: %v, "+
			"File path: %s",
		a.ResinUUID,
		a.Name,
		a.BoardType,
		a.Commit,
		a.TargetCommit,
		a.Config,
		a.FilePath)
}

func Load() []error {
	bytes, errs := supervisor.DependentApplicationsList()
	if errs != nil {
		return errs
	}

	var buffer []Application
	if err := json.Unmarshal(bytes, &buffer); err != nil {
		return []error{err}
	}

	for _, application := range List {
		application.deleteFlag = true
	}

	for key, value := range buffer {
		resinUUID := value.ResinUUID

		application, exists := List[resinUUID]

		if exists {
			application.deleteFlag = false
			application.Config = value.Config
		} else {
			List[resinUUID] = &buffer[key]
			application := List[resinUUID]

			if _, exists := application.Config["RESIN_HOST_TYPE"]; exists {
				application.BoardType = (board.Type)(application.Config["RESIN_HOST_TYPE"].(string))
			}
		}

		application = List[resinUUID]
		if errs := application.GetDevices(); errs != nil {
			return errs
		}
	}

	for key, application := range List {
		if application.deleteFlag {
			delete(List, key)
		}
	}

	return nil
}

func (a *Application) GetOnlineDevices() error {
	board, err := board.Create(a.BoardType, "", nil)
	if err != nil {
		return err
	}

	if a.OnlineDevices, err = board.Scan(a.ResinUUID); err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"Number of online devices": len(a.OnlineDevices),
	}).Info("Processing application")

	if log.GetLevel() != log.DebugLevel {
		return nil
	}

	for localUUID := range a.OnlineDevices {
		log.WithFields(log.Fields{
			"Local UUID": localUUID,
		}).Debug("Online device")
	}

	return nil
}

func (a *Application) ProvisionDevices() []error {
	for localUUID := range a.OnlineDevices {
		if _, exists := a.Devices[localUUID]; exists {
			log.WithFields(log.Fields{
				"Local UUID": localUUID,
			}).Debug("Device already provisioned")
			continue
		}

		log.WithFields(log.Fields{
			"Local UUID": localUUID,
		}).Info("Provisioning device")

		resinUUID, name, errs := supervisor.DependentDeviceProvision(a.ResinUUID)
		if errs != nil {
			return errs
		}

		d, err := device.Create(a.BoardType, name, localUUID, resinUUID, a.ResinUUID, a.Name, a.Commit)
		if err != nil {
			return []error{err}
		}

		a.Devices[d.LocalUUID] = d

		log.WithFields(log.Fields{
			"Name":       d.Name,
			"Local UUID": localUUID,
		}).Info("Provisioned device")
	}

	return nil
}

func (a *Application) SetOfflineDeviceStatus() []error {
	for _, d := range a.Devices {
		if _, exists := a.OnlineDevices[d.LocalUUID]; !exists {
			if errs := d.SetStatus(status.OFFLINE); errs != nil {
				return errs
			}
		}
	}

	return nil
}

func (a *Application) UpdateOnlineDevices() []error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		online, err := d.Board.Online()
		if err != nil {
			return []error{err}
		}

		if !online {
			d.SetStatus(status.OFFLINE)
			return nil
		}

		d.SetStatus(status.IDLE)

		if d.Commit == d.TargetCommit {
			log.WithFields(log.Fields{
				"Device": d,
			}).Debug("Device up to date")
			continue
		}

		log.WithFields(log.Fields{
			"Device": d,
		}).Debug("Device not up to date")

		if err := a.checkCommit(); err != nil {
			return []error{err}
		}

		d.SetStatus(status.INSTALLING)

		for i := 1; i <= 3; i++ {
			log.WithFields(log.Fields{
				"Name":    d.Name,
				"Attempt": i,
			}).Info("Starting update")

			if err := d.Board.Update(a.FilePath); err != nil {
				log.WithFields(log.Fields{
					"Name":  d.Name,
					"Error": err,
				}).Error("Update failed")
				bluetooth.ResetDevice()
				continue
			} else {
				d.Commit = d.TargetCommit

				log.WithFields(log.Fields{
					"Name": d.Name,
				}).Info("Finished update")
				break
			}
		}
		d.SetStatus(status.IDLE)
	}

	return nil
}

func (a *Application) UpdateConfigOnlineDevices() []error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		online, err := d.Board.Online()
		if err != nil {
			return []error{err}
		}

		if !online {
			return nil
		}

		if reflect.DeepEqual(d.Config, d.TargetConfig) {
			log.WithFields(log.Fields{
				"Device": d,
			}).Debug("Device config up to date")
			continue
		}

		log.WithFields(log.Fields{
			"Device": d,
		}).Debug("Device config not up to date")

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Starting config update")

		if err := d.Board.UpdateConfig(d.TargetConfig); err != nil {
			if err.Error() != "Update config not implemented" {
				log.WithFields(log.Fields{
					"Name": d.Name,
				}).Error("Update config failed")
				return []error{err}
			}
		}

		d.Config = d.TargetConfig

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Finished config update")
	}

	return nil
}

func (a *Application) UpdateEnvironmentOnlineDevices() []error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		online, err := d.Board.Online()
		if err != nil {
			return []error{err}
		}

		if !online {
			return nil
		}

		if reflect.DeepEqual(d.Environment, d.TargetEnvironment) {
			log.WithFields(log.Fields{
				"Device": d,
			}).Debug("Device environment up to date")
			continue
		}

		log.WithFields(log.Fields{
			"Device": d,
		}).Debug("Device environment not up to date")

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Starting environment update")

		if err := d.Board.UpdateEnvironment(d.TargetEnvironment); err != nil {
			if err.Error() != "Update environment not implemented" {
				log.WithFields(log.Fields{
					"Name": d.Name,
				}).Error("Update environment failed")
				return []error{err}
			}
		}

		d.Environment = d.TargetEnvironment

		log.WithFields(log.Fields{
			"Name": d.Name,
		}).Info("Finished environment update")
	}

	return nil
}

func (a *Application) HandleDeleteFlag() error {
	for key, d := range a.Devices {
		if !d.DeleteFlag {
			continue
		}

		delete(a.Devices, key)

		if err := database.DeleteDevice(a.ResinUUID, d.ResinUUID); err != nil {
			log.WithFields(log.Fields{
				"Error": err,
			}).Error("Unable to delete device")
			return err
		}

		log.WithFields(log.Fields{
			"name": d.Name,
		}).Info("Device deleted")
	}

	return nil
}

func (a *Application) HandleFlags() error {
	if err := a.handleRestartFlag(); err != nil {
		return err
	}

	return nil
}

func (a *Application) PutDevices() error {
	buffer := make(map[string][]byte)
	for _, d := range a.Devices {
		bytes, err := d.Marshall()
		if err != nil {
			return err
		}
		buffer[d.ResinUUID] = bytes
	}

	return database.PutDevices(a.ResinUUID, buffer)
}

func (a *Application) GetDevices() []error {
	if a.Devices == nil {
		a.Devices = make(map[string]*device.Device)
	}

	buffer, err := database.GetDevices(a.ResinUUID)
	if err != nil {
		return []error{err}
	}

	for _, bytes := range buffer {
		d, err := device.Unmarshall(bytes)
		if err != nil {
			return []error{err}
		}

		// Only add the device from the DB if it does not already exist
		if _, exists := a.Devices[d.LocalUUID]; !exists {
			a.Devices[d.LocalUUID] = d
		}

		// Sync device with resin
		if errs := a.Devices[d.LocalUUID].Sync(); errs != nil {
			return errs
		}
	}

	return nil
}

func init() {
	log.SetLevel(config.GetLogLevel())

	List = make(map[int]*Application)

	log.Debug("Initialised applications")
}

func (a *Application) checkCommit() error {
	if a.Commit == a.TargetCommit {
		return nil
	}

	if err := supervisor.DependentApplicationUpdate(a.ResinUUID, a.TargetCommit); err != nil {
		return err
	}

	a.FilePath = config.GetAssetsDir()
	a.FilePath = path.Join(a.FilePath, strconv.Itoa(a.ResinUUID))
	a.FilePath = path.Join(a.FilePath, a.TargetCommit)
	tarPath := path.Join(a.FilePath, "binary.tar")

	if err := tarinator.UnTarinate(a.FilePath, tarPath); err != nil {
		return err
	}

	a.Commit = a.TargetCommit

	log.Info("Application firmware extracted")

	return nil
}

func (a *Application) handleRestartFlag() error {
	for localUUID := range a.OnlineDevices {
		d := a.Devices[localUUID]

		if !d.RestartFlag {
			continue
		}

		online, err := d.Board.Online()
		if err != nil {
			return err
		}

		if !online {
			d.SetStatus(status.OFFLINE)
			return nil
		}

		d.SetStatus(status.IDLE)

		if err = d.Board.Restart(); err != nil {
			return err
		}

		d.RestartFlag = false

		log.WithFields(log.Fields{
			"name": d.Name,
		}).Info("Device restarted")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The code in this file is adapted from src/mime/multipart/writer.go in the Go
// repository at commit c007ce824d9a4fccb148f9204e04c23ed2984b71. The LICENSE
// file there says:
//
// Copyright (c) 2012 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package httputil

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io"
)

type ContentTypedReader struct {
	ContentType string
	Reader      io.Reader
}

// Create a reader that streams an HTTP multipart body (see RFC 2388) composed
// of the contents of each component reader in sequence, each with a
// Content-Type header as specified.
//
// Unlike multipart.Writer from the standard library, this can be used directly
// as http.Request.Body without bending over backwards to convert an io.Writer
// to an io.Reader.
func NewMultipartReader(readers []ContentTypedReader) *MultipartReader

// MultipartReader is an io.Reader that generates HTTP multipart bodies. See
// NewMultipartReader for details.
type MultipartReader struct {
}

func (mr *MultipartReader) Read(p []byte) (n int, err error) {
	err = fmt.Errorf("TODO: MultipartReader.Read")
	return
}

// Return the value to use for the Content-Type header in an HTTP request that
// uses mr as the body.
func (mr *MultipartReader) ContentType() string {
	return fmt.Sprintf("multipart/related; boundary=%s", mr.boundary)
}

func randomBoundary() string {
	var buf [30]byte
	_, err := io.ReadFull(rand.Reader, buf[:])
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", buf[:])
}

// Create a reader for a single part and the boundary in front of it. first
// specifies whether this is the first part.
func makePartReader(
	ctr ContentTypedReader,
	first bool,
	boundary string) (r io.Reader) {
	// Set up a buffer containing the boundary.
	var b bytes.Buffer
	if first {
		fmt.Fprintf(&b, "--%s\r\n", boundary)
	} else {
		fmt.Fprintf(&b, "\r\n--%s\r\n", boundary)
	}

	fmt.Fprintf(&b, "Content-Type: %s\r\n", ctr.ContentType)

	// Read the boundary followed by the content.
	r = io.MultiReader(&b, ctr.Reader)

	return
}

// Create a reader for the trailing boundary.
func makeTrailerReader(boundary string) (r io.Reader) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "\r\n--%s--\r\n", boundary)

	r = &b
	return
}
<commit_msg>NewMultipartReader<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The code in this file is adapted from src/mime/multipart/writer.go in the Go
// repository at commit c007ce824d9a4fccb148f9204e04c23ed2984b71. The LICENSE
// file there says:
//
// Copyright (c) 2012 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package httputil

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io"
)

type ContentTypedReader struct {
	ContentType string
	Reader      io.Reader
}

// Create a reader that streams an HTTP multipart body (see RFC 2388) composed
// of the contents of each component reader in sequence, each with a
// Content-Type header as specified.
//
// Unlike multipart.Writer from the standard library, this can be used directly
// as http.Request.Body without bending over backwards to convert an io.Writer
// to an io.Reader.
func NewMultipartReader(ctrs []ContentTypedReader) *MultipartReader {
	boundary := randomBoundary()

	// Read each part followed by the trailer.
	var readers []io.Reader
	for i, ctr := range ctrs {
		readers = append(readers, makePartReader(ctr, i == 0, boundary))
	}

	readers = append(readers, makeTrailerReader(boundary))

	return &MultipartReader{
		boundary: boundary,
		r:        io.MultiReader(readers...),
	}
}

// MultipartReader is an io.Reader that generates HTTP multipart bodies. See
// NewMultipartReader for details.
type MultipartReader struct {
	boundary string
	r        io.Reader
}

func (mr *MultipartReader) Read(p []byte) (n int, err error) {
	n, err = mr.r.Read(p)
	return
}

// Return the value to use for the Content-Type header in an HTTP request that
// uses mr as the body.
func (mr *MultipartReader) ContentType() string {
	return fmt.Sprintf("multipart/related; boundary=%s", mr.boundary)
}

func randomBoundary() string {
	var buf [30]byte
	_, err := io.ReadFull(rand.Reader, buf[:])
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", buf[:])
}

// Create a reader for a single part and the boundary in front of it. first
// specifies whether this is the first part.
func makePartReader(
	ctr ContentTypedReader,
	first bool,
	boundary string) (r io.Reader) {
	// Set up a buffer containing the boundary.
	var b bytes.Buffer
	if first {
		fmt.Fprintf(&b, "--%s\r\n", boundary)
	} else {
		fmt.Fprintf(&b, "\r\n--%s\r\n", boundary)
	}

	fmt.Fprintf(&b, "Content-Type: %s\r\n", ctr.ContentType)

	// Read the boundary followed by the content.
	r = io.MultiReader(&b, ctr.Reader)

	return
}

// Create a reader for the trailing boundary.
func makeTrailerReader(boundary string) (r io.Reader) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "\r\n--%s--\r\n", boundary)

	r = &b
	return
}
<|endoftext|>