<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kube

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/ghodss/yaml"
	"github.com/sirupsen/logrus"
)

const (
	inClusterBaseURL = "https://kubernetes.default"
	maxRetries       = 8
	retryDelay       = 2 * time.Second
	EmptySelector    = ""
)

type Logger interface {
	Debugf(s string, v ...interface{})
}

// Client interacts with the Kubernetes api-server.
type Client struct {
	// If logger is non-nil, log all method calls with it.
	logger Logger

	baseURL   string
	client    *http.Client
	token     string
	namespace string
	fake      bool
}

// Namespace returns a copy of the client pointing at the specified namespace.
func (c *Client) Namespace(ns string) *Client {
	nc := *c
	nc.namespace = ns
	return &nc
}

func (c *Client) log(methodName string, args ...interface{}) {
	if c.logger == nil {
		return
	}
	var as []string
	for _, arg := range args {
		as = append(as, fmt.Sprintf("%v", arg))
	}
	c.logger.Debugf("%s(%s)", methodName, strings.Join(as, ", "))
}

type ConflictError struct {
	e error
}

func (e ConflictError) Error() string {
	return e.e.Error()
}

func NewConflictError(e error) ConflictError {
	return ConflictError{e: e}
}

type UnprocessableEntityError struct {
	e error
}

func (e UnprocessableEntityError) Error() string {
	return e.e.Error()
}

func NewUnprocessableEntityError(e error) UnprocessableEntityError {
	return UnprocessableEntityError{e: e}
}

type request struct {
	method      string
	path        string
	query       map[string]string
	requestBody interface{}
}

func (c *Client) request(r *request, ret interface{}) error {
	out, err := c.requestRetry(r)
	if err != nil {
		return err
	}
	if ret != nil {
		if err := json.Unmarshal(out, ret); err != nil {
			return err
		}
	}
	return nil
}

func (c *Client) retry(r *request) (*http.Response, error) {
	var resp *http.Response
	var err error
	backoff := retryDelay
	for retries := 0; retries < maxRetries; retries++ {
		resp, err = c.doRequest(r.method, r.path, r.query, r.requestBody)
		if err == nil {
			if resp.StatusCode < 500 {
				break
			}
			resp.Body.Close()
		}

		time.Sleep(backoff)
		backoff *= 2
	}
	return resp, err
}

// Retry on transport failures. Does not retry on 500s.
func (c *Client) requestRetryStream(r *request) (io.ReadCloser, error) {
	if c.fake {
		return nil, nil
	}
	resp, err := c.retry(r)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == 409 {
		return nil, NewConflictError(fmt.Errorf("body cannot be streamed"))
	} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return nil, fmt.Errorf("response has status \"%s\"", resp.Status)
	}
	return resp.Body, nil
}

// Retry on transport failures. Does not retry on 500s.
func (c *Client) requestRetry(r *request) ([]byte, error) {
	if c.fake {
		return []byte("{}"), nil
	}
	resp, err := c.retry(r)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	rb, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == 409 {
		return nil, NewConflictError(fmt.Errorf("body: %s", string(rb)))
	} else if resp.StatusCode == 422 {
		return nil, NewUnprocessableEntityError(fmt.Errorf("body: %s", string(rb)))
	} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return nil, fmt.Errorf("response has status \"%s\" and body \"%s\"", resp.Status, string(rb))
	}
	return rb, nil
}

func (c *Client) doRequest(method, urlPath string, query map[string]string, body interface{}) (*http.Response, error) {
	url := c.baseURL + urlPath
	var buf io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		buf = bytes.NewBuffer(b)
	}
	req, err := http.NewRequest(method, url, buf)
	if err != nil {
		return nil, err
	}
	if c.token != "" {
		req.Header.Set("Authorization", "Bearer "+c.token)
	}
	if method == http.MethodPatch {
		req.Header.Set("Content-Type", "application/strategic-merge-patch+json")
	} else {
		req.Header.Set("Content-Type", "application/json")
	}

	q := req.URL.Query()
	for k, v := range query {
		q.Add(k, v)
	}
	req.URL.RawQuery = q.Encode()

	return c.client.Do(req)
}

// NewFakeClient creates a client that doesn't do anything.
func NewFakeClient() *Client {
	return &Client{
		namespace: "default",
		fake:      true,
	}
}

// NewClientInCluster creates a Client that works from within a pod.
func NewClientInCluster(namespace string) (*Client, error) {
	tokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token"
	token, err := ioutil.ReadFile(tokenFile)
	if err != nil {
		return nil, err
	}

	rootCAFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
	certData, err := ioutil.ReadFile(rootCAFile)
	if err != nil {
		return nil, err
	}

	cp := x509.NewCertPool()
	cp.AppendCertsFromPEM(certData)

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			MinVersion: tls.VersionTLS12,
			RootCAs:    cp,
		},
	}
	c := &http.Client{Transport: tr}
	return &Client{
		logger:    logrus.WithField("client", "kube"),
		baseURL:   inClusterBaseURL,
		client:    c,
		token:     string(token),
		namespace: namespace,
	}, nil
}

// Cluster represents the information necessary to talk to a Kubernetes
// master endpoint.
type Cluster struct {
	// The IP address of the cluster's master endpoint.
	Endpoint string `yaml:"endpoint"`
	// Base64-encoded public cert used by clients to authenticate to the
	// cluster endpoint.
	ClientCertificate string `yaml:"clientCertificate"`
	// Base64-encoded private key used by clients..
	ClientKey string `yaml:"clientKey"`
	// Base64-encoded public certificate that is the root of trust for the
	// cluster.
	ClusterCACertificate string `yaml:"clusterCaCertificate"`
}

// NewClientFromFile reads a Cluster object at clusterPath and returns an
// authenticated client using the keys within.
func NewClientFromFile(clusterPath, namespace string) (*Client, error) {
	data, err := ioutil.ReadFile(clusterPath)
	if err != nil {
		return nil, err
	}
	var c Cluster
	if err := yaml.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return NewClient(&c, namespace)
}

// NewClient returns an authenticated Client using the keys in the Cluster.
func NewClient(c *Cluster, namespace string) (*Client, error) {
	cc, err := base64.StdEncoding.DecodeString(c.ClientCertificate)
	if err != nil {
		return nil, err
	}
	ck, err := base64.StdEncoding.DecodeString(c.ClientKey)
	if err != nil {
		return nil, err
	}
	ca, err := base64.StdEncoding.DecodeString(c.ClusterCACertificate)
	if err != nil {
		return nil, err
	}

	cert, err := tls.X509KeyPair(cc, ck)
	if err != nil {
		return nil, err
	}

	cp := x509.NewCertPool()
	cp.AppendCertsFromPEM(ca)

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			MinVersion:   tls.VersionTLS12,
			Certificates: []tls.Certificate{cert},
			RootCAs:      cp,
		},
	}
	return &Client{
		logger:    logrus.WithField("client", "kube"),
		baseURL:   c.Endpoint,
		client:    &http.Client{Transport: tr},
		namespace: namespace,
	}, nil
}

func (c *Client) GetPod(name string) (Pod, error) {
	c.log("GetPod", name)
	var retPod Pod
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", c.namespace, name),
	}, &retPod)
	return retPod, err
}

func (c *Client) ListPods(selector string) ([]Pod, error) {
	c.log("ListPods", selector)
	var pl struct {
		Items []Pod `json:"items"`
	}
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods", c.namespace),
		query:  map[string]string{"labelSelector": selector},
	}, &pl)
	return pl.Items, err
}

func (c *Client) DeletePod(name string) error {
	c.log("DeletePod", name)
	return c.request(&request{
		method: http.MethodDelete,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", c.namespace, name),
	}, nil)
}

func (c *Client) CreateProwJob(j ProwJob) (ProwJob, error) {
	c.log("CreateProwJob", j)
	var retJob ProwJob
	err := c.request(&request{
		method:      http.MethodPost,
		path:        fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs", c.namespace),
		requestBody: &j,
	}, &retJob)
	return retJob, err
}

func (c *Client) GetProwJob(name string) (ProwJob, error) {
	c.log("GetProwJob", name)
	var pj ProwJob
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name),
	}, &pj)
	return pj, err
}

func (c *Client) ListProwJobs(selector string) ([]ProwJob, error) {
	c.log("ListProwJobs", selector)
	var jl struct {
		Items []ProwJob `json:"items"`
	}
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs", c.namespace),
		query:  map[string]string{"labelSelector": selector},
	}, &jl)
	return jl.Items, err
}

func (c *Client) DeleteProwJob(name string) error {
	c.log("DeleteProwJob", name)
	return c.request(&request{
		method: http.MethodDelete,
		path:   fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name),
	}, nil)
}

func (c *Client) ReplaceProwJob(name string, job ProwJob) (ProwJob, error) {
	c.log("ReplaceProwJob", name, job)
	var retJob ProwJob
	err := c.request(&request{
		method:      http.MethodPut,
		path:        fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name),
		requestBody: &job,
	}, &retJob)
	return retJob, err
}

func (c *Client) CreatePod(p Pod) (Pod, error) {
	c.log("CreatePod", p)
	var retPod Pod
	err := c.request(&request{
		method:      http.MethodPost,
		path:        fmt.Sprintf("/api/v1/namespaces/%s/pods", c.namespace),
		requestBody: &p,
	}, &retPod)
	return retPod, err
}

func (c *Client) GetLog(pod string) ([]byte, error) {
	c.log("GetLog", pod)
	return c.requestRetry(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod),
	})
}

func (c *Client) GetLogStream(pod string, options map[string]string) (io.ReadCloser, error) {
	c.log("GetLogStream", pod)
	return c.requestRetryStream(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod),
		query:  options,
	})
}

func (c *Client) CreateConfigMap(content ConfigMap) (ConfigMap, error) {
	c.log("CreateConfigMap")
	var retConfigMap ConfigMap
	err := c.request(&request{
		method:      http.MethodPost,
		path:        fmt.Sprintf("/api/v1/namespaces/%s/configmaps", c.namespace),
		requestBody: &content,
	}, &retConfigMap)

	return retConfigMap, err
}

func (c *Client) ReplaceConfigMap(name string, config ConfigMap) (ConfigMap, error) {
	c.log("ReplaceConfigMap", name)
	var retConfigMap ConfigMap
	err := c.request(&request{
		method:      http.MethodPut,
		path:        fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", c.namespace, name),
		requestBody: &config,
	}, &retConfigMap)

	return retConfigMap, err
}
<commit_msg>Add timeout to all Prow to API server requests.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kube

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/ghodss/yaml"
	"github.com/sirupsen/logrus"
)

const (
	inClusterBaseURL = "https://kubernetes.default"
	maxRetries       = 8
	retryDelay       = 2 * time.Second
	requestTimeout   = time.Minute

	EmptySelector = ""
)

type Logger interface {
	Debugf(s string, v ...interface{})
}

// Client interacts with the Kubernetes api-server.
type Client struct {
	// If logger is non-nil, log all method calls with it.
	logger Logger

	baseURL   string
	client    *http.Client
	token     string
	namespace string
	fake      bool
}

// Namespace returns a copy of the client pointing at the specified namespace.
func (c *Client) Namespace(ns string) *Client {
	nc := *c
	nc.namespace = ns
	return &nc
}

func (c *Client) log(methodName string, args ...interface{}) {
	if c.logger == nil {
		return
	}
	var as []string
	for _, arg := range args {
		as = append(as, fmt.Sprintf("%v", arg))
	}
	c.logger.Debugf("%s(%s)", methodName, strings.Join(as, ", "))
}

type ConflictError struct {
	e error
}

func (e ConflictError) Error() string {
	return e.e.Error()
}

func NewConflictError(e error) ConflictError {
	return ConflictError{e: e}
}

type UnprocessableEntityError struct {
	e error
}

func (e UnprocessableEntityError) Error() string {
	return e.e.Error()
}

func NewUnprocessableEntityError(e error) UnprocessableEntityError {
	return UnprocessableEntityError{e: e}
}

type request struct {
	method      string
	path        string
	query       map[string]string
	requestBody interface{}
}

func (c *Client) request(r *request, ret interface{}) error {
	out, err := c.requestRetry(r)
	if err != nil {
		return err
	}
	if ret != nil {
		if err := json.Unmarshal(out, ret); err != nil {
			return err
		}
	}
	return nil
}

func (c *Client) retry(r *request) (*http.Response, error) {
	var resp *http.Response
	var err error
	backoff := retryDelay
	for retries := 0; retries < maxRetries; retries++ {
		resp, err = c.doRequest(r.method, r.path, r.query, r.requestBody)
		if err == nil {
			if resp.StatusCode < 500 {
				break
			}
			resp.Body.Close()
		}

		time.Sleep(backoff)
		backoff *= 2
	}
	return resp, err
}

// Retry on transport failures. Does not retry on 500s.
func (c *Client) requestRetryStream(r *request) (io.ReadCloser, error) {
	if c.fake {
		return nil, nil
	}
	resp, err := c.retry(r)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == 409 {
		return nil, NewConflictError(fmt.Errorf("body cannot be streamed"))
	} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return nil, fmt.Errorf("response has status \"%s\"", resp.Status)
	}
	return resp.Body, nil
}

// Retry on transport failures. Does not retry on 500s.
func (c *Client) requestRetry(r *request) ([]byte, error) {
	if c.fake {
		return []byte("{}"), nil
	}
	resp, err := c.retry(r)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	rb, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode == 409 {
		return nil, NewConflictError(fmt.Errorf("body: %s", string(rb)))
	} else if resp.StatusCode == 422 {
		return nil, NewUnprocessableEntityError(fmt.Errorf("body: %s", string(rb)))
	} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return nil, fmt.Errorf("response has status \"%s\" and body \"%s\"", resp.Status, string(rb))
	}
	return rb, nil
}

func (c *Client) doRequest(method, urlPath string, query map[string]string, body interface{}) (*http.Response, error) {
	url := c.baseURL + urlPath
	var buf io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		buf = bytes.NewBuffer(b)
	}
	req, err := http.NewRequest(method, url, buf)
	if err != nil {
		return nil, err
	}
	if c.token != "" {
		req.Header.Set("Authorization", "Bearer "+c.token)
	}
	if method == http.MethodPatch {
		req.Header.Set("Content-Type", "application/strategic-merge-patch+json")
	} else {
		req.Header.Set("Content-Type", "application/json")
	}

	q := req.URL.Query()
	for k, v := range query {
		q.Add(k, v)
	}
	req.URL.RawQuery = q.Encode()

	return c.client.Do(req)
}

// NewFakeClient creates a client that doesn't do anything.
func NewFakeClient() *Client {
	return &Client{
		namespace: "default",
		fake:      true,
	}
}

// NewClientInCluster creates a Client that works from within a pod.
func NewClientInCluster(namespace string) (*Client, error) {
	tokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token"
	token, err := ioutil.ReadFile(tokenFile)
	if err != nil {
		return nil, err
	}

	rootCAFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
	certData, err := ioutil.ReadFile(rootCAFile)
	if err != nil {
		return nil, err
	}

	cp := x509.NewCertPool()
	cp.AppendCertsFromPEM(certData)

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			MinVersion: tls.VersionTLS12,
			RootCAs:    cp,
		},
	}
	return &Client{
		logger:    logrus.WithField("client", "kube"),
		baseURL:   inClusterBaseURL,
		client:    &http.Client{Transport: tr, Timeout: requestTimeout},
		token:     string(token),
		namespace: namespace,
	}, nil
}

// Cluster represents the information necessary to talk to a Kubernetes
// master endpoint.
type Cluster struct {
	// The IP address of the cluster's master endpoint.
	Endpoint string `yaml:"endpoint"`
	// Base64-encoded public cert used by clients to authenticate to the
	// cluster endpoint.
	ClientCertificate string `yaml:"clientCertificate"`
	// Base64-encoded private key used by clients..
	ClientKey string `yaml:"clientKey"`
	// Base64-encoded public certificate that is the root of trust for the
	// cluster.
	ClusterCACertificate string `yaml:"clusterCaCertificate"`
}

// NewClientFromFile reads a Cluster object at clusterPath and returns an
// authenticated client using the keys within.
func NewClientFromFile(clusterPath, namespace string) (*Client, error) {
	data, err := ioutil.ReadFile(clusterPath)
	if err != nil {
		return nil, err
	}
	var c Cluster
	if err := yaml.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return NewClient(&c, namespace)
}

// NewClient returns an authenticated Client using the keys in the Cluster.
func NewClient(c *Cluster, namespace string) (*Client, error) {
	cc, err := base64.StdEncoding.DecodeString(c.ClientCertificate)
	if err != nil {
		return nil, err
	}
	ck, err := base64.StdEncoding.DecodeString(c.ClientKey)
	if err != nil {
		return nil, err
	}
	ca, err := base64.StdEncoding.DecodeString(c.ClusterCACertificate)
	if err != nil {
		return nil, err
	}

	cert, err := tls.X509KeyPair(cc, ck)
	if err != nil {
		return nil, err
	}

	cp := x509.NewCertPool()
	cp.AppendCertsFromPEM(ca)

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{
			MinVersion:   tls.VersionTLS12,
			Certificates: []tls.Certificate{cert},
			RootCAs:      cp,
		},
	}
	return &Client{
		logger:    logrus.WithField("client", "kube"),
		baseURL:   c.Endpoint,
		client:    &http.Client{Transport: tr, Timeout: requestTimeout},
		namespace: namespace,
	}, nil
}

func (c *Client) GetPod(name string) (Pod, error) {
	c.log("GetPod", name)
	var retPod Pod
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", c.namespace, name),
	}, &retPod)
	return retPod, err
}

func (c *Client) ListPods(selector string) ([]Pod, error) {
	c.log("ListPods", selector)
	var pl struct {
		Items []Pod `json:"items"`
	}
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods", c.namespace),
		query:  map[string]string{"labelSelector": selector},
	}, &pl)
	return pl.Items, err
}

func (c *Client) DeletePod(name string) error {
	c.log("DeletePod", name)
	return c.request(&request{
		method: http.MethodDelete,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", c.namespace, name),
	}, nil)
}

func (c *Client) CreateProwJob(j ProwJob) (ProwJob, error) {
	c.log("CreateProwJob", j)
	var retJob ProwJob
	err := c.request(&request{
		method:      http.MethodPost,
		path:        fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs", c.namespace),
		requestBody: &j,
	}, &retJob)
	return retJob, err
}

func (c *Client) GetProwJob(name string) (ProwJob, error) {
	c.log("GetProwJob", name)
	var pj ProwJob
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name),
	}, &pj)
	return pj, err
}

func (c *Client) ListProwJobs(selector string) ([]ProwJob, error) {
	c.log("ListProwJobs", selector)
	var jl struct {
		Items []ProwJob `json:"items"`
	}
	err := c.request(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs", c.namespace),
		query:  map[string]string{"labelSelector": selector},
	}, &jl)
	return jl.Items, err
}

func (c *Client) DeleteProwJob(name string) error {
	c.log("DeleteProwJob", name)
	return c.request(&request{
		method: http.MethodDelete,
		path:   fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name),
	}, nil)
}

func (c *Client) ReplaceProwJob(name string, job ProwJob) (ProwJob, error) {
	c.log("ReplaceProwJob", name, job)
	var retJob ProwJob
	err := c.request(&request{
		method:      http.MethodPut,
		path:        fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name),
		requestBody: &job,
	}, &retJob)
	return retJob, err
}

func (c *Client) CreatePod(p Pod) (Pod, error) {
	c.log("CreatePod", p)
	var retPod Pod
	err := c.request(&request{
		method:      http.MethodPost,
		path:        fmt.Sprintf("/api/v1/namespaces/%s/pods", c.namespace),
		requestBody: &p,
	}, &retPod)
	return retPod, err
}

func (c *Client) GetLog(pod string) ([]byte, error) {
	c.log("GetLog", pod)
	return c.requestRetry(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod),
	})
}

func (c *Client) GetLogStream(pod string, options map[string]string) (io.ReadCloser, error) {
	c.log("GetLogStream", pod)
	return c.requestRetryStream(&request{
		method: http.MethodGet,
		path:   fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod),
		query:  options,
	})
}

func (c *Client) CreateConfigMap(content ConfigMap) (ConfigMap, error) {
	c.log("CreateConfigMap")
	var retConfigMap ConfigMap
	err := c.request(&request{
		method:      http.MethodPost,
		path:        fmt.Sprintf("/api/v1/namespaces/%s/configmaps", c.namespace),
		requestBody: &content,
	}, &retConfigMap)

	return retConfigMap, err
}

func (c *Client) ReplaceConfigMap(name string, config ConfigMap) (ConfigMap, error) {
	c.log("ReplaceConfigMap", name)
	var retConfigMap ConfigMap
	err := c.request(&request{
		method:      http.MethodPut,
		path:        fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", c.namespace, name),
		requestBody: &config,
	}, &retConfigMap)

	return retConfigMap, err
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"github.com/yaricom/goNEAT/v2/neat"
)

// Link is a connection from one node to another with an associated weight.
// It can be marked as recurrent.
type Link struct {
	// Weight of connection
	Weight float64
	// NNode inputting into the link
	InNode *NNode
	// NNode that the link affects
	OutNode *NNode
	// If TRUE the link is recurrent
	IsRecurrent bool
	// If TRUE the link is time delayed
	IsTimeDelayed bool

	// Points to a trait of parameters for genetic creation
	Trait *neat.Trait

	/* ************ LEARNING PARAMETERS *********** */
	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes
	Params []float64
	// The amount of weight adjustment
	AddedWeight float64
}

// NewLink Creates new link with specified weight, input and output neurons connected recurrently or not.
func NewLink(weight float64, inputNode, outNode *NNode, recurrent bool) *Link {
	link := newLink(weight)
	link.InNode = inputNode
	link.OutNode = outNode
	link.IsRecurrent = recurrent
	return link
}

// NewLinkWithTrait Creates new Link with specified Trait
func NewLinkWithTrait(trait *neat.Trait, weight float64, inputNode, outNode *NNode, recurrent bool) *Link {
	link := newLink(weight)
	link.InNode = inputNode
	link.OutNode = outNode
	link.IsRecurrent = recurrent
	link.Trait = trait
	link.deriveTrait(trait)
	return link
}

// NewLinkCopy The copy constructor to create new link with parameters taken from provided ones and connecting specified nodes
func NewLinkCopy(l *Link, inputNode, outNode *NNode) *Link {
	link := newLink(l.Weight)
	link.InNode = inputNode
	link.OutNode = outNode
	link.Trait = l.Trait
	link.deriveTrait(l.Trait)
	link.IsRecurrent = l.IsRecurrent
	return link
}

// The private default constructor
func newLink(weight float64) *Link {
	return &Link{
		Weight: weight,
	}
}

// IsEqualGenetically Checks if this link is genetically equal to provided one, i.e. connects nodes with the same IDs and has equal
// recurrent flag. I.e. if both links represent the same Gene.
func (l *Link) IsEqualGenetically(ol *Link) bool {
	sameInNode := l.InNode.Id == ol.InNode.Id
	sameOutNode := l.OutNode.Id == ol.OutNode.Id
	sameRecurrent := l.IsRecurrent == ol.IsRecurrent

	return sameInNode && sameOutNode && sameRecurrent
}

// The Link methods implementation
func (l *Link) String() string {
	return fmt.Sprintf("[Link: (%s <-> %s), weight: %.3f, recurrent: %t, time delayed: %t]",
		l.InNode, l.OutNode, l.Weight, l.IsRecurrent, l.IsTimeDelayed)
}

// Copy trait parameters into this link's parameters
func (l *Link) deriveTrait(t *neat.Trait) {
	if t != nil {
		l.Params = make([]float64, len(t.Params))
		for i, p := range t.Params {
			l.Params[i] = p
		}
	}
}
<commit_msg>Implemented WeightedEdge<commit_after>package network

import (
	"fmt"
	"github.com/yaricom/goNEAT/v2/neat"
	"gonum.org/v1/gonum/graph"
)

// Link is a connection from one node to another with an associated weight.
// It can be marked as recurrent.
type Link struct {
	// weight of connection
	ConnectionWeight float64
	// NNode inputting into the link
	InNode *NNode
	// NNode that the link affects
	OutNode *NNode
	// If TRUE the link is recurrent
	IsRecurrent bool
	// If TRUE the link is time delayed
	IsTimeDelayed bool

	// Points to a trait of parameters for genetic creation
	Trait *neat.Trait

	/* ************ LEARNING PARAMETERS *********** */
	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes
	Params []float64
	// The amount of weight adjustment
	AddedWeight float64
}

// NewLink Creates new link with specified weight, input and output neurons connected recurrently or not.
func NewLink(weight float64, inputNode, outNode *NNode, recurrent bool) *Link {
	link := newLink(weight)
	link.InNode = inputNode
	link.OutNode = outNode
	link.IsRecurrent = recurrent
	return link
}

// NewLinkWithTrait Creates new Link with specified Trait
func NewLinkWithTrait(trait *neat.Trait, weight float64, inputNode, outNode *NNode, recurrent bool) *Link {
	link := newLink(weight)
	link.InNode = inputNode
	link.OutNode = outNode
	link.IsRecurrent = recurrent
	link.Trait = trait
	link.deriveTrait(trait)
	return link
}

// NewLinkCopy The copy constructor to create new link with parameters taken from provided ones and connecting specified nodes
func NewLinkCopy(l *Link, inputNode, outNode *NNode) *Link {
	link := newLink(l.ConnectionWeight)
	link.InNode = inputNode
	link.OutNode = outNode
	link.Trait = l.Trait
	link.deriveTrait(l.Trait)
	link.IsRecurrent = l.IsRecurrent
	return link
}

// The private default constructor
func newLink(weight float64) *Link {
	return &Link{
		ConnectionWeight: weight,
	}
}

// IsEqualGenetically Checks if this link is genetically equal to provided one, i.e. connects nodes with the same IDs and has equal
// recurrent flag. I.e. if both links represent the same Gene.
func (l *Link) IsEqualGenetically(ol *Link) bool {
	sameInNode := l.InNode.Id == ol.InNode.Id
	sameOutNode := l.OutNode.Id == ol.OutNode.Id
	sameRecurrent := l.IsRecurrent == ol.IsRecurrent

	return sameInNode && sameOutNode && sameRecurrent
}

// The Link methods implementation
func (l *Link) String() string {
	return fmt.Sprintf("[Link: (%s <-> %s), weight: %.3f, recurrent: %t, time delayed: %t]",
		l.InNode, l.OutNode, l.ConnectionWeight, l.IsRecurrent, l.IsTimeDelayed)
}

// The Gonum Graph specific

// From returns the from node of the edge. Implements graph.Edge From method
func (l *Link) From() graph.Node {
	return l.InNode
}

// To returns the to node of the edge. Implements graph.To From method
func (l *Link) To() graph.Node {
	return l.OutNode
}

// Weight returns weight of this link
func (l *Link) Weight() float64 {
	return l.ConnectionWeight
}

// ReversedEdge returns the edge reversal of the receiver
// if a reversal is valid for the data type.
// When a reversal is valid an edge of the same type as
// the receiver with nodes of the receiver swapped should
// be returned, otherwise the receiver should be returned
// unaltered.
func (l *Link) ReversedEdge() graph.Edge {
	// the reversal is not valid - returning the same
	return l
}

// Copy trait parameters into this link's parameters
func (l *Link) deriveTrait(t *neat.Trait) {
	if t != nil {
		l.Params = make([]float64, len(t.Params))
		for i, p := range t.Params {
			l.Params[i] = p
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package geoip

import (
	"net"

	"github.com/corestoreio/csfw/util/errors"
	"github.com/oschwald/geoip2-golang"
)

// think about that ...
//type CountryTrimmed struct {
//	// IP contains the request IP address even if we run behind a proxy
//	IP      net.IP `json:"ip,omitempty"`
//	Country struct {
//		Confidence int               `json:"confidence,omitempty"`
//		GeoNameID  uint              `json:"geoname_id,omitempty"`
//		IsoCode    string            `json:"iso_code,omitempty"`
//	} `json:"country,omitempty"`
//	MaxMind struct {
//		QueriesRemaining int `json:"queries_remaining,omitempty"`
//	} `json:"maxmind,omitempty"`
//}

// The Country structure corresponds to the data in the GeoIP2/GeoLite2
// Country databases.
type Country struct {
	// IP contains the request IP address even if we run behind a proxy
	IP   net.IP `json:"ip,omitempty"`
	City struct {
		Confidence int               `json:"confidence,omitempty"`
		GeoNameID  uint              `json:"geoname_id,omitempty"`
		Names      map[string]string `json:"names,omitempty"`
	} `json:"city,omitempty"`
	Continent struct {
		Code      string            `json:"code,omitempty"`
		GeoNameID uint              `json:"geoname_id,omitempty"`
		Names     map[string]string `json:"names,omitempty"`
	} `json:"continent,omitempty"`
	Country struct {
		Confidence int               `json:"confidence,omitempty"`
		GeoNameID  uint              `json:"geoname_id,omitempty"`
		IsoCode    string            `json:"iso_code,omitempty"`
		Names      map[string]string `json:"names,omitempty"`
	} `json:"country,omitempty"`
	Location struct {
		AccuracyRadius    int     `json:"accuracy_radius,omitempty"`
		AverageIncome     int     `json:"average_income,omitempty"`
		Latitude          float64 `json:"latitude,omitempty"`
		Longitude         float64 `json:"longitude,omitempty"`
		MetroCode         int     `json:"metro_code,omitempty"`
		PopulationDensity int     `json:"population_density,omitempty"`
		TimeZone          string  `json:"time_zone,omitempty"`
	} `json:"location,omitempty"`
	Postal struct {
		Code       string `json:"code,omitempty"`
		Confidence int    `json:"confidence,omitempty"`
	} `json:"postal,omitempty"`
	RegisteredCountry struct {
		GeoNameID uint              `json:"geoname_id,omitempty"`
		IsoCode   string            `json:"iso_code,omitempty"`
		Names     map[string]string `json:"names,omitempty"`
	} `json:"registered_country,omitempty"`
	RepresentedCountry struct {
		GeoNameID uint              `json:"geoname_id,omitempty"`
		IsoCode   string            `json:"iso_code,omitempty"`
		Names     map[string]string `json:"names,omitempty"`
		Type      string            `json:"type,omitempty"`
	} `json:"represented_country,omitempty"`
	Subdivision []struct {
		Confidence int               `json:"confidence,omitempty"`
		GeoNameID  uint              `json:"geoname_id,omitempty"`
		IsoCode    string            `json:"iso_code,omitempty"`
		Names      map[string]string `json:"names,omitempty"`
	} `json:"subdivisions,omitempty"`
	Traits struct {
		AutonomousSystemNumber       int    `json:"autonomous_system_number,omitempty"`
		AutonomousSystemOrganization string `json:"autonomous_system_organization,omitempty"`
		Domain                       string `json:"domain,omitempty"`
		IsAnonymousProxy             bool   `json:"is_anonymous_proxy,omitempty"`
		IsSatelliteProvider          bool   `json:"is_satellite_provider,omitempty"`
		Isp                          string `json:"isp,omitempty"`
		IPAddress                    string `json:"ip_address,omitempty"`
		Organization                 string `json:"organization,omitempty"`
		UserType                     string `json:"user_type,omitempty"`
	} `json:"traits,omitempty"`
	MaxMind struct {
		QueriesRemaining int `json:"queries_remaining,omitempty"`
	} `json:"maxmind,omitempty"`
}

// CountryRetriever implements how to lookup the Country for an IP address.
// Supports IPv4 and IPv6 addresses.
type CountryRetriever interface {
	Country(net.IP) (*Country, error)
	// Close may be called on shutdown of the overall app and terminates
	// the underlying lookup service.
	Close() error
}

// mmdb internal wrapper between geoip2 and our interface
type mmdb struct {
	r *geoip2.Reader
}

func newMMDBByFile(filename string) (*mmdb, error) {
	r, err := geoip2.Open(filename)
	return &mmdb{r}, errors.NewNotValid(err, "[geoip] Maxmind Open")
}

func (mm *mmdb) Country(ipAddress net.IP) (*Country, error) {
	c, err := mm.r.Country(ipAddress)
	if err != nil {
		return nil, errors.NewNotValid(err, "[geoip] mmdb.Country")
	}
	c2 := &Country{
		IP: ipAddress,
	}
	c2.Continent.Code = c.Continent.Code
	c2.Continent.GeoNameID = c.Continent.GeoNameID
	c2.Continent.Names = c.Continent.Names // ! a map those names, should maybe copied away

	c2.Country.GeoNameID = c.Country.GeoNameID
	c2.Country.IsoCode = c.Country.IsoCode
	c2.Country.Names = c.Country.Names

	c2.RegisteredCountry.GeoNameID = c.RegisteredCountry.GeoNameID
	c2.RegisteredCountry.IsoCode = c.RegisteredCountry.IsoCode
	c2.RegisteredCountry.Names = c.RegisteredCountry.Names

	c2.RepresentedCountry.GeoNameID = c.RepresentedCountry.GeoNameID
	c2.RepresentedCountry.IsoCode = c.RepresentedCountry.IsoCode
	c2.RepresentedCountry.Names = c.RepresentedCountry.Names
	c2.RepresentedCountry.Type = c.RepresentedCountry.Type

	c2.Traits.IsAnonymousProxy = c.Traits.IsAnonymousProxy
	c2.Traits.IsSatelliteProvider = c.Traits.IsSatelliteProvider

	return c2, nil
}

func (mm *mmdb) Close() error {
	return mm.r.Close()
}
<commit_msg>net/geoip: Add todo to add a context when calling a webservice<commit_after>// Copyright 2015-2016, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package geoip

import (
	"net"

	"github.com/corestoreio/csfw/util/errors"
	"github.com/oschwald/geoip2-golang"
)

// think about that ...
//type CountryTrimmed struct {
//	// IP contains the request IP address even if we run behind a proxy
//	IP      net.IP `json:"ip,omitempty"`
//	Country struct {
//		Confidence int               `json:"confidence,omitempty"`
//		GeoNameID  uint              `json:"geoname_id,omitempty"`
//		IsoCode    string            `json:"iso_code,omitempty"`
//	} `json:"country,omitempty"`
//	MaxMind struct {
//		QueriesRemaining int `json:"queries_remaining,omitempty"`
//	} `json:"maxmind,omitempty"`
//}

// The Country structure corresponds to the data in the GeoIP2/GeoLite2
// Country databases.
type Country struct {
	// IP contains the request IP address even if we run behind a proxy
	IP   net.IP `json:"ip,omitempty"`
	City struct {
		Confidence int               `json:"confidence,omitempty"`
		GeoNameID  uint              `json:"geoname_id,omitempty"`
		Names      map[string]string `json:"names,omitempty"`
	} `json:"city,omitempty"`
	Continent struct {
		Code      string            `json:"code,omitempty"`
		GeoNameID uint              `json:"geoname_id,omitempty"`
		Names     map[string]string `json:"names,omitempty"`
	} `json:"continent,omitempty"`
	Country struct {
		Confidence int               `json:"confidence,omitempty"`
		GeoNameID  uint              `json:"geoname_id,omitempty"`
		IsoCode    string            `json:"iso_code,omitempty"`
		Names      map[string]string `json:"names,omitempty"`
	} `json:"country,omitempty"`
	Location struct {
		AccuracyRadius    int     `json:"accuracy_radius,omitempty"`
		AverageIncome     int     `json:"average_income,omitempty"`
		Latitude          float64 `json:"latitude,omitempty"`
		Longitude         float64 `json:"longitude,omitempty"`
		MetroCode         int     `json:"metro_code,omitempty"`
		PopulationDensity int     `json:"population_density,omitempty"`
		TimeZone          string  `json:"time_zone,omitempty"`
	} `json:"location,omitempty"`
	Postal struct {
		Code       string `json:"code,omitempty"`
		Confidence int    `json:"confidence,omitempty"`
	} `json:"postal,omitempty"`
	RegisteredCountry struct {
		GeoNameID uint              `json:"geoname_id,omitempty"`
		IsoCode   string            `json:"iso_code,omitempty"`
		Names     map[string]string `json:"names,omitempty"`
	} `json:"registered_country,omitempty"`
	RepresentedCountry struct {
		GeoNameID uint              `json:"geoname_id,omitempty"`
		IsoCode   string            `json:"iso_code,omitempty"`
		Names     map[string]string `json:"names,omitempty"`
		Type      string            `json:"type,omitempty"`
	} `json:"represented_country,omitempty"`
	Subdivision []struct {
		Confidence int               `json:"confidence,omitempty"`
		GeoNameID  uint              `json:"geoname_id,omitempty"`
		IsoCode    string            `json:"iso_code,omitempty"`
		Names      map[string]string `json:"names,omitempty"`
	} `json:"subdivisions,omitempty"`
	Traits struct {
		AutonomousSystemNumber       int    `json:"autonomous_system_number,omitempty"`
		AutonomousSystemOrganization string `json:"autonomous_system_organization,omitempty"`
		Domain                       string `json:"domain,omitempty"`
		IsAnonymousProxy             bool   `json:"is_anonymous_proxy,omitempty"`
		IsSatelliteProvider          bool   `json:"is_satellite_provider,omitempty"`
		Isp                          string `json:"isp,omitempty"`
		IPAddress                    string `json:"ip_address,omitempty"`
		Organization                 string `json:"organization,omitempty"`
		UserType                     string `json:"user_type,omitempty"`
	} `json:"traits,omitempty"`
	MaxMind struct {
		QueriesRemaining int `json:"queries_remaining,omitempty"`
	} `json:"maxmind,omitempty"`
}

// CountryRetriever implements how to lookup the Country for an IP address.
// Supports IPv4 and IPv6 addresses.
type CountryRetriever interface {
	// Country todo add context for cancelling
	Country(net.IP) (*Country, error)
	// Close may be called on shutdown of the overall app and terminates
	// the underlying lookup service.
	Close() error
}

// mmdb internal wrapper between geoip2 and our interface
type mmdb struct {
	r *geoip2.Reader
}

func newMMDBByFile(filename string) (*mmdb, error) {
	r, err := geoip2.Open(filename)
	return &mmdb{r}, errors.NewNotValid(err, "[geoip] Maxmind Open")
}

func (mm *mmdb) Country(ipAddress net.IP) (*Country, error) {
	c, err := mm.r.Country(ipAddress)
	if err != nil {
		return nil, errors.NewNotValid(err, "[geoip] mmdb.Country")
	}
	c2 := &Country{
		IP: ipAddress,
	}
	c2.Continent.Code = c.Continent.Code
	c2.Continent.GeoNameID = c.Continent.GeoNameID
	c2.Continent.Names = c.Continent.Names // ! a map those names, should maybe copied away

	c2.Country.GeoNameID = c.Country.GeoNameID
	c2.Country.IsoCode = c.Country.IsoCode
	c2.Country.Names = c.Country.Names

	c2.RegisteredCountry.GeoNameID = c.RegisteredCountry.GeoNameID
	c2.RegisteredCountry.IsoCode = c.RegisteredCountry.IsoCode
	c2.RegisteredCountry.Names = c.RegisteredCountry.Names

	c2.RepresentedCountry.GeoNameID = c.RepresentedCountry.GeoNameID
	c2.RepresentedCountry.IsoCode = c.RepresentedCountry.IsoCode
	c2.RepresentedCountry.Names = c.RepresentedCountry.Names
	c2.RepresentedCountry.Type = c.RepresentedCountry.Type

	c2.Traits.IsAnonymousProxy = c.Traits.IsAnonymousProxy
	c2.Traits.IsSatelliteProvider = c.Traits.IsSatelliteProvider

	return c2, nil
}

func (mm *mmdb) Close() error {
	return mm.r.Close()
}
<|endoftext|>


<commit_before>//
// Copyright 2015 Gregory Trubetskoy. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package daemon

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/tgres/tgres/misc"
	"github.com/tgres/tgres/rrd"
	"github.com/tgres/tgres/serde"
)

type Config struct { // Needs to be exported for TOML to work
	PidPath                  string   `toml:"pid-file"`
	LogPath                  string   `toml:"log-file"`
	LogCycle                 duration `toml:"log-cycle-interval"`
	DbConnectString          string   `toml:"db-connect-string"`
	MinStep                  duration `toml:"min-step"`
	MaxReceiverQueueSize     int      `toml:"max-receiver-queue-size"`
	GraphiteTextListenSpec   string   `toml:"graphite-text-listen-spec"`
	GraphiteUdpListenSpec    string   `toml:"graphite-udp-listen-spec"`
	GraphitePickleListenSpec string   `toml:"graphite-pickle-listen-spec"`
	StatsdTextListenSpec     string   `toml:"statsd-text-listen-spec"`
	StatsdUdpListenSpec      string   `toml:"statsd-udp-listen-spec"`
	HttpListenSpec           string   `toml:"http-listen-spec"`
	Workers                  int
	DSs                      []ConfigDSSpec `toml:"ds"`
	StatFlush                duration       `toml:"stat-flush-interval"`
	StatsNamePrefix          string         `toml:"stats-name-prefix"`
}

type regex struct{ *regexp.Regexp }

func (r *regex) UnmarshalText(text []byte) (err error) {
	r.Regexp, err = regexp.Compile(string(text))
	return err
}

type duration struct{ time.Duration }

func (d *duration) UnmarshalText(text []byte) (err error) {
	d.Duration, err = time.ParseDuration(string(text))
	return err
}

// Needs to be exported for TOML
type ConfigDSSpec struct {
	Regexp    regex
	Step      duration
	Heartbeat duration
	RRAs      []ConfigRRASpec
}
type ConfigRRASpec struct {
	Function rrd.Consolidation
	Step     time.Duration
	Span     time.Duration
	Xff      float64
}

func (r *ConfigRRASpec) UnmarshalText(text []byte) error {
	r.Xff = 0.5
	parts := strings.SplitN(string(text), ":", 4)
	if len(parts) < 2 || len(parts) > 4 {
		return fmt.Errorf("Invalid RRA specification (not enough or too many elements): %q", string(text))
	}

	// If first character of first part is a digit, assume we're
	// skipping CF and default to WMEAN.
	if len(parts[0]) > 0 && strings.Contains("0123456789", string(parts[0][0])) {
		parts = append([]string{"WMEAN"}, parts...)
	}

	switch strings.ToUpper(parts[0]) {
	case "WMEAN":
		r.Function = rrd.WMEAN
	case "MIN":
		r.Function = rrd.MIN
	case "MAX":
		r.Function = rrd.MAX
	case "LAST":
		r.Function = rrd.LAST
	default:
		return fmt.Errorf("Invalid consolidation: %q (valid funcs: wmean, min, max, last)", parts[0])
	}

	var err error
	if r.Step, err = misc.BetterParseDuration(parts[1]); err != nil {
		return fmt.Errorf("Invalid Step: %q (%v)", parts[1], err)
	}
	if r.Span, err = misc.BetterParseDuration(parts[2]); err != nil {
		return fmt.Errorf("Invalid Size: %q (%v)", parts[2], err)
	}
	if (r.Span.Nanoseconds() % r.Step.Nanoseconds()) != 0 {
		newSpan := time.Duration(r.Span.Nanoseconds()/r.Step.Nanoseconds()*r.Step.Nanoseconds()) * time.Nanosecond
		log.Printf("Span (%q) is not a multiple of step (%q), auto adjusting span to %v.", parts[2], parts[1], newSpan)
		r.Span = newSpan
		if newSpan.Nanoseconds() == 0 {
			return fmt.Errorf("invalid Size (%v)", newSpan)
		}
	}
	if len(parts) == 4 {
		var err error
		if r.Xff, err = strconv.ParseFloat(parts[3], 64); err != nil {
			return fmt.Errorf("Invalid XFF: %q (%v)", parts[3], err)
		}
	}
	return nil
}

var readConfig = func(cfgPath string) (*Config, error) {
	cfg := &Config{}
	_, err := toml.DecodeFile(cfgPath, cfg)
	if err != nil {
		return nil, err
	}
	return cfg, nil
}

func (c *Config) processConfigPidFile(wd string) error {
	if c.PidPath == "" {
		return fmt.Errorf("pid-file setting empty")
	}
	if !filepath.IsAbs(c.PidPath) {
		if wd == "" {
			return fmt.Errorf("pid-file must be absolute path if working directory cannot be determined")
		}
		c.PidPath = filepath.Join(wd, c.PidPath)
	}
	pidDir, _ := filepath.Split(c.PidPath)
	if err := os.MkdirAll(pidDir, 0755); err != nil {
		return errors.New(fmt.Sprintf("Unable to create directory: '%s' (%v).", pidDir, err))
	}
	return nil
}

func (c *Config) processConfigLogFile(wd string) error {
	if os.Getenv("TGRES_LOG") != "" {
		c.LogPath = os.Getenv("TGRES_LOG")
	}
	if c.LogPath == "" {
		return fmt.Errorf("log-file setting empty")
	}
	if !filepath.IsAbs(c.LogPath) {
		if wd == "" {
			return fmt.Errorf("log-file must be absolute path if working directory cannot be determined")
		}
		c.LogPath = filepath.Join(wd, c.LogPath)
	}
	logDir, _ := filepath.Split(c.LogPath)
	if err := os.MkdirAll(logDir, 0755); err != nil {
		return errors.New(fmt.Sprintf("Unable to create directory: '%s' (%v).", logDir, err))
	}

	log.Printf("Logs will be written to '%s'.", c.LogPath)
	return nil
}

func (c *Config) processConfigLogCycleInterval() error {
	if c.LogCycle.Duration == 0 {
		return fmt.Errorf("log-cycle-interval setting empty")
	}
	log.Printf("Will cycle logs every %v (log-cycle-interval).", c.LogCycle.Duration)

	logDir, _ := filepath.Split(c.LogPath)
	log.Printf("All further status messages will be written to log file(s) in '%s'.", logDir)
	logFileCycler(c.LogPath, c.LogCycle.Duration)
	log.Print("Server starting.")

	return nil
}

func (c *Config) processDbConnectString() error {
	if os.Getenv("TGRES_DB_CONNECT") != "" {
		c.DbConnectString = os.Getenv("TGRES_DB_CONNECT")
	}
	if c.DbConnectString == "" {
		return fmt.Errorf("db-connect-string empty")
	}
	return nil
}

func (c *Config) processMinStep() error {
	if c.MinStep.Duration == 0 {
		return fmt.Errorf("min-step is missing")
	} else {
		log.Printf("Smallest step allowed: %v (min-step).", c.MinStep.Duration)
	}
	return nil
}

func (c *Config) processMaxReceiverQueueSize() error {
	if c.MaxReceiverQueueSize == 0 {
		return fmt.Errorf("max-receiver-queue-size unspecified, defaults to 0 (unlimited)")
	} else if c.MaxReceiverQueueSize <= 0 {
		log.Printf("Receiver Queue Size is unlimited (%d) (max-receiver-queue-size).", c.MaxReceiverQueueSize)
	} else {
		log.Printf("Receiver Queue Size is limited to %d (max-receiver-queue-size).", c.MaxReceiverQueueSize)
	}
	return nil
}

func (c *Config) processStatFlushInterval() error {
	if c.StatFlush.Duration == 0 {
		return fmt.Errorf("stat-flush-interval is missing")
	} else {
		log.Printf("Stats (a la statsd) will be flushed every %v (stat-flush-interval).", c.StatFlush.Duration)
	}
	return nil
}

func (c *Config) processStatsNamePrefix() error {
	if c.StatsNamePrefix == "" {
		log.Printf("stats-name-prefix is empty, defaulting to 'stats'")
		c.StatsNamePrefix = "stats"

	}
	return nil
}

func (c *Config) processWorkers() error {
	if c.Workers == 0 {
		return fmt.Errorf("workers missing, must be an integer")
	}
	log.Printf("Number of workers (and flushers) will be %d.", c.Workers)
	return nil
}

func (c *Config) processDSSpec() error {
	// TODO validate function, regular expression, all that
	for _, ds := range c.DSs {
		for _, rra := range ds.RRAs {
			if (rra.Step.Nanoseconds() % c.MinStep.Nanoseconds()) != 0 {
				return fmt.Errorf("DS %q: invalid Step (%v), must be one or multiple min-step (%v).", ds.Regexp.String(), rra.Step, c.MinStep)
			}
			if (rra.Step.Nanoseconds() % ds.Step.Duration.Nanoseconds()) != 0 {
				newStep := time.Duration(rra.Step.Nanoseconds()/ds.Step.Duration.Nanoseconds()*ds.Step.Duration.Nanoseconds()) * time.Nanosecond
				log.Printf("DS %q: RRA step (%v) is not a multiple of DS Step (%v), auto adjusting Step to %v.", ds.Regexp.String(), rra.Step, ds.Step.Duration, newStep)
				if newStep.Nanoseconds() == 0 {
					return fmt.Errorf("DS %q: invalid Step (%v)", ds.Regexp.String(), newStep)
				}
				rra.Step = newStep
			}
		}
	}
	// TODO xff?
	return nil
}

func (c *Config) FindMatchingDSSpec(ident serde.Ident) *rrd.DSSpec {
	for _, dsSpec := range c.DSs {
		name := ident["name"]
		if dsSpec.Regexp.Regexp.MatchString(name) {
			return convertDSSpec(&dsSpec)
		}
	}
	return nil
}

func convertDSSpec(dsSpec *ConfigDSSpec) *rrd.DSSpec {
	serdeDSSpec := &rrd.DSSpec{
		Step:      dsSpec.Step.Duration,
		Heartbeat: dsSpec.Heartbeat.Duration,
		RRAs:      make([]rrd.RRASpec, len(dsSpec.RRAs)),
	}
	for i, r := range dsSpec.RRAs {
		serdeDSSpec.RRAs[i] = rrd.RRASpec{
			Function: r.Function,
			Step:     r.Step,
			Span:     r.Span,
			Xff:      float32(r.Xff),
		}
	}
	return serdeDSSpec
}

type configer interface {
	processConfigPidFile(string) error
	processConfigLogFile(string) error
	processConfigLogCycleInterval() error
	processDbConnectString() error
	processMinStep() error
	processMaxReceiverQueueSize() error
	processStatFlushInterval() error
	processStatsNamePrefix() error
	processWorkers() error
	processDSSpec() error
}

var processConfig = func(c configer, wd string) error {

	if err := c.processConfigPidFile(wd); err != nil {
		return err
	}
	if err := c.processConfigLogFile(wd); err != nil {
		return err
	}
	if err := c.processConfigLogCycleInterval(); err != nil {
		return err
	}
	if err := c.processDbConnectString(); err != nil {
		return err
	}
	if err := c.processMinStep(); err != nil {
		return err
	}
	if err := c.processMaxReceiverQueueSize(); err != nil {
		return err
	}
	if err := c.processStatFlushInterval(); err != nil {
		return err
	}
	if err := c.processStatsNamePrefix(); err != nil {
		return err
	}
	if err := c.processWorkers(); err != nil {
		return err
	}
	if err := c.processDSSpec(); err != nil {
		return err
	}
	return nil
}
<commit_msg>max-receiver-queue-size should be optional in the config<commit_after>//
// Copyright 2015 Gregory Trubetskoy. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package daemon

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/tgres/tgres/misc"
	"github.com/tgres/tgres/rrd"
	"github.com/tgres/tgres/serde"
)

type Config struct { // Needs to be exported for TOML to work
	PidPath                  string   `toml:"pid-file"`
	LogPath                  string   `toml:"log-file"`
	LogCycle                 duration `toml:"log-cycle-interval"`
	DbConnectString          string   `toml:"db-connect-string"`
	MinStep                  duration `toml:"min-step"`
	MaxReceiverQueueSize     int      `toml:"max-receiver-queue-size"`
	GraphiteTextListenSpec   string   `toml:"graphite-text-listen-spec"`
	GraphiteUdpListenSpec    string   `toml:"graphite-udp-listen-spec"`
	GraphitePickleListenSpec string   `toml:"graphite-pickle-listen-spec"`
	StatsdTextListenSpec     string   `toml:"statsd-text-listen-spec"`
	StatsdUdpListenSpec      string   `toml:"statsd-udp-listen-spec"`
	HttpListenSpec           string   `toml:"http-listen-spec"`
	Workers                  int
	DSs                      []ConfigDSSpec `toml:"ds"`
	StatFlush                duration       `toml:"stat-flush-interval"`
	StatsNamePrefix          string         `toml:"stats-name-prefix"`
}

type regex struct{ *regexp.Regexp }

func (r *regex) UnmarshalText(text []byte) (err error) {
	r.Regexp, err = regexp.Compile(string(text))
	return err
}

type duration struct{ time.Duration }

func (d *duration) UnmarshalText(text []byte) (err error) {
	d.Duration, err = time.ParseDuration(string(text))
	return err
}

// Needs to be exported for TOML
type ConfigDSSpec struct {
	Regexp    regex
	Step      duration
	Heartbeat duration
	RRAs      []ConfigRRASpec
}
type ConfigRRASpec struct {
	Function rrd.Consolidation
	Step     time.Duration
	Span     time.Duration
	Xff      float64
}

func (r *ConfigRRASpec) UnmarshalText(text []byte) error {
	r.Xff = 0.5
	parts := strings.SplitN(string(text), ":", 4)
	if len(parts) < 2 || len(parts) > 4 {
		return fmt.Errorf("Invalid RRA specification (not enough or too many elements): %q", string(text))
	}

	// If first character of first part is a digit, assume we're
	// skipping CF and default to WMEAN.
	if len(parts[0]) > 0 && strings.Contains("0123456789", string(parts[0][0])) {
		parts = append([]string{"WMEAN"}, parts...)
	}

	switch strings.ToUpper(parts[0]) {
	case "WMEAN":
		r.Function = rrd.WMEAN
	case "MIN":
		r.Function = rrd.MIN
	case "MAX":
		r.Function = rrd.MAX
	case "LAST":
		r.Function = rrd.LAST
	default:
		return fmt.Errorf("Invalid consolidation: %q (valid funcs: wmean, min, max, last)", parts[0])
	}

	var err error
	if r.Step, err = misc.BetterParseDuration(parts[1]); err != nil {
		return fmt.Errorf("Invalid Step: %q (%v)", parts[1], err)
	}
	if r.Span, err = misc.BetterParseDuration(parts[2]); err != nil {
		return fmt.Errorf("Invalid Size: %q (%v)", parts[2], err)
	}
	if (r.Span.Nanoseconds() % r.Step.Nanoseconds()) != 0 {
		newSpan := time.Duration(r.Span.Nanoseconds()/r.Step.Nanoseconds()*r.Step.Nanoseconds()) * time.Nanosecond
		log.Printf("Span (%q) is not a multiple of step (%q), auto adjusting span to %v.", parts[2], parts[1], newSpan)
		r.Span = newSpan
		if newSpan.Nanoseconds() == 0 {
			return fmt.Errorf("invalid Size (%v)", newSpan)
		}
	}
	if len(parts) == 4 {
		var err error
		if r.Xff, err = strconv.ParseFloat(parts[3], 64); err != nil {
			return fmt.Errorf("Invalid XFF: %q (%v)", parts[3], err)
		}
	}
	return nil
}

var readConfig = func(cfgPath string) (*Config, error) {
	cfg := &Config{}
	_, err := toml.DecodeFile(cfgPath, cfg)
	if err != nil {
		return nil, err
	}
	return cfg, nil
}

func (c *Config) processConfigPidFile(wd string) error {
	if c.PidPath == "" {
		return fmt.Errorf("pid-file setting empty")
	}
	if !filepath.IsAbs(c.PidPath) {
		if wd == "" {
			return fmt.Errorf("pid-file must be absolute path if working directory cannot be determined")
		}
		c.PidPath = filepath.Join(wd, c.PidPath)
	}
	pidDir, _ := filepath.Split(c.PidPath)
	if err := os.MkdirAll(pidDir, 0755); err != nil {
		return errors.New(fmt.Sprintf("Unable to create directory: '%s' (%v).", pidDir, err))
	}
	return nil
}

func (c *Config) processConfigLogFile(wd string) error {
	if os.Getenv("TGRES_LOG") != "" {
		c.LogPath = os.Getenv("TGRES_LOG")
	}
	if c.LogPath == "" {
		return fmt.Errorf("log-file setting empty")
	}
	if !filepath.IsAbs(c.LogPath) {
		if wd == "" {
			return fmt.Errorf("log-file must be absolute path if working directory cannot be determined")
		}
		c.LogPath = filepath.Join(wd, c.LogPath)
	}
	logDir, _ := filepath.Split(c.LogPath)
	if err := os.MkdirAll(logDir, 0755); err != nil {
		return errors.New(fmt.Sprintf("Unable to create directory: '%s' (%v).", logDir, err))
	}

	log.Printf("Logs will be written to '%s'.", c.LogPath)
	return nil
}

func (c *Config) processConfigLogCycleInterval() error {
	if c.LogCycle.Duration == 0 {
		return fmt.Errorf("log-cycle-interval setting empty")
	}
	log.Printf("Will cycle logs every %v (log-cycle-interval).", c.LogCycle.Duration)

	logDir, _ := filepath.Split(c.LogPath)
	log.Printf("All further status messages will be written to log file(s) in '%s'.", logDir)
	logFileCycler(c.LogPath, c.LogCycle.Duration)
	log.Print("Server starting.")

	return nil
}

func (c *Config) processDbConnectString() error {
	if os.Getenv("TGRES_DB_CONNECT") != "" {
		c.DbConnectString = os.Getenv("TGRES_DB_CONNECT")
	}
	if c.DbConnectString == "" {
		return fmt.Errorf("db-connect-string empty")
	}
	return nil
}

func (c *Config) processMinStep() error {
	if c.MinStep.Duration == 0 {
		return fmt.Errorf("min-step is missing")
	} else {
		log.Printf("Smallest step allowed: %v (min-step).", c.MinStep.Duration)
	}
	return nil
}

func (c *Config) processMaxReceiverQueueSize() error {
	if c.MaxReceiverQueueSize == 0 {
		log.Printf("max-receiver-queue-size unspecified, defaults to 0 (unlimited)")
	} else if c.MaxReceiverQueueSize <= 0 {
		log.Printf("Receiver Queue Size is unlimited (%d) (max-receiver-queue-size).", c.MaxReceiverQueueSize)
	} else {
		log.Printf("Receiver Queue Size is limited to %d (max-receiver-queue-size).", c.MaxReceiverQueueSize)
	}
	return nil
}

func (c *Config) processStatFlushInterval() error {
	if c.StatFlush.Duration == 0 {
		return fmt.Errorf("stat-flush-interval is missing")
	} else {
		log.Printf("Stats (a la statsd) will be flushed every %v (stat-flush-interval).", c.StatFlush.Duration)
	}
	return nil
}

func (c *Config) processStatsNamePrefix() error {
	if c.StatsNamePrefix == "" {
		log.Printf("stats-name-prefix is empty, defaulting to 'stats'")
		c.StatsNamePrefix = "stats"

	}
	return nil
}

func (c *Config) processWorkers() error {
	if c.Workers == 0 {
		return fmt.Errorf("workers missing, must be an integer")
	}
	log.Printf("Number of workers (and flushers) will be %d.", c.Workers)
	return nil
}

func (c *Config) processDSSpec() error {
	// TODO validate function, regular expression, all that
	for _, ds := range c.DSs {
		for _, rra := range ds.RRAs {
			if (rra.Step.Nanoseconds() % c.MinStep.Nanoseconds()) != 0 {
				return fmt.Errorf("DS %q: invalid Step (%v), must be one or multiple min-step (%v).", ds.Regexp.String(), rra.Step, c.MinStep)
			}
			if (rra.Step.Nanoseconds() % ds.Step.Duration.Nanoseconds()) != 0 {
				newStep := time.Duration(rra.Step.Nanoseconds()/ds.Step.Duration.Nanoseconds()*ds.Step.Duration.Nanoseconds()) * time.Nanosecond
				log.Printf("DS %q: RRA step (%v) is not a multiple of DS Step (%v), auto adjusting Step to %v.", ds.Regexp.String(), rra.Step, ds.Step.Duration, newStep)
				if newStep.Nanoseconds() == 0 {
					return fmt.Errorf("DS %q: invalid Step (%v)", ds.Regexp.String(), newStep)
				}
				rra.Step = newStep
			}
		}
	}
	// TODO xff?
	return nil
}

func (c *Config) FindMatchingDSSpec(ident serde.Ident) *rrd.DSSpec {
	for _, dsSpec := range c.DSs {
		name := ident["name"]
		if dsSpec.Regexp.Regexp.MatchString(name) {
			return convertDSSpec(&dsSpec)
		}
	}
	return nil
}

func convertDSSpec(dsSpec *ConfigDSSpec) *rrd.DSSpec {
	serdeDSSpec := &rrd.DSSpec{
		Step:      dsSpec.Step.Duration,
		Heartbeat: dsSpec.Heartbeat.Duration,
		RRAs:      make([]rrd.RRASpec, len(dsSpec.RRAs)),
	}
	for i, r := range dsSpec.RRAs {
		serdeDSSpec.RRAs[i] = rrd.RRASpec{
			Function: r.Function,
			Step:     r.Step,
			Span:     r.Span,
			Xff:      float32(r.Xff),
		}
	}
	return serdeDSSpec
}

type configer interface {
	processConfigPidFile(string) error
	processConfigLogFile(string) error
	processConfigLogCycleInterval() error
	processDbConnectString() error
	processMinStep() error
	processMaxReceiverQueueSize() error
	processStatFlushInterval() error
	processStatsNamePrefix() error
	processWorkers() error
	processDSSpec() error
}

var processConfig = func(c configer, wd string) error {

	if err := c.processConfigPidFile(wd); err != nil {
		return err
	}
	if err := c.processConfigLogFile(wd); err != nil {
		return err
	}
	if err := c.processConfigLogCycleInterval(); err != nil {
		return err
	}
	if err := c.processDbConnectString(); err != nil {
		return err
	}
	if err := c.processMinStep(); err != nil {
		return err
	}
	if err := c.processMaxReceiverQueueSize(); err != nil {
		return err
	}
	if err := c.processStatFlushInterval(); err != nil {
		return err
	}
	if err := c.processStatsNamePrefix(); err != nil {
		return err
	}
	if err := c.processWorkers(); err != nil {
		return err
	}
	if err := c.processDSSpec(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package daemon

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ant0ine/go-json-rest"
	"github.com/nytlabs/streamtools/blocks"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

const (
	READ_MAX = 1024768
)

var (
	// channel that returns the next ID
	idChan chan string
)

// Daemon keeps track of all the blocks and connections
type Daemon struct {
	blockMap map[string]*blocks.Block
}

// The rootHandler returns information about the whole system
func (d *Daemon) rootHandler(w *rest.ResponseWriter, r *rest.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(index())
}

// The createHandler creates new blocks
func (d *Daemon) createHandler(w *rest.ResponseWriter, r *rest.Request) {
	err := r.ParseForm()
	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	var id string
	var blockType string
	fType, typeExists := r.Form["blockType"]
	fID, idExists := r.Form["id"]

	if typeExists == false {
		ApiResponse(w, 500, "MISSING_BLOCKTYPE")
		return
	} else {
		blockType = fType[0]
	}

	_, inLibrary := blocks.Library[blockType]
	if inLibrary == false {
		ApiResponse(w, 500, "INVALID_BLOCKTYPE")
		return
	}

	if idExists == false {
		id = <-idChan
	} else {

		if len(strings.TrimSpace(fID[0])) == 0 {
			ApiResponse(w, 500, "BAD_BLOCK_ID")
			return
		}

		_, notUnique := d.blockMap[fID[0]]
		if notUnique == true {
			ApiResponse(w, 500, "BLOCK_ID_ALREADY_EXISTS")
			return
		} else {
			id = fID[0]
		}
	}

	d.CreateBlock(blockType, id)

	ApiResponse(w, 200, "BLOCK_CREATED")
}

func (d *Daemon) deleteHandler(w *rest.ResponseWriter, r *rest.Request) {
	err := r.ParseForm()
	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	_, hasID := r.Form["id"]
	if hasID == false {
		ApiResponse(w, 500, "MISSING_BLOCK_ID")
		return
	}

	id := r.Form["id"][0]

	err = d.DeleteBlock(id)

	if err != nil {
		ApiResponse(w, 500, err.Error())
		return
	}

	ApiResponse(w, 200, "BLOCK_DELETED")
}

func (d *Daemon) DeleteBlock(id string) error {
	block, ok := d.blockMap[id]
	if ok == false {
		return errors.New("BLOCK_NOT_FOUND")
	}

	// delete inbound channels
	for k, _ := range block.InBlocks {
		d.blockMap[k].AddChan <- &blocks.OutChanMsg{
			Action: blocks.DELETE_OUT_CHAN,
			ID:     block.ID,
		}

		log.Println("disconnecting \"" + block.ID + "\" from \"" + k + "\"")

		delete(d.blockMap[k].OutBlocks, block.ID)

		if d.blockMap[k].BlockType == "connection" {
			d.DeleteBlock(k)
		}
	}

	// delete outbound channels
	for k, _ := range block.OutBlocks {
		delete(d.blockMap[k].InBlocks, block.ID)
		if d.blockMap[k].BlockType == "connection" {
			d.DeleteBlock(k)
		}
	}

	// delete the block itself
	block.QuitChan <- true
	delete(d.blockMap, id)

	return nil
}

// The connectHandler connects together two blocks
func (d *Daemon) connectHandler(w *rest.ResponseWriter, r *rest.Request) {
	var id string

	err := r.ParseForm()
	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	_, hasFrom := r.Form["from"]
	if hasFrom == false {
		ApiResponse(w, 500, "MISSING_FROM_BLOCK_ID")
		return
	}

	_, hasTo := r.Form["to"]
	if hasTo == false {
		ApiResponse(w, 500, "MISSING_TO_BLOCK_ID")
		return
	}

	fID, hasID := r.Form["id"]
	if hasID == false {
		id = <-idChan
	} else {

		if len(strings.TrimSpace(fID[0])) == 0 {
			ApiResponse(w, 500, "BAD_CONNECTION_ID")
			return
		}

		_, ok := d.blockMap[fID[0]]
		if ok == false {
			id = fID[0]
		} else {
			ApiResponse(w, 500, "BLOCK_ID_ALREADY_EXISTS")
			return
		}
	}

	from := r.Form["from"][0]
	to := r.Form["to"][0]

	if len(from) == 0 {
		ApiResponse(w, 500, "MISSING_FROM_BLOCK_ID")
		return
	}

	if len(to) == 0 {
		ApiResponse(w, 500, "MISSING_TO_BLOCK_ID")
		return
	}

	_, exists := d.blockMap[from]
	if exists == false {
		ApiResponse(w, 500, "FROM_BLOCK_NOT_FOUND")
		return
	}

	_, exists = d.blockMap[strings.Split(to, "/")[0]]
	if exists == false {
		ApiResponse(w, 500, "TO_BLOCK_NOT_FOUND")
		return
	}

	err = d.CreateConnection(from, to, id)
	if err != nil {
		ApiResponse(w, 500, "TO_ROUTE_NOT_FOUND")
		return
	}

	ApiResponse(w, 200, "CONNECTION_CREATED")
}

// The routeHandler deals with any incoming message sent to an arbitrary block endpoint
func (d *Daemon) routeHandler(w *rest.ResponseWriter, r *rest.Request) {
	//id := strings.Split(r.URL.Path, "/")[2]
	//route := strings.Split(r.URL.Path, "/")[3]

	id, ok := r.PathParams["id"]
	if ok == false {
		ApiResponse(w, 500, "MISSING_BLOCK_ID")
		return
	}

	route, ok := r.PathParams["route"]
	if ok == false {
		ApiResponse(w, 500, "MISSING_ROUTE")
		return
	}

	_, ok = d.blockMap[id]
	if ok == false {
		ApiResponse(w, 500, "BLOCK_ID_NOT_FOUND")
		return
	}

	_, ok = d.blockMap[id].Routes[route]
	if ok == false {
		ApiResponse(w, 500, "ROUTE_NOT_FOUND")
		return
	}

	msg, err := ioutil.ReadAll(io.LimitReader(r.Body, READ_MAX))

	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	var outMsg blocks.BMsg

	if len(msg) > 0 {
		err = json.Unmarshal(msg, &outMsg)
		if err != nil {
			log.Println(msg)
			ApiResponse(w, 500, "BAD_JSON")
			return
		}
	}

	ResponseChan := make(chan blocks.BMsg)
	blockRouteChan := d.blockMap[id].Routes[route]
	blockRouteChan <- blocks.RouteResponse{
		Msg:          outMsg,
		ResponseChan: ResponseChan,
	}
	respMsg := <-ResponseChan

	respJson, err := json.Marshal(respMsg)
	if err != nil {
		ApiResponse(w, 500, "BAD_RESPONSE_FROM_BLOCK")
		return
	}

	DataResponse(w, respJson)
}

func (d *Daemon) libraryHandler(w *rest.ResponseWriter, r *rest.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(len(blocks.LibraryBlob)))
	fmt.Fprint(w, blocks.LibraryBlob)
}

func (d *Daemon) listHandler(w *rest.ResponseWriter, r *rest.Request) {
	blockList := []map[string]interface{}{}
	for _, v := range d.blockMap {
		blockItem := make(map[string]interface{})
		blockItem["BlockType"] = v.BlockType
		blockItem["ID"] = v.ID
		blockItem["InBlocks"] = []string{}
		blockItem["OutBlocks"] = []string{}
		blockItem["Routes"] = []string{}
		for k, _ := range v.InBlocks {
			blockItem["InBlocks"] = append(blockItem["InBlocks"].([]string), k)
		}
		for k, _ := range v.OutBlocks {
			blockItem["OutBlocks"] = append(blockItem["OutBlocks"].([]string), k)
		}
		for k, _ := range v.Routes {
			blockItem["Routes"] = append(blockItem["Routes"].([]string), k)
		}
		blockList = append(blockList, blockItem)
	}
	blob, _ := json.Marshal(blockList)

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(len(blob)))
	fmt.Fprint(w, string(blob))
}

func (d *Daemon) CreateConnection(from string, to string, ID string) error {
	d.CreateBlock("connection", ID)

	d.blockMap[from].AddChan <- &blocks.OutChanMsg{
		Action:  blocks.CREATE_OUT_CHAN,
		OutChan: d.blockMap[ID].InChan,
		ID:      ID,
	}

	/*d.blockMap[ID].AddChan <- &blocks.OutChanMsg{
		Action:  blocks.CREATE_OUT_CHAN,
		OutChan: d.blockMap[to].InChan,
		ID:      to,
	}*/
	toParts := strings.Split(to, "/")
	switch len(toParts) {
	case 1:
		d.blockMap[ID].AddChan <- &blocks.OutChanMsg{
			Action:  blocks.CREATE_OUT_CHAN,
			OutChan: d.blockMap[to].InChan,
			ID:      to,
		}
	case 2:
		d.blockMap[ID].AddChan <- &blocks.OutChanMsg{
			Action:  blocks.CREATE_OUT_CHAN,
			OutChan: d.blockMap[toParts[0]].Routes[toParts[1]],
			ID:      to,
		}
	default:
		err := errors.New("malformed to route specification")
		return err
	}

	// add the from block to the list of inblocks for connection.
	d.blockMap[from].OutBlocks[ID] = true
	d.blockMap[ID].InBlocks[from] = true
	d.blockMap[ID].OutBlocks[toParts[0]] = true
	d.blockMap[toParts[0]].InBlocks[ID] = true

	log.Println("connected", d.blockMap[from].ID, "to", d.blockMap[toParts[0]].ID)

	return nil
}

func (d *Daemon) CreateBlock(name string, ID string) {
	// TODO: Clean this up.
	//
	// In order to avoid data races the blocks held in daemon's blockMap
	// are not the same blocks held in each block routine. When CreateBlock
	// is called, we actually create two blocks: one to store in daemon's
	// blockMap and one to send to the block routine.
	//
	// The block stored in daemon's blockmap doesn't make use of OutChans as
	// a block's OutChans can be dynamically modified when connections are
	// added or deleted. All of the other fields, such as ID, name, and all
	// the channels that go into the block (inChan, Routes) are the SAME
	// in both the daemon blockMap block and the blockroutine block.
	//
	// Becauase of this very minor difference it would be a huge semantic help
	// if the type going to the blockroutines was actually different than the
	// type being kept in daemon's blockmap.
	//
	// Modifications to blocks in daemon's blockMap will obviously not
	// proliferate to blockroutines and all changes (such as adding outchans)
	// can only be done through messages. A future daemon block type might
	// want to restrict how daemon blocks can be used, such as creating
	// getters and no setters. Or perhaps a setter automatically takes care
	// of sending a message to the blockroutine to emulate the manipulation
	// of a single variable.

	// create the block that will be stored in blockMap
	b, _ := blocks.NewBlock(name, ID)
	//d.createRoutes(b)
	d.blockMap[b.ID] = b
	d.blockMap[b.ID].InBlocks = make(map[string]bool)
	d.blockMap[b.ID].OutBlocks = make(map[string]bool)

	// create the block that will be sent to the blockroutine and copy all
	// chan references from the previously created block
	c, _ := blocks.NewBlock(name, ID)
	for k, v := range b.Routes {
		c.Routes[k] = v
	}

	c.InChan = b.InChan
	c.AddChan = b.AddChan
	c.QuitChan = b.QuitChan

	//create outchans for use only by blockroutine block.
	c.OutChans = make(map[string]chan blocks.BMsg)

	go blocks.Library[name].Routine(c)

	log.Println("started block \"" + ID + "\" of type " + name)
}

func (d *Daemon) Run(port string) {

	// start the ID Service
	idChan = make(chan string)
	go IDService(idChan)

	// start the library service
	blocks.BuildLibrary()

	// initialise the block maps
	d.blockMap = make(map[string]*blocks.Block)

	handler := rest.ResourceHandler{
		EnableRelaxedContentType: true,
	}

	//TODO: make this a _real_ restful API
	handler.SetRoutes(
		rest.Route{"GET", "/", d.rootHandler},
		rest.Route{"GET", "/library", d.libraryHandler},
		rest.Route{"GET", "/list", d.listHandler},
		rest.Route{"GET", "/save", d.saveHandler},
		rest.Route{"GET", "/create", d.createHandler},
		rest.Route{"GET", "/delete", d.deleteHandler},
		rest.Route{"GET", "/connect", d.connectHandler},
		rest.Route{"GET", "/blocks/:id/:route", d.routeHandler},
		rest.Route{"POST", "/blocks/:id/:route", d.routeHandler},
	)
	log.Println("starting stream tools on port", port)
	err := http.ListenAndServe(":"+port, &handler)
	if err != nil {
		log.Fatalf(err.Error())
	}
}
<commit_msg>removed save handler for another pr<commit_after>package daemon

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ant0ine/go-json-rest"
	"github.com/nytlabs/streamtools/blocks"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
)

const (
	READ_MAX = 1024768
)

var (
	// channel that returns the next ID
	idChan chan string
)

// Daemon keeps track of all the blocks and connections
type Daemon struct {
	blockMap map[string]*blocks.Block
}

// The rootHandler returns information about the whole system
func (d *Daemon) rootHandler(w *rest.ResponseWriter, r *rest.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(index())
}

// The createHandler creates new blocks
func (d *Daemon) createHandler(w *rest.ResponseWriter, r *rest.Request) {
	err := r.ParseForm()
	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	var id string
	var blockType string
	fType, typeExists := r.Form["blockType"]
	fID, idExists := r.Form["id"]

	if typeExists == false {
		ApiResponse(w, 500, "MISSING_BLOCKTYPE")
		return
	} else {
		blockType = fType[0]
	}

	_, inLibrary := blocks.Library[blockType]
	if inLibrary == false {
		ApiResponse(w, 500, "INVALID_BLOCKTYPE")
		return
	}

	if idExists == false {
		id = <-idChan
	} else {

		if len(strings.TrimSpace(fID[0])) == 0 {
			ApiResponse(w, 500, "BAD_BLOCK_ID")
			return
		}

		_, notUnique := d.blockMap[fID[0]]
		if notUnique == true {
			ApiResponse(w, 500, "BLOCK_ID_ALREADY_EXISTS")
			return
		} else {
			id = fID[0]
		}
	}

	d.CreateBlock(blockType, id)

	ApiResponse(w, 200, "BLOCK_CREATED")
}

func (d *Daemon) deleteHandler(w *rest.ResponseWriter, r *rest.Request) {
	err := r.ParseForm()
	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	_, hasID := r.Form["id"]
	if hasID == false {
		ApiResponse(w, 500, "MISSING_BLOCK_ID")
		return
	}

	id := r.Form["id"][0]

	err = d.DeleteBlock(id)

	if err != nil {
		ApiResponse(w, 500, err.Error())
		return
	}

	ApiResponse(w, 200, "BLOCK_DELETED")
}

func (d *Daemon) DeleteBlock(id string) error {
	block, ok := d.blockMap[id]
	if ok == false {
		return errors.New("BLOCK_NOT_FOUND")
	}

	// delete inbound channels
	for k, _ := range block.InBlocks {
		d.blockMap[k].AddChan <- &blocks.OutChanMsg{
			Action: blocks.DELETE_OUT_CHAN,
			ID:     block.ID,
		}

		log.Println("disconnecting \"" + block.ID + "\" from \"" + k + "\"")

		delete(d.blockMap[k].OutBlocks, block.ID)

		if d.blockMap[k].BlockType == "connection" {
			d.DeleteBlock(k)
		}
	}

	// delete outbound channels
	for k, _ := range block.OutBlocks {
		delete(d.blockMap[k].InBlocks, block.ID)
		if d.blockMap[k].BlockType == "connection" {
			d.DeleteBlock(k)
		}
	}

	// delete the block itself
	block.QuitChan <- true
	delete(d.blockMap, id)

	return nil
}

// The connectHandler connects together two blocks
func (d *Daemon) connectHandler(w *rest.ResponseWriter, r *rest.Request) {
	var id string

	err := r.ParseForm()
	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	_, hasFrom := r.Form["from"]
	if hasFrom == false {
		ApiResponse(w, 500, "MISSING_FROM_BLOCK_ID")
		return
	}

	_, hasTo := r.Form["to"]
	if hasTo == false {
		ApiResponse(w, 500, "MISSING_TO_BLOCK_ID")
		return
	}

	fID, hasID := r.Form["id"]
	if hasID == false {
		id = <-idChan
	} else {

		if len(strings.TrimSpace(fID[0])) == 0 {
			ApiResponse(w, 500, "BAD_CONNECTION_ID")
			return
		}

		_, ok := d.blockMap[fID[0]]
		if ok == false {
			id = fID[0]
		} else {
			ApiResponse(w, 500, "BLOCK_ID_ALREADY_EXISTS")
			return
		}
	}

	from := r.Form["from"][0]
	to := r.Form["to"][0]

	if len(from) == 0 {
		ApiResponse(w, 500, "MISSING_FROM_BLOCK_ID")
		return
	}

	if len(to) == 0 {
		ApiResponse(w, 500, "MISSING_TO_BLOCK_ID")
		return
	}

	_, exists := d.blockMap[from]
	if exists == false {
		ApiResponse(w, 500, "FROM_BLOCK_NOT_FOUND")
		return
	}

	_, exists = d.blockMap[strings.Split(to, "/")[0]]
	if exists == false {
		ApiResponse(w, 500, "TO_BLOCK_NOT_FOUND")
		return
	}

	err = d.CreateConnection(from, to, id)
	if err != nil {
		ApiResponse(w, 500, "TO_ROUTE_NOT_FOUND")
		return
	}

	ApiResponse(w, 200, "CONNECTION_CREATED")
}

// The routeHandler deals with any incoming message sent to an arbitrary block endpoint
func (d *Daemon) routeHandler(w *rest.ResponseWriter, r *rest.Request) {
	//id := strings.Split(r.URL.Path, "/")[2]
	//route := strings.Split(r.URL.Path, "/")[3]

	id, ok := r.PathParams["id"]
	if ok == false {
		ApiResponse(w, 500, "MISSING_BLOCK_ID")
		return
	}

	route, ok := r.PathParams["route"]
	if ok == false {
		ApiResponse(w, 500, "MISSING_ROUTE")
		return
	}

	_, ok = d.blockMap[id]
	if ok == false {
		ApiResponse(w, 500, "BLOCK_ID_NOT_FOUND")
		return
	}

	_, ok = d.blockMap[id].Routes[route]
	if ok == false {
		ApiResponse(w, 500, "ROUTE_NOT_FOUND")
		return
	}

	msg, err := ioutil.ReadAll(io.LimitReader(r.Body, READ_MAX))

	if err != nil {
		ApiResponse(w, 500, "BAD_REQUEST")
		return
	}

	var outMsg blocks.BMsg

	if len(msg) > 0 {
		err = json.Unmarshal(msg, &outMsg)
		if err != nil {
			log.Println(msg)
			ApiResponse(w, 500, "BAD_JSON")
			return
		}
	}

	ResponseChan := make(chan blocks.BMsg)
	blockRouteChan := d.blockMap[id].Routes[route]
	blockRouteChan <- blocks.RouteResponse{
		Msg:          outMsg,
		ResponseChan: ResponseChan,
	}
	respMsg := <-ResponseChan

	respJson, err := json.Marshal(respMsg)
	if err != nil {
		ApiResponse(w, 500, "BAD_RESPONSE_FROM_BLOCK")
		return
	}

	DataResponse(w, respJson)
}

func (d *Daemon) libraryHandler(w *rest.ResponseWriter, r *rest.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(len(blocks.LibraryBlob)))
	fmt.Fprint(w, blocks.LibraryBlob)
}

func (d *Daemon) listHandler(w *rest.ResponseWriter, r *rest.Request) {
	blockList := []map[string]interface{}{}
	for _, v := range d.blockMap {
		blockItem := make(map[string]interface{})
		blockItem["BlockType"] = v.BlockType
		blockItem["ID"] = v.ID
		blockItem["InBlocks"] = []string{}
		blockItem["OutBlocks"] = []string{}
		blockItem["Routes"] = []string{}
		for k, _ := range v.InBlocks {
			blockItem["InBlocks"] = append(blockItem["InBlocks"].([]string), k)
		}
		for k, _ := range v.OutBlocks {
			blockItem["OutBlocks"] = append(blockItem["OutBlocks"].([]string), k)
		}
		for k, _ := range v.Routes {
			blockItem["Routes"] = append(blockItem["Routes"].([]string), k)
		}
		blockList = append(blockList, blockItem)
	}
	blob, _ := json.Marshal(blockList)

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(len(blob)))
	fmt.Fprint(w, string(blob))
}

func (d *Daemon) CreateConnection(from string, to string, ID string) error {
	d.CreateBlock("connection", ID)

	d.blockMap[from].AddChan <- &blocks.OutChanMsg{
		Action:  blocks.CREATE_OUT_CHAN,
		OutChan: d.blockMap[ID].InChan,
		ID:      ID,
	}

	/*d.blockMap[ID].AddChan <- &blocks.OutChanMsg{
		Action:  blocks.CREATE_OUT_CHAN,
		OutChan: d.blockMap[to].InChan,
		ID:      to,
	}*/
	toParts := strings.Split(to, "/")
	switch len(toParts) {
	case 1:
		d.blockMap[ID].AddChan <- &blocks.OutChanMsg{
			Action:  blocks.CREATE_OUT_CHAN,
			OutChan: d.blockMap[to].InChan,
			ID:      to,
		}
	case 2:
		d.blockMap[ID].AddChan <- &blocks.OutChanMsg{
			Action:  blocks.CREATE_OUT_CHAN,
			OutChan: d.blockMap[toParts[0]].Routes[toParts[1]],
			ID:      to,
		}
	default:
		err := errors.New("malformed to route specification")
		return err
	}

	// add the from block to the list of inblocks for connection.
	d.blockMap[from].OutBlocks[ID] = true
	d.blockMap[ID].InBlocks[from] = true
	d.blockMap[ID].OutBlocks[toParts[0]] = true
	d.blockMap[toParts[0]].InBlocks[ID] = true

	log.Println("connected", d.blockMap[from].ID, "to", d.blockMap[toParts[0]].ID)

	return nil
}

func (d *Daemon) CreateBlock(name string, ID string) {
	// TODO: Clean this up.
	//
	// In order to avoid data races the blocks held in daemon's blockMap
	// are not the same blocks held in each block routine. When CreateBlock
	// is called, we actually create two blocks: one to store in daemon's
	// blockMap and one to send to the block routine.
	//
	// The block stored in daemon's blockmap doesn't make use of OutChans as
	// a block's OutChans can be dynamically modified when connections are
	// added or deleted. All of the other fields, such as ID, name, and all
	// the channels that go into the block (inChan, Routes) are the SAME
	// in both the daemon blockMap block and the blockroutine block.
	//
	// Becauase of this very minor difference it would be a huge semantic help
	// if the type going to the blockroutines was actually different than the
	// type being kept in daemon's blockmap.
	//
	// Modifications to blocks in daemon's blockMap will obviously not
	// proliferate to blockroutines and all changes (such as adding outchans)
	// can only be done through messages. A future daemon block type might
	// want to restrict how daemon blocks can be used, such as creating
	// getters and no setters. Or perhaps a setter automatically takes care
	// of sending a message to the blockroutine to emulate the manipulation
	// of a single variable.

	// create the block that will be stored in blockMap
	b, _ := blocks.NewBlock(name, ID)
	//d.createRoutes(b)
	d.blockMap[b.ID] = b
	d.blockMap[b.ID].InBlocks = make(map[string]bool)
	d.blockMap[b.ID].OutBlocks = make(map[string]bool)

	// create the block that will be sent to the blockroutine and copy all
	// chan references from the previously created block
	c, _ := blocks.NewBlock(name, ID)
	for k, v := range b.Routes {
		c.Routes[k] = v
	}

	c.InChan = b.InChan
	c.AddChan = b.AddChan
	c.QuitChan = b.QuitChan

	//create outchans for use only by blockroutine block.
	c.OutChans = make(map[string]chan blocks.BMsg)

	go blocks.Library[name].Routine(c)

	log.Println("started block \"" + ID + "\" of type " + name)
}

func (d *Daemon) Run(port string) {

	// start the ID Service
	idChan = make(chan string)
	go IDService(idChan)

	// start the library service
	blocks.BuildLibrary()

	// initialise the block maps
	d.blockMap = make(map[string]*blocks.Block)

	handler := rest.ResourceHandler{
		EnableRelaxedContentType: true,
	}

	//TODO: make this a _real_ restful API
	handler.SetRoutes(
		rest.Route{"GET", "/", d.rootHandler},
		rest.Route{"GET", "/library", d.libraryHandler},
		rest.Route{"GET", "/list", d.listHandler},
		rest.Route{"GET", "/create", d.createHandler},
		rest.Route{"GET", "/delete", d.deleteHandler},
		rest.Route{"GET", "/connect", d.connectHandler},
		rest.Route{"GET", "/blocks/:id/:route", d.routeHandler},
		rest.Route{"POST", "/blocks/:id/:route", d.routeHandler},
	)
	log.Println("starting stream tools on port", port)
	err := http.ListenAndServe(":"+port, &handler)
	if err != nil {
		log.Fatalf(err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package jwalterweatherman

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync/atomic"
)

// Level describes the chosen log level between
// debug and critical.
type Level int

type NotePad struct {
	Handle  io.Writer
	Level   Level
	Prefix  string
	Logger  **log.Logger
	counter uint64
}

func (n *NotePad) incr() {
	atomic.AddUint64(&n.counter, 1)
}

func (n *NotePad) resetCounter() {
	atomic.StoreUint64(&n.counter, 0)
}

func (n *NotePad) getCount() uint64 {
	return atomic.LoadUint64(&n.counter)
}

type countingWriter struct {
	incrFunc func()
}

func (cw *countingWriter) Write(p []byte) (n int, err error) {
	cw.incrFunc()

	return 0, nil
}

// Feedback is special. It writes plainly to the output while
// logging with the standard extra information (date, file, etc)
// Only Println and Printf are currently provided for this
type Feedback struct{}

const (
	LevelTrace Level = iota
	LevelDebug
	LevelInfo
	LevelWarn
	LevelError
	LevelCritical
	LevelFatal
	DefaultLogThreshold    = LevelWarn
	DefaultStdoutThreshold = LevelError
)

var (
	TRACE      *log.Logger
	DEBUG      *log.Logger
	INFO       *log.Logger
	WARN       *log.Logger
	ERROR      *log.Logger
	CRITICAL   *log.Logger
	FATAL      *log.Logger
	LOG        *log.Logger
	FEEDBACK   Feedback
	LogHandle  io.Writer  = ioutil.Discard
	OutHandle  io.Writer  = os.Stdout
	BothHandle io.Writer  = io.MultiWriter(LogHandle, OutHandle)
	NotePads   []*NotePad = []*NotePad{trace, debug, info, warn, err, critical, fatal}

	trace           *NotePad = &NotePad{Level: LevelTrace, Handle: os.Stdout, Logger: &TRACE, Prefix: "TRACE: "}
	debug           *NotePad = &NotePad{Level: LevelDebug, Handle: os.Stdout, Logger: &DEBUG, Prefix: "DEBUG: "}
	info            *NotePad = &NotePad{Level: LevelInfo, Handle: os.Stdout, Logger: &INFO, Prefix: "INFO: "}
	warn            *NotePad = &NotePad{Level: LevelWarn, Handle: os.Stdout, Logger: &WARN, Prefix: "WARN: "}
	err             *NotePad = &NotePad{Level: LevelError, Handle: os.Stdout, Logger: &ERROR, Prefix: "ERROR: "}
	critical        *NotePad = &NotePad{Level: LevelCritical, Handle: os.Stdout, Logger: &CRITICAL, Prefix: "CRITICAL: "}
	fatal           *NotePad = &NotePad{Level: LevelFatal, Handle: os.Stdout, Logger: &FATAL, Prefix: "FATAL: "}
	logThreshold    Level    = DefaultLogThreshold
	outputThreshold Level    = DefaultStdoutThreshold
)

const (
	DATE  = log.Ldate
	TIME  = log.Ltime
	SFILE = log.Lshortfile
	LFILE = log.Llongfile
	MSEC  = log.Lmicroseconds
)

var logFlags = DATE | TIME | SFILE

func init() {
	SetStdoutThreshold(DefaultStdoutThreshold)
}

// initialize will setup the jWalterWeatherman standard approach of providing the user
// some feedback and logging a potentially different amount based on independent log and output thresholds.
// By default the output has a lower threshold than logged
// Don't use if you have manually set the Handles of the different levels as it will overwrite them.
func initialize() {
	BothHandle = io.MultiWriter(LogHandle, OutHandle)

	for _, n := range NotePads {
		if n.Level < outputThreshold && n.Level < logThreshold {
			n.Handle = ioutil.Discard
		} else if n.Level >= outputThreshold && n.Level >= logThreshold {
			n.Handle = BothHandle
		} else if n.Level >= outputThreshold && n.Level < logThreshold {
			n.Handle = OutHandle
		} else {
			n.Handle = LogHandle
		}
	}

	for _, n := range NotePads {
		n.Handle = io.MultiWriter(n.Handle, &countingWriter{n.incr})
		*n.Logger = log.New(n.Handle, n.Prefix, logFlags)
	}

	LOG = log.New(LogHandle,
		"LOG:   ",
		logFlags)
}

// Set the log Flags (Available flag: DATE, TIME, SFILE, LFILE and MSEC)
func SetLogFlag(flags int) {
	logFlags = flags
}

// Level returns the current global log threshold.
func LogThreshold() Level {
	return logThreshold
}

// Level returns the current global output threshold.
func StdoutThreshold() Level {
	return outputThreshold
}

// Ensures that the level provided is within the bounds of available levels
func levelCheck(level Level) Level {
	switch {
	case level <= LevelTrace:
		return LevelTrace
	case level >= LevelFatal:
		return LevelFatal
	default:
		return level
	}
}

// Establishes a threshold where anything matching or above will be logged
func SetLogThreshold(level Level) {
	logThreshold = levelCheck(level)
	initialize()
}

// Establishes a threshold where anything matching or above will be output
func SetStdoutThreshold(level Level) {
	outputThreshold = levelCheck(level)
	initialize()
}

// Conveniently Sets the Log Handle to a io.writer created for the file behind the given filepath
// Will only append to this file
func SetLogFile(path string) {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		CRITICAL.Println("Failed to open log file:", path, err)
		os.Exit(-1)
	}

	INFO.Println("Logging to", file.Name())

	LogHandle = file
	initialize()
}

// Conveniently Creates a temporary file and sets the Log Handle to a io.writer created for it
func UseTempLogFile(prefix string) {
	file, err := ioutil.TempFile(os.TempDir(), prefix)
	if err != nil {
		CRITICAL.Println(err)
	}

	INFO.Println("Logging to", file.Name())

	LogHandle = file
	initialize()
}

// LogCountForLevel returns the number of log invocations for a given level.
func LogCountForLevel(l Level) uint64 {
	for _, np := range NotePads {
		if np.Level == l {
			return np.getCount()
		}
	}
	return 0
}

// LogCountForLevelsGreaterThanorEqualTo returns the number of log invocations
// greater than or equal to a given level threshold.
func LogCountForLevelsGreaterThanorEqualTo(threshold Level) uint64 {
	var cnt uint64
	for _, np := range NotePads {
		if np.Level >= threshold {
			cnt += np.getCount()
		}
	}
	return cnt
}

// ResetLogCounters resets the invocation counters for all levels.
func ResetLogCounters() {
	for _, np := range NotePads {
		np.resetCounter()
	}
}

// Disables logging for the entire JWW system
func DiscardLogging() {
	LogHandle = ioutil.Discard
	initialize()
}

// Feedback is special. It writes plainly to the output while
// logging with the standard extra information (date, file, etc)
// Only Println and Printf are currently provided for this
func (fb *Feedback) Println(v ...interface{}) {
	s := fmt.Sprintln(v...)
	fmt.Print(s)
	LOG.Output(2, s)
}

// Feedback is special. It writes plainly to the output while
// logging with the standard extra information (date, file, etc)
// Only Println and Printf are currently provided for this
func (fb *Feedback) Printf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	fmt.Print(s)
	LOG.Output(2, s)
}
<commit_msg>Call initialize after changing logFlags<commit_after>// Copyright © 2016 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package jwalterweatherman

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync/atomic"
)

// Level describes the chosen log level between
// debug and critical.
type Level int

type NotePad struct {
	Handle  io.Writer
	Level   Level
	Prefix  string
	Logger  **log.Logger
	counter uint64
}

func (n *NotePad) incr() {
	atomic.AddUint64(&n.counter, 1)
}

func (n *NotePad) resetCounter() {
	atomic.StoreUint64(&n.counter, 0)
}

func (n *NotePad) getCount() uint64 {
	return atomic.LoadUint64(&n.counter)
}

type countingWriter struct {
	incrFunc func()
}

func (cw *countingWriter) Write(p []byte) (n int, err error) {
	cw.incrFunc()

	return 0, nil
}

// Feedback is special. It writes plainly to the output while
// logging with the standard extra information (date, file, etc)
// Only Println and Printf are currently provided for this
type Feedback struct{}

const (
	LevelTrace Level = iota
	LevelDebug
	LevelInfo
	LevelWarn
	LevelError
	LevelCritical
	LevelFatal
	DefaultLogThreshold    = LevelWarn
	DefaultStdoutThreshold = LevelError
)

var (
	TRACE      *log.Logger
	DEBUG      *log.Logger
	INFO       *log.Logger
	WARN       *log.Logger
	ERROR      *log.Logger
	CRITICAL   *log.Logger
	FATAL      *log.Logger
	LOG        *log.Logger
	FEEDBACK   Feedback
	LogHandle  io.Writer  = ioutil.Discard
	OutHandle  io.Writer  = os.Stdout
	BothHandle io.Writer  = io.MultiWriter(LogHandle, OutHandle)
	NotePads   []*NotePad = []*NotePad{trace, debug, info, warn, err, critical, fatal}

	trace           *NotePad = &NotePad{Level: LevelTrace, Handle: os.Stdout, Logger: &TRACE, Prefix: "TRACE: "}
	debug           *NotePad = &NotePad{Level: LevelDebug, Handle: os.Stdout, Logger: &DEBUG, Prefix: "DEBUG: "}
	info            *NotePad = &NotePad{Level: LevelInfo, Handle: os.Stdout, Logger: &INFO, Prefix: "INFO: "}
	warn            *NotePad = &NotePad{Level: LevelWarn, Handle: os.Stdout, Logger: &WARN, Prefix: "WARN: "}
	err             *NotePad = &NotePad{Level: LevelError, Handle: os.Stdout, Logger: &ERROR, Prefix: "ERROR: "}
	critical        *NotePad = &NotePad{Level: LevelCritical, Handle: os.Stdout, Logger: &CRITICAL, Prefix: "CRITICAL: "}
	fatal           *NotePad = &NotePad{Level: LevelFatal, Handle: os.Stdout, Logger: &FATAL, Prefix: "FATAL: "}
	logThreshold    Level    = DefaultLogThreshold
	outputThreshold Level    = DefaultStdoutThreshold
)

const (
	DATE  = log.Ldate
	TIME  = log.Ltime
	SFILE = log.Lshortfile
	LFILE = log.Llongfile
	MSEC  = log.Lmicroseconds
)

var logFlags = DATE | TIME | SFILE

func init() {
	SetStdoutThreshold(DefaultStdoutThreshold)
}

// initialize will setup the jWalterWeatherman standard approach of providing the user
// some feedback and logging a potentially different amount based on independent log and output thresholds.
// By default the output has a lower threshold than logged
// Don't use if you have manually set the Handles of the different levels as it will overwrite them.
func initialize() {
	BothHandle = io.MultiWriter(LogHandle, OutHandle)

	for _, n := range NotePads {
		if n.Level < outputThreshold && n.Level < logThreshold {
			n.Handle = ioutil.Discard
		} else if n.Level >= outputThreshold && n.Level >= logThreshold {
			n.Handle = BothHandle
		} else if n.Level >= outputThreshold && n.Level < logThreshold {
			n.Handle = OutHandle
		} else {
			n.Handle = LogHandle
		}
	}

	for _, n := range NotePads {
		n.Handle = io.MultiWriter(n.Handle, &countingWriter{n.incr})
		*n.Logger = log.New(n.Handle, n.Prefix, logFlags)
	}

	LOG = log.New(LogHandle,
		"LOG:   ",
		logFlags)
}

// Set the log Flags (Available flag: DATE, TIME, SFILE, LFILE and MSEC)
func SetLogFlag(flags int) {
	logFlags = flags
	initialize()
}

// Level returns the current global log threshold.
func LogThreshold() Level {
	return logThreshold
}

// Level returns the current global output threshold.
func StdoutThreshold() Level {
	return outputThreshold
}

// Ensures that the level provided is within the bounds of available levels
func levelCheck(level Level) Level {
	switch {
	case level <= LevelTrace:
		return LevelTrace
	case level >= LevelFatal:
		return LevelFatal
	default:
		return level
	}
}

// Establishes a threshold where anything matching or above will be logged
func SetLogThreshold(level Level) {
	logThreshold = levelCheck(level)
	initialize()
}

// Establishes a threshold where anything matching or above will be output
func SetStdoutThreshold(level Level) {
	outputThreshold = levelCheck(level)
	initialize()
}

// Conveniently Sets the Log Handle to a io.writer created for the file behind the given filepath
// Will only append to this file
func SetLogFile(path string) {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		CRITICAL.Println("Failed to open log file:", path, err)
		os.Exit(-1)
	}

	INFO.Println("Logging to", file.Name())

	LogHandle = file
	initialize()
}

// Conveniently Creates a temporary file and sets the Log Handle to a io.writer created for it
func UseTempLogFile(prefix string) {
	file, err := ioutil.TempFile(os.TempDir(), prefix)
	if err != nil {
		CRITICAL.Println(err)
	}

	INFO.Println("Logging to", file.Name())

	LogHandle = file
	initialize()
}

// LogCountForLevel returns the number of log invocations for a given level.
func LogCountForLevel(l Level) uint64 {
	for _, np := range NotePads {
		if np.Level == l {
			return np.getCount()
		}
	}
	return 0
}

// LogCountForLevelsGreaterThanorEqualTo returns the number of log invocations
// greater than or equal to a given level threshold.
func LogCountForLevelsGreaterThanorEqualTo(threshold Level) uint64 {
	var cnt uint64
	for _, np := range NotePads {
		if np.Level >= threshold {
			cnt += np.getCount()
		}
	}
	return cnt
}

// ResetLogCounters resets the invocation counters for all levels.
func ResetLogCounters() {
	for _, np := range NotePads {
		np.resetCounter()
	}
}

// Disables logging for the entire JWW system
func DiscardLogging() {
	LogHandle = ioutil.Discard
	initialize()
}

// Feedback is special. It writes plainly to the output while
// logging with the standard extra information (date, file, etc)
// Only Println and Printf are currently provided for this
func (fb *Feedback) Println(v ...interface{}) {
	s := fmt.Sprintln(v...)
	fmt.Print(s)
	LOG.Output(2, s)
}

// Feedback is special. It writes plainly to the output while
// logging with the standard extra information (date, file, etc)
// Only Println and Printf are currently provided for this
func (fb *Feedback) Printf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	fmt.Print(s)
	LOG.Output(2, s)
}
<|endoftext|>