<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fastcgi

import (
	"context"
	"crypto/tls"
	"fmt"
	"net/http"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
	"github.com/caddyserver/caddy/v2/modules/caddytls"

	"github.com/caddyserver/caddy/v2"
)

func init() {
	caddy.RegisterModule(Transport{})
}

// Transport facilitates FastCGI communication.
type Transport struct {
	// TODO: Populate these
	softwareName    string
	softwareVersion string
	serverName      string
	serverPort      string

	// Use this directory as the fastcgi root directory. Defaults to the root
	// directory of the parent virtual host.
	Root string `json:"root,omitempty"`

	// The path in the URL will be split into two, with the first piece ending
	// with the value of SplitPath. The first piece will be assumed as the
	// actual resource (CGI script) name, and the second piece will be set to
	// PATH_INFO for the CGI script to use.
	// Future enhancements should be careful to avoid CVE-2019-11043,
	// which can be mitigated with use of a try_files-like behavior
	// that 404's if the fastcgi path info is not found.
	SplitPath string `json:"split_path,omitempty"`

	// Extra environment variables.
	EnvVars map[string]string `json:"env,omitempty"`

	// The duration used to set a deadline when connecting to an upstream.
	DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`

	// The duration used to set a deadline when reading from the FastCGI server.
	ReadTimeout caddy.Duration `json:"read_timeout,omitempty"`

	// The duration used to set a deadline when sending to the FastCGI server.
	WriteTimeout caddy.Duration `json:"write_timeout,omitempty"`
}

// CaddyModule returns the Caddy module information.
func (Transport) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		ID:  "http.reverse_proxy.transport.fastcgi",
		New: func() caddy.Module { return new(Transport) },
	}
}

// Provision sets up t.
func (t *Transport) Provision(_ caddy.Context) error {
	if t.Root == "" {
		t.Root = "{http.vars.root}"
	}
	return nil
}

// RoundTrip implements http.RoundTripper.
func (t Transport) RoundTrip(r *http.Request) (*http.Response, error) {
	env, err := t.buildEnv(r)
	if err != nil {
		return nil, fmt.Errorf("building environment: %v", err)
	}

	// TODO: doesn't dialer have a Timeout field?
	ctx := r.Context()
	if t.DialTimeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, time.Duration(t.DialTimeout))
		defer cancel()
	}

	// extract dial information from request (this
	// should embedded by the reverse proxy)
	network, address := "tcp", r.URL.Host
	if dialInfoVal := ctx.Value(reverseproxy.DialInfoCtxKey); dialInfoVal != nil {
		dialInfo := dialInfoVal.(reverseproxy.DialInfo)
		network = dialInfo.Network
		address = dialInfo.Address
	}

	fcgiBackend, err := DialContext(ctx, network, address)
	if err != nil {
		// TODO: wrap in a special error type if the dial failed, so retries can happen if enabled
		return nil, fmt.Errorf("dialing backend: %v", err)
	}
	// fcgiBackend gets closed when response body is closed (see clientCloser)

	// read/write timeouts
	if err := fcgiBackend.SetReadTimeout(time.Duration(t.ReadTimeout)); err != nil {
		return nil, fmt.Errorf("setting read timeout: %v", err)
	}
	if err := fcgiBackend.SetWriteTimeout(time.Duration(t.WriteTimeout)); err != nil {
		return nil, fmt.Errorf("setting write timeout: %v", err)
	}

	contentLength := r.ContentLength
	if contentLength == 0 {
		contentLength, _ = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
	}

	var resp *http.Response
	switch r.Method {
	case http.MethodHead:
		resp, err = fcgiBackend.Head(env)
	case http.MethodGet:
		resp, err = fcgiBackend.Get(env, r.Body, contentLength)
	case http.MethodOptions:
		resp, err = fcgiBackend.Options(env)
	default:
		resp, err = fcgiBackend.Post(env, r.Method, r.Header.Get("Content-Type"), r.Body, contentLength)
	}

	return resp, err
}

// buildEnv returns a set of CGI environment variables for the request.
func (t Transport) buildEnv(r *http.Request) (map[string]string, error) {
	repl := r.Context().Value(caddy.ReplacerCtxKey).(caddy.Replacer)

	var env map[string]string

	// Separate remote IP and port; more lenient than net.SplitHostPort
	var ip, port string
	if idx := strings.LastIndex(r.RemoteAddr, ":"); idx > -1 {
		ip = r.RemoteAddr[:idx]
		port = r.RemoteAddr[idx+1:]
	} else {
		ip = r.RemoteAddr
	}

	// Remove [] from IPv6 addresses
	ip = strings.Replace(ip, "[", "", 1)
	ip = strings.Replace(ip, "]", "", 1)

	root := repl.ReplaceAll(t.Root, ".")
	fpath := r.URL.Path

	// Split path in preparation for env variables.
	// Previous canSplit checks ensure this can never be -1.
	// TODO: I haven't brought over canSplit; make sure this doesn't break
	splitPos := t.splitPos(fpath)

	// Request has the extension; path was split successfully
	docURI := fpath[:splitPos+len(t.SplitPath)]
	pathInfo := fpath[splitPos+len(t.SplitPath):]
	scriptName := fpath

	// Strip PATH_INFO from SCRIPT_NAME
	scriptName = strings.TrimSuffix(scriptName, pathInfo)

	// SCRIPT_FILENAME is the absolute path of SCRIPT_NAME
	scriptFilename := filepath.Join(root, scriptName)

	// Add vhost path prefix to scriptName. Otherwise, some PHP software will
	// have difficulty discovering its URL.
	pathPrefix, _ := r.Context().Value(caddy.CtxKey("path_prefix")).(string)
	scriptName = path.Join(pathPrefix, scriptName)

	// Get the request URL from context. The context stores the original URL in case
	// it was changed by a middleware such as rewrite. By default, we pass the
	// original URI in as the value of REQUEST_URI (the user can overwrite this
	// if desired). Most PHP apps seem to want the original URI. Besides, this is
	// how nginx defaults: http://stackoverflow.com/a/12485156/1048862
	origReq, ok := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
	if !ok {
		// some requests, like active health checks, don't add this to
		// the request context, so we can just use the current URL
		origReq = *r
	}
	reqURL := origReq.URL

	requestScheme := "http"
	if r.TLS != nil {
		requestScheme = "https"
	}

	// Some variables are unused but cleared explicitly to prevent
	// the parent environment from interfering.
	env = map[string]string{
		// Variables defined in CGI 1.1 spec
		"AUTH_TYPE":         "", // Not used
		"CONTENT_LENGTH":    r.Header.Get("Content-Length"),
		"CONTENT_TYPE":      r.Header.Get("Content-Type"),
		"GATEWAY_INTERFACE": "CGI/1.1",
		"PATH_INFO":         pathInfo,
		"QUERY_STRING":      r.URL.RawQuery,
		"REMOTE_ADDR":       ip,
		"REMOTE_HOST":       ip, // For speed, remote host lookups disabled
		"REMOTE_PORT":       port,
		"REMOTE_IDENT":      "", // Not used
		"REMOTE_USER":       "", // TODO: once there are authentication handlers, populate this
		"REQUEST_METHOD":    r.Method,
		"REQUEST_SCHEME":    requestScheme,
		"SERVER_NAME":       t.serverName,
		"SERVER_PORT":       t.serverPort,
		"SERVER_PROTOCOL":   r.Proto,
		"SERVER_SOFTWARE":   t.softwareName + "/" + t.softwareVersion,

		// Other variables
		"DOCUMENT_ROOT":   root,
		"DOCUMENT_URI":    docURI,
		"HTTP_HOST":       r.Host, // added here, since not always part of headers
		"REQUEST_URI":     reqURL.RequestURI(),
		"SCRIPT_FILENAME": scriptFilename,
		"SCRIPT_NAME":     scriptName,
	}

	// compliance with the CGI specification requires that
	// PATH_TRANSLATED should only exist if PATH_INFO is defined.
	// Info: https://www.ietf.org/rfc/rfc3875 Page 14
	if env["PATH_INFO"] != "" {
		env["PATH_TRANSLATED"] = filepath.Join(root, pathInfo) // Info: http://www.oreilly.com/openbook/cgi/ch02_04.html
	}

	// Some web apps rely on knowing HTTPS or not
	if r.TLS != nil {
		env["HTTPS"] = "on"
		// and pass the protocol details in a manner compatible with apache's mod_ssl
		// (which is why these have a SSL_ prefix and not TLS_).
		v, ok := tlsProtocolStrings[r.TLS.Version]
		if ok {
			env["SSL_PROTOCOL"] = v
		}
		// and pass the cipher suite in a manner compatible with apache's mod_ssl
		for k, v := range caddytls.SupportedCipherSuites {
			if v == r.TLS.CipherSuite {
				env["SSL_CIPHER"] = k
				break
			}
		}
	}

	// Add env variables from config (with support for placeholders in values)
	for key, value := range t.EnvVars {
		env[key] = repl.ReplaceAll(value, "")
	}

	// Add all HTTP headers to env variables
	for field, val := range r.Header {
		header := strings.ToUpper(field)
		header = headerNameReplacer.Replace(header)
		env["HTTP_"+header] = strings.Join(val, ", ")
	}
	return env, nil
}

// splitPos returns the index where path should
// be split based on t.SplitPath.
func (t Transport) splitPos(path string) int {
	// TODO:
	// if httpserver.CaseSensitivePath {
	// 	return strings.Index(path, r.SplitPath)
	// }
	return strings.Index(strings.ToLower(path), strings.ToLower(t.SplitPath))
}

// TODO:
// Map of supported protocols to Apache ssl_mod format
// Note that these are slightly different from SupportedProtocols in caddytls/config.go
var tlsProtocolStrings = map[uint16]string{
	tls.VersionTLS10: "TLSv1",
	tls.VersionTLS11: "TLSv1.1",
	tls.VersionTLS12: "TLSv1.2",
	tls.VersionTLS13: "TLSv1.3",
}

var headerNameReplacer = strings.NewReplacer(" ", "_", "-", "_")

// Interface guards
var (
	_ caddy.Provisioner = (*Transport)(nil)
	_ http.RoundTripper = (*Transport)(nil)
)
<commit_msg>fastcgi: Set SERVER_SOFTWARE, _NAME, and _PORT properly (fixes #2952)<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fastcgi

import (
	"context"
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
	"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
	"github.com/caddyserver/caddy/v2/modules/caddytls"

	"github.com/caddyserver/caddy/v2"
)

func init() {
	caddy.RegisterModule(Transport{})
}

// Transport facilitates FastCGI communication.
type Transport struct {
	// Use this directory as the fastcgi root directory. Defaults to the root
	// directory of the parent virtual host.
	Root string `json:"root,omitempty"`

	// The path in the URL will be split into two, with the first piece ending
	// with the value of SplitPath. The first piece will be assumed as the
	// actual resource (CGI script) name, and the second piece will be set to
	// PATH_INFO for the CGI script to use.
	// Future enhancements should be careful to avoid CVE-2019-11043,
	// which can be mitigated with use of a try_files-like behavior
	// that 404's if the fastcgi path info is not found.
	SplitPath string `json:"split_path,omitempty"`

	// Extra environment variables.
	EnvVars map[string]string `json:"env,omitempty"`

	// The duration used to set a deadline when connecting to an upstream.
	DialTimeout caddy.Duration `json:"dial_timeout,omitempty"`

	// The duration used to set a deadline when reading from the FastCGI server.
	ReadTimeout caddy.Duration `json:"read_timeout,omitempty"`

	// The duration used to set a deadline when sending to the FastCGI server.
	WriteTimeout caddy.Duration `json:"write_timeout,omitempty"`

	serverSoftware string
}

// CaddyModule returns the Caddy module information.
func (Transport) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		ID:  "http.reverse_proxy.transport.fastcgi",
		New: func() caddy.Module { return new(Transport) },
	}
}

// Provision sets up t.
func (t *Transport) Provision(_ caddy.Context) error {
	if t.Root == "" {
		t.Root = "{http.vars.root}"
	}
	t.serverSoftware = "Caddy"
	if mod := caddy.GoModule(); mod.Version != "" {
		t.serverSoftware += "/" + mod.Version
	}
	return nil
}

// RoundTrip implements http.RoundTripper.
func (t Transport) RoundTrip(r *http.Request) (*http.Response, error) {
	env, err := t.buildEnv(r)
	if err != nil {
		return nil, fmt.Errorf("building environment: %v", err)
	}

	// TODO: doesn't dialer have a Timeout field?
	ctx := r.Context()
	if t.DialTimeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, time.Duration(t.DialTimeout))
		defer cancel()
	}

	// extract dial information from request (this
	// should embedded by the reverse proxy)
	network, address := "tcp", r.URL.Host
	if dialInfoVal := ctx.Value(reverseproxy.DialInfoCtxKey); dialInfoVal != nil {
		dialInfo := dialInfoVal.(reverseproxy.DialInfo)
		network = dialInfo.Network
		address = dialInfo.Address
	}

	fcgiBackend, err := DialContext(ctx, network, address)
	if err != nil {
		// TODO: wrap in a special error type if the dial failed, so retries can happen if enabled
		return nil, fmt.Errorf("dialing backend: %v", err)
	}
	// fcgiBackend gets closed when response body is closed (see clientCloser)

	// read/write timeouts
	if err := fcgiBackend.SetReadTimeout(time.Duration(t.ReadTimeout)); err != nil {
		return nil, fmt.Errorf("setting read timeout: %v", err)
	}
	if err := fcgiBackend.SetWriteTimeout(time.Duration(t.WriteTimeout)); err != nil {
		return nil, fmt.Errorf("setting write timeout: %v", err)
	}

	contentLength := r.ContentLength
	if contentLength == 0 {
		contentLength, _ = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
	}

	var resp *http.Response
	switch r.Method {
	case http.MethodHead:
		resp, err = fcgiBackend.Head(env)
	case http.MethodGet:
		resp, err = fcgiBackend.Get(env, r.Body, contentLength)
	case http.MethodOptions:
		resp, err = fcgiBackend.Options(env)
	default:
		resp, err = fcgiBackend.Post(env, r.Method, r.Header.Get("Content-Type"), r.Body, contentLength)
	}

	return resp, err
}

// buildEnv returns a set of CGI environment variables for the request.
func (t Transport) buildEnv(r *http.Request) (map[string]string, error) {
	repl := r.Context().Value(caddy.ReplacerCtxKey).(caddy.Replacer)

	var env map[string]string

	// Separate remote IP and port; more lenient than net.SplitHostPort
	var ip, port string
	if idx := strings.LastIndex(r.RemoteAddr, ":"); idx > -1 {
		ip = r.RemoteAddr[:idx]
		port = r.RemoteAddr[idx+1:]
	} else {
		ip = r.RemoteAddr
	}

	// Remove [] from IPv6 addresses
	ip = strings.Replace(ip, "[", "", 1)
	ip = strings.Replace(ip, "]", "", 1)

	root := repl.ReplaceAll(t.Root, ".")
	fpath := r.URL.Path

	// Split path in preparation for env variables.
	// Previous canSplit checks ensure this can never be -1.
	// TODO: I haven't brought over canSplit; make sure this doesn't break
	splitPos := t.splitPos(fpath)

	// Request has the extension; path was split successfully
	docURI := fpath[:splitPos+len(t.SplitPath)]
	pathInfo := fpath[splitPos+len(t.SplitPath):]
	scriptName := fpath

	// Strip PATH_INFO from SCRIPT_NAME
	scriptName = strings.TrimSuffix(scriptName, pathInfo)

	// SCRIPT_FILENAME is the absolute path of SCRIPT_NAME
	scriptFilename := filepath.Join(root, scriptName)

	// Add vhost path prefix to scriptName. Otherwise, some PHP software will
	// have difficulty discovering its URL.
	pathPrefix, _ := r.Context().Value(caddy.CtxKey("path_prefix")).(string)
	scriptName = path.Join(pathPrefix, scriptName)

	// Get the request URL from context. The context stores the original URL in case
	// it was changed by a middleware such as rewrite. By default, we pass the
	// original URI in as the value of REQUEST_URI (the user can overwrite this
	// if desired). Most PHP apps seem to want the original URI. Besides, this is
	// how nginx defaults: http://stackoverflow.com/a/12485156/1048862
	origReq, ok := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
	if !ok {
		// some requests, like active health checks, don't add this to
		// the request context, so we can just use the current URL
		origReq = *r
	}
	reqURL := origReq.URL

	requestScheme := "http"
	if r.TLS != nil {
		requestScheme = "https"
	}

	reqHost, reqPort, err := net.SplitHostPort(r.Host)
	if err != nil {
		// whatever, just assume there was no port
		reqHost = r.Host
	}

	// Some variables are unused but cleared explicitly to prevent
	// the parent environment from interfering.
	env = map[string]string{
		// Variables defined in CGI 1.1 spec
		"AUTH_TYPE":         "", // Not used
		"CONTENT_LENGTH":    r.Header.Get("Content-Length"),
		"CONTENT_TYPE":      r.Header.Get("Content-Type"),
		"GATEWAY_INTERFACE": "CGI/1.1",
		"PATH_INFO":         pathInfo,
		"QUERY_STRING":      r.URL.RawQuery,
		"REMOTE_ADDR":       ip,
		"REMOTE_HOST":       ip, // For speed, remote host lookups disabled
		"REMOTE_PORT":       port,
		"REMOTE_IDENT":      "", // Not used
		"REMOTE_USER":       "", // TODO: once there are authentication handlers, populate this
		"REQUEST_METHOD":    r.Method,
		"REQUEST_SCHEME":    requestScheme,
		"SERVER_NAME":       reqHost,
		"SERVER_PORT":       reqPort,
		"SERVER_PROTOCOL":   r.Proto,
		"SERVER_SOFTWARE":   t.serverSoftware,

		// Other variables
		"DOCUMENT_ROOT":   root,
		"DOCUMENT_URI":    docURI,
		"HTTP_HOST":       r.Host, // added here, since not always part of headers
		"REQUEST_URI":     reqURL.RequestURI(),
		"SCRIPT_FILENAME": scriptFilename,
		"SCRIPT_NAME":     scriptName,
	}

	// compliance with the CGI specification requires that
	// PATH_TRANSLATED should only exist if PATH_INFO is defined.
	// Info: https://www.ietf.org/rfc/rfc3875 Page 14
	if env["PATH_INFO"] != "" {
		env["PATH_TRANSLATED"] = filepath.Join(root, pathInfo) // Info: http://www.oreilly.com/openbook/cgi/ch02_04.html
	}

	// Some web apps rely on knowing HTTPS or not
	if r.TLS != nil {
		env["HTTPS"] = "on"
		// and pass the protocol details in a manner compatible with apache's mod_ssl
		// (which is why these have a SSL_ prefix and not TLS_).
		v, ok := tlsProtocolStrings[r.TLS.Version]
		if ok {
			env["SSL_PROTOCOL"] = v
		}
		// and pass the cipher suite in a manner compatible with apache's mod_ssl
		for k, v := range caddytls.SupportedCipherSuites {
			if v == r.TLS.CipherSuite {
				env["SSL_CIPHER"] = k
				break
			}
		}
	}

	// Add env variables from config (with support for placeholders in values)
	for key, value := range t.EnvVars {
		env[key] = repl.ReplaceAll(value, "")
	}

	// Add all HTTP headers to env variables
	for field, val := range r.Header {
		header := strings.ToUpper(field)
		header = headerNameReplacer.Replace(header)
		env["HTTP_"+header] = strings.Join(val, ", ")
	}
	return env, nil
}

// splitPos returns the index where path should
// be split based on t.SplitPath.
func (t Transport) splitPos(path string) int {
	// TODO:
	// if httpserver.CaseSensitivePath {
	// 	return strings.Index(path, r.SplitPath)
	// }
	return strings.Index(strings.ToLower(path), strings.ToLower(t.SplitPath))
}

// TODO:
// Map of supported protocols to Apache ssl_mod format
// Note that these are slightly different from SupportedProtocols in caddytls/config.go
var tlsProtocolStrings = map[uint16]string{
	tls.VersionTLS10: "TLSv1",
	tls.VersionTLS11: "TLSv1.1",
	tls.VersionTLS12: "TLSv1.2",
	tls.VersionTLS13: "TLSv1.3",
}

var headerNameReplacer = strings.NewReplacer(" ", "_", "-", "_")

// Interface guards
var (
	_ caddy.Provisioner = (*Transport)(nil)
	_ http.RoundTripper = (*Transport)(nil)
)
<|endoftext|>


<commit_before>package messages

import (
	"bytes"
	fio "github.com/cucumber/messages-go/v10/io"
	gio "github.com/gogo/protobuf/io"
	"github.com/stretchr/testify/require"
	"math"
	"testing"
)

func TestMessages(t *testing.T) {
	t.Run("builds a pickle doc string", func(t *testing.T) {
		pickleDocString := PickleStepArgument_PickleDocString{
			MediaType: "text/plain",
			Content:   "some\ncontent\n",
		}

		b := &bytes.Buffer{}
		writer := gio.NewDelimitedWriter(b)
		require.NoError(t, writer.WriteMsg(&pickleDocString))

		r := gio.NewDelimitedReader(b, math.MaxInt32)
		var decoded PickleStepArgument_PickleDocString
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, "some\ncontent\n", decoded.Content)
	})

	t.Run("builds a step", func(t *testing.T) {
		step := &GherkinDocument_Feature_Step{
			Keyword: "Given",
			Text:    "the following message:",
			Location: &Location{
				Line:   11,
				Column: 4,
			},

			Argument: &GherkinDocument_Feature_Step_DocString_{
				DocString: &GherkinDocument_Feature_Step_DocString{
					Content: "Hello",
					Location: &Location{
						Line:   12,
						Column: 6,
					},
				},
			},
		}

		b := &bytes.Buffer{}
		writer := gio.NewDelimitedWriter(b)
		require.NoError(t, writer.WriteMsg(step))

		r := gio.NewDelimitedReader(b, 4096)
		var decoded GherkinDocument_Feature_Step
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, "Hello", decoded.GetDocString().Content)
	})

	t.Run("reads an attachment with a tiny string as NDJSON", func(t *testing.T) {
		attachment := &Attachment{
			Body: &Attachment_Text{Text: "Hello"},
		}
		b := &bytes.Buffer{}
		writer := fio.NewNdjsonWriter(b)
		require.NoError(t, writer.WriteMsg(attachment))
		r := fio.NewNdjsonReader(b)
		var decoded Attachment
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, "Hello", decoded.GetText())
	})

	t.Run("reads an attachment with a 70k string as NDJSON", func(t *testing.T) {
		ba := make([]byte, 70000)
		for i := range ba {
			ba[i] = "x"[0]
		}
		s := string(ba)
		attachment := &Attachment{
			Body: &Attachment_Text{Text: s},
		}
		b := &bytes.Buffer{}
		writer := fio.NewNdjsonWriter(b)
		require.NoError(t, writer.WriteMsg(attachment))
		r := fio.NewNdjsonReader(b)
		var decoded Attachment
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, s, decoded.GetText())
	})
}
<commit_msg>More rigorous test<commit_after>package messages

import (
	"bytes"
	fio "github.com/cucumber/messages-go/v10/io"
	gio "github.com/gogo/protobuf/io"
	"github.com/stretchr/testify/require"
	"math"
	"testing"
)

func TestMessages(t *testing.T) {
	t.Run("builds a pickle doc string", func(t *testing.T) {
		pickleDocString := PickleStepArgument_PickleDocString{
			MediaType: "text/plain",
			Content:   "some\ncontent\n",
		}

		b := &bytes.Buffer{}
		writer := gio.NewDelimitedWriter(b)
		require.NoError(t, writer.WriteMsg(&pickleDocString))

		r := gio.NewDelimitedReader(b, math.MaxInt32)
		var decoded PickleStepArgument_PickleDocString
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, "some\ncontent\n", decoded.Content)
	})

	t.Run("builds a step", func(t *testing.T) {
		step := &GherkinDocument_Feature_Step{
			Keyword: "Given",
			Text:    "the following message:",
			Location: &Location{
				Line:   11,
				Column: 4,
			},

			Argument: &GherkinDocument_Feature_Step_DocString_{
				DocString: &GherkinDocument_Feature_Step_DocString{
					Content: "Hello",
					Location: &Location{
						Line:   12,
						Column: 6,
					},
				},
			},
		}

		b := &bytes.Buffer{}
		writer := gio.NewDelimitedWriter(b)
		require.NoError(t, writer.WriteMsg(step))

		r := gio.NewDelimitedReader(b, 4096)
		var decoded GherkinDocument_Feature_Step
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, "Hello", decoded.GetDocString().Content)
	})

	t.Run("reads an attachment with a tiny string as NDJSON", func(t *testing.T) {
		attachment := &Attachment{
			Body: &Attachment_Text{Text: "Hello"},
		}
		b := &bytes.Buffer{}
		writer := fio.NewNdjsonWriter(b)
		require.NoError(t, writer.WriteMsg(attachment))
		r := fio.NewNdjsonReader(b)
		var decoded Attachment
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, "Hello", decoded.GetText())
	})

	t.Run("reads an attachment with a 9Mb string as NDJSON", func(t *testing.T) {
		ba := make([]byte, 9*1024*1024)
		for i := range ba {
			ba[i] = "x"[0]
		}
		s := string(ba)
		attachment := &Attachment{
			Body: &Attachment_Text{Text: s},
		}
		b := &bytes.Buffer{}
		writer := fio.NewNdjsonWriter(b)
		require.NoError(t, writer.WriteMsg(attachment))
		r := fio.NewNdjsonReader(b)
		var decoded Attachment
		require.NoError(t, r.ReadMsg(&decoded))
		require.Equal(t, s, decoded.GetText())
	})
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"regexp"
	"syscall"
	"time"

	"github.com/cubicdaiya/cachectl/cachectl"
)

func purgePages(target cachectl.SectionTarget, re *regexp.Regexp) error {
	fi, err := os.Stat(target.Path)
	if err != nil {
		return err
	}

	verbose := false

	if fi.IsDir() {
		err := cachectl.WalkPurgePages(target.Path, re, target.Rate, verbose)
		if err != nil {
			return fmt.Errorf("failed to walk in %s.", fi.Name())
		}
	} else {
		if !fi.Mode().IsRegular() {
			return fmt.Errorf("%s is not regular file", fi.Name())
		}

		err := cachectl.RunPurgePages(target.Path, fi.Size(), target.Rate, verbose)
		if err != nil {
			return fmt.Errorf("%s: %s", fi.Name(), err.Error())
		}
	}

	return nil
}

func scheduledPurgePages(target cachectl.SectionTarget, purgeOnStart bool) {

	if target.PurgeInterval == -1 {
		log.Printf("cachectld runs for the target(path:%s, filter:%s) when only received USR1\n",
			target.Path, target.Filter)
		return
	}

	re := regexp.MustCompile(target.Filter)

	if purgeOnStart {
		err := purgePages(target, re)
		if err != nil {
			log.Println(err)
		}
	}

	for {
		timer := time.NewTimer(time.Second * time.Duration(target.PurgeInterval))
		<-timer.C

		err := purgePages(target, re)
		if err != nil {
			log.Println(err)
		}
	}
}

func waitSignal() int {
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan,
		syscall.SIGUSR1,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	var exitcode int

	s := <-sigchan

	switch s {
	case syscall.SIGUSR1:
		// not exit
		exitcode = -1
	case syscall.SIGHUP:
		fallthrough
	case syscall.SIGINT:
		fallthrough
	case syscall.SIGTERM:
		fallthrough
	case syscall.SIGQUIT:
		exitcode = 0
	default:
		exitcode = 1
	}

	return exitcode
}

func main() {

	// Parse flags
	version := flag.Bool("v", false, "show version")
	purgeOnStart := flag.Bool("a", false, "run all targets at the startup time")
	confPath := flag.String("c", "", "configuration file for cachectld")
	flag.Parse()

	if *version {
		cachectl.PrintVersion(cachectl.Cachectld)
		return
	}

	var confCachectld cachectl.ConfToml
	err := cachectl.LoadConf(*confPath, &confCachectld)
	if err != nil {
		log.Fatal(err)
	}

	err = cachectl.ValidateConf(&confCachectld)
	if err != nil {
		log.Fatal(err)
	}

	for _, target := range confCachectld.Targets {
		go scheduledPurgePages(target, *purgeOnStart)
	}

waitSignalLoop:
	code := waitSignal()

	// When received SIGUSR1,
	// cachectld runs purgePages().
	if code == -1 {
		log.Println("Run all targets with SIGUSR1.")
		for _, target := range confCachectld.Targets {
			re := regexp.MustCompile(target.Filter)
			err := purgePages(target, re)
			if err != nil {
				log.Println(err)
			}
		}
		goto waitSignalLoop
	}

	os.Exit(code)
}
<commit_msg>use regexp.Compile() instead of regexp.MustCompile().<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"regexp"
	"syscall"
	"time"

	"github.com/cubicdaiya/cachectl/cachectl"
)

func purgePages(target cachectl.SectionTarget, re *regexp.Regexp) error {
	fi, err := os.Stat(target.Path)
	if err != nil {
		return err
	}

	verbose := false

	if fi.IsDir() {
		err := cachectl.WalkPurgePages(target.Path, re, target.Rate, verbose)
		if err != nil {
			return fmt.Errorf("failed to walk in %s.", fi.Name())
		}
	} else {
		if !fi.Mode().IsRegular() {
			return fmt.Errorf("%s is not regular file", fi.Name())
		}

		err := cachectl.RunPurgePages(target.Path, fi.Size(), target.Rate, verbose)
		if err != nil {
			return fmt.Errorf("%s: %s", fi.Name(), err.Error())
		}
	}

	return nil
}

func scheduledPurgePages(target cachectl.SectionTarget, purgeOnStart bool) {

	if target.PurgeInterval == -1 {
		log.Printf("cachectld runs for the target(path:%s, filter:%s) when only received USR1\n",
			target.Path, target.Filter)
		return
	}

	re, err := regexp.Compile(target.Filter)
	if err != nil {
		log.Printf("target: %s, filter is invalid: %s.",
			target.Path, target.Filter)
		return
	}

	if purgeOnStart {
		err := purgePages(target, re)
		if err != nil {
			log.Println(err)
		}
	}

	for {
		timer := time.NewTimer(time.Second * time.Duration(target.PurgeInterval))
		<-timer.C

		err := purgePages(target, re)
		if err != nil {
			log.Println(err)
		}
	}
}

func waitSignal() int {
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan,
		syscall.SIGUSR1,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	var exitcode int

	s := <-sigchan

	switch s {
	case syscall.SIGUSR1:
		// not exit
		exitcode = -1
	case syscall.SIGHUP:
		fallthrough
	case syscall.SIGINT:
		fallthrough
	case syscall.SIGTERM:
		fallthrough
	case syscall.SIGQUIT:
		exitcode = 0
	default:
		exitcode = 1
	}

	return exitcode
}

func main() {

	// Parse flags
	version := flag.Bool("v", false, "show version")
	purgeOnStart := flag.Bool("a", false, "run all targets at the startup time")
	confPath := flag.String("c", "", "configuration file for cachectld")
	flag.Parse()

	if *version {
		cachectl.PrintVersion(cachectl.Cachectld)
		return
	}

	var confCachectld cachectl.ConfToml
	err := cachectl.LoadConf(*confPath, &confCachectld)
	if err != nil {
		log.Fatal(err)
	}

	err = cachectl.ValidateConf(&confCachectld)
	if err != nil {
		log.Fatal(err)
	}

	for _, target := range confCachectld.Targets {
		go scheduledPurgePages(target, *purgeOnStart)
	}

waitSignalLoop:
	code := waitSignal()

	// When received SIGUSR1,
	// cachectld runs purgePages().
	if code == -1 {
		log.Println("Run all targets with SIGUSR1.")
		for _, target := range confCachectld.Targets {
			re, err := regexp.Compile(target.Filter)
			if err != nil {
				log.Println(err)
			}
			if err := purgePages(target, re); err != nil {
				log.Println(err)
			}
		}
		goto waitSignalLoop
	}

	os.Exit(code)
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"encoding/json"
	"fmt"
	asrt "github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"
)

// TestShareCmd tests `ddev share`
func TestShareCmd(t *testing.T) {
	if os.Getenv("DDEV_TEST_SHARE_CMD") != "true" {
		t.Skip("Skipping because DDEV_TEST_SHARE_CMD != true")
	}
	if runtime.GOOS == "windows" {
		t.Skip("Skipping because unreliable on Windows due to DNS lookup failure")
	}
	if os.Getenv("GITHUB_ACTIONS") == "true" {
		t.Skip("Skipping on GitHub actions because no auth can be provided")
	}
	assert := asrt.New(t)

	site := TestSites[0]
	defer site.Chdir()()

	// Configure ddev/ngrok to use json output to stdout
	cmd := exec.Command(DdevBin, "config", "--ngrok-args", "-log stdout -log-format=json")
	err := cmd.Start()
	require.NoError(t, err)
	err = cmd.Wait()
	require.NoError(t, err)

	cmd = exec.Command(DdevBin, "share")
	cmdReader, err := cmd.StdoutPipe()
	require.NoError(t, err)
	scanner := bufio.NewScanner(cmdReader)

	// Make absolutely sure the ngrok process gets killed off, because otherwise
	// the testbot (windows) can remain occupied forever.
	// nolint: errcheck
	t.Cleanup(func() {
		err = pKill(cmd)
		assert.NoError(err)
		_ = cmd.Wait()
		_ = cmdReader.Close()

		if err != nil && !strings.Contains(err.Error(), "process already finished") {
			assert.NoError(err)
		}
	})
	logData := make(map[string]string)

	scanDone := make(chan bool, 1)
	defer close(scanDone)

	// Read through the ngrok json output until we get the url it has opened
	go func() {
		for scanner.Scan() {
			logLine := scanner.Text()

			err := json.Unmarshal([]byte(logLine), &logData)
			if err != nil {
				switch err.(type) {
				case *json.SyntaxError:
					continue
				default:
					t.Errorf("failed unmarshalling %v: %v", logLine, err)
					break
				}
			}
			if logErr, ok := logData["err"]; ok && logErr != "<nil>" {
				if strings.Contains(logErr, "Your account is limited to 1 simultaneous") {
					t.Errorf("Failed because ngrok account in use elsewhere: %s", logErr)
					break
				}
			}
			if _, ok := logData["url"]; ok {
				break
			}
		}
		scanDone <- true
	}()
	err = cmd.Start()
	require.NoError(t, err)
	select {
	case <-scanDone:
		fmt.Printf("Scanning all done at %v\n", time.Now())
	case <-time.After(20 * time.Second):
		t.Fatal("Timed out waiting for reads\n", time.Now())
	}
	// If URL is provided, try to hit it and look for expected response
	if url, ok := logData["url"]; ok {
		resp, err := http.Get(url + site.Safe200URIWithExpectation.URI)
		if err != nil {
			t.Logf("http.Get on url=%s failed, err=%v", url+site.Safe200URIWithExpectation.URI, err)
			err = pKill(cmd)
			assert.NoError(err)
			return
		}
		//nolint: errcheck
		defer resp.Body.Close()
		body, err := io.ReadAll(resp.Body)
		assert.NoError(err)
		assert.Contains(string(body), site.Safe200URIWithExpectation.Expect)
	} else {
		t.Errorf("no URL found: %v", logData)
	}
}

// pKill kills a started cmd; If windows, it shells out to the
// taskkill command.
func pKill(cmd *exec.Cmd) error {
	var err error
	if cmd == nil {
		return fmt.Errorf("pKill: cmd is nill")
	}
	if runtime.GOOS == "windows" {
		// Windows has a completely different process model, no SIGCHLD,
		// no killing of subprocesses. I wasn't successful in finding a way
		// to properly kill a process set using golang; rfay 20190622
		kill := exec.Command("TASKKILL", "/T", "/F", "/PID", strconv.Itoa(cmd.Process.Pid))
		kill.Stderr = os.Stderr
		kill.Stdout = os.Stdout
		err = kill.Run()
	} else {
		err = cmd.Process.Kill()
	}
	return err
}
<commit_msg>[tests only] Try again to make sure ngrok dies (#3538)<commit_after>package cmd

import (
	"bufio"
	"encoding/json"
	"fmt"
	exec2 "github.com/drud/ddev/pkg/exec"
	asrt "github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"
)

// TestShareCmd tests `ddev share`
func TestShareCmd(t *testing.T) {
	if os.Getenv("DDEV_TEST_SHARE_CMD") != "true" {
		t.Skip("Skipping because DDEV_TEST_SHARE_CMD != true")
	}
	if runtime.GOOS == "windows" {
		t.Skip("Skipping because unreliable on Windows due to DNS lookup failure")
	}
	if os.Getenv("GITHUB_ACTIONS") == "true" {
		t.Skip("Skipping on GitHub actions because no auth can be provided")
	}
	assert := asrt.New(t)

	site := TestSites[0]
	defer site.Chdir()()

	// Configure ddev/ngrok to use json output to stdout
	cmd := exec.Command(DdevBin, "config", "--ngrok-args", "-log stdout -log-format=json")
	err := cmd.Start()
	require.NoError(t, err)
	err = cmd.Wait()
	require.NoError(t, err)

	cmd = exec.Command(DdevBin, "share")
	cmdReader, err := cmd.StdoutPipe()
	require.NoError(t, err)
	scanner := bufio.NewScanner(cmdReader)

	// Make absolutely sure the ngrok process gets killed off, because otherwise
	// the testbot (windows) can remain occupied forever.
	// nolint: errcheck
	t.Cleanup(func() {
		err = pKill(cmd)
		assert.NoError(err)
		_ = cmd.Wait()
		_ = cmdReader.Close()
		_, err = exec.LookPath("killall")
		// Try to kill ngrok any way we can, avoid having two run at same time.
		if err == nil {
			_, _ = exec2.RunHostCommand("killall", "-9", "ngrok")
		}

		if err != nil && !strings.Contains(err.Error(), "process already finished") {
			assert.NoError(err)
		}
	})
	logData := make(map[string]string)

	scanDone := make(chan bool, 1)
	defer close(scanDone)

	// Read through the ngrok json output until we get the url it has opened
	go func() {
		for scanner.Scan() {
			logLine := scanner.Text()

			err := json.Unmarshal([]byte(logLine), &logData)
			if err != nil {
				switch err.(type) {
				case *json.SyntaxError:
					continue
				default:
					t.Errorf("failed unmarshalling %v: %v", logLine, err)
					break
				}
			}
			if logErr, ok := logData["err"]; ok && logErr != "<nil>" {
				if strings.Contains(logErr, "Your account is limited to 1 simultaneous") {
					t.Errorf("Failed because ngrok account in use elsewhere: %s", logErr)
					break
				}
			}
			if _, ok := logData["url"]; ok {
				break
			}
		}
		scanDone <- true
	}()
	err = cmd.Start()
	require.NoError(t, err)
	select {
	case <-scanDone:
		fmt.Printf("Scanning all done at %v\n", time.Now())
	case <-time.After(20 * time.Second):
		t.Fatal("Timed out waiting for reads\n", time.Now())
	}
	// If URL is provided, try to hit it and look for expected response
	if url, ok := logData["url"]; ok {
		resp, err := http.Get(url + site.Safe200URIWithExpectation.URI)
		if err != nil {
			t.Logf("http.Get on url=%s failed, err=%v", url+site.Safe200URIWithExpectation.URI, err)
			err = pKill(cmd)
			assert.NoError(err)
			return
		}
		//nolint: errcheck
		defer resp.Body.Close()
		body, err := io.ReadAll(resp.Body)
		assert.NoError(err)
		assert.Contains(string(body), site.Safe200URIWithExpectation.Expect)
	} else {
		t.Errorf("no URL found: %v", logData)
	}
}

// pKill kills a started cmd; If windows, it shells out to the
// taskkill command.
func pKill(cmd *exec.Cmd) error {
	var err error
	if cmd == nil {
		return fmt.Errorf("pKill: cmd is nill")
	}
	if runtime.GOOS == "windows" {
		// Windows has a completely different process model, no SIGCHLD,
		// no killing of subprocesses. I wasn't successful in finding a way
		// to properly kill a process set using golang; rfay 20190622
		kill := exec.Command("TASKKILL", "/T", "/F", "/PID", strconv.Itoa(cmd.Process.Pid))
		kill.Stderr = os.Stderr
		kill.Stdout = os.Stdout
		err = kill.Run()
	} else {
		err = cmd.Process.Kill()
	}
	return err
}
<|endoftext|>


<commit_before>package run

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
)

const logo = `
 8888888           .d888 888                   8888888b.  888888b.
   888            d88P"  888                   888  "Y88b 888  "88b
   888            888    888                   888    888 888  .88P
   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
   888   888  888 888    888 888  888   X88K   888    888 888    888
   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"

`

// Command represents the command executed by "influxd run".
type Command struct {
	Version string
	Branch  string
	Commit  string

	closing chan struct{}
	Closed  chan struct{}

	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	Server *Server
}

// NewCommand return a new instance of Command.
func NewCommand() *Command {
	return &Command{
		closing: make(chan struct{}),
		Closed:  make(chan struct{}),
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
	}
}

// Run parses the config from args and runs the server.
func (cmd *Command) Run(args ...string) error {
	// Parse the command line flags.
	options, err := cmd.ParseFlags(args...)
	if err != nil {
		return err
	}

	// Print sweet InfluxDB logo.
	fmt.Print(logo)

	// Write the PID file.
	if err := cmd.writePIDFile(options.PIDFile); err != nil {
		return fmt.Errorf("write pid file: %s", err)
	}

	// Turn on block profiling to debug stuck databases
	runtime.SetBlockProfileRate(int(1 * time.Second))

	// Parse config
	config, err := cmd.ParseConfig(options.ConfigPath)
	if err != nil {
		return fmt.Errorf("parse config: %s", err)
	}

	// Apply any environment variables on top of the parsed config
	if err := config.ApplyEnvOverrides(); err != nil {
		return fmt.Errorf("apply env config: %v", err)
	}

	// Override config hostname if specified in the command line args.
	if options.Hostname != "" {
		config.Meta.Hostname = options.Hostname
	}

	if options.Join != "" {
		config.Meta.Peers = strings.Split(options.Join, ",")
	}

	// Validate the configuration.
	if err := config.Validate(); err != nil {
		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`.", err)
	}

	// Create server from config and start it.
	s, err := NewServer(config, cmd.Version)
	if err != nil {
		return fmt.Errorf("create server: %s", err)
	}
	s.CPUProfile = options.CPUProfile
	s.MemProfile = options.MemProfile
	if err := s.Open(); err != nil {
		return fmt.Errorf("open server: %s", err)
	}
	cmd.Server = s

	// Mark start-up in log.
	log.Printf("InfluxDB starting, version %s, branch %s, commit %s", cmd.Version, cmd.Branch, cmd.Commit)
	log.Printf("Go version %s, GOMAXPROCS set to %d", runtime.Version(), runtime.GOMAXPROCS(0))

	// Begin monitoring the server's error channel.
	go cmd.monitorServerErrors()

	return nil
}

// Close shuts down the server.
func (cmd *Command) Close() error {
	defer close(cmd.Closed)
	close(cmd.closing)
	if cmd.Server != nil {
		return cmd.Server.Close()
	}
	return nil
}

func (cmd *Command) monitorServerErrors() {
	logger := log.New(cmd.Stderr, "", log.LstdFlags)
	for {
		select {
		case err := <-cmd.Server.Err():
			logger.Println(err)
		case <-cmd.closing:
			return
		}
	}
}

// ParseFlags parses the command line flags from args and returns an options set.
func (cmd *Command) ParseFlags(args ...string) (Options, error) {
	var options Options
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.StringVar(&options.ConfigPath, "config", "", "")
	fs.StringVar(&options.PIDFile, "pidfile", "", "")
	fs.StringVar(&options.Hostname, "hostname", "", "")
	fs.StringVar(&options.Join, "join", "", "")
	fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
	fs.StringVar(&options.MemProfile, "memprofile", "", "")
	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) }
	if err := fs.Parse(args); err != nil {
		return Options{}, err
	}
	return options, nil
}

// writePIDFile writes the process ID to path.
func (cmd *Command) writePIDFile(path string) error {
	// Ignore if path is not set.
	if path == "" {
		return nil
	}

	// Ensure the required directory structure exists.
	err := os.MkdirAll(filepath.Dir(path), 0777)
	if err != nil {
		return fmt.Errorf("mkdir: %s", err)
	}

	// Retrieve the PID and write it.
	pid := strconv.Itoa(os.Getpid())
	if err := ioutil.WriteFile(path, []byte(pid), 0666); err != nil {
		return fmt.Errorf("write file: %s", err)
	}

	return nil
}

// ParseConfig parses the config at path.
// Returns a demo configuration if path is blank.
func (cmd *Command) ParseConfig(path string) (*Config, error) {
	// Use demo configuration if no config path is specified.
	if path == "" {
		fmt.Fprintln(cmd.Stdout, "no configuration provided, using default settings")
		return NewDemoConfig()
	}

	fmt.Fprintf(cmd.Stdout, "Using configuration at: %s\n", path)

	config := NewConfig()
	if _, err := toml.DecodeFile(path, &config); err != nil {
		return nil, err
	}

	return config, nil
}

var usage = `usage: run [flags]

run starts the broker and data node server. If this is the first time running
the command then a new cluster will be initialized unless the -join argument
is used.

        -config <path>
                          Set the path to the configuration file.

        -hostname <name>
                          Override the hostname, the 'hostname' configuration
                          option will be overridden.

        -join <url>
                          Joins the server to an existing cluster.

        -pidfile <path>
                          Write process ID to a file.
`

// Options represents the command line options that can be parsed.
type Options struct {
	ConfigPath string
	PIDFile    string
	Hostname   string
	Join       string
	CPUProfile string
	MemProfile string
}
<commit_msg>Make the startup log message actually first<commit_after>package run

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/BurntSushi/toml"
)

const logo = `
 8888888           .d888 888                   8888888b.  888888b.
   888            d88P"  888                   888  "Y88b 888  "88b
   888            888    888                   888    888 888  .88P
   888   88888b.  888888 888 888  888 888  888 888    888 8888888K.
   888   888 "88b 888    888 888  888  Y8bd8P' 888    888 888  "Y88b
   888   888  888 888    888 888  888   X88K   888    888 888    888
   888   888  888 888    888 Y88b 888 .d8""8b. 888  .d88P 888   d88P
 8888888 888  888 888    888  "Y88888 888  888 8888888P"  8888888P"

`

// Command represents the command executed by "influxd run".
type Command struct {
	Version string
	Branch  string
	Commit  string

	closing chan struct{}
	Closed  chan struct{}

	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	Server *Server
}

// NewCommand return a new instance of Command.
func NewCommand() *Command {
	return &Command{
		closing: make(chan struct{}),
		Closed:  make(chan struct{}),
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
	}
}

// Run parses the config from args and runs the server.
func (cmd *Command) Run(args ...string) error {
	// Parse the command line flags.
	options, err := cmd.ParseFlags(args...)
	if err != nil {
		return err
	}

	// Print sweet InfluxDB logo.
	fmt.Print(logo)

	// Mark start-up in log.
	log.Printf("InfluxDB starting, version %s, branch %s, commit %s", cmd.Version, cmd.Branch, cmd.Commit)
	log.Printf("Go version %s, GOMAXPROCS set to %d", runtime.Version(), runtime.GOMAXPROCS(0))

	// Write the PID file.
	if err := cmd.writePIDFile(options.PIDFile); err != nil {
		return fmt.Errorf("write pid file: %s", err)
	}

	// Turn on block profiling to debug stuck databases
	runtime.SetBlockProfileRate(int(1 * time.Second))

	// Parse config
	config, err := cmd.ParseConfig(options.ConfigPath)
	if err != nil {
		return fmt.Errorf("parse config: %s", err)
	}

	// Apply any environment variables on top of the parsed config
	if err := config.ApplyEnvOverrides(); err != nil {
		return fmt.Errorf("apply env config: %v", err)
	}

	// Override config hostname if specified in the command line args.
	if options.Hostname != "" {
		config.Meta.Hostname = options.Hostname
	}

	if options.Join != "" {
		config.Meta.Peers = strings.Split(options.Join, ",")
	}

	// Validate the configuration.
	if err := config.Validate(); err != nil {
		return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`.", err)
	}

	// Create server from config and start it.
	s, err := NewServer(config, cmd.Version)
	if err != nil {
		return fmt.Errorf("create server: %s", err)
	}
	s.CPUProfile = options.CPUProfile
	s.MemProfile = options.MemProfile
	if err := s.Open(); err != nil {
		return fmt.Errorf("open server: %s", err)
	}
	cmd.Server = s

	// Begin monitoring the server's error channel.
	go cmd.monitorServerErrors()

	return nil
}

// Close shuts down the server.
func (cmd *Command) Close() error {
	defer close(cmd.Closed)
	close(cmd.closing)
	if cmd.Server != nil {
		return cmd.Server.Close()
	}
	return nil
}

func (cmd *Command) monitorServerErrors() {
	logger := log.New(cmd.Stderr, "", log.LstdFlags)
	for {
		select {
		case err := <-cmd.Server.Err():
			logger.Println(err)
		case <-cmd.closing:
			return
		}
	}
}

// ParseFlags parses the command line flags from args and returns an options set.
func (cmd *Command) ParseFlags(args ...string) (Options, error) {
	var options Options
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.StringVar(&options.ConfigPath, "config", "", "")
	fs.StringVar(&options.PIDFile, "pidfile", "", "")
	fs.StringVar(&options.Hostname, "hostname", "", "")
	fs.StringVar(&options.Join, "join", "", "")
	fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
	fs.StringVar(&options.MemProfile, "memprofile", "", "")
	fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) }
	if err := fs.Parse(args); err != nil {
		return Options{}, err
	}
	return options, nil
}

// writePIDFile writes the process ID to path.
func (cmd *Command) writePIDFile(path string) error {
	// Ignore if path is not set.
	if path == "" {
		return nil
	}

	// Ensure the required directory structure exists.
	err := os.MkdirAll(filepath.Dir(path), 0777)
	if err != nil {
		return fmt.Errorf("mkdir: %s", err)
	}

	// Retrieve the PID and write it.
	pid := strconv.Itoa(os.Getpid())
	if err := ioutil.WriteFile(path, []byte(pid), 0666); err != nil {
		return fmt.Errorf("write file: %s", err)
	}

	return nil
}

// ParseConfig parses the config at path.
// Returns a demo configuration if path is blank.
func (cmd *Command) ParseConfig(path string) (*Config, error) {
	// Use demo configuration if no config path is specified.
	if path == "" {
		log.Println("no configuration provided, using default settings")
		return NewDemoConfig()
	}

	log.Printf("Using configuration at: %s\n", path)

	config := NewConfig()
	if _, err := toml.DecodeFile(path, &config); err != nil {
		return nil, err
	}

	return config, nil
}

var usage = `usage: run [flags]

run starts the broker and data node server. If this is the first time running
the command then a new cluster will be initialized unless the -join argument
is used.

        -config <path>
                          Set the path to the configuration file.

        -hostname <name>
                          Override the hostname, the 'hostname' configuration
                          option will be overridden.

        -join <url>
                          Joins the server to an existing cluster.

        -pidfile <path>
                          Write process ID to a file.
`

// Options represents the command line options that can be parsed.
type Options struct {
	ConfigPath string
	PIDFile    string
	Hostname   string
	Join       string
	CPUProfile string
	MemProfile string
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/google/go-github/github"
	"github.com/joshdk/go-junit"
	"go.uber.org/zap"
	"golang.org/x/oauth2"
)

const (

	// Keys of required environment variables
	projectUsernameKey = "CIRCLE_PROJECT_USERNAME"
	projectRepoNameKey = "CIRCLE_PROJECT_REPONAME"
	circleBuildURLKey  = "CIRCLE_BUILD_URL"
	jobNameKey         = "CIRCLE_JOB"
	githubAPITokenKey  = "GITHUB_TOKEN"
)

func main() {
	pathToArtifacts := ""
	if len(os.Args) > 1 {
		pathToArtifacts = os.Args[1]
	}

	rg := newReportGenerator(pathToArtifacts)

	// Look for existing open GitHub Issue that resulted from previous
	// failures of this job.
	rg.logger.Info("Searching GitHub for existing Issues")
	existingIssue := rg.getExistingIssue()

	if existingIssue == nil {
		// If none exists, create a new GitHub Issue for the failure.
		rg.logger.Info("No existing Issues found, creating a new one.")
		createdIssue := rg.createIssue()
		rg.logger.Info("New GitHub Issue created", zap.String("html_url", *createdIssue.HTMLURL))
	} else {
		// Otherwise, add a comment to the existing Issue.
		rg.logger.Info(
			"Updating GitHub Issue with latest failure",
			zap.String("html_url", *existingIssue.HTMLURL),
		)
		createdIssueComment := rg.commentOnIssue(existingIssue)
		rg.logger.Info("GitHub Issue updated", zap.String("html_url", *createdIssueComment.HTMLURL))
	}
}

func newReportGenerator(pathToArtifacts string) *reportGenerator {
	logger, err := zap.NewDevelopment()
	if err != nil {
		fmt.Printf("Failed to set up logger: %v", err)
		os.Exit(1)
	}

	rg := &reportGenerator{
		ctx:    context.Background(),
		logger: logger,
	}

	rg.getRequiredEnv()

	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: rg.envVariables[githubAPITokenKey]})
	tc := oauth2.NewClient(rg.ctx, ts)
	rg.client = github.NewClient(tc)

	if pathToArtifacts != "" {
		rg.logger.Info("Ingesting test reports", zap.String("path", pathToArtifacts))
		suites, err := junit.IngestFile(pathToArtifacts)
		if err != nil {
			rg.logger.Warn(
				"Failed to ingest JUnit xml, omitting test results from report",
				zap.Error(err),
			)
		}

		rg.testSuites = suites
	}

	return rg
}

type reportGenerator struct {
	ctx          context.Context
	logger       *zap.Logger
	client       *github.Client
	envVariables map[string]string
	testSuites   []junit.Suite
}

// getRequiredEnv loads required environment variables for the main method.
// Some of the environment variables are built-in in CircleCI, whereas others
// need to be configured. See https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
// for a list of built-in environment variables.
func (rg *reportGenerator) getRequiredEnv() {
	env := map[string]string{}

	env[projectUsernameKey] = os.Getenv(projectUsernameKey)
	env[projectRepoNameKey] = os.Getenv(projectRepoNameKey)
	env[jobNameKey] = os.Getenv(jobNameKey)
	env[githubAPITokenKey] = os.Getenv(githubAPITokenKey)

	for k, v := range env {
		if v == "" {
			rg.logger.Fatal(
				"Required environment variable not set",
				zap.String("env_var", k),
			)
		}
	}

	rg.envVariables = env
}

const (
	issueTitleTemplate = `Bug report for failed CircleCI build (job: ${jobName})`
	issueBodyTemplate  = `
Auto-generated report for ${jobName} job build.

Link to failed build: ${linkToBuild}

${failedTests}

**Note**: Information about any subsequent build failures that happen while
this issue is open, will be added as comments with more information to this issue.
`
	issueCommentTemplate = `
Link to latest failed build: ${linkToBuild}

${failedTests}
`
)

func (rg reportGenerator) templateHelper(param string) string {
	switch param {
	case "jobName":
		return "`" + rg.envVariables[jobNameKey] + "`"
	case "linkToBuild":
		return os.Getenv(circleBuildURLKey)
	case "failedTests":
		return rg.getFailedTests()
	default:
		return ""
	}
}

// getExistingIssues gathers an existing GitHub Issue related to previous failures
// of the same job.
func (rg *reportGenerator) getExistingIssue() *github.Issue {
	issues, response, err := rg.client.Issues.ListByRepo(
		rg.ctx,
		rg.envVariables[projectUsernameKey],
		rg.envVariables[projectRepoNameKey],
		&github.IssueListByRepoOptions{
			State: "open",
		},
	)
	if err != nil {
		rg.logger.Fatal("Failed to search GitHub Issues", zap.Error(err))
	}

	if response.StatusCode != http.StatusOK {
		rg.handleBadResponses(response)
	}

	requiredTitle := rg.getIssueTitle()
	for _, issue := range issues {
		if *issue.Title == requiredTitle {
			return issue
		}
	}

	return nil
}

// commentOnIssue adds a new comment on an existing GitHub issue with
// information about the latest failure. This method is expected to be
// called only if there's an existing open Issue for the current job.
func (rg *reportGenerator) commentOnIssue(issue *github.Issue) *github.IssueComment {
	body := os.Expand(issueCommentTemplate, rg.templateHelper)

	issueComment, response, err := rg.client.Issues.CreateComment(
		rg.ctx,
		rg.envVariables[projectUsernameKey],
		rg.envVariables[projectRepoNameKey],
		*issue.Number,
		&github.IssueComment{
			Body: &body,
		},
	)
	if err != nil {
		rg.logger.Fatal("Failed to search GitHub Issues", zap.Error(err))
	}

	if response.StatusCode != http.StatusCreated {
		rg.handleBadResponses(response)
	}

	return issueComment
}

// createIssue creates a new GitHub Issue corresponding to a build failure.
func (rg *reportGenerator) createIssue() *github.Issue {
	title := rg.getIssueTitle()
	body := os.Expand(issueBodyTemplate, rg.templateHelper)

	issue, response, err := rg.client.Issues.Create(
		rg.ctx,
		rg.envVariables[projectUsernameKey],
		rg.envVariables[projectRepoNameKey],
		&github.IssueRequest{
			Title: &title,
			Body:  &body,
			// TODO: Set Assignees and labels
		})
	if err != nil {
		rg.logger.Fatal("Failed to create GitHub Issue", zap.Error(err))
	}

	if response.StatusCode != http.StatusCreated {
		rg.handleBadResponses(response)
	}

	return issue
}

func (rg reportGenerator) getIssueTitle() string {
	return strings.Replace(issueTitleTemplate, "${jobName}", rg.envVariables[jobNameKey], 1)
}

// getFailedTests returns information about failed tests if available, otherwise
// an empty string.
func (rg reportGenerator) getFailedTests() string {
	if len(rg.testSuites) == 0 {
		return ""
	}

	var sb strings.Builder
	sb.WriteString("#### Test Failures\n")

	for _, s := range rg.testSuites {
		for _, t := range s.Tests {
			if t.Status != junit.StatusFailed {
				continue
			}
			sb.WriteString("-  " + t.Name + "\n")
		}
	}

	return sb.String()
}

func (rg reportGenerator) handleBadResponses(response *github.Response) {
	body, _ := ioutil.ReadAll(response.Body)
	rg.logger.Fatal(
		"Unexpected response from GitHub",
		zap.String("status_code", string(response.StatusCode)),
		zap.String("response", string(body)),
		zap.String("url", response.Request.URL.String()),
	)
}
<commit_msg>Use zap int argument for int values instead of conversion (#1779)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/google/go-github/github"
	"github.com/joshdk/go-junit"
	"go.uber.org/zap"
	"golang.org/x/oauth2"
)

const (

	// Keys of required environment variables
	projectUsernameKey = "CIRCLE_PROJECT_USERNAME"
	projectRepoNameKey = "CIRCLE_PROJECT_REPONAME"
	circleBuildURLKey  = "CIRCLE_BUILD_URL"
	jobNameKey         = "CIRCLE_JOB"
	githubAPITokenKey  = "GITHUB_TOKEN"
)

func main() {
	pathToArtifacts := ""
	if len(os.Args) > 1 {
		pathToArtifacts = os.Args[1]
	}

	rg := newReportGenerator(pathToArtifacts)

	// Look for existing open GitHub Issue that resulted from previous
	// failures of this job.
	rg.logger.Info("Searching GitHub for existing Issues")
	existingIssue := rg.getExistingIssue()

	if existingIssue == nil {
		// If none exists, create a new GitHub Issue for the failure.
		rg.logger.Info("No existing Issues found, creating a new one.")
		createdIssue := rg.createIssue()
		rg.logger.Info("New GitHub Issue created", zap.String("html_url", *createdIssue.HTMLURL))
	} else {
		// Otherwise, add a comment to the existing Issue.
		rg.logger.Info(
			"Updating GitHub Issue with latest failure",
			zap.String("html_url", *existingIssue.HTMLURL),
		)
		createdIssueComment := rg.commentOnIssue(existingIssue)
		rg.logger.Info("GitHub Issue updated", zap.String("html_url", *createdIssueComment.HTMLURL))
	}
}

func newReportGenerator(pathToArtifacts string) *reportGenerator {
	logger, err := zap.NewDevelopment()
	if err != nil {
		fmt.Printf("Failed to set up logger: %v", err)
		os.Exit(1)
	}

	rg := &reportGenerator{
		ctx:    context.Background(),
		logger: logger,
	}

	rg.getRequiredEnv()

	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: rg.envVariables[githubAPITokenKey]})
	tc := oauth2.NewClient(rg.ctx, ts)
	rg.client = github.NewClient(tc)

	if pathToArtifacts != "" {
		rg.logger.Info("Ingesting test reports", zap.String("path", pathToArtifacts))
		suites, err := junit.IngestFile(pathToArtifacts)
		if err != nil {
			rg.logger.Warn(
				"Failed to ingest JUnit xml, omitting test results from report",
				zap.Error(err),
			)
		}

		rg.testSuites = suites
	}

	return rg
}

type reportGenerator struct {
	ctx          context.Context
	logger       *zap.Logger
	client       *github.Client
	envVariables map[string]string
	testSuites   []junit.Suite
}

// getRequiredEnv loads required environment variables for the main method.
// Some of the environment variables are built-in in CircleCI, whereas others
// need to be configured. See https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
// for a list of built-in environment variables.
func (rg *reportGenerator) getRequiredEnv() {
	env := map[string]string{}

	env[projectUsernameKey] = os.Getenv(projectUsernameKey)
	env[projectRepoNameKey] = os.Getenv(projectRepoNameKey)
	env[jobNameKey] = os.Getenv(jobNameKey)
	env[githubAPITokenKey] = os.Getenv(githubAPITokenKey)

	for k, v := range env {
		if v == "" {
			rg.logger.Fatal(
				"Required environment variable not set",
				zap.String("env_var", k),
			)
		}
	}

	rg.envVariables = env
}

const (
	issueTitleTemplate = `Bug report for failed CircleCI build (job: ${jobName})`
	issueBodyTemplate  = `
Auto-generated report for ${jobName} job build.

Link to failed build: ${linkToBuild}

${failedTests}

**Note**: Information about any subsequent build failures that happen while
this issue is open, will be added as comments with more information to this issue.
`
	issueCommentTemplate = `
Link to latest failed build: ${linkToBuild}

${failedTests}
`
)

func (rg reportGenerator) templateHelper(param string) string {
	switch param {
	case "jobName":
		return "`" + rg.envVariables[jobNameKey] + "`"
	case "linkToBuild":
		return os.Getenv(circleBuildURLKey)
	case "failedTests":
		return rg.getFailedTests()
	default:
		return ""
	}
}

// getExistingIssues gathers an existing GitHub Issue related to previous failures
// of the same job.
func (rg *reportGenerator) getExistingIssue() *github.Issue {
	issues, response, err := rg.client.Issues.ListByRepo(
		rg.ctx,
		rg.envVariables[projectUsernameKey],
		rg.envVariables[projectRepoNameKey],
		&github.IssueListByRepoOptions{
			State: "open",
		},
	)
	if err != nil {
		rg.logger.Fatal("Failed to search GitHub Issues", zap.Error(err))
	}

	if response.StatusCode != http.StatusOK {
		rg.handleBadResponses(response)
	}

	requiredTitle := rg.getIssueTitle()
	for _, issue := range issues {
		if *issue.Title == requiredTitle {
			return issue
		}
	}

	return nil
}

// commentOnIssue adds a new comment on an existing GitHub issue with
// information about the latest failure. This method is expected to be
// called only if there's an existing open Issue for the current job.
func (rg *reportGenerator) commentOnIssue(issue *github.Issue) *github.IssueComment {
	body := os.Expand(issueCommentTemplate, rg.templateHelper)

	issueComment, response, err := rg.client.Issues.CreateComment(
		rg.ctx,
		rg.envVariables[projectUsernameKey],
		rg.envVariables[projectRepoNameKey],
		*issue.Number,
		&github.IssueComment{
			Body: &body,
		},
	)
	if err != nil {
		rg.logger.Fatal("Failed to search GitHub Issues", zap.Error(err))
	}

	if response.StatusCode != http.StatusCreated {
		rg.handleBadResponses(response)
	}

	return issueComment
}

// createIssue creates a new GitHub Issue corresponding to a build failure.
func (rg *reportGenerator) createIssue() *github.Issue {
	title := rg.getIssueTitle()
	body := os.Expand(issueBodyTemplate, rg.templateHelper)

	issue, response, err := rg.client.Issues.Create(
		rg.ctx,
		rg.envVariables[projectUsernameKey],
		rg.envVariables[projectRepoNameKey],
		&github.IssueRequest{
			Title: &title,
			Body:  &body,
			// TODO: Set Assignees and labels
		})
	if err != nil {
		rg.logger.Fatal("Failed to create GitHub Issue", zap.Error(err))
	}

	if response.StatusCode != http.StatusCreated {
		rg.handleBadResponses(response)
	}

	return issue
}

func (rg reportGenerator) getIssueTitle() string {
	return strings.Replace(issueTitleTemplate, "${jobName}", rg.envVariables[jobNameKey], 1)
}

// getFailedTests returns information about failed tests if available, otherwise
// an empty string.
func (rg reportGenerator) getFailedTests() string {
	if len(rg.testSuites) == 0 {
		return ""
	}

	var sb strings.Builder
	sb.WriteString("#### Test Failures\n")

	for _, s := range rg.testSuites {
		for _, t := range s.Tests {
			if t.Status != junit.StatusFailed {
				continue
			}
			sb.WriteString("-  " + t.Name + "\n")
		}
	}

	return sb.String()
}

func (rg reportGenerator) handleBadResponses(response *github.Response) {
	body, _ := ioutil.ReadAll(response.Body)
	rg.logger.Fatal(
		"Unexpected response from GitHub",
		zap.Int("status_code", response.StatusCode),
		zap.String("response", string(body)),
		zap.String("url", response.Request.URL.String()),
	)
}
<|endoftext|>


<commit_before>package auth

import (
	"github.com/qor/auth"
	"github.com/qor/auth/database"
	"github.com/qor/auth/oauth/github"
	"github.com/qor/auth/oauth/google"
	"github.com/qor/auth/phone"
	"github.com/qor/qor-example/app/models"
	"github.com/qor/qor-example/config"
	"github.com/qor/qor-example/db"
)

var Auth = auth.New(&auth.Config{
	DB:        db.DB,
	Render:    config.View,
	UserModel: models.User{},
})

func init() {
	Auth.RegisterProvider(database.New(nil))
	Auth.RegisterProvider(phone.New())
	Auth.RegisterProvider(github.New(&config.Config.Github))
	Auth.RegisterProvider(google.New(&config.Config.Google))
}
<commit_msg>Add clean theme<commit_after>package auth

import (
	"github.com/qor/auth"
	"github.com/qor/auth_themes/clean"
	"github.com/qor/qor-example/app/models"
	"github.com/qor/qor-example/config"
	"github.com/qor/qor-example/db"
)

var Auth = auth.New(&auth.Config{
	DB:        db.DB,
	Render:    config.View,
	UserModel: models.User{},
})

func init() {
	clean.New(Auth)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

func (this *Gateway) helpHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Header().Set(ContentTypeHeader, ContentTypeText)
	w.Write([]byte(strings.TrimSpace(fmt.Sprintf(`
pub server: %s
sub server: %s
man server: %s
dbg server: %s

pub:
POST /topics/:topic/:ver?key=mykey&async=1
POST /ws/topics/:topic/:ver
 GET /raw/topics/:topic/:ver

sub:
 GET /topics/:appid/:topic/:ver/:group?limit=1&reset=newest
 GET /ws/topics/:appid/:topic/:ver/:group
 GET /raw/topics/:appid/:topic/:ver

man:
 GET /help
 GET /status
 GET /clusters
POST /topics/:cluster/:appid/:topic/:ver

dbg:
 GET /debug/pprof
 GET /debug/vars
`,
		options.pubHttpAddr, options.subHttpAddr,
		options.manHttpAddr, options.debugHttpAddr))))

}

func (this *Gateway) statusHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Write([]byte(fmt.Sprintf("ver:%s, build:%s, uptime:%s",
		gafka.Version, gafka.BuildId, time.Since(this.startedAt))))
}

func (this *Gateway) clustersHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Header().Set(ContentTypeHeader, ContentTypeJson)
	w.WriteHeader(http.StatusOK)
	b, _ := json.Marshal(meta.Default.Clusters())
	w.Write(b)
}

// /topics/:cluster/:appid/:topic/:ver?partitions=1&replicas=2
func (this *Gateway) addTopicHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)

	topic := params.ByName(UrlParamTopic)
	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName("appid")
	appid := r.Header.Get(HttpHeaderAppid)
	if !meta.Default.AuthPub(appid, r.Header.Get(HttpHeaderPubkey), topic) {
		this.writeAuthFailure(w)
		return
	}

	ver := params.ByName(UrlParamVersion)

	replicas, partitions := 2, 1
	query := r.URL.Query()
	partitionsArg := query.Get("partitions")
	if partitionsArg != "" {
		partitions, _ = strconv.Atoi(partitionsArg)
	}
	replicasArg := query.Get("replicas")
	if replicasArg != "" {
		replicas, _ = strconv.Atoi(replicasArg)
	}

	log.Info("%s add topic: {appid:%s, cluster:%s, topic:%s, ver:%s query:%s}",
		appid, hisAppid, cluster, topic, ver, query.Encode())

	topic = meta.KafkaTopic(hisAppid, topic, ver)
	lines, err := meta.Default.ZkCluster(cluster).AddTopic(topic, replicas, partitions)
	if err != nil {
		log.Info("%s add topic: %s", appid, err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	ok := false
	for _, l := range lines {
		log.Info("%s add topic: %s", appid, l)
		if strings.Contains(l, "Created topic") {
			ok = true
		}
	}

	if ok {
		w.Write(ResponseOk)
	} else {
		http.Error(w, strings.Join(lines, "\n"), http.StatusInternalServerError)
	}
}
<commit_msg>logging<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

func (this *Gateway) helpHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Header().Set(ContentTypeHeader, ContentTypeText)
	w.Write([]byte(strings.TrimSpace(fmt.Sprintf(`
pub server: %s
sub server: %s
man server: %s
dbg server: %s

pub:
POST /topics/:topic/:ver?key=mykey&async=1
POST /ws/topics/:topic/:ver
 GET /raw/topics/:topic/:ver

sub:
 GET /topics/:appid/:topic/:ver/:group?limit=1&reset=newest
 GET /ws/topics/:appid/:topic/:ver/:group
 GET /raw/topics/:appid/:topic/:ver

man:
 GET /help
 GET /status
 GET /clusters
POST /topics/:cluster/:appid/:topic/:ver

dbg:
 GET /debug/pprof
 GET /debug/vars
`,
		options.pubHttpAddr, options.subHttpAddr,
		options.manHttpAddr, options.debugHttpAddr))))

}

func (this *Gateway) statusHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Write([]byte(fmt.Sprintf("ver:%s, build:%s, uptime:%s",
		gafka.Version, gafka.BuildId, time.Since(this.startedAt))))
}

func (this *Gateway) clustersHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)
	w.Header().Set(ContentTypeHeader, ContentTypeJson)
	w.WriteHeader(http.StatusOK)
	b, _ := json.Marshal(meta.Default.Clusters())
	w.Write(b)
}

// /topics/:cluster/:appid/:topic/:ver?partitions=1&replicas=2
func (this *Gateway) addTopicHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	this.writeKatewayHeader(w)

	topic := params.ByName(UrlParamTopic)
	cluster := params.ByName(UrlParamCluster)
	hisAppid := params.ByName("appid")
	appid := r.Header.Get(HttpHeaderAppid)
	if !meta.Default.AuthPub(appid, r.Header.Get(HttpHeaderPubkey), topic) {
		this.writeAuthFailure(w)
		return
	}

	ver := params.ByName(UrlParamVersion)

	replicas, partitions := 2, 1
	query := r.URL.Query()
	partitionsArg := query.Get("partitions")
	if partitionsArg != "" {
		partitions, _ = strconv.Atoi(partitionsArg)
	}
	replicasArg := query.Get("replicas")
	if replicasArg != "" {
		replicas, _ = strconv.Atoi(replicasArg)
	}

	log.Info("%s add topic: {appid:%s, cluster:%s, topic:%s, ver:%s query:%s}",
		appid, hisAppid, cluster, topic, ver, query.Encode())

	topic = meta.KafkaTopic(hisAppid, topic, ver)
	lines, err := meta.Default.ZkCluster(cluster).AddTopic(topic, replicas, partitions)
	if err != nil {
		log.Info("%s add topic: %s", appid, err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	ok := false
	for _, l := range lines {
		log.Info("%s add topic in cluster %s: %s", appid, cluster, l)
		if strings.Contains(l, "Created topic") {
			ok = true
		}
	}

	if ok {
		w.Write(ResponseOk)
	} else {
		http.Error(w, strings.Join(lines, "\n"), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/spf13/cobra"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/registry"
	"k8s.io/kops/pkg/client/simple"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/kutil"
	"k8s.io/kops/util/pkg/tables"
	"k8s.io/kops/util/pkg/vfs"
)

type DeleteClusterCmd struct {
	Yes        bool
	Region     string
	External   bool
	Unregister bool
}

var deleteCluster DeleteClusterCmd

func init() {
	cmd := &cobra.Command{
		Use:   "cluster CLUSTERNAME [--yes]",
		Short: "Delete cluster",
		Long:  `Deletes a k8s cluster.`,
		Run: func(cmd *cobra.Command, args []string) {
			err := deleteCluster.Run(args)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	deleteCmd.AddCommand(cmd)
	argString := strings.ToLower(strings.Join(os.Args, " "))
	if strings.Contains(argString, "--yes") || strings.Contains(argString, "-y") {
		deleteCluster.Yes = true
	}

	cmd.Flags().BoolVar(&deleteCluster.Unregister, "unregister", false, "Don't delete cloud resources, just unregister the cluster")
	cmd.Flags().BoolVar(&deleteCluster.External, "external", false, "Delete an external cluster")

	cmd.Flags().StringVar(&deleteCluster.Region, "region", "", "region")
}

type getter func(o interface{}) interface{}

func (c *DeleteClusterCmd) Run(args []string) error {
	var configBase vfs.Path
	var clientset simple.Clientset

	err := rootCommand.ProcessArgs(args)
	if err != nil {
		return err
	}

	clusterName := rootCommand.clusterName
	if clusterName == "" {
		return fmt.Errorf("--name is required (for safety)")
	}

	var cloud fi.Cloud
	var cluster *api.Cluster

	if c.External {
		region := c.Region
		if region == "" {
			return fmt.Errorf("--region is required (when --external)")
		}

		tags := map[string]string{"KubernetesCluster": clusterName}
		cloud, err = awsup.NewAWSCloud(region, tags)
		if err != nil {
			return fmt.Errorf("error initializing AWS client: %v", err)
		}
	} else {
		clientset, err = rootCommand.Clientset()
		if err != nil {
			return err
		}

		cluster, err = clientset.Clusters().Get(clusterName)
		if err != nil {
			return err
		}

		if cluster == nil {
			return fmt.Errorf("cluster %q not found", clusterName)
		}

		if clusterName != cluster.ObjectMeta.Name {
			return fmt.Errorf("sanity check failed: cluster name mismatch")
		}

		configBase, err = registry.ConfigBase(cluster)
		if err != nil {
			return err
		}
	}

	wouldDeleteCloudResources := false

	if !c.Unregister {
		if cloud == nil {
			cloud, err = cloudup.BuildCloud(cluster)
			if err != nil {
				return err
			}
		}

		d := &kutil.DeleteCluster{}
		d.ClusterName = clusterName
		d.Cloud = cloud

		resources, err := d.ListResources()
		if err != nil {
			return err
		}

		if len(resources) == 0 {
			fmt.Printf("No cloud resources to delete\n")
		} else {
			wouldDeleteCloudResources = true

			t := &tables.Table{}
			t.AddColumn("TYPE", func(r *kutil.ResourceTracker) string {
				return r.Type
			})
			t.AddColumn("ID", func(r *kutil.ResourceTracker) string {
				return r.ID
			})
			t.AddColumn("NAME", func(r *kutil.ResourceTracker) string {
				return r.Name
			})
			var l []*kutil.ResourceTracker
			for _, v := range resources {
				l = append(l, v)
			}

			err := t.Render(l, os.Stdout, "TYPE", "NAME", "ID")
			if err != nil {
				return err
			}

			if !c.Yes {
				return fmt.Errorf("Must specify --yes to delete")
			}

			fmt.Fprintf(os.Stdout, "\n")

			err = d.DeleteResources(resources)
			if err != nil {
				return err
			}
		}
	}

	if !c.External {
		if !c.Yes {
			if wouldDeleteCloudResources {
				fmt.Printf("\nMust specify --yes to delete cloud resources & unregister cluster\n")
			} else {
				fmt.Printf("\nMust specify --yes to unregister the cluster\n")
			}
			return nil
		}
		err := registry.DeleteAllClusterState(configBase)
		if err != nil {
			return fmt.Errorf("error removing cluster from state store: %v", err)
		}
	}

	b := kutil.NewKubeconfigBuilder()
	b.Context = cluster.ObjectMeta.Name
	b.DeleteKubeConfig()

	fmt.Printf("\nCluster deleted\n")
	return nil
}
<commit_msg>slightly better way as it alloes other flags that may start with 'y'<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/spf13/cobra"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/registry"
	"k8s.io/kops/pkg/client/simple"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
	"k8s.io/kops/upup/pkg/kutil"
	"k8s.io/kops/util/pkg/tables"
	"k8s.io/kops/util/pkg/vfs"
)

type DeleteClusterCmd struct {
	Yes        bool
	Region     string
	External   bool
	Unregister bool
}

var deleteCluster DeleteClusterCmd

func init() {
	cmd := &cobra.Command{
		Use:   "cluster CLUSTERNAME [--yes]",
		Short: "Delete cluster",
		Long:  `Deletes a k8s cluster.`,
		Run: func(cmd *cobra.Command, args []string) {
			err := deleteCluster.Run(args)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	deleteCmd.AddCommand(cmd)
	for _, arg := range os.Args {
		arg = strings.ToLower(arg)
		if arg == "-y" || arg == "--yes" {
			deleteCluster.Yes = true
			break
		}
	}

	cmd.Flags().BoolVar(&deleteCluster.Unregister, "unregister", false, "Don't delete cloud resources, just unregister the cluster")
	cmd.Flags().BoolVar(&deleteCluster.External, "external", false, "Delete an external cluster")

	cmd.Flags().StringVar(&deleteCluster.Region, "region", "", "region")
}

type getter func(o interface{}) interface{}

func (c *DeleteClusterCmd) Run(args []string) error {
	var configBase vfs.Path
	var clientset simple.Clientset

	err := rootCommand.ProcessArgs(args)
	if err != nil {
		return err
	}

	clusterName := rootCommand.clusterName
	if clusterName == "" {
		return fmt.Errorf("--name is required (for safety)")
	}

	var cloud fi.Cloud
	var cluster *api.Cluster

	if c.External {
		region := c.Region
		if region == "" {
			return fmt.Errorf("--region is required (when --external)")
		}

		tags := map[string]string{"KubernetesCluster": clusterName}
		cloud, err = awsup.NewAWSCloud(region, tags)
		if err != nil {
			return fmt.Errorf("error initializing AWS client: %v", err)
		}
	} else {
		clientset, err = rootCommand.Clientset()
		if err != nil {
			return err
		}

		cluster, err = clientset.Clusters().Get(clusterName)
		if err != nil {
			return err
		}

		if cluster == nil {
			return fmt.Errorf("cluster %q not found", clusterName)
		}

		if clusterName != cluster.ObjectMeta.Name {
			return fmt.Errorf("sanity check failed: cluster name mismatch")
		}

		configBase, err = registry.ConfigBase(cluster)
		if err != nil {
			return err
		}
	}

	wouldDeleteCloudResources := false

	if !c.Unregister {
		if cloud == nil {
			cloud, err = cloudup.BuildCloud(cluster)
			if err != nil {
				return err
			}
		}

		d := &kutil.DeleteCluster{}
		d.ClusterName = clusterName
		d.Cloud = cloud

		resources, err := d.ListResources()
		if err != nil {
			return err
		}

		if len(resources) == 0 {
			fmt.Printf("No cloud resources to delete\n")
		} else {
			wouldDeleteCloudResources = true

			t := &tables.Table{}
			t.AddColumn("TYPE", func(r *kutil.ResourceTracker) string {
				return r.Type
			})
			t.AddColumn("ID", func(r *kutil.ResourceTracker) string {
				return r.ID
			})
			t.AddColumn("NAME", func(r *kutil.ResourceTracker) string {
				return r.Name
			})
			var l []*kutil.ResourceTracker
			for _, v := range resources {
				l = append(l, v)
			}

			err := t.Render(l, os.Stdout, "TYPE", "NAME", "ID")
			if err != nil {
				return err
			}

			if !c.Yes {
				return fmt.Errorf("Must specify --yes to delete")
			}

			fmt.Fprintf(os.Stdout, "\n")

			err = d.DeleteResources(resources)
			if err != nil {
				return err
			}
		}
	}

	if !c.External {
		if !c.Yes {
			if wouldDeleteCloudResources {
				fmt.Printf("\nMust specify --yes to delete cloud resources & unregister cluster\n")
			} else {
				fmt.Printf("\nMust specify --yes to unregister the cluster\n")
			}
			return nil
		}
		err := registry.DeleteAllClusterState(configBase)
		if err != nil {
			return fmt.Errorf("error removing cluster from state store: %v", err)
		}
	}

	b := kutil.NewKubeconfigBuilder()
	b.Context = cluster.ObjectMeta.Name
	b.DeleteKubeConfig()

	fmt.Printf("\nCluster deleted\n")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"

	"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/cluster"
	"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/config"
	"github.com/spf13/cobra"
)

var (
	cmdUp = &cobra.Command{
		Use:          "up",
		Short:        "Create a new Kubernetes cluster",
		Long:         ``,
		RunE:         runCmdUp,
		SilenceUsage: true,
	}

	upOpts = struct {
		awsDebug, export, update bool
	}{}
)

func init() {
	cmdRoot.AddCommand(cmdUp)
	cmdUp.Flags().BoolVar(&upOpts.export, "export", false, "Don't create cluster, instead export cloudformation stack file")
	//	cmdUp.Flags().BoolVar(&upOpts.update, "update", false, "update existing cluster with new cloudformation stack")
	cmdUp.Flags().BoolVar(&upOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
}

func runCmdUp(cmd *cobra.Command, args []string) error {
	conf, err := config.ClusterFromFile(configPath)
	if err != nil {
		return fmt.Errorf("Failed to read cluster config: %v", err)
	}

	data, err := conf.RenderStackTemplate(stackTemplateOptions)
	if err != nil {
		return fmt.Errorf("Failed to render stack template: %v", err)
	}

	if upOpts.export {
		templatePath := fmt.Sprintf("%s.stack-template.json", conf.ClusterName)
		fmt.Printf("Exporting %s\n", templatePath)
		if err := ioutil.WriteFile(templatePath, data, 0600); err != nil {
			return fmt.Errorf("Error writing %s : %v", templatePath, err)
		}
		if conf.KMSKeyARN == "" {
			fmt.Printf("BEWARE: %s contains your TLS secrets!\n", templatePath)
		}
		return nil
	}

	cluster := cluster.New(conf, upOpts.awsDebug)
	if upOpts.update {
		report, err := cluster.Update(string(data))
		if err != nil {
			return fmt.Errorf("Error updating cluster: %v", err)
		}
		if report != "" {
			fmt.Printf("Update stack: %s\n", report)
		}
	} else {
		fmt.Printf("Creating AWS resources. This make take several minutes.\n")
		if err := cluster.Create(string(data)); err != nil {
			return fmt.Errorf("Error creating cluster: %v", err)
		}
	}

	info, err := cluster.Info()
	if err != nil {
		return fmt.Errorf("Failed fetching cluster info: %v", err)
	}

	successMsg :=
		`Success! Your AWS resources have been created:
%s
The containers that power your cluster are now being dowloaded.

You should be able to access the Kubernetes API once the containers finish downloading.
`
	fmt.Printf(successMsg, info.String())

	return nil
}
<commit_msg>kube-aws: validate user data in `kube-aws up`<commit_after>package main

import (
	"fmt"
	"io/ioutil"

	"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/cluster"
	"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/config"
	"github.com/spf13/cobra"
)

var (
	cmdUp = &cobra.Command{
		Use:          "up",
		Short:        "Create a new Kubernetes cluster",
		Long:         ``,
		RunE:         runCmdUp,
		SilenceUsage: true,
	}

	upOpts = struct {
		awsDebug, export, update bool
	}{}
)

func init() {
	cmdRoot.AddCommand(cmdUp)
	cmdUp.Flags().BoolVar(&upOpts.export, "export", false, "Don't create cluster, instead export cloudformation stack file")
	//	cmdUp.Flags().BoolVar(&upOpts.update, "update", false, "update existing cluster with new cloudformation stack")
	cmdUp.Flags().BoolVar(&upOpts.awsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library")
}

func runCmdUp(cmd *cobra.Command, args []string) error {
	conf, err := config.ClusterFromFile(configPath)
	if err != nil {
		return fmt.Errorf("Failed to read cluster config: %v", err)
	}

	if err := conf.ValidateUserData(stackTemplateOptions); err != nil {
		return err
	}

	data, err := conf.RenderStackTemplate(stackTemplateOptions)
	if err != nil {
		return fmt.Errorf("Failed to render stack template: %v", err)
	}

	if upOpts.export {
		templatePath := fmt.Sprintf("%s.stack-template.json", conf.ClusterName)
		fmt.Printf("Exporting %s\n", templatePath)
		if err := ioutil.WriteFile(templatePath, data, 0600); err != nil {
			return fmt.Errorf("Error writing %s : %v", templatePath, err)
		}
		if conf.KMSKeyARN == "" {
			fmt.Printf("BEWARE: %s contains your TLS secrets!\n", templatePath)
		}
		return nil
	}

	cluster := cluster.New(conf, upOpts.awsDebug)
	if upOpts.update {
		report, err := cluster.Update(string(data))
		if err != nil {
			return fmt.Errorf("Error updating cluster: %v", err)
		}
		if report != "" {
			fmt.Printf("Update stack: %s\n", report)
		}
	} else {
		fmt.Printf("Creating AWS resources. This make take several minutes.\n")
		if err := cluster.Create(string(data)); err != nil {
			return fmt.Errorf("Error creating cluster: %v", err)
		}
	}

	info, err := cluster.Info()
	if err != nil {
		return fmt.Errorf("Failed fetching cluster info: %v", err)
	}

	successMsg :=
		`Success! Your AWS resources have been created:
%s
The containers that power your cluster are now being dowloaded.

You should be able to access the Kubernetes API once the containers finish downloading.
`
	fmt.Printf(successMsg, info.String())

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/datawire/teleproxy/pkg/k8s"
	"github.com/datawire/teleproxy/pkg/limiter"
	"github.com/datawire/teleproxy/pkg/tpu"
	"github.com/spf13/cobra"
)

type Syncer struct {
	Watcher     *k8s.Watcher
	SyncCommand string
	Kinds       []string
	Mux         *sync.Mutex // protects the whole data structure
	Changed     *sync.Cond
	Dirty       bool
	Limiter     limiter.Limiter
	ModTime     time.Time
	SyncCount   int
	WarmupDelay time.Duration
	router      *http.ServeMux
	port        string
	snapshotMux sync.Mutex // protects just the snapshot map
	snapshots   map[string]map[string][]byte
}

func (s *Syncer) maybeSync() {
	if s.Dirty {
		delay := s.Limiter.Limit(s.ModTime)
		if delay == 0 {
			s.Dirty = false
			s.sync()
		} else if delay > 0 {
			// if we are delaying an event we need an
			// artificial wakeup just in case there are no
			// more events to trigger syncing... if there
			// are events prior to this, then this should
			// end up being a noop because s.Dirty will be
			// false
			log.Printf("rate limiting, will sync after %s", delay.String())
			time.AfterFunc(delay, func() {
				s.Mux.Lock()
				defer s.Mux.Unlock()
				log.Printf("triggering delayed sync")
				s.ModTime = time.Now()
				s.Changed.Broadcast()
			})
		}
	}
}

func (s *Syncer) sync() {
	s.SyncCount += 1
	snapshot_id := s.write()
	s.invoke(snapshot_id)
}

func (s *Syncer) write() string {
	s.snapshotMux.Lock()
	defer s.snapshotMux.Unlock()
	snapshot_id := fmt.Sprintf("%d", s.SyncCount)
	s.snapshots[snapshot_id] = make(map[string][]byte)
	for _, kind := range s.Kinds {
		resources := s.Watcher.List(kind)
		bytes, err := k8s.MarshalResources(resources)
		if err != nil {
			panic(err)
		}
		s.snapshots[snapshot_id][kind] = bytes
		for _, rsrc := range resources {
			qname := path.Join(kind, rsrc.Namespace(), rsrc.Name())
			bytes, err := k8s.MarshalResource(rsrc)
			if err != nil {
				panic(err)
			}
			s.snapshots[snapshot_id][qname] = bytes
		}
	}
	s.cleanup()
	return snapshot_id
}

func (s *Syncer) cleanup() {
	for k := range s.snapshots {
		keep := false
		for c := s.SyncCount; c > s.SyncCount-10; c-- {
			id := fmt.Sprintf("%d", c)
			if id == k {
				keep = true
			}
		}
		if !keep {
			delete(s.snapshots, k)
			log.Printf("deleting snapshot %s", k)
		}
	}
}

func (s *Syncer) invoke(snapshot_id string) {
	k := tpu.NewKeeper("SYNC", fmt.Sprintf("%s http://localhost:%s/api/snapshot/%s", s.SyncCommand, s.port, snapshot_id))
	k.Limit = 1
	k.Start()
	k.Wait()
}

func (s *Syncer) Run() {
	go func() {
		time.Sleep(s.WarmupDelay)
		s.Mux.Lock()
		defer s.Mux.Unlock()
		for {
			s.Changed.Wait()
			s.maybeSync()
		}
	}()

	for _, k := range s.Kinds {
		// this alias is important so the func picks up the
		// value from the current iteration instead of the
		// value from the last iteration
		kind := k
		err := s.Watcher.WatchNamespace(NAMESPACE, kind, func(_ *k8s.Watcher) {
			s.Mux.Lock()
			defer s.Mux.Unlock()
			s.Dirty = true
			s.ModTime = time.Now()
			s.Changed.Broadcast()
		})
		if err != nil {
			panic(fmt.Sprintf("kubewatch: %v", err))
		}
	}
	s.Watcher.Start()
	s.serve()
}

func (s *Syncer) serve() {
	s.routes()

	ln, err := net.Listen("tcp", fmt.Sprintf(":%s", PORT))
	if err != nil {
		// Error starting or closing listener:
		panic(fmt.Sprintf("kubewatch: %v", err))
	}

	_, port, err := net.SplitHostPort(ln.Addr().String())
	if err != nil {
		panic(fmt.Sprintf("kubewatch: %v", err))
	}
	s.port = port

	server := http.Server{
		Handler: s.router,
	}

	if err := server.Serve(ln); err != http.ErrServerClosed {
		// Error starting or closing listener:
		panic(fmt.Sprintf("kubewatch: %v", err))
	}
}

func (s *Syncer) routes() {
	s.router.HandleFunc("/api/snapshot/", s.safe(s.handleSnapshot()))
}

func (s *Syncer) safe(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if r := recover(); r != nil {
				var msg string
				switch e := r.(type) {
				case error:
					msg = e.Error()
				default:
					msg = fmt.Sprintf("%v", r)
				}
				http.Error(w, fmt.Sprintf("Server Error: %s", msg), http.StatusInternalServerError)
			}
		}()
		h(w, r)
	}
}

func (s *Syncer) handleSnapshot() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		s.snapshotMux.Lock()
		defer s.snapshotMux.Unlock()
		parts := strings.Split(r.URL.Path, "/")
		parts = parts[3:]
		snapshot_id := parts[0]
		snapshot, ok := s.snapshots[snapshot_id]
		if !ok {
			http.NotFound(w, r)
			return
		}
		qname := strings.Join(parts[1:], "/")
		var body []byte
		if qname != "" {
			body, ok = snapshot[qname]
			if !ok {
				http.NotFound(w, r)
			}
		} else {
			var keys []string
			for k := range snapshot {
				keys = append(keys, k)
			}
			body = []byte(fmt.Sprintf("Available snapshot keys:\n - %s\n", strings.Join(keys, "\n - ")))
		}
		w.Write(body)
	}
}

var Version = "(unknown version)"

var KUBEWATCH = &cobra.Command{
	Use:  "kubewatch [options] <resources>",
	Args: cobra.MinimumNArgs(1),
	Run:  kubewatch,
}

func init() {
	KUBEWATCH.Version = Version
	KUBEWATCH.Flags().StringVarP(&PORT, "port", "p", "0", "port for kubewatch api")
	KUBEWATCH.Flags().StringVarP(&SYNC_COMMAND, "sync", "s", "curl", "sync command")
	KUBEWATCH.Flags().StringVarP(&NAMESPACE, "namespace", "n", "", "namespace to watch (defaults to all)")
	KUBEWATCH.Flags().DurationVarP(&MIN_INTERVAL, "min-interval", "m", 250*time.Millisecond, "min sync interval")
	KUBEWATCH.Flags().DurationVarP(&WARMUP_DELAY, "warmup-delay", "w", 0, "warmup delay")
}

var (
	PORT         string
	SYNC_COMMAND string
	NAMESPACE    string
	MIN_INTERVAL time.Duration
	WARMUP_DELAY time.Duration
)

func kubewatch(cmd *cobra.Command, args []string) {
	mux := &sync.Mutex{}
	cond := sync.NewCond(mux)
	s := Syncer{
		Mux:         mux,
		Changed:     cond,
		Watcher:     k8s.MustNewWatcher(nil),
		SyncCommand: SYNC_COMMAND,
		Kinds:       args,
		Limiter:     limiter.NewInterval(MIN_INTERVAL),
		WarmupDelay: WARMUP_DELAY,
		router:      http.NewServeMux(),
		snapshots:   make(map[string]map[string][]byte),
	}

	s.Run()
}

func main() {
	KUBEWATCH.Execute()
}
<commit_msg>rm -rf cmd/kubewatch<commit_after><|endoftext|>