<commit_before>package http

import (
	"bufio"
	"context"
	"io"
	"net"
	"net/http"
	"runtime"
	"strconv"
	"strings"
	"time"

	"v2ray.com/core/app"
	"v2ray.com/core/app/dispatcher"
	"v2ray.com/core/app/log"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/errors"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/transport/internet"
)

// Server is a HTTP proxy server.
type Server struct {
	config *ServerConfig
}

// NewServer creates a new HTTP inbound handler.
func NewServer(ctx context.Context, config *ServerConfig) (*Server, error) {
	space := app.SpaceFromContext(ctx)
	if space == nil {
		return nil, newError("no space in context.")
	}
	s := &Server{
		config: config,
	}
	return s, nil
}

func (*Server) Network() v2net.NetworkList {
	return v2net.NetworkList{
		Network: []v2net.Network{v2net.Network_TCP},
	}
}

func parseHost(rawHost string, defaultPort v2net.Port) (v2net.Destination, error) {
	port := defaultPort
	host, rawPort, err := net.SplitHostPort(rawHost)
	if err != nil {
		if addrError, ok := err.(*net.AddrError); ok && strings.Contains(addrError.Err, "missing port") {
			host = rawHost
		} else {
			return v2net.Destination{}, err
		}
	} else {
		intPort, err := strconv.Atoi(rawPort)
		if err != nil {
			return v2net.Destination{}, err
		}
		port = v2net.Port(intPort)
	}

	if ip := net.ParseIP(host); ip != nil {
		return v2net.TCPDestination(v2net.IPAddress(ip), port), nil
	}
	return v2net.TCPDestination(v2net.DomainAddress(host), port), nil
}

func isTimeout(err error) bool {
	nerr, ok := err.(net.Error)
	return ok && nerr.Timeout()
}

func (s *Server) Process(ctx context.Context, network v2net.Network, conn internet.Connection, dispatcher dispatcher.Interface) error {
	reader := bufio.NewReaderSize(conn, 8192)

Start:
	conn.SetReadDeadline(time.Now().Add(time.Second * 16))

	request, err := http.ReadRequest(reader)
	if err != nil {
		trace := newError("failed to read http request").Base(err)
		if errors.Cause(err) != io.EOF && !isTimeout(errors.Cause(err)) {
			trace.AtWarning()
		}
		return trace
	}
	log.Trace(newError("request to Method [", request.Method, "] Host [", request.Host, "] with URL [", request.URL, "]"))
	conn.SetReadDeadline(time.Time{})

	defaultPort := v2net.Port(80)
	if strings.ToLower(request.URL.Scheme) == "https" {
		defaultPort = v2net.Port(443)
	}
	host := request.Host
	if len(host) == 0 {
		host = request.URL.Host
	}
	dest, err := parseHost(host, defaultPort)
	if err != nil {
		return newError("malformed proxy host: ", host).AtWarning().Base(err)
	}
	log.Access(conn.RemoteAddr(), request.URL, log.AccessAccepted, "")

	if strings.ToUpper(request.Method) == "CONNECT" {
		return s.handleConnect(ctx, request, reader, conn, dest, dispatcher)
	}

	keepAlive := (strings.TrimSpace(strings.ToLower(request.Header.Get("Proxy-Connection"))) == "keep-alive")

	err = s.handlePlainHTTP(ctx, request, reader, conn, dest, dispatcher)
	if err == errWaitAnother {
		if keepAlive {
			goto Start
		}
		err = nil
	}

	return err
}

func (s *Server) handleConnect(ctx context.Context, request *http.Request, reader io.Reader, writer io.Writer, dest v2net.Destination, dispatcher dispatcher.Interface) error {
	_, err := writer.Write([]byte("HTTP/1.1 200 Connection established\n\n"))
	if err != nil {
		return newError("failed to write back OK response").Base(err)
	}

	timeout := time.Second * time.Duration(s.config.Timeout)
	if timeout == 0 {
		timeout = time.Minute * 2
	}
	ctx, timer := signal.CancelAfterInactivity(ctx, timeout)
	ray, err := dispatcher.Dispatch(ctx, dest)
	if err != nil {
		return err
	}

	requestDone := signal.ExecuteAsync(func() error {
		defer ray.InboundInput().Close()

		v2reader := buf.NewReader(reader)
		if err := buf.Copy(timer, v2reader, ray.InboundInput()); err != nil {
			return err
		}
		return nil
	})

	responseDone := signal.ExecuteAsync(func() error {
		v2writer := buf.NewWriter(writer)
		if err := buf.Copy(timer, ray.InboundOutput(), v2writer); err != nil {
			return err
		}
		return nil
	})

	if err := signal.ErrorOrFinish2(ctx, requestDone, responseDone); err != nil {
		ray.InboundInput().CloseError()
		ray.InboundOutput().CloseError()
		return newError("connection ends").Base(err)
	}

	runtime.KeepAlive(timer)

	return nil
}

// @VisibleForTesting
func StripHopByHopHeaders(header http.Header) {
	// Strip hop-by-hop header basaed on RFC:
	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1
	// https://www.mnot.net/blog/2011/07/11/what_proxies_must_do

	header.Del("Proxy-Connection")
	header.Del("Proxy-Authenticate")
	header.Del("Proxy-Authorization")
	header.Del("TE")
	header.Del("Trailers")
	header.Del("Transfer-Encoding")
	header.Del("Upgrade")

	connections := header.Get("Connection")
	header.Del("Connection")
	if len(connections) == 0 {
		return
	}
	for _, h := range strings.Split(connections, ",") {
		header.Del(strings.TrimSpace(h))
	}
}

var errWaitAnother = newError("keep alive")

func (s *Server) handlePlainHTTP(ctx context.Context, request *http.Request, reader io.Reader, writer io.Writer, dest v2net.Destination, dispatcher dispatcher.Interface) error {
	if len(request.URL.Host) <= 0 {
		response := &http.Response{
			Status:        "Bad Request",
			StatusCode:    400,
			Proto:         "HTTP/1.1",
			ProtoMajor:    1,
			ProtoMinor:    1,
			Header:        http.Header(make(map[string][]string)),
			Body:          nil,
			ContentLength: 0,
			Close:         true,
		}
		response.Header.Set("Proxy-Connection", "close")
		response.Header.Set("Connection", "close")
		return response.Write(writer)
	}

	request.Host = request.URL.Host
	StripHopByHopHeaders(request.Header)

	ray, err := dispatcher.Dispatch(ctx, dest)
	if err != nil {
		return err
	}
	input := ray.InboundInput()
	output := ray.InboundOutput()
	defer input.Close()

	requestDone := signal.ExecuteAsync(func() error {
		request.Header.Set("Connection", "close")

		requestWriter := buf.ToBytesWriter(ray.InboundInput())
		if err := request.Write(requestWriter); err != nil {
			return err
		}
		return nil
	})

	responseDone := signal.ExecuteAsync(func() error {
		responseReader := bufio.NewReaderSize(buf.ToBytesReader(ray.InboundOutput()), 8192)
		response, err := http.ReadResponse(responseReader, request)
		if err == nil {
			StripHopByHopHeaders(response.Header)
			response.Header.Set("Proxy-Connection", "keep-alive")
			response.Header.Set("Connection", "keep-alive")
			response.Header.Set("Keep-Alive", "timeout=4")
			response.Close = false
		} else {
			log.Trace(newError("failed to read response from ", request.Host).Base(err).AtWarning())
			response = &http.Response{
				Status:        "Service Unavailable",
				StatusCode:    503,
				Proto:         "HTTP/1.1",
				ProtoMajor:    1,
				ProtoMinor:    1,
				Header:        http.Header(make(map[string][]string)),
				Body:          nil,
				ContentLength: 0,
				Close:         true,
			}
			response.Header.Set("Connection", "close")
			response.Header.Set("Proxy-Connection", "close")
		}
		if err := response.Write(writer); err != nil {
			return newError("failed to write response").Base(err)
		}
		return nil
	})

	if err := signal.ErrorOrFinish2(ctx, requestDone, responseDone); err != nil {
		input.CloseError()
		output.CloseError()
		return newError("connection ends").Base(err)
	}

	return errWaitAnother
}

func init() {
	common.Must(common.RegisterConfig((*ServerConfig)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return NewServer(ctx, config.(*ServerConfig))
	}))
}
<commit_msg>only keep connection alive when content length is known<commit_after>package http

import (
	"bufio"
	"context"
	"io"
	"net"
	"net/http"
	"runtime"
	"strconv"
	"strings"
	"time"

	"v2ray.com/core/app"
	"v2ray.com/core/app/dispatcher"
	"v2ray.com/core/app/log"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/errors"
	v2net "v2ray.com/core/common/net"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/transport/internet"
)

// Server is a HTTP proxy server.
type Server struct {
	config *ServerConfig
}

// NewServer creates a new HTTP inbound handler.
func NewServer(ctx context.Context, config *ServerConfig) (*Server, error) {
	space := app.SpaceFromContext(ctx)
	if space == nil {
		return nil, newError("no space in context.")
	}
	s := &Server{
		config: config,
	}
	return s, nil
}

func (*Server) Network() v2net.NetworkList {
	return v2net.NetworkList{
		Network: []v2net.Network{v2net.Network_TCP},
	}
}

func parseHost(rawHost string, defaultPort v2net.Port) (v2net.Destination, error) {
	port := defaultPort
	host, rawPort, err := net.SplitHostPort(rawHost)
	if err != nil {
		if addrError, ok := err.(*net.AddrError); ok && strings.Contains(addrError.Err, "missing port") {
			host = rawHost
		} else {
			return v2net.Destination{}, err
		}
	} else {
		intPort, err := strconv.Atoi(rawPort)
		if err != nil {
			return v2net.Destination{}, err
		}
		port = v2net.Port(intPort)
	}

	if ip := net.ParseIP(host); ip != nil {
		return v2net.TCPDestination(v2net.IPAddress(ip), port), nil
	}
	return v2net.TCPDestination(v2net.DomainAddress(host), port), nil
}

func isTimeout(err error) bool {
	nerr, ok := err.(net.Error)
	return ok && nerr.Timeout()
}

func (s *Server) Process(ctx context.Context, network v2net.Network, conn internet.Connection, dispatcher dispatcher.Interface) error {
	reader := bufio.NewReaderSize(conn, 8192)

Start:
	conn.SetReadDeadline(time.Now().Add(time.Second * 16))

	request, err := http.ReadRequest(reader)
	if err != nil {
		trace := newError("failed to read http request").Base(err)
		if errors.Cause(err) != io.EOF && !isTimeout(errors.Cause(err)) {
			trace.AtWarning()
		}
		return trace
	}
	log.Trace(newError("request to Method [", request.Method, "] Host [", request.Host, "] with URL [", request.URL, "]"))
	conn.SetReadDeadline(time.Time{})

	defaultPort := v2net.Port(80)
	if strings.ToLower(request.URL.Scheme) == "https" {
		defaultPort = v2net.Port(443)
	}
	host := request.Host
	if len(host) == 0 {
		host = request.URL.Host
	}
	dest, err := parseHost(host, defaultPort)
	if err != nil {
		return newError("malformed proxy host: ", host).AtWarning().Base(err)
	}
	log.Access(conn.RemoteAddr(), request.URL, log.AccessAccepted, "")

	if strings.ToUpper(request.Method) == "CONNECT" {
		return s.handleConnect(ctx, request, reader, conn, dest, dispatcher)
	}

	keepAlive := (strings.TrimSpace(strings.ToLower(request.Header.Get("Proxy-Connection"))) == "keep-alive")

	err = s.handlePlainHTTP(ctx, request, reader, conn, dest, dispatcher)
	if err == errWaitAnother {
		if keepAlive {
			goto Start
		}
		err = nil
	}

	return err
}

func (s *Server) handleConnect(ctx context.Context, request *http.Request, reader io.Reader, writer io.Writer, dest v2net.Destination, dispatcher dispatcher.Interface) error {
	_, err := writer.Write([]byte("HTTP/1.1 200 Connection established\n\n"))
	if err != nil {
		return newError("failed to write back OK response").Base(err)
	}

	timeout := time.Second * time.Duration(s.config.Timeout)
	if timeout == 0 {
		timeout = time.Minute * 2
	}
	ctx, timer := signal.CancelAfterInactivity(ctx, timeout)
	ray, err := dispatcher.Dispatch(ctx, dest)
	if err != nil {
		return err
	}

	requestDone := signal.ExecuteAsync(func() error {
		defer ray.InboundInput().Close()

		v2reader := buf.NewReader(reader)
		if err := buf.Copy(timer, v2reader, ray.InboundInput()); err != nil {
			return err
		}
		return nil
	})

	responseDone := signal.ExecuteAsync(func() error {
		v2writer := buf.NewWriter(writer)
		if err := buf.Copy(timer, ray.InboundOutput(), v2writer); err != nil {
			return err
		}
		return nil
	})

	if err := signal.ErrorOrFinish2(ctx, requestDone, responseDone); err != nil {
		ray.InboundInput().CloseError()
		ray.InboundOutput().CloseError()
		return newError("connection ends").Base(err)
	}

	runtime.KeepAlive(timer)

	return nil
}

// @VisibleForTesting
func StripHopByHopHeaders(header http.Header) {
	// Strip hop-by-hop header basaed on RFC:
	// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1
	// https://www.mnot.net/blog/2011/07/11/what_proxies_must_do

	header.Del("Proxy-Connection")
	header.Del("Proxy-Authenticate")
	header.Del("Proxy-Authorization")
	header.Del("TE")
	header.Del("Trailers")
	header.Del("Transfer-Encoding")
	header.Del("Upgrade")

	connections := header.Get("Connection")
	header.Del("Connection")
	if len(connections) == 0 {
		return
	}
	for _, h := range strings.Split(connections, ",") {
		header.Del(strings.TrimSpace(h))
	}
}

var errWaitAnother = newError("keep alive")

func (s *Server) handlePlainHTTP(ctx context.Context, request *http.Request, reader io.Reader, writer io.Writer, dest v2net.Destination, dispatcher dispatcher.Interface) error {
	if len(request.URL.Host) <= 0 {
		response := &http.Response{
			Status:        "Bad Request",
			StatusCode:    400,
			Proto:         "HTTP/1.1",
			ProtoMajor:    1,
			ProtoMinor:    1,
			Header:        http.Header(make(map[string][]string)),
			Body:          nil,
			ContentLength: 0,
			Close:         true,
		}
		response.Header.Set("Proxy-Connection", "close")
		response.Header.Set("Connection", "close")
		return response.Write(writer)
	}

	request.Host = request.URL.Host
	StripHopByHopHeaders(request.Header)

	ray, err := dispatcher.Dispatch(ctx, dest)
	if err != nil {
		return err
	}
	input := ray.InboundInput()
	output := ray.InboundOutput()
	defer input.Close()

	requestDone := signal.ExecuteAsync(func() error {
		request.Header.Set("Connection", "close")

		requestWriter := buf.ToBytesWriter(ray.InboundInput())
		if err := request.Write(requestWriter); err != nil {
			return err
		}
		return nil
	})

	responseDone := signal.ExecuteAsync(func() error {
		responseReader := bufio.NewReaderSize(buf.ToBytesReader(ray.InboundOutput()), 8192)
		response, err := http.ReadResponse(responseReader, request)
		if err == nil {
			StripHopByHopHeaders(response.Header)
			if response.ContentLength >= 0 {
				response.Header.Set("Proxy-Connection", "keep-alive")
				response.Header.Set("Connection", "keep-alive")
				response.Header.Set("Keep-Alive", "timeout=4")
				response.Close = false
			}
		} else {
			log.Trace(newError("failed to read response from ", request.Host).Base(err).AtWarning())
			response = &http.Response{
				Status:        "Service Unavailable",
				StatusCode:    503,
				Proto:         "HTTP/1.1",
				ProtoMajor:    1,
				ProtoMinor:    1,
				Header:        http.Header(make(map[string][]string)),
				Body:          nil,
				ContentLength: 0,
				Close:         true,
			}
			response.Header.Set("Connection", "close")
			response.Header.Set("Proxy-Connection", "close")
		}
		if err := response.Write(writer); err != nil {
			return newError("failed to write response").Base(err)
		}
		return nil
	})

	if err := signal.ErrorOrFinish2(ctx, requestDone, responseDone); err != nil {
		input.CloseError()
		output.CloseError()
		return newError("connection ends").Base(err)
	}

	return errWaitAnother
}

func init() {
	common.Must(common.RegisterConfig((*ServerConfig)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return NewServer(ctx, config.(*ServerConfig))
	}))
}
<|endoftext|>


<commit_before>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2019] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

// Last.Backend Open-source API
//
// Open-source system for automating deployment, scaling, and management of containerized applications.
//
// Terms Of Service:
//
// https://lastbackend.com/legal/terms/
//
//     Schemes: https
//     Host: api.lastbackend.com
//     BasePath: /
//     Version: 0.9.4
//     License: MIT http://opensource.org/licenses/MIT
//     Contact: Last.Backend Teams <team@lastbackend.com> https://lastbackend.com
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Security:
//     - bearerToken:
//
//     SecurityDefinitions:
//       bearerToken:
//         description: Bearer Token authentication
//         type: apiKey
//         name: authorization
//         in: header
//
//     Extensions:
//     x-meta-value: value
//     x-meta-array:
//       - value1
//       - value2
//     x-meta-array-obj:
//       - name: obj
//         value: field
//
// swagger:meta
package main

import (
	"fmt"
	"github.com/lastbackend/lastbackend/pkg/node"
	flag "github.com/spf13/pflag"
	"github.com/spf13/viper"
	"strings"
)

const default_env_prefix = "LB"
const default_config_type = "yaml"
const default_config_name = "config"

var (
	flags = []struct {
		// flag name
		Name string
		// flag short name
		Short string
		// flag value
		Value interface{}
		// flag description
		Desc string
		// viper name for binding from flag
		Bind string
	}{
		{Name: "access-token", Short: "", Value: "", Desc: "Access token to API server", Bind: "token"},
		{Name: "workdir", Short: "", Value: "", Desc: "Node workdir for runtime", Bind: "workdir"},
		{Name: "manifest-path", Short: "", Value: "", Desc: "Node manifest(s) path", Bind: "manifest.dir"},
		{Name: "bind-interface", Short: "", Value: "eth0", Desc: "Exporter bind network interface", Bind: "network.interface"},
		{Name: "network-proxy", Short: "", Value: "ipvs", Desc: "Network proxy driver (ipvs by default)", Bind: "network.cpi.type"},
		{Name: "network-proxy-iface-internal", Short: "", Value: "docker0", Desc: "Network proxy internal interface binding", Bind: "network.cpi.interface.internal"},
		{Name: "network-proxy-iface-external", Short: "", Value: "eth0", Desc: "Network proxy external interface binding", Bind: "network.cpi.interface.external"},
		{Name: "network-driver", Short: "", Value: "vxlan", Desc: "Network driver (vxlan by default)", Bind: "network.cni.type"},
		{Name: "network-driver-iface-external", Short: "", Value: "eth0", Desc: "Container overlay network external interface for host communication", Bind: "network.cni.interface.external"},
		{Name: "network-driver-iface-internal", Short: "", Value: "docker0", Desc: "Container overlay network internal bridge interface for container intercommunications", Bind: "network.cni.interface.internal"},
		{Name: "container-runtime", Short: "", Value: "docker", Desc: "Node container runtime", Bind: "container.cri.type"},
		{Name: "container-runtime-docker-version", Short: "", Value: "", Desc: "Set docker version for docker container runtime", Bind: "container.cri.docker.version"},
		{Name: "container-storage-root", Short: "", Value: "/var/run/lastbackend", Desc: "Node container storage root", Bind: "container.csi.dir.root"},
		{Name: "container-image-runtime", Short: "", Value: "docker", Desc: "Node container images runtime", Bind: "container.iri.type"},
		{Name: "bind-address", Short: "", Value: "0.0.0.0", Desc: "Node bind address", Bind: "server.host"},
		{Name: "bind-port", Short: "", Value: 2965, Desc: "Node listening port binding", Bind: "server.port"},
		{Name: "tls-cert-file", Short: "", Value: "", Desc: "Node cert file path", Bind: "server.tls.cert"},
		{Name: "tls-private-key-file", Short: "", Value: "", Desc: "Node private key file path", Bind: "server.tls.key"},
		{Name: "tls-ca-file", Short: "", Value: "", Desc: "Node certificate authority file path", Bind: "server.tls.ca"},
		{Name: "api-uri", Short: "", Value: "", Desc: "REST API endpoint", Bind: "api.uri"},
		{Name: "api-tls-cert-file", Short: "", Value: "", Desc: "REST API TLS certificate file path", Bind: "api.tls.cert"},
		{Name: "api-tls-private-key-file", Short: "", Value: "", Desc: "REST API TLS private key file path", Bind: "api.tls.key"},
		{Name: "api-tls-ca-file", Short: "", Value: "", Desc: "REST API TSL certificate authority file path", Bind: "api.tls.ca"},
		{Name: "verbose", Short: "v", Value: 0, Desc: "Set log level from 0 to 7", Bind: "verbose"},
		{Name: "config", Short: "c", Value: "", Desc: "Path for the configuration file", Bind: "config"},
	}
)

func main() {

	for _, item := range flags {
		switch item.Value.(type) {
		case string:
			flag.StringP(item.Name, item.Short, item.Value.(string), item.Desc)
		case int:
			flag.IntP(item.Name, item.Short, item.Value.(int), item.Desc)
		case []string:
			flag.StringSliceP(item.Name, item.Short, item.Value.([]string), item.Desc)
		default:
			panic(fmt.Sprintf("bad %s argument value", item.Name))
		}
	}

	flag.Parse()

	v := viper.New()

	v.AutomaticEnv()
	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	v.SetEnvPrefix(default_env_prefix)

	for _, item := range flags {
		if err := v.BindPFlag(item.Bind, flag.Lookup(item.Name)); err != nil {
			panic(err)
		}

		name := strings.Replace(strings.ToUpper(item.Name), "-", "_", -1)
		name = strings.Join([]string{default_env_prefix, name}, "_")

		if err := v.BindEnv(item.Bind, name); err != nil {
			panic(err)
		}

		if item.Value != nil {
			v.SetDefault(item.Bind, item.Value)
		}
	}

	v.SetConfigType(default_config_type)
	v.SetConfigFile(v.GetString(default_config_name))

	if len(v.GetString("config")) != 0 {
		if err := v.ReadInConfig(); err != nil {
			panic(fmt.Sprintf("Read config err: %v", err))
		}
	}

	node.Daemon(v)
}
<commit_msg>update node configs<commit_after>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2019] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

// Last.Backend Open-source API
//
// Open-source system for automating deployment, scaling, and management of containerized applications.
//
// Terms Of Service:
//
// https://lastbackend.com/legal/terms/
//
//     Schemes: https
//     Host: api.lastbackend.com
//     BasePath: /
//     Version: 0.9.4
//     License: MIT http://opensource.org/licenses/MIT
//     Contact: Last.Backend Teams <team@lastbackend.com> https://lastbackend.com
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Security:
//     - bearerToken:
//
//     SecurityDefinitions:
//       bearerToken:
//         description: Bearer Token authentication
//         type: apiKey
//         name: authorization
//         in: header
//
//     Extensions:
//     x-meta-value: value
//     x-meta-array:
//       - value1
//       - value2
//     x-meta-array-obj:
//       - name: obj
//         value: field
//
// swagger:meta
package main

import (
	"fmt"
	"github.com/lastbackend/lastbackend/pkg/node"
	flag "github.com/spf13/pflag"
	"github.com/spf13/viper"
	"strings"
)

const default_env_prefix = "LB"
const default_config_type = "yaml"
const default_config_name = "config"

var (
	flags = []struct {
		// flag name
		Name string
		// flag short name
		Short string
		// flag value
		Value interface{}
		// flag description
		Desc string
		// viper name for binding from flag
		Bind string
	}{
		{Name: "access-token", Short: "", Value: "", Desc: "Access token to API server", Bind: "token"},
		{Name: "workdir", Short: "", Value: "", Desc: "Node workdir for runtime", Bind: "workdir"},
		{Name: "manifest-path", Short: "", Value: "", Desc: "Node manifest(s) path", Bind: "manifest.dir"},
		{Name: "bind-interface", Short: "", Value: "eth0", Desc: "Exporter bind network interface", Bind: "network.interface"},
		{Name: "network-proxy", Short: "", Value: "ipvs", Desc: "Network proxy driver (ipvs by default)", Bind: "network.cpi.type"},
		{Name: "network-proxy-iface-internal", Short: "", Value: "docker0", Desc: "Network proxy internal interface binding", Bind: "network.cpi.interface.internal"},
		{Name: "network-proxy-iface-external", Short: "", Value: "eth0", Desc: "Network proxy external interface binding", Bind: "network.cpi.interface.external"},
		{Name: "network-driver", Short: "", Value: "vxlan", Desc: "Network driver (vxlan by default)", Bind: "network.cni.type"},
		{Name: "network-driver-iface-external", Short: "", Value: "eth0", Desc: "Container overlay network external interface for host communication", Bind: "network.cni.interface.external"},
		{Name: "network-driver-iface-internal", Short: "", Value: "docker0", Desc: "Container overlay network internal bridge interface for container intercommunications", Bind: "network.cni.interface.internal"},
		{Name: "container-runtime", Short: "", Value: "docker", Desc: "Node container runtime", Bind: "container.cri.type"},
		{Name: "container-runtime-docker-version", Short: "", Value: "1.38", Desc: "Set docker version for docker container runtime", Bind: "container.cri.docker.version"},
		{Name: "container-storage-root", Short: "", Value: "/var/run/lastbackend", Desc: "Node container storage root", Bind: "container.csi.dir.root"},
		{Name: "container-image-runtime", Short: "", Value: "docker", Desc: "Node container images runtime", Bind: "container.iri.type"},
		{Name: "container-image-runtime-docker-version", Short: "", Value: "1.38", Desc: "Set docker version for docker container image runtime", Bind: "container.iri.docker.version"},
		{Name: "bind-address", Short: "", Value: "0.0.0.0", Desc: "Node bind address", Bind: "server.host"},
		{Name: "bind-port", Short: "", Value: 2965, Desc: "Node listening port binding", Bind: "server.port"},
		{Name: "tls-cert-file", Short: "", Value: "", Desc: "Node cert file path", Bind: "server.tls.cert"},
		{Name: "tls-private-key-file", Short: "", Value: "", Desc: "Node private key file path", Bind: "server.tls.key"},
		{Name: "tls-ca-file", Short: "", Value: "", Desc: "Node certificate authority file path", Bind: "server.tls.ca"},
		{Name: "api-uri", Short: "", Value: "", Desc: "REST API endpoint", Bind: "api.uri"},
		{Name: "api-tls-cert-file", Short: "", Value: "", Desc: "REST API TLS certificate file path", Bind: "api.tls.cert"},
		{Name: "api-tls-private-key-file", Short: "", Value: "", Desc: "REST API TLS private key file path", Bind: "api.tls.key"},
		{Name: "api-tls-ca-file", Short: "", Value: "", Desc: "REST API TSL certificate authority file path", Bind: "api.tls.ca"},
		{Name: "verbose", Short: "v", Value: 0, Desc: "Set log level from 0 to 7", Bind: "verbose"},
		{Name: "config", Short: "c", Value: "", Desc: "Path for the configuration file", Bind: "config"},
	}
)

func main() {

	for _, item := range flags {
		switch item.Value.(type) {
		case string:
			flag.StringP(item.Name, item.Short, item.Value.(string), item.Desc)
		case int:
			flag.IntP(item.Name, item.Short, item.Value.(int), item.Desc)
		case []string:
			flag.StringSliceP(item.Name, item.Short, item.Value.([]string), item.Desc)
		default:
			panic(fmt.Sprintf("bad %s argument value", item.Name))
		}
	}

	flag.Parse()

	v := viper.New()

	v.AutomaticEnv()
	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	v.SetEnvPrefix(default_env_prefix)

	for _, item := range flags {
		if err := v.BindPFlag(item.Bind, flag.Lookup(item.Name)); err != nil {
			panic(err)
		}

		name := strings.Replace(strings.ToUpper(item.Name), "-", "_", -1)
		name = strings.Join([]string{default_env_prefix, name}, "_")

		if err := v.BindEnv(item.Bind, name); err != nil {
			panic(err)
		}

		if item.Value != nil {
			v.SetDefault(item.Bind, item.Value)
		}
	}

	v.SetConfigType(default_config_type)
	v.SetConfigFile(v.GetString(default_config_name))

	if len(v.GetString("config")) != 0 {
		if err := v.ReadInConfig(); err != nil {
			panic(fmt.Sprintf("Read config err: %v", err))
		}
	}

	node.Daemon(v)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"). You
 * may not use this file except in compliance with the License. A copy of
 * the License is located at
 *
 * 	http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 * ANY KIND, either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */
package ecr

import (
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/containerd/containerd/reference"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

const (
	refPrefix           = "ecr.aws/"
	repositoryDelimiter = "/"
)

var (
	invalidARN = errors.New("ref: invalid ARN")
	splitRe    = regexp.MustCompile(`[:@]`)
)

type ECRSpec struct {
	Repository string
	Object     string
	arn        arn.ARN
}

// ParseRef parses an ECR reference into its constituent parts
func ParseRef(ref string) (ECRSpec, error) {
	if !strings.HasPrefix(ref, refPrefix) {
		return ECRSpec{}, invalidARN
	}
	stripped := ref[len(refPrefix):]
	return parseARN(stripped)
}

func (spec ECRSpec) Partition() string {
	return spec.arn.Partition
}

func (spec ECRSpec) Region() string {
	return spec.arn.Region
}

func (spec ECRSpec) Registry() string {
	return spec.arn.AccountID
}

// parseARN parses an ECR ARN into its constituent parts
// An example ARN is arn:aws:ecr:us-west-2:123456789012:repository/foo/bar
func parseARN(a string) (ECRSpec, error) {
	parsed, err := arn.Parse(a)
	if err != nil {
		return ECRSpec{}, err
	}

	// remove label & digest
	var object string
	if delimiterIndex := splitRe.FindStringIndex(parsed.Resource); delimiterIndex != nil {
		// This allows us to retain the @ to signify digests or shortened digests in
		// the object.
		object = parsed.Resource[delimiterIndex[0]:]
		// trim leading :
		if object[:1] == ":" {
			object = object[1:]
		}
		parsed.Resource = parsed.Resource[:delimiterIndex[0]]
	}

	// strip "repository/" prefix
	repositorySections := strings.SplitN(parsed.Resource, repositoryDelimiter, 2)
	if len(repositorySections) != 2 {
		return ECRSpec{}, invalidARN
	}
	return ECRSpec{
		arn:        parsed,
		Repository: repositorySections[1],
		Object:     object,
	}, nil

}

// Canonical returns the canonical representation
func (spec ECRSpec) Canonical() string {
	object := ""
	if len(spec.Object) != 0 {
		if spec.Object[0] != '@' {
			object = ":"
		}
		object = object + spec.Object
	}
	return refPrefix + spec.ARN() + object
}

// ARN returns the canonical representation of the ECR ARN
func (spec ECRSpec) ARN() string {
	return spec.arn.String()
}

// Spec returns a reference.Spec
func (spec ECRSpec) Spec() reference.Spec {
	return reference.Spec{Locator: "", Object: spec.Object}
}

// ImageID returns an ecr.ImageIdentifier suitable for using in calls to ECR
func (spec ECRSpec) ImageID() *ecr.ImageIdentifier {
	imageID := ecr.ImageIdentifier{}
	tag, digest := spec.TagDigest()
	if tag != "" {
		imageID.ImageTag = aws.String(tag)
	}
	if digest != "" {
		imageID.ImageDigest = aws.String(string(digest))
	}
	return &imageID
}

func (spec ECRSpec) TagDigest() (string, digest.Digest) {
	return reference.SplitObject(spec.Object)
}
<commit_msg>ref: create valid imageID for refs with digests<commit_after>/*
 * Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"). You
 * may not use this file except in compliance with the License. A copy of
 * the License is located at
 *
 * 	http://aws.amazon.com/apache2.0/
 *
 * or in the "license" file accompanying this file. This file is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 * ANY KIND, either express or implied. See the License for the specific
 * language governing permissions and limitations under the License.
 */
package ecr

import (
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/arn"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/containerd/containerd/reference"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

const (
	refPrefix           = "ecr.aws/"
	repositoryDelimiter = "/"
)

var (
	invalidARN = errors.New("ref: invalid ARN")
	splitRe    = regexp.MustCompile(`[:@]`)
)

type ECRSpec struct {
	Repository string
	Object     string
	arn        arn.ARN
}

// ParseRef parses an ECR reference into its constituent parts
func ParseRef(ref string) (ECRSpec, error) {
	if !strings.HasPrefix(ref, refPrefix) {
		return ECRSpec{}, invalidARN
	}
	stripped := ref[len(refPrefix):]
	return parseARN(stripped)
}

func (spec ECRSpec) Partition() string {
	return spec.arn.Partition
}

func (spec ECRSpec) Region() string {
	return spec.arn.Region
}

func (spec ECRSpec) Registry() string {
	return spec.arn.AccountID
}

// parseARN parses an ECR ARN into its constituent parts
// An example ARN is arn:aws:ecr:us-west-2:123456789012:repository/foo/bar
func parseARN(a string) (ECRSpec, error) {
	parsed, err := arn.Parse(a)
	if err != nil {
		return ECRSpec{}, err
	}

	// remove label & digest
	var object string
	if delimiterIndex := splitRe.FindStringIndex(parsed.Resource); delimiterIndex != nil {
		// This allows us to retain the @ to signify digests or shortened digests in
		// the object.
		object = parsed.Resource[delimiterIndex[0]:]
		// trim leading :
		if object[:1] == ":" {
			object = object[1:]
		}
		parsed.Resource = parsed.Resource[:delimiterIndex[0]]
	}

	// strip "repository/" prefix
	repositorySections := strings.SplitN(parsed.Resource, repositoryDelimiter, 2)
	if len(repositorySections) != 2 {
		return ECRSpec{}, invalidARN
	}
	return ECRSpec{
		arn:        parsed,
		Repository: repositorySections[1],
		Object:     object,
	}, nil

}

// Canonical returns the canonical representation
func (spec ECRSpec) Canonical() string {
	object := ""
	if len(spec.Object) != 0 {
		if spec.Object[0] != '@' {
			object = ":"
		}
		object = object + spec.Object
	}
	return refPrefix + spec.ARN() + object
}

// ARN returns the canonical representation of the ECR ARN
func (spec ECRSpec) ARN() string {
	return spec.arn.String()
}

// Spec returns a reference.Spec
func (spec ECRSpec) Spec() reference.Spec {
	return reference.Spec{Locator: "", Object: spec.Object}
}

// ImageID returns an ecr.ImageIdentifier suitable for using in calls to ECR
func (spec ECRSpec) ImageID() *ecr.ImageIdentifier {
	imageID := ecr.ImageIdentifier{}
	tag, digest := spec.TagDigest()
	if tag != "" {
		imageID.ImageTag = aws.String(tag)
	}
	if digest != "" {
		imageID.ImageDigest = aws.String(string(digest))
	}
	return &imageID
}

func (spec ECRSpec) TagDigest() (string, digest.Digest) {
	tag, digest := reference.SplitObject(spec.Object)
	tag = strings.TrimSuffix(tag, "@")
	return tag, digest
}
<|endoftext|>


<commit_before>package ps

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"strings"
)

// Table represents CSV tables.
type Table struct {
	// Columns maps its column name to its column index.
	Columns     map[string]int
	ColumnSlice []string
	Rows        [][]string

	minTS int64
	maxTS int64
}

var (
	ColumnsPS = make(map[string]int)
)

func init() {
	for i, v := range append([]string{"unix_ts"}, ProcessTableColumns...) {
		ColumnsPS[v] = i
	}
}

// ReadCSV reads a csv file for ps csv file.
// It assumes the results from one single program.
func ReadCSV(columns map[string]int, fpath string) (Table, error) {
	f, err := openToRead(fpath)
	if err != nil {
		return Table{}, err
	}
	defer f.Close()

	columnSlice := make([]string, len(columns))
	for k, v := range columns {
		columnSlice[v] = k
	}

	rd := csv.NewReader(f)

	// in case that rows have different number of fields
	rd.FieldsPerRecord = -1

	rows, err := rd.ReadAll()
	if err != nil {
		return Table{}, err
	}
	if len(rows) > 0 && len(rows[0]) > 0 {
		if rows[0][0] == "unix_ts" {
			rows = rows[1:]
		}
	}
	min, err := strconv.ParseInt(rows[0][0], 10, 64)
	if err != nil {
		return Table{}, err
	}
	max, err := strconv.ParseInt(rows[len(rows)-1][0], 10, 64)
	if err != nil {
		return Table{}, err
	}
	return Table{
		Columns:     columns,
		ColumnSlice: columnSlice,
		Rows:        rows,
		minTS:       min,
		maxTS:       max,
	}, nil
}

// ReadCSVs reads multiple csv files, including only unix timestamps,
// CPU usage, and VmRSS in MB. It assumes the results from one single program.
func ReadCSVs(columns map[string]int, fpaths ...string) (Table, error) {
	tbs := []Table{}
	var minTS, maxTS int64
	for i, fpath := range fpaths {
		tb, err := ReadCSV(columns, fpath)
		if err != nil {
			return Table{}, err
		}
		tbs = append(tbs, tb)

		if i == 0 {
			minTS = tb.minTS
			maxTS = tb.maxTS
		}
		if minTS > tb.minTS {
			minTS = tb.minTS
		}
		if maxTS < tb.maxTS {
			maxTS = tb.maxTS
		}
	}

	ntb := Table{}
	ntb.Columns = make(map[string]int)
	ntb.Columns["unix_tx"] = 0
	for i := range fpaths {
		ntb.Columns[fmt.Sprintf("cpu_%d", i+1)] = 2*i + 1
		ntb.Columns[fmt.Sprintf("memory_mb_%d", i+1)] = 2*i + 2
	}
	columnSlice := make([]string, len(ntb.Columns))
	for k, v := range ntb.Columns {
		columnSlice[v] = k
	}
	ntb.ColumnSlice = columnSlice

	cIdx, mIdx := tbs[0].Columns["CpuUsageFloat64"], tbs[0].Columns["VmRSSBytes"]

	nrows := [][]string{}
	for i, tb := range tbs {
		if minTS < tb.minTS { // need to fill in top rows
			// push-front from minTS to tb.minTS-1
			rows := make([][]string, tb.minTS-minTS)
			for i := range rows {
				emptyRow := append([]string{fmt.Sprintf("%d", minTS+int64(i))}, strings.Split(strings.Repeat("0.00_", len(ProcessTableColumns)), "_")...)
				rows[i] = emptyRow
			}
			tb.Rows = append(rows, tb.Rows...)
		}
		if maxTS > tb.maxTS { // need to fill in bottom rows
			// push-back from tb.maxTS+1 to maxTS
			rows := make([][]string, maxTS-tb.maxTS)
			for i := range rows {
				emptyRow := append([]string{fmt.Sprintf("%d", tb.maxTS+int64(i))}, strings.Split(strings.Repeat("0.00_", len(ProcessTableColumns)), "_")...)
				rows[i] = emptyRow
			}
			tb.Rows = append(tb.Rows, rows...)
		}

		if i == 0 {
			for _, row := range tb.Rows {
				if row[0] == tb.ColumnSlice[0] {
					continue
				}
				mf, _ := strconv.ParseFloat(row[mIdx], 64)
				nrows = append(nrows, []string{row[0], row[cIdx], fmt.Sprintf("%.2f", mf/(1000*1000))}) // memory usage in mega-bytes
			}
		} else {
			for rowNum := range nrows {
				trow := tb.Rows[rowNum]
				if trow[0] == tb.ColumnSlice[0] {
					continue
				}
				mf, _ := strconv.ParseFloat(trow[mIdx], 64)
				nrows[rowNum] = append(nrows[rowNum], []string{trow[cIdx], fmt.Sprintf("%.2f", mf/(1000*1000))}...)
			}
		}
	}
	ntb.Rows = nrows

	return ntb, nil
}

// ToRows returns rows from the table.
func (t Table) ToRows() [][]string {
	return append([][]string{t.ColumnSlice}, t.Rows...)
}

// ToCSV saves the table to csv file.
func (t Table) ToCSV(fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()

	wr := csv.NewWriter(f)

	if err := wr.Write(t.ColumnSlice); err != nil {
		return err
	}

	if err := wr.WriteAll(t.Rows); err != nil {
		return err
	}

	wr.Flush()
	return wr.Error()
}
<commit_msg>ps: use columnSlice to match header<commit_after>package ps

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"strings"
)

// Table represents CSV tables.
type Table struct {
	// Columns maps its column name to its column index.
	Columns     map[string]int
	ColumnSlice []string
	Rows        [][]string

	minTS int64
	maxTS int64
}

var (
	ColumnsPS = make(map[string]int)
)

func init() {
	for i, v := range append([]string{"unix_ts"}, ProcessTableColumns...) {
		ColumnsPS[v] = i
	}
}

// ReadCSV reads a csv file for ps csv file.
// It assumes the results from one single program.
func ReadCSV(columns map[string]int, fpath string) (Table, error) {
	f, err := openToRead(fpath)
	if err != nil {
		return Table{}, err
	}
	defer f.Close()

	columnSlice := make([]string, len(columns))
	for k, v := range columns {
		columnSlice[v] = k
	}

	rd := csv.NewReader(f)

	// in case that rows have different number of fields
	rd.FieldsPerRecord = -1

	rows, err := rd.ReadAll()
	if err != nil {
		return Table{}, err
	}
	if len(rows) > 0 && len(rows[0]) > 0 {
		if rows[0][0] == columnSlice[0] {
			rows = rows[1:]
		}
	}
	min, err := strconv.ParseInt(rows[0][0], 10, 64)
	if err != nil {
		return Table{}, err
	}
	max, err := strconv.ParseInt(rows[len(rows)-1][0], 10, 64)
	if err != nil {
		return Table{}, err
	}
	return Table{
		Columns:     columns,
		ColumnSlice: columnSlice,
		Rows:        rows,
		minTS:       min,
		maxTS:       max,
	}, nil
}

// ReadCSVs reads multiple csv files, including only unix timestamps,
// CPU usage, and VmRSS in MB. It assumes the results from one single program.
func ReadCSVs(columns map[string]int, fpaths ...string) (Table, error) {
	tbs := []Table{}
	var minTS, maxTS int64
	for i, fpath := range fpaths {
		tb, err := ReadCSV(columns, fpath)
		if err != nil {
			return Table{}, err
		}
		tbs = append(tbs, tb)

		if i == 0 {
			minTS = tb.minTS
			maxTS = tb.maxTS
		}
		if minTS > tb.minTS {
			minTS = tb.minTS
		}
		if maxTS < tb.maxTS {
			maxTS = tb.maxTS
		}
	}

	ntb := Table{}
	ntb.Columns = make(map[string]int)
	ntb.Columns["unix_tx"] = 0
	for i := range fpaths {
		ntb.Columns[fmt.Sprintf("cpu_%d", i+1)] = 2*i + 1
		ntb.Columns[fmt.Sprintf("memory_mb_%d", i+1)] = 2*i + 2
	}
	columnSlice := make([]string, len(ntb.Columns))
	for k, v := range ntb.Columns {
		columnSlice[v] = k
	}
	ntb.ColumnSlice = columnSlice

	cIdx, mIdx := tbs[0].Columns["CpuUsageFloat64"], tbs[0].Columns["VmRSSBytes"]

	nrows := [][]string{}
	for i, tb := range tbs {
		if minTS < tb.minTS { // need to fill in top rows
			// push-front from minTS to tb.minTS-1
			rows := make([][]string, tb.minTS-minTS)
			for i := range rows {
				emptyRow := append([]string{fmt.Sprintf("%d", minTS+int64(i))}, strings.Split(strings.Repeat("0.00_", len(ProcessTableColumns)), "_")...)
				rows[i] = emptyRow
			}
			tb.Rows = append(rows, tb.Rows...)
		}
		if maxTS > tb.maxTS { // need to fill in bottom rows
			// push-back from tb.maxTS+1 to maxTS
			rows := make([][]string, maxTS-tb.maxTS)
			for i := range rows {
				emptyRow := append([]string{fmt.Sprintf("%d", tb.maxTS+int64(i))}, strings.Split(strings.Repeat("0.00_", len(ProcessTableColumns)), "_")...)
				rows[i] = emptyRow
			}
			tb.Rows = append(tb.Rows, rows...)
		}

		if i == 0 {
			for _, row := range tb.Rows {
				if row[0] == tb.ColumnSlice[0] {
					continue
				}
				mf, _ := strconv.ParseFloat(row[mIdx], 64)
				nrows = append(nrows, []string{row[0], row[cIdx], fmt.Sprintf("%.2f", mf/(1000*1000))}) // memory usage in mega-bytes
			}
		} else {
			for rowNum := range nrows {
				trow := tb.Rows[rowNum]
				if trow[0] == tb.ColumnSlice[0] {
					continue
				}
				mf, _ := strconv.ParseFloat(trow[mIdx], 64)
				nrows[rowNum] = append(nrows[rowNum], []string{trow[cIdx], fmt.Sprintf("%.2f", mf/(1000*1000))}...)
			}
		}
	}
	ntb.Rows = nrows

	return ntb, nil
}

// ToRows returns rows from the table.
func (t Table) ToRows() [][]string {
	return append([][]string{t.ColumnSlice}, t.Rows...)
}

// ToCSV saves the table to csv file.
func (t Table) ToCSV(fpath string) error {
	f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
	if err != nil {
		f, err = os.Create(fpath)
		if err != nil {
			return err
		}
	}
	defer f.Close()

	wr := csv.NewWriter(f)

	if err := wr.Write(t.ColumnSlice); err != nil {
		return err
	}

	if err := wr.WriteAll(t.Rows); err != nil {
		return err
	}

	wr.Flush()
	return wr.Error()
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018-2019 The original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"os"

	"github.com/projectriff/riff/pkg/core"
	"github.com/projectriff/riff/pkg/riff/commands"
)

var (
	manifests = map[string]*core.Manifest{
		// validated, compatible versions of Knative
		"stable": {
			ManifestVersion: "0.1",
			Istio: []string{
				// diff https://storage.googleapis.com/projectriff/istio/istio-riff-knative-serving-v0-5-0.diff
				"https://storage.googleapis.com/projectriff/istio/istio-riff-knative-serving-v0-5-0.yaml",
			},
			Knative: []string{
				"https://storage.googleapis.com/knative-releases/build/previous/v0.4.0/build.yaml",
				"https://storage.googleapis.com/knative-releases/serving/previous/v0.5.0/serving.yaml",
				"https://raw.githubusercontent.com/knative/serving/v0.5.0/third_party/config/build/clusterrole.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/previous/v0.4.0/eventing.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/previous/v0.4.0/in-memory-channel.yaml",
				// TODO update to a release version before releasing riff
				"https://storage.googleapis.com/projectriff/riff-buildtemplate/riff-cnb-clusterbuildtemplate-0.2.0-snapshot-ci-c957e7e83d23.yaml",
			},
		},
		// most recent release of Knative. This manifest is not tested
		"latest": {
			ManifestVersion: "0.1",
			Istio: []string{
				"https://storage.googleapis.com/knative-releases/serving/latest/istio.yaml",
			},
			Knative: []string{
				"https://storage.googleapis.com/knative-releases/build/latest/build.yaml",
				"https://storage.googleapis.com/knative-releases/serving/latest/serving.yaml",
				"https://raw.githubusercontent.com/knative/serving/master/third_party/config/build/clusterrole.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/latest/eventing.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/latest/in-memory-channel.yaml",
				"https://storage.googleapis.com/projectriff/riff-buildtemplate/riff-cnb-clusterbuildtemplate.yaml",
			},
		},
		// most recent build of Knative from master. This manifest is not tested
		"nightly": {
			ManifestVersion: "0.1",
			Istio: []string{
				"https://storage.googleapis.com/knative-nightly/serving/latest/istio.yaml",
			},
			Knative: []string{
				"https://storage.googleapis.com/knative-nightly/build/latest/build.yaml",
				"https://storage.googleapis.com/knative-nightly/serving/latest/serving.yaml",
				"https://raw.githubusercontent.com/knative/serving/master/third_party/config/build/clusterrole.yaml",
				"https://storage.googleapis.com/knative-nightly/eventing/latest/eventing.yaml",
				"https://storage.googleapis.com/knative-nightly/eventing/latest/in-memory-channel.yaml",
				"https://storage.googleapis.com/projectriff/riff-buildtemplate/riff-cnb-clusterbuildtemplate.yaml",
			},
		},
	}
)

func main() {

	root := commands.CreateAndWireRootCommand(manifests)

	sub, err := root.ExecuteC()
	if err != nil {
		if !sub.SilenceUsage { // May have been switched to true once we're past PreRunE()
			sub.Help()
		}
		fmt.Fprintf(os.Stderr, "\nError: %v\n", err)
		os.Exit(1)
	}
}
<commit_msg>Update Knative Build to v0.5.0<commit_after>/*
 * Copyright 2018-2019 The original author or authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"os"

	"github.com/projectriff/riff/pkg/core"
	"github.com/projectriff/riff/pkg/riff/commands"
)

var (
	manifests = map[string]*core.Manifest{
		// validated, compatible versions of Knative
		"stable": {
			ManifestVersion: "0.1",
			Istio: []string{
				// diff https://storage.googleapis.com/projectriff/istio/istio-riff-knative-serving-v0-5-0.diff
				"https://storage.googleapis.com/projectriff/istio/istio-riff-knative-serving-v0-5-0.yaml",
			},
			Knative: []string{
				"https://storage.googleapis.com/knative-releases/build/previous/v0.5.0/build.yaml",
				"https://storage.googleapis.com/knative-releases/serving/previous/v0.5.0/serving.yaml",
				"https://raw.githubusercontent.com/knative/serving/v0.5.0/third_party/config/build/clusterrole.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/previous/v0.4.0/eventing.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/previous/v0.4.0/in-memory-channel.yaml",
				// TODO update to a release version before releasing riff
				"https://storage.googleapis.com/projectriff/riff-buildtemplate/riff-cnb-clusterbuildtemplate-0.2.0-snapshot-ci-c957e7e83d23.yaml",
			},
		},
		// most recent release of Knative. This manifest is not tested
		"latest": {
			ManifestVersion: "0.1",
			Istio: []string{
				"https://storage.googleapis.com/knative-releases/serving/latest/istio.yaml",
			},
			Knative: []string{
				"https://storage.googleapis.com/knative-releases/build/latest/build.yaml",
				"https://storage.googleapis.com/knative-releases/serving/latest/serving.yaml",
				"https://raw.githubusercontent.com/knative/serving/master/third_party/config/build/clusterrole.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/latest/eventing.yaml",
				"https://storage.googleapis.com/knative-releases/eventing/latest/in-memory-channel.yaml",
				"https://storage.googleapis.com/projectriff/riff-buildtemplate/riff-cnb-clusterbuildtemplate.yaml",
			},
		},
		// most recent build of Knative from master. This manifest is not tested
		"nightly": {
			ManifestVersion: "0.1",
			Istio: []string{
				"https://storage.googleapis.com/knative-nightly/serving/latest/istio.yaml",
			},
			Knative: []string{
				"https://storage.googleapis.com/knative-nightly/build/latest/build.yaml",
				"https://storage.googleapis.com/knative-nightly/serving/latest/serving.yaml",
				"https://raw.githubusercontent.com/knative/serving/master/third_party/config/build/clusterrole.yaml",
				"https://storage.googleapis.com/knative-nightly/eventing/latest/eventing.yaml",
				"https://storage.googleapis.com/knative-nightly/eventing/latest/in-memory-channel.yaml",
				"https://storage.googleapis.com/projectriff/riff-buildtemplate/riff-cnb-clusterbuildtemplate.yaml",
			},
		},
	}
)

func main() {

	root := commands.CreateAndWireRootCommand(manifests)

	sub, err := root.ExecuteC()
	if err != nil {
		if !sub.SilenceUsage { // May have been switched to true once we're past PreRunE()
			sub.Help()
		}
		fmt.Fprintf(os.Stderr, "\nError: %v\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2015 Douglas Thrift
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/douglaswth/rsrdp/win32"
)

func rdpLaunchNative(instance *Instance, private bool, index int, arguments []string, prompt bool, username string) error {
	ipAddress, err := instance.IpAddress(private, index)
	if err != nil {
		return err
	}

	count := 3 + len(arguments)
	if !prompt {
		count += 2
	}
	args := make([]string, 0, count)

	if !prompt {
		credential := win32.CREDENTIAL{
			Type:           win32.CRED_TYPE_GENERIC,
			TargetName:     ipAddress,
			Comment:        "Temporary RSRDP credential",
			CredentialBlob: instance.AdminPassword,
			Persist:        win32.CRED_PERSIST_SESSION,
			UserName:       username,
		}
		err = win32.CredWrite(&credential, 0)
		if err != nil {
			return fmt.Errorf("Error storing credential: %s", err)
		}
		args = append(args, "--credential", ipAddress)
	}

	file, err := rdpCreateFile(instance, private, index, username, false)
	if err != nil {
		return err
	}
	args = append(args, "--temporary", filepath.Dir(file), "--", "mstsc", file)
	args = append(args, arguments...)

	executable, err := rdpFindRunExecutable()
	if err != nil {
		return err
	}

	command := exec.Command(executable, args...)
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr

	err = command.Start()
	if err != nil {
		return err
	}
	err = command.Process.Release()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Oops! Missed that for Windows because my Linux VM was too small to install mingw-w64.<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2015 Douglas Thrift
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/douglaswth/rsrdp/win32"
)

func rdpLaunchNative(instance *Instance, private bool, index int, arguments []string, prompt bool, username string) error {
	ipAddress, err := instance.IpAddress(private, index)
	if err != nil {
		return err
	}

	count := 3 + len(arguments)
	if !prompt {
		count += 2
	}
	args := make([]string, 0, count)

	if !prompt {
		credential := win32.CREDENTIAL{
			Type:           win32.CRED_TYPE_GENERIC,
			TargetName:     ipAddress,
			Comment:        "Temporary RSRDP credential",
			CredentialBlob: instance.AdminPassword,
			Persist:        win32.CRED_PERSIST_SESSION,
			UserName:       username,
		}
		err = win32.CredWrite(&credential, 0)
		if err != nil {
			return fmt.Errorf("Error storing credential: %s", err)
		}
		args = append(args, "--credential", ipAddress)
	}

	file, err := rdpCreateFile(instance, private, index, username, false)
	if err != nil {
		return err
	}
	args = append(args, "--temporary", filepath.Dir(file), "--", "mstsc", file)
	args = append(args, arguments...)

	executable, err := rdpFindRunExecutable()
	if err != nil {
		return err
	}

	command := exec.Command(executable, args...)
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr

	err = command.Start()
	if err != nil {
		return err
	}
	err = command.Process.Release()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

// Benchmark the scanning of a file-system tree.

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/fsbench"
	"github.com/Symantec/Dominator/sub/fsrateio"
	"github.com/Symantec/Dominator/sub/scanner"
	"os"
	"syscall"
	"time"
)

var (
	interval = flag.Uint("interval", 0, "Seconds to sleep after each scan")
	numScans = flag.Int("numScans", 1,
		"The number of scans to run (infinite: < 0)")
	rootDir = flag.String("rootDir", "/",
		"Name of root of directory tree to scan")
)

func main() {
	flag.Parse()
	var err error
	bytesPerSecond, blocksPerSecond, err := fsbench.GetReadSpeed(*rootDir)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	ctx := fsrateio.NewContext(bytesPerSecond, blocksPerSecond)
	fmt.Println(ctx)
	syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10)
	var prev_fs *scanner.FileSystem
	sleepDuration, _ := time.ParseDuration(fmt.Sprintf("%ds", *interval))
	for iter := 1; *numScans < 0 || iter <= *numScans; iter++ {
		timeStart := time.Now()
		fs, err := scanner.ScanFileSystem(*rootDir, "", ctx)
		timeStop := time.Now()
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
		fmt.Print(fs)
		var tread uint64 = 0
		for _, inode := range fs.InodeTable {
			tread += inode.Size
		}
		fmt.Printf("Total scanned: %s,\t", fsrateio.FormatBytes(tread))
		bytesPerSecond := uint64(float64(tread) /
			timeStop.Sub(timeStart).Seconds())
		fmt.Printf("%s/s\n", fsrateio.FormatBytes(bytesPerSecond))
		if prev_fs != nil {
			if !scanner.Compare(prev_fs, fs, os.Stdout) {
				fmt.Println("Scan results different from last run")
			}
		}
		prev_fs = fs
		time.Sleep(sleepDuration)
	}
}
<commit_msg>Add simple memory status output to scan utility.<commit_after>package main

// Benchmark the scanning of a file-system tree.

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/fsbench"
	"github.com/Symantec/Dominator/sub/fsrateio"
	"github.com/Symantec/Dominator/sub/scanner"
	"io"
	"os"
	"runtime"
	"syscall"
	"time"
)

var (
	interval = flag.Uint("interval", 0, "Seconds to sleep after each scan")
	numScans = flag.Int("numScans", 1,
		"The number of scans to run (infinite: < 0)")
	rootDir = flag.String("rootDir", "/",
		"Name of root of directory tree to scan")
)

func writeNamedStat(writer io.Writer, name string, value uint64) {
	fmt.Fprintf(writer, "  %s=%s\n", name, fsrateio.FormatBytes(value))
}

func writeMemoryStats(writer io.Writer) {
	var memStats runtime.MemStats
	runtime.ReadMemStats(&memStats)
	fmt.Fprintln(writer, "MemStats:")
	writeNamedStat(writer, "Alloc", memStats.Alloc)
	writeNamedStat(writer, "TotalAlloc", memStats.TotalAlloc)
	writeNamedStat(writer, "Sys", memStats.Sys)
}

func main() {
	flag.Parse()
	var err error
	bytesPerSecond, blocksPerSecond, err := fsbench.GetReadSpeed(*rootDir)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	ctx := fsrateio.NewContext(bytesPerSecond, blocksPerSecond)
	fmt.Println(ctx)
	syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10)
	var prev_fs *scanner.FileSystem
	sleepDuration, _ := time.ParseDuration(fmt.Sprintf("%ds", *interval))
	for iter := 1; *numScans < 0 || iter <= *numScans; iter++ {
		timeStart := time.Now()
		fs, err := scanner.ScanFileSystem(*rootDir, "", ctx)
		timeStop := time.Now()
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
		fmt.Print(fs)
		var tread uint64 = 0
		for _, inode := range fs.InodeTable {
			tread += inode.Size
		}
		fmt.Printf("Total scanned: %s,\t", fsrateio.FormatBytes(tread))
		bytesPerSecond := uint64(float64(tread) /
			timeStop.Sub(timeStart).Seconds())
		fmt.Printf("%s/s\n", fsrateio.FormatBytes(bytesPerSecond))
		if prev_fs != nil {
			if !scanner.Compare(prev_fs, fs, os.Stdout) {
				fmt.Println("Scan results different from last run")
			}
		}
		writeMemoryStats(os.Stdout)
		prev_fs = fs
		time.Sleep(sleepDuration)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2020 Karim Radhouani <medkarimrdi@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sort"
	"sync"
	"time"

	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/formatters"
	"github.com/manifoldco/promptui"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

const defaultRetryTimer = 10 * time.Second

var subscriptionModes = [][2]string{
	{"once", "a single request/response channel. The target creates the relevant update messages, transmits them, and subsequently closes the RPC"},
	{"stream", "long-lived subscriptions which continue to transmit updates relating to the set of paths that are covered within the subscription indefinitely"},
	{"poll", "on-demand retrieval of data items via long-lived RPCs"},
}

var streamSubscriptionModes = [][2]string{
	{"target-defined", "the target MUST determine the best type of subscription to be created on a per-leaf basis"},
	{"sample", "the value of the data item(s) MUST be sent once per sample interval to the client"},
	{"on-change", "data updates are only sent when the value of the data item changes"},
}

// subscribeCmd represents the subscribe command
func newSubscribeCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "subscribe",
		Aliases: []string{"sub"},
		Short:   "subscribe to gnmi updates on targets",
		Annotations: map[string]string{
			"--path":        "XPATH",
			"--prefix":      "PREFIX",
			"--model":       "MODEL",
			"--mode":        "SUBSC_MODE",
			"--stream-mode": "STREAM_MODE",
			"--name":        "SUBSCRIPTION",
			"--output":      "OUTPUT",
		},
		PreRun: func(cmd *cobra.Command, args []string) {
			cli.config.SetLocalFlagsFromFile(cmd)
		},
		RunE: cli.subscribeRunE,
		PostRun: func(cmd *cobra.Command, args []string) {
			cmd.ResetFlags()
			initSubscribeFlags(cmd)
		},
		SilenceUsage: true,
	}
	initSubscribeFlags(cmd)
	return cmd
}

// used to init or reset subscribeCmd flags for gnmic-prompt mode
func initSubscribeFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribePrefix, "prefix", "", "", "subscribe request prefix")
	cmd.Flags().StringArrayVarP(&cli.config.LocalFlags.SubscribePath, "path", "", []string{}, "subscribe request paths")
	//cmd.MarkFlagRequired("path")
	cmd.Flags().Uint32VarP(&cli.config.LocalFlags.SubscribeQos, "qos", "q", 0, "qos marking")
	cmd.Flags().BoolVarP(&cli.config.LocalFlags.SubscribeUpdatesOnly, "updates-only", "", false, "only updates to current state should be sent")
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribeMode, "mode", "", "stream", "one of: once, stream, poll")
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribeStreamMode, "stream-mode", "", "target-defined", "one of: on-change, sample, target-defined")
	cmd.Flags().DurationVarP(&cli.config.LocalFlags.SubscribeSampleInteral, "sample-interval", "i", 0,
		"sample interval as a decimal number and a suffix unit, such as \"10s\" or \"1m30s\"")
	cmd.Flags().BoolVarP(&cli.config.LocalFlags.SubscribeSuppressRedundant, "suppress-redundant", "", false, "suppress redundant update if the subscribed value didn't not change")
	cmd.Flags().DurationVarP(&cli.config.LocalFlags.SubscribeHeartbearInterval, "heartbeat-interval", "", 0, "heartbeat interval in case suppress-redundant is enabled")
	cmd.Flags().StringSliceVarP(&cli.config.LocalFlags.SubscribeModel, "model", "", []string{}, "subscribe request used model(s)")
	cmd.Flags().BoolVar(&cli.config.LocalFlags.SubscribeQuiet, "quiet", false, "suppress stdout printing")
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribeTarget, "target", "", "", "subscribe request target")
	cmd.Flags().StringSliceVarP(&cli.config.LocalFlags.SubscribeName, "name", "n", []string{}, "reference subscriptions by name, must be defined in gnmic config file")
	cmd.Flags().StringSliceVarP(&cli.config.LocalFlags.SubscribeOutput, "output", "", []string{}, "reference to output groups by name, must be defined in gnmic config file")
	//
	cmd.LocalFlags().VisitAll(func(flag *pflag.Flag) {
		cli.config.FileConfig.BindPFlag(fmt.Sprintf("%s-%s", cmd.Name(), flag.Name), flag)
	})
}

func (c *CLI) subscribeRunE(cmd *cobra.Command, args []string) error {
	gctx, gcancel = context.WithCancel(context.Background())
	setupCloseHandler(gcancel)
	targetsConfig, err := c.config.GetTargets()
	if err != nil {
		return fmt.Errorf("failed getting targets config: %v", err)
	}

	subscriptionsConfig, err := c.config.GetSubscriptions(cmd)
	if err != nil {
		return fmt.Errorf("failed getting subscriptions config: %v", err)
	}
	outs, err := c.config.GetOutputs()
	if err != nil {
		return err
	}
	epconfig, err := c.config.GetEventProcessors()
	if err != nil {
		return err
	}

	if c.collector == nil {
		cfg := &collector.Config{
			PrometheusAddress:   c.config.Globals.PrometheusAddress,
			Debug:               c.config.Globals.Debug,
			Format:              c.config.Globals.Format,
			TargetReceiveBuffer: c.config.Globals.TargetBufferSize,
			RetryTimer:          c.config.Globals.Retry,
		}

		c.collector = collector.NewCollector(cfg, targetsConfig,
			collector.WithDialOptions(createCollectorDialOpts()),
			collector.WithSubscriptions(subscriptionsConfig),
			collector.WithOutputs(outs),
			collector.WithLogger(c.logger),
			collector.WithEventProcessors(epconfig),
		)
	} else {
		// prompt mode
		for name, outCfg := range outs {
			err = c.collector.AddOutput(name, outCfg)
			if err != nil {
				c.logger.Printf("%v", err)
			}
		}
		for _, sc := range subscriptionsConfig {
			err = c.collector.AddSubscriptionConfig(sc)
			if err != nil {
				c.logger.Printf("%v", err)
			}
		}
		for _, tc := range targetsConfig {
			c.collector.AddTarget(tc)
			if err != nil {
				c.logger.Printf("%v", err)
			}
		}
	}

	c.collector.InitOutputs(gctx)

	go c.collector.Start(gctx)

	wg := new(sync.WaitGroup)
	wg.Add(len(c.collector.Targets))
	for name := range c.collector.Targets {
		go func(tn string) {
			defer wg.Done()
			tRetryTimer := c.collector.Targets[tn].Config.RetryTimer
			for {
				err = c.collector.Subscribe(gctx, tn)
				if err != nil {
					if errors.Is(err, context.DeadlineExceeded) {
						c.logger.Printf("failed to initialize target '%s' timeout (%s) reached", tn, targetsConfig[tn].Timeout)
					} else {
						c.logger.Printf("failed to initialize target '%s': %v", tn, err)
					}
					c.logger.Printf("retrying target %s in %s", tn, tRetryTimer)
					time.Sleep(tRetryTimer)
					continue
				}
				return
			}
		}(name)
	}
	wg.Wait()
	polledTargetsSubscriptions := c.collector.PolledSubscriptionsTargets()
	if len(polledTargetsSubscriptions) > 0 {
		pollTargets := make([]string, 0, len(polledTargetsSubscriptions))
		for t := range polledTargetsSubscriptions {
			pollTargets = append(pollTargets, t)
		}
		sort.Slice(pollTargets, func(i, j int) bool {
			return pollTargets[i] < pollTargets[j]
		})
		s := promptui.Select{
			Label:        "select target to poll",
			Items:        pollTargets,
			HideSelected: true,
		}
		waitChan := make(chan struct{}, 1)
		waitChan <- struct{}{}
		mo := &formatters.MarshalOptions{
			Multiline: true,
			Indent:    "  ",
			Format:    cli.config.Globals.Format,
		}
		go func() {
			for {
				select {
				case <-waitChan:
					_, name, err := s.Run()
					if err != nil {
						fmt.Printf("failed selecting target to poll: %v\n", err)
						continue
					}
					ss := promptui.Select{
						Label:        "select subscription to poll",
						Items:        polledTargetsSubscriptions[name],
						HideSelected: true,
					}
					_, subName, err := ss.Run()
					if err != nil {
						fmt.Printf("failed selecting subscription to poll: %v\n", err)
						continue
					}
					response, err := c.collector.TargetPoll(name, subName)
					if err != nil && err != io.EOF {
						fmt.Printf("target '%s', subscription '%s': poll response error:%v\n", name, subName, err)
						continue
					}
					if response == nil {
						fmt.Printf("received empty response from target '%s'\n", name)
						continue
					}
					switch rsp := response.Response.(type) {
					case *gnmi.SubscribeResponse_SyncResponse:
						fmt.Printf("received sync response '%t' from '%s'\n", rsp.SyncResponse, name)
						waitChan <- struct{}{}
						continue
					}
					b, err := mo.Marshal(response, nil)
					if err != nil {
						fmt.Printf("target '%s', subscription '%s': poll response formatting error:%v\n", name, subName, err)
						fmt.Println(string(b))
						waitChan <- struct{}{}
						continue
					}
					fmt.Println(string(b))
					waitChan <- struct{}{}
				case <-gctx.Done():
					return
				}
			}
		}()
	}

	if cli.promptMode {
		return nil
	}
	for range gctx.Done() {
		return gctx.Err()
	}
	return nil
}
<commit_msg>use CLI for subscribe<commit_after>// Copyright © 2020 Karim Radhouani <medkarimrdi@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"errors"
	"fmt"
	"io"
	"sort"
	"time"

	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/formatters"
	"github.com/manifoldco/promptui"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

const defaultRetryTimer = 10 * time.Second

var subscriptionModes = [][2]string{
	{"once", "a single request/response channel. The target creates the relevant update messages, transmits them, and subsequently closes the RPC"},
	{"stream", "long-lived subscriptions which continue to transmit updates relating to the set of paths that are covered within the subscription indefinitely"},
	{"poll", "on-demand retrieval of data items via long-lived RPCs"},
}

var streamSubscriptionModes = [][2]string{
	{"target-defined", "the target MUST determine the best type of subscription to be created on a per-leaf basis"},
	{"sample", "the value of the data item(s) MUST be sent once per sample interval to the client"},
	{"on-change", "data updates are only sent when the value of the data item changes"},
}

// subscribeCmd represents the subscribe command
func newSubscribeCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:     "subscribe",
		Aliases: []string{"sub"},
		Short:   "subscribe to gnmi updates on targets",
		Annotations: map[string]string{
			"--path":        "XPATH",
			"--prefix":      "PREFIX",
			"--model":       "MODEL",
			"--mode":        "SUBSC_MODE",
			"--stream-mode": "STREAM_MODE",
			"--name":        "SUBSCRIPTION",
			"--output":      "OUTPUT",
		},
		PreRun: func(cmd *cobra.Command, args []string) {
			cli.config.SetLocalFlagsFromFile(cmd)
		},
		RunE: cli.subscribeRunE,
		PostRun: func(cmd *cobra.Command, args []string) {
			cmd.ResetFlags()
			initSubscribeFlags(cmd)
		},
		SilenceUsage: true,
	}
	initSubscribeFlags(cmd)
	return cmd
}

// used to init or reset subscribeCmd flags for gnmic-prompt mode
func initSubscribeFlags(cmd *cobra.Command) {
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribePrefix, "prefix", "", "", "subscribe request prefix")
	cmd.Flags().StringArrayVarP(&cli.config.LocalFlags.SubscribePath, "path", "", []string{}, "subscribe request paths")
	//cmd.MarkFlagRequired("path")
	cmd.Flags().Uint32VarP(&cli.config.LocalFlags.SubscribeQos, "qos", "q", 0, "qos marking")
	cmd.Flags().BoolVarP(&cli.config.LocalFlags.SubscribeUpdatesOnly, "updates-only", "", false, "only updates to current state should be sent")
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribeMode, "mode", "", "stream", "one of: once, stream, poll")
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribeStreamMode, "stream-mode", "", "target-defined", "one of: on-change, sample, target-defined")
	cmd.Flags().DurationVarP(&cli.config.LocalFlags.SubscribeSampleInteral, "sample-interval", "i", 0,
		"sample interval as a decimal number and a suffix unit, such as \"10s\" or \"1m30s\"")
	cmd.Flags().BoolVarP(&cli.config.LocalFlags.SubscribeSuppressRedundant, "suppress-redundant", "", false, "suppress redundant update if the subscribed value didn't not change")
	cmd.Flags().DurationVarP(&cli.config.LocalFlags.SubscribeHeartbearInterval, "heartbeat-interval", "", 0, "heartbeat interval in case suppress-redundant is enabled")
	cmd.Flags().StringSliceVarP(&cli.config.LocalFlags.SubscribeModel, "model", "", []string{}, "subscribe request used model(s)")
	cmd.Flags().BoolVar(&cli.config.LocalFlags.SubscribeQuiet, "quiet", false, "suppress stdout printing")
	cmd.Flags().StringVarP(&cli.config.LocalFlags.SubscribeTarget, "target", "", "", "subscribe request target")
	cmd.Flags().StringSliceVarP(&cli.config.LocalFlags.SubscribeName, "name", "n", []string{}, "reference subscriptions by name, must be defined in gnmic config file")
	cmd.Flags().StringSliceVarP(&cli.config.LocalFlags.SubscribeOutput, "output", "", []string{}, "reference to output groups by name, must be defined in gnmic config file")
	//
	cmd.LocalFlags().VisitAll(func(flag *pflag.Flag) {
		cli.config.FileConfig.BindPFlag(fmt.Sprintf("%s-%s", cmd.Name(), flag.Name), flag)
	})
}

func (c *CLI) subscribeRunE(cmd *cobra.Command, args []string) error {
	gctx, gcancel = context.WithCancel(context.Background())
	setupCloseHandler(gcancel)
	targetsConfig, err := c.config.GetTargets()
	if err != nil {
		return fmt.Errorf("failed getting targets config: %v", err)
	}

	subscriptionsConfig, err := c.config.GetSubscriptions(cmd)
	if err != nil {
		return fmt.Errorf("failed getting subscriptions config: %v", err)
	}
	outs, err := c.config.GetOutputs()
	if err != nil {
		return err
	}
	epconfig, err := c.config.GetEventProcessors()
	if err != nil {
		return err
	}

	if c.collector == nil {
		cfg := &collector.Config{
			PrometheusAddress:   c.config.Globals.PrometheusAddress,
			Debug:               c.config.Globals.Debug,
			Format:              c.config.Globals.Format,
			TargetReceiveBuffer: c.config.Globals.TargetBufferSize,
			RetryTimer:          c.config.Globals.Retry,
		}

		c.collector = collector.NewCollector(cfg, targetsConfig,
			collector.WithDialOptions(createCollectorDialOpts()),
			collector.WithSubscriptions(subscriptionsConfig),
			collector.WithOutputs(outs),
			collector.WithLogger(c.logger),
			collector.WithEventProcessors(epconfig),
		)
	} else {
		// prompt mode
		for name, outCfg := range outs {
			err = c.collector.AddOutput(name, outCfg)
			if err != nil {
				c.logger.Printf("%v", err)
			}
		}
		for _, sc := range subscriptionsConfig {
			err = c.collector.AddSubscriptionConfig(sc)
			if err != nil {
				c.logger.Printf("%v", err)
			}
		}
		for _, tc := range targetsConfig {
			c.collector.AddTarget(tc)
			if err != nil {
				c.logger.Printf("%v", err)
			}
		}
	}

	c.collector.InitOutputs(gctx)

	go c.collector.Start(gctx)

	c.wg.Add(len(c.collector.Targets))
	for _, target := range c.collector.Targets {
		go c.subscribe(gctx, target.Config)
	}
	c.wg.Wait()
	polledTargetsSubscriptions := c.collector.PolledSubscriptionsTargets()
	if len(polledTargetsSubscriptions) > 0 {
		pollTargets := make([]string, 0, len(polledTargetsSubscriptions))
		for t := range polledTargetsSubscriptions {
			pollTargets = append(pollTargets, t)
		}
		sort.Slice(pollTargets, func(i, j int) bool {
			return pollTargets[i] < pollTargets[j]
		})
		s := promptui.Select{
			Label:        "select target to poll",
			Items:        pollTargets,
			HideSelected: true,
		}
		waitChan := make(chan struct{}, 1)
		waitChan <- struct{}{}
		mo := &formatters.MarshalOptions{
			Multiline: true,
			Indent:    "  ",
			Format:    cli.config.Globals.Format,
		}
		go func() {
			for {
				select {
				case <-waitChan:
					_, name, err := s.Run()
					if err != nil {
						fmt.Printf("failed selecting target to poll: %v\n", err)
						continue
					}
					ss := promptui.Select{
						Label:        "select subscription to poll",
						Items:        polledTargetsSubscriptions[name],
						HideSelected: true,
					}
					_, subName, err := ss.Run()
					if err != nil {
						fmt.Printf("failed selecting subscription to poll: %v\n", err)
						continue
					}
					response, err := c.collector.TargetPoll(name, subName)
					if err != nil && err != io.EOF {
						fmt.Printf("target '%s', subscription '%s': poll response error:%v\n", name, subName, err)
						continue
					}
					if response == nil {
						fmt.Printf("received empty response from target '%s'\n", name)
						continue
					}
					switch rsp := response.Response.(type) {
					case *gnmi.SubscribeResponse_SyncResponse:
						fmt.Printf("received sync response '%t' from '%s'\n", rsp.SyncResponse, name)
						waitChan <- struct{}{}
						continue
					}
					b, err := mo.Marshal(response, nil)
					if err != nil {
						fmt.Printf("target '%s', subscription '%s': poll response formatting error:%v\n", name, subName, err)
						fmt.Println(string(b))
						waitChan <- struct{}{}
						continue
					}
					fmt.Println(string(b))
					waitChan <- struct{}{}
				case <-gctx.Done():
					return
				}
			}
		}()
	}

	if cli.promptMode {
		return nil
	}
	for range gctx.Done() {
		return gctx.Err()
	}
	return nil
}

func (c *CLI) subscribe(ctx context.Context, tConf *collector.TargetConfig) {
	defer c.wg.Done()
	var err error
	for {
		err = c.collector.Subscribe(gctx, tConf.Name)
		if err != nil {
			if errors.Is(err, context.DeadlineExceeded) {
				c.logger.Printf("failed to initialize target '%s' timeout (%s) reached", tConf.Name, tConf.Timeout)
			} else {
				c.logger.Printf("failed to initialize target '%s': %v", tConf.Name, err)
			}
			c.logger.Printf("retrying target %s in %s", tConf.Name, tConf.RetryTimer)
			time.Sleep(tConf.RetryTimer)
			continue
		}
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

type godocBuilder struct{}

func prefix8(s string) string {
	if len(s) < 8 {
		return s
	}
	return s[:8]
}

func (b godocBuilder) Signature(heads map[string]string) string {
	return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"]))
}

func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {

	goDir := filepath.Join(dir, "go")
	toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
	logger.Printf("checking out go repo ...")
	if err := checkout(repoURL+"go", heads["go"], goDir); err != nil {
		return nil, fmt.Errorf("checkout of go: %v", err)
	}
	logger.Printf("checking out tools repo ...")
	if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil {
		return nil, fmt.Errorf("checkout of tools: %v", err)
	}

	var logWriter io.Writer = toLoggerWriter{logger}

	make := exec.Command(filepath.Join(goDir, "src/make.bash"))
	make.Dir = filepath.Join(goDir, "src")
	make.Stdout = logWriter
	make.Stderr = logWriter
	logger.Printf("running make.bash in %s ...", make.Dir)
	if err := make.Run(); err != nil {
		return nil, fmt.Errorf("running make.bash: %v", err)
	}

	logger.Printf("installing godoc ...")
	goBin := filepath.Join(goDir, "bin/go")
	goPath := filepath.Join(dir, "gopath")
	install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc")
	install.Stdout = logWriter
	install.Stderr = logWriter
	install.Env = append(os.Environ(),
		"GOROOT="+goDir,
		"GOPATH="+goPath,
		"GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"),
	)
	if err := install.Run(); err != nil {
		return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err)
	}

	logger.Printf("starting godoc ...")
	godocBin := filepath.Join(goPath, "bin/godoc")
	godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play")
	godoc.Env = append(os.Environ(), "GOROOT="+goDir)
	godoc.Stdout = logWriter
	godoc.Stderr = logWriter
	if err := godoc.Start(); err != nil {
		return nil, fmt.Errorf("starting godoc: %v", err)
	}
	return godoc, nil
}

var indexingMsg = []byte("Indexing in progress: result may be inaccurate")

func (b godocBuilder) HealthCheck(hostport string) error {
	body, err := getOK(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", hostport))
	if err != nil {
		return err
	}
	if bytes.Contains(body, indexingMsg) {
		return errors.New("still indexing")
	}
	return nil
}
<commit_msg>cmd/tip: also fetch x/net repository before building cmd/godoc<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

type godocBuilder struct{}

func prefix8(s string) string {
	if len(s) < 8 {
		return s
	}
	return s[:8]
}

func (b godocBuilder) Signature(heads map[string]string) string {
	// x/net is intentionally not a part of the signature, because
	// at this time it does not contribute substantially to the deployed
	// website, and so we don't want tip.golang.org redeployed whenever
	// x/net changes. This will no longer matter when the Go website uses
	// modules to pin its dependencies to specific versions.
	return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"]))
}

func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
	goDir := filepath.Join(dir, "go")
	toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
	netDir := filepath.Join(dir, "gopath/src/golang.org/x/net")
	logger.Printf("checking out go repo ...")
	if err := checkout(repoURL+"go", heads["go"], goDir); err != nil {
		return nil, fmt.Errorf("checkout of go: %v", err)
	}
	logger.Printf("checking out tools repo ...")
	if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil {
		return nil, fmt.Errorf("checkout of tools: %v", err)
	}
	logger.Printf("checking out net repo ...")
	if err := checkout(repoURL+"net", heads["net"], netDir); err != nil {
		return nil, fmt.Errorf("checkout of net: %v", err)
	}

	var logWriter io.Writer = toLoggerWriter{logger}

	make := exec.Command(filepath.Join(goDir, "src/make.bash"))
	make.Dir = filepath.Join(goDir, "src")
	make.Stdout = logWriter
	make.Stderr = logWriter
	logger.Printf("running make.bash in %s ...", make.Dir)
	if err := make.Run(); err != nil {
		return nil, fmt.Errorf("running make.bash: %v", err)
	}

	logger.Printf("installing godoc ...")
	goBin := filepath.Join(goDir, "bin/go")
	goPath := filepath.Join(dir, "gopath")
	install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc")
	install.Stdout = logWriter
	install.Stderr = logWriter
	install.Env = append(os.Environ(),
		"GOROOT="+goDir,
		"GOPATH="+goPath,
		"GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"),
	)
	if err := install.Run(); err != nil {
		return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err)
	}

	logger.Printf("starting godoc ...")
	godocBin := filepath.Join(goPath, "bin/godoc")
	godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play")
	godoc.Env = append(os.Environ(), "GOROOT="+goDir)
	godoc.Stdout = logWriter
	godoc.Stderr = logWriter
	if err := godoc.Start(); err != nil {
		return nil, fmt.Errorf("starting godoc: %v", err)
	}
	return godoc, nil
}

var indexingMsg = []byte("Indexing in progress: result may be inaccurate")

func (b godocBuilder) HealthCheck(hostport string) error {
	body, err := getOK(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", hostport))
	if err != nil {
		return err
	}
	if bytes.Contains(body, indexingMsg) {
		return errors.New("still indexing")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"text/template"

	"github.com/go-clix/cli"

	"github.com/grafana/tanka/pkg/tanka"
)

func exportCmd() *cli.Command {
	args := workflowArgs
	args.Validator = cli.ValidateExact(2)

	cmd := &cli.Command{
		Use:   "export <environment> <outputDir>",
		Short: "write each resources as a YAML file",
		Args:  args,
	}

	vars := workflowFlags(cmd.Flags())
	getExtCode := extCodeParser(cmd.Flags())
	format := cmd.Flags().String("format", "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames")
	extension := cmd.Flags().String("extension", "yaml", "File extension")

	templateFuncMap := template.FuncMap{
		"lower": func(s string) string {
			return strings.ToLower(s)
		},
	}

	cmd.Run = func(cmd *cli.Command, args []string) error {
		// dir must be empty
		to := args[1]
		empty, err := dirEmpty(to)
		if err != nil {
			return fmt.Errorf("Checking target dir: %s", err)
		}
		if !empty {
			return fmt.Errorf("Target dir `%s` not empty. Aborting.", to)
		}

		// exit early if the template is bad
		tmpl, err := template.New("").Funcs(templateFuncMap).Parse(*format)
		if err != nil {
			return fmt.Errorf("Parsing name format: %s", err)
		}

		// get the manifests
		res, err := tanka.Show(args[0],
			tanka.WithExtCode(getExtCode()),
			tanka.WithTargets(stringsToRegexps(vars.targets)),
		)
		if err != nil {
			return err
		}

		// write each to a file
		for _, m := range res {
			buf := bytes.Buffer{}
			if err := tmpl.Execute(&buf, m); err != nil {
				log.Fatalln("executing name template:", err)
			}
			name := strings.Replace(buf.String(), "/", "-", -1)

			data := m.String()
			if err := ioutil.WriteFile(filepath.Join(to, name+"."+*extension), []byte(data), 0644); err != nil {
				return fmt.Errorf("Writing manifest: %s", err)
			}
		}

		return nil
	}
	return cmd
}

func dirEmpty(dir string) (bool, error) {
	f, err := os.Open(dir)
	if os.IsNotExist(err) {
		return true, os.MkdirAll(dir, os.ModePerm)
	} else if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}
<commit_msg>feat(cli): allow creating subfolders during export (#300)<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"text/template"

	"github.com/go-clix/cli"

	"github.com/grafana/tanka/pkg/tanka"
)

// BelRune is a string of the Ascii character BEL which made computers ring in ancient times
// We use it as "magic" char for the subfolder creation as it is a non printable character and thereby will never be
// in a valid filepath by accident. Only when we include it.
const BelRune = string(rune(7))

func exportCmd() *cli.Command {
	args := workflowArgs
	args.Validator = cli.ValidateExact(2)

	cmd := &cli.Command{
		Use:   "export <environment> <outputDir>",
		Short: "write each resources as a YAML file",
		Args:  args,
	}

	vars := workflowFlags(cmd.Flags())
	getExtCode := extCodeParser(cmd.Flags())
	format := cmd.Flags().String("format", "{{.apiVersion}}.{{.kind}}-{{.metadata.name}}", "https://tanka.dev/exporting#filenames")
	extension := cmd.Flags().String("extension", "yaml", "File extension")

	templateFuncMap := template.FuncMap{
		"lower": func(s string) string {
			return strings.ToLower(s)
		},
	}

	cmd.Run = func(cmd *cli.Command, args []string) error {
		// dir must be empty
		to := args[1]
		empty, err := dirEmpty(to)
		if err != nil {
			return fmt.Errorf("Checking target dir: %s", err)
		}
		if !empty {
			return fmt.Errorf("Target dir `%s` not empty. Aborting.", to)
		}

		// exit early if the template is bad

		// Replace all os.path separators in string with BelRune for creating subfolders
		replacedFormat := strings.Replace(*format, string(os.PathSeparator), BelRune, -1)

		tmpl, err := template.New("").Funcs(templateFuncMap).Parse(replacedFormat)
		if err != nil {
			return fmt.Errorf("Parsing name format: %s", err)
		}

		// get the manifests
		res, err := tanka.Show(args[0],
			tanka.WithExtCode(getExtCode()),
			tanka.WithTargets(stringsToRegexps(vars.targets)),
		)
		if err != nil {
			return err
		}

		// write each to a file
		for _, m := range res {
			buf := bytes.Buffer{}
			if err := tmpl.Execute(&buf, m); err != nil {
				log.Fatalln("executing name template:", err)
			}

			// Replace all os.path separators in string in order to not accidentally create subfolders
			name := strings.Replace(buf.String(), string(os.PathSeparator), "-", -1)
			// Replace the BEL character inserted with a path separator again in order to create a subfolder
			name = strings.Replace(name, BelRune, string(os.PathSeparator), -1)

			// Create all subfolders in path
			path := filepath.Join(to, name+"."+*extension)
			if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
				return fmt.Errorf("creating filepath '%s': %s", filepath.Dir(path), err)
			}

			data := m.String()
			if err := ioutil.WriteFile(path, []byte(data), 0644); err != nil {
				return fmt.Errorf("writing manifest: %s", err)
			}
		}

		return nil
	}
	return cmd
}

func dirEmpty(dir string) (bool, error) {
	f, err := os.Open(dir)
	if os.IsNotExist(err) {
		return true, os.MkdirAll(dir, os.ModePerm)
	} else if err != nil {
		return false, err
	}
	defer f.Close()

	_, err = f.Readdirnames(1)
	if err == io.EOF {
		return true, nil
	}
	return false, err
}
<|endoftext|>