<commit_before>// Copyright 2014 The intelengine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"path"
	"sync"

	"github.com/jroimartin/rpcmq"
)

const (
	sep = '|'
)

type worker struct {
	cfg    config
	server *rpcmq.Server

	mu       sync.RWMutex
	commands map[string]*command
}

func newWorker(cfg config) *worker {
	return &worker{cfg: cfg}
}

func (w *worker) start() error {
	if w.cfg.Worker.CmdDir == "" || w.cfg.Broker.URI == "" || w.cfg.Broker.Queue == "" {
		return errors.New("missing configuration parameters")
	}

	var tlsConfig *tls.Config
	if w.cfg.Broker.CAFile != "" && w.cfg.Broker.CertFile != "" &&
		w.cfg.Broker.KeyFile != "" {
		cert, err := tls.LoadX509KeyPair(w.cfg.Broker.CertFile, w.cfg.Broker.KeyFile)
		if err != nil {
			return fmt.Errorf("LoadX509KeyPair: %v", err)
		}
		caCert, err := ioutil.ReadFile(w.cfg.Broker.CAFile)
		if err != nil {
			return fmt.Errorf("ReadFile: %v", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
	}
	w.server = rpcmq.NewServer(w.cfg.Broker.URI, w.cfg.Broker.Queue)
	w.server.TLSConfig = tlsConfig
	if err := w.server.Init(); err != nil {
		return fmt.Errorf("Init: %v", err)
	}
	defer w.server.Shutdown()

	w.refreshCommands()
	if err := w.server.Register("listCommands", w.listCommands); err != nil {
		return err
	}
	if err := w.server.Register("execCommand", w.execCommand); err != nil {
		return err
	}

	select {}
}

func (w *worker) listCommands(data []byte) ([]byte, error) {
	w.refreshCommands()

	w.mu.RLock()
	defer w.mu.RUnlock()

	b, err := json.Marshal(w.commands)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal commands: %v", err)
	}
	return b, nil
}

func (w *worker) refreshCommands() {
	w.mu.Lock()
	defer w.mu.Unlock()

	w.commands = map[string]*command{}

	files, err := ioutil.ReadDir(w.cfg.Worker.CmdDir)
	if err != nil {
		log.Println("refreshCommands warning:", err)
		return
	}

	for _, f := range files {
		if f.IsDir() || path.Ext(f.Name()) != cmdExt {
			continue
		}

		filename := path.Join(w.cfg.Worker.CmdDir, f.Name())
		cmd, err := newCommand(filename)
		if err != nil {
			log.Printf("refreshCommands warning (%v): %v\n", f.Name(), err)
			continue
		}

		w.commands[cmd.Name] = cmd
		log.Println("command registered:", cmd.Name)
	}
}

func (w *worker) execCommand(data []byte) ([]byte, error) {
	w.mu.RLock()
	defer w.mu.RUnlock()

	sepIdx := bytes.IndexByte(data, sep)
	if sepIdx < 0 {
		return nil, errors.New("separator not found")
	}
	name := string(data[:sepIdx])
	br := bytes.NewReader(data[sepIdx+1:])

	cmd := w.command(name)
	if cmd == nil {
		return nil, fmt.Errorf("command not found: %v", name)
	}

	out, err := cmd.exec(br)
	if err != nil {
		return nil, fmt.Errorf("command execution error: %v", err)
	}

	return out, nil
}

func (w *worker) command(name string) *command {
	w.mu.RLock()
	defer w.mu.RUnlock()

	if cmd, ok := w.commands[name]; ok {
		return cmd
	}
	return nil
}
<commit_msg>Allow to use complex directory structures for cmd files<commit_after>// Copyright 2014 The intelengine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"

	"github.com/jroimartin/rpcmq"
)

const (
	sep = '|'
)

type worker struct {
	cfg    config
	server *rpcmq.Server

	mu       sync.RWMutex
	commands map[string]*command
}

func newWorker(cfg config) *worker {
	return &worker{cfg: cfg}
}

func (w *worker) start() error {
	if w.cfg.Worker.CmdDir == "" || w.cfg.Broker.URI == "" || w.cfg.Broker.Queue == "" {
		return errors.New("missing configuration parameters")
	}

	var tlsConfig *tls.Config
	if w.cfg.Broker.CAFile != "" && w.cfg.Broker.CertFile != "" &&
		w.cfg.Broker.KeyFile != "" {
		cert, err := tls.LoadX509KeyPair(w.cfg.Broker.CertFile, w.cfg.Broker.KeyFile)
		if err != nil {
			return fmt.Errorf("LoadX509KeyPair: %v", err)
		}
		caCert, err := ioutil.ReadFile(w.cfg.Broker.CAFile)
		if err != nil {
			return fmt.Errorf("ReadFile: %v", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig = &tls.Config{
			Certificates: []tls.Certificate{cert},
			RootCAs:      caCertPool,
		}
	}
	w.server = rpcmq.NewServer(w.cfg.Broker.URI, w.cfg.Broker.Queue)
	w.server.TLSConfig = tlsConfig
	if err := w.server.Init(); err != nil {
		return fmt.Errorf("Init: %v", err)
	}
	defer w.server.Shutdown()

	w.refreshCommands()
	if err := w.server.Register("listCommands", w.listCommands); err != nil {
		return err
	}
	if err := w.server.Register("execCommand", w.execCommand); err != nil {
		return err
	}

	select {}
}

func (w *worker) listCommands(data []byte) ([]byte, error) {
	if err := w.refreshCommands(); err != nil {
		return nil, fmt.Errorf("cannot refresh commands: %v", err)
	}

	w.mu.RLock()
	defer w.mu.RUnlock()

	b, err := json.Marshal(w.commands)
	if err != nil {
		return nil, fmt.Errorf("cannot marshal commands: %v", err)
	}
	return b, nil
}

func (w *worker) refreshCommands() error {
	w.mu.Lock()
	defer w.mu.Unlock()

	w.commands = map[string]*command{}
	if err := filepath.Walk(w.cfg.Worker.CmdDir, w.handleFile); err != nil {
		return err
	}
	return nil
}

func (w *worker) handleFile(filepath string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	if info.IsDir() || path.Ext(info.Name()) != cmdExt {
		return nil
	}

	cmd, err := newCommand(filepath)
	if err != nil {
		log.Printf("handleFile warning (%v): %v\n", info.Name(), err)
		return nil
	}

	w.commands[cmd.Name] = cmd
	log.Println("command registered:", cmd.Name)
	return nil
}

func (w *worker) execCommand(data []byte) ([]byte, error) {
	w.mu.RLock()
	defer w.mu.RUnlock()

	sepIdx := bytes.IndexByte(data, sep)
	if sepIdx < 0 {
		return nil, errors.New("separator not found")
	}
	name := string(data[:sepIdx])
	br := bytes.NewReader(data[sepIdx+1:])

	cmd := w.command(name)
	if cmd == nil {
		return nil, fmt.Errorf("command not found: %v", name)
	}

	out, err := cmd.exec(br)
	if err != nil {
		return nil, fmt.Errorf("command execution error: %v", err)
	}

	return out, nil
}

func (w *worker) command(name string) *command {
	w.mu.RLock()
	defer w.mu.RUnlock()

	if cmd, ok := w.commands[name]; ok {
		return cmd
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"math"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/mikioh/ipaddr"
	"github.com/mikioh/ipoam"
	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv4"
	"golang.org/x/net/ipv6"
)

var cvUsageTmpl = `Usage:
	ipoam {{.Name}} [flags] destination

destination
	A hostname, DNS reg-name, IP address, IP address prefix, or
	comma-separated list of IP addresses and/or IP address prefixes.
	A combination of unicast and multicast addresses is prohibited.

`

var (
	cmdCV = &Command{
		Func:      cvMain,
		Usage:     cmdUsage,
		UsageTmpl: cvUsageTmpl,
		CanonName: "cv",
		Aliases:   []string{"ping"},
		Descr:     "Verify IP-layer connectivity",
	}

	cvPayload []byte
	cvData    = []byte("0123456789abcdefghijklmnopqrstuvwxyz")

	cvIPv4only    bool
	cvIPv6only    bool
	cvNoRevLookup bool
	cvQuiet       bool
	cvXmitOnly    bool
	cvVerbose     bool

	cvCount         int
	cvHops          int
	cvMulticastHops int
	cvTC            int
	cvPayloadLen    int
	cvWait          int // allow to run "hidden flooding mode" when cvWait is a negative integer

	cvOutboundIf string
	cvSrc        string
)

func init() {
	cmdCV.Flag.BoolVar(&cvIPv4only, "4", false, "Run IPv4 test only")
	cmdCV.Flag.BoolVar(&cvIPv6only, "6", false, "Run IPv6 test only")
	cmdCV.Flag.BoolVar(&cvNoRevLookup, "n", false, "Don't use DNS reverse lookup")
	cmdCV.Flag.BoolVar(&cvQuiet, "q", false, "Quiet output except summary")
	cmdCV.Flag.BoolVar(&cvXmitOnly, "x", false, "Run transmission only")
	cmdCV.Flag.BoolVar(&cvVerbose, "v", false, "Show verbose information")

	cmdCV.Flag.IntVar(&cvCount, "count", 0, "Iteration count, less than or equal to zero will run until interrupted")
	cmdCV.Flag.IntVar(&cvHops, "hops", 64, "IPv4 TTL or IPv6 hop-limit on outgoing unicast packets")
	cmdCV.Flag.IntVar(&cvMulticastHops, "mchops", 5, "IPv4 TTL or IPv6 hop-limit on outgoing multicast packets")
	cmdCV.Flag.IntVar(&cvTC, "tc", 0, "IPv4 TOS or IPv6 traffic-class on outgoing packets")
	cmdCV.Flag.IntVar(&cvPayloadLen, "pldlen", 56, "ICMP echo payload length")
	cmdCV.Flag.IntVar(&cvWait, "wait", 1, "Seconds between transmitting each echo")

	cmdCV.Flag.StringVar(&cvOutboundIf, "if", "", "Outbound interface name")
	cmdCV.Flag.StringVar(&cvSrc, "src", "", "Source IP address")
}

func cvMain(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.Flag.Usage()
	}

	bw := bufio.NewWriter(os.Stdout)

	c, ifi, err := parseDsts(args[0], cvIPv4only, cvIPv6only)
	if err != nil {
		cmd.fatal(err)
	}

	cvPayload = bytes.Repeat(cvData, int(cvPayloadLen)/len(cvData)+1)
	cvPayload = cvPayload[:cvPayloadLen]
	if cvWait == 0 {
		cvWait = 1
	}
	if cvOutboundIf != "" {
		oif, err := net.InterfaceByName(cvOutboundIf)
		if err == nil {
			ifi = oif
		}
	}
	var src net.IP
	if cvSrc != "" {
		src = net.ParseIP(cvSrc)
		if src.To4() != nil {
			cvIPv4only = true
		}
		if src.To16() != nil && src.To4() == nil {
			cvIPv6only = true
		}
	}

	var ipts = [2]struct {
		t *ipoam.Tester
		r <-chan ipoam.Report
	}{}
	for _, p := range c.List() {
		if !cvIPv6only && p.IP.To4() != nil && ipts[0].t == nil {
			address := "0.0.0.0"
			if src != nil {
				address = src.String()
			}
			ipts[0].t, err = ipoam.NewTester("ip4:icmp", address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipts[0].t.Close()
			ipts[0].r = ipts[0].t.Report()
			if cvXmitOnly {
				ipts[0].t.StopReport()
			}
			if p := ipts[0].t.IPv4PacketConn(); p != nil {
				if cvHops >= 0 {
					p.SetTTL(cvHops)
				}
				if cvMulticastHops >= 0 {
					p.SetMulticastTTL(cvMulticastHops)
				}
				if cvTC >= 0 {
					p.SetTOS(cvTC)
				}
			}
		}
		if !cvIPv4only && p.IP.To16() != nil && p.IP.To4() == nil && ipts[1].t == nil {
			address := "::"
			if src != nil {
				address = src.String()
			}
			ipts[1].t, err = ipoam.NewTester("ip6:ipv6-icmp", address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipts[1].t.Close()
			ipts[1].r = ipts[1].t.Report()
			if cvXmitOnly {
				ipts[1].t.StopReport()
			}
			if p := ipts[1].t.IPv6PacketConn(); p != nil {
				if cvHops >= 0 {
					p.SetHopLimit(cvHops)
				}
				if cvMulticastHops >= 0 {
					p.SetMulticastHopLimit(cvHops)
				}
				if cvTC >= 0 {
					p.SetTrafficClass(cvTC)
				}
			}
		}
		if ipts[0].t != nil && ipts[1].t != nil {
			break
		}
	}

	printCVBanner(bw, args[0], c)

	stats := make(cvStats)
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
	var onlink ipoam.Report
	cm := ipoam.ControlMessage{ID: os.Getpid() & 0xffff}
	for i := 1; ; i++ {
		t := time.NewTimer(time.Duration(cvWait) * time.Second)
		begin := time.Now()
		cm.Seq = i
		for pos := c.First(); pos != nil; pos = c.Next() {
			if !cvIPv6only && pos.IP.To4() != nil {
				onlink.Error = ipts[0].t.Probe(cvPayload, &cm, pos.IP, ifi)
				stats.get(pos.IP.String()).onDeparture(&onlink)
				if onlink.Error != nil {
					printCVReport(bw, 0, &onlink)
					continue
				}
			}
			if !cvIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil {
				onlink.Error = ipts[1].t.Probe(cvPayload, &cm, pos.IP, ifi)
				stats.get(pos.IP.String()).onDeparture(&onlink)
				if onlink.Error != nil {
					printCVReport(bw, 0, &onlink)
					continue
				}
			}
		}
		c.Reset(nil)

	loop:
		for {
			select {
			case <-sig:
				if cvVerbose {
					printCVSummary(bw, args[0], stats)
				}
				os.Exit(0)
			case <-t.C:
				break loop
			case r := <-ipts[0].r:
				rtt := time.Since(begin)
				printCVReport(bw, rtt, &r)
				stats.get(r.Src.String()).onArrival(rtt, &r)
			case r := <-ipts[1].r:
				rtt := time.Since(begin)
				printCVReport(bw, rtt, &r)
				stats.get(r.Src.String()).onArrival(rtt, &r)
			}
		}
		t.Stop()

		if cvCount > 0 && i == cvCount {
			if cvVerbose {
				printCVSummary(bw, args[0], stats)
			}
			os.Exit(0)
		}
	}
}

type cvStats map[string]*cvStat

func (stats cvStats) get(s string) *cvStat {
	st := stats[s]
	if st == nil {
		st = &cvStat{minRTT: math.MaxInt64}
		stats[s] = st
	}
	return st
}

type cvStat struct {
	received    uint64
	transmitted uint64
	opErrors    uint64
	icmpErrors  uint64

	minRTT time.Duration
	maxRTT time.Duration
	rttSum time.Duration
	rttSq  float64
}

func (st *cvStat) onArrival(rtt time.Duration, r *ipoam.Report) {
	if r.Error != nil {
		st.opErrors++
		return
	}
	if r.ICMP.Type != ipv4.ICMPTypeEchoReply && r.ICMP.Type != ipv6.ICMPTypeEchoReply {
		st.icmpErrors++
		return
	}
	st.received++
	if rtt < st.minRTT {
		st.minRTT = rtt
	}
	if rtt > st.maxRTT {
		st.maxRTT = rtt
	}
	st.rttSum += rtt
	st.rttSq += float64(rtt) * float64(rtt)
}

func (st *cvStat) onDeparture(r *ipoam.Report) {
	st.transmitted++
	if r.Error != nil {
		st.opErrors++
	}
}

func printCVBanner(bw *bufio.Writer, dsts string, c *ipaddr.Cursor) {
	fmt.Fprintf(bw, "Connectivity verification for %s", dsts)
	if cvVerbose {
		fmt.Fprintf(bw, " [")
		printed := false
		for pos := c.First(); pos != nil; pos = c.Next() {
			if !cvIPv6only && pos.IP.To4() != nil || !cvIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil {
				if printed {
					fmt.Fprintf(bw, " ")
				}
				fmt.Fprintf(bw, "%v", pos.IP)
				printed = true
			} else {
				printed = false
			}
		}
		fmt.Fprintf(bw, "]")
		c.Reset(nil)
	}
	fmt.Fprintf(bw, ": %d bytes payload\n", len(cvPayload))
	bw.Flush()
}

func printCVReport(bw *bufio.Writer, rtt time.Duration, r *ipoam.Report) {
	if cvQuiet {
		return
	}
	if r.Error != nil {
		fmt.Fprintf(bw, "error=%q\n", r.Error)
		bw.Flush()
		return
	}
	if r.ICMP.Type != ipv4.ICMPTypeEchoReply && r.ICMP.Type != ipv6.ICMPTypeEchoReply {
		fmt.Fprintf(bw, "from=%s icmp.type=%q icmp.code=%d rtt=%v\n", literalOrName(r.Src.String(), cvNoRevLookup), r.ICMP.Type, r.ICMP.Code, rtt)
		bw.Flush()
		return
	}
	echo, _ := r.ICMP.Body.(*icmp.Echo)
	fmt.Fprintf(bw, "%d bytes", len(echo.Data))
	if !cvVerbose {
		fmt.Fprintf(bw, " from=%s echo.seq=%d rtt=%v\n", literalOrName(r.Src.String(), cvNoRevLookup), echo.Seq, rtt)
		bw.Flush()
		return
	}
	if r.Dst == nil {
		fmt.Fprintf(bw, " from=%s", literalOrName(r.Src.String(), cvNoRevLookup))
	} else {
		fmt.Fprintf(bw, " tc=%#x hops=%d from=%s to=%s", r.TC, r.Hops, literalOrName(r.Src.String(), cvNoRevLookup), literalOrName(r.Dst.String(), cvNoRevLookup))
	}
	if r.Interface != nil {
		fmt.Fprintf(bw, " if=%s", r.Interface.Name)
	}
	fmt.Fprintf(bw, " echo.id=%d echo.seq=%d rtt=%v\n", echo.ID, echo.Seq, rtt)
	bw.Flush()
}

func printCVSummary(bw *bufio.Writer, dsts string, stats cvStats) {
	fmt.Fprintf(bw, "\nStatistical information for %s:\n", dsts)
	for ip, st := range stats {
		var avg time.Duration
		var stddev float64
		if st.received > 0 {
			avg = st.rttSum / time.Duration(st.received)
			stddev = math.Sqrt(float64(st.rttSq)/float64(st.received) - float64(avg)*float64(avg))
		} else {
			st.minRTT = 0
		}
		fmt.Fprintf(bw, "%s:", literalOrName(ip, cvNoRevLookup))
		if st.transmitted > 0 && st.received <= st.transmitted {
			fmt.Fprintf(bw, " loss=%.1f%%", float64(st.transmitted-st.received)*100.0/float64(st.transmitted))
		}
		fmt.Fprintf(bw, " rcvd=%d sent=%d op.err=%d icmp.err=%d", st.received, st.transmitted, st.opErrors, st.icmpErrors)
		fmt.Fprintf(bw, " min=%v avg=%v max=%v stddev=%v\n", st.minRTT, avg, st.maxRTT, time.Duration(stddev))
	}
	bw.Flush()
}
<commit_msg>cmd/ipoam: tweak variable names<commit_after>// Copyright 2015 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"math"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/mikioh/ipaddr"
	"github.com/mikioh/ipoam"
	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv4"
	"golang.org/x/net/ipv6"
)

var cvUsageTmpl = `Usage:
	ipoam {{.Name}} [flags] destination

destination
	A hostname, DNS reg-name, IP address, IP address prefix, or
	comma-separated list of IP addresses and/or IP address prefixes.
	A combination of unicast and multicast addresses is prohibited.

`

var (
	cmdCV = &Command{
		Func:      cvMain,
		Usage:     cmdUsage,
		UsageTmpl: cvUsageTmpl,
		CanonName: "cv",
		Aliases:   []string{"ping"},
		Descr:     "Verify IP-layer connectivity",
	}

	cvPayload []byte
	cvData    = []byte("0123456789abcdefghijklmnopqrstuvwxyz")

	cvIPv4only    bool
	cvIPv6only    bool
	cvNoRevLookup bool
	cvQuiet       bool
	cvXmitOnly    bool
	cvVerbose     bool

	cvCount         int
	cvHops          int
	cvMulticastHops int
	cvTC            int
	cvPayloadLen    int
	cvWait          int // allow to run "hidden flooding mode" when cvWait is a negative integer

	cvOutboundIf string
	cvSrc        string
)

func init() {
	cmdCV.Flag.BoolVar(&cvIPv4only, "4", false, "Run IPv4 test only")
	cmdCV.Flag.BoolVar(&cvIPv6only, "6", false, "Run IPv6 test only")
	cmdCV.Flag.BoolVar(&cvNoRevLookup, "n", false, "Don't use DNS reverse lookup")
	cmdCV.Flag.BoolVar(&cvQuiet, "q", false, "Quiet output except summary")
	cmdCV.Flag.BoolVar(&cvXmitOnly, "x", false, "Run transmission only")
	cmdCV.Flag.BoolVar(&cvVerbose, "v", false, "Show verbose information")

	cmdCV.Flag.IntVar(&cvCount, "count", 0, "Iteration count, less than or equal to zero will run until interrupted")
	cmdCV.Flag.IntVar(&cvHops, "hops", 64, "IPv4 TTL or IPv6 hop-limit on outgoing unicast packets")
	cmdCV.Flag.IntVar(&cvMulticastHops, "mchops", 5, "IPv4 TTL or IPv6 hop-limit on outgoing multicast packets")
	cmdCV.Flag.IntVar(&cvTC, "tc", 0, "IPv4 TOS or IPv6 traffic-class on outgoing packets")
	cmdCV.Flag.IntVar(&cvPayloadLen, "pldlen", 56, "ICMP echo payload length")
	cmdCV.Flag.IntVar(&cvWait, "wait", 1, "Seconds between transmitting each echo")

	cmdCV.Flag.StringVar(&cvOutboundIf, "if", "", "Outbound interface name")
	cmdCV.Flag.StringVar(&cvSrc, "src", "", "Source IP address")
}

func cvMain(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.Flag.Usage()
	}

	bw := bufio.NewWriter(os.Stdout)

	c, ifi, err := parseDsts(args[0], cvIPv4only, cvIPv6only)
	if err != nil {
		cmd.fatal(err)
	}

	cvPayload = bytes.Repeat(cvData, int(cvPayloadLen)/len(cvData)+1)
	cvPayload = cvPayload[:cvPayloadLen]
	if cvWait == 0 {
		cvWait = 1
	}
	if cvOutboundIf != "" {
		oif, err := net.InterfaceByName(cvOutboundIf)
		if err == nil {
			ifi = oif
		}
	}
	var src net.IP
	if cvSrc != "" {
		src = net.ParseIP(cvSrc)
		if src.To4() != nil {
			cvIPv4only = true
		}
		if src.To16() != nil && src.To4() == nil {
			cvIPv6only = true
		}
	}

	var ipts = [2]struct {
		t *ipoam.Tester
		r <-chan ipoam.Report
	}{}
	for _, pos := range c.List() {
		if !cvIPv6only && pos.IP.To4() != nil && ipts[0].t == nil {
			address := "0.0.0.0"
			if src != nil {
				address = src.String()
			}
			ipts[0].t, err = ipoam.NewTester("ip4:icmp", address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipts[0].t.Close()
			ipts[0].r = ipts[0].t.Report()
			if cvXmitOnly {
				ipts[0].t.StopReport()
			}
			if p := ipts[0].t.IPv4PacketConn(); p != nil {
				if cvHops >= 0 {
					p.SetTTL(cvHops)
				}
				if cvMulticastHops >= 0 {
					p.SetMulticastTTL(cvMulticastHops)
				}
				if cvTC >= 0 {
					p.SetTOS(cvTC)
				}
			}
		}
		if !cvIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil && ipts[1].t == nil {
			address := "::"
			if src != nil {
				address = src.String()
			}
			ipts[1].t, err = ipoam.NewTester("ip6:ipv6-icmp", address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipts[1].t.Close()
			ipts[1].r = ipts[1].t.Report()
			if cvXmitOnly {
				ipts[1].t.StopReport()
			}
			if p := ipts[1].t.IPv6PacketConn(); p != nil {
				if cvHops >= 0 {
					p.SetHopLimit(cvHops)
				}
				if cvMulticastHops >= 0 {
					p.SetMulticastHopLimit(cvHops)
				}
				if cvTC >= 0 {
					p.SetTrafficClass(cvTC)
				}
			}
		}
		if ipts[0].t != nil && ipts[1].t != nil {
			break
		}
	}

	printCVBanner(bw, args[0], c)

	stats := make(cvStats)
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
	var onlink ipoam.Report
	cm := ipoam.ControlMessage{ID: os.Getpid() & 0xffff}
	for i := 1; ; i++ {
		t := time.NewTimer(time.Duration(cvWait) * time.Second)
		begin := time.Now()
		cm.Seq = i
		for pos := c.First(); pos != nil; pos = c.Next() {
			if !cvIPv6only && pos.IP.To4() != nil {
				onlink.Error = ipts[0].t.Probe(cvPayload, &cm, pos.IP, ifi)
				stats.get(pos.IP.String()).onDeparture(&onlink)
				if onlink.Error != nil {
					printCVReport(bw, 0, &onlink)
					continue
				}
			}
			if !cvIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil {
				onlink.Error = ipts[1].t.Probe(cvPayload, &cm, pos.IP, ifi)
				stats.get(pos.IP.String()).onDeparture(&onlink)
				if onlink.Error != nil {
					printCVReport(bw, 0, &onlink)
					continue
				}
			}
		}
		c.Reset(nil)

	loop:
		for {
			select {
			case <-sig:
				if cvVerbose {
					printCVSummary(bw, args[0], stats)
				}
				os.Exit(0)
			case <-t.C:
				break loop
			case r := <-ipts[0].r:
				rtt := time.Since(begin)
				printCVReport(bw, rtt, &r)
				stats.get(r.Src.String()).onArrival(rtt, &r)
			case r := <-ipts[1].r:
				rtt := time.Since(begin)
				printCVReport(bw, rtt, &r)
				stats.get(r.Src.String()).onArrival(rtt, &r)
			}
		}
		t.Stop()

		if cvCount > 0 && i == cvCount {
			if cvVerbose {
				printCVSummary(bw, args[0], stats)
			}
			os.Exit(0)
		}
	}
}

type cvStats map[string]*cvStat

func (stats cvStats) get(s string) *cvStat {
	st := stats[s]
	if st == nil {
		st = &cvStat{minRTT: math.MaxInt64}
		stats[s] = st
	}
	return st
}

type cvStat struct {
	received    uint64
	transmitted uint64
	opErrors    uint64
	icmpErrors  uint64

	minRTT time.Duration
	maxRTT time.Duration
	rttSum time.Duration
	rttSq  float64
}

func (st *cvStat) onArrival(rtt time.Duration, r *ipoam.Report) {
	if r.Error != nil {
		st.opErrors++
		return
	}
	if r.ICMP.Type != ipv4.ICMPTypeEchoReply && r.ICMP.Type != ipv6.ICMPTypeEchoReply {
		st.icmpErrors++
		return
	}
	st.received++
	if rtt < st.minRTT {
		st.minRTT = rtt
	}
	if rtt > st.maxRTT {
		st.maxRTT = rtt
	}
	st.rttSum += rtt
	st.rttSq += float64(rtt) * float64(rtt)
}

func (st *cvStat) onDeparture(r *ipoam.Report) {
	st.transmitted++
	if r.Error != nil {
		st.opErrors++
	}
}

func printCVBanner(bw *bufio.Writer, dsts string, c *ipaddr.Cursor) {
	fmt.Fprintf(bw, "Connectivity verification for %s", dsts)
	if cvVerbose {
		fmt.Fprintf(bw, " [")
		printed := false
		for pos := c.First(); pos != nil; pos = c.Next() {
			if !cvIPv6only && pos.IP.To4() != nil || !cvIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil {
				if printed {
					fmt.Fprintf(bw, " ")
				}
				fmt.Fprintf(bw, "%v", pos.IP)
				printed = true
			} else {
				printed = false
			}
		}
		fmt.Fprintf(bw, "]")
		c.Reset(nil)
	}
	fmt.Fprintf(bw, ": %d bytes payload\n", len(cvPayload))
	bw.Flush()
}

func printCVReport(bw *bufio.Writer, rtt time.Duration, r *ipoam.Report) {
	if cvQuiet {
		return
	}
	if r.Error != nil {
		fmt.Fprintf(bw, "error=%q\n", r.Error)
		bw.Flush()
		return
	}
	if r.ICMP.Type != ipv4.ICMPTypeEchoReply && r.ICMP.Type != ipv6.ICMPTypeEchoReply {
		fmt.Fprintf(bw, "from=%s icmp.type=%q icmp.code=%d rtt=%v\n", literalOrName(r.Src.String(), cvNoRevLookup), r.ICMP.Type, r.ICMP.Code, rtt)
		bw.Flush()
		return
	}
	echo, _ := r.ICMP.Body.(*icmp.Echo)
	fmt.Fprintf(bw, "%d bytes", len(echo.Data))
	if !cvVerbose {
		fmt.Fprintf(bw, " from=%s echo.seq=%d rtt=%v\n", literalOrName(r.Src.String(), cvNoRevLookup), echo.Seq, rtt)
		bw.Flush()
		return
	}
	if r.Dst == nil {
		fmt.Fprintf(bw, " from=%s", literalOrName(r.Src.String(), cvNoRevLookup))
	} else {
		fmt.Fprintf(bw, " tc=%#x hops=%d from=%s to=%s", r.TC, r.Hops, literalOrName(r.Src.String(), cvNoRevLookup), literalOrName(r.Dst.String(), cvNoRevLookup))
	}
	if r.Interface != nil {
		fmt.Fprintf(bw, " if=%s", r.Interface.Name)
	}
	fmt.Fprintf(bw, " echo.id=%d echo.seq=%d rtt=%v\n", echo.ID, echo.Seq, rtt)
	bw.Flush()
}

func printCVSummary(bw *bufio.Writer, dsts string, stats cvStats) {
	fmt.Fprintf(bw, "\nStatistical information for %s:\n", dsts)
	for ip, st := range stats {
		var avg time.Duration
		var stddev float64
		if st.received > 0 {
			avg = st.rttSum / time.Duration(st.received)
			stddev = math.Sqrt(float64(st.rttSq)/float64(st.received) - float64(avg)*float64(avg))
		} else {
			st.minRTT = 0
		}
		fmt.Fprintf(bw, "%s:", literalOrName(ip, cvNoRevLookup))
		if st.transmitted > 0 && st.received <= st.transmitted {
			fmt.Fprintf(bw, " loss=%.1f%%", float64(st.transmitted-st.received)*100.0/float64(st.transmitted))
		}
		fmt.Fprintf(bw, " rcvd=%d sent=%d op.err=%d icmp.err=%d", st.received, st.transmitted, st.opErrors, st.icmpErrors)
		fmt.Fprintf(bw, " min=%v avg=%v max=%v stddev=%v\n", st.minRTT, avg, st.maxRTT, time.Duration(stddev))
	}
	bw.Flush()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/Shopify/sarama"
	"github.com/mathpl/go-metrics"
	"github.com/mathpl/kafka_httpcat"
)

var flagConf = flag.String("c", "", "Location of configuration file.")

type Config struct {
	// Target urls
	Hosts []string

	//Context path
	ContextPath string

	//Headers to add to the request
	Headers map[string][]string

	//HTTP method to use
	Method string

	//Expected http response deoces
	ExpectedResponses []int

	//Broker
	BrokerList []string

	//Topic
	Topic string

	ConsumerGroup string
	ConsumerID    string

	Partitions  []int32
	StartOffset string
	BufferSize  int

	//PayloadSize sent between each Kafka commit
	OffsetCommitThreshold int64

	//OpentsdbReport
	MetricsReport string
}

func readConf(filename string) (conf *Config) {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatalf("Unable to open config file: %s", err)
	}
	defer f.Close()

	host, err := os.Hostname()
	if err != nil {
		log.Fatalf("Unable to get hostname: %s", err)
	}

	cid := fmt.Sprintf("%s-%d", host, os.Getpid())

	conf = &Config{StartOffset: "newest", BufferSize: 16, ConsumerID: cid, OffsetCommitThreshold: 1e3}

	md, err := toml.DecodeReader(f, conf)
	if err != nil {
		log.Fatal("Unable to parse config file: %s", err)
	}
	if u := md.Undecoded(); len(u) > 0 {
		log.Fatal("Extra keys in config file: %v", u)
	}

	log.Printf("ConsumerID: %s", conf.ConsumerID)

	return
}

func getPartitions(conf *Config, c sarama.Consumer) ([]int32, error) {
	if len(conf.Partitions) == 0 {
		return c.Partitions(conf.Topic)
	}

	return conf.Partitions, nil
}

func generateConsumerLag(r metrics.TaggedRegistry) {
	valsSent := make(map[string]int64, 0)
	valsHWM := make(map[string]int64, 0)

	fn := func(n string, tm metrics.StandardTaggedMetric) {
		switch n {
		case "kafka_httpcat.consumer.sent":
			if m, ok := tm.Metric.(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsSent[tm.Tags["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.high_water_mark":
			if m, ok := tm.Metric.(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsHWM[tm.Tags["partition"]] = m.Value()
			}
		}
	}

	r.Each(fn)

	for partition, sentOffset := range valsSent {
		if partitionHWM, ok := valsHWM[partition]; ok {
			i := r.GetOrRegister("consumer.offset_lag", metrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := partitionHWM - sentOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}
}

func main() {
	flag.Parse()
	conf := readConf(*flagConf)

	var initialOffset int64

	switch conf.StartOffset {
	case "oldest":
		initialOffset = sarama.OffsetOldest
	case "newest":
		initialOffset = sarama.OffsetNewest
	default:
		log.Fatal("offset should be `oldest` or `newest`")
	}

	metricsRegistry := metrics.NewPrefixedTaggedRegistry("kafka_httpcat", metrics.Tags{"topic": conf.Topic})
	metricsTsdb := metrics.TaggedOpenTSDBConfig{Addr: conf.MetricsReport, Registry: metricsRegistry, FlushInterval: 15 * time.Second, DurationUnit: time.Millisecond, Format: metrics.Json}

	log.Printf("Connecting to: %s", conf.BrokerList)

	saramaConfig := kafka_httpcat.GetDefaultSaramaConfig()

	c, err := sarama.NewConsumer(conf.BrokerList, saramaConfig)
	if err != nil {
		log.Fatalf("Failed to start consumer: %s", err)
	}

	partitionList, err := getPartitions(conf, c)
	if err != nil {
		log.Fatalf("Failed to get the list of partitions: %s", err)
	}

	var (
		messages = make(chan *sarama.ConsumerMessage, conf.BufferSize)
		closing  = make(chan struct{})
		wg       sync.WaitGroup
	)

	om := kafka_httpcat.NewOffsetManager(metricsRegistry, conf.BrokerList, partitionList, conf.Topic, conf.ConsumerGroup, conf.ConsumerID, initialOffset, conf.OffsetCommitThreshold)

	go func() {
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Kill, os.Interrupt)
		<-signals
		log.Println("Initiating shutdown of consumer...")
		close(closing)
	}()

	for _, partition := range partitionList {
		offset := om.GetCurrentOffset(partition)
		log.Printf("Starting consumer on topic %s partition %d offset %d", conf.Topic, partition, offset)
		pc, err := c.ConsumePartition(conf.Topic, partition, offset)
		if err != nil {
			log.Fatalf("Failed to start consumer for partition %d: %s", partition, err)
		}

		m := metrics.NewGauge()
		m.Update(pc.HighWaterMarkOffset())
		metricsRegistry.GetOrRegister("consumer.high_water_mark", metrics.Tags{"partition": fmt.Sprintf("%d", partition)}, m)

		go func(pc sarama.PartitionConsumer) {
			<-closing
			pc.AsyncClose()
		}(pc)

		wg.Add(1)
		go func(pc sarama.PartitionConsumer) {
			defer wg.Done()
			for message := range pc.Messages() {
				m.Update(pc.HighWaterMarkOffset())
				messages <- message
			}
		}(pc)
	}

	go metrics.TaggedOpenTSDBWithConfigAndPreprocessing(metricsTsdb, []func(metrics.TaggedRegistry){generateConsumerLag})

	go func() {
		for msg := range messages {
			sender := kafka_httpcat.NewHTTPSender(conf.Hosts, conf.ContextPath, conf.Method, conf.Headers, conf.ExpectedResponses)
			for {
				if err := sender.RRSend(msg.Value); err != nil {
					log.Printf("Error send data: %s", err)
				} else {
					break
				}
			}

			om.Add(msg.Partition, msg.Offset)
		}
	}()

	wg.Wait()
	log.Println("Done consuming topic", conf.Topic)
	close(messages)

	if err := c.Close(); err != nil {
		log.Println("Failed to close consumer: ", err)
	}

	om.CommitAll()
}
<commit_msg>Adding consumer group as tag.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/Shopify/sarama"
	"github.com/mathpl/go-metrics"
	"github.com/mathpl/kafka_httpcat"
)

var flagConf = flag.String("c", "", "Location of configuration file.")

type Config struct {
	// Target urls
	Hosts []string

	//Context path
	ContextPath string

	//Headers to add to the request
	Headers map[string][]string

	//HTTP method to use
	Method string

	//Expected http response deoces
	ExpectedResponses []int

	//Broker
	BrokerList []string

	//Topic
	Topic string

	ConsumerGroup string
	ConsumerID    string

	Partitions  []int32
	StartOffset string
	BufferSize  int

	//PayloadSize sent between each Kafka commit
	OffsetCommitThreshold int64

	//OpentsdbReport
	MetricsReport string
}

func readConf(filename string) (conf *Config) {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatalf("Unable to open config file: %s", err)
	}
	defer f.Close()

	host, err := os.Hostname()
	if err != nil {
		log.Fatalf("Unable to get hostname: %s", err)
	}

	cid := fmt.Sprintf("%s-%d", host, os.Getpid())

	conf = &Config{StartOffset: "newest", BufferSize: 16, ConsumerID: cid, OffsetCommitThreshold: 1e3}

	md, err := toml.DecodeReader(f, conf)
	if err != nil {
		log.Fatal("Unable to parse config file: %s", err)
	}
	if u := md.Undecoded(); len(u) > 0 {
		log.Fatal("Extra keys in config file: %v", u)
	}

	log.Printf("ConsumerID: %s", conf.ConsumerID)

	return
}

func getPartitions(conf *Config, c sarama.Consumer) ([]int32, error) {
	if len(conf.Partitions) == 0 {
		return c.Partitions(conf.Topic)
	}

	return conf.Partitions, nil
}

func generateConsumerLag(r metrics.TaggedRegistry) {
	valsSent := make(map[string]int64, 0)
	valsHWM := make(map[string]int64, 0)

	fn := func(n string, tm metrics.StandardTaggedMetric) {
		switch n {
		case "kafka_httpcat.consumer.sent":
			if m, ok := tm.Metric.(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsSent[tm.Tags["partition"]] = m.Value()
			}
		case "kafka_httpcat.consumer.high_water_mark":
			if m, ok := tm.Metric.(metrics.Gauge); !ok {
				log.Printf("Unexpect metric type.")
			} else {
				valsHWM[tm.Tags["partition"]] = m.Value()
			}
		}
	}

	r.Each(fn)

	for partition, sentOffset := range valsSent {
		if partitionHWM, ok := valsHWM[partition]; ok {
			i := r.GetOrRegister("consumer.offset_lag", metrics.Tags{"partition": partition}, metrics.NewGauge())
			if m, ok := i.(metrics.Gauge); ok {
				offsetLag := partitionHWM - sentOffset
				m.Update(offsetLag)
			} else {
				log.Print("Unexpected metric type")
			}
		}
	}
}

func main() {
	flag.Parse()
	conf := readConf(*flagConf)

	var initialOffset int64

	switch conf.StartOffset {
	case "oldest":
		initialOffset = sarama.OffsetOldest
	case "newest":
		initialOffset = sarama.OffsetNewest
	default:
		log.Fatal("offset should be `oldest` or `newest`")
	}

	metricsRegistry := metrics.NewPrefixedTaggedRegistry("kafka_httpcat", metrics.Tags{"topic": conf.Topic})
	metricsTsdb := metrics.TaggedOpenTSDBConfig{Addr: conf.MetricsReport, Registry: metricsRegistry, FlushInterval: 15 * time.Second, DurationUnit: time.Millisecond, Format: metrics.Json}

	log.Printf("Connecting to: %s", conf.BrokerList)

	saramaConfig := kafka_httpcat.GetDefaultSaramaConfig()

	c, err := sarama.NewConsumer(conf.BrokerList, saramaConfig)
	if err != nil {
		log.Fatalf("Failed to start consumer: %s", err)
	}

	partitionList, err := getPartitions(conf, c)
	if err != nil {
		log.Fatalf("Failed to get the list of partitions: %s", err)
	}

	var (
		messages = make(chan *sarama.ConsumerMessage, conf.BufferSize)
		closing  = make(chan struct{})
		wg       sync.WaitGroup
	)

	om := kafka_httpcat.NewOffsetManager(metricsRegistry, conf.BrokerList, partitionList, conf.Topic, conf.ConsumerGroup, conf.ConsumerID, initialOffset, conf.OffsetCommitThreshold)

	go func() {
		signals := make(chan os.Signal, 1)
		signal.Notify(signals, os.Kill, os.Interrupt)
		<-signals
		log.Println("Initiating shutdown of consumer...")
		close(closing)
	}()

	for _, partition := range partitionList {
		offset := om.GetCurrentOffset(partition)
		log.Printf("Starting consumer on topic %s partition %d offset %d", conf.Topic, partition, offset)
		pc, err := c.ConsumePartition(conf.Topic, partition, offset)
		if err != nil {
			log.Fatalf("Failed to start consumer for partition %d: %s", partition, err)
		}

		m := metrics.NewGauge()
		m.Update(pc.HighWaterMarkOffset())
		metricsRegistry.GetOrRegister("consumer.high_water_mark", metrics.Tags{"partition": fmt.Sprintf("%d", partition), "consumergroup": conf.ConsumerGroup}, m)

		go func(pc sarama.PartitionConsumer) {
			<-closing
			pc.AsyncClose()
		}(pc)

		wg.Add(1)
		go func(pc sarama.PartitionConsumer) {
			defer wg.Done()
			for message := range pc.Messages() {
				m.Update(pc.HighWaterMarkOffset())
				messages <- message
			}
		}(pc)
	}

	go metrics.TaggedOpenTSDBWithConfigAndPreprocessing(metricsTsdb, []func(metrics.TaggedRegistry){generateConsumerLag})

	go func() {
		for msg := range messages {
			sender := kafka_httpcat.NewHTTPSender(conf.Hosts, conf.ContextPath, conf.Method, conf.Headers, conf.ExpectedResponses)
			for {
				if err := sender.RRSend(msg.Value); err != nil {
					log.Printf("Error send data: %s", err)
				} else {
					break
				}
			}

			om.Add(msg.Partition, msg.Offset)
		}
	}()

	wg.Wait()
	log.Println("Done consuming topic", conf.Topic)
	close(messages)

	if err := c.Close(); err != nil {
		log.Println("Failed to close consumer: ", err)
	}

	om.CommitAll()
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 by Thorsten von Eicken, see LICENSE file for details

package main

import (
	"fmt"

	"github.com/tve/devices/sx1231"
	"github.com/tve/devices/varint"
)

// JeeLabs rfm69 ACK protocol

const group = 6 // this needs to come from config!

// jlAck takes a raw rfm69 packet using the JeeLabs format, looks whether it
// requests an ack and if so publishes one. The ACK contains the node ID as dest, the GW ID
// as source, a byte with the RX SNR (0..63), and a byte with the FEI/128.
// The top 2 bits of the rssi byte are effectively unused.
func jlAck(m *RawRxMessage, pub pubFunc, debug LogPrintf) {
	src, _, ack, payload, err := sx1231.JLDecode(group, m.Payload.Packet)
	if err != nil {
		debug("Can't decode JL packet: %s", err)
		return
	}
	if !ack {
		debug("no ACK needed")
		return // no ack requested
	}
	// Send an ack back.
	debug("ACK reply to node %d!", src)
	ackPkt := sx1231.MakeJLAck(group, payload)
	snr := m.Payload.Snr
	switch {
	case snr < 0:
		snr = 0
	case snr > 63:
		snr = 63
	}
	ackPkt = append(ackPkt, byte(snr), byte(m.Payload.Fei/128))
	txPkt := RawTxPacket{Packet: ackPkt}
	pub("", txPkt)
}

func init() {
	RegisterModule(module{"jl-ack", jlAck})
}

//=====

// jlDecode decodes a packet using the JeeLabs protocol and having a type byte as the first byte in
// the payload. It publishes to a topic by adding "/<type>".
func jlDecode(m *RawRxMessage, pub pubFunc, debug LogPrintf) {
	src, dst, ack, payload, err := sx1231.JLDecode(group, m.Payload.Packet)
	if err != nil {
		debug("Can't decode JL packet: %s", err)
		return
	}
	if len(payload) < 1 {
		return
	}
	txPkt := jlRxPacket{RawRxPacket: m.Payload, Src: src, Dst: dst, Ack: ack, Type: payload[0]}
	txPkt.Packet = payload[1:]
	topic := fmt.Sprintf("/%d", txPkt.Type)
	pub(topic, txPkt)
}

func init() {
	RegisterModule(module{"jl-decode", jlDecode})
}

type jlRxPacket struct {
	RawRxPacket
	Src  byte `json:"src"`
	Dst  byte `json:"dst"`
	Ack  bool `json:"ack"`
	Type byte `json:"type"`
}

type jlRxMessage struct {
	Topic   string
	Payload jlRxPacket
}

//=====

// jlviDecode decodes varints in the payload of a packet.
func jlviDecode(m *jlRxMessage, pub pubFunc, debug LogPrintf) {
	pub("", varintRxPacket{jlRxPacket: m.Payload, Data: varint.Decode(m.Payload.Packet)})
}

func init() {
	RegisterModule(module{"jl-varint", jlviDecode})
}

type varintRxPacket struct {
	jlRxPacket
	Data []int `json:"data"`
}
<commit_msg>improve documentation<commit_after>// Copyright (c) 2016 by Thorsten von Eicken, see LICENSE file for details

package main

import (
	"fmt"

	"github.com/tve/devices/sx1231"
	"github.com/tve/devices/varint"
)

//===== JeeLabs rfm69 ACK protocol

const group = 6 // FIXME: this needs to come from config!

// jlAck takes a raw rfm69 packet using the JeeLabs format, looks whether it
// requests an ack and if so publishes one. The ACK contains the node ID as dest, the GW ID
// as source, a byte with the RX SNR (0..63), and a byte with the FEI/128.
// The SNR is the difference between the received RSSI and the RSSI threshold
// configured into the radio (the driver adjusts this threshold dynamically).
// The top 2 bits of the rssi byte are effectively unused.
// TODO: if the RX RSSI wasn't measured it redults in a 0 SNR, we should omit the two
// ACK payload bytes instead.
func jlAck(m *RawRxMessage, pub pubFunc, debug LogPrintf) {
	src, _, ack, payload, err := sx1231.JLDecode(group, m.Payload.Packet)
	if err != nil {
		debug("Can't decode JL packet: %s", err)
		return
	}
	if !ack {
		debug("no ACK needed")
		return // no ack requested
	}
	// Send an ack back.
	debug("ACK reply to node %d!", src)
	ackPkt := sx1231.MakeJLAck(group, payload)
	snr := m.Payload.Snr
	switch {
	case snr < 0:
		snr = 0
	case snr > 63:
		snr = 63
	}
	ackPkt = append(ackPkt, byte(snr), byte(m.Payload.Fei/128))
	txPkt := RawTxPacket{Packet: ackPkt}
	pub("", txPkt)
}

func init() {
	RegisterModule(module{"jl-ack", jlAck})
}

//===== JeeLabs rfm69 packet decoder

// jlDecode decodes a packet using the JeeLabs protocol and having a type byte as the first byte in
// the payload. It publishes to a topic by adding "/<type>" to the configured publication topic.
// This is intended to allow further decoding by having modules subscribe to their packet type.
// The format of the packet published to MQTT is described by the jlRxPacket struct.
func jlDecode(m *RawRxMessage, pub pubFunc, debug LogPrintf) {
	src, dst, ack, payload, err := sx1231.JLDecode(group, m.Payload.Packet)
	if err != nil {
		debug("Can't decode JL packet: %s", err)
		return
	}
	if len(payload) < 1 {
		return
	}
	txPkt := jlRxPacket{RawRxPacket: m.Payload, Src: src, Dst: dst, Ack: ack, Type: payload[0]}
	txPkt.Packet = payload[1:]
	topic := fmt.Sprintf("/%d", txPkt.Type)
	pub(topic, txPkt)
}

func init() {
	RegisterModule(module{"jl-decode", jlDecode})
}

// jlRxPacket is the structure of the packets published to MQTT by the jl-decode module.
type jlRxPacket struct {
	RawRxPacket
	Src  byte `json:"src"`
	Dst  byte `json:"dst"`
	Ack  bool `json:"ack"`
	Type byte `json:"type"`
}

type jlRxMessage struct {
	Topic   string
	Payload jlRxPacket
}

//===== JeeLabs rfm69 varint decoder

// jlviDecode decodes varints in the payload of a packet. It expects a decoded packet whose
// paylaod consists entirely of varints.
func jlviDecode(m *jlRxMessage, pub pubFunc, debug LogPrintf) {
	pub("", varintRxPacket{jlRxPacket: m.Payload, Data: varint.Decode(m.Payload.Packet)})
}

func init() {
	RegisterModule(module{"jl-varint", jlviDecode})
}

// varintRxPacket is the structure of packets published to MQTT by the jl-varint decoder.
type varintRxPacket struct {
	jlRxPacket
	Data []int `json:"data"`
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/boltdb/bolt"
	"github.com/scjalliance/resourceful/guardian"
	"github.com/scjalliance/resourceful/lease"
	"github.com/scjalliance/resourceful/provider/boltprov"
	"github.com/scjalliance/resourceful/provider/cacheprov"
	"github.com/scjalliance/resourceful/provider/fsprov"
	"github.com/scjalliance/resourceful/provider/memprov"
)

const (
	defaultLeaseStorage = "memory"
	defaultBoltPath     = "resourceful.boltdb"
)

func daemon(command string, args []string) {
	prepareConsole(false)

	logger := log.New(os.Stderr, "", log.LstdFlags)

	var (
		leaseStorage = os.Getenv("LEASE_STORE")
		boltPath     = os.Getenv("BOLT_PATH")
	)

	if leaseStorage == "" {
		leaseStorage = defaultLeaseStorage
	}
	if boltPath == "" {
		boltPath = defaultBoltPath
	}

	fs := flag.NewFlagSet(command, flag.ExitOnError)
	fs.StringVar(&leaseStorage, "leasestore", leaseStorage, "lease storage type [\"bolt\", \"memory\"]")
	fs.StringVar(&boltPath, "boltpath", boltPath, "bolt database file path")
	fs.Parse(args)

	// Detect the working directory, which is the source of policy files
	wd, err := os.Getwd()
	if err != nil {
		logger.Printf("Unable to detect working directory: %v", err)
		os.Exit(2)
	}

	logger.Println("Starting resourceful guardian daemon")

	leaseProvider, err := createLeaseProvider(leaseStorage, boltPath)
	if err != nil {
		logger.Printf("Unable to create lease provider: %v", err)
		os.Exit(2)
	}

	policyProvider := cacheprov.New(fsprov.New(wd))

	cfg := guardian.ServerConfig{
		ListenSpec:      ":5877",
		PolicyProvider:  policyProvider,
		LeaseProvider:   leaseProvider,
		ShutdownTimeout: 5 * time.Second,
		Logger:          logger,
	}

	logger.Printf("Created providers (policy: %s, lease: %s)", policyProvider.ProviderName(), leaseProvider.ProviderName())

	logger.Printf("Policy source directory: %s\n", wd)
	// Verify that we're starting with a good policy set
	policies, err := cfg.PolicyProvider.Policies()
	if err != nil {
		logger.Printf("Failed to load policy set: %v", err)
		os.Exit(2)
	}

	count := len(policies)
	switch count {
	case 1:
		logger.Printf("1 policy loaded")
	default:
		logger.Printf("%d policies loaded", count)
	}

	ctx, shutdown := context.WithCancel(context.Background())
	defer shutdown()
	go func() {
		waitForSignal(logger)
		shutdown()
	}()

	err = guardian.Run(ctx, cfg)

	if provErr := leaseProvider.Close(); provErr != nil {
		logger.Printf("The lease provider did not shut down correctly: %v", provErr)
	}
	if provErr := policyProvider.Close(); provErr != nil {
		logger.Printf("The policy provider did not shut down correctly: %v", provErr)
	}

	if err != http.ErrServerClosed {
		logger.Printf("Stopped resourceful guardian daemon due to error: %v", err)
		os.Exit(2)
	}

	logger.Printf("Stopped resourceful guardian daemon")
}

func createLeaseProvider(storage string, boltPath string) (lease.Provider, error) {
	switch strings.ToLower(storage) {
	case "mem", "memory":
		return memprov.New(), nil
	case "bolt", "boltdb":
		boltdb, err := bolt.Open(boltPath, 0666, nil)
		if err != nil {
			return nil, fmt.Errorf("unable to open or create bolt database \"%s\": %v", boltPath, err)
		}
		return boltprov.New(boltdb), nil
	default:
		return nil, fmt.Errorf("unknown lease storage type: %s", storage)
	}
}

func waitForSignal(logger *log.Logger) {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	s := <-ch
	logger.Printf("Got signal: %v, exiting.", s)
}
<commit_msg>Added support for alternate policy directory paths<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/boltdb/bolt"
	"github.com/scjalliance/resourceful/guardian"
	"github.com/scjalliance/resourceful/lease"
	"github.com/scjalliance/resourceful/provider/boltprov"
	"github.com/scjalliance/resourceful/provider/cacheprov"
	"github.com/scjalliance/resourceful/provider/fsprov"
	"github.com/scjalliance/resourceful/provider/memprov"
)

const (
	defaultLeaseStorage = "memory"
	defaultBoltPath     = "resourceful.boltdb"
)

func daemon(command string, args []string) {
	prepareConsole(false)

	logger := log.New(os.Stderr, "", log.LstdFlags)

	var (
		leaseStorage = os.Getenv("LEASE_STORE")
		boltPath     = os.Getenv("BOLT_PATH")
		policyPath   = os.Getenv("POLICY_PATH")
		err          error
	)

	if leaseStorage == "" {
		leaseStorage = defaultLeaseStorage
	}
	if boltPath == "" {
		boltPath = defaultBoltPath
	}
	if policyPath == "" {
		// Use the working directory as the default source for policy files
		policyPath, err = os.Getwd()
		if err != nil {
			logger.Printf("Unable to detect working directory: %v", err)
			os.Exit(2)
		}
	}

	fs := flag.NewFlagSet(command, flag.ExitOnError)
	fs.StringVar(&leaseStorage, "leasestore", leaseStorage, "lease storage type [\"bolt\", \"memory\"]")
	fs.StringVar(&boltPath, "boltpath", boltPath, "bolt database file path")
	fs.StringVar(&policyPath, "policypath", policyPath, "policy directory path")
	fs.Parse(args)

	logger.Println("Starting resourceful guardian daemon")

	leaseProvider, err := createLeaseProvider(leaseStorage, boltPath)
	if err != nil {
		logger.Printf("Unable to create lease provider: %v", err)
		os.Exit(2)
	}

	policyProvider := cacheprov.New(fsprov.New(policyPath))

	cfg := guardian.ServerConfig{
		ListenSpec:      ":5877",
		PolicyProvider:  policyProvider,
		LeaseProvider:   leaseProvider,
		ShutdownTimeout: 5 * time.Second,
		Logger:          logger,
	}

	logger.Printf("Created providers (policy: %s, lease: %s)", policyProvider.ProviderName(), leaseProvider.ProviderName())

	logger.Printf("Policy source directory: %s\n", policyPath)
	// Verify that we're starting with a good policy set
	policies, err := cfg.PolicyProvider.Policies()
	if err != nil {
		logger.Printf("Failed to load policy set: %v", err)
		os.Exit(2)
	}

	count := len(policies)
	switch count {
	case 1:
		logger.Printf("1 policy loaded")
	default:
		logger.Printf("%d policies loaded", count)
	}

	ctx, shutdown := context.WithCancel(context.Background())
	defer shutdown()
	go func() {
		waitForSignal(logger)
		shutdown()
	}()

	err = guardian.Run(ctx, cfg)

	if provErr := leaseProvider.Close(); provErr != nil {
		logger.Printf("The lease provider did not shut down correctly: %v", provErr)
	}
	if provErr := policyProvider.Close(); provErr != nil {
		logger.Printf("The policy provider did not shut down correctly: %v", provErr)
	}

	if err != http.ErrServerClosed {
		logger.Printf("Stopped resourceful guardian daemon due to error: %v", err)
		os.Exit(2)
	}

	logger.Printf("Stopped resourceful guardian daemon")
}

func createLeaseProvider(storage string, boltPath string) (lease.Provider, error) {
	switch strings.ToLower(storage) {
	case "mem", "memory":
		return memprov.New(), nil
	case "bolt", "boltdb":
		boltdb, err := bolt.Open(boltPath, 0666, nil)
		if err != nil {
			return nil, fmt.Errorf("unable to open or create bolt database \"%s\": %v", boltPath, err)
		}
		return boltprov.New(boltdb), nil
	default:
		return nil, fmt.Errorf("unknown lease storage type: %s", storage)
	}
}

func waitForSignal(logger *log.Logger) {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
	s := <-ch
	logger.Printf("Got signal: %v, exiting.", s)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains tests for the printf checker.

package testdata

import (
	"fmt"
	"os"
	"unsafe" // just for test case printing unsafe.Pointer
)

func UnsafePointerPrintfTest() {
	var up unsafe.Pointer
	fmt.Printf("%p, %x %X", up, up, up)
}

// Error methods that do not satisfy the Error interface and should be checked.
type errorTest1 int

func (errorTest1) Error(...interface{}) string {
	return "hi"
}

type errorTest2 int // Analogous to testing's *T type.
func (errorTest2) Error(...interface{}) {
}

type errorTest3 int

func (errorTest3) Error() { // No return value.
}

type errorTest4 int

func (errorTest4) Error() int { // Different return type.
	return 3
}

type errorTest5 int

func (errorTest5) error() { // niladic; don't complain if no args (was bug)
}

// This function never executes, but it serves as a simple test for the program.
// Test with make test.
func PrintfTests() {
	var b bool
	var i int
	var r rune
	var s string
	var x float64
	var p *int
	var imap map[int]int
	var fslice []float64
	var c complex64
	// Some good format/argtypes
	fmt.Printf("")
	fmt.Printf("%b %b %b", 3, i, x)
	fmt.Printf("%c %c %c %c", 3, i, 'x', r)
	fmt.Printf("%d %d %d", 3, i, imap)
	fmt.Printf("%e %e %e %e", 3e9, x, fslice, c)
	fmt.Printf("%E %E %E %E", 3e9, x, fslice, c)
	fmt.Printf("%f %f %f %f", 3e9, x, fslice, c)
	fmt.Printf("%F %F %F %F", 3e9, x, fslice, c)
	fmt.Printf("%g %g %g %g", 3e9, x, fslice, c)
	fmt.Printf("%G %G %G %G", 3e9, x, fslice, c)
	fmt.Printf("%o %o", 3, i)
	fmt.Printf("%p %p", p, nil)
	fmt.Printf("%q %q %q %q", 3, i, 'x', r)
	fmt.Printf("%s %s %s", "hi", s, []byte{65})
	fmt.Printf("%t %t", true, b)
	fmt.Printf("%T %T", 3, i)
	fmt.Printf("%U %U", 3, i)
	fmt.Printf("%v %v", 3, i)
	fmt.Printf("%x %x %x %x", 3, i, "hi", s)
	fmt.Printf("%X %X %X %X", 3, i, "hi", s)
	fmt.Printf("%.*s %d %g", 3, "hi", 23, 2.3)
	fmt.Printf("%s", &stringerv)
	fmt.Printf("%v", &stringerv)
	fmt.Printf("%T", &stringerv)
	fmt.Printf("%v", notstringerv)
	fmt.Printf("%T", notstringerv)
	fmt.Printf("%q", stringerarrayv)
	fmt.Printf("%v", stringerarrayv)
	fmt.Printf("%s", stringerarrayv)
	fmt.Printf("%v", notstringerarrayv)
	fmt.Printf("%T", notstringerarrayv)
	fmt.Printf("%*%", 2)               // Ridiculous but allowed.
	fmt.Printf("%s", interface{}(nil)) // Nothing useful we can say.

	fmt.Printf("%g", 1+2i)
	// Some bad format/argTypes
	fmt.Printf("%b", "hi")                     // ERROR "arg .hi. for printf verb %b of wrong type"
	fmt.Printf("%b", c)                        // ERROR "arg c for printf verb %b of wrong type"
	fmt.Printf("%b", 1+2i)                     // ERROR "arg 1 \+ 2i for printf verb %b of wrong type"
	fmt.Printf("%c", 2.3)                      // ERROR "arg 2.3 for printf verb %c of wrong type"
	fmt.Printf("%d", 2.3)                      // ERROR "arg 2.3 for printf verb %d of wrong type"
	fmt.Printf("%e", "hi")                     // ERROR "arg .hi. for printf verb %e of wrong type"
	fmt.Printf("%E", true)                     // ERROR "arg true for printf verb %E of wrong type"
	fmt.Printf("%f", "hi")                     // ERROR "arg .hi. for printf verb %f of wrong type"
	fmt.Printf("%F", 'x')                      // ERROR "arg 'x' for printf verb %F of wrong type"
	fmt.Printf("%g", "hi")                     // ERROR "arg .hi. for printf verb %g of wrong type"
	fmt.Printf("%g", imap)                     // ERROR "arg imap for printf verb %g of wrong type"
	fmt.Printf("%G", i)                        // ERROR "arg i for printf verb %G of wrong type"
	fmt.Printf("%o", x)                        // ERROR "arg x for printf verb %o of wrong type"
	fmt.Printf("%p", 23)                       // ERROR "arg 23 for printf verb %p of wrong type"
	fmt.Printf("%q", x)                        // ERROR "arg x for printf verb %q of wrong type"
	fmt.Printf("%s", b)                        // ERROR "arg b for printf verb %s of wrong type"
	fmt.Printf("%s", byte(65))                 // ERROR "arg byte\(65\) for printf verb %s of wrong type"
	fmt.Printf("%t", 23)                       // ERROR "arg 23 for printf verb %t of wrong type"
	fmt.Printf("%U", x)                        // ERROR "arg x for printf verb %U of wrong type"
	fmt.Printf("%x", nil)                      // ERROR "arg nil for printf verb %x of wrong type"
	fmt.Printf("%X", 2.3)                      // ERROR "arg 2.3 for printf verb %X of wrong type"
	fmt.Printf("%s", stringerv)                // ERROR "arg stringerv for printf verb %s of wrong type"
	fmt.Printf("%t", stringerv)                // ERROR "arg stringerv for printf verb %t of wrong type"
	fmt.Printf("%q", notstringerv)             // ERROR "arg notstringerv for printf verb %q of wrong type"
	fmt.Printf("%t", notstringerv)             // ERROR "arg notstringerv for printf verb %t of wrong type"
	fmt.Printf("%t", stringerarrayv)           // ERROR "arg stringerarrayv for printf verb %t of wrong type"
	fmt.Printf("%t", notstringerarrayv)        // ERROR "arg notstringerarrayv for printf verb %t of wrong type"
	fmt.Printf("%q", notstringerarrayv)        // ERROR "arg notstringerarrayv for printf verb %q of wrong type"
	fmt.Printf("%s", nonemptyinterface)        // NOTERROR "for printf verb %s of wrong type" (Disabled temporarily because of bug in IsAssignableTo)
	fmt.Printf("%.*s %d %g", 3, "hi", 23, 'x') // ERROR "arg 'x' for printf verb %g of wrong type"
	fmt.Println()                              // not an error
	fmt.Println("%s", "hi")                    // ERROR "possible formatting directive in Println call"
	fmt.Printf("%s", "hi", 3)                  // ERROR "wrong number of args for format in Printf call"
	fmt.Sprintf("%"+("s"), "hi", 3)            // ERROR "wrong number of args for format in Sprintf call"
	fmt.Printf("%s%%%d", "hi", 3)              // correct
	fmt.Printf("%08s", "woo")                  // correct
	fmt.Printf("% 8s", "woo")                  // correct
	fmt.Printf("%.*d", 3, 3)                   // correct
	fmt.Printf("%.*d", 3, 3, 3)                // ERROR "wrong number of args for format in Printf call"
	fmt.Printf("%.*d", "hi", 3)                // ERROR "arg .hi. for \* in printf format not of type int"
	fmt.Printf("%.*d", i, 3)                   // correct
	fmt.Printf("%.*d", s, 3)                   // ERROR "arg s for \* in printf format not of type int"
	fmt.Printf("%*%", 0.22)                    // ERROR "arg 0.22 for \* in printf format not of type int"
	fmt.Printf("%q %q", multi()...)            // ok
	fmt.Printf("%#q", `blah`)                  // ok
	printf("now is the time", "buddy")         // ERROR "no formatting directive"
	Printf("now is the time", "buddy")         // ERROR "no formatting directive"
	Printf("hi")                               // ok
	const format = "%s %s\n"
	Printf(format, "hi", "there")
	Printf(format, "hi") // ERROR "missing argument for Printf verb %s: need 2, have 1"
	f := new(stringer)
	f.Warn(0, "%s", "hello", 3)  // ERROR "possible formatting directive in Warn call"
	f.Warnf(0, "%s", "hello", 3) // ERROR "wrong number of args for format in Warnf call"
	f.Warnf(0, "%r", "hello")    // ERROR "unrecognized printf verb"
	f.Warnf(0, "%#s", "hello")   // ERROR "unrecognized printf flag"
	Printf("d%", 2)              // ERROR "missing verb at end of format string in Printf call"
	Printf("%d", percentDV)
	Printf("%d", &percentDV)
	Printf("%d", notPercentDV)  // ERROR "arg notPercentDV for printf verb %d of wrong type"
	Printf("%d", &notPercentDV) // ERROR "arg &notPercentDV for printf verb %d of wrong type"
	Printf("%p", &notPercentDV) // Works regardless: we print it as a pointer.
	Printf("%s", percentSV)
	Printf("%s", &percentSV)
	// Good argument reorderings.
	Printf("%[1]d", 3)
	Printf("%[1]*d", 3, 1)
	Printf("%[2]*[1]d", 1, 3)
	Printf("%[2]*.[1]*[3]d", 2, 3, 4)
	fmt.Fprintf(os.Stderr, "%[2]*.[1]*[3]d", 2, 3, 4) // Use Fprintf to make sure we count arguments correctly.
	// Bad argument reorderings.
	Printf("%[xd", 3)                    // ERROR "illegal syntax for printf argument index"
	Printf("%[x]d", 3)                   // ERROR "illegal syntax for printf argument index"
	Printf("%[3]*s", "hi", 2)            // ERROR "missing argument for Printf indirect \*: need 3, have 2"
	fmt.Sprintf("%[3]d", 2)              // ERROR "missing argument for Sprintf verb %d: need 3, have 1"
	Printf("%[2]*.[1]*[3]d", 2, "hi", 4) // ERROR "arg .hi. for \* in printf format not of type int"
	// Something that satisfies the error interface.
	var e error
	fmt.Println(e.Error()) // ok
	// Something that looks like an error interface but isn't, such as the (*T).Error method
	// in the testing package.
	var et1 errorTest1
	fmt.Println(et1.Error())        // ERROR "no args in Error call"
	fmt.Println(et1.Error("hi"))    // ok
	fmt.Println(et1.Error("%d", 3)) // ERROR "possible formatting directive in Error call"
	var et2 errorTest2
	et2.Error()        // ERROR "no args in Error call"
	et2.Error("hi")    // ok, not an error method.
	et2.Error("%d", 3) // ERROR "possible formatting directive in Error call"
	var et3 errorTest3
	et3.Error() // ok, not an error method.
	var et4 errorTest4
	et4.Error() // ok, not an error method.
	var et5 errorTest5
	et5.error() // ok, not an error method.
}

// Printf is used by the test so we must declare it.
func Printf(format string, args ...interface{}) {
	panic("don't call - testing only")
}

// printf is used by the test so we must declare it.
func printf(format string, args ...interface{}) {
	panic("don't call - testing only")
}

// multi is used by the test.
func multi() []interface{} {
	panic("don't call - testing only")
}

type stringer float64

var stringerv stringer

func (*stringer) String() string {
	return "string"
}

func (*stringer) Warn(int, ...interface{}) string {
	return "warn"
}

func (*stringer) Warnf(int, string, ...interface{}) string {
	return "warnf"
}

type notstringer struct {
	f float64
}

var notstringerv notstringer

type stringerarray [4]float64

func (stringerarray) String() string {
	return "string"
}

var stringerarrayv stringerarray

type notstringerarray [4]float64

var notstringerarrayv notstringerarray

var nonemptyinterface = interface {
	f()
}(nil)

// A data type we can print with "%d".
type percentDStruct struct {
	a int
	b []byte
	c *float64
}

var percentDV percentDStruct

// A data type we cannot print correctly with "%d".
type notPercentDStruct struct {
	a int
	b []byte
	c bool
}

var notPercentDV notPercentDStruct

// A data type we can print with "%s".
type percentSStruct struct {
	a string
	b []byte
	c stringerarray
}

var percentSV percentSStruct
<commit_msg>go.tools/cmd/vet: enable test (fix build)<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains tests for the printf checker.

package testdata

import (
	"fmt"
	"os"
	"unsafe" // just for test case printing unsafe.Pointer
)

func UnsafePointerPrintfTest() {
	var up unsafe.Pointer
	fmt.Printf("%p, %x %X", up, up, up)
}

// Error methods that do not satisfy the Error interface and should be checked.
type errorTest1 int

func (errorTest1) Error(...interface{}) string {
	return "hi"
}

type errorTest2 int // Analogous to testing's *T type.
func (errorTest2) Error(...interface{}) {
}

type errorTest3 int

func (errorTest3) Error() { // No return value.
}

type errorTest4 int

func (errorTest4) Error() int { // Different return type.
	return 3
}

type errorTest5 int

func (errorTest5) error() { // niladic; don't complain if no args (was bug)
}

// This function never executes, but it serves as a simple test for the program.
// Test with make test.
func PrintfTests() {
	var b bool
	var i int
	var r rune
	var s string
	var x float64
	var p *int
	var imap map[int]int
	var fslice []float64
	var c complex64
	// Some good format/argtypes
	fmt.Printf("")
	fmt.Printf("%b %b %b", 3, i, x)
	fmt.Printf("%c %c %c %c", 3, i, 'x', r)
	fmt.Printf("%d %d %d", 3, i, imap)
	fmt.Printf("%e %e %e %e", 3e9, x, fslice, c)
	fmt.Printf("%E %E %E %E", 3e9, x, fslice, c)
	fmt.Printf("%f %f %f %f", 3e9, x, fslice, c)
	fmt.Printf("%F %F %F %F", 3e9, x, fslice, c)
	fmt.Printf("%g %g %g %g", 3e9, x, fslice, c)
	fmt.Printf("%G %G %G %G", 3e9, x, fslice, c)
	fmt.Printf("%o %o", 3, i)
	fmt.Printf("%p %p", p, nil)
	fmt.Printf("%q %q %q %q", 3, i, 'x', r)
	fmt.Printf("%s %s %s", "hi", s, []byte{65})
	fmt.Printf("%t %t", true, b)
	fmt.Printf("%T %T", 3, i)
	fmt.Printf("%U %U", 3, i)
	fmt.Printf("%v %v", 3, i)
	fmt.Printf("%x %x %x %x", 3, i, "hi", s)
	fmt.Printf("%X %X %X %X", 3, i, "hi", s)
	fmt.Printf("%.*s %d %g", 3, "hi", 23, 2.3)
	fmt.Printf("%s", &stringerv)
	fmt.Printf("%v", &stringerv)
	fmt.Printf("%T", &stringerv)
	fmt.Printf("%v", notstringerv)
	fmt.Printf("%T", notstringerv)
	fmt.Printf("%q", stringerarrayv)
	fmt.Printf("%v", stringerarrayv)
	fmt.Printf("%s", stringerarrayv)
	fmt.Printf("%v", notstringerarrayv)
	fmt.Printf("%T", notstringerarrayv)
	fmt.Printf("%*%", 2)               // Ridiculous but allowed.
	fmt.Printf("%s", interface{}(nil)) // Nothing useful we can say.

	fmt.Printf("%g", 1+2i)
	// Some bad format/argTypes
	fmt.Printf("%b", "hi")                     // ERROR "arg .hi. for printf verb %b of wrong type"
	fmt.Printf("%b", c)                        // ERROR "arg c for printf verb %b of wrong type"
	fmt.Printf("%b", 1+2i)                     // ERROR "arg 1 \+ 2i for printf verb %b of wrong type"
	fmt.Printf("%c", 2.3)                      // ERROR "arg 2.3 for printf verb %c of wrong type"
	fmt.Printf("%d", 2.3)                      // ERROR "arg 2.3 for printf verb %d of wrong type"
	fmt.Printf("%e", "hi")                     // ERROR "arg .hi. for printf verb %e of wrong type"
	fmt.Printf("%E", true)                     // ERROR "arg true for printf verb %E of wrong type"
	fmt.Printf("%f", "hi")                     // ERROR "arg .hi. for printf verb %f of wrong type"
	fmt.Printf("%F", 'x')                      // ERROR "arg 'x' for printf verb %F of wrong type"
	fmt.Printf("%g", "hi")                     // ERROR "arg .hi. for printf verb %g of wrong type"
	fmt.Printf("%g", imap)                     // ERROR "arg imap for printf verb %g of wrong type"
	fmt.Printf("%G", i)                        // ERROR "arg i for printf verb %G of wrong type"
	fmt.Printf("%o", x)                        // ERROR "arg x for printf verb %o of wrong type"
	fmt.Printf("%p", 23)                       // ERROR "arg 23 for printf verb %p of wrong type"
	fmt.Printf("%q", x)                        // ERROR "arg x for printf verb %q of wrong type"
	fmt.Printf("%s", b)                        // ERROR "arg b for printf verb %s of wrong type"
	fmt.Printf("%s", byte(65))                 // ERROR "arg byte\(65\) for printf verb %s of wrong type"
	fmt.Printf("%t", 23)                       // ERROR "arg 23 for printf verb %t of wrong type"
	fmt.Printf("%U", x)                        // ERROR "arg x for printf verb %U of wrong type"
	fmt.Printf("%x", nil)                      // ERROR "arg nil for printf verb %x of wrong type"
	fmt.Printf("%X", 2.3)                      // ERROR "arg 2.3 for printf verb %X of wrong type"
	fmt.Printf("%s", stringerv)                // ERROR "arg stringerv for printf verb %s of wrong type"
	fmt.Printf("%t", stringerv)                // ERROR "arg stringerv for printf verb %t of wrong type"
	fmt.Printf("%q", notstringerv)             // ERROR "arg notstringerv for printf verb %q of wrong type"
	fmt.Printf("%t", notstringerv)             // ERROR "arg notstringerv for printf verb %t of wrong type"
	fmt.Printf("%t", stringerarrayv)           // ERROR "arg stringerarrayv for printf verb %t of wrong type"
	fmt.Printf("%t", notstringerarrayv)        // ERROR "arg notstringerarrayv for printf verb %t of wrong type"
	fmt.Printf("%q", notstringerarrayv)        // ERROR "arg notstringerarrayv for printf verb %q of wrong type"
	fmt.Printf("%s", nonemptyinterface)        // ERROR "for printf verb %s of wrong type" (Disabled temporarily because of bug in IsAssignableTo)
	fmt.Printf("%.*s %d %g", 3, "hi", 23, 'x') // ERROR "arg 'x' for printf verb %g of wrong type"
	fmt.Println()                              // not an error
	fmt.Println("%s", "hi")                    // ERROR "possible formatting directive in Println call"
	fmt.Printf("%s", "hi", 3)                  // ERROR "wrong number of args for format in Printf call"
	fmt.Sprintf("%"+("s"), "hi", 3)            // ERROR "wrong number of args for format in Sprintf call"
	fmt.Printf("%s%%%d", "hi", 3)              // correct
	fmt.Printf("%08s", "woo")                  // correct
	fmt.Printf("% 8s", "woo")                  // correct
	fmt.Printf("%.*d", 3, 3)                   // correct
	fmt.Printf("%.*d", 3, 3, 3)                // ERROR "wrong number of args for format in Printf call"
	fmt.Printf("%.*d", "hi", 3)                // ERROR "arg .hi. for \* in printf format not of type int"
	fmt.Printf("%.*d", i, 3)                   // correct
	fmt.Printf("%.*d", s, 3)                   // ERROR "arg s for \* in printf format not of type int"
	fmt.Printf("%*%", 0.22)                    // ERROR "arg 0.22 for \* in printf format not of type int"
	fmt.Printf("%q %q", multi()...)            // ok
	fmt.Printf("%#q", `blah`)                  // ok
	printf("now is the time", "buddy")         // ERROR "no formatting directive"
	Printf("now is the time", "buddy")         // ERROR "no formatting directive"
	Printf("hi")                               // ok
	const format = "%s %s\n"
	Printf(format, "hi", "there")
	Printf(format, "hi") // ERROR "missing argument for Printf verb %s: need 2, have 1"
	f := new(stringer)
	f.Warn(0, "%s", "hello", 3)  // ERROR "possible formatting directive in Warn call"
	f.Warnf(0, "%s", "hello", 3) // ERROR "wrong number of args for format in Warnf call"
	f.Warnf(0, "%r", "hello")    // ERROR "unrecognized printf verb"
	f.Warnf(0, "%#s", "hello")   // ERROR "unrecognized printf flag"
	Printf("d%", 2)              // ERROR "missing verb at end of format string in Printf call"
	Printf("%d", percentDV)
	Printf("%d", &percentDV)
	Printf("%d", notPercentDV)  // ERROR "arg notPercentDV for printf verb %d of wrong type"
	Printf("%d", &notPercentDV) // ERROR "arg &notPercentDV for printf verb %d of wrong type"
	Printf("%p", &notPercentDV) // Works regardless: we print it as a pointer.
	Printf("%s", percentSV)
	Printf("%s", &percentSV)
	// Good argument reorderings.
	Printf("%[1]d", 3)
	Printf("%[1]*d", 3, 1)
	Printf("%[2]*[1]d", 1, 3)
	Printf("%[2]*.[1]*[3]d", 2, 3, 4)
	fmt.Fprintf(os.Stderr, "%[2]*.[1]*[3]d", 2, 3, 4) // Use Fprintf to make sure we count arguments correctly.
	// Bad argument reorderings.
	Printf("%[xd", 3)                    // ERROR "illegal syntax for printf argument index"
	Printf("%[x]d", 3)                   // ERROR "illegal syntax for printf argument index"
	Printf("%[3]*s", "hi", 2)            // ERROR "missing argument for Printf indirect \*: need 3, have 2"
	fmt.Sprintf("%[3]d", 2)              // ERROR "missing argument for Sprintf verb %d: need 3, have 1"
	Printf("%[2]*.[1]*[3]d", 2, "hi", 4) // ERROR "arg .hi. for \* in printf format not of type int"
	// Something that satisfies the error interface.
	var e error
	fmt.Println(e.Error()) // ok
	// Something that looks like an error interface but isn't, such as the (*T).Error method
	// in the testing package.
	var et1 errorTest1
	fmt.Println(et1.Error())        // ERROR "no args in Error call"
	fmt.Println(et1.Error("hi"))    // ok
	fmt.Println(et1.Error("%d", 3)) // ERROR "possible formatting directive in Error call"
	var et2 errorTest2
	et2.Error()        // ERROR "no args in Error call"
	et2.Error("hi")    // ok, not an error method.
	et2.Error("%d", 3) // ERROR "possible formatting directive in Error call"
	var et3 errorTest3
	et3.Error() // ok, not an error method.
	var et4 errorTest4
	et4.Error() // ok, not an error method.
	var et5 errorTest5
	et5.error() // ok, not an error method.
}

// Printf is used by the test so we must declare it.
func Printf(format string, args ...interface{}) {
	panic("don't call - testing only")
}

// printf is used by the test so we must declare it.
func printf(format string, args ...interface{}) {
	panic("don't call - testing only")
}

// multi is used by the test.
func multi() []interface{} {
	panic("don't call - testing only")
}

type stringer float64

var stringerv stringer

func (*stringer) String() string {
	return "string"
}

func (*stringer) Warn(int, ...interface{}) string {
	return "warn"
}

func (*stringer) Warnf(int, string, ...interface{}) string {
	return "warnf"
}

type notstringer struct {
	f float64
}

var notstringerv notstringer

type stringerarray [4]float64

func (stringerarray) String() string {
	return "string"
}

var stringerarrayv stringerarray

type notstringerarray [4]float64

var notstringerarrayv notstringerarray

var nonemptyinterface = interface {
	f()
}(nil)

// A data type we can print with "%d".
type percentDStruct struct {
	a int
	b []byte
	c *float64
}

var percentDV percentDStruct

// A data type we cannot print correctly with "%d".
type notPercentDStruct struct {
	a int
	b []byte
	c bool
}

var notPercentDV notPercentDStruct

// A data type we can print with "%s".
type percentSStruct struct {
	a string
	b []byte
	c stringerarray
}

var percentSV percentSStruct
<|endoftext|>


<commit_before>package render

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"

	cmd_helm "helm.sh/helm/v3/cmd/helm"
	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/chart"
	"helm.sh/helm/v3/pkg/chart/loader"
	"helm.sh/helm/v3/pkg/cli/values"

	"github.com/werf/logboek"
	"github.com/werf/logboek/pkg/level"

	"github.com/werf/werf/cmd/werf/common"
	"github.com/werf/werf/pkg/build"
	"github.com/werf/werf/pkg/container_runtime"
	"github.com/werf/werf/pkg/deploy/helm"
	"github.com/werf/werf/pkg/deploy/helm/chart_extender"
	"github.com/werf/werf/pkg/deploy/secrets_manager"
	"github.com/werf/werf/pkg/docker"
	"github.com/werf/werf/pkg/git_repo"
	"github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/ssh_agent"
	"github.com/werf/werf/pkg/storage"
	"github.com/werf/werf/pkg/storage/manager"
	"github.com/werf/werf/pkg/tmp_manager"
	"github.com/werf/werf/pkg/true_git"
	"github.com/werf/werf/pkg/werf"
	"github.com/werf/werf/pkg/werf/global_warnings"
)

var cmdData struct {
	Timeout      int
	AutoRollback bool
	RenderOutput string
}

var commonCmdData common.CmdData

func NewCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:                   "render",
		Short:                 "Render Kubernetes templates",
		Long:                  common.GetLongCommandDescription(`Render Kubernetes templates. This command will calculate digests and build (if needed) all images defined in the werf.yaml.`),
		DisableFlagsInUseLine: true,
		Annotations: map[string]string{
			common.CmdEnvAnno: common.EnvsDescription(common.WerfDebugAnsibleArgs, common.WerfSecretKey),
		},
		RunE: func(cmd *cobra.Command, args []string) error {
			defer global_warnings.PrintGlobalWarnings(common.BackgroundContext())

			logboek.Streams().Mute()
			logboek.SetAcceptedLevel(level.Error)

			if err := common.ProcessLogOptionsDefaultQuiet(&commonCmdData); err != nil {
				common.PrintHelp(cmd)
				return err
			}

			common.LogVersion()

			return common.LogRunningTime(runRender)
		},
	}

	common.SetupDir(&commonCmdData, cmd)
	common.SetupGitWorkTree(&commonCmdData, cmd)
	common.SetupConfigTemplatesDir(&commonCmdData, cmd)
	common.SetupConfigPath(&commonCmdData, cmd)
	common.SetupEnvironment(&commonCmdData, cmd)

	common.SetupGiterminismInspectorOptions(&commonCmdData, cmd)

	common.SetupTmpDir(&commonCmdData, cmd)
	common.SetupHomeDir(&commonCmdData, cmd)
	common.SetupSSHKey(&commonCmdData, cmd)

	common.SetupIntrospectAfterError(&commonCmdData, cmd)
	common.SetupIntrospectBeforeError(&commonCmdData, cmd)
	common.SetupIntrospectStage(&commonCmdData, cmd)

	common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd)
	common.SetupStagesStorageOptions(&commonCmdData, cmd)

	common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images")
	common.SetupInsecureRegistry(&commonCmdData, cmd)
	common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd)

	common.SetupLogOptionsDefaultQuiet(&commonCmdData, cmd)
	common.SetupLogProjectDir(&commonCmdData, cmd)

	common.SetupSynchronization(&commonCmdData, cmd)

	common.SetupRelease(&commonCmdData, cmd)
	common.SetupNamespace(&commonCmdData, cmd)
	common.SetupAddAnnotations(&commonCmdData, cmd)
	common.SetupAddLabels(&commonCmdData, cmd)

	common.SetupSetDockerConfigJsonValue(&commonCmdData, cmd)
	common.SetupSet(&commonCmdData, cmd)
	common.SetupSetString(&commonCmdData, cmd)
	common.SetupSetFile(&commonCmdData, cmd)
	common.SetupValues(&commonCmdData, cmd)
	common.SetupSecretValues(&commonCmdData, cmd)
	common.SetupIgnoreSecretKey(&commonCmdData, cmd)

	common.SetupReportPath(&commonCmdData, cmd)
	common.SetupReportFormat(&commonCmdData, cmd)

	common.SetupVirtualMerge(&commonCmdData, cmd)
	common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
	common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

	common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)

	common.SetupSkipBuild(&commonCmdData, cmd)

	cmd.Flags().IntVarP(&cmdData.Timeout, "timeout", "t", 0, "Resources tracking timeout in seconds")
	cmd.Flags().BoolVarP(&cmdData.AutoRollback, "auto-rollback", "R", common.GetBoolEnvironmentDefaultFalse("WERF_AUTO_ROLLBACK"), "Enable auto rollback of the failed release to the previous deployed release version when current deploy process have failed ($WERF_AUTO_ROLLBACK by default)")
	cmd.Flags().BoolVarP(&cmdData.AutoRollback, "atomic", "", common.GetBoolEnvironmentDefaultFalse("WERF_ATOMIC"), "Enable auto rollback of the failed release to the previous deployed release version when current deploy process have failed ($WERF_ATOMIC by default)")

	cmd.Flags().StringVarP(&cmdData.RenderOutput, "output", "", os.Getenv("WERF_RENDER_OUTPUT"), "Write render output to the specified file instead of stdout ($WERF_RENDER_OUTPUT by default)")

	return cmd
}

func runRender() error {
	ctx := common.BackgroundContext()

	if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
		return fmt.Errorf("initialization error: %s", err)
	}

	if err := git_repo.Init(); err != nil {
		return err
	}

	if err := image.Init(); err != nil {
		return err
	}

	if err := true_git.Init(true_git.Options{LiveGitOutput: *commonCmdData.LogVerbose || *commonCmdData.LogDebug}); err != nil {
		return err
	}

	if err := common.DockerRegistryInit(&commonCmdData); err != nil {
		return err
	}

	if err := docker.Init(ctx, *commonCmdData.DockerConfig, *commonCmdData.LogVerbose, *commonCmdData.LogDebug); err != nil {
		return err
	}

	ctxWithDockerCli, err := docker.NewContext(ctx)
	if err != nil {
		return err
	}
	ctx = ctxWithDockerCli

	giterminismManager, err := common.GetGiterminismManager(&commonCmdData)
	if err != nil {
		return err
	}

	common.ProcessLogProjectDir(&commonCmdData, giterminismManager.ProjectDir())

	werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, true))
	if err != nil {
		return fmt.Errorf("unable to load werf config: %s", err)
	}

	projectName := werfConfig.Meta.Project

	chartDir, err := common.GetHelmChartDir(werfConfig, giterminismManager)
	if err != nil {
		return fmt.Errorf("getting helm chart dir failed: %s", err)
	}

	projectTmpDir, err := tmp_manager.CreateProjectDir(ctx)
	if err != nil {
		return fmt.Errorf("getting project tmp dir failed: %s", err)
	}
	defer tmp_manager.ReleaseProjectDir(projectTmpDir)

	if err := ssh_agent.Init(ctx, common.GetSSHKey(&commonCmdData)); err != nil {
		return fmt.Errorf("cannot initialize ssh agent: %s", err)
	}
	defer func() {
		err := ssh_agent.Terminate()
		if err != nil {
			logboek.Warn().LogF("WARNING: ssh agent termination failed: %s\n", err)
		}
	}()

	releaseName, err := common.GetHelmRelease(*commonCmdData.Release, *commonCmdData.Environment, werfConfig)
	if err != nil {
		return err
	}

	namespace, err := common.GetKubernetesNamespace(*commonCmdData.Namespace, *commonCmdData.Environment, werfConfig)
	if err != nil {
		return err
	}

	userExtraAnnotations, err := common.GetUserExtraAnnotations(&commonCmdData)
	if err != nil {
		return err
	}

	userExtraLabels, err := common.GetUserExtraLabels(&commonCmdData)
	if err != nil {
		return err
	}

	buildOptions, err := common.GetBuildOptions(&commonCmdData, werfConfig)
	if err != nil {
		return err
	}

	logboek.LogOptionalLn()

	var imagesInfoGetters []*image.InfoGetter
	var imagesRepository string
	var isStub bool

	if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 {
		stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData)

		if stagesStorageAddress != storage.LocalStorageAddress {
			containerRuntime := &container_runtime.LocalDockerServerRuntime{} // TODO
			stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerRuntime, &commonCmdData)
			if err != nil {
				return err
			}
			synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage)
			if err != nil {
				return err
			}
			stagesStorageCache, err := common.GetStagesStorageCache(synchronization)
			if err != nil {
				return err
			}
			storageLockManager, err := common.GetStorageLockManager(ctx, synchronization)
			if err != nil {
				return err
			}
			secondaryStagesStorageList, err := common.GetSecondaryStagesStorageList(stagesStorage, containerRuntime, &commonCmdData)
			if err != nil {
				return err
			}

			storageManager := manager.NewStorageManager(projectName, stagesStorage, secondaryStagesStorageList, storageLockManager, stagesStorageCache)

			imagesRepository = storageManager.StagesStorage.String()

			conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions)
			if err != nil {
				return err
			}

			conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerRuntime, storageManager, storageLockManager, conveyorOptions)
			defer conveyorWithRetry.Terminate()

			if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
				if *commonCmdData.SkipBuild {
					if err := c.ShouldBeBuilt(ctx); err != nil {
						return err
					}
				} else {
					if err := c.Build(ctx, buildOptions); err != nil {
						return err
					}
				}

				imagesInfoGetters = c.GetImageInfoGetters()

				return nil
			}); err != nil {
				return err
			}

			logboek.LogOptionalLn()
		} else {
			imagesRepository = "REPO"
			isStub = true
		}
	}

	secretsManager := secrets_manager.NewSecretsManager(giterminismManager.ProjectDir(), secrets_manager.SecretsManagerOptions{DisableSecretsDecryption: *commonCmdData.IgnoreSecretKey})

	wc := chart_extender.NewWerfChart(ctx, giterminismManager, secretsManager, chartDir, cmd_helm.Settings, chart_extender.WerfChartOptions{
		SecretValueFiles: common.GetSecretValues(&commonCmdData),
		ExtraAnnotations: userExtraAnnotations,
		ExtraLabels:      userExtraLabels,
	})

	if err := wc.SetEnv(*commonCmdData.Environment); err != nil {
		return err
	}
	if err := wc.SetWerfConfig(werfConfig); err != nil {
		return err
	}
	if vals, err := chart_extender.GetServiceValues(ctx, werfConfig.Meta.Project, imagesRepository, imagesInfoGetters, chart_extender.ServiceValuesOptions{Namespace: namespace, Env: *commonCmdData.Environment, IsStub: isStub}); err != nil {
		return fmt.Errorf("error creating service values: %s", err)
	} else if err := wc.SetServiceValues(vals); err != nil {
		return err
	}

	if *commonCmdData.SetDockerConfigJsonValue {
		if err := chart_extender.WriteDockerConfigJsonValue(ctx, wc.GetExtraValues(), *commonCmdData.DockerConfig); err != nil {
			return fmt.Errorf("error writing docker config value into werf chart extra values: %s", err)
		}
	}

	actionConfig := new(action.Configuration)
	if err := helm.InitActionConfig(ctx, nil, namespace, cmd_helm.Settings, actionConfig, helm.InitActionConfigOptions{}); err != nil {
		return err
	}

	var output io.Writer
	if cmdData.RenderOutput != "" {
		if f, err := os.Create(cmdData.RenderOutput); err != nil {
			return fmt.Errorf("unable to open file %q: %s", cmdData.RenderOutput, err)
		} else {
			defer f.Close()
			output = f
		}
	} else {
		output = os.Stdout
	}

	cmd_helm.Settings.Debug = *commonCmdData.LogDebug

	loader.GlobalLoadOptions = &loader.LoadOptions{
		ChartExtender:               wc,
		SubchartExtenderFactoryFunc: func() chart.ChartExtender { return chart_extender.NewWerfSubchart() },
	}

	postRenderer, err := wc.GetPostRenderer()
	if err != nil {
		return err
	}

	helmTemplateCmd, _ := cmd_helm.NewTemplateCmd(actionConfig, output, cmd_helm.TemplateCmdOptions{
		PostRenderer: postRenderer,
		ValueOpts: &values.Options{
			ValueFiles:   common.GetValues(&commonCmdData),
			StringValues: common.GetSetString(&commonCmdData),
			Values:       common.GetSet(&commonCmdData),
			FileValues:   common.GetSetFile(&commonCmdData),
		},
	})
	return helmTemplateCmd.RunE(helmTemplateCmd, []string{releaseName, filepath.Join(giterminismManager.ProjectDir(), chartDir)})
}
<commit_msg>Add --validate and include-crds opts for werf-render cmd<commit_after>package render

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"

	cmd_helm "helm.sh/helm/v3/cmd/helm"
	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/chart"
	"helm.sh/helm/v3/pkg/chart/loader"
	"helm.sh/helm/v3/pkg/cli/values"

	"github.com/werf/logboek"
	"github.com/werf/logboek/pkg/level"

	"github.com/werf/werf/cmd/werf/common"
	"github.com/werf/werf/pkg/build"
	"github.com/werf/werf/pkg/container_runtime"
	"github.com/werf/werf/pkg/deploy/helm"
	"github.com/werf/werf/pkg/deploy/helm/chart_extender"
	"github.com/werf/werf/pkg/deploy/secrets_manager"
	"github.com/werf/werf/pkg/docker"
	"github.com/werf/werf/pkg/git_repo"
	"github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/ssh_agent"
	"github.com/werf/werf/pkg/storage"
	"github.com/werf/werf/pkg/storage/manager"
	"github.com/werf/werf/pkg/tmp_manager"
	"github.com/werf/werf/pkg/true_git"
	"github.com/werf/werf/pkg/werf"
	"github.com/werf/werf/pkg/werf/global_warnings"
)

var cmdData struct {
	RenderOutput string
	Validate     bool
	IncludeCRDs  bool
}

var commonCmdData common.CmdData

func NewCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:                   "render",
		Short:                 "Render Kubernetes templates",
		Long:                  common.GetLongCommandDescription(`Render Kubernetes templates. This command will calculate digests and build (if needed) all images defined in the werf.yaml.`),
		DisableFlagsInUseLine: true,
		Annotations: map[string]string{
			common.CmdEnvAnno: common.EnvsDescription(common.WerfDebugAnsibleArgs, common.WerfSecretKey),
		},
		RunE: func(cmd *cobra.Command, args []string) error {
			defer global_warnings.PrintGlobalWarnings(common.BackgroundContext())

			logboek.Streams().Mute()
			logboek.SetAcceptedLevel(level.Error)

			if err := common.ProcessLogOptionsDefaultQuiet(&commonCmdData); err != nil {
				common.PrintHelp(cmd)
				return err
			}

			common.LogVersion()

			return common.LogRunningTime(runRender)
		},
	}

	common.SetupDir(&commonCmdData, cmd)
	common.SetupGitWorkTree(&commonCmdData, cmd)
	common.SetupConfigTemplatesDir(&commonCmdData, cmd)
	common.SetupConfigPath(&commonCmdData, cmd)
	common.SetupEnvironment(&commonCmdData, cmd)

	common.SetupGiterminismInspectorOptions(&commonCmdData, cmd)

	common.SetupTmpDir(&commonCmdData, cmd)
	common.SetupHomeDir(&commonCmdData, cmd)
	common.SetupSSHKey(&commonCmdData, cmd)

	common.SetupIntrospectAfterError(&commonCmdData, cmd)
	common.SetupIntrospectBeforeError(&commonCmdData, cmd)
	common.SetupIntrospectStage(&commonCmdData, cmd)

	common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd)
	common.SetupStagesStorageOptions(&commonCmdData, cmd)

	common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images")
	common.SetupInsecureRegistry(&commonCmdData, cmd)
	common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd)

	common.SetupLogOptionsDefaultQuiet(&commonCmdData, cmd)
	common.SetupLogProjectDir(&commonCmdData, cmd)

	common.SetupSynchronization(&commonCmdData, cmd)

	common.SetupRelease(&commonCmdData, cmd)
	common.SetupNamespace(&commonCmdData, cmd)
	common.SetupAddAnnotations(&commonCmdData, cmd)
	common.SetupAddLabels(&commonCmdData, cmd)

	common.SetupSetDockerConfigJsonValue(&commonCmdData, cmd)
	common.SetupSet(&commonCmdData, cmd)
	common.SetupSetString(&commonCmdData, cmd)
	common.SetupSetFile(&commonCmdData, cmd)
	common.SetupValues(&commonCmdData, cmd)
	common.SetupSecretValues(&commonCmdData, cmd)
	common.SetupIgnoreSecretKey(&commonCmdData, cmd)

	common.SetupReportPath(&commonCmdData, cmd)
	common.SetupReportFormat(&commonCmdData, cmd)

	common.SetupVirtualMerge(&commonCmdData, cmd)
	common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
	common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

	common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)

	common.SetupSkipBuild(&commonCmdData, cmd)

	cmd.Flags().BoolVarP(&cmdData.Validate, "validate", "", common.GetBoolEnvironmentDefaultFalse("WERF_VALIDATE"), "Validate your manifests against the Kubernetes cluster you are currently pointing at (default $WERF_VALIDATE)")
	cmd.Flags().BoolVarP(&cmdData.IncludeCRDs, "include-crds", "", common.GetBoolEnvironmentDefaultTrue("WERF_INCLUDE_CRDS"), "Include CRDs in the templated output (default $WERF_INCLUDE_CRDS)")

	cmd.Flags().StringVarP(&cmdData.RenderOutput, "output", "", os.Getenv("WERF_RENDER_OUTPUT"), "Write render output to the specified file instead of stdout ($WERF_RENDER_OUTPUT by default)")

	return cmd
}

func runRender() error {
	ctx := common.BackgroundContext()

	if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
		return fmt.Errorf("initialization error: %s", err)
	}

	if err := git_repo.Init(); err != nil {
		return err
	}

	if err := image.Init(); err != nil {
		return err
	}

	if err := true_git.Init(true_git.Options{LiveGitOutput: *commonCmdData.LogVerbose || *commonCmdData.LogDebug}); err != nil {
		return err
	}

	if err := common.DockerRegistryInit(&commonCmdData); err != nil {
		return err
	}

	if err := docker.Init(ctx, *commonCmdData.DockerConfig, *commonCmdData.LogVerbose, *commonCmdData.LogDebug); err != nil {
		return err
	}

	ctxWithDockerCli, err := docker.NewContext(ctx)
	if err != nil {
		return err
	}
	ctx = ctxWithDockerCli

	giterminismManager, err := common.GetGiterminismManager(&commonCmdData)
	if err != nil {
		return err
	}

	common.ProcessLogProjectDir(&commonCmdData, giterminismManager.ProjectDir())

	werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, true))
	if err != nil {
		return fmt.Errorf("unable to load werf config: %s", err)
	}

	projectName := werfConfig.Meta.Project

	chartDir, err := common.GetHelmChartDir(werfConfig, giterminismManager)
	if err != nil {
		return fmt.Errorf("getting helm chart dir failed: %s", err)
	}

	projectTmpDir, err := tmp_manager.CreateProjectDir(ctx)
	if err != nil {
		return fmt.Errorf("getting project tmp dir failed: %s", err)
	}
	defer tmp_manager.ReleaseProjectDir(projectTmpDir)

	if err := ssh_agent.Init(ctx, common.GetSSHKey(&commonCmdData)); err != nil {
		return fmt.Errorf("cannot initialize ssh agent: %s", err)
	}
	defer func() {
		err := ssh_agent.Terminate()
		if err != nil {
			logboek.Warn().LogF("WARNING: ssh agent termination failed: %s\n", err)
		}
	}()

	releaseName, err := common.GetHelmRelease(*commonCmdData.Release, *commonCmdData.Environment, werfConfig)
	if err != nil {
		return err
	}

	namespace, err := common.GetKubernetesNamespace(*commonCmdData.Namespace, *commonCmdData.Environment, werfConfig)
	if err != nil {
		return err
	}

	userExtraAnnotations, err := common.GetUserExtraAnnotations(&commonCmdData)
	if err != nil {
		return err
	}

	userExtraLabels, err := common.GetUserExtraLabels(&commonCmdData)
	if err != nil {
		return err
	}

	buildOptions, err := common.GetBuildOptions(&commonCmdData, werfConfig)
	if err != nil {
		return err
	}

	logboek.LogOptionalLn()

	var imagesInfoGetters []*image.InfoGetter
	var imagesRepository string
	var isStub bool

	if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 {
		stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData)

		if stagesStorageAddress != storage.LocalStorageAddress {
			containerRuntime := &container_runtime.LocalDockerServerRuntime{} // TODO
			stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerRuntime, &commonCmdData)
			if err != nil {
				return err
			}
			synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage)
			if err != nil {
				return err
			}
			stagesStorageCache, err := common.GetStagesStorageCache(synchronization)
			if err != nil {
				return err
			}
			storageLockManager, err := common.GetStorageLockManager(ctx, synchronization)
			if err != nil {
				return err
			}
			secondaryStagesStorageList, err := common.GetSecondaryStagesStorageList(stagesStorage, containerRuntime, &commonCmdData)
			if err != nil {
				return err
			}

			storageManager := manager.NewStorageManager(projectName, stagesStorage, secondaryStagesStorageList, storageLockManager, stagesStorageCache)

			imagesRepository = storageManager.StagesStorage.String()

			conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions)
			if err != nil {
				return err
			}

			conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerRuntime, storageManager, storageLockManager, conveyorOptions)
			defer conveyorWithRetry.Terminate()

			if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error {
				if *commonCmdData.SkipBuild {
					if err := c.ShouldBeBuilt(ctx); err != nil {
						return err
					}
				} else {
					if err := c.Build(ctx, buildOptions); err != nil {
						return err
					}
				}

				imagesInfoGetters = c.GetImageInfoGetters()

				return nil
			}); err != nil {
				return err
			}

			logboek.LogOptionalLn()
		} else {
			imagesRepository = "REPO"
			isStub = true
		}
	}

	secretsManager := secrets_manager.NewSecretsManager(giterminismManager.ProjectDir(), secrets_manager.SecretsManagerOptions{DisableSecretsDecryption: *commonCmdData.IgnoreSecretKey})

	wc := chart_extender.NewWerfChart(ctx, giterminismManager, secretsManager, chartDir, cmd_helm.Settings, chart_extender.WerfChartOptions{
		SecretValueFiles: common.GetSecretValues(&commonCmdData),
		ExtraAnnotations: userExtraAnnotations,
		ExtraLabels:      userExtraLabels,
	})

	if err := wc.SetEnv(*commonCmdData.Environment); err != nil {
		return err
	}
	if err := wc.SetWerfConfig(werfConfig); err != nil {
		return err
	}
	if vals, err := chart_extender.GetServiceValues(ctx, werfConfig.Meta.Project, imagesRepository, imagesInfoGetters, chart_extender.ServiceValuesOptions{Namespace: namespace, Env: *commonCmdData.Environment, IsStub: isStub}); err != nil {
		return fmt.Errorf("error creating service values: %s", err)
	} else if err := wc.SetServiceValues(vals); err != nil {
		return err
	}

	if *commonCmdData.SetDockerConfigJsonValue {
		if err := chart_extender.WriteDockerConfigJsonValue(ctx, wc.GetExtraValues(), *commonCmdData.DockerConfig); err != nil {
			return fmt.Errorf("error writing docker config value into werf chart extra values: %s", err)
		}
	}

	actionConfig := new(action.Configuration)
	if err := helm.InitActionConfig(ctx, nil, namespace, cmd_helm.Settings, actionConfig, helm.InitActionConfigOptions{}); err != nil {
		return err
	}

	var output io.Writer
	if cmdData.RenderOutput != "" {
		if f, err := os.Create(cmdData.RenderOutput); err != nil {
			return fmt.Errorf("unable to open file %q: %s", cmdData.RenderOutput, err)
		} else {
			defer f.Close()
			output = f
		}
	} else {
		output = os.Stdout
	}

	cmd_helm.Settings.Debug = *commonCmdData.LogDebug

	loader.GlobalLoadOptions = &loader.LoadOptions{
		ChartExtender:               wc,
		SubchartExtenderFactoryFunc: func() chart.ChartExtender { return chart_extender.NewWerfSubchart() },
	}

	postRenderer, err := wc.GetPostRenderer()
	if err != nil {
		return err
	}

	helmTemplateCmd, _ := cmd_helm.NewTemplateCmd(actionConfig, output, cmd_helm.TemplateCmdOptions{
		PostRenderer: postRenderer,
		ValueOpts: &values.Options{
			ValueFiles:   common.GetValues(&commonCmdData),
			StringValues: common.GetSetString(&commonCmdData),
			Values:       common.GetSet(&commonCmdData),
			FileValues:   common.GetSetFile(&commonCmdData),
		},
		Validate:    &cmdData.Validate,
		IncludeCrds: &cmdData.IncludeCRDs,
	})
	return helmTemplateCmd.RunE(helmTemplateCmd, []string{releaseName, filepath.Join(giterminismManager.ProjectDir(), chartDir)})
}
<|endoftext|>