<commit_before>// Copyright 2018 The Go Cloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package driver defines a set of interfaces that the pubsub package uses to
// interact with the underlying pubsub services.
package driver

import (
	"context"
)

// Batcher should gather items into batches to be sent to the pubsub service.
type Batcher interface {
	// Add should add an item to the batcher.
	Add(ctx context.Context, item interface{}) error

	// AddNoWait should add an item to the batcher without blocking.
	AddNoWait(item interface{}) <-chan error

	// Shutdown should wait for all active calls to Add to finish, then
	// return. After Shutdown is called, all calls to Add should fail.
	Shutdown()
}

// AckID is the identifier of a message for purposes of acknowledgement.
type AckID interface{}

// Message is data to be published (sent) to a topic and later received from
// subscriptions on that topic.
type Message struct {
	// Body contains the content of the message.
	Body []byte

	// Metadata has key/value pairs describing the message.
	Metadata map[string]string

	// AckID should be set to something identifying the message on the
	// server. It may be passed to Subscription.SendAcks() to acknowledge
	// the message. This field should only be set by methods implementing
	// Subscription.ReceiveBatch.
	AckID AckID
}

// Topic publishes messages.
type Topic interface {
	// SendBatch publishes all the messages in ms. This method should
	// return only after all the messages are sent, an error occurs, or the
	// context is done.
	//
	// Only the Body and (optionally) Metadata fields of the Messages in ms
	// should be set by the caller of SendBatch.
	//
	// Only one RPC should be made to send the messages, and the returned
	// error should be based on the result of that RPC. Implementations
	// that send only one message at a time should return a non-nil error
	// if len(ms) != 1.
	//
	// If there is a transient failure, this method should not retry but should
	// return an error. The concrete API will take care of retry logic.
	//
	// The slice ms should not be retained past the end of the call to
	// SendBatch.
	//
	// SendBatch should be safe for concurrent access from multiple goroutines.
	SendBatch(ctx context.Context, ms []*Message) error

	// IsRetryable should report whether err can be retried.
	// err will always be a non-nil error returned from SendBatch.
	IsRetryable(err error) bool
}

// Subscription receives published messages.
type Subscription interface {
	// ReceiveBatch should return a batch of messages that have queued up
	// for the subscription on the server, up to maxMessages.
	//
	// If there is a transient failure, this method should not retry but
	// should return a nil slice and an error. The concrete API will take
	// care of retry logic.
	//
	// If the service returns no messages for some other reason, this
	// method should return the empty slice of messages and not attempt to
	// retry.
	//
	// Implementations of ReceiveBatch should request that the underlying
	// service wait some non-zero amount of time before returning, if there
	// are no messages yet.
	//
	// ReceiveBatch should be safe for concurrent access from multiple goroutines.
	ReceiveBatch(ctx context.Context, maxMessages int) ([]*Message, error)

	// SendAcks should acknowledge the messages with the given ackIDs on
	// the server so that they will not be received again for this
	// subscription if the server gets the acks before their deadlines.
	// This method should return only after all the ackIDs are sent, an
	// error occurs, or the context is done.
	//
	// Only one RPC should be made to send the messages, and the returned
	// error should be based on the result of that RPC.  Implementations
	// that send only one ack at a time should return a non-nil error if
	// len(ackIDs) != 1.
	//
	// SendAcks should be safe for concurrent access from multiple goroutines.
	SendAcks(ctx context.Context, ackIDs []AckID) error

	// IsRetryable should report whether err can be retried.
	// err will always be a non-nil error returned from ReceiveBatch or SendAcks.
	IsRetryable(err error) bool
}
<commit_msg>pubsub: clarify and expand docstring+semantics for SendBatch (#849)<commit_after>// Copyright 2018 The Go Cloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package driver defines a set of interfaces that the pubsub package uses to
// interact with the underlying pubsub services.
package driver

import (
	"context"
)

// Batcher should gather items into batches to be sent to the pubsub service.
type Batcher interface {
	// Add should add an item to the batcher.
	Add(ctx context.Context, item interface{}) error

	// AddNoWait should add an item to the batcher without blocking.
	AddNoWait(item interface{}) <-chan error

	// Shutdown should wait for all active calls to Add to finish, then
	// return. After Shutdown is called, all calls to Add should fail.
	Shutdown()
}

// AckID is the identifier of a message for purposes of acknowledgement.
type AckID interface{}

// Message is data to be published (sent) to a topic and later received from
// subscriptions on that topic.
type Message struct {
	// Body contains the content of the message.
	Body []byte

	// Metadata has key/value pairs describing the message.
	Metadata map[string]string

	// AckID should be set to something identifying the message on the
	// server. It may be passed to Subscription.SendAcks() to acknowledge
	// the message. This field should only be set by methods implementing
	// Subscription.ReceiveBatch.
	AckID AckID
}

// Topic publishes messages.
type Topic interface {
	// SendBatch publishes all the messages in ms. This method should
	// return only after all the messages are sent, an error occurs, or the
	// context is done.
	//
	// Only the Body and (optionally) Metadata fields of the Messages in ms
	// will be set by the caller of SendBatch.
	//
	// If any message in the batch fails to send, SendBatch should return an
	// error.
	//
	// If there is a transient failure, this method should not retry but
	// should return an error for which IsRetryable returns true. The
	// concrete API takes care of retry logic.
	//
	// The slice ms should not be retained past the end of the call to
	// SendBatch.
	//
	// SendBatch may be called concurrently from multiple goroutines.
	SendBatch(ctx context.Context, ms []*Message) error

	// IsRetryable should report whether err can be retried.
	// err will always be a non-nil error returned from SendBatch.
	IsRetryable(err error) bool
}

// Subscription receives published messages.
type Subscription interface {
	// ReceiveBatch should return a batch of messages that have queued up
	// for the subscription on the server, up to maxMessages.
	//
	// If there is a transient failure, this method should not retry but
	// should return a nil slice and an error. The concrete API will take
	// care of retry logic.
	//
	// If the service returns no messages for some other reason, this
	// method should return the empty slice of messages and not attempt to
	// retry.
	//
	// Implementations of ReceiveBatch should request that the underlying
	// service wait some non-zero amount of time before returning, if there
	// are no messages yet.
	//
	// ReceiveBatch should be safe for concurrent access from multiple goroutines.
	ReceiveBatch(ctx context.Context, maxMessages int) ([]*Message, error)

	// SendAcks should acknowledge the messages with the given ackIDs on
	// the server so that they will not be received again for this
	// subscription if the server gets the acks before their deadlines.
	// This method should return only after all the ackIDs are sent, an
	// error occurs, or the context is done.
	//
	// Only one RPC should be made to send the messages, and the returned
	// error should be based on the result of that RPC.  Implementations
	// that send only one ack at a time should return a non-nil error if
	// len(ackIDs) != 1.
	//
	// SendAcks should be safe for concurrent access from multiple goroutines.
	SendAcks(ctx context.Context, ackIDs []AckID) error

	// IsRetryable should report whether err can be retried.
	// err will always be a non-nil error returned from ReceiveBatch or SendAcks.
	IsRetryable(err error) bool
}
<|endoftext|>


<commit_before>// myLG is command line looking glass that written with Go language
// it tries from its own icmp and external looking glasses tools
package main

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/briandowns/spinner"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/mehrdadrad/mylg/disc"
	"github.com/mehrdadrad/mylg/http/ping"
	"github.com/mehrdadrad/mylg/icmp"
	"github.com/mehrdadrad/mylg/lg"
	"github.com/mehrdadrad/mylg/ns"
	"github.com/mehrdadrad/mylg/packet"
	"github.com/mehrdadrad/mylg/peeringdb"
	"github.com/mehrdadrad/mylg/scan"
	"github.com/mehrdadrad/mylg/services/httpd"
	"github.com/mehrdadrad/mylg/whois"
)

const (
	version = "0.2.4"
)

// Provider represents looking glass
type Provider interface {
	Set(host, version string)
	GetDefaultNode() string
	GetNodes() []string
	ChangeNode(node string) bool
	Ping() (string, error)
	Trace() chan string
	BGP() chan string
}

var (
	pNames = providerNames()
	req    = make(chan string, 1)
	nxt    = make(chan struct{}, 1)
	spin   = spinner.New(spinner.CharSets[26], 220*time.Millisecond)
	eArgs  = os.Args
	args   string
	prompt string
	cPName string
	noIf   bool = true
	cfg    cli.Config
	nsr    *ns.Request
	c      *cli.Readline

	// register looking glass hosts
	providers = map[string]Provider{
		"telia":  new(lg.Telia),  // telia
		"level3": new(lg.Level3), // level3
		"cogent": new(lg.Cogent), //cogent
	}

	// map cmd to function
	cmdFunc = map[string]func(){
		"web":     web,          // web dashboard
		"dump":    dump,         // dump traffic
		"disc":    discovery,    // network discovery
		"scan":    scanPorts,    // network scan
		"mode":    mode,         // editor mode
		"ping":    pingQuery,    // ping
		"trace":   trace,        // trace route
		"bgp":     BGP,          // BGP
		"whois":   whoisLookup,  // whois / dns lookup
		"peering": peeringDB,    // peering DB
		"hping":   hping,        // hping
		"dig":     dig,          // dig
		"node":    node,         // change node
		"connect": connect,      // connect to a country or LG
		"local":   local,        // local
		"help":    help,         // help
		"exit":    cleanUp,      // clean up
		"quit":    cleanUp,      // clean up
		"show":    show,         // show config
		"set":     setConfig,    // set config
		"lg":      setLG,        // prepare looking glass
		"ns":      setNS,        // prepare name server
		"version": printVersion, // prints version
	}
)

// init
func init() {
	// load configuration
	cfg = cli.LoadConfig()
	// with interface
	if len(eArgs) == 1 {
		// initialize cli
		c = cli.Init("local", version)
		go c.Run(req, nxt)
		// start web server
		go httpd.Run(cfg)
		// set interface enabled
		noIf = false
	}
	// initialize name server
	nsr = ns.NewRequest()
	go nsr.Init()
	// set default provider, promot
	cPName = "local"
	prompt = "local"
}

func main() {
	// command line w/o interface
	if noIf {
		cmd := eArgs[1]
		args = strings.Join(eArgs[2:], " ")
		if f, ok := cmdFunc[cmd]; ok {
			f()
		} else {
			println("Invalid command please try mylg help")
		}
		return
	}
	// command like w/ interface
LOOP:
	for {
		select {
		case request, ok := <-req:
			if !ok {
				break LOOP
			}
			if len(request) < 1 {
				c.Next()
				continue
			}
			subReq := cli.CMDReg.FindStringSubmatch(request)
			if len(subReq) == 0 {
				println("syntax error")
				c.Next()
				continue
			}
			prompt = c.GetPrompt()
			args = strings.TrimSpace(subReq[2])
			cmd := strings.TrimSpace(subReq[1])
			if f, ok := cmdFunc[cmd]; ok {
				f()
			} else {
				println("Invalid command please try help")
			}
			c.Next()
		}
	}
}

// providerName
func providerNames() []string {
	pNames := []string{}
	for p := range providers {
		pNames = append(pNames, p)
	}
	return pNames
}

// validateProvider
func validateProvider(p string) (string, error) {
	pNames := []string{}
	match, _ := regexp.MatchString("("+strings.Join(pNames, "|")+")", p)
	p = strings.ToLower(p)
	if match {
		return p, nil
	}
	return "", errors.New("provider not support")

}

// node handles node cmd
func node() {
	switch {
	case strings.HasPrefix(prompt, "lg"):
		if _, ok := providers[cPName]; ok {
			if providers[cPName].ChangeNode(args) {
				c.UpdatePromptN(args, 3)
				return
			}
		}
		println("the specified node doesn't support")
	case strings.HasPrefix(prompt, "ns"):
		if !nsr.ChkNode(args) {
			println("error: argument is not valid")
		} else {
			c.UpdatePromptN(args, 3)
		}
	default:
		if cPName == "local" {
			println("local doesn't support node")
		}

	}
}

// dig gets dig info
func dig() {
	if ok := nsr.SetOptions(args, prompt); ok {
		nsr.Dig()
	}
}

// web tries to open web interface at default web browser
func web() {
	var openCmd = "open"
	println("opening default web broswer ...")
	if runtime.GOOS != "darwin" {
		openCmd = "xdg-open"
	}
	cmd := exec.Command(openCmd, fmt.Sprintf("http://%s:%d", cfg.Web.Address, cfg.Web.Port))
	err := cmd.Start()
	if err != nil {
		println("error opening default browser")
	}

}

// dump provides decoding packets
func dump() {
	p, err := packet.NewPacket(args)
	if p == nil || err != nil {
		return
	}
	println(p.Banner())
	for l := range p.Open() {
		l.PrintPretty()
	}
}

// connect handles connect cmd
func connect() {
	var (
		pName string
		err   error
	)
	switch {
	case strings.HasPrefix(prompt, "lg"):
		if pName, err = validateProvider(args); err != nil {
			println("provider not available")
			c.Next()
			return
		}
		cPName = pName
		if _, ok := providers[cPName]; ok {
			c.UpdatePromptN(cPName+"/"+providers[cPName].GetDefaultNode(), 2)
			go func() {
				c.UpdateCompleter("node", providers[cPName].GetNodes())
			}()
		} else {
			println("it doesn't support")
		}
	case strings.HasPrefix(prompt, "ns"):
		if !nsr.ChkCountry(args) {
			println("error: argument is not valid")
		} else {
			c.SetPrompt("ns/" + args)
			c.UpdateCompleter("node", nsr.NodeList())
		}
	}
}

// mode set editor mode
func mode() {
	if args == "vim" {
		c.SetVim()
	} else if args == "emacs" {
		c.SetEmacs()
	} else {
		println("the request mode doesn't support")
	}
}

// trace tries to trace from local and lg
func trace() {
	switch {
	case strings.HasPrefix(prompt, "local"):
		trace, err := icmp.NewTrace(args, cfg)
		if err != nil {
			println(err.Error())
		}
		if trace == nil {
			break
		}
		trace.Print()
	case strings.HasPrefix(prompt, "lg"):
		providers[cPName].Set(args, "ipv4")
		for l := range providers[cPName].Trace() {
			println(l)
		}
	}
}

// hping tries to ping a web server by http
func hping() {
	// it should work at local mode
	if cPName != "local" {
		return
	}
	p, err := ping.NewPing(args, cfg)
	if err != nil {
		println(err.Error())
	} else {
		p.Run()
	}
}

// pingQuery runs ping command (local/LG)
func pingQuery() {
	if cPName == "local" {
		pingLocal()
	} else {
		pingLG()
	}
}

// pingLG tries to ping through a looking glass
func pingLG() {
	spin.Prefix = "please wait "
	spin.Start()
	providers[cPName].Set(args, "ipv4")
	m, err := providers[cPName].Ping()
	spin.Stop()
	if err != nil {
		println(err.Error())
	} else {
		println(m)
	}
}

// pingLocal tries to ping from local source ip
func pingLocal() {
	p, err := icmp.NewPing(args, cfg)
	if err != nil {
		println(err.Error())
	}
	if p == nil {
		return
	}
	if !p.IsCIDR() {
		resp := p.Run()
		p.PrintPretty(resp)
	}
}

// scanPorts tries to scan tcp/ip ports
func scanPorts() {
	scan, err := scan.NewScan(args, cfg)
	if err != nil {
		println(err.Error())
	} else {
		scan.Run()
	}
}

// BGP tries to get BGP lookup from a LG
func BGP() {
	if cPName == "local" {
		println("no provider selected")
		return
	}
	providers[cPName].Set(args, "ipv4")
	for l := range providers[cPName].BGP() {
		println(l)
	}
}

// discovery handles disc command
func discovery() {
	var (
		wg sync.WaitGroup
		//ts = time.Now()
	)

	d := disc.New(args)
	// help requested
	if d == nil {
		return
	}

	spin.Prefix = "please wait "
	spin.Start()

	// load OUI async
	go func() {
		wg.Add(1)
		d.LoadOUI()
		wg.Done()
	}()
	d.PingLan()
	//s := 10 - int(time.Since(ts).Seconds())
	time.Sleep(5 * time.Second)
	if err := d.GetARPTable(); err != nil {
		println(err.Error())
		return
	}
	wg.Wait()
	spin.Stop()

	println("Network LAN Discovery")
	d.PrintPretty()
}

// setConfig
func setConfig() {
	cli.SetConfig(args, &cfg)
}

// showConfig
func show() {
	if args == "config" {
		cli.ShowConfig(&cfg)
	}
}

// setLG set lg prompt and completer
func setLG() {
	c.SetPrompt("lg")
	c.UpdateCompleter("connect", pNames)
}

// setNS set ns prompt and update completers
func setNS() {
	c.UpdateCompleter("connect", nsr.CountryList())
	c.UpdateCompleter("node", []string{})
	c.SetPrompt("ns")
}

// peeringDB gets peer info
func peeringDB() {
	peeringdb.Search(args)
}

// whoisLookup gets ANS/Prefix info
func whoisLookup() {
	whois.Lookup(args)
}

// local set prompts to local
func local() {
	nsr.Local()
	cPName = "local"
	c.SetPrompt(cPName)
}

// printVersion prints version and exits
func printVersion() {
	fmt.Printf("myLG v%s\n", version)
}

// cleanUp
func cleanUp() {
	c.Close(nxt)
	close(req)
}

// help
func help() {
	if noIf {
		// without command line
		h := `
              ***** TRY IT WITHOUT ANYTHING TO HAVE INTERFACE *****
        Usage:
              mylg [command] [args...]

              Available commands:

              ping                        ping ip address or domain name
              trace                       trace ip address or domain name (real-time w/ -r option)
              dig                         name server looking up
              whois                       resolve AS number/IP/CIDR to holder (provides by ripe ncc)
              hping                       Ping through HTTP/HTTPS w/ GET/HEAD methods
              scan                        scan tcp ports (you can provide range >scan host minport maxport)
              dump                        prints out a description of the contents of packets on a network interface
              disc                        discover all the devices on a LAN
              peering                     peering information (provides by peeringdb.com)
              version                     shows mylg version

        Example:
              mylg trace freebsd.org -r
              mylg whois 8.8.8.8
              mylg scan 127.0.0.1
              mylg dig google.com +trace
		`
		fmt.Println(h)
	} else {
		// with command line interface

		c.Help()
	}
}
<commit_msg>minor change  based on the cli package<commit_after>// myLG is command line looking glass that written with Go language
// it tries from its own icmp and external looking glasses tools
package main

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/briandowns/spinner"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/mehrdadrad/mylg/disc"
	"github.com/mehrdadrad/mylg/http/ping"
	"github.com/mehrdadrad/mylg/icmp"
	"github.com/mehrdadrad/mylg/lg"
	"github.com/mehrdadrad/mylg/ns"
	"github.com/mehrdadrad/mylg/packet"
	"github.com/mehrdadrad/mylg/peeringdb"
	"github.com/mehrdadrad/mylg/scan"
	"github.com/mehrdadrad/mylg/services/httpd"
	"github.com/mehrdadrad/mylg/whois"
)

const (
	version = "0.2.4"
)

// Provider represents looking glass
type Provider interface {
	Set(host, version string)
	GetDefaultNode() string
	GetNodes() []string
	ChangeNode(node string) bool
	Ping() (string, error)
	Trace() chan string
	BGP() chan string
}

var (
	pNames = providerNames()
	req    = make(chan string, 1)
	nxt    = make(chan struct{}, 1)
	spin   = spinner.New(spinner.CharSets[26], 220*time.Millisecond)
	eArgs  = os.Args
	args   string
	prompt string
	cPName string
	noIf   bool = true
	cfg    cli.Config
	nsr    *ns.Request
	c      *cli.Readline

	// register looking glass hosts
	providers = map[string]Provider{
		"telia":  new(lg.Telia),  // telia
		"level3": new(lg.Level3), // level3
		"cogent": new(lg.Cogent), //cogent
	}

	// map cmd to function
	cmdFunc = map[string]func(){
		"web":     web,          // web dashboard
		"dump":    dump,         // dump traffic
		"disc":    discovery,    // network discovery
		"scan":    scanPorts,    // network scan
		"mode":    mode,         // editor mode
		"ping":    pingQuery,    // ping
		"trace":   trace,        // trace route
		"bgp":     BGP,          // BGP
		"whois":   whoisLookup,  // whois / dns lookup
		"peering": peeringDB,    // peering DB
		"hping":   hping,        // hping
		"dig":     dig,          // dig
		"node":    node,         // change node
		"connect": connect,      // connect to a country or LG
		"local":   local,        // local
		"help":    help,         // help
		"exit":    cleanUp,      // clean up
		"quit":    cleanUp,      // clean up
		"show":    show,         // show config
		"set":     setConfig,    // set config
		"lg":      setLG,        // prepare looking glass
		"ns":      setNS,        // prepare name server
		"version": printVersion, // prints version
	}
)

// init
func init() {
	// load configuration
	cfg = cli.LoadConfig()
	// with interface
	if len(eArgs) == 1 {
		// initialize cli
		c = cli.Init(version)
		go c.Run(req, nxt)
		// start web server
		go httpd.Run(cfg)
		// set interface enabled
		noIf = false
	}
	// initialize name server
	nsr = ns.NewRequest()
	go nsr.Init()
	// set local as default
	local()
}

func main() {
	// command line w/o interface
	if noIf {
		cmd := eArgs[1]
		args = strings.Join(eArgs[2:], " ")
		if f, ok := cmdFunc[cmd]; ok {
			f()
		} else {
			println("Invalid command please try mylg help")
		}
		return
	}
	// command like w/ interface
LOOP:
	for {
		select {
		case request, ok := <-req:
			if !ok {
				break LOOP
			}
			if len(request) < 1 {
				c.Next()
				continue
			}
			subReq := cli.CMDRgx().FindStringSubmatch(request)
			if len(subReq) == 0 {
				println("syntax error")
				c.Next()
				continue
			}
			prompt = c.GetPrompt()
			args = strings.TrimSpace(subReq[2])
			cmd := strings.TrimSpace(subReq[1])
			if f, ok := cmdFunc[cmd]; ok {
				f()
			} else {
				println("Invalid command please try help")
			}
			c.Next()
		}
	}
}

// providerName
func providerNames() []string {
	pNames := []string{}
	for p := range providers {
		pNames = append(pNames, p)
	}
	return pNames
}

// validateProvider
func validateProvider(p string) (string, error) {
	pNames := []string{}
	match, _ := regexp.MatchString("("+strings.Join(pNames, "|")+")", p)
	p = strings.ToLower(p)
	if match {
		return p, nil
	}
	return "", errors.New("provider not support")

}

// node handles node cmd
func node() {
	switch {
	case strings.HasPrefix(prompt, "lg"):
		if _, ok := providers[cPName]; ok {
			if providers[cPName].ChangeNode(args) {
				c.UpdatePromptN(args, 3)
				return
			}
		}
		println("the specified node doesn't support")
	case strings.HasPrefix(prompt, "ns"):
		if !nsr.ChkNode(args) {
			println("error: argument is not valid")
		} else {
			c.UpdatePromptN(args, 3)
		}
	default:
		if cPName == "local" {
			println("local doesn't support node")
		}

	}
}

// dig gets dig info
func dig() {
	if ok := nsr.SetOptions(args, prompt); ok {
		nsr.Dig()
	}
}

// web tries to open web interface at default web browser
func web() {
	var openCmd = "open"
	println("opening default web broswer ...")
	if runtime.GOOS != "darwin" {
		openCmd = "xdg-open"
	}
	cmd := exec.Command(openCmd, fmt.Sprintf("http://%s:%d", cfg.Web.Address, cfg.Web.Port))
	err := cmd.Start()
	if err != nil {
		println("error opening default browser")
	}

}

// dump provides decoding packets
func dump() {
	p, err := packet.NewPacket(args)
	if p == nil || err != nil {
		return
	}
	println(p.Banner())
	for l := range p.Open() {
		l.PrintPretty()
	}
}

// connect handles connect cmd
func connect() {
	var (
		pName string
		err   error
	)
	switch {
	case strings.HasPrefix(prompt, "lg"):
		if pName, err = validateProvider(args); err != nil {
			println("provider not available")
			c.Next()
			return
		}
		cPName = pName
		if _, ok := providers[cPName]; ok {
			c.UpdatePromptN(cPName+"/"+providers[cPName].GetDefaultNode(), 2)
			go func() {
				c.UpdateCompleter("node", providers[cPName].GetNodes())
			}()
		} else {
			println("it doesn't support")
		}
	case strings.HasPrefix(prompt, "ns"):
		if !nsr.ChkCountry(args) {
			println("error: argument is not valid")
		} else {
			c.SetPrompt("ns/" + args)
			c.UpdateCompleter("node", nsr.NodeList())
		}
	}
}

// mode set editor mode
func mode() {
	if args == "vim" {
		c.SetVim()
	} else if args == "emacs" {
		c.SetEmacs()
	} else {
		println("the request mode doesn't support")
	}
}

// trace tries to trace from local and lg
func trace() {
	switch {
	case strings.HasPrefix(prompt, "local"):
		trace, err := icmp.NewTrace(args, cfg)
		if err != nil {
			println(err.Error())
		}
		if trace == nil {
			break
		}
		trace.Print()
	case strings.HasPrefix(prompt, "lg"):
		providers[cPName].Set(args, "ipv4")
		for l := range providers[cPName].Trace() {
			println(l)
		}
	}
}

// hping tries to ping a web server by http
func hping() {
	// it should work at local mode
	if cPName != "local" {
		return
	}
	p, err := ping.NewPing(args, cfg)
	if err != nil {
		println(err.Error())
	} else {
		p.Run()
	}
}

// pingQuery runs ping command (local/LG)
func pingQuery() {
	if cPName == "local" {
		pingLocal()
	} else {
		pingLG()
	}
}

// pingLG tries to ping through a looking glass
func pingLG() {
	spin.Prefix = "please wait "
	spin.Start()
	providers[cPName].Set(args, "ipv4")
	m, err := providers[cPName].Ping()
	spin.Stop()
	if err != nil {
		println(err.Error())
	} else {
		println(m)
	}
}

// pingLocal tries to ping from local source ip
func pingLocal() {
	p, err := icmp.NewPing(args, cfg)
	if err != nil {
		println(err.Error())
	}
	if p == nil {
		return
	}
	if !p.IsCIDR() {
		resp := p.Run()
		p.PrintPretty(resp)
	}
}

// scanPorts tries to scan tcp/ip ports
func scanPorts() {
	scan, err := scan.NewScan(args, cfg)
	if err != nil {
		println(err.Error())
	} else {
		scan.Run()
	}
}

// BGP tries to get BGP lookup from a LG
func BGP() {
	if cPName == "local" {
		println("no provider selected")
		return
	}
	providers[cPName].Set(args, "ipv4")
	for l := range providers[cPName].BGP() {
		println(l)
	}
}

// discovery handles disc command
func discovery() {
	var (
		wg sync.WaitGroup
		//ts = time.Now()
	)

	d := disc.New(args)
	// help requested
	if d == nil {
		return
	}

	spin.Prefix = "please wait "
	spin.Start()

	// load OUI async
	go func() {
		wg.Add(1)
		d.LoadOUI()
		wg.Done()
	}()
	d.PingLan()
	//s := 10 - int(time.Since(ts).Seconds())
	time.Sleep(5 * time.Second)
	if err := d.GetARPTable(); err != nil {
		println(err.Error())
		return
	}
	wg.Wait()
	spin.Stop()

	println("Network LAN Discovery")
	d.PrintPretty()
}

// setConfig
func setConfig() {
	cli.SetConfig(args, &cfg)
}

// showConfig
func show() {
	if args == "config" {
		cli.ShowConfig(&cfg)
	}
}

// setLG set lg prompt and completer
func setLG() {
	c.SetPrompt("lg")
	c.UpdateCompleter("connect", pNames)
}

// setNS set ns prompt and update completers
func setNS() {
	c.UpdateCompleter("connect", nsr.CountryList())
	c.UpdateCompleter("node", []string{})
	c.SetPrompt("ns")
}

// peeringDB gets peer info
func peeringDB() {
	peeringdb.Search(args)
}

// whoisLookup gets ANS/Prefix info
func whoisLookup() {
	whois.Lookup(args)
}

// local set prompts to local
func local() {
	nsr.Local()
	cPName = "local"
	c.SetPrompt(cPName)
}

// printVersion prints version and exits
func printVersion() {
	fmt.Printf("myLG v%s\n", version)
}

// cleanUp
func cleanUp() {
	c.Close(nxt)
	close(req)
}

// help
func help() {
	if noIf {
		// without command line
		h := `
              ***** TRY IT WITHOUT ANYTHING TO HAVE INTERFACE *****
        Usage:
              mylg [command] [args...]

              Available commands:

              ping                        ping ip address or domain name
              trace                       trace ip address or domain name (real-time w/ -r option)
              dig                         name server looking up
              whois                       resolve AS number/IP/CIDR to holder (provides by ripe ncc)
              hping                       Ping through HTTP/HTTPS w/ GET/HEAD methods
              scan                        scan tcp ports (you can provide range >scan host minport maxport)
              dump                        prints out a description of the contents of packets on a network interface
              disc                        discover all the devices on a LAN
              peering                     peering information (provides by peeringdb.com)
              version                     shows mylg version

        Example:
              mylg trace freebsd.org -r
              mylg whois 8.8.8.8
              mylg scan 127.0.0.1
              mylg dig google.com +trace
		`
		fmt.Println(h)
	} else {
		// with command line interface

		c.Help()
	}
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/minhttp"
	"github.com/minio/minio/pkg/probe"
)

var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start Minio cloud storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "min-free-disk, M",
			Value: "5%",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTION VALUE] PATH

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}

ENVIRONMENT VARIABLES:
  MINIO_ACCESS_KEY, MINIO_SECRET_KEY: Access and secret key to use.

EXAMPLES:
  1. Start minio server on Linux.
      $ minio {{.Name}} /home/shared

  2. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio --address 192.168.1.101:9000 {{.Name}} /home/shared

  4. Start minio server with minimum free disk threshold to 5%
      $ minio {{.Name}} --min-free-disk 5% /home/shared/Pictures

`,
}

// cloudServerConfig - http server config
type cloudServerConfig struct {
	/// HTTP server options
	Address   string // Address:Port listening
	AccessLog bool   // Enable access log handler

	// Credentials.
	AccessKeyID     string // Access key id.
	SecretAccessKey string // Secret access key.
	Region          string // Region string.

	/// FS options
	Path        string // Path to export for cloud storage
	MinFreeDisk int64  // Minimum free disk space for filesystem
	MaxBuckets  int    // Maximum number of buckets suppported by filesystem.

	/// TLS service
	TLS      bool   // TLS on when certs are specified
	CertFile string // Domain certificate
	KeyFile  string // Domain key
}

// configureServer configure a new server instance
func configureServer(conf cloudServerConfig) (*http.Server, *probe.Error) {
	// Minio server config
	apiServer := &http.Server{
		Addr:           conf.Address,
		Handler:        serverHandler(conf),
		MaxHeaderBytes: 1 << 20,
	}

	if conf.TLS {
		var err error
		apiServer.TLSConfig = &tls.Config{}
		apiServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
		apiServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(conf.CertFile, conf.KeyFile)
		if err != nil {
			return nil, probe.NewError(err)
		}
	}
	return apiServer, nil
}

func printServerMsg(serverConf *http.Server) {
	host, port, e := net.SplitHostPort(serverConf.Addr)
	fatalIf(probe.NewError(e), "Unable to split host port.", nil)

	var hosts []string
	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, e := net.InterfaceAddrs()
		fatalIf(probe.NewError(e), "Unable to get interface address.", nil)

		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	for _, host := range hosts {
		if serverConf.TLSConfig != nil {
			console.Printf("    https://%s:%s\n", host, port)
		} else {
			console.Printf("    http://%s:%s\n", host, port)
		}
	}
}

// parse input string with percent to int64
func parsePercentToInt(s string, bitSize int) (int64, *probe.Error) {
	i := strings.Index(s, "%")
	if i < 0 {
		// no percentage string found try to parse the whole string anyways
		p, e := strconv.ParseInt(s, 10, bitSize)
		if e != nil {
			return 0, probe.NewError(e)
		}
		return p, nil
	}
	p, e := strconv.ParseInt(s[:i], 10, bitSize)
	if e != nil {
		return 0, probe.NewError(e)
	}
	return p, nil
}
func setLogger(conf *configV2) *probe.Error {
	if conf.IsMongoLoggingEnabled() {
		if err := log2Mongo(conf.MongoLogger.Addr, conf.MongoLogger.DB, conf.MongoLogger.Collection); err != nil {
			return err.Trace(conf.MongoLogger.Addr, conf.MongoLogger.DB, conf.MongoLogger.Collection)
		}
	}
	if conf.IsSysloggingEnabled() {
		if err := log2Syslog(conf.SyslogLogger.Network, conf.SyslogLogger.Addr); err != nil {
			return err.Trace(conf.SyslogLogger.Network, conf.SyslogLogger.Addr)
		}
	}
	if conf.IsFileLoggingEnabled() {
		if err := log2File(conf.FileLogger.Filename); err != nil {
			return err.Trace(conf.FileLogger.Filename)
		}
	}
	return nil
}

// Generates config if it doesn't exist, otherwise returns back the saved ones.
func getConfig() (*configV2, *probe.Error) {
	if err := createConfigPath(); err != nil {
		return nil, err.Trace()
	}
	if err := createBucketsConfigPath(); err != nil {
		return nil, err.Trace()
	}
	config, err := loadConfigV2()
	if err != nil {
		if os.IsNotExist(err.ToGoError()) {
			// Initialize new config, since config file doesn't exist yet
			config = &configV2{}
			config.Version = "2"
			config.Credentials.AccessKeyID = string(mustGenerateAccessKeyID())
			config.Credentials.SecretAccessKey = string(mustGenerateSecretAccessKey())
			config.Credentials.Region = "us-east-1"
			if err = saveConfig(config); err != nil {
				return nil, err.Trace()
			}
			return config, nil
		}
		return nil, err.Trace()
	}
	return config, nil
}

type accessKeys struct {
	*configV2
}

func (a accessKeys) String() string {
	magenta := color.New(color.FgMagenta, color.Bold).SprintFunc()
	white := color.New(color.FgWhite, color.Bold).SprintfFunc()
	return fmt.Sprint(magenta("AccessKey: ") + white(a.Credentials.AccessKeyID) + "  " + magenta("SecretKey: ") + white(a.Credentials.SecretAccessKey) + "  " + magenta("Region: ") + white(a.Credentials.Region))
}

// JSON - json formatted output
func (a accessKeys) JSON() string {
	b, e := json.Marshal(a)
	errorIf(probe.NewError(e), "Unable to marshal json", nil)
	return string(b)
}

// initServer initialize server
func initServer() (*configV2, *probe.Error) {
	conf, err := getConfig()
	if err != nil {
		return nil, err.Trace()
	}
	if err := setLogger(conf); err != nil {
		return nil, err.Trace()
	}
	if conf != nil {
		console.Println()
		console.Println(accessKeys{conf})
	}
	return conf, nil
}

func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
	if len(c.Args()) > 1 {
		fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil)
	}
	path := strings.TrimSpace(c.Args().Last())
	if path == "" {
		fatalIf(probe.NewError(errInvalidArgument), "Path argument cannot be empty.", nil)
	}
}

// extract port number from address.
// address should be of the form host:port
func getPort(address string) int {
	_, portStr, e := net.SplitHostPort(address)
	fatalIf(probe.NewError(e), "Unable to split host port.", nil)
	portInt, e := strconv.Atoi(portStr)
	fatalIf(probe.NewError(e), "Invalid port number.", nil)
	return portInt
}

// Make sure that none of the other processes are listening on the
// specified port on any of the interfaces.
//
// On linux if a process is listening on 127.0.0.1:9000 then Listen()
// on ":9000" fails with the error "port already in use".
// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
// This causes confusion on Mac OSX that minio server is not reachable
// on 127.0.0.1 even though minio server is running. So before we start
// the minio server we make sure that the port is free on all the IPs.
func checkPortAvailability(port int) {
	ifcs, e := net.Interfaces()
	if e != nil {
		fatalIf(probe.NewError(e), "Unable to list interfaces.", nil)
	}
	for _, ifc := range ifcs {
		addrs, e := ifc.Addrs()
		if e != nil {
			fatalIf(probe.NewError(e), fmt.Sprintf("Unable to list addresses on interface %s.", ifc.Name), nil)
		}
		for _, addr := range addrs {
			ip := addr.(*net.IPNet).IP
			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
			l, e := net.ListenTCP("tcp", &tcpAddr)
			if e != nil {
				fatalIf(probe.NewError(e), fmt.Sprintf("Unable to listen on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil)
			}
			e = l.Close()
			if e != nil {
				fatalIf(probe.NewError(e), fmt.Sprintf("Unable to close listener on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil)
			}
		}
	}
}

func serverMain(c *cli.Context) {
	checkServerSyntax(c)
	address := c.GlobalString("address")
	checkPortAvailability(getPort(address))

	conf, err := initServer()
	fatalIf(err.Trace(), "Failed to read config for minio.", nil)

	certFile := c.GlobalString("cert")
	keyFile := c.GlobalString("key")
	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
		fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil)
	}

	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey(accessKey) {
			fatalIf(probe.NewError(errInvalidArgument), "Access key does not have required length", nil)
		}
		if !isValidSecretKey(secretKey) {
			fatalIf(probe.NewError(errInvalidArgument), "Secret key does not have required length", nil)
		}

		conf.Credentials.AccessKeyID = accessKey
		conf.Credentials.SecretAccessKey = secretKey

		err = saveConfig(conf)
		fatalIf(err.Trace(), "Unable to save credentials to config.", nil)
	}

	minFreeDisk, err := parsePercentToInt(c.String("min-free-disk"), 64)
	fatalIf(err.Trace(c.String("min-free-disk")), "Invalid minium free disk size "+c.String("min-free-disk")+" passed.", nil)

	path := strings.TrimSpace(c.Args().Last())
	// Last argument is always path
	if _, err := os.Stat(path); err != nil {
		fatalIf(probe.NewError(err), "Unable to validate the path", nil)
	}
	region := conf.Credentials.Region
	if region == "" {
		region = "us-east-1"
	}
	tls := (certFile != "" && keyFile != "")
	serverConfig := cloudServerConfig{
		Address:         address,
		AccessLog:       c.GlobalBool("enable-accesslog"),
		AccessKeyID:     conf.Credentials.AccessKeyID,
		SecretAccessKey: conf.Credentials.SecretAccessKey,
		Region:          region,
		Path:            path,
		MinFreeDisk:     minFreeDisk,
		TLS:             tls,
		CertFile:        certFile,
		KeyFile:         keyFile,
	}

	// configure server.
	apiServer, err := configureServer(serverConfig)
	errorIf(err.Trace(), "Failed to configure API server.", nil)

	console.Println("\nMinio Object Storage:")
	printServerMsg(apiServer)

	console.Println("\nMinio Browser:")
	printServerMsg(apiServer)

	console.Println("\nTo configure Minio Client:")
	if runtime.GOOS == "windows" {
		console.Println("    Download \"mc\" from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
		console.Println("    $ mc.exe config host add myminio http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
	} else {
		console.Println("    $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
		console.Println("    $ chmod 755 mc")
		console.Println("    $ ./mc config host add myminio http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
	}

	// Start server.
	err = minhttp.ListenAndServe(apiServer)
	errorIf(err.Trace(), "Failed to start the minio server.", nil)
}
<commit_msg>startup: specify the network - tcp4/tcp6 for ListenTCP()<commit_after>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/minhttp"
	"github.com/minio/minio/pkg/probe"
)

var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start Minio cloud storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "min-free-disk, M",
			Value: "5%",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTION VALUE] PATH

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}

ENVIRONMENT VARIABLES:
  MINIO_ACCESS_KEY, MINIO_SECRET_KEY: Access and secret key to use.

EXAMPLES:
  1. Start minio server on Linux.
      $ minio {{.Name}} /home/shared

  2. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio --address 192.168.1.101:9000 {{.Name}} /home/shared

  4. Start minio server with minimum free disk threshold to 5%
      $ minio {{.Name}} --min-free-disk 5% /home/shared/Pictures

`,
}

// cloudServerConfig - http server config
type cloudServerConfig struct {
	/// HTTP server options
	Address   string // Address:Port listening
	AccessLog bool   // Enable access log handler

	// Credentials.
	AccessKeyID     string // Access key id.
	SecretAccessKey string // Secret access key.
	Region          string // Region string.

	/// FS options
	Path        string // Path to export for cloud storage
	MinFreeDisk int64  // Minimum free disk space for filesystem
	MaxBuckets  int    // Maximum number of buckets suppported by filesystem.

	/// TLS service
	TLS      bool   // TLS on when certs are specified
	CertFile string // Domain certificate
	KeyFile  string // Domain key
}

// configureServer configure a new server instance
func configureServer(conf cloudServerConfig) (*http.Server, *probe.Error) {
	// Minio server config
	apiServer := &http.Server{
		Addr:           conf.Address,
		Handler:        serverHandler(conf),
		MaxHeaderBytes: 1 << 20,
	}

	if conf.TLS {
		var err error
		apiServer.TLSConfig = &tls.Config{}
		apiServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
		apiServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(conf.CertFile, conf.KeyFile)
		if err != nil {
			return nil, probe.NewError(err)
		}
	}
	return apiServer, nil
}

func printServerMsg(serverConf *http.Server) {
	host, port, e := net.SplitHostPort(serverConf.Addr)
	fatalIf(probe.NewError(e), "Unable to split host port.", nil)

	var hosts []string
	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, e := net.InterfaceAddrs()
		fatalIf(probe.NewError(e), "Unable to get interface address.", nil)

		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	for _, host := range hosts {
		if serverConf.TLSConfig != nil {
			console.Printf("    https://%s:%s\n", host, port)
		} else {
			console.Printf("    http://%s:%s\n", host, port)
		}
	}
}

// parse input string with percent to int64
func parsePercentToInt(s string, bitSize int) (int64, *probe.Error) {
	i := strings.Index(s, "%")
	if i < 0 {
		// no percentage string found try to parse the whole string anyways
		p, e := strconv.ParseInt(s, 10, bitSize)
		if e != nil {
			return 0, probe.NewError(e)
		}
		return p, nil
	}
	p, e := strconv.ParseInt(s[:i], 10, bitSize)
	if e != nil {
		return 0, probe.NewError(e)
	}
	return p, nil
}
func setLogger(conf *configV2) *probe.Error {
	if conf.IsMongoLoggingEnabled() {
		if err := log2Mongo(conf.MongoLogger.Addr, conf.MongoLogger.DB, conf.MongoLogger.Collection); err != nil {
			return err.Trace(conf.MongoLogger.Addr, conf.MongoLogger.DB, conf.MongoLogger.Collection)
		}
	}
	if conf.IsSysloggingEnabled() {
		if err := log2Syslog(conf.SyslogLogger.Network, conf.SyslogLogger.Addr); err != nil {
			return err.Trace(conf.SyslogLogger.Network, conf.SyslogLogger.Addr)
		}
	}
	if conf.IsFileLoggingEnabled() {
		if err := log2File(conf.FileLogger.Filename); err != nil {
			return err.Trace(conf.FileLogger.Filename)
		}
	}
	return nil
}

// Generates config if it doesn't exist, otherwise returns back the saved ones.
func getConfig() (*configV2, *probe.Error) {
	if err := createConfigPath(); err != nil {
		return nil, err.Trace()
	}
	if err := createBucketsConfigPath(); err != nil {
		return nil, err.Trace()
	}
	config, err := loadConfigV2()
	if err != nil {
		if os.IsNotExist(err.ToGoError()) {
			// Initialize new config, since config file doesn't exist yet
			config = &configV2{}
			config.Version = "2"
			config.Credentials.AccessKeyID = string(mustGenerateAccessKeyID())
			config.Credentials.SecretAccessKey = string(mustGenerateSecretAccessKey())
			config.Credentials.Region = "us-east-1"
			if err = saveConfig(config); err != nil {
				return nil, err.Trace()
			}
			return config, nil
		}
		return nil, err.Trace()
	}
	return config, nil
}

type accessKeys struct {
	*configV2
}

func (a accessKeys) String() string {
	magenta := color.New(color.FgMagenta, color.Bold).SprintFunc()
	white := color.New(color.FgWhite, color.Bold).SprintfFunc()
	return fmt.Sprint(magenta("AccessKey: ") + white(a.Credentials.AccessKeyID) + "  " + magenta("SecretKey: ") + white(a.Credentials.SecretAccessKey) + "  " + magenta("Region: ") + white(a.Credentials.Region))
}

// JSON - json formatted output
func (a accessKeys) JSON() string {
	b, e := json.Marshal(a)
	errorIf(probe.NewError(e), "Unable to marshal json", nil)
	return string(b)
}

// initServer initialize server
func initServer() (*configV2, *probe.Error) {
	conf, err := getConfig()
	if err != nil {
		return nil, err.Trace()
	}
	if err := setLogger(conf); err != nil {
		return nil, err.Trace()
	}
	if conf != nil {
		console.Println()
		console.Println(accessKeys{conf})
	}
	return conf, nil
}

func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
	if len(c.Args()) > 1 {
		fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil)
	}
	path := strings.TrimSpace(c.Args().Last())
	if path == "" {
		fatalIf(probe.NewError(errInvalidArgument), "Path argument cannot be empty.", nil)
	}
}

// extract port number from address.
// address should be of the form host:port
func getPort(address string) int {
	_, portStr, e := net.SplitHostPort(address)
	fatalIf(probe.NewError(e), "Unable to split host port.", nil)
	portInt, e := strconv.Atoi(portStr)
	fatalIf(probe.NewError(e), "Invalid port number.", nil)
	return portInt
}

// Make sure that none of the other processes are listening on the
// specified port on any of the interfaces.
//
// On linux if a process is listening on 127.0.0.1:9000 then Listen()
// on ":9000" fails with the error "port already in use".
// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
// This causes confusion on Mac OSX that minio server is not reachable
// on 127.0.0.1 even though minio server is running. So before we start
// the minio server we make sure that the port is free on all the IPs.
func checkPortAvailability(port int) {
	ifcs, e := net.Interfaces()
	if e != nil {
		fatalIf(probe.NewError(e), "Unable to list interfaces.", nil)
	}
	for _, ifc := range ifcs {
		addrs, e := ifc.Addrs()
		if e != nil {
			fatalIf(probe.NewError(e), fmt.Sprintf("Unable to list addresses on interface %s.", ifc.Name), nil)
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				errorIf(probe.NewError(errors.New("")), "Interface type assertion to (*net.IPNet) failed.", nil)
				continue
			}
			ip := ipnet.IP
			network := "tcp4"
			if ip.To4() == nil {
				network = "tcp6"
			}
			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
			l, e := net.ListenTCP(network, &tcpAddr)
			if e != nil {
				fatalIf(probe.NewError(e), fmt.Sprintf("Unable to listen on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil)
			}
			e = l.Close()
			if e != nil {
				fatalIf(probe.NewError(e), fmt.Sprintf("Unable to close listener on IP %s, port %.d", tcpAddr.IP, tcpAddr.Port), nil)
			}
		}
	}
}

func serverMain(c *cli.Context) {
	checkServerSyntax(c)
	address := c.GlobalString("address")
	checkPortAvailability(getPort(address))

	conf, err := initServer()
	fatalIf(err.Trace(), "Failed to read config for minio.", nil)

	certFile := c.GlobalString("cert")
	keyFile := c.GlobalString("key")
	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
		fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil)
	}

	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey(accessKey) {
			fatalIf(probe.NewError(errInvalidArgument), "Access key does not have required length", nil)
		}
		if !isValidSecretKey(secretKey) {
			fatalIf(probe.NewError(errInvalidArgument), "Secret key does not have required length", nil)
		}

		conf.Credentials.AccessKeyID = accessKey
		conf.Credentials.SecretAccessKey = secretKey

		err = saveConfig(conf)
		fatalIf(err.Trace(), "Unable to save credentials to config.", nil)
	}

	minFreeDisk, err := parsePercentToInt(c.String("min-free-disk"), 64)
	fatalIf(err.Trace(c.String("min-free-disk")), "Invalid minium free disk size "+c.String("min-free-disk")+" passed.", nil)

	path := strings.TrimSpace(c.Args().Last())
	// Last argument is always path
	if _, err := os.Stat(path); err != nil {
		fatalIf(probe.NewError(err), "Unable to validate the path", nil)
	}
	region := conf.Credentials.Region
	if region == "" {
		region = "us-east-1"
	}
	tls := (certFile != "" && keyFile != "")
	serverConfig := cloudServerConfig{
		Address:         address,
		AccessLog:       c.GlobalBool("enable-accesslog"),
		AccessKeyID:     conf.Credentials.AccessKeyID,
		SecretAccessKey: conf.Credentials.SecretAccessKey,
		Region:          region,
		Path:            path,
		MinFreeDisk:     minFreeDisk,
		TLS:             tls,
		CertFile:        certFile,
		KeyFile:         keyFile,
	}

	// configure server.
	apiServer, err := configureServer(serverConfig)
	errorIf(err.Trace(), "Failed to configure API server.", nil)

	console.Println("\nMinio Object Storage:")
	printServerMsg(apiServer)

	console.Println("\nMinio Browser:")
	printServerMsg(apiServer)

	console.Println("\nTo configure Minio Client:")
	if runtime.GOOS == "windows" {
		console.Println("    Download \"mc\" from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
		console.Println("    $ mc.exe config host add myminio http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
	} else {
		console.Println("    $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
		console.Println("    $ chmod 755 mc")
		console.Println("    $ ./mc config host add myminio http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
	}

	// Start server.
	err = minhttp.ListenAndServe(apiServer)
	errorIf(err.Trace(), "Failed to start the minio server.", nil)
}
<|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"sync"
)

var (
	// The default size of the ring buffer
	defaultCap = 60
)

// RingBuff is a data structure which is a circular list based on slices
type RingBuff struct {
	head int
	buff []interface{}

	lock sync.RWMutex
}

// NewRingBuff creates a new ring buffer of the specified size
func NewRingBuff(capacity int) (*RingBuff, error) {
	if capacity < 1 {
		return nil, fmt.Errorf("can not create a ring buffer with capacity: %v", capacity)
	}
	return &RingBuff{buff: make([]interface{}, capacity), head: -1}, nil
}

// Enqueue queues a new value in the ring buffer. This operation would
// over-write an older value if the list has reached it's capacity
func (r *RingBuff) Enqueue(value interface{}) {
	r.lock.Lock()
	defer r.lock.Unlock()
	r.head += 1
	if r.head == len(r.buff) {
		r.head = 0
	}
	r.buff[r.head] = value
}

// Peek returns the last value enqueued in the ring buffer
func (r *RingBuff) Peek() interface{} {
	r.lock.RLock()
	defer r.lock.RUnlock()
	if r.head == -1 {
		return nil
	}
	return r.buff[r.head]
}

// Values returns all the values in the buffer.
func (r *RingBuff) Values() []interface{} {
	r.lock.RLock()
	defer r.lock.RUnlock()
	if r.head == len(r.buff)-1 {
		vals := make([]interface{}, len(r.buff))
		copy(vals, r.buff)
		return vals
	}

	slice1 := r.buff[r.head+1:]
	slice2 := r.buff[:r.head+1]
	return append(slice1, slice2...)
}
<commit_msg>Initializing the ring buffer with no cells<commit_after>package stats

import (
	"fmt"
	"sync"
)

var (
	// The default size of the ring buffer
	defaultCap = 60
)

// RingBuff is a data structure which is a circular list based on slices
type RingBuff struct {
	head int
	buff []interface{}

	lock sync.RWMutex
}

// NewRingBuff creates a new ring buffer of the specified size
func NewRingBuff(capacity int) (*RingBuff, error) {
	if capacity < 1 {
		return nil, fmt.Errorf("can not create a ring buffer with capacity: %v", capacity)
	}
	return &RingBuff{buff: make([]interface{}, 0, capacity), head: -1}, nil
}

// Enqueue queues a new value in the ring buffer. This operation would
// over-write an older value if the list has reached it's capacity
func (r *RingBuff) Enqueue(value interface{}) {
	r.lock.Lock()
	defer r.lock.Unlock()
	if len(r.buff) < cap(r.buff) {
		r.buff = append(r.buff, struct{}{})
	}
	r.head += 1
	if r.head == cap(r.buff) {
		r.head = 0
	}
	r.buff[r.head] = value
}

// Peek returns the last value enqueued in the ring buffer
func (r *RingBuff) Peek() interface{} {
	r.lock.RLock()
	defer r.lock.RUnlock()
	if r.head == -1 {
		return nil
	}
	return r.buff[r.head]
}

// Values returns all the values in the buffer.
func (r *RingBuff) Values() []interface{} {
	r.lock.RLock()
	defer r.lock.RUnlock()
	if r.head == len(r.buff)-1 {
		vals := make([]interface{}, len(r.buff))
		copy(vals, r.buff)
		return vals
	}

	slice1 := r.buff[r.head+1:]
	slice2 := r.buff[:r.head+1]
	return append(slice1, slice2...)
}
<|endoftext|>


<commit_before>package dsunit

import (
	"fmt"
	"github.com/viant/dsunit"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
)

var converter = toolbox.NewColumnConverter("yyyy-MM-dd HH:ss")

const (
	//ServiceID represents a data store unit service id
	ServiceID = "dsunit"
)

//PopulateDatastoreEvent represents a populate Datastore event
type PopulateDatastoreEvent struct {
	Datastore string `required:"true" description:"register datastore name"` //target host
	Table     string
	Rows      int
}

//RunSQLcriptEvent represents run script event
type RunSQLcriptEvent struct {
	Datastore string
	URL       string
}

type service struct {
	*endly.AbstractService
	Service dsunit.Service
}

const (
	dsunitMySQLInitExample = `{
  "Datastore": "mydb",
  "Config": {
    "DriverName": "mysql",
    "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true",
    "Credentials": "$mysqlCredentials"
  },
  "Admin": {
    "Datastore": "mysql",
    "Config": {
      "DriverName": "mysql",
      "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true",
      "Credentials": "$mysqlCredentials"
    }
  },
  "Scripts": [
    {
      "URL": "datastore/mydb/schema.ddl"
    }
  ],
  "Recreate": "true"
}`

	dsunitAerospikeRegisterExample = `{
  "Datastore": "db",
  "Config": {
    "DriverName": "aerospike",
    "Descriptor": "tcp([host]:3000)/[namespace]",
    "Parameters": {
      "dateFormat": "yyyy-MM-dd hh:mm:ss",
      "dbname": "db",
      "host": "127.0.0.1",
      "keyColumnName": "id",
      "namespace": "db",
      "port": "3000"
    }
  }
}`

	dsunitBigQueryRegisterExample = `{
  "Datastore": "db1",
  "Config": {
    "DriverName": "bigquery",
    "Descriptor": "bq/[datasetId]",
	"Credentials": "${env.HOME}/.secret/bq.json",
    "Parameters": {
      "datasetId": "db1",
      "dateFormat": "yyyy-MM-dd HH:mm:ss.SSSZ",
      "projectId": "xxxxx"
    }
  }

}`

	dsunitMySQLRegisterExample = `{
  "Datastore": "db1",
  "Config": {
    "DriverName": "mysql",
  	"Credentials": "${env.HOME}/.secret/mysql.json",
    "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true"
  }
}`

	dsunitServiceSQLExample = `{
		"Datastore": "db1",
		"Scripts": [
			{
				"URL": "datastore/db1/schema.ddl"
			}
		]
	}`

	dsunitServiceMappingExample = ` {
		"Mappings": [
			{
				"URL": "config/mapping/v_asset.json"
			}
		]
	}`

	dsunitServiceSequenceExample = `{
		"Datastore": "db1",
		"Tables": [
			"table1",
			"table2"
		]
	}`

	dsunitServiceStaticDataPrepareExample = `{
    "Datastore": "db1",
    "URL": "datastore/db1/dictionary"
  }`

	dsunitDataPrepareExaple = ` {
		"Datastore": "db1",
		"Data": {
			"table1": [
				{
					"id": 1,
					"name": "test 1",
					"type": "pivot"
				},
				{
					"id": 2,
					"name": "test 2",
					"type": "pivot"
				}
			],
			"table2": [
				{
					"id": 1,
					"name": "test 1",
					"type": "pivot"
				},
				{
					"id": 2,
					"name": "test 2",
					"type": "pivot"
				}
			]
		}
	}`

	dsunitServiceExpectAction = `{
    "Datastore": "db1",
    "URL": "datastore/db1/use_case2/",
	"Prefix":"expect_"
  }`
)

func (s *service) registerRoutes() {

	s.Register(&endly.Route{
		Action: "register",
		RequestInfo: &endly.ActionInfo{
			Description: "register database connection",
			Examples: []*endly.UseCase{
				{
					Description: "aerospike datastore registration",
					Data:        dsunitAerospikeRegisterExample,
				},
				{
					Description: "BigQuery datastore registration",
					Data:        dsunitBigQueryRegisterExample,
				},

				{
					Description: "MySQL datastore registration",
					Data:        dsunitMySQLRegisterExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &RegisterRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RegisterResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {

			if req, ok := request.(*RegisterRequest); ok {
				var dsRequest = dsunit.RegisterRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.RegisterRequest); ok {
				resp := s.Service.Register(req)
				response := RegisterResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "recreate",
		RequestInfo: &endly.ActionInfo{
			Description: "create datastore",
			Examples:    []*endly.UseCase{},
		},
		RequestProvider: func() interface{} {
			return &RecreateRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RecreateResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*RecreateRequest); ok {
				var dsRequest = dsunit.RecreateRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.RecreateRequest); ok {
				resp := s.Service.Recreate(req)
				response := RecreateResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "script",
		RequestInfo: &endly.ActionInfo{
			Description: "run SQL script",
			Examples: []*endly.UseCase{
				{
					Description: "run script",
					Data:        dsunitServiceSQLExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &RunScriptRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RunSQLResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*RunScriptRequest); ok {
				var dsRequest = dsunit.RunScriptRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.RunScriptRequest); ok {
				resp := s.Service.RunScript(req)
				response := RunSQLResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "sql",
		RequestInfo: &endly.ActionInfo{
			Description: "run SQL",
			Examples:    []*endly.UseCase{},
		},
		RequestProvider: func() interface{} {
			return &RunSQLRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RunSQLResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {

			if req, ok := request.(*RunSQLRequest); ok {
				var dsRequest = dsunit.RunSQLRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.RunSQLRequest); ok {
				resp := s.Service.RunSQL(req)
				response := RunSQLResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "mapping",
		RequestInfo: &endly.ActionInfo{
			Description: "register database table mapping (view)",
			Examples:    []*endly.UseCase{},
		},
		RequestProvider: func() interface{} {
			return &MappingRequest{}
		},
		ResponseProvider: func() interface{} {
			return &MappingResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*MappingRequest); ok {
				var dsRequest = dsunit.MappingRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.MappingRequest); ok {
				resp := s.Service.AddTableMapping(req)
				response := MappingResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "init",
		RequestInfo: &endly.ActionInfo{
			Description: "initialize datastore (register, recreated, run sql, add mapping)",

			Examples: []*endly.UseCase{
				{
					Description: "mysql init",
					Data:        dsunitMySQLInitExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &InitRequest{}
		},
		ResponseProvider: func() interface{} {
			return &InitResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*InitRequest); ok {
				var dsRequest = dsunit.InitRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.InitRequest); ok {
				resp := s.Service.Init(req)
				response := InitResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "prepare",
		RequestInfo: &endly.ActionInfo{
			Description: "populate databstore with provided data",
			Examples: []*endly.UseCase{
				{
					Description: "static data prepare",
					Data:        dsunitServiceStaticDataPrepareExample,
				},
				{
					Description: "data prepare",
					Data:        dsunitDataPrepareExaple,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &PrepareRequest{}
		},
		ResponseProvider: func() interface{} {
			return &PrepareResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*PrepareRequest); ok {
				var dsRequest = dsunit.PrepareRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.PrepareRequest); ok {
				resp := s.Service.Prepare(req)
				response := PrepareResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "expect",
		RequestInfo: &endly.ActionInfo{
			Description: "verify databstore with provided data",
			Examples: []*endly.UseCase{
				{
					Description: "static data expect",
					Data:        dsunitServiceExpectAction,
				},
				{
					Description: "data expect",
					Data:        dsunitDataPrepareExaple,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &dsunit.ExpectRequest{
				DatasetResource: &dsunit.DatasetResource{
					DatastoreDatasets: &dsunit.DatastoreDatasets{},
				},
			}
		},
		ResponseProvider: func() interface{} {
			return &dsunit.ExpectResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*ExpectRequest); ok {
				var dsRequest = dsunit.ExpectRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.ExpectRequest); ok {
				resp := s.Service.Expect(req)
				response := ExpectResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "query",
		RequestInfo: &endly.ActionInfo{
			Description: "run SQL query",
		},
		RequestProvider: func() interface{} {
			return &QueryRequest{}
		},
		ResponseProvider: func() interface{} {
			return &QueryResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*QueryRequest); ok {
				var dsRequest = dsunit.QueryRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.QueryRequest); ok {
				resp := s.Service.Query(req)
				response := QueryResponse(*resp)
				var err = response.Error()
				if req.IgnoreError {
					err = nil
				}
				return &response, err
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "sequence",
		RequestInfo: &endly.ActionInfo{
			Description: "get sequence for supplied tables",
			Examples: []*endly.UseCase{
				{
					Description: "sequence",
					Data:        dsunitServiceSequenceExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &SequenceRequest{}
		},
		ResponseProvider: func() interface{} {
			return &SequenceResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*SequenceRequest); ok {
				var dsRequest = dsunit.SequenceRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.SequenceRequest); ok {
				resp := s.Service.Sequence(req)
				response := SequenceResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})
}

func (s service) Run(context *endly.Context, request interface{}) *endly.ServiceResponse {
	var state = context.State()
	context.Context.Replace((*data.Map)(nil), &state)
	s.Service.SetContext(context.Context)
	return s.AbstractService.Run(context, request)
}

//New creates a new Datastore unit service
func New() endly.Service {
	var result = &service{
		AbstractService: endly.NewAbstractService(ServiceID),
		Service:         dsunit.New(),
	}
	result.AbstractService.Service = result
	result.registerRoutes()
	return result
}
<commit_msg>added script URL expansion<commit_after>package dsunit

import (
	"fmt"
	"github.com/viant/dsunit"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
)

var converter = toolbox.NewColumnConverter("yyyy-MM-dd HH:ss")

const (
	//ServiceID represents a data store unit service id
	ServiceID = "dsunit"
)

//PopulateDatastoreEvent represents a populate Datastore event
type PopulateDatastoreEvent struct {
	Datastore string `required:"true" description:"register datastore name"` //target host
	Table     string
	Rows      int
}

//RunSQLcriptEvent represents run script event
type RunSQLcriptEvent struct {
	Datastore string
	URL       string
}

type service struct {
	*endly.AbstractService
	Service dsunit.Service
}

const (
	dsunitMySQLInitExample = `{
  "Datastore": "mydb",
  "Config": {
    "DriverName": "mysql",
    "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true",
    "Credentials": "$mysqlCredentials"
  },
  "Admin": {
    "Datastore": "mysql",
    "Config": {
      "DriverName": "mysql",
      "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true",
      "Credentials": "$mysqlCredentials"
    }
  },
  "Scripts": [
    {
      "URL": "datastore/mydb/schema.ddl"
    }
  ],
  "Recreate": "true"
}`

	dsunitAerospikeRegisterExample = `{
  "Datastore": "db",
  "Config": {
    "DriverName": "aerospike",
    "Descriptor": "tcp([host]:3000)/[namespace]",
    "Parameters": {
      "dateFormat": "yyyy-MM-dd hh:mm:ss",
      "dbname": "db",
      "host": "127.0.0.1",
      "keyColumnName": "id",
      "namespace": "db",
      "port": "3000"
    }
  }
}`

	dsunitBigQueryRegisterExample = `{
  "Datastore": "db1",
  "Config": {
    "DriverName": "bigquery",
    "Descriptor": "bq/[datasetId]",
	"Credentials": "${env.HOME}/.secret/bq.json",
    "Parameters": {
      "datasetId": "db1",
      "dateFormat": "yyyy-MM-dd HH:mm:ss.SSSZ",
      "projectId": "xxxxx"
    }
  }

}`

	dsunitMySQLRegisterExample = `{
  "Datastore": "db1",
  "Config": {
    "DriverName": "mysql",
  	"Credentials": "${env.HOME}/.secret/mysql.json",
    "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true"
  }
}`

	dsunitServiceSQLExample = `{
		"Datastore": "db1",
		"Scripts": [
			{
				"URL": "datastore/db1/schema.ddl"
			}
		]
	}`

	dsunitServiceMappingExample = ` {
		"Mappings": [
			{
				"URL": "config/mapping/v_asset.json"
			}
		]
	}`

	dsunitServiceSequenceExample = `{
		"Datastore": "db1",
		"Tables": [
			"table1",
			"table2"
		]
	}`

	dsunitServiceStaticDataPrepareExample = `{
    "Datastore": "db1",
    "URL": "datastore/db1/dictionary"
  }`

	dsunitDataPrepareExaple = ` {
		"Datastore": "db1",
		"Data": {
			"table1": [
				{
					"id": 1,
					"name": "test 1",
					"type": "pivot"
				},
				{
					"id": 2,
					"name": "test 2",
					"type": "pivot"
				}
			],
			"table2": [
				{
					"id": 1,
					"name": "test 1",
					"type": "pivot"
				},
				{
					"id": 2,
					"name": "test 2",
					"type": "pivot"
				}
			]
		}
	}`

	dsunitServiceExpectAction = `{
    "Datastore": "db1",
    "URL": "datastore/db1/use_case2/",
	"Prefix":"expect_"
  }`
)

func (s *service) registerRoutes() {

	s.Register(&endly.Route{
		Action: "register",
		RequestInfo: &endly.ActionInfo{
			Description: "register database connection",
			Examples: []*endly.UseCase{
				{
					Description: "aerospike datastore registration",
					Data:        dsunitAerospikeRegisterExample,
				},
				{
					Description: "BigQuery datastore registration",
					Data:        dsunitBigQueryRegisterExample,
				},

				{
					Description: "MySQL datastore registration",
					Data:        dsunitMySQLRegisterExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &RegisterRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RegisterResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {

			if req, ok := request.(*RegisterRequest); ok {
				var dsRequest = dsunit.RegisterRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.RegisterRequest); ok {
				resp := s.Service.Register(req)
				response := RegisterResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "recreate",
		RequestInfo: &endly.ActionInfo{
			Description: "create datastore",
			Examples:    []*endly.UseCase{},
		},
		RequestProvider: func() interface{} {
			return &RecreateRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RecreateResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*RecreateRequest); ok {
				var dsRequest = dsunit.RecreateRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.RecreateRequest); ok {
				resp := s.Service.Recreate(req)
				response := RecreateResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "script",
		RequestInfo: &endly.ActionInfo{
			Description: "run SQL script",
			Examples: []*endly.UseCase{
				{
					Description: "run script",
					Data:        dsunitServiceSQLExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &RunScriptRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RunSQLResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*RunScriptRequest); ok {
				var dsRequest = dsunit.RunScriptRequest(*req)
				request = &dsRequest
			}
			var err error
			if req, ok := request.(*dsunit.RunScriptRequest); ok {
				for i, script := range req.Scripts {
					req.Scripts[i], err  = context.ExpandResource(script)
					if err != nil {
						return nil, err
					}
				}
				resp := s.Service.RunScript(req)
				response := RunSQLResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "sql",
		RequestInfo: &endly.ActionInfo{
			Description: "run SQL",
			Examples:    []*endly.UseCase{},
		},
		RequestProvider: func() interface{} {
			return &RunSQLRequest{}
		},
		ResponseProvider: func() interface{} {
			return &RunSQLResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {

			if req, ok := request.(*RunSQLRequest); ok {
				var dsRequest = dsunit.RunSQLRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.RunSQLRequest); ok {
				resp := s.Service.RunSQL(req)
				response := RunSQLResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "mapping",
		RequestInfo: &endly.ActionInfo{
			Description: "register database table mapping (view)",
			Examples:    []*endly.UseCase{},
		},
		RequestProvider: func() interface{} {
			return &MappingRequest{}
		},
		ResponseProvider: func() interface{} {
			return &MappingResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*MappingRequest); ok {
				var dsRequest = dsunit.MappingRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.MappingRequest); ok {
				resp := s.Service.AddTableMapping(req)
				response := MappingResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "init",
		RequestInfo: &endly.ActionInfo{
			Description: "initialize datastore (register, recreated, run sql, add mapping)",

			Examples: []*endly.UseCase{
				{
					Description: "mysql init",
					Data:        dsunitMySQLInitExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &InitRequest{}
		},
		ResponseProvider: func() interface{} {
			return &InitResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*InitRequest); ok {
				var dsRequest = dsunit.InitRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.InitRequest); ok {
				resp := s.Service.Init(req)
				response := InitResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "prepare",
		RequestInfo: &endly.ActionInfo{
			Description: "populate databstore with provided data",
			Examples: []*endly.UseCase{
				{
					Description: "static data prepare",
					Data:        dsunitServiceStaticDataPrepareExample,
				},
				{
					Description: "data prepare",
					Data:        dsunitDataPrepareExaple,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &PrepareRequest{}
		},
		ResponseProvider: func() interface{} {
			return &PrepareResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*PrepareRequest); ok {
				var dsRequest = dsunit.PrepareRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.PrepareRequest); ok {
				resp := s.Service.Prepare(req)
				response := PrepareResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "expect",
		RequestInfo: &endly.ActionInfo{
			Description: "verify databstore with provided data",
			Examples: []*endly.UseCase{
				{
					Description: "static data expect",
					Data:        dsunitServiceExpectAction,
				},
				{
					Description: "data expect",
					Data:        dsunitDataPrepareExaple,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &dsunit.ExpectRequest{
				DatasetResource: &dsunit.DatasetResource{
					DatastoreDatasets: &dsunit.DatastoreDatasets{},
				},
			}
		},
		ResponseProvider: func() interface{} {
			return &dsunit.ExpectResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*ExpectRequest); ok {
				var dsRequest = dsunit.ExpectRequest(*req)
				request = &dsRequest
			}

			if req, ok := request.(*dsunit.ExpectRequest); ok {
				resp := s.Service.Expect(req)
				response := ExpectResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "query",
		RequestInfo: &endly.ActionInfo{
			Description: "run SQL query",
		},
		RequestProvider: func() interface{} {
			return &QueryRequest{}
		},
		ResponseProvider: func() interface{} {
			return &QueryResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*QueryRequest); ok {
				var dsRequest = dsunit.QueryRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.QueryRequest); ok {
				resp := s.Service.Query(req)
				response := QueryResponse(*resp)
				var err = response.Error()
				if req.IgnoreError {
					err = nil
				}
				return &response, err
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})

	s.Register(&endly.Route{
		Action: "sequence",
		RequestInfo: &endly.ActionInfo{
			Description: "get sequence for supplied tables",
			Examples: []*endly.UseCase{
				{
					Description: "sequence",
					Data:        dsunitServiceSequenceExample,
				},
			},
		},
		RequestProvider: func() interface{} {
			return &SequenceRequest{}
		},
		ResponseProvider: func() interface{} {
			return &SequenceResponse{}
		},
		Handler: func(context *endly.Context, request interface{}) (interface{}, error) {
			if req, ok := request.(*SequenceRequest); ok {
				var dsRequest = dsunit.SequenceRequest(*req)
				request = &dsRequest
			}
			if req, ok := request.(*dsunit.SequenceRequest); ok {
				resp := s.Service.Sequence(req)
				response := SequenceResponse(*resp)
				return &response, response.Error()
			}
			return nil, fmt.Errorf("unsupported request type: %T", request)
		},
	})
}

func (s service) Run(context *endly.Context, request interface{}) *endly.ServiceResponse {
	var state = context.State()
	context.Context.Replace((*data.Map)(nil), &state)
	s.Service.SetContext(context.Context)
	return s.AbstractService.Run(context, request)
}

//New creates a new Datastore unit service
func New() endly.Service {
	var result = &service{
		AbstractService: endly.NewAbstractService(ServiceID),
		Service:         dsunit.New(),
	}
	result.AbstractService.Service = result
	result.registerRoutes()
	return result
}
<|endoftext|>


<commit_before>package examples

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/pborman/uuid"

	"github.com/BTrDB/btrdb"
)

func TestInsertingProceduralData(t *testing.T) {
	//First connect to the cluster. In BTrDB v4 we are advocating that all
	//programs use environment variables to specify the endpoint rather
	//than assuming specific addresses:
	//Set $BTRDB_ENDPOINTS to
	//"server1:4410;server2:4410..."
	//Note that not all endpoints need be listed, but it will make this
	//program more resilient if you specify more or all of the endpoints
	db, err := btrdb.Connect(context.TODO(), btrdb.EndpointsFromEnv()...)
	if err != nil {
		t.Fatalf("Unexpected connection error: %v", err)
	}

	//Streams must be created before use
	uu := uuid.NewRandom()
	//A collection is a small group of streams (<100 is best) generally associated
	//with a single device or service. BTrDB is designed for lots of small collections
	//not small numbers of big collections
	collection := fmt.Sprintf("test/inserting_procedural_data.%d", time.Now().UnixNano())
	//Tags are used to identify streams within a collection
	tags := btrdb.M{"key": "value", "anotherkey": "anothervalue"}
	//The annotation is used to store (mutable) extra data with the stream. It
	//is technically just a byte array, but we prefer people use msgpacked objects.
	//the tooling is not quite there to make this easy, so its ok to make this nil
	//for now
	var annotation map[string]string = nil

	stream, err := db.Create(context.TODO(), uu, collection, tags, annotation)
	if err != nil {
		t.Fatalf("Unexpected creation error: %v", err)
	}

	//Now you manipulate the stream:
	err = stream.InsertTV(context.TODO(),
		[]int64{100e6, 200e6, 300e6, 400e6},
		[]float64{1.1, 2.2, 3.3, 4.4})
	if err != nil {
		t.Fatalf("Unexpected insert error: %v", err)
	}

	//Start = -1000ns, End = 1000ns, Width = 150ns, Depth = 2^0 (all the way), Version = latest
	rvchan, ver, errc := stream.Windows(context.TODO(), -1000, 1000, 150, 0, btrdb.LatestVersion)
	_ = ver //don't use this, that's ok
	for result := range rvchan {
		fmt.Printf("Window @%d min=%.2f mean=%.2f max=%.2f count=%d\n",
			result.Time, result.Min, result.Mean, result.Max, result.Count)
	}
	if e := <-errc; e != nil {
		t.Fatalf("Got an error: %v", e)
	}
}
<commit_msg>tweak<commit_after>package examples

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/pborman/uuid"

	"github.com/BTrDB/btrdb"
)

func TestInsertingProceduralData(t *testing.T) {
	//First connect to the cluster. In BTrDB v4 we are advocating that all
	//programs use environment variables to specify the endpoint rather
	//than assuming specific addresses:
	//Set $BTRDB_ENDPOINTS to
	//"server1:4410;server2:4410..."
	//Note that not all endpoints need be listed, but it will make this
	//program more resilient if you specify more or all of the endpoints
	db, err := btrdb.Connect(context.TODO(), btrdb.EndpointsFromEnv()...)
	if err != nil {
		t.Fatalf("Unexpected connection error: %v", err)
	}

	//Streams must be created before use
	uu := uuid.NewRandom()
	//A collection is a small group of streams (<100 is best) generally associated
	//with a single device or service. BTrDB is designed for lots of small collections
	//not small numbers of big collections
	collection := fmt.Sprintf("test/inserting_procedural_data.%d", time.Now().UnixNano())
	//Tags are used to identify streams within a collection
	tags := btrdb.M{"key": "value", "anotherkey": "anothervalue"}
	//The annotation is used to store (mutable) extra data with the stream. It
	//is technically just a byte array, but we prefer people use msgpacked objects.
	//the tooling is not quite there to make this easy, so its ok to make this nil
	//for now
	var annotation map[string]string = nil

	stream, err := db.Create(context.TODO(), uu, collection, tags, annotation)
	if err != nil {
		t.Fatalf("Unexpected creation error: %v", err)
	}

	//Now you manipulate the stream:
	err = stream.InsertTV(context.TODO(),
		[]int64{100e6, 200e6, 300e6, 400e6},
		[]float64{1.1, 2.2, 3.3, 4.4})
	if err != nil {
		t.Fatalf("Unexpected insert error: %v", err)
	}

	//Start = -1000ns, End = 1000ns, Width = 150ns, Depth = 2^0 (all the way), Version = latest
	rvchan, ver, errc := stream.Windows(context.TODO(), -1000e6, 1000e6, 150e6, 0, btrdb.LatestVersion)
	_ = ver //don't use this, that's ok
	for result := range rvchan {
		fmt.Printf("Window @%d min=%.2f mean=%.2f max=%.2f count=%d\n",
			result.Time, result.Min, result.Mean, result.Max, result.Count)
	}
	if e := <-errc; e != nil {
		t.Fatalf("Got an error: %v", e)
	}
}
<|endoftext|>


<commit_before>package examples

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/pborman/uuid"

	"github.com/BTrDB/btrdb"
)

func TestInsertingProceduralData(t *testing.T) {
	//First connect to the cluster. In BTrDB v4 we are advocating that all
	//programs use environment variables to specify the endpoint rather
	//than assuming specific addresses:
	//Set $BTRDB_ENDPOINTS to
	//"server1:4410;server2:4410..."
	//Note that not all endpoints need be listed, but it will make this
	//program more resilient if you specify more or all of the endpoints
	db, err := btrdb.Connect(context.TODO(), btrdb.EndpointsFromEnv()...)
	if err != nil {
		t.Fatalf("Unexpected connection error: %v", err)
	}

	//Streams must be created before use
	uu := uuid.NewRandom()
	//A collection is a small group of streams (<100 is best) generally associated
	//with a single device or service. BTrDB is designed for lots of small collections
	//not small numbers of big collections
	collection := fmt.Sprintf("test.inserting_procedural_data.%d", time.Now().UnixNano())
	//Tags are used to identify streams within a collection
	tags := btrdb.M{"key": "value", "anotherkey": "anothervalue"}
	//The annotation is used to store (mutable) extra data with the stream. It
	//is technically just a byte array, but we prefer people use msgpacked objects.
	//the tooling is not quite there to make this easy, so its ok to make this nil
	//for now
	var annotation map[string]string = nil

	stream, err := db.Create(context.TODO(), uu, collection, tags, annotation)
	if err != nil {
		t.Fatalf("Unexpected creation error: %v", err)
	}

	//Now you manipulate the stream:
	err = stream.InsertTV(context.TODO(),
		[]int64{100, 200, 300, 400},
		[]float64{1.1, 2.2, 3.3, 4.4})
	if err != nil {
		t.Fatalf("Unexpected insert error: %v", err)
	}

	//Start = -1000ns, End = 1000ns, Width = 150ns, Depth = 2^0 (all the way), Version = latest
	rvchan, ver, errc := stream.Windows(context.TODO(), -1000, 1000, 150, 0, btrdb.LatestVersion)
	_ = ver //don't use this, that's ok
	for result := range rvchan {
		fmt.Printf("Window @%d min=%.2f mean=%.2f max=%.2f count=%d\n",
			result.Time, result.Min, result.Mean, result.Max, result.Count)
	}
	if e := <-errc; e != nil {
		t.Fatalf("Got an error: %v", e)
	}
}
<commit_msg>Update example to produce a stream that works for mrplotter<commit_after>package examples

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/pborman/uuid"

	"github.com/BTrDB/btrdb"
)

func TestInsertingProceduralData(t *testing.T) {
	//First connect to the cluster. In BTrDB v4 we are advocating that all
	//programs use environment variables to specify the endpoint rather
	//than assuming specific addresses:
	//Set $BTRDB_ENDPOINTS to
	//"server1:4410;server2:4410..."
	//Note that not all endpoints need be listed, but it will make this
	//program more resilient if you specify more or all of the endpoints
	db, err := btrdb.Connect(context.TODO(), btrdb.EndpointsFromEnv()...)
	if err != nil {
		t.Fatalf("Unexpected connection error: %v", err)
	}

	//Streams must be created before use
	uu := uuid.NewRandom()
	//A collection is a small group of streams (<100 is best) generally associated
	//with a single device or service. BTrDB is designed for lots of small collections
	//not small numbers of big collections
	collection := fmt.Sprintf("test/inserting_procedural_data.%d", time.Now().UnixNano())
	//Tags are used to identify streams within a collection
	tags := btrdb.M{"key": "value", "anotherkey": "anothervalue"}
	//The annotation is used to store (mutable) extra data with the stream. It
	//is technically just a byte array, but we prefer people use msgpacked objects.
	//the tooling is not quite there to make this easy, so its ok to make this nil
	//for now
	var annotation map[string]string = nil

	stream, err := db.Create(context.TODO(), uu, collection, tags, annotation)
	if err != nil {
		t.Fatalf("Unexpected creation error: %v", err)
	}

	//Now you manipulate the stream:
	err = stream.InsertTV(context.TODO(),
		[]int64{100e6, 200e6, 300e6, 400e6},
		[]float64{1.1, 2.2, 3.3, 4.4})
	if err != nil {
		t.Fatalf("Unexpected insert error: %v", err)
	}

	//Start = -1000ns, End = 1000ns, Width = 150ns, Depth = 2^0 (all the way), Version = latest
	rvchan, ver, errc := stream.Windows(context.TODO(), -1000, 1000, 150, 0, btrdb.LatestVersion)
	_ = ver //don't use this, that's ok
	for result := range rvchan {
		fmt.Printf("Window @%d min=%.2f mean=%.2f max=%.2f count=%d\n",
			result.Time, result.Min, result.Mean, result.Max, result.Count)
	}
	if e := <-errc; e != nil {
		t.Fatalf("Got an error: %v", e)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"net"
	"regexp"

	"google.golang.org/genproto/googleapis/bytestream"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	_ "google.golang.org/grpc/encoding/gzip" // Register gzip support.
	"google.golang.org/grpc/status"

	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	"github.com/bazelbuild/remote-apis/build/bazel/semver"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
)

const (
	hashKeyLength = 64
	emptySha256   = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
)

var (
	// Cache keys must be lower case asciified SHA256 sums.
	hashKeyRegex = regexp.MustCompile("^[a-f0-9]{64}$")
)

type grpcServer struct {
	cache        *disk.DiskCache
	accessLogger cache.Logger
	errorLogger  cache.Logger
	depsCheck    bool
}

func ListenAndServeGRPC(addr string, opts []grpc.ServerOption,
	validateACDeps bool,
	c *disk.DiskCache, a cache.Logger, e cache.Logger) error {

	listener, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}

	return serveGRPC(listener, opts, validateACDeps, c, a, e)
}

func serveGRPC(l net.Listener, opts []grpc.ServerOption,
	validateACDepsCheck bool,
	c *disk.DiskCache, a cache.Logger, e cache.Logger) error {

	srv := grpc.NewServer(opts...)
	s := &grpcServer{
		cache: c, accessLogger: a, errorLogger: e,
		depsCheck: validateACDepsCheck,
	}
	pb.RegisterActionCacheServer(srv, s)
	pb.RegisterCapabilitiesServer(srv, s)
	pb.RegisterContentAddressableStorageServer(srv, s)
	bytestream.RegisterByteStreamServer(srv, s)
	return srv.Serve(l)
}

// Capabilities interface:

func (s *grpcServer) GetCapabilities(ctx context.Context,
	req *pb.GetCapabilitiesRequest) (*pb.ServerCapabilities, error) {

	// Instance name is currently ignored.

	resp := pb.ServerCapabilities{
		CacheCapabilities: &pb.CacheCapabilities{
			DigestFunction: []pb.DigestFunction_Value{pb.DigestFunction_SHA256},
			ActionCacheUpdateCapabilities: &pb.ActionCacheUpdateCapabilities{
				UpdateEnabled: true,
			},
			CachePriorityCapabilities: &pb.PriorityCapabilities{
				Priorities: []*pb.PriorityCapabilities_PriorityRange{
					{
						MinPriority: 0,
						MaxPriority: 0,
					},
				},
			},
			MaxBatchTotalSizeBytes:      0, // "no limit"
			SymlinkAbsolutePathStrategy: pb.SymlinkAbsolutePathStrategy_ALLOWED,
		},
		LowApiVersion:  &semver.SemVer{Major: int32(2)},
		HighApiVersion: &semver.SemVer{Major: int32(2), Minor: int32(1)},
	}

	s.accessLogger.Printf("GRPC GETCAPABILITIES")

	return &resp, nil
}

// Return an error if `hash` is not a valid cache key.
func (s *grpcServer) validateHash(hash string, size int64, logPrefix string) error {
	if size == int64(0) {
		if hash == emptySha256 {
			return nil
		}

		msg := "Invalid zero-length SHA256 hash"
		s.accessLogger.Printf("%s %s: %s", logPrefix, hash, msg)
		return status.Error(codes.InvalidArgument, msg)
	}

	if len(hash) != hashKeyLength {
		msg := fmt.Sprintf("Hash length must be length %d", hashKeyLength)
		s.accessLogger.Printf("%s %s: %s", logPrefix, hash, msg)
		return status.Error(codes.InvalidArgument, msg)
	}

	if !hashKeyRegex.MatchString(hash) {
		msg := "Malformed hash"
		s.accessLogger.Printf("%s %s: %s", logPrefix, hash, msg)
		return status.Error(codes.InvalidArgument, msg)
	}

	return nil
}
<commit_msg>grpc.go: add godoc-style comment for ListenAndServeGRPC<commit_after>package server

import (
	"context"
	"fmt"
	"net"
	"regexp"

	"google.golang.org/genproto/googleapis/bytestream"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	_ "google.golang.org/grpc/encoding/gzip" // Register gzip support.
	"google.golang.org/grpc/status"

	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	"github.com/bazelbuild/remote-apis/build/bazel/semver"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
)

const (
	hashKeyLength = 64
	emptySha256   = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
)

var (
	// Cache keys must be lower case asciified SHA256 sums.
	hashKeyRegex = regexp.MustCompile("^[a-f0-9]{64}$")
)

type grpcServer struct {
	cache        *disk.DiskCache
	accessLogger cache.Logger
	errorLogger  cache.Logger
	depsCheck    bool
}

// ListenAndServeGRPC creates a new gRPC server and listens on the given
// address. This function either returns an error quickly, or triggers a
// blocking call to https://godoc.org/google.golang.org/grpc#Server.Serve
func ListenAndServeGRPC(addr string, opts []grpc.ServerOption,
	validateACDeps bool,
	c *disk.DiskCache, a cache.Logger, e cache.Logger) error {

	listener, err := net.Listen("tcp", addr)
	if err != nil {
		return err
	}

	return serveGRPC(listener, opts, validateACDeps, c, a, e)
}

func serveGRPC(l net.Listener, opts []grpc.ServerOption,
	validateACDepsCheck bool,
	c *disk.DiskCache, a cache.Logger, e cache.Logger) error {

	srv := grpc.NewServer(opts...)
	s := &grpcServer{
		cache: c, accessLogger: a, errorLogger: e,
		depsCheck: validateACDepsCheck,
	}
	pb.RegisterActionCacheServer(srv, s)
	pb.RegisterCapabilitiesServer(srv, s)
	pb.RegisterContentAddressableStorageServer(srv, s)
	bytestream.RegisterByteStreamServer(srv, s)
	return srv.Serve(l)
}

// Capabilities interface:

func (s *grpcServer) GetCapabilities(ctx context.Context,
	req *pb.GetCapabilitiesRequest) (*pb.ServerCapabilities, error) {

	// Instance name is currently ignored.

	resp := pb.ServerCapabilities{
		CacheCapabilities: &pb.CacheCapabilities{
			DigestFunction: []pb.DigestFunction_Value{pb.DigestFunction_SHA256},
			ActionCacheUpdateCapabilities: &pb.ActionCacheUpdateCapabilities{
				UpdateEnabled: true,
			},
			CachePriorityCapabilities: &pb.PriorityCapabilities{
				Priorities: []*pb.PriorityCapabilities_PriorityRange{
					{
						MinPriority: 0,
						MaxPriority: 0,
					},
				},
			},
			MaxBatchTotalSizeBytes:      0, // "no limit"
			SymlinkAbsolutePathStrategy: pb.SymlinkAbsolutePathStrategy_ALLOWED,
		},
		LowApiVersion:  &semver.SemVer{Major: int32(2)},
		HighApiVersion: &semver.SemVer{Major: int32(2), Minor: int32(1)},
	}

	s.accessLogger.Printf("GRPC GETCAPABILITIES")

	return &resp, nil
}

// Return an error if `hash` is not a valid cache key.
func (s *grpcServer) validateHash(hash string, size int64, logPrefix string) error {
	if size == int64(0) {
		if hash == emptySha256 {
			return nil
		}

		msg := "Invalid zero-length SHA256 hash"
		s.accessLogger.Printf("%s %s: %s", logPrefix, hash, msg)
		return status.Error(codes.InvalidArgument, msg)
	}

	if len(hash) != hashKeyLength {
		msg := fmt.Sprintf("Hash length must be length %d", hashKeyLength)
		s.accessLogger.Printf("%s %s: %s", logPrefix, hash, msg)
		return status.Error(codes.InvalidArgument, msg)
	}

	if !hashKeyRegex.MatchString(hash) {
		msg := "Malformed hash"
		s.accessLogger.Printf("%s %s: %s", logPrefix, hash, msg)
		return status.Error(codes.InvalidArgument, msg)
	}

	return nil
}
<|endoftext|>