<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/rand"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"

	"upspin.io/flags"
	"upspin.io/upspin"
)

// This file implements the initial configuration for a new domain.

func (s *State) setupdomain(args ...string) {
	const (
		help = `
Setupdomain generates keys and rc files for the Upspin users upspin-dir@domain
and upspin-store@domain, and generates a signature to be added as a DNS TXT
record to prove that the calling Upspin user has control over domain.

If any state exists at the given location (-where) then the command aborts.

If you intend to deploy to a Google Cloud Platform project you must specify the
project ID with -project. This permits later steps to find the generated keys
and configuration files.

TODO: how to complete the process with 'upspin user -put'

Once the domain has been set up and its servers deployed, use setupwriters to
set access controls.
`
	)
	fs := flag.NewFlagSet("setupdomain", flag.ExitOnError)
	where := fs.String("where", filepath.Join(os.Getenv("HOME"), "upspin", "deploy"), "`directory` to store private configuration files")
	curveName := fs.String("curve", "p256", "cryptographic curve `name`: p256, p384, or p521")
	putUsers := fs.Bool("put-users", false, "put server users to the key server")
	s.parseFlags(fs, args, help, "[-project=<gcp_project_name>] setupdomain [-where=$HOME/upspin/deploy] <domain_name>")
	if fs.NArg() != 1 {
		fs.Usage()
	}
	if *where == "" {
		s.failf("the -where flag must not be empty")
		fs.Usage()
	}
	domain := fs.Arg(0)
	if domain == "" {
		s.failf("domain must be provided")
		fs.Usage()
	}
	switch *curveName {
	case "p256", "p384", "p521":
		// OK
	default:
		s.exitf("no such curve %q", *curveName)
	}

	dstDir := *where
	dstDir = filepath.Join(dstDir, flags.Project)

	dirServerPath := filepath.Join(dstDir, "dirserver")
	storeServerPath := filepath.Join(dstDir, "storeserver")

	if *putUsers {
		s.exitf("at the present moment in time this is not now currently implemented, yet")
		// However it may be, one day.
		return
	}

	s.shouldNotExist(dirServerPath)
	s.shouldNotExist(storeServerPath)
	s.mkdirAllLocal(dirServerPath)
	s.mkdirAllLocal(storeServerPath)

	// Generate keys for the dirserver and the storeserver.
	var noProquint string
	dirPublic, dirPrivate, _, err := createKeys(*curveName, noProquint)
	if err != nil {
		s.exit(err)
	}
	storePublic, storePrivate, _, err := createKeys(*curveName, noProquint)
	if err != nil {
		s.exit(err)
	}
	err = writeKeys(dirServerPath, dirPublic, dirPrivate)
	if err != nil {
		s.exit(err)
	}
	err = writeKeys(storeServerPath, storePublic, storePrivate)
	if err != nil {
		s.exit(err)
	}

	// Generate and write symmetric key for DirServer data.
	var symmSecret [32]byte
	_, err = rand.Read(symmSecret[:])
	if err != nil {
		s.exit(err)
	}
	err = ioutil.WriteFile(filepath.Join(dirServerPath, "symmsecret.upspinkey"), symmSecret[:], 0600)
	if err != nil {
		s.exit(err)
	}

	// Generate rc files for those users.
	err = ioutil.WriteFile(filepath.Join(storeServerPath, "rc"),
		[]byte(fmt.Sprintf(rcFormat, "upspin-store", domain, domain, domain, "plain", storeServerPath)), 0600)
	if err != nil {
		s.exit(err)
	}
	err = ioutil.WriteFile(filepath.Join(dirServerPath, "rc"),
		[]byte(fmt.Sprintf(rcFormat, "upspin-dir", domain, domain, domain, "symm", dirServerPath)), 0600)
	if err != nil {
		s.exit(err)
	}

	// Generate signature.
	msg := "upspin-domain:" + domain + "-" + string(s.context.UserName())
	sig, err := s.context.Factotum().Sign([]byte(msg))
	if err != nil {
		s.exit(err)
	}

	err = setupDomainTemplate.Execute(os.Stdout, setupDomainData{
		Dir:       dstDir,
		Where:     *where,
		Domain:    domain,
		Project:   flags.Project,
		UserName:  s.context.UserName(),
		Signature: fmt.Sprintf("%x-%x", sig.R, sig.S),
	})
	if err != nil {
		s.exit(err)
	}
}

const (
	rcFormat = `username: %s@%s
dirserver: remote,dir.%s
storeserver: remote,store.%s
packing: %s
secrets: %s
`
)

type setupDomainData struct {
	Dir, Where string
	Domain     string
	Project    string
	UserName   upspin.UserName
	Signature  string
}

var setupDomainTemplate = template.Must(template.New("setupdomain").Parse(`
Keys and rc files for the users
	upspin-dir@{{.Domain}}
	upspin-store@{{.Domain}}
were generated and placed under the directory:
	{{.Dir}}

To prove that {{.UserName}} is the owner of {{.Domain}},
add the following record to {{.Domain}}'s DNS zone:

	NAME	TYPE	TTL	DATA
	@	TXT	1h	upspin:{{.Signature}}

Once the DNS change propagates the key server will use the TXT record to verify
that {{.UserName}} is authorized to register users under {{.Domain}}.
To register the users listed above, run this command:

	$ upspin -project={{.Project}} setupdomain -where={{.Where}} -put-users {{.Domain}}
`))
<commit_msg>cmd/upspin: implement setupdomain -put-users<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/rand"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"

	yaml "gopkg.in/yaml.v2"

	"upspin.io/context"
	"upspin.io/flags"
	"upspin.io/upspin"
)

// This file implements the initial configuration for a new domain.

func (s *State) setupdomain(args ...string) {
	const (
		help = `
Setupdomain generates keys and rc files for the Upspin users upspin-dir@domain
and upspin-store@domain, and generates a signature to be added as a DNS TXT
record to prove that the calling Upspin user has control over domain.

If any state exists at the given location (-where) then the command aborts.

If you intend to deploy to a Google Cloud Platform project you must specify the
project ID with -project. This permits later steps to find the generated keys
and configuration files.

TODO: how to complete the process with 'upspin user -put'

Once the domain has been set up and its servers deployed, use setupwriters to
set access controls.
`
	)
	fs := flag.NewFlagSet("setupdomain", flag.ExitOnError)
	where := fs.String("where", filepath.Join(os.Getenv("HOME"), "upspin", "deploy"), "`directory` to store private configuration files")
	curveName := fs.String("curve", "p256", "cryptographic curve `name`: p256, p384, or p521")
	putUsers := fs.Bool("put-users", false, "put server users to the key server")
	s.parseFlags(fs, args, help, "[-project=<gcp_project_name>] setupdomain [-where=$HOME/upspin/deploy] <domain_name>")
	if fs.NArg() != 1 {
		fs.Usage()
	}
	if *where == "" {
		s.failf("the -where flag must not be empty")
		fs.Usage()
	}
	domain := fs.Arg(0)
	if domain == "" {
		s.failf("domain must be provided")
		fs.Usage()
	}
	switch *curveName {
	case "p256", "p384", "p521":
		// OK
	default:
		s.exitf("no such curve %q", *curveName)
	}

	var (
		dirServerPath   = filepath.Join(*where, flags.Project, "dirserver")
		storeServerPath = filepath.Join(*where, flags.Project, "storeserver")
		dirRC           = filepath.Join(dirServerPath, "rc")
		storeRC         = filepath.Join(storeServerPath, "rc")
	)

	if *putUsers {
		dirFile, dirUser, err := writeUserFile(dirRC)
		if err != nil {
			s.exit(err)
		}
		storeFile, storeUser, err := writeUserFile(storeRC)
		if err != nil {
			s.exit(err)
		}
		s.user("-put", "-in", dirFile)
		os.Remove(dirFile)
		s.user("-put", "-in", storeFile)
		os.Remove(storeFile)
		fmt.Printf("Successfully put %q and %q to the key server.\n", dirUser, storeUser)
		return
	}

	s.shouldNotExist(dirServerPath)
	s.shouldNotExist(storeServerPath)
	s.mkdirAllLocal(dirServerPath)
	s.mkdirAllLocal(storeServerPath)

	// Generate keys for the dirserver and the storeserver.
	var noProquint string
	dirPublic, dirPrivate, _, err := createKeys(*curveName, noProquint)
	if err != nil {
		s.exit(err)
	}
	storePublic, storePrivate, _, err := createKeys(*curveName, noProquint)
	if err != nil {
		s.exit(err)
	}
	err = writeKeys(dirServerPath, dirPublic, dirPrivate)
	if err != nil {
		s.exit(err)
	}
	err = writeKeys(storeServerPath, storePublic, storePrivate)
	if err != nil {
		s.exit(err)
	}

	// Generate and write symmetric key for DirServer data.
	var symmSecret [32]byte
	_, err = rand.Read(symmSecret[:])
	if err != nil {
		s.exit(err)
	}
	err = ioutil.WriteFile(filepath.Join(dirServerPath, "symmsecret.upspinkey"), symmSecret[:], 0600)
	if err != nil {
		s.exit(err)
	}

	// Generate rc files for those users.
	err = ioutil.WriteFile(storeRC,
		[]byte(fmt.Sprintf(rcFormat, "upspin-store", domain, domain, domain, "plain", storeServerPath)), 0600)
	if err != nil {
		s.exit(err)
	}
	err = ioutil.WriteFile(dirRC,
		[]byte(fmt.Sprintf(rcFormat, "upspin-dir", domain, domain, domain, "symm", dirServerPath)), 0600)
	if err != nil {
		s.exit(err)
	}

	// Generate signature.
	msg := "upspin-domain:" + domain + "-" + string(s.context.UserName())
	sig, err := s.context.Factotum().Sign([]byte(msg))
	if err != nil {
		s.exit(err)
	}

	err = setupDomainTemplate.Execute(os.Stdout, setupDomainData{
		Dir:       filepath.Join(*where, flags.Project),
		Where:     *where,
		Domain:    domain,
		Project:   flags.Project,
		UserName:  s.context.UserName(),
		Signature: fmt.Sprintf("%x-%x", sig.R, sig.S),
	})
	if err != nil {
		s.exit(err)
	}
}

const (
	rcFormat = `username: %s@%s
dirserver: remote,dir.%s
storeserver: remote,store.%s
packing: %s
secrets: %s
`
)

type setupDomainData struct {
	Dir, Where string
	Domain     string
	Project    string
	UserName   upspin.UserName
	Signature  string
}

var setupDomainTemplate = template.Must(template.New("setupdomain").Parse(`
Keys and rc files for the users
	upspin-dir@{{.Domain}}
	upspin-store@{{.Domain}}
were generated and placed under the directory:
	{{.Dir}}

To prove that {{.UserName}} is the owner of {{.Domain}},
add the following record to {{.Domain}}'s DNS zone:

	NAME	TYPE	TTL	DATA
	@	TXT	1h	upspin:{{.Signature}}

Once the DNS change propagates the key server will use the TXT record to verify
that {{.UserName}} is authorized to register users under {{.Domain}}.
To register the users listed above, run this command:

	$ upspin -project={{.Project}} setupdomain -where={{.Where}} -put-users {{.Domain}}
`))

// writeUserFile reads the specified rc file and writes a YAML-encoded
// upspin.User to userFile. It also returns the username.
func writeUserFile(rcFile string) (userFile string, u upspin.UserName, err error) {
	ctx, err := context.FromFile(rcFile)
	if err != nil {
		return "", "", err
	}
	b, err := yaml.Marshal(context.User(ctx))
	if err != nil {
		return "", "", err
	}
	f, err := ioutil.TempFile("", "setupdomain-user")
	if err != nil {
		return "", "", err
	}
	if _, err := f.Write(b); err != nil {
		os.Remove(f.Name())
		return "", "", err
	}
	if err := f.Close(); err != nil {
		os.Remove(f.Name())
		return "", "", err
	}
	return f.Name(), ctx.UserName(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"path"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

/* This function takes a string and returns
   a (potentially nil) error object */
func TTWS(filename string, verbose bool) error {
	/* Open the input file */
	inf, err := os.Open(filename);
	/* In case this function generates a "panic", be sure to close this file */
	defer inf.Close();
	/* Did we open it successfully?  If not, close and return. */
	if (err!=nil) { return err; }

	data, err := ioutil.ReadAll(inf);
	inf.Close();

	fileType := http.DetectContentType(data);
	if (!strings.Contains(fileType, "text/plain")) {
		if (verbose) { fmt.Printf("Skipping file of type '%v': %v\n", fileType, filename); }
		return nil;
	}

	/* Open the output file in system temp dir*/
	outf, err := ioutil.TempFile("","");
	/* In case this function generates a "panic", be sure to close this file */
	defer outf.Close();
	/* Did we open it succesfully?  If not, close all and return. */
	if (err!=nil) { return err; }

	for _, line := range(strings.Split(string(data), "\n")) {
		line = strings.TrimRight(line, " \t")+"\n"
		outf.Write([]byte(line));
	}

	outf.Close();

	/* Replace the source file by the trimmed file */
	err = os.Rename(outf.Name(), filename);
	if (err!=nil) { return err; }

	if (verbose) { fmt.Printf("Trimmed %s\n", filename); }
	/* No errors, so we return nil */
	return nil;
}

var blacklist = []string{".bzr", ".cvs", ".git", ".hg", ".svn"}

func processNode(node string, verbose bool) error {
	fi, err := os.Lstat(node)
	if (err!=nil) { return err; }

	if (fi.IsDir()) {
		if contains(fi.Name(), blacklist) { return nil; }
		contents, err := ioutil.ReadDir(node);
		if (err!=nil) { return err; }
		for _, n := range(contents) {
			serr := processNode(path.Join(node, n.Name()), verbose);
			if (serr!=nil) { return serr; }
		}
		return nil;
	} else {
		return TTWS(node, verbose);
	}
}

func contains(x string, a []string) bool {
	for _, e := range(a) {
		if (x==e) { return true; }
	}
	return false;
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	err := processNode(root, false);
	fmt.Printf("processNode("+root+") returned %v\n", err);
}
<commit_msg>Added profiling support...but it doesn't seem to work<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"path"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"runtime/pprof"
)

/* This function takes a string and returns
   a (potentially nil) error object */
func TTWS(filename string, verbose bool) error {
	/* Open the input file */
	inf, err := os.Open(filename);
	/* In case this function generates a "panic", be sure to close this file */
	defer inf.Close();
	/* Did we open it successfully?  If not, close and return. */
	if (err!=nil) { return err; }

	data, err := ioutil.ReadAll(inf);
	inf.Close();

	fileType := http.DetectContentType(data);
	if (!strings.Contains(fileType, "text/plain")) {
		if (verbose) { fmt.Printf("Skipping file of type '%v': %v\n", fileType, filename); }
		return nil;
	}

	/* Open the output file in system temp dir*/
	outf, err := ioutil.TempFile("","");
	/* In case this function generates a "panic", be sure to close this file */
	defer outf.Close();
	/* Did we open it succesfully?  If not, close all and return. */
	if (err!=nil) { return err; }

	lines := strings.Split(string(data), "\n")
	nlines := len(lines)
	for i, line := range(lines) {
		//fmt.Println("Original line: '"+line+"'");

		/* Trim whitespace */
		line = strings.TrimRight(line, " \t")
		/* Don't add a \n to the last line if it is empty */
		if (i<nlines-1 || len(line)>0) { line = line+"\n" }
		outf.Write([]byte(line));

		//fmt.Println(" Trimmed line: '"+line+"'");
	}

	outf.Close();

	/* Replace the source file by the trimmed file */
	err = os.Rename(outf.Name(), filename);
	if (err!=nil) { return err; }

	if (verbose) { fmt.Printf("Trimmed %s\n", filename); }
	/* No errors, so we return nil */
	return nil;
}

var blacklist = []string{".bzr", ".cvs", ".git", ".hg", ".svn"}

func processNode(node string, verbose bool) error {
	fi, err := os.Lstat(node)
	if (err!=nil) { return err; }

	if (fi.IsDir()) {
		if contains(fi.Name(), blacklist) { return nil; }
		contents, err := ioutil.ReadDir(node);
		if (err!=nil) { return err; }
		for _, n := range(contents) {
			serr := processNode(path.Join(node, n.Name()), verbose);
			if (serr!=nil) { return serr; }
		}
		return nil;
	} else {
		return TTWS(node, verbose);
	}
}

func contains(x string, a []string) bool {
	for _, e := range(a) {
		if (x==e) { return true; }
	}
	return false;
}

func run(root string, verbose bool) {
	err := processNode(root, verbose);
	fmt.Printf("processNode("+root+") returned %v\n", err);
}

func main() {
	var verbose = flag.Bool("verbose", false, "request verbose output")
	var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

	flag.Parse()

	root := flag.Arg(0)

	if (*cpuprofile!="") {
		f, err := os.Create(*cpuprofile)
        if err != nil {
            log.Fatal(err)
        }

		fmt.Println("Starting profiling");
		pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()

		run(root, *verbose);

		pprof.StopCPUProfile();
		fmt.Println("...done profiling");
		f.Close();
	} else {
		run(root, *verbose);
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/go-sql-driver/mysql"
	"database/sql/driver"
	"fmt"
	"time"
	"sync"
	"runtime"
	"flag"
//	"database/sql"
//  _ "github.com/go-sql-driver/mysql"
//  _ "net/http/pprof"
//  "net/http"
	"math/big"
	"math"
)

type TestResult struct {
	connOk bool
	queryOk bool
	connTime time.Duration
	queryTime time.Duration
}

type SummeryResult struct {
	count                 int64

	connFailCount         int64
	totalConnTime         time.Duration
	totalSquareConnTime   big.Int
	maxConnTime           time.Duration
	minConnTime           time.Duration
	avgConnTime           time.Duration
	stddevConnTime        time.Duration

	queryFailCount        int64
	totalQueryTime        time.Duration
	totalSquareQueryTime  big.Int
	maxQueryTime          time.Duration
	minQueryTime          time.Duration
	avgQueryTime          time.Duration
	stddevQueryTime       time.Duration
}

func testOnce(dsn, query string) TestResult {
	result := TestResult{}
	beforeConn := time.Now()
	db, err := (mysql.MySQLDriver{}).Open(dsn)
	if err != nil {
		//fmt.Println(err.Error())
		result.connOk = false
		return result
	}
	result.connOk = true
	afterConn := time.Now()
	result.connTime = afterConn.Sub(beforeConn)
	defer db.Close()

	rows, err := db.(driver.Queryer).Query(query, []driver.Value{})
	if err != nil {
		//fmt.Println(err.Error())
		result.queryOk = false
		return result
	}
	result.queryOk = true
	defer rows.Close()
	afterQuery := time.Now()
	result.queryTime = afterQuery.Sub(afterConn)
	return result
}

func testRoutine(dsn, query string, n int, outChan chan<- TestResult) {
	for i := 0; i < n; i++ {
		outChan <- testOnce(dsn, query)
	}
}

func summeryRoutine(inChan <-chan TestResult, outChan chan<- SummeryResult, summeryIntervalSecond int) {
	var ret   SummeryResult
	var bigA  big.Int
	ret.minConnTime = math.MaxInt64
	ret.minQueryTime = math.MaxInt64
	var ticker *time.Ticker
	if summeryIntervalSecond > 0 {
		summeryInterval := time.Second * time.Duration(summeryIntervalSecond)
		ticker = time.NewTicker(summeryInterval)
	}
	for result := range inChan {
		ret.count++
		if result.connOk {
			if result.connTime > ret.maxConnTime {
				ret.maxConnTime = result.connTime
			}
			if result.connTime < ret.minConnTime {
				ret.minConnTime = result.connTime
			}
			ret.totalConnTime+= result.connTime
			bigA.SetInt64((int64)(result.connTime)).Mul(&bigA, &bigA)
			ret.totalSquareConnTime.Add(&ret.totalSquareConnTime, &bigA)
		} else {
			ret.connFailCount++
		}
		if result.queryOk {
			if result.queryTime > ret.maxQueryTime {
				ret.maxQueryTime = result.queryTime
			}
			if result.queryTime < ret.minQueryTime {
				ret.minQueryTime = result.queryTime
			}
			ret.totalQueryTime+= result.queryTime
			bigA.SetInt64((int64)(result.queryTime)).Mul(&bigA, &bigA)
			ret.totalSquareQueryTime.Add(&ret.totalSquareQueryTime, &bigA)
		} else {
			ret.queryFailCount++
		}
		if summeryIntervalSecond > 0 {
			if _, ok := <-ticker.C; ok {
				ret.Summery()
				outChan<-ret
				ret = SummeryResult{}
			}
		}
	}
	ret.Summery()
	outChan<-ret
	return
}

func (self *SummeryResult) Summery() {
	var bigA, big2 big.Int
	var bigR1, bigR2, big1N, big1N1 big.Rat
	big2.SetInt64(2)
	//  ∑(i-miu)2 = ∑(i2)-(∑i)2/n
	n := self.count - self.connFailCount
	if n > 1 {
		self.avgConnTime = (time.Duration)((int64)(self.totalConnTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalConnTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareConnTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevConnTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}

	n = self.count - self.queryFailCount
	if n > 1 {
		self.avgQueryTime = (time.Duration)((int64)(self.totalQueryTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalQueryTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareQueryTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevQueryTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}

	if self.minConnTime == math.MaxInt64 {
		self.minConnTime = 0
	}
	if self.minQueryTime == math.MaxInt64 {
		self.minQueryTime = 0
	}
}

func msStr(t time.Duration) string {
	return fmt.Sprintf("%0.3f ms", float64(int64(t) / 1000) / 1000.0)
}

type NullLogger struct{}
func (*NullLogger) Print(v ...interface{}) {
}

// mysqlburst -c 2000 -r 30 -d 'mha:M616VoUJBnYFi0L02Y24@tcp(10.200.180.54:3342)/x?timeout=5s&readTimeout=3s&writeTimeout=3s'
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//go func() {
	//      http.ListenAndServe("localhost:6060", nil)
	//}()

	procs := 0
	rounds := 0
	dsn := ""
	query := ""
	summeryIntervalSec := 0
	flag.IntVar(&procs, "c", 1000, "concurrency")
	flag.IntVar(&rounds, "r", 100, "rounds")
	flag.StringVar(&dsn, "d", "mysql:@tcp(127.0.0.1:3306)/mysql?timeout=5s&readTimeout=5s&writeTimeout=5s", "dsn")
	flag.StringVar(&query, "q", "select 1", "sql")
	flag.IntVar(&summeryIntervalSec, "i", 0, "summery interval (sec)")
	flag.Parse()

	mysql.SetLogger(&NullLogger{})

	wg := sync.WaitGroup{}
	wg.Add(procs)
	resultChan := make(chan TestResult, 5000)
	summeryChan := make(chan SummeryResult, 10)

	go func() {
		for i := 0; i < procs; i++ {
			go func() {
				testRoutine(dsn, query, rounds, resultChan)
				wg.Done()
			}()
		}
		wg.Wait()
		close(resultChan)
	}()
	go summeryRoutine(resultChan, summeryChan, summeryIntervalSec)

	testBegin := time.Now()
	for summery := range summeryChan {
		testEnd := time.Now()
		duration:= testEnd.Sub(testBegin)
		testBegin = testEnd

		fmt.Printf("test time: %s\n", duration.String())
		fmt.Printf("total tests: %d\n", summery.count);
		fmt.Printf("failed connections: %d\n", summery.connFailCount);
		fmt.Printf("failed queries: %d\n", summery.queryFailCount);

		fmt.Println("connect time")
		fmt.Printf("avg: %s\n", msStr(summery.avgConnTime))
		fmt.Printf("min: %s\n", msStr(summery.minConnTime))
		fmt.Printf("max: %s\n", msStr(summery.maxConnTime))
		fmt.Printf("stddev: %s\n", msStr(summery.stddevConnTime))
		fmt.Println()
		fmt.Println("query time")
		fmt.Printf("avg: %s\n", msStr(summery.avgQueryTime))
		fmt.Printf("min: %s\n", msStr(summery.minQueryTime))
		fmt.Printf("max: %s\n", msStr(summery.maxQueryTime))
		fmt.Printf("stddev: %s\n", msStr(summery.stddevQueryTime))
		fmt.Println()
	}

}

<commit_msg>query time; read time; extract stages<commit_after>package main

import (
	"github.com/go-sql-driver/mysql"
	"database/sql/driver"
	"fmt"
	"time"
	"sync"
	"runtime"
	"flag"
//	"database/sql"
//  _ "github.com/go-sql-driver/mysql"
//  _ "net/http/pprof"
//  "net/http"
	"math/big"
	"math"
	"os"
	"io"
)

/*
type TestResult struct {
	connOk bool
	queryOk bool
	connTime time.Duration
	queryTime time.Duration
}*/

const (
	STAGE_CONN  byte = 0
	STAGE_QUERY byte = 1
	STAGE_READ  byte = 2
	STAGE_TOTAL byte = 3
	//
	STAGE_MAX   byte = 4
)

type TestResult struct {
	stage                  byte
	ok                     bool
	time                   time.Duration
}

/*
type SummeryResult struct {
	count                 int64

	connFailCount         int64
	totalConnTime         time.Duration
	totalSquareConnTime   big.Int
	maxConnTime           time.Duration
	minConnTime           time.Duration
	avgConnTime           time.Duration
	stddevConnTime        time.Duration

	queryFailCount        int64
	totalQueryTime        time.Duration
	totalSquareQueryTime  big.Int
	maxQueryTime          time.Duration
	minQueryTime          time.Duration
	avgQueryTime          time.Duration
	stddevQueryTime       time.Duration
}*/

type SummeryResult struct {
	stage             byte
	count             int64

	failCount         int64
	totalTime         time.Duration
	totalSquareTime   big.Int
	maxTime           time.Duration
	minTime           time.Duration
	avgTime           time.Duration
	stddevTime        time.Duration
}

func getColumnCount(dsn, query string) (int, error) {
	db, err := (mysql.MySQLDriver{}).Open(dsn)
	if err != nil {
		return 0, err
	}
	defer db.Close()

	rows, err := db.(driver.Queryer).Query(query, []driver.Value{})
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	return len(rows.Columns()), nil
}

func testOnce(dsn, query string, row []driver.Value, result [STAGE_MAX]TestResult) {
	result[STAGE_TOTAL].ok = false
	beforeConn := time.Now()
	db, err := (mysql.MySQLDriver{}).Open(dsn)
	if err != nil {
		//fmt.Println(err.Error())
		result[STAGE_CONN].ok = false
		return
	}
	result[STAGE_CONN].ok = true
	afterConn := time.Now()
	result[STAGE_CONN].time = afterConn.Sub(beforeConn)
	defer db.Close()

	rows, err := db.(driver.Queryer).Query(query, []driver.Value{})
	if err != nil {
		//fmt.Println(err.Error())
		result[STAGE_QUERY].ok = false
		return
	}
	afterQuery := time.Now()
	result[STAGE_QUERY].ok = true
	result[STAGE_QUERY].time = afterQuery.Sub(afterConn)
	defer rows.Close()
	for {
		err = rows.Next(row)
		if err != nil {
			break
		}
	}
	if err != io.EOF {
		result[STAGE_QUERY].ok = false
	} else {
		afterRead := time.Now()
		result[STAGE_READ].ok = true
		result[STAGE_TOTAL].ok = true
		result[STAGE_READ].time = afterRead.Sub(afterQuery)
		result[STAGE_TOTAL].time = afterRead.Sub(beforeConn)
	}
}

/*
func testOnce(dsn, query string, row []driver.Value) TestResult {
	result := TestResult{}
	beforeConn := time.Now()
	db, err := (mysql.MySQLDriver{}).Open(dsn)
	if err != nil {
		//fmt.Println(err.Error())
		result.connOk = false
		return result
	}
	result.connOk = true
	afterConn := time.Now()
	result.connTime = afterConn.Sub(beforeConn)
	defer db.Close()

	rows, err := db.(driver.Queryer).Query(query, []driver.Value{})
	if err != nil {
		//fmt.Println(err.Error())
		result.queryOk = false
		return result
	}
	afterQuery := time.Now()
	result.queryTime = afterQuery.Sub(afterConn)
	result.queryOk = true
	defer rows.Close()
	for {
		err = rows.Next(row)
		if err != nil {
			break
		}
	}
	// result.queryTime = afterQuery.Sub(afterConn)
	return result
}*/

func testRoutine(dsn, query string, n int, colNum int, outChan chan<- [STAGE_MAX]TestResult) {
	var result [STAGE_MAX]TestResult
	result[STAGE_CONN].stage = STAGE_CONN
	result[STAGE_QUERY].stage = STAGE_QUERY
	result[STAGE_READ].stage = STAGE_READ
	result[STAGE_TOTAL].stage = STAGE_TOTAL

	row := make([]driver.Value, colNum)
	for i := 0; i < n; i++ {
		testOnce(dsn, query, row, result)
		outChan <-result
	}
}

func summeryRoutine(inChan <-chan [STAGE_MAX]TestResult, outChan chan<- [STAGE_MAX]SummeryResult, summeryIntervalSecond int) {
	var ret   [STAGE_MAX]SummeryResult
	var bigA  big.Int
	var ticker *time.Ticker
	for i := byte(0); i < STAGE_MAX; i++ {
		ret[i].minTime = math.MaxInt64
		ret[i].stage = (byte)(i)
	}

	if summeryIntervalSecond > 0 {
		summeryInterval := time.Second * time.Duration(summeryIntervalSecond)
		ticker = time.NewTicker(summeryInterval)
	}
	for result := range inChan {
		for i := byte(0); i < STAGE_MAX; i++ {
			ret[i].count++
			if result[i].ok {
				if result[i].time > ret[i].maxTime {
					ret[i].maxTime = result[i].time
				}
				if result[i].time < ret[i].minTime {
					ret[i].minTime = result[i].time
				}
				ret[i].totalTime+= result[i].time
				bigA.SetInt64((int64)(result[i].time)).Mul(&bigA, &bigA)
				ret[i].totalSquareTime.Add(&ret[i].totalSquareTime, &bigA)
			} else {
				ret[i].failCount++
			}

			if summeryIntervalSecond > 0 {
				if _, ok := <-ticker.C; ok {
					ret[i].Summery()
					outChan<-ret
					ret = [STAGE_MAX]SummeryResult{}
				}
			}
		}
	}
	for i := byte(0); i < STAGE_MAX; i++ {
		ret[i].Summery()
	}
	outChan<-ret
	return
}

func (self *SummeryResult) Summery() {
	var bigA, big2 big.Int
	var bigR1, bigR2, big1N, big1N1 big.Rat
	big2.SetInt64(2)
	//  ∑(i-miu)2 = ∑(i2)-(∑i)2/n
	n := self.count - self.failCount
	if n > 1 {
		self.avgTime = (time.Duration)((int64)(self.totalTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}
	if self.minTime == math.MaxInt64 {
		self.minTime = 0
	}
}

/*
func summeryRoutine(inChan <-chan TestResult, outChan chan<- SummeryResult, summeryIntervalSecond int) {
	var ret   SummeryResult
	var bigA  big.Int
	ret.minConnTime = math.MaxInt64
	ret.minQueryTime = math.MaxInt64
	var ticker *time.Ticker
	if summeryIntervalSecond > 0 {
		summeryInterval := time.Second * time.Duration(summeryIntervalSecond)
		ticker = time.NewTicker(summeryInterval)
	}
	for result := range inChan {
		ret.count++
		if result.connOk {
			if result.connTime > ret.maxConnTime {
				ret.maxConnTime = result.connTime
			}
			if result.connTime < ret.minConnTime {
				ret.minConnTime = result.connTime
			}
			ret.totalConnTime+= result.connTime
			bigA.SetInt64((int64)(result.connTime)).Mul(&bigA, &bigA)
			ret.totalSquareConnTime.Add(&ret.totalSquareConnTime, &bigA)
		} else {
			ret.connFailCount++
		}
		if result.queryOk {
			if result.queryTime > ret.maxQueryTime {
				ret.maxQueryTime = result.queryTime
			}
			if result.queryTime < ret.minQueryTime {
				ret.minQueryTime = result.queryTime
			}
			ret.totalQueryTime+= result.queryTime
			bigA.SetInt64((int64)(result.queryTime)).Mul(&bigA, &bigA)
			ret.totalSquareQueryTime.Add(&ret.totalSquareQueryTime, &bigA)
		} else {
			ret.queryFailCount++
		}
		if summeryIntervalSecond > 0 {
			if _, ok := <-ticker.C; ok {
				ret.Summery()
				outChan<-ret
				ret = SummeryResult{}
			}
		}
	}
	ret.Summery()
	outChan<-ret
	return
}

func (self *SummeryResult) Summery() {
	var bigA, big2 big.Int
	var bigR1, bigR2, big1N, big1N1 big.Rat
	big2.SetInt64(2)
	//  ∑(i-miu)2 = ∑(i2)-(∑i)2/n
	n := self.count - self.connFailCount
	if n > 1 {
		self.avgConnTime = (time.Duration)((int64)(self.totalConnTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalConnTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareConnTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevConnTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}

	n = self.count - self.queryFailCount
	if n > 1 {
		self.avgQueryTime = (time.Duration)((int64)(self.totalQueryTime) / n)

		big1N.SetInt64(n).Inv(&big1N) // 1/n
		big1N1.SetInt64(n-1).Inv(&big1N1) // 1/(n-1)
		bigA.SetInt64((int64)(self.totalQueryTime)).Mul(&bigA, &bigA) // (∑i)2
		bigR1.SetInt(&bigA).Mul(&bigR1, &big1N) // (∑i)2/n
		bigR2.SetInt(&self.totalSquareQueryTime).Sub(&bigR2, &bigR1)
		s2, _ := bigR2.Mul(&bigR2, &big1N1).Float64()
		self.stddevQueryTime = (time.Duration)((int64)(math.Sqrt(s2)))
	}

	if self.minConnTime == math.MaxInt64 {
		self.minConnTime = 0
	}
	if self.minQueryTime == math.MaxInt64 {
		self.minQueryTime = 0
	}
}*/

func msStr(t time.Duration) string {
	return fmt.Sprintf("%0.3f ms", float64(int64(t) / 1000) / 1000.0)
}

type NullLogger struct{}
func (*NullLogger) Print(v ...interface{}) {
}

// mysqlburst -c 2000 -r 30 -d 'mha:M616VoUJBnYFi0L02Y24@tcp(10.200.180.54:3342)/x?timeout=5s&readTimeout=3s&writeTimeout=3s'
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	//go func() {
	//      http.ListenAndServe("localhost:6060", nil)
	//}()

	procs := 0
	rounds := 0
	dsn := ""
	query := ""
	summeryIntervalSec := 0
	flag.IntVar(&procs, "c", 1000, "concurrency")
	flag.IntVar(&rounds, "r", 100, "rounds")
	flag.StringVar(&dsn, "d", "mysql:@tcp(127.0.0.1:3306)/mysql?timeout=5s&readTimeout=5s&writeTimeout=5s", "dsn")
	flag.StringVar(&query, "q", "select 1", "sql")
	flag.IntVar(&summeryIntervalSec, "i", 0, "summery interval (sec)")
	flag.Parse()

	mysql.SetLogger(&NullLogger{})

	colNum, err := getColumnCount(dsn, query)
	if err != nil {
		fmt.Printf("init failed: %s", err)
		os.Exit(2)
	}
	wg := sync.WaitGroup{}
	wg.Add(procs)
	resultChan := make(chan [STAGE_MAX]TestResult, 5000)
	summeryChan := make(chan [STAGE_MAX]SummeryResult, 10)

	go func() {
		for i := 0; i < procs; i++ {
			go func() {
				testRoutine(dsn, query, rounds, colNum, resultChan)
				wg.Done()
			}()
		}
		wg.Wait()
		close(resultChan)
	}()
	go summeryRoutine(resultChan, summeryChan, summeryIntervalSec)

	testBegin := time.Now()
	titles := [STAGE_MAX]string{
		"connect", "query", "read", "total",
	}
	for summery := range summeryChan {
		testEnd := time.Now()
		duration:= testEnd.Sub(testBegin)
		testBegin = testEnd

		fmt.Printf("time: %s\n", duration.String())
		//fmt.Printf("tests: %d\n", summery.count);
		for i, title := range titles {
			fmt.Println(title)
			fmt.Printf("tests: %d\n", summery[i].count);
			fmt.Printf("failed: %d\n", summery[i].failCount);
			fmt.Printf("avg time: %s\n", msStr(summery[i].avgTime))
			fmt.Printf("min time: %s\n", msStr(summery[i].minTime))
			fmt.Printf("max time: %s\n", msStr(summery[i].maxTime))
			fmt.Printf("stddev time: %s\n", msStr(summery[i].stddevTime))
		}
		/*
		fmt.Println("connect time")
		fmt.Printf("failed: %d\n", summery.connFailCount);
		fmt.Printf("avg: %s\n", msStr(summery.avgConnTime))
		fmt.Printf("min: %s\n", msStr(summery.minConnTime))
		fmt.Printf("max: %s\n", msStr(summery.maxConnTime))
		fmt.Printf("stddev: %s\n", msStr(summery.stddevConnTime))

		fmt.Println()
		fmt.Println("query time")
		fmt.Printf("failed: %d\n", summery.queryFailCount);
		fmt.Printf("avg: %s\n", msStr(summery.avgQueryTime))
		fmt.Printf("min: %s\n", msStr(summery.minQueryTime))
		fmt.Printf("max: %s\n", msStr(summery.maxQueryTime))
		fmt.Printf("stddev: %s\n", msStr(summery.stddevQueryTime))
		fmt.Println()
		*/
	}

}

<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	fabric "github.com/nimona/go-nimona-fabric"
)

func main() {
	ctx := context.Background()

	peerA, err := newPeer(0, 0, "PeerA")
	if err != nil {
		log.Fatal("Could not create peer A", err)
	}

	peerB,  err := newPeer(0, 0, "PeerB")
	if err != nil {
		log.Fatal("Could not create peer B", err)
	}

	log.Println("Peer A address:", peerA.GetAddresses())

	for _, addr := range peerA.GetAddresses() {
		endpoint := addr + "/tls/router/ping"
		log.Println("-------- Dialing", endpoint)
		if _, _, err := peerB.DialContext(context.Background(), endpoint); err != nil {
			log.Fatal("Dial error", err)
		}
	}
}

func newPeer(tcpPort, wsPort int, peerID string) (*fabric.Fabric,  error) {
	crt, err := GenX509KeyPair()
	if err != nil {
		fmt.Println("Cert creation error", err)
		return nil, err
	}

	yamux := &fabric.YamuxMiddleware{}
	router := &fabric.RouterMiddleware{}
	identity := &fabric.IdentityMiddleware{Local: peerID}
	tls := &fabric.SecMiddleware{
		Config: tls.Config{
			Certificates:       []tls.Certificate{crt},
			InsecureSkipVerify: true,
		},
	}
	ping := &Ping{}

	f := fabric.New(tls, router)
	f.AddTransport(fabric.NewTransportTCP("0.0.0.0", tcpPort))
	f.AddTransport(fabric.NewTransportWebsocket(fmt.Sprintf("0.0.0.0:%d", wsPort)))

	f.AddMiddleware(yamux)
	f.AddMiddleware(identity)
	f.AddMiddleware(ping)

	f.AddHandlerFunc("ping", ping.Handle)
	f.AddHandlerFunc("identity/ping", ping.Handle)

	if err := peerA.Listen(ctx); err != nil {
		log.Fatal("Could not listen for peer A", err)
	}

	return f,  nil
}
<commit_msg>Fix example<commit_after>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	fabric "github.com/nimona/go-nimona-fabric"
)

func main() {
	peerA, err := newPeer(0, 0, "PeerA")
	if err != nil {
		log.Fatal("Could not create peer A", err)
	}

	peerB, err := newPeer(0, 0, "PeerB")
	if err != nil {
		log.Fatal("Could not create peer B", err)
	}

	log.Println("Peer A address:", peerA.GetAddresses())

	for _, addr := range peerA.GetAddresses() {
		endpoint := addr + "/tls/router/ping"
		log.Println("-------- Dialing", endpoint)
		if _, _, err := peerB.DialContext(context.Background(), endpoint); err != nil {
			log.Fatal("Dial error", err)
		}
	}
}

func newPeer(tcpPort, wsPort int, peerID string) (*fabric.Fabric, error) {
	ctx := context.Background()
	crt, err := GenX509KeyPair()
	if err != nil {
		fmt.Println("Cert creation error", err)
		return nil, err
	}

	yamux := &fabric.YamuxMiddleware{}
	router := &fabric.RouterMiddleware{}
	identity := &fabric.IdentityMiddleware{Local: peerID}
	tls := &fabric.SecMiddleware{
		Config: tls.Config{
			Certificates:       []tls.Certificate{crt},
			InsecureSkipVerify: true,
		},
	}
	ping := &Ping{}

	f := fabric.New(tls, router)
	f.AddTransport(fabric.NewTransportTCP("0.0.0.0", tcpPort))
	f.AddTransport(fabric.NewTransportWebsocket(fmt.Sprintf("0.0.0.0:%d", wsPort)))

	f.AddMiddleware(yamux)
	f.AddMiddleware(identity)
	f.AddMiddleware(ping)

	f.AddHandlerFunc("ping", ping.Handle)
	f.AddHandlerFunc("identity/ping", ping.Handle)

	if err := f.Listen(ctx); err != nil {
		log.Fatal("Could not listen for peer A", err)
	}

	return f, nil
}
<|endoftext|>


<commit_before>package config

import (
	"time"

	"github.com/corestoreio/csfw/store/scope"
)

// ScopedGetter is equal to Getter but the underlying implementation takes
// care of providing the correct scope: default, website or store and bubbling
// up the scope chain from store -> website -> default.
//
// This interface is mainly implemented in the store package. The functions
// should be the same as in Getter but only the different is the paths
// argument. A path can be either one string containing a valid path like a/b/c
// or it can consists of 3 path parts like "a", "b", "c". All other arguments
// are invalid. Returned error is mostly of ErrKeyNotFound.
type ScopedGetter interface {
	// Scope tells you the current underlying scope and its website, group or store ID
	Scope() (scope.Scope, int64)
	String(paths ...string) (string, error)
	Bool(paths ...string) (bool, error)
	Float64(paths ...string) (float64, error)
	Int(paths ...string) (int, error)
	DateTime(paths ...string) (time.Time, error)
}

type scopedService struct {
	root      Getter
	websiteID int64
	groupID   int64
	storeID   int64
}

var _ ScopedGetter = (*scopedService)(nil)

func newScopedService(r Getter, websiteID, groupID, storeID int64) scopedService {
	return scopedService{
		root:      r,
		websiteID: websiteID,
		groupID:   groupID,
		storeID:   storeID,
	}
}

// Scope tells you the current underlying scope and its website, group or store ID
func (ss scopedService) Scope() (scope.Scope, int64) {
	switch {
	case ss.storeID > 0:
		return scope.StoreID, ss.storeID
	case ss.groupID > 0:
		return scope.GroupID, ss.groupID
	case ss.websiteID > 0:
		return scope.WebsiteID, ss.websiteID
	default:
		return scope.DefaultID, 0
	}
}

// String returns a string. Enable debug logging to see possible errors.
func (ss scopedService) String(paths ...string) (string, error) {
	return ss.root.String(Scope(ss.Scope()), Path(paths...))
}

// Bool returns a bool value. Enable debug logging to see possible errors.
func (ss scopedService) Bool(paths ...string) (bool, error) {
	return ss.root.Bool(Scope(ss.Scope()), Path(paths...))
}

// Float64 returns a float number. Enable debug logging for possible errors.
func (ss scopedService) Float64(paths ...string) (float64, error) {
	return ss.root.Float64(Scope(ss.Scope()), Path(paths...))
}

// Int returns an int. Enable debug logging for possible errors.
func (ss scopedService) Int(paths ...string) (int, error) {
	return ss.root.Int(Scope(ss.Scope()), Path(paths...))
}

// DateTime returns a time. Enable debug logging for possible errors.
func (ss scopedService) DateTime(paths ...string) (time.Time, error) {
	return ss.root.DateTime(Scope(ss.Scope()), Path(paths...))
}
<commit_msg>config: Interface ScopedGetter embeds scope.Scoper for smaller interfaces<commit_after>package config

import (
	"time"

	"github.com/corestoreio/csfw/store/scope"
)

// ScopedGetter is equal to Getter but the underlying implementation takes
// care of providing the correct scope: default, website or store and bubbling
// up the scope chain from store -> website -> default.
//
// This interface is mainly implemented in the store package. The functions
// should be the same as in Getter but only the different is the paths
// argument. A path can be either one string containing a valid path like a/b/c
// or it can consists of 3 path parts like "a", "b", "c". All other arguments
// are invalid. Returned error is mostly of ErrKeyNotFound.
type ScopedGetter interface {
	scope.Scoper
	String(paths ...string) (string, error)
	Bool(paths ...string) (bool, error)
	Float64(paths ...string) (float64, error)
	Int(paths ...string) (int, error)
	DateTime(paths ...string) (time.Time, error)
}

type scopedService struct {
	root      Getter
	websiteID int64
	groupID   int64
	storeID   int64
}

var _ ScopedGetter = (*scopedService)(nil)

func newScopedService(r Getter, websiteID, groupID, storeID int64) scopedService {
	return scopedService{
		root:      r,
		websiteID: websiteID,
		groupID:   groupID,
		storeID:   storeID,
	}
}

// Scope tells you the current underlying scope and its website, group or store ID
func (ss scopedService) Scope() (scope.Scope, int64) {
	switch {
	case ss.storeID > 0:
		return scope.StoreID, ss.storeID
	case ss.groupID > 0:
		return scope.GroupID, ss.groupID
	case ss.websiteID > 0:
		return scope.WebsiteID, ss.websiteID
	default:
		return scope.DefaultID, 0
	}
}

// String returns a string. Enable debug logging to see possible errors.
func (ss scopedService) String(paths ...string) (string, error) {
	return ss.root.String(Scope(ss.Scope()), Path(paths...))
}

// Bool returns a bool value. Enable debug logging to see possible errors.
func (ss scopedService) Bool(paths ...string) (bool, error) {
	return ss.root.Bool(Scope(ss.Scope()), Path(paths...))
}

// Float64 returns a float number. Enable debug logging for possible errors.
func (ss scopedService) Float64(paths ...string) (float64, error) {
	return ss.root.Float64(Scope(ss.Scope()), Path(paths...))
}

// Int returns an int. Enable debug logging for possible errors.
func (ss scopedService) Int(paths ...string) (int, error) {
	return ss.root.Int(Scope(ss.Scope()), Path(paths...))
}

// DateTime returns a time. Enable debug logging for possible errors.
func (ss scopedService) DateTime(paths ...string) (time.Time, error) {
	return ss.root.DateTime(Scope(ss.Scope()), Path(paths...))
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"
	"go.uber.org/thriftrw/compile"
)

// TypeConverter can generate a function body that converts two thriftrw
// FieldGroups from one to another. It's assumed that the converted code
// operates on two variables, "in" and "out" and that both are a go struct.
type TypeConverter struct {
	Lines  []string
	Helper PackageNameResolver
}

// PackageNameResolver interface allows for resolving what the
// package name for a thrift file is. This depends on where the
// thrift-based structs are generated.
type PackageNameResolver interface {
	TypePackageName(thriftFile string) (string, error)
}

func (c *TypeConverter) getGoTypeName(
	valueType compile.TypeSpec,
) (string, error) {
	switch s := valueType.(type) {
	case *compile.BoolSpec:
		return "bool", nil
	case *compile.I8Spec:
		return "int8", nil
	case *compile.I16Spec:
		return "int16", nil
	case *compile.I32Spec:
		return "int32", nil
	case *compile.I64Spec:
		return "int64", nil
	case *compile.DoubleSpec:
		return "float64", nil
	case *compile.StringSpec:
		return "string", nil
	case *compile.BinarySpec:
		return "[]byte", nil
	case *compile.MapSpec:
		panic("Not Implemented")
	case *compile.SetSpec:
		panic("Not Implemented")
	case *compile.ListSpec:
		panic("Not Implemented")
	case *compile.EnumSpec, *compile.StructSpec, *compile.TypedefSpec:
		return c.getIdentifierName(s)
	default:
		panic(fmt.Sprintf("Unknown type (%T) %v", valueType, valueType))
	}
}

func (c *TypeConverter) getIdentifierName(
	fieldType compile.TypeSpec,
) (string, error) {
	pkgName, err := c.Helper.TypePackageName(fieldType.ThriftFile())
	if err != nil {
		return "", errors.Wrapf(
			err,
			"could not lookup fieldType when building converter for %s :",
			fieldType.ThriftName(),
		)
	}
	typeName := pkgName + "." + fieldType.ThriftName()
	return typeName, nil
}

func (c *TypeConverter) genConverterForStruct(
	toFieldName string,
	toFieldType *compile.StructSpec,
	fromFieldType compile.TypeSpec,
	fromIdentifier string,
	keyPrefix string,
	indent string,
) error {
	toIdentifier := indent + "out." + keyPrefix

	typeName, err := c.getIdentifierName(toFieldType)
	if err != nil {
		return err
	}
	subToFields := toFieldType.Fields

	fromFieldStruct, ok := fromFieldType.(*compile.StructSpec)
	if !ok {
		return errors.Errorf(
			"could not convert struct fields, "+
				"incompatible type for %s :",
			toFieldName,
		)
	}

	line := "if " + fromIdentifier + " != nil {"
	c.Lines = append(c.Lines, line)

	line = "	" + toIdentifier + " = &" + typeName + "{}"
	c.Lines = append(c.Lines, line)

	subFromFields := fromFieldStruct.Fields
	err = c.genStructConverter(
		keyPrefix+".",
		indent+"	",
		subFromFields,
		subToFields,
	)
	if err != nil {
		return err
	}

	line = "} else {"
	c.Lines = append(c.Lines, line)

	line = "	" + toIdentifier + " = nil"
	c.Lines = append(c.Lines, line)

	line = "}"
	c.Lines = append(c.Lines, line)

	return nil
}

func (c *TypeConverter) genConverterForPrimitive(
	toField *compile.FieldSpec,
	toIdentifier string,
	fromIdentifier string,
) error {
	var line string
	typeName, err := c.getGoTypeName(toField.Type)
	if err != nil {
		return err
	}

	if toField.Required {
		line = toIdentifier + " = " + typeName + "(" + fromIdentifier + ")"
	} else {
		line = toIdentifier + " = (*" + typeName + ")(" + fromIdentifier + ")"
	}
	c.Lines = append(c.Lines, line)
	return nil
}

func (c *TypeConverter) genStructConverter(
	keyPrefix string,
	indent string,
	fromFields []*compile.FieldSpec,
	toFields []*compile.FieldSpec,
) error {
	for i := 0; i < len(toFields); i++ {
		toField := toFields[i]

		var fromField *compile.FieldSpec
		for j := 0; j < len(fromFields); j++ {
			if fromFields[j].Name == toField.Name {
				fromField = fromFields[j]
				break
			}
		}

		if fromField == nil {
			return errors.Errorf(
				"cannot map by name for the field %s",
				toField.Name,
			)
		}

		toIdentifier := indent + "out." + keyPrefix + strings.Title(toField.Name)
		fromIdentifier := "in." + keyPrefix + strings.Title(fromField.Name)

		// Override thrift type names to avoid naming collisions between endpoint
		// and client types.
		switch toFieldType := toField.Type.(type) {
		case
			*compile.BoolSpec,
			*compile.I8Spec,
			*compile.I16Spec,
			*compile.I32Spec,
			*compile.EnumSpec,
			*compile.I64Spec,
			*compile.DoubleSpec,
			*compile.StringSpec:

			err := c.genConverterForPrimitive(
				toField, toIdentifier, fromIdentifier,
			)
			if err != nil {
				return err
			}
		case *compile.BinarySpec:
			line := toIdentifier + " = []byte(" + fromIdentifier + ")"
			c.Lines = append(c.Lines, line)
		case *compile.TypedefSpec:
			typeName, err := c.getIdentifierName(toField.Type)
			if err != nil {
				return err
			}

			var line string
			// TODO: typedef for struct is invalid here ...
			if toField.Required {
				line = toIdentifier + " = " + typeName + "(" + fromIdentifier + ")"
			} else {
				line = toIdentifier + " = (*" + typeName + ")(" + fromIdentifier + ")"
			}
			c.Lines = append(c.Lines, line)

		case *compile.StructSpec:
			err := c.genConverterForStruct(
				toField.Name,
				toFieldType,
				fromField.Type,
				fromIdentifier,
				keyPrefix+strings.Title(toField.Name),
				indent,
			)
			if err != nil {
				return err
			}
		case *compile.ListSpec:
			typeName, err := c.getGoTypeName(toFieldType.ValueSpec)
			if err != nil {
				return err
			}

			valueStruct, isStruct := toFieldType.ValueSpec.(*compile.StructSpec)
			if isStruct {
				line := toIdentifier + " = make([]*" +
					typeName + ", len(" + fromIdentifier + "))"
				c.Lines = append(c.Lines, line)
			} else {
				line := toIdentifier + " = make([]" +
					typeName + ", len(" + fromIdentifier + "))"
				c.Lines = append(c.Lines, line)
			}

			line := "for index, value := range " + fromIdentifier + " {"
			c.Lines = append(c.Lines, line)

			if isStruct {
				fromFieldType, ok := fromField.Type.(*compile.ListSpec)
				if !ok {
					return errors.Errorf(
						"Could not convert field (%s): type is not list",
						fromField.Name,
					)
				}

				err = c.genConverterForStruct(
					toField.Name,
					valueStruct,
					fromFieldType.ValueSpec,
					"value",
					keyPrefix+strings.Title(toField.Name)+"[index]",
					indent,
				)
				if err != nil {
					return err
				}
			} else {
				line = toIdentifier + "[index] = " + typeName + "(value)"
				c.Lines = append(c.Lines, line)
			}

			line = "}"
			c.Lines = append(c.Lines, line)

		case *compile.MapSpec:
			typeName, err := c.getGoTypeName(toFieldType.ValueSpec)
			if err != nil {
				return err
			}

			_, isStringKey := toFieldType.KeySpec.(*compile.StringSpec)
			if !isStringKey {
				return errors.Errorf(
					"could not convert key (%s), map is not string-keyed.",
					toField.Name,
				)
			}

			_, isStruct := toFieldType.ValueSpec.(*compile.StructSpec)
			if isStruct {
				line := toIdentifier + " = make(map[string]*" +
					typeName + ", len(" + fromIdentifier + "))"
				c.Lines = append(c.Lines, line)
			} else {
				line := toIdentifier + " = make(map[string]" +
					typeName + ", len(" + fromIdentifier + "))"
				c.Lines = append(c.Lines, line)
			}

			line := "for key, value := range " + fromIdentifier + " {"
			c.Lines = append(c.Lines, line)

			if isStruct {
				// TODO: need to deep copy struct here.
				line = toIdentifier + "[key] = (*" + typeName + ")(value)"
				c.Lines = append(c.Lines, line)
			} else {
				line = toIdentifier + "[key] = " + typeName + "(value)"
				c.Lines = append(c.Lines, line)
			}

			line = "}"
			c.Lines = append(c.Lines, line)

		default:
			// fmt.Printf("Unknown type %s for field %s \n",
			// 	toField.Type.TypeCode().String(), toField.Name,
			// )

			// pkgName, err := h.TypePackageName(toField.Type.ThriftFile())
			// if err != nil {
			// 	return nil, err
			// }
			// typeName := pkgName + "." + toField.Type.ThriftName()
			// line := prefix + "(*" + typeName + ")" + postfix
			// c.Lines = append(c.Lines, line)
		}
	}

	return nil
}

// GenStructConverter will add lines to the TypeConverter for mapping
// from one go struct to another based on two thriftrw.FieldGroups
func (c *TypeConverter) GenStructConverter(
	fromFields []*compile.FieldSpec,
	toFields []*compile.FieldSpec,
) error {
	err := c.genStructConverter("", "", fromFields, toFields)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>codegen/type_converter: move list generation to a method<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package codegen

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"
	"go.uber.org/thriftrw/compile"
)

// TypeConverter can generate a function body that converts two thriftrw
// FieldGroups from one to another. It's assumed that the converted code
// operates on two variables, "in" and "out" and that both are a go struct.
type TypeConverter struct {
	Lines  []string
	Helper PackageNameResolver
}

// PackageNameResolver interface allows for resolving what the
// package name for a thrift file is. This depends on where the
// thrift-based structs are generated.
type PackageNameResolver interface {
	TypePackageName(thriftFile string) (string, error)
}

func (c *TypeConverter) getGoTypeName(
	valueType compile.TypeSpec,
) (string, error) {
	switch s := valueType.(type) {
	case *compile.BoolSpec:
		return "bool", nil
	case *compile.I8Spec:
		return "int8", nil
	case *compile.I16Spec:
		return "int16", nil
	case *compile.I32Spec:
		return "int32", nil
	case *compile.I64Spec:
		return "int64", nil
	case *compile.DoubleSpec:
		return "float64", nil
	case *compile.StringSpec:
		return "string", nil
	case *compile.BinarySpec:
		return "[]byte", nil
	case *compile.MapSpec:
		panic("Not Implemented")
	case *compile.SetSpec:
		panic("Not Implemented")
	case *compile.ListSpec:
		panic("Not Implemented")
	case *compile.EnumSpec, *compile.StructSpec, *compile.TypedefSpec:
		return c.getIdentifierName(s)
	default:
		panic(fmt.Sprintf("Unknown type (%T) %v", valueType, valueType))
	}
}

func (c *TypeConverter) getIdentifierName(
	fieldType compile.TypeSpec,
) (string, error) {
	pkgName, err := c.Helper.TypePackageName(fieldType.ThriftFile())
	if err != nil {
		return "", errors.Wrapf(
			err,
			"could not lookup fieldType when building converter for %s :",
			fieldType.ThriftName(),
		)
	}
	typeName := pkgName + "." + fieldType.ThriftName()
	return typeName, nil
}

func (c *TypeConverter) genConverterForStruct(
	toFieldName string,
	toFieldType *compile.StructSpec,
	fromFieldType compile.TypeSpec,
	fromIdentifier string,
	keyPrefix string,
	indent string,
) error {
	toIdentifier := indent + "out." + keyPrefix

	typeName, err := c.getIdentifierName(toFieldType)
	if err != nil {
		return err
	}
	subToFields := toFieldType.Fields

	fromFieldStruct, ok := fromFieldType.(*compile.StructSpec)
	if !ok {
		return errors.Errorf(
			"could not convert struct fields, "+
				"incompatible type for %s :",
			toFieldName,
		)
	}

	line := "if " + fromIdentifier + " != nil {"
	c.Lines = append(c.Lines, line)

	line = "	" + toIdentifier + " = &" + typeName + "{}"
	c.Lines = append(c.Lines, line)

	subFromFields := fromFieldStruct.Fields
	err = c.genStructConverter(
		keyPrefix+".",
		indent+"	",
		subFromFields,
		subToFields,
	)
	if err != nil {
		return err
	}

	line = "} else {"
	c.Lines = append(c.Lines, line)

	line = "	" + toIdentifier + " = nil"
	c.Lines = append(c.Lines, line)

	line = "}"
	c.Lines = append(c.Lines, line)

	return nil
}

func (c *TypeConverter) genConverterForPrimitive(
	toField *compile.FieldSpec,
	toIdentifier string,
	fromIdentifier string,
) error {
	var line string
	typeName, err := c.getGoTypeName(toField.Type)
	if err != nil {
		return err
	}

	if toField.Required {
		line = toIdentifier + " = " + typeName + "(" + fromIdentifier + ")"
	} else {
		line = toIdentifier + " = (*" + typeName + ")(" + fromIdentifier + ")"
	}
	c.Lines = append(c.Lines, line)
	return nil
}

func (c *TypeConverter) genConverterForList(
	toFieldType *compile.ListSpec,
	toField *compile.FieldSpec,
	fromField *compile.FieldSpec,
	toIdentifier string,
	fromIdentifier string,
	keyPrefix string,
	indent string,
) error {
	typeName, err := c.getGoTypeName(toFieldType.ValueSpec)
	if err != nil {
		return err
	}

	valueStruct, isStruct := toFieldType.ValueSpec.(*compile.StructSpec)
	if isStruct {
		line := toIdentifier + " = make([]*" +
			typeName + ", len(" + fromIdentifier + "))"
		c.Lines = append(c.Lines, line)
	} else {
		line := toIdentifier + " = make([]" +
			typeName + ", len(" + fromIdentifier + "))"
		c.Lines = append(c.Lines, line)
	}

	line := "for index, value := range " + fromIdentifier + " {"
	c.Lines = append(c.Lines, line)

	if isStruct {
		fromFieldType, ok := fromField.Type.(*compile.ListSpec)
		if !ok {
			return errors.Errorf(
				"Could not convert field (%s): type is not list",
				fromField.Name,
			)
		}

		err = c.genConverterForStruct(
			toField.Name,
			valueStruct,
			fromFieldType.ValueSpec,
			"value",
			keyPrefix+strings.Title(toField.Name)+"[index]",
			indent,
		)
		if err != nil {
			return err
		}
	} else {
		line = toIdentifier + "[index] = " + typeName + "(value)"
		c.Lines = append(c.Lines, line)
	}

	line = "}"
	c.Lines = append(c.Lines, line)
	return nil
}

func (c *TypeConverter) genStructConverter(
	keyPrefix string,
	indent string,
	fromFields []*compile.FieldSpec,
	toFields []*compile.FieldSpec,
) error {
	for i := 0; i < len(toFields); i++ {
		toField := toFields[i]

		var fromField *compile.FieldSpec
		for j := 0; j < len(fromFields); j++ {
			if fromFields[j].Name == toField.Name {
				fromField = fromFields[j]
				break
			}
		}

		if fromField == nil {
			return errors.Errorf(
				"cannot map by name for the field %s",
				toField.Name,
			)
		}

		toIdentifier := indent + "out." + keyPrefix + strings.Title(toField.Name)
		fromIdentifier := "in." + keyPrefix + strings.Title(fromField.Name)

		// Override thrift type names to avoid naming collisions between endpoint
		// and client types.
		switch toFieldType := toField.Type.(type) {
		case
			*compile.BoolSpec,
			*compile.I8Spec,
			*compile.I16Spec,
			*compile.I32Spec,
			*compile.EnumSpec,
			*compile.I64Spec,
			*compile.DoubleSpec,
			*compile.StringSpec:

			err := c.genConverterForPrimitive(
				toField, toIdentifier, fromIdentifier,
			)
			if err != nil {
				return err
			}
		case *compile.BinarySpec:
			line := toIdentifier + " = []byte(" + fromIdentifier + ")"
			c.Lines = append(c.Lines, line)
		case *compile.TypedefSpec:
			typeName, err := c.getIdentifierName(toField.Type)
			if err != nil {
				return err
			}

			var line string
			// TODO: typedef for struct is invalid here ...
			if toField.Required {
				line = toIdentifier + " = " + typeName + "(" + fromIdentifier + ")"
			} else {
				line = toIdentifier + " = (*" + typeName + ")(" + fromIdentifier + ")"
			}
			c.Lines = append(c.Lines, line)

		case *compile.StructSpec:
			err := c.genConverterForStruct(
				toField.Name,
				toFieldType,
				fromField.Type,
				fromIdentifier,
				keyPrefix+strings.Title(toField.Name),
				indent,
			)
			if err != nil {
				return err
			}
		case *compile.ListSpec:
			err := c.genConverterForList(
				toFieldType,
				toField,
				fromField,
				toIdentifier,
				fromIdentifier,
				keyPrefix,
				indent,
			)
			if err != nil {
				return err
			}
		case *compile.MapSpec:
			typeName, err := c.getGoTypeName(toFieldType.ValueSpec)
			if err != nil {
				return err
			}

			_, isStringKey := toFieldType.KeySpec.(*compile.StringSpec)
			if !isStringKey {
				return errors.Errorf(
					"could not convert key (%s), map is not string-keyed.",
					toField.Name,
				)
			}

			_, isStruct := toFieldType.ValueSpec.(*compile.StructSpec)
			if isStruct {
				line := toIdentifier + " = make(map[string]*" +
					typeName + ", len(" + fromIdentifier + "))"
				c.Lines = append(c.Lines, line)
			} else {
				line := toIdentifier + " = make(map[string]" +
					typeName + ", len(" + fromIdentifier + "))"
				c.Lines = append(c.Lines, line)
			}

			line := "for key, value := range " + fromIdentifier + " {"
			c.Lines = append(c.Lines, line)

			if isStruct {
				// TODO: need to deep copy struct here.
				line = toIdentifier + "[key] = (*" + typeName + ")(value)"
				c.Lines = append(c.Lines, line)
			} else {
				line = toIdentifier + "[key] = " + typeName + "(value)"
				c.Lines = append(c.Lines, line)
			}

			line = "}"
			c.Lines = append(c.Lines, line)

		default:
			// fmt.Printf("Unknown type %s for field %s \n",
			// 	toField.Type.TypeCode().String(), toField.Name,
			// )

			// pkgName, err := h.TypePackageName(toField.Type.ThriftFile())
			// if err != nil {
			// 	return nil, err
			// }
			// typeName := pkgName + "." + toField.Type.ThriftName()
			// line := prefix + "(*" + typeName + ")" + postfix
			// c.Lines = append(c.Lines, line)
		}
	}

	return nil
}

// GenStructConverter will add lines to the TypeConverter for mapping
// from one go struct to another based on two thriftrw.FieldGroups
func (c *TypeConverter) GenStructConverter(
	fromFields []*compile.FieldSpec,
	toFields []*compile.FieldSpec,
) error {
	err := c.genStructConverter("", "", fromFields, toFields)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package common

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	cleanhttp "github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey             string `mapstructure:"access_key"`
	CustomEndpointEc2     string `mapstructure:"custom_endpoint_ec2"`
	DecodeAuthZMessages   bool   `mapstructure:"decode_authorization_messages"`
	InsecureSkipTLSVerify bool   `mapstructure:"insecure_skip_tls_verify"`
	MFACode               string `mapstructure:"mfa_code"`
	ProfileName           string `mapstructure:"profile"`
	RawRegion             string `mapstructure:"region"`
	SecretKey             string `mapstructure:"secret_key"`
	SkipValidation        bool   `mapstructure:"skip_region_validation"`
	SkipMetadataApiCheck  bool   `mapstructure:"skip_metadata_api_check"`
	Token                 string `mapstructure:"token"`
	session               *session.Session

	getEC2Connection func() ec2iface.EC2API
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
		config.WithCredentials(staticCreds)
	}

	// default is 3, and when it was causing failures for users being throttled
	// retries are exponentially backed off.
	config = config.WithMaxRetries(8)

	if c.RawRegion != "" {
		config = config.WithRegion(c.RawRegion)
	}

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	if c.InsecureSkipTLSVerify {
		config := config.WithHTTPClient(cleanhttp.DefaultClient())
		transport := config.HTTPClient.Transport.(*http.Transport)
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}

	if c.ProfileName != "" {
		opts.Profile = c.ProfileName
	}

	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}

	sess, err := session.NewSessionWithOptions(opts)
	if err != nil {
		return nil, err
	}
	log.Printf("Found region %s", *sess.Config.Region)
	c.session = sess

	cp, err := c.session.Config.Credentials.Get()
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
			return nil, fmt.Errorf("No valid credential sources found for AWS Builder. " +
				"Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " +
				"for more information on providing credentials for the AWS Builder.")
		} else {
			return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
		}
	}
	log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)

	if c.DecodeAuthZMessages {
		DecodeAuthZMessages(c.session)
	}
	LogEnvOverrideWarnings()

	return c.session, nil
}

func (c *AccessConfig) SessionRegion() string {
	if c.session == nil {
		panic("access config session should be set.")
	}
	return aws.StringValue(c.session.Config.Region)
}

func (c *AccessConfig) IsGovCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
}

func (c *AccessConfig) IsChinaCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "cn-")
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error

	if c.SkipMetadataApiCheck {
		log.Println("(WARN) skip_metadata_api_check ignored.")
	}
	// Either both access and secret key must be set or neither of them should
	// be.
	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
		errs = append(errs,
			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))
	}

	return errs
}

func (c *AccessConfig) NewEC2Connection() (ec2iface.EC2API, error) {
	if c.getEC2Connection != nil {
		return c.getEC2Connection(), nil
	}
	sess, err := c.Session()
	if err != nil {
		return nil, err
	}
	return ec2.New(sess), nil
}
<commit_msg>remove default max retries config of 8, it will now be used from env<commit_after>package common

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	cleanhttp "github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey             string `mapstructure:"access_key"`
	CustomEndpointEc2     string `mapstructure:"custom_endpoint_ec2"`
	DecodeAuthZMessages   bool   `mapstructure:"decode_authorization_messages"`
	InsecureSkipTLSVerify bool   `mapstructure:"insecure_skip_tls_verify"`
	MFACode               string `mapstructure:"mfa_code"`
	ProfileName           string `mapstructure:"profile"`
	RawRegion             string `mapstructure:"region"`
	SecretKey             string `mapstructure:"secret_key"`
	SkipValidation        bool   `mapstructure:"skip_region_validation"`
	SkipMetadataApiCheck  bool   `mapstructure:"skip_metadata_api_check"`
	Token                 string `mapstructure:"token"`
	session               *session.Session

	getEC2Connection func() ec2iface.EC2API
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
		config.WithCredentials(staticCreds)
	}

	if c.RawRegion != "" {
		config = config.WithRegion(c.RawRegion)
	}

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	if c.InsecureSkipTLSVerify {
		config := config.WithHTTPClient(cleanhttp.DefaultClient())
		transport := config.HTTPClient.Transport.(*http.Transport)
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}

	if c.ProfileName != "" {
		opts.Profile = c.ProfileName
	}

	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}

	sess, err := session.NewSessionWithOptions(opts)
	if err != nil {
		return nil, err
	}
	log.Printf("Found region %s", *sess.Config.Region)
	c.session = sess

	cp, err := c.session.Config.Credentials.Get()
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
			return nil, fmt.Errorf("No valid credential sources found for AWS Builder. " +
				"Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " +
				"for more information on providing credentials for the AWS Builder.")
		} else {
			return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
		}
	}
	log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)

	if c.DecodeAuthZMessages {
		DecodeAuthZMessages(c.session)
	}
	LogEnvOverrideWarnings()

	return c.session, nil
}

func (c *AccessConfig) SessionRegion() string {
	if c.session == nil {
		panic("access config session should be set.")
	}
	return aws.StringValue(c.session.Config.Region)
}

func (c *AccessConfig) IsGovCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
}

func (c *AccessConfig) IsChinaCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "cn-")
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error

	if c.SkipMetadataApiCheck {
		log.Println("(WARN) skip_metadata_api_check ignored.")
	}
	// Either both access and secret key must be set or neither of them should
	// be.
	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
		errs = append(errs,
			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))
	}

	return errs
}

func (c *AccessConfig) NewEC2Connection() (ec2iface.EC2API, error) {
	if c.getEC2Connection != nil {
		return c.getEC2Connection(), nil
	}
	sess, err := c.Session()
	if err != nil {
		return nil, err
	}
	return ec2.New(sess), nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey            string `mapstructure:"access_key"`
	CustomEndpointEc2    string `mapstructure:"custom_endpoint_ec2"`
	DecodeAuthZMessages  bool   `mapstructure:"decode_authorization_messages"`
	MFACode              string `mapstructure:"mfa_code"`
	ProfileName          string `mapstructure:"profile"`
	RawRegion            string `mapstructure:"region"`
	SecretKey            string `mapstructure:"secret_key"`
	SkipValidation       bool   `mapstructure:"skip_region_validation"`
	SkipMetadataApiCheck bool   `mapstructure:"skip_metadata_api_check"`
	Token                string `mapstructure:"token"`
	session              *session.Session

	getEC2Connection func() ec2iface.EC2API
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
		config.WithCredentials(staticCreds)
	}

	// default is 3, and when it was causing failures for users being throttled
	config = config.WithMaxRetries(20)

	if c.RawRegion != "" {
		config = config.WithRegion(c.RawRegion)
	} else if region := c.metadataRegion(); region != "" {
		config = config.WithRegion(region)
	}

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}

	if c.ProfileName != "" {
		opts.Profile = c.ProfileName
	}

	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}

	if sess, err := session.NewSessionWithOptions(opts); err != nil {
		return nil, err
	} else if *sess.Config.Region == "" {
		return nil, fmt.Errorf("Could not find AWS region, make sure it's set.")
	} else {
		log.Printf("Found region %s", *sess.Config.Region)
		c.session = sess

		cp, err := c.session.Config.Credentials.Get()
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
				return nil, fmt.Errorf("No valid credential sources found for AWS Builder. " +
					"Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " +
					"for more information on providing credentials for the AWS Builder.")
			} else {
				return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
			}
		}
		log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)
	}

	if c.DecodeAuthZMessages {
		DecodeAuthZMessages(c.session)
	}

	return c.session, nil
}

func (c *AccessConfig) SessionRegion() string {
	if c.session == nil {
		panic("access config session should be set.")
	}
	return aws.StringValue(c.session.Config.Region)
}

func (c *AccessConfig) IsGovCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
}

func (c *AccessConfig) IsChinaCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "cn-")
}

// metadataRegion returns the region from the metadata service
func (c *AccessConfig) metadataRegion() string {

	client := cleanhttp.DefaultClient()

	// Keep the default timeout (100ms) low as we don't want to wait in non-EC2 environments
	client.Timeout = 100 * time.Millisecond
	ec2meta := ec2metadata.New(session.New(), &aws.Config{
		HTTPClient: client,
	})
	region, err := ec2meta.Region()
	if err != nil {
		log.Println("Error getting region from metadata service, "+
			"probably because we're not running on AWS.", err)
		return ""
	}
	return region
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error

	if c.SkipMetadataApiCheck {
		log.Println("(WARN) skip_metadata_api_check ignored.")
	}
	// Either both access and secret key must be set or neither of them should
	// be.
	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
		errs = append(errs,
			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))
	}

	if c.RawRegion != "" && !c.SkipValidation {
		err := c.ValidateRegion(c.RawRegion)
		if err != nil {
			errs = append(errs, fmt.Errorf("error validating region: %s", err.Error()))
		}
	}

	return errs
}

func (c *AccessConfig) NewEC2Connection() (ec2iface.EC2API, error) {
	if c.getEC2Connection != nil {
		return c.getEC2Connection(), nil
	}
	sess, err := c.Session()
	if err != nil {
		return nil, err
	}
	return ec2.New(sess), nil
}
<commit_msg>aws: better error handling of region guessing from metadata<commit_after>package common

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/packer/template/interpolate"
)

// AccessConfig is for common configuration related to AWS access
type AccessConfig struct {
	AccessKey            string `mapstructure:"access_key"`
	CustomEndpointEc2    string `mapstructure:"custom_endpoint_ec2"`
	DecodeAuthZMessages  bool   `mapstructure:"decode_authorization_messages"`
	MFACode              string `mapstructure:"mfa_code"`
	ProfileName          string `mapstructure:"profile"`
	RawRegion            string `mapstructure:"region"`
	SecretKey            string `mapstructure:"secret_key"`
	SkipValidation       bool   `mapstructure:"skip_region_validation"`
	SkipMetadataApiCheck bool   `mapstructure:"skip_metadata_api_check"`
	Token                string `mapstructure:"token"`
	session              *session.Session

	getEC2Connection func() ec2iface.EC2API
}

// Config returns a valid aws.Config object for access to AWS services, or
// an error if the authentication and region couldn't be resolved
func (c *AccessConfig) Session() (*session.Session, error) {
	if c.session != nil {
		return c.session, nil
	}

	config := aws.NewConfig().WithCredentialsChainVerboseErrors(true)
	staticCreds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
	if _, err := staticCreds.Get(); err != credentials.ErrStaticCredentialsEmpty {
		config.WithCredentials(staticCreds)
	}

	// default is 3, and when it was causing failures for users being throttled
	config = config.WithMaxRetries(20)

	region, err := c.region()
	if err != nil {
		return nil, fmt.Errorf("Could not get region, "+
			"probably because it's not set or we're not running on AWS. %s", err)
	}
	config = config.WithRegion(region)

	if c.CustomEndpointEc2 != "" {
		config = config.WithEndpoint(c.CustomEndpointEc2)
	}

	opts := session.Options{
		SharedConfigState: session.SharedConfigEnable,
		Config:            *config,
	}

	if c.ProfileName != "" {
		opts.Profile = c.ProfileName
	}

	if c.MFACode != "" {
		opts.AssumeRoleTokenProvider = func() (string, error) {
			return c.MFACode, nil
		}
	}

	if sess, err := session.NewSessionWithOptions(opts); err != nil {
		return nil, err
	} else if *sess.Config.Region == "" {
		return nil, fmt.Errorf("Could not find AWS region, make sure it's set.")
	} else {
		log.Printf("Found region %s", *sess.Config.Region)
		c.session = sess

		cp, err := c.session.Config.Credentials.Get()
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
				return nil, fmt.Errorf("No valid credential sources found for AWS Builder. " +
					"Please see https://www.packer.io/docs/builders/amazon.html#specifying-amazon-credentials " +
					"for more information on providing credentials for the AWS Builder.")
			} else {
				return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
			}
		}
		log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)
	}

	if c.DecodeAuthZMessages {
		DecodeAuthZMessages(c.session)
	}

	return c.session, nil
}

func (c *AccessConfig) SessionRegion() string {
	if c.session == nil {
		panic("access config session should be set.")
	}
	return aws.StringValue(c.session.Config.Region)
}

func (c *AccessConfig) IsGovCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
}

func (c *AccessConfig) IsChinaCloud() bool {
	return strings.HasPrefix(c.SessionRegion(), "cn-")
}

// metadataRegion returns the region from the metadata service
func (c *AccessConfig) metadataRegion() (string, error) {

	client := cleanhttp.DefaultClient()

	// Keep the default timeout (100ms) low as we don't want to wait in non-EC2 environments
	client.Timeout = 100 * time.Millisecond
	ec2meta := ec2metadata.New(session.New(), &aws.Config{
		HTTPClient: client,
	})
	return ec2meta.Region()
}

func (c *AccessConfig) region() (string, error) {
	if c.RawRegion != "" {
		return c.RawRegion, nil
	}
	return c.metadataRegion()
}

func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error {
	var errs []error

	if c.SkipMetadataApiCheck {
		log.Println("(WARN) skip_metadata_api_check ignored.")
	}
	// Either both access and secret key must be set or neither of them should
	// be.
	if (len(c.AccessKey) > 0) != (len(c.SecretKey) > 0) {
		errs = append(errs,
			fmt.Errorf("`access_key` and `secret_key` must both be either set or not set."))
	}

	if c.RawRegion != "" && !c.SkipValidation {
		err := c.ValidateRegion(c.RawRegion)
		if err != nil {
			errs = append(errs, fmt.Errorf("error validating region: %s", err.Error()))
		}
	}

	return errs
}

func (c *AccessConfig) NewEC2Connection() (ec2iface.EC2API, error) {
	if c.getEC2Connection != nil {
		return c.getEC2Connection(), nil
	}
	sess, err := c.Session()
	if err != nil {
		return nil, err
	}
	return ec2.New(sess), nil
}
<|endoftext|>


<commit_before>package extensions

import (
	"encoding/json"
	"fmt"
	"github.com/pawelszydlo/papa-bot"
	"github.com/pawelszydlo/papa-bot/events"
	"github.com/pawelszydlo/papa-bot/utils"
	"golang.org/x/text/language"
	text_message "golang.org/x/text/message"
	"net/url"
	"regexp"
	"strconv"
	"text/template"
	"time"
)

// ExtensionYoutube - extension for getting basic video information.
type ExtensionYoutube struct {
	youTubeRe *regexp.Regexp
	bot       *papaBot.Bot
	Texts     *ExtensionYoutubeTexts
}

type ExtensionYoutubeTexts struct {
	TempNotice *template.Template
}

// Init inits the extension.
func (ext *ExtensionYoutube) Init(bot *papaBot.Bot) error {
	ext.youTubeRe = regexp.MustCompile(`(?i)youtu(?:be\.com/watch\?v=|\.be/)([\w\-_]*)(&(amp;)?[\w?=]*)?`)
	ext.bot = bot
	// Load texts.
	texts := new(ExtensionYoutubeTexts)
	if err := bot.LoadTexts("youtube", texts); err != nil {
		return err
	}
	ext.Texts = texts
	bot.EventDispatcher.RegisterListener(events.EventURLFound, ext.UrlListener)
	return nil
}

// UrlListener will try to get more info on GitHub links.
func (ext *ExtensionYoutube) UrlListener(message events.EventMessage) {
	match := ext.youTubeRe.FindStringSubmatch(message.Message)
	if len(match) < 2 {
		return
	}
	go func() {
		video_no := match[1]
		// Get response
		err, _, body := ext.bot.GetPageBody(fmt.Sprintf("https://youtube.com/get_video_info?video_id=%s", video_no), nil)
		if err != nil {
			ext.bot.Log.Warningf("Error getting response from YouTube: %s", err)
			return
		}
		// Extract data from www-from-urlencoded.
		params, err := url.ParseQuery(string(body))
		if err != nil {
			ext.bot.Log.Error(err)
			return
		}
		// Intersting stuff is only in the "player_response" -> "videoDetails".
		response, ok := params["player_response"]
		if !ok {
			ext.bot.Log.Error("Player response not found.")
		}
		// Convert from JSON
		var raw_data interface{}
		if err := json.Unmarshal([]byte(response[0]), &raw_data); err != nil {
			ext.bot.Log.Warningf("Error parsing JSON from YouTube get info: %s", err)
			return
		}
		data := raw_data.(map[string]interface{})["videoDetails"].(map[string]interface{})

		// Map that the user will be able to use for formatting.
		duration, err := time.ParseDuration(fmt.Sprintf("%ss", data["lengthSeconds"]))

		p := text_message.NewPrinter(language.English)
		views, _ := strconv.Atoi(data["viewCount"].(string))

		values := map[string]string{
			"title":       fmt.Sprintf("%s", data["title"]),
			"length":      ext.bot.Humanizer.SecondsToTimeString(int64(duration.Seconds())),
			"description": fmt.Sprintf("%s", data["shortDescription"]),
			"rating":      fmt.Sprintf("%.2f", data["averageRating"]),
			"views":       p.Sprintf("%d", views),
			"author":      fmt.Sprintf("%s", data["author"]),
		}

		// Add "more".
		ext.bot.AddMoreInfo(message.TransportName, message.Channel, values["description"])

		// Send the notice.
		ext.bot.SendNotice(&message, utils.Format(ext.Texts.TempNotice, values))
	}()
}
<commit_msg>Use humanizer for youtube views.<commit_after>package extensions

import (
	"encoding/json"
	"fmt"
	"github.com/pawelszydlo/papa-bot"
	"github.com/pawelszydlo/papa-bot/events"
	"github.com/pawelszydlo/papa-bot/utils"
	"net/url"
	"regexp"
	"strconv"
	"text/template"
	"time"
)

// ExtensionYoutube - extension for getting basic video information.
type ExtensionYoutube struct {
	youTubeRe *regexp.Regexp
	bot       *papaBot.Bot
	Texts     *ExtensionYoutubeTexts
}

type ExtensionYoutubeTexts struct {
	TempNotice *template.Template
}

// Init inits the extension.
func (ext *ExtensionYoutube) Init(bot *papaBot.Bot) error {
	ext.youTubeRe = regexp.MustCompile(`(?i)youtu(?:be\.com/watch\?v=|\.be/)([\w\-_]*)(&(amp;)?[\w?=]*)?`)
	ext.bot = bot
	// Load texts.
	texts := new(ExtensionYoutubeTexts)
	if err := bot.LoadTexts("youtube", texts); err != nil {
		return err
	}
	ext.Texts = texts
	bot.EventDispatcher.RegisterListener(events.EventURLFound, ext.UrlListener)
	return nil
}

// UrlListener will try to get more info on GitHub links.
func (ext *ExtensionYoutube) UrlListener(message events.EventMessage) {
	match := ext.youTubeRe.FindStringSubmatch(message.Message)
	if len(match) < 2 {
		return
	}
	go func() {
		video_no := match[1]
		// Get response
		err, _, body := ext.bot.GetPageBody(fmt.Sprintf("https://youtube.com/get_video_info?video_id=%s", video_no), nil)
		if err != nil {
			ext.bot.Log.Warningf("Error getting response from YouTube: %s", err)
			return
		}
		// Extract data from www-from-urlencoded.
		params, err := url.ParseQuery(string(body))
		if err != nil {
			ext.bot.Log.Error(err)
			return
		}
		// Intersting stuff is only in the "player_response" -> "videoDetails".
		response, ok := params["player_response"]
		if !ok {
			ext.bot.Log.Error("Player response not found.")
		}
		// Convert from JSON
		var raw_data interface{}
		if err := json.Unmarshal([]byte(response[0]), &raw_data); err != nil {
			ext.bot.Log.Warningf("Error parsing JSON from YouTube get info: %s", err)
			return
		}
		data := raw_data.(map[string]interface{})["videoDetails"].(map[string]interface{})

		// Map that the user will be able to use for formatting.
		duration, err := time.ParseDuration(fmt.Sprintf("%ss", data["lengthSeconds"]))
		views, _ := strconv.Atoi(data["viewCount"].(string))

		values := map[string]string{
			"title":       fmt.Sprintf("%s", data["title"]),
			"length":      ext.bot.Humanizer.SecondsToTimeString(int64(duration.Seconds())),
			"description": fmt.Sprintf("%s", data["shortDescription"]),
			"rating":      fmt.Sprintf("%.2f", data["averageRating"]),
			"views":       ext.bot.Humanizer.HumanizeNumber(float64(views), 0),
			"author":      fmt.Sprintf("%s", data["author"]),
		}

		// Add "more".
		ext.bot.AddMoreInfo(message.TransportName, message.Channel, values["description"])

		// Send the notice.
		ext.bot.SendNotice(&message, utils.Format(ext.Texts.TempNotice, values))
	}()
}
<|endoftext|>


<commit_before>package parser

//go:generate goyacc -o parser.go parser.y

import (
	"bytes"
	"fmt"
	"unicode/utf8"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/token"
	"github.com/elpinal/coco3/extra/types"
)

const eof = 0

type exprLexer struct {
	src []byte // source
	r   rune   // current character

	off    uint // starts from 0
	line   uint // starts from 1
	column uint // starts from 1

	// information for the start position of current token
	tokLine   uint
	tokColumn uint

	// result
	expr *ast.Command

	// channel for error
	errCh chan *ParseError
}

func newLexer(src []byte) *exprLexer {
	l := &exprLexer{
		src:   src,
		line:  1,
		errCh: make(chan *ParseError),
	}
	l.next()
	return l
}

func isAlphabet(c rune) bool {
	return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
}

func isIdent(c rune) bool {
	return isAlphabet(c) || isNumber(c) || c == '-'
}

func isNumber(c rune) bool {
	return '0' <= c && c <= '9'
}

func isQuote(c rune) bool {
	return c == '\''
}

func (l *exprLexer) emitError(format string, args ...interface{}) {
	select {
	case l.errCh <- l.errorAtHere(format, args...):
	default:
		// If errCh is blocked (i.e. another error had occurred), this
		// error message is ignored.
	}
}

func (l *exprLexer) errorAtHere(format string, args ...interface{}) *ParseError {
	return &ParseError{
		Line:   l.line,
		Column: l.column,
		Msg:    fmt.Sprintf(format, args...),
	}
}

func (l *exprLexer) Lex(yylval *yySymType) int {
	for {
		l.tokLine = l.line
		l.tokColumn = l.column
		c := l.r
		// TODO: set yylval.token on all cases.
		switch c {
		case eof:
			return eof
		case ' ', '\n':
			l.next()
		case '\'':
			l.next()
			return l.str(yylval)
		case '[':
			l.next()
			return LBRACK
		case ']':
			l.next()
			return RBRACK
		case ':':
			l.next()
			return COLON
		case ',':
			l.next()
			return COMMA
		case '!', '(', ')':
			l.next()
			yylval.token = token.Token{
				Kind:   types.Ident,
				Lit:    string(c),
				Line:   l.tokLine,
				Column: l.tokColumn,
			}
			return int(c)
		default:
			if isAlphabet(c) {
				return l.ident(yylval)
			}
			if isNumber(c) {
				return l.num(yylval)
			}
			l.emitError("invalid character: %[1]U %[1]q", c)
			return ILLEGAL
		}
	}
}

func (l *exprLexer) ident(yylval *yySymType) int {
	l.takeWhile(types.Ident, isIdent, yylval)
	return IDENT
}

func (l *exprLexer) str(yylval *yySymType) int {
	add := func(b *bytes.Buffer, c rune) {
		if _, err := b.WriteRune(c); err != nil {
			l.emitError("WriteRune: %s", err)
		}
	}
	var b bytes.Buffer
	for !isQuote(l.r) {
		if l.r == eof {
			l.errCh <- &ParseError{
				Line:   l.tokLine,
				Column: l.tokColumn,
				Msg:    "string literal not terminated: unexpected EOF",
			}
			return STRING
		}
		if l.r == '\\' {
			line := l.line
			column := l.column
			l.next()
			switch l.r {
			case '\'', '\\':
				add(&b, l.r)
				l.next()
			case 'n':
				add(&b, '\n')
				l.next()
			case eof:
				l.errCh <- &ParseError{
					Line:   line,
					Column: column,
					Msg:    "string literal not terminated: unexpected EOF",
				}
				return STRING
			default:
				l.errCh <- &ParseError{
					Line:   line,
					Column: column,
					Msg:    fmt.Sprintf("unknown escape sequence: \\%c", l.r),
				}
				l.next()
				return STRING
			}
			continue
		}
		add(&b, l.r)
		l.next()
	}
	yylval.token = token.Token{
		Kind:   types.String,
		Lit:    b.String(),
		Line:   l.tokLine,
		Column: l.tokColumn,
	}
	l.next()
	return STRING
}

func (l *exprLexer) num(yylval *yySymType) int {
	l.takeWhile(types.Int, isNumber, yylval)
	return NUM
}

func (l *exprLexer) takeWhile(kind types.Type, f func(rune) bool, yylval *yySymType) {
	add := func(b *bytes.Buffer, c rune) {
		if _, err := b.WriteRune(c); err != nil {
			l.emitError("WriteRune: %s", err)
		}
	}
	var b bytes.Buffer
	for f(l.r) && l.r != eof {
		add(&b, l.r)
		l.next()
	}
	yylval.token = token.Token{
		Kind:   kind,
		Lit:    b.String(),
		Line:   l.tokLine,
		Column: l.tokColumn,
	}
}

func (l *exprLexer) next() {
	if len(l.src) == 0 {
		l.r = eof
		return
	}
	c, size := utf8.DecodeRune(l.src)
	l.src = l.src[size:]
	l.off++
	if c == '\n' {
		l.line++
		l.column = 0
	} else {
		l.column++
	}
	if c == utf8.RuneError && size == 1 {
		l.emitError("next: invalid utf8")
		l.next()
		return
	}
	l.r = c
}

func (l *exprLexer) Error(s string) {
	l.errCh <- &ParseError{
		Line:   l.tokLine,
		Column: l.tokColumn,
		Msg:    s,
	}
}

func (l *exprLexer) run() <-chan struct{} {
	done := make(chan struct{})
	go func() {
		yyParse(l)
		done <- struct{}{}
	}()
	return done
}

func init() {
	yyErrorVerbose = true
}

func Parse(src []byte) (*ast.Command, error) {
	l := newLexer(src)
	done := l.run()
	select {
	case err := <-l.errCh:
		err.Src = string(src)
		return nil, err
	case <-done:
	}
	return l.expr, nil
}
<commit_msg>Update lexer<commit_after>package parser

//go:generate goyacc -o parser.go parser.y

import (
	"bytes"
	"fmt"
	"unicode/utf8"

	"github.com/elpinal/coco3/extra/ast"
	"github.com/elpinal/coco3/extra/token"
	"github.com/elpinal/coco3/extra/types"
)

const eof = 0

type exprLexer struct {
	src []byte // source
	r   rune   // current character

	off    uint // starts from 0
	line   uint // starts from 1
	column uint // starts from 1

	// information for the start position of current token
	tokLine   uint
	tokColumn uint

	// result
	expr *ast.Command

	// channel for error
	errCh chan *ParseError
}

func newLexer(src []byte) *exprLexer {
	l := &exprLexer{
		src:   src,
		line:  1,
		errCh: make(chan *ParseError),
	}
	l.next()
	return l
}

func isAlphabet(c rune) bool {
	return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
}

func isIdent(c rune) bool {
	return isAlphabet(c) || isNumber(c) || c == '-'
}

func isNumber(c rune) bool {
	return '0' <= c && c <= '9'
}

func isQuote(c rune) bool {
	return c == '\''
}

func (l *exprLexer) emitError(format string, args ...interface{}) {
	select {
	case l.errCh <- l.errorAtHere(format, args...):
	default:
		// If errCh is blocked (i.e. another error had occurred), this
		// error message is ignored.
	}
}

func (l *exprLexer) errorAtHere(format string, args ...interface{}) *ParseError {
	return &ParseError{
		Line:   l.line,
		Column: l.column,
		Msg:    fmt.Sprintf(format, args...),
	}
}

func (l *exprLexer) Lex(yylval *yySymType) int {
	for {
		l.tokLine = l.line
		l.tokColumn = l.column
		c := l.r
		// TODO: set yylval.token on all cases.
		switch c {
		case eof:
			return eof
		case ' ', '\n':
			l.next()
		case '\'':
			l.next()
			return l.str(yylval)
		case '[':
			l.next()
			return LBRACK
		case ']':
			l.next()
			return RBRACK
		case ':':
			l.next()
			return COLON
		case ',':
			l.next()
			return COMMA
		case '!', '(', ')':
			l.next()
			yylval.token = token.Token{
				Kind:   token.KindOf(types.Ident),
				Lit:    string(c),
				Line:   l.tokLine,
				Column: l.tokColumn,
			}
			return int(c)
		default:
			if isAlphabet(c) {
				return l.ident(yylval)
			}
			if isNumber(c) {
				return l.num(yylval)
			}
			l.emitError("invalid character: %[1]U %[1]q", c)
			return ILLEGAL
		}
	}
}

func (l *exprLexer) ident(yylval *yySymType) int {
	l.takeWhile(types.Ident, isIdent, yylval)
	return IDENT
}

func (l *exprLexer) str(yylval *yySymType) int {
	add := func(b *bytes.Buffer, c rune) {
		if _, err := b.WriteRune(c); err != nil {
			l.emitError("WriteRune: %s", err)
		}
	}
	var b bytes.Buffer
	for !isQuote(l.r) {
		if l.r == eof {
			l.errCh <- &ParseError{
				Line:   l.tokLine,
				Column: l.tokColumn,
				Msg:    "string literal not terminated: unexpected EOF",
			}
			return STRING
		}
		if l.r == '\\' {
			line := l.line
			column := l.column
			l.next()
			switch l.r {
			case '\'', '\\':
				add(&b, l.r)
				l.next()
			case 'n':
				add(&b, '\n')
				l.next()
			case eof:
				l.errCh <- &ParseError{
					Line:   line,
					Column: column,
					Msg:    "string literal not terminated: unexpected EOF",
				}
				return STRING
			default:
				l.errCh <- &ParseError{
					Line:   line,
					Column: column,
					Msg:    fmt.Sprintf("unknown escape sequence: \\%c", l.r),
				}
				l.next()
				return STRING
			}
			continue
		}
		add(&b, l.r)
		l.next()
	}
	yylval.token = token.Token{
		Kind:   token.KindOf(types.String),
		Lit:    b.String(),
		Line:   l.tokLine,
		Column: l.tokColumn,
	}
	l.next()
	return STRING
}

func (l *exprLexer) num(yylval *yySymType) int {
	l.takeWhile(types.Int, isNumber, yylval)
	return NUM
}

func (l *exprLexer) takeWhile(kind types.Type, f func(rune) bool, yylval *yySymType) {
	add := func(b *bytes.Buffer, c rune) {
		if _, err := b.WriteRune(c); err != nil {
			l.emitError("WriteRune: %s", err)
		}
	}
	var b bytes.Buffer
	for f(l.r) && l.r != eof {
		add(&b, l.r)
		l.next()
	}
	yylval.token = token.Token{
		Kind:   token.KindOf(kind),
		Lit:    b.String(),
		Line:   l.tokLine,
		Column: l.tokColumn,
	}
}

func (l *exprLexer) next() {
	if len(l.src) == 0 {
		l.r = eof
		return
	}
	c, size := utf8.DecodeRune(l.src)
	l.src = l.src[size:]
	l.off++
	if c == '\n' {
		l.line++
		l.column = 0
	} else {
		l.column++
	}
	if c == utf8.RuneError && size == 1 {
		l.emitError("next: invalid utf8")
		l.next()
		return
	}
	l.r = c
}

func (l *exprLexer) Error(s string) {
	l.errCh <- &ParseError{
		Line:   l.tokLine,
		Column: l.tokColumn,
		Msg:    s,
	}
}

func (l *exprLexer) run() <-chan struct{} {
	done := make(chan struct{})
	go func() {
		yyParse(l)
		done <- struct{}{}
	}()
	return done
}

func init() {
	yyErrorVerbose = true
}

func Parse(src []byte) (*ast.Command, error) {
	l := newLexer(src)
	done := l.run()
	select {
	case err := <-l.errCh:
		err.Src = string(src)
		return nil, err
	case <-done:
	}
	return l.expr, nil
}
<|endoftext|>