<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fnproject/fn/fnlb/lb"
)

const VERSION = "0.0.3"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.Parse()

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	var wg sync.WaitGroup
	wg.Add(1)
	defer wg.Wait()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		defer wg.Done()
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.4 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fnproject/fn/fnlb/lb"
)

const VERSION = "0.0.4"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.Parse()

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	var wg sync.WaitGroup
	wg.Add(1)
	defer wg.Wait()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		defer wg.Done()
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.150"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.117", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.151 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.151"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.118", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/hatofmonkeys/cloudfocker/focker"
	"log"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "Focker"
	app.Usage = "Fock the Cloud, run apps locally!"
	app.Action = func(c *cli.Context) {

	}

	app.Commands = []cli.Command{
		{
			Name:  "docker",
			Usage: "print the local Docker info",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.DockerVersion(os.Stdout)
			},
		},
		{
			Name:  "this",
			Usage: "Download the Cloud Foundry base image",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.ImportRootfsImage(os.Stdout)
			},
		},
		{
			Name:  "dockerfile",
			Usage: "Create a dockerfile for the application",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.WriteDockerfile(os.Stdout)
			},
		},
		{
			Name:  "build",
			Usage: "Create an image for the application",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.BuildImage(os.Stdout)
			},
		},
		{
			Name:  "up",
			Usage: "Start the container for the application",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				pwd, err := os.Getwd()
				if err != nil {
					log.Fatalf(" %s", err)
				} else {
					focker.RunStager(os.Stdout, pwd)
				}
				focker.RunRuntime(os.Stdout)
			},
		},
		{
			Name:  "off",
			Usage: "Stop the container and remove it",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.StopRuntime()
			},
		},
		{
			Name:  "add-buildpack",
			Usage: "add-buildpack [URL] - add a buildpack from a GitHub URL",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				if url := c.Args().First(); url != "" {
					focker.AddBuildpack(os.Stdout, url)
				} else {
					fmt.Println("Please supply a GitHub URL to download")
				}
			},
		},
		{
			Name:  "stage",
			Usage: "stage an application",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				pwd, err := os.Getwd()
				if err != nil {
					log.Fatalf(" %s", err)
				} else {
					focker.RunStager(os.Stdout, pwd)
				}
			},
		},
		{
			Name:  "run",
			Usage: "Run the staged container",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.RunRuntime(os.Stdout)
			},
		},
		{
			Name:  "stage-internal",
			Usage: "used to stage the app inside a container",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.StageApp(os.Stdout)
			},
			HideHelp: true,
		},
	}

	app.Run(os.Args)
}
<commit_msg>Tidied up CLI<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/hatofmonkeys/cloudfocker/focker"
	"log"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "Focker"
	app.Usage = "Fock the Cloud, run apps locally!"
	app.Action = func(c *cli.Context) {

	}

	app.Commands = []cli.Command{
		{
			Name:  "docker",
			Usage: "print the local Docker info",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.DockerVersion(os.Stdout)
			},
		},
		{
			Name:  "this",
			Usage: "Download the Cloud Foundry base image",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.ImportRootfsImage(os.Stdout)
			},
		},
		{
			Name:  "up",
			Usage: "Start the container for the application",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				pwd, err := os.Getwd()
				if err != nil {
					log.Fatalf(" %s", err)
				} else {
					focker.RunStager(os.Stdout, pwd)
				}
				focker.RunRuntime(os.Stdout)
			},
		},
		{
			Name:  "off",
			Usage: "Stop the container and remove it",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.StopRuntime()
			},
		},
		{
			Name:  "add-buildpack",
			Usage: "add-buildpack [URL] - add a buildpack from a GitHub URL",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				if url := c.Args().First(); url != "" {
					focker.AddBuildpack(os.Stdout, url)
				} else {
					fmt.Println("Please supply a GitHub URL to download")
				}
			},
		},
		{
			Name:  "stage",
			Usage: "stage an application",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				pwd, err := os.Getwd()
				if err != nil {
					log.Fatalf(" %s", err)
				} else {
					focker.RunStager(os.Stdout, pwd)
				}
			},
		},
		{
			Name:  "run",
			Usage: "run the staged container",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.RunRuntime(os.Stdout)
			},
		},
		{
			Name:  "stage-internal",
			Action: func(c *cli.Context) {
				focker := focker.NewFocker()
				focker.StageApp(os.Stdout)
			},
			HideHelp: true,
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// parsing mathematical expressions
//
//go:generate nex lexer.nex
//go:generate go tool yacc -o parser.y.go parser.y
package parser

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"text/template"
)

func Formula(n int) string {
	if true {
		var terms []string
		for i := 0; i < n; i++ {
			terms = append(terms, fmt.Sprintf("x%d", i))
		}
		return "f:=" + strings.Join(terms, "*")
	}
	var terms []string
	for i := 0; i < n; i++ {
		term := fmt.Sprintf("%f * x%d", rand.Float64(), i)
		terms = append(terms, term)
	}
	return fmt.Sprintf("f := sqrt(a*x) + pow(a,b) + a-b+c/d + log(1 + exp(-y * (%s)))", strings.Join(terms, "+"))
}

func computeDerivatives(w io.Writer, steps []Step, private string) {
	n := len(steps)
	for i := 0; i < n; i++ {
		j := n - i - 1
		step := steps[j]
		if i == 0 {
			fmt.Fprintf(w, "const b_%s = 1.0\n", step.lhs)
		} else {
			fmt.Fprintf(w, "b_%s := 0.0\n", step.lhs)
		}
		for k := j + 1; k < n; k++ {
			s3 := steps[k]
			if d := derivative(s3, step, private); len(d) > 0 {
				fmt.Fprintf(w, "b_%s += b_%s * (%s)\n", step.lhs, s3.lhs, d)
			}
		}
		if step.decl {
			fmt.Fprintf(w, "grad_%s[\"%s\"] = b_%s\n", private, step.rhs, step.lhs)
		}
	}
}

type Step struct {
	decl bool
	lhs  string
	rhs  string
	f    string
	args []string
}

func derivative(num, denom Step, private string) string {
	var args []int
	for i, a := range num.args {
		if a == denom.lhs {
			args = append(args, i)
		}
	}
	if len(args) == 0 {
		return ""
	}
	var list []string
	for _, a := range args {
		list = append(list, fmt.Sprintf("d_%s_%s(%d,%s)", num.f, private, a, strings.Join(num.args, ",")))
	}
	return strings.Join(list, "+")
}

type NodeType string

const (
	numberNT     NodeType = "NUM"
	identifierNT NodeType = "IDENT"
	functionNT   NodeType = "FUNC"
)

type Node struct {
	Type     NodeType `json:"T,omitempty"`
	S        string   `json:",omitempty"`
	F        float64  `json:",omitempty"`
	Children []*Node  `json:"C,omitempty"`
	Name     string   `json:"N,omitempty"`
}

func (n Node) String() string {
	buf, _ := json.Marshal(n)
	return string(buf)
}

func Run(args []string) {
	var private, templates, formula, output string
	var dx float64
	flags := flag.NewFlagSet("parse", flag.ExitOnError)
	flags.StringVar(&formula, "formula", Formula(10), "the formula to parse")
	flags.StringVar(&private, "private", defaultPrivateString, "the private variable string")
	flags.StringVar(&templates, "templates", defaultTemplates, "directory of go template functions")
	flags.StringVar(&output, "output", "compute.go", "name of go program to output")
	flags.Float64Var(&dx, "dx", defaultDx, "infinitesimal for numerical differentiation")
	flags.Parse(args)

	code, err := Parse(private, templates, formula, 0.00001)
	if err != nil {
		log.Fatal(err)
	}
	f, err := os.Create(output)
	if err != nil {
		log.Fatal(err)
	}
	f.Write(code)
	f.Close()
}

const (
	defaultPrivateString = "pvt"
	defaultDx            = 0.00001
	defaultTemplates     = "src/xoba/ad/parser/templates"
)

func Parse(private, templates, formula string, dx float64) ([]byte, error) {
	if len(private) == 0 {
		private = defaultPrivateString
	}
	if dx == 0 {
		dx = defaultDx
	}
	if len(templates) == 0 {
		templates = defaultTemplates
	}
	lex := NewContext(NewLexer(strings.NewReader(formula)))
	yyParse(lex)
	if len(lex.errors) > 0 {
		return nil, lex.errors[0]
	}
	if lex.lhs == nil || lex.rhs == nil {
		return nil, fmt.Errorf("parse error: lhs or rhs is nil")
	}
	vars := make(map[string]string)
	vp := &VarParser{vars: vars}
	sp := &StepParser{vars: vars}
	var steps []Step
	steps = append(steps, vp.getVars(lex.rhs, private)...)
	steps = append(steps, sp.program(lex.rhs, private)...)
	y := steps[len(steps)-1].lhs

	checker := new(bytes.Buffer)
	{
		fmt.Fprintf(checker, "const delta_%s = %f\n", private, dx)
		fmt.Fprintf(checker, `calc_%s := func() float64 {
%s
return %s
}
`, private, formula, lex.lhs.S)
		fmt.Fprintf(checker, "tmp1_%s := calc_%s()\n", private, private)
		for k := range vars {
			fmt.Fprintln(checker, "{")
			fmt.Fprintf(checker, "%s += delta_%s\n", k, private)
			fmt.Fprintf(checker, "tmp2_%s := calc_%s()\n", private, private)
			fmt.Fprintf(checker, "%s -= delta_%s\n", k, private)
			fmt.Fprintf(checker, "grad_%s[%q] = (tmp2_%s - tmp1_%s)/delta_%s\n", private, k, private, private, private)
			fmt.Fprintln(checker, "}")
		}
	}

	pgm := new(bytes.Buffer)
	for _, s := range steps {
		fmt.Fprintln(pgm, s)
	}
	computeDerivatives(pgm, steps, private)
	var list []string
	for v := range vars {
		list = append(list, v)
	}
	sort.Strings(list)

	f := new(bytes.Buffer)

	decls := new(bytes.Buffer)
	for _, v := range list {
		fmt.Fprintf(decls, "%s := rand.Float64();\n", v)
		fmt.Fprintf(decls, "fmt.Printf(\"setting %s = %%.20f\\n\",%s)\n", v, v)
	}
	fmt.Fprintln(decls, `fmt.Println();`)

	var imports Imports
	imports.Add("fmt")
	imports.Add("math")
	imports.Add("time")
	imports.Add("math/rand")

	t := template.Must(template.New("output.go").Parse(`package main
{{.imports}}

// automatically compute the value and gradient of {{.qformula}}
func ComputeAD({{.vars}} float64) (float64,map[string]float64) {
grad_{{.private}} := make(map[string]float64)
{{.program}} return {{.y}},grad_{{.private}};
}

func main() {
fmt.Println("running autodiff code on {{.formula}}\n");
rand.Seed(time.Now().UTC().UnixNano())
{{.decls}}

	c1, grad1 := ComputeAD({{.vars}})
	fmt.Printf("autodiff value   : %f\n", c1)
	fmt.Printf("autodiff gradient: %v\n\n", grad1)

	c2, grad2 := ComputeNumerical({{.vars}})
	fmt.Printf("numeric value   : %f\n", c2)
	fmt.Printf("numeric gradient: %v\n\n", grad2)

	var total float64
	add := func(n string, x float64) {
		fmt.Printf("%s difference: %f\n", n, x)
		total += math.Abs(x)
	}
	add("value", c1-c2)
	for k, v := range grad2 {
		add(fmt.Sprintf("grad[%3s]", k), grad1[k]-v)
	}
fmt.Printf("\nsum of absolute differences: %f\n",total);
}

// numerically compute the value and gradient of {{.qformula}}
func ComputeNumerical({{.vars}} float64) (float64,map[string]float64) {
grad_{{.private}} := make(map[string]float64)
{{.checker}} return tmp1_{{.private}},grad_{{.private}};
}

{{.funcs}}

`))

	templateImports, code, err := GenTemplates(templates, private)
	if err != nil {
		return nil, err
	}
	imports.AddAll(templateImports)

	t.Execute(f, map[string]interface{}{
		"decls":    decls.String(),
		"formula":  formula,
		"qformula": fmt.Sprintf("%q", formula),
		"lhs":      lex.lhs.S,
		"vars":     strings.Join(list, ", "),
		"program":  pgm.String(),
		"checker":  checker.String(),
		"y":        y,
		"funcs":    code,
		"private":  private,
		"imports":  imports.String(),
	})

	return GofmtBuffer(f.Bytes())
}

type Imports struct {
	list []string
}

func (i Imports) String() string {
	m := make(map[string]bool)
	for _, x := range i.list {
		x = strings.Replace(x, `"`, ``, -1)
		m[x] = true
	}
	out := new(bytes.Buffer)
	fmt.Fprintf(out, "import (\n")
	for k := range m {
		fmt.Fprintf(out, "%q\n", k)
	}
	fmt.Fprintf(out, ")\n")
	return out.String()
}

func (i *Imports) Add(x string) {
	i.list = append(i.list, x)
}

func (i *Imports) AddAll(list []string) {
	for _, x := range list {
		i.Add(x)
	}
}

func formatImports(list []string) string {
	var out []string
	m := make(map[string]bool)
	for _, x := range list {
		m[x] = true
	}
	for k := range m {
		out = append(out, k)
	}
	return strings.Join(out, "\n")
}

func Gofmt(p string) error {
	cmd := exec.Command("gofmt", "-w", p)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func GofmtBuffer(code []byte) ([]byte, error) {
	out := new(bytes.Buffer)
	cmd := exec.Command("gofmt")
	cmd.Stdin = bytes.NewReader(code)
	cmd.Stdout = out
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	return out.Bytes(), nil
}

type VarParser struct {
	i    int
	vars map[string]string
}

// gets all the vars and name them in-place
func (v *VarParser) getVars(rhs *Node, private string) (out []Step) {
	switch rhs.Type {
	case identifierNT:
		if _, ok := v.vars[rhs.S]; !ok {
			rhs.Name = fmt.Sprintf("v_%d_%s", v.i, private)
			v.vars[rhs.S] = rhs.Name
			v.i++
			step := Step{
				decl: true,
				lhs:  rhs.Name,
				rhs:  rhs.S,
			}
			out = append(out, step)
		} else {
			rhs.Name = v.vars[rhs.S]
		}
	case functionNT:
		for _, n := range rhs.Children {
			out = append(out, v.getVars(n, private)...)
		}
	}
	return
}

type StepParser struct {
	i    int
	vars map[string]string
}

func (s Step) String() string {
	return fmt.Sprintf("%s := %s;", s.lhs, s.rhs)
}

func (s *StepParser) program(rhs *Node, private string) (out []Step) {
	switch rhs.Type {
	case numberNT:
		rhs.Name = fmt.Sprintf("%f", rhs.F)
	case functionNT:
		var args []string
		for _, n := range rhs.Children {
			steps := s.program(n, private)
			out = append(out, steps...)
			args = append(args, n.Name)
		}
		step := Step{
			lhs:  fmt.Sprintf("s_%d_%s", s.i, private),
			rhs:  fmt.Sprintf("%s_%s(%s)", rhs.S, private, strings.Join(args, ",")),
			f:    rhs.S,
			args: args,
		}
		out = append(out, step)
		rhs.Name = step.lhs
		s.i++
	}
	return out
}

func Function(ident string, args ...*Node) *Node {
	return &Node{
		Type:     functionNT,
		S:        ident,
		Children: args,
	}
}

func Number(n float64) *Node {
	return &Node{
		Type: numberNT,
		F:    n,
	}
}

func LexIdentifier(s string) *Node {
	return &Node{
		Type: identifierNT,
		S:    s,
	}
}

func LexNumber(s string) *Node {
	n, _ := strconv.ParseFloat(s, 64)
	return Number(n)
}

func Negate(a *Node) *Node {
	return Function("multiply", Number(-1), a)
}

type context struct {
	lhs *Node
	rhs *Node
	yyLexer
	errors []error
}

func NewContext(y yyLexer) *context {
	return &context{yyLexer: y}
}

func (c *context) Error(e string) {
	c.Error2(fmt.Errorf("%s", e))
}

func (c *context) Error2(e error) {
	log.Printf("oops: %v\n", e)
	c.errors = append(c.errors, e)
}
<commit_msg>comments and more description<commit_after>// parsing mathematical expressions
//
//go:generate nex lexer.nex
//go:generate go tool yacc -o parser.y.go parser.y
package parser

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"text/template"
	"time"
)

func Formula(n int) string {
	if true {
		var terms []string
		for i := 0; i < n; i++ {
			terms = append(terms, fmt.Sprintf("x%d", i))
		}
		return "f:=" + strings.Join(terms, "*")
	}
	var terms []string
	for i := 0; i < n; i++ {
		term := fmt.Sprintf("%f * x%d", rand.Float64(), i)
		terms = append(terms, term)
	}
	return fmt.Sprintf("f := sqrt(a*x) + pow(a,b) + a-b+c/d + log(1 + exp(-y * (%s)))", strings.Join(terms, "+"))
}

func computeDerivatives(w io.Writer, steps []Step, private string) {
	n := len(steps)
	for i := 0; i < n; i++ {
		j := n - i - 1
		step := steps[j]
		if i == 0 {
			fmt.Fprintf(w, "const b_%s = 1.0\n", step.lhs)
		} else {
			fmt.Fprintf(w, "b_%s := 0.0\n", step.lhs)
		}
		for k := j + 1; k < n; k++ {
			s3 := steps[k]
			if d := derivative(s3, step, private); len(d) > 0 {
				fmt.Fprintf(w, "b_%s += b_%s * (%s)\n", step.lhs, s3.lhs, d)
			}
		}
		if step.decl {
			fmt.Fprintf(w, "grad_%s[\"%s\"] = b_%s\n", private, step.rhs, step.lhs)
		}
	}
}

type Step struct {
	decl bool
	lhs  string
	rhs  string
	f    string
	args []string
}

func derivative(num, denom Step, private string) string {
	var args []int
	for i, a := range num.args {
		if a == denom.lhs {
			args = append(args, i)
		}
	}
	if len(args) == 0 {
		return ""
	}
	var list []string
	for _, a := range args {
		list = append(list, fmt.Sprintf("d_%s_%s(%d,%s)", num.f, private, a, strings.Join(num.args, ",")))
	}
	return strings.Join(list, "+")
}

type NodeType string

const (
	numberNT     NodeType = "NUM"
	identifierNT NodeType = "IDENT"
	functionNT   NodeType = "FUNC"
)

type Node struct {
	Type     NodeType `json:"T,omitempty"`
	S        string   `json:",omitempty"`
	F        float64  `json:",omitempty"`
	Children []*Node  `json:"C,omitempty"`
	Name     string   `json:"N,omitempty"`
}

func (n Node) String() string {
	buf, _ := json.Marshal(n)
	return string(buf)
}

func Run(args []string) {
	var private, templates, formula, output string
	var dx float64
	flags := flag.NewFlagSet("parse", flag.ExitOnError)
	flags.StringVar(&formula, "formula", Formula(10), "the formula to parse")
	flags.StringVar(&private, "private", defaultPrivateString, "the private variable string")
	flags.StringVar(&templates, "templates", defaultTemplates, "directory of go template functions")
	flags.StringVar(&output, "output", "compute.go", "name of go program to output")
	flags.Float64Var(&dx, "dx", defaultDx, "infinitesimal for numerical differentiation")
	flags.Parse(args)

	code, err := Parse(private, templates, formula, 0.00001)
	if err != nil {
		log.Fatal(err)
	}
	f, err := os.Create(output)
	if err != nil {
		log.Fatal(err)
	}
	f.Write(code)
	f.Close()
}

const (
	defaultPrivateString = "pvt"
	defaultDx            = 0.00001
	defaultTemplates     = "src/xoba/ad/parser/templates"
)

func Parse(private, templates, formula string, dx float64) ([]byte, error) {
	if len(private) == 0 {
		private = defaultPrivateString
	}
	if dx == 0 {
		dx = defaultDx
	}
	if len(templates) == 0 {
		templates = defaultTemplates
	}
	lex := NewContext(NewLexer(strings.NewReader(formula)))
	yyParse(lex)
	if len(lex.errors) > 0 {
		return nil, lex.errors[0]
	}
	if lex.lhs == nil || lex.rhs == nil {
		return nil, fmt.Errorf("parse error: lhs or rhs is nil")
	}
	vars := make(map[string]string)
	vp := &VarParser{vars: vars}
	sp := &StepParser{vars: vars}
	var steps []Step
	steps = append(steps, vp.getVars(lex.rhs, private)...)
	steps = append(steps, sp.program(lex.rhs, private)...)
	y := steps[len(steps)-1].lhs

	checker := new(bytes.Buffer)
	{
		fmt.Fprintf(checker, "const delta_%s = %f\n", private, dx)
		fmt.Fprintf(checker, `calc_%s := func() float64 {
%s
return %s
}
`, private, formula, lex.lhs.S)
		fmt.Fprintf(checker, "tmp1_%s := calc_%s()\n", private, private)
		for k := range vars {
			fmt.Fprintln(checker, "{")
			fmt.Fprintf(checker, "%s += delta_%s\n", k, private)
			fmt.Fprintf(checker, "tmp2_%s := calc_%s()\n", private, private)
			fmt.Fprintf(checker, "%s -= delta_%s\n", k, private)
			fmt.Fprintf(checker, "grad_%s[%q] = (tmp2_%s - tmp1_%s)/delta_%s\n", private, k, private, private, private)
			fmt.Fprintln(checker, "}")
		}
	}

	pgm := new(bytes.Buffer)
	for _, s := range steps {
		fmt.Fprintln(pgm, s)
	}
	computeDerivatives(pgm, steps, private)
	var list []string
	for v := range vars {
		list = append(list, v)
	}
	sort.Strings(list)

	f := new(bytes.Buffer)

	decls := new(bytes.Buffer)
	for _, v := range list {
		fmt.Fprintf(decls, "%s := rand.Float64();\n", v)
		fmt.Fprintf(decls, "fmt.Printf(\"setting %s = %%.20f\\n\",%s)\n", v, v)
	}
	fmt.Fprintln(decls, `fmt.Println();`)

	var imports Imports
	imports.Add("fmt")
	imports.Add("math")
	imports.Add("time")
	imports.Add("math/rand")

	t := template.Must(template.New("output.go").Parse(`// created {{.time}}
// see https://github.com/xoba/ad

package main
{{.imports}}

// automatically compute the value and gradient of {{.qformula}}
func ComputeAD({{.vars}} float64) (float64,map[string]float64) {
grad_{{.private}} := make(map[string]float64)
{{.program}} return {{.y}},grad_{{.private}};
}

func main() {
fmt.Printf("running autodiff code of {{.time}} on %q\n\n", {{ printf "%q" .formula }});
rand.Seed(time.Now().UTC().UnixNano())
{{.decls}}

	c1, grad1 := ComputeAD({{.vars}})
	fmt.Printf("autodiff value   : %f\n", c1)
	fmt.Printf("autodiff gradient: %v\n\n", grad1)

	c2, grad2 := ComputeNumerical({{.vars}})
	fmt.Printf("numeric value   : %f\n", c2)
	fmt.Printf("numeric gradient: %v\n\n", grad2)

	var total float64
	add := func(n string, x float64) {
		fmt.Printf("%s difference: %f\n", n, x)
		total += math.Abs(x)
	}
	add("value", c1-c2)
	for k, v := range grad2 {
		add(fmt.Sprintf("grad[%3s]", k), grad1[k]-v)
	}
fmt.Printf("\nsum of absolute differences: %f\n",total);
}

// numerically compute the value and gradient of {{.qformula}}
func ComputeNumerical({{.vars}} float64) (float64,map[string]float64) {
grad_{{.private}} := make(map[string]float64)
{{.checker}} return tmp1_{{.private}},grad_{{.private}};
}

{{.funcs}}

`))

	templateImports, code, err := GenTemplates(templates, private)
	if err != nil {
		return nil, err
	}
	imports.AddAll(templateImports)

	t.Execute(f, map[string]interface{}{
		"time":     time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
		"decls":    decls.String(),
		"formula":  formula,
		"qformula": fmt.Sprintf("%q", formula),
		"lhs":      lex.lhs.S,
		"vars":     strings.Join(list, ", "),
		"program":  pgm.String(),
		"checker":  checker.String(),
		"y":        y,
		"funcs":    code,
		"private":  private,
		"imports":  imports.String(),
	})

	return GofmtBuffer(f.Bytes())
}

type Imports struct {
	list []string
}

func (i Imports) String() string {
	m := make(map[string]bool)
	for _, x := range i.list {
		x = strings.Replace(x, `"`, ``, -1)
		m[x] = true
	}
	out := new(bytes.Buffer)
	fmt.Fprintf(out, "import (\n")
	for k := range m {
		fmt.Fprintf(out, "%q\n", k)
	}
	fmt.Fprintf(out, ")\n")
	return out.String()
}

func (i *Imports) Add(x string) {
	i.list = append(i.list, x)
}

func (i *Imports) AddAll(list []string) {
	for _, x := range list {
		i.Add(x)
	}
}

func formatImports(list []string) string {
	var out []string
	m := make(map[string]bool)
	for _, x := range list {
		m[x] = true
	}
	for k := range m {
		out = append(out, k)
	}
	return strings.Join(out, "\n")
}

func Gofmt(p string) error {
	cmd := exec.Command("gofmt", "-w", p)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func GofmtBuffer(code []byte) ([]byte, error) {
	out := new(bytes.Buffer)
	cmd := exec.Command("gofmt")
	cmd.Stdin = bytes.NewReader(code)
	cmd.Stdout = out
	if err := cmd.Run(); err != nil {
		return nil, err
	}
	return out.Bytes(), nil
}

type VarParser struct {
	i    int
	vars map[string]string
}

// gets all the vars and name them in-place
func (v *VarParser) getVars(rhs *Node, private string) (out []Step) {
	switch rhs.Type {
	case identifierNT:
		if _, ok := v.vars[rhs.S]; !ok {
			rhs.Name = fmt.Sprintf("v_%d_%s", v.i, private)
			v.vars[rhs.S] = rhs.Name
			v.i++
			step := Step{
				decl: true,
				lhs:  rhs.Name,
				rhs:  rhs.S,
			}
			out = append(out, step)
		} else {
			rhs.Name = v.vars[rhs.S]
		}
	case functionNT:
		for _, n := range rhs.Children {
			out = append(out, v.getVars(n, private)...)
		}
	}
	return
}

type StepParser struct {
	i    int
	vars map[string]string
}

func (s Step) String() string {
	return fmt.Sprintf("%s := %s;", s.lhs, s.rhs)
}

func (s *StepParser) program(rhs *Node, private string) (out []Step) {
	switch rhs.Type {
	case numberNT:
		rhs.Name = fmt.Sprintf("%f", rhs.F)
	case functionNT:
		var args []string
		for _, n := range rhs.Children {
			steps := s.program(n, private)
			out = append(out, steps...)
			args = append(args, n.Name)
		}
		step := Step{
			lhs:  fmt.Sprintf("s_%d_%s", s.i, private),
			rhs:  fmt.Sprintf("%s_%s(%s)", rhs.S, private, strings.Join(args, ",")),
			f:    rhs.S,
			args: args,
		}
		out = append(out, step)
		rhs.Name = step.lhs
		s.i++
	}
	return out
}

func Function(ident string, args ...*Node) *Node {
	return &Node{
		Type:     functionNT,
		S:        ident,
		Children: args,
	}
}

func Number(n float64) *Node {
	return &Node{
		Type: numberNT,
		F:    n,
	}
}

func LexIdentifier(s string) *Node {
	return &Node{
		Type: identifierNT,
		S:    s,
	}
}

func LexNumber(s string) *Node {
	n, _ := strconv.ParseFloat(s, 64)
	return Number(n)
}

func Negate(a *Node) *Node {
	return Function("multiply", Number(-1), a)
}

type context struct {
	lhs *Node
	rhs *Node
	yyLexer
	errors []error
}

func NewContext(y yyLexer) *context {
	return &context{yyLexer: y}
}

func (c *context) Error(e string) {
	c.Error2(fmt.Errorf("%s", e))
}

func (c *context) Error2(e error) {
	log.Printf("oops: %v\n", e)
	c.errors = append(c.errors, e)
}
<|endoftext|>


<commit_before>// Package jsutil provides utility functions for interacting with
// native JavaScript APIs via syscall/js API.
// It has support for common types in honnef.co/go/js/dom/v2.
package jsutil

import (
	"encoding/json"
	"fmt"
	"reflect"
	"syscall/js"

	"honnef.co/go/js/dom/v2"
)

// Wrap returns a wrapper func that handles the conversion from native JavaScript js.Value parameters
// to the following types.
//
// It supports js.Value (left unmodified), dom.Document, dom.Element, dom.Event, dom.HTMLElement, dom.Node.
// It has to be one of those types exactly; it can't be another type that implements the interface like *dom.BasicElement.
//
// For other types, the input is assumed to be a JSON string which is then unmarshalled into that type.
//
// If the number of arguments provided to the wrapped func doesn't match
// the number of arguments for original func, it panics.
//
// Here is example usage:
//
// 	Example
//
// 	func Handler(event dom.Event, htmlElement dom.HTMLElement, data someStruct) {
// 		data.Foo = ... // Use event, htmlElement, data.
// 	}
//
// 	func main() {
// 		js.Global().Set("Handler", jsutil.Wrap(Handler))
// 	}
func Wrap(fn interface{}) js.Func {
	v := reflect.ValueOf(fn)
	return js.FuncOf(func(_ js.Value, args []js.Value) interface{} {
		if len(args) != v.Type().NumIn() {
			panic(fmt.Errorf("wrapped %v got %v arguments, want %v", v.Type().String(), len(args), v.Type().NumIn()))
		}
		in := make([]reflect.Value, v.Type().NumIn())
		for i := range in {
			switch t := v.Type().In(i); t {
			// js.Value is passed through.
			case typeOf((*js.Value)(nil)):
				in[i] = reflect.ValueOf(args[i])

			// dom types are wrapped.
			case typeOf((*dom.Document)(nil)):
				in[i] = reflect.ValueOf(dom.WrapDocument(args[i]))
			case typeOf((*dom.Element)(nil)):
				in[i] = reflect.ValueOf(dom.WrapElement(args[i]))
			case typeOf((*dom.Event)(nil)):
				in[i] = reflect.ValueOf(dom.WrapEvent(args[i]))
			case typeOf((*dom.HTMLElement)(nil)):
				in[i] = reflect.ValueOf(dom.WrapHTMLElement(args[i]))
			case typeOf((*dom.Node)(nil)):
				in[i] = reflect.ValueOf(dom.WrapNode(args[i]))

			// Unmarshal incoming encoded JSON into the Go type.
			default:
				if args[i].Type() != js.TypeString {
					panic(fmt.Errorf("jsutil: incoming value type is %s; want a string with JSON content", args[i].Type()))
				}
				p := reflect.New(t)
				err := json.Unmarshal([]byte(args[i].String()), p.Interface())
				if err != nil {
					panic(fmt.Errorf("jsutil: unmarshaling JSON %q into type %s failed: %v", args[i], t, err))
				}
				in[i] = reflect.Indirect(p)
			}
		}
		v.Call(in)
		return nil
	})
}

// typeOf returns the reflect.Type of what the pointer points to.
func typeOf(pointer interface{}) reflect.Type {
	return reflect.TypeOf(pointer).Elem()
}
<commit_msg>gopherjs_http/jsutil/v2: add js build constraint<commit_after>// +build js

// Package jsutil provides utility functions for interacting with
// native JavaScript APIs via syscall/js API.
// It has support for common types in honnef.co/go/js/dom/v2.
package jsutil

import (
	"encoding/json"
	"fmt"
	"reflect"
	"syscall/js"

	"honnef.co/go/js/dom/v2"
)

// Wrap returns a wrapper func that handles the conversion from native JavaScript js.Value parameters
// to the following types.
//
// It supports js.Value (left unmodified), dom.Document, dom.Element, dom.Event, dom.HTMLElement, dom.Node.
// It has to be one of those types exactly; it can't be another type that implements the interface like *dom.BasicElement.
//
// For other types, the input is assumed to be a JSON string which is then unmarshalled into that type.
//
// If the number of arguments provided to the wrapped func doesn't match
// the number of arguments for original func, it panics.
//
// Here is example usage:
//
// 	Example
//
// 	func Handler(event dom.Event, htmlElement dom.HTMLElement, data someStruct) {
// 		data.Foo = ... // Use event, htmlElement, data.
// 	}
//
// 	func main() {
// 		js.Global().Set("Handler", jsutil.Wrap(Handler))
// 	}
func Wrap(fn interface{}) js.Func {
	v := reflect.ValueOf(fn)
	return js.FuncOf(func(_ js.Value, args []js.Value) interface{} {
		if len(args) != v.Type().NumIn() {
			panic(fmt.Errorf("wrapped %v got %v arguments, want %v", v.Type().String(), len(args), v.Type().NumIn()))
		}
		in := make([]reflect.Value, v.Type().NumIn())
		for i := range in {
			switch t := v.Type().In(i); t {
			// js.Value is passed through.
			case typeOf((*js.Value)(nil)):
				in[i] = reflect.ValueOf(args[i])

			// dom types are wrapped.
			case typeOf((*dom.Document)(nil)):
				in[i] = reflect.ValueOf(dom.WrapDocument(args[i]))
			case typeOf((*dom.Element)(nil)):
				in[i] = reflect.ValueOf(dom.WrapElement(args[i]))
			case typeOf((*dom.Event)(nil)):
				in[i] = reflect.ValueOf(dom.WrapEvent(args[i]))
			case typeOf((*dom.HTMLElement)(nil)):
				in[i] = reflect.ValueOf(dom.WrapHTMLElement(args[i]))
			case typeOf((*dom.Node)(nil)):
				in[i] = reflect.ValueOf(dom.WrapNode(args[i]))

			// Unmarshal incoming encoded JSON into the Go type.
			default:
				if args[i].Type() != js.TypeString {
					panic(fmt.Errorf("jsutil: incoming value type is %s; want a string with JSON content", args[i].Type()))
				}
				p := reflect.New(t)
				err := json.Unmarshal([]byte(args[i].String()), p.Interface())
				if err != nil {
					panic(fmt.Errorf("jsutil: unmarshaling JSON %q into type %s failed: %v", args[i], t, err))
				}
				in[i] = reflect.Indirect(p)
			}
		}
		v.Call(in)
		return nil
	})
}

// typeOf returns the reflect.Type of what the pointer points to.
func typeOf(pointer interface{}) reflect.Type {
	return reflect.TypeOf(pointer).Elem()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"code.google.com/p/go.exp/fsnotify"
	"github.com/hishboy/gocommons/lang"
)

func isMember(element string, array []string) bool {
	for i := 0; i < len(array); i++ {
		if array[i] == element {
			return true
		}
	}
	return false
}

func collectPaths(path []string) []string {
	// paths to be returned
	collectedPaths := make([]string, 1, 1)

	//
	for _, thisPath := range paths {
		err := filepath.Walk(thisPath, func(path string, info os.FileInfo, err error) error {
			if info.IsDir() {
				collectedPaths = append(collectedPaths, path)
			}
			return nil
		})

		if err != nil {
			fmt.Println(err)
		}
	}

	return collectedPaths
}

func SetupWatch(paths []string, excludes []string) (int, *fsnotify.Watcher) {
	var watchedCount int

	paths = collectPaths(paths)
	excludes = collectPaths(excludes)

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Println("Error establishing watcher: ", err)
	}

	// establish watches
	for _, path := range paths {
		if !(isMember(path, excludes)) {
			err = watcher.Watch(path)
			if err != nil {
				fmt.Println("Error: ", err, "  establishing watch on: ", path)
			}
			watchedCount++
		}
	}
	return watcher, watchedCount
}
<commit_msg>Removed more useless imports. Fixed variable name mismatch.<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"code.google.com/p/go.exp/fsnotify"
)

func isMember(element string, array []string) bool {
	for i := 0; i < len(array); i++ {
		if array[i] == element {
			return true
		}
	}
	return false
}

func collectPaths(paths []string) []string {
	// paths to be returned
	collectedPaths := make([]string, 1, 1)

	//
	for _, thisPath := range paths {
		err := filepath.Walk(thisPath, func(path string, info os.FileInfo, err error) error {
			if info.IsDir() {
				collectedPaths = append(collectedPaths, path)
			}
			return nil
		})

		if err != nil {
			fmt.Println(err)
		}
	}

	return collectedPaths
}

func SetupWatch(paths []string, excludes []string) (int, *fsnotify.Watcher) {
	var watchedCount int

	paths = collectPaths(paths)
	excludes = collectPaths(excludes)

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		fmt.Println("Error establishing watcher: ", err)
	}

	// establish watches
	for _, path := range paths {
		if !(isMember(path, excludes)) {
			err = watcher.Watch(path)
			if err != nil {
				fmt.Println("Error: ", err, "  establishing watch on: ", path)
			}
			watchedCount++
		}
	}
	return watchedCount, watcher
}
<|endoftext|>


<commit_before>package clone

import (
	"errors"
	_ "fmt"
	"github.com/jeffail/tunny"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type WOFClone struct {
	Source     string
	Dest       string
	Success    int64
	Error      int64
	Skipped    int64
	Scheduled  int64
	Completed  int64
	MaxRetries float64 // max percentage of errors over scheduled
	Failed     []string
	Logger     *log.WOFLogger
	client     *http.Client
	retries    *pool.LIFOPool
	workpool   *tunny.WorkPool
	timer      time.Time
	done       chan bool
}

func NewWOFClone(source string, dest string, procs int, logger *log.WOFLogger) (*WOFClone, error) {

	// https://golang.org/src/net/http/filetransport.go

	u, err := url.Parse(source)

	if err != nil {
		return nil, err
	}

	var cl *http.Client

	if u.Scheme == "file" {

		// See also: https://code.google.com/p/go/issues/detail?id=2113

		root := u.Path

		if !strings.HasSuffix(root, "/") {
			root = root + "/"
		}

		// See this:
		// root = "/"
		// That's what necessary to make cloning local files work absent
		// tweaking the code below to see whether we are a "file" source
		// and modifying the request URL. The fear of blindly opening up
		// the root level directory on the file system in this context
		// seems a bit premature (not to mention silly) but measure twice
		// and all that good stuff... (20160112/thisisaaronland)

		t := &http.Transport{}
		t.RegisterProtocol("file", http.NewFileTransport(http.Dir(root)))

		cl = &http.Client{Transport: t}
	} else {
		cl = &http.Client{}
	}

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()
	retries := pool.NewLIFOPool()

	ch := make(chan bool)

	c := WOFClone{
		Success:    0,
		Error:      0,
		Skipped:    0,
		Source:     source,
		Dest:       dest,
		Logger:     logger,
		MaxRetries: 25.0, // maybe allow this to be user-defined ?
		client:     cl,
		workpool:   workpool,
		retries:    retries,
		timer:      time.Now(),
		done:       ch,
	}

	go func(c *WOFClone) {

		for {
			select {

			case <-c.done:
				break
			case <-time.After(1 * time.Second):
				c.Status()
			}
		}
	}(&c)

	return &c, nil
}

func (c *WOFClone) CloneMetaFile(file string, skip_existing bool, force_updates bool) error {

	abs_path, _ := filepath.Abs(file)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.Logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	c.timer = time.Now()

	for {

		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		ensure_changes := true
		has_changes := true
		carry_on := false

		remote := c.Source + rel_path
		local := path.Join(c.Dest, rel_path)

		_, err = os.Stat(local)

		if !os.IsNotExist(err) {

			if force_updates {

				c.Logger.Debug("%s already but we are forcing updates", local)
			} else if skip_existing {

				c.Logger.Debug("%s already exists and we are skipping things that exist", local)
				carry_on = true

			} else {

				file_hash, ok := row["file_hash"]

				t1 := time.Now()

				if ok {
					c.Logger.Debug("comparing hardcoded hash (%s) for %s", file_hash, local)
					has_changes, _ = c.HasHashChanged(file_hash, remote)
				} else {
					has_changes, _ = c.HasChanged(local, remote)
				}

				if !has_changes {
					c.Logger.Info("no changes to %s", local)
					carry_on = true
				}

				t2 := time.Since(t1)

				c.Logger.Debug("time to determine whether %s has changed (%t), %v", local, has_changes, t2)
			}

			if carry_on {

				atomic.AddInt64(&c.Scheduled, 1)
				atomic.AddInt64(&c.Completed, 1)
				atomic.AddInt64(&c.Skipped, 1)
				continue
			}

			ensure_changes = false
		}

		wg.Add(1)
		atomic.AddInt64(&c.Scheduled, 1)

		go func(c *WOFClone, rel_path string, ensure_changes bool) {

			defer wg.Done()

			_, err = c.workpool.SendWork(func() {

				t1 := time.Now()
				cl_err := c.ClonePath(rel_path, ensure_changes)

				t2 := time.Since(t1)

				c.Logger.Debug("time to process %s : %v", rel_path, t2)

				if cl_err != nil {
					atomic.AddInt64(&c.Error, 1)
					c.retries.Push(&pool.PoolString{String: rel_path})
				} else {
					atomic.AddInt64(&c.Success, 1)
				}

				atomic.AddInt64(&c.Completed, 1)
			})

		}(c, rel_path, ensure_changes)
	}

	wg.Wait()

	ok := c.ProcessRetries()

	if !ok {
		c.Logger.Warning("failed to process retries")
		return errors.New("One of file failed to be cloned")
	}

	return nil
}

func (c *WOFClone) ProcessRetries() bool {

	to_retry := c.retries.Length()

	if to_retry > 0 {

		scheduled_f := float64(c.Scheduled)
		retry_f := float64(to_retry)

		pct := (retry_f / scheduled_f) * 100.0

		if pct > c.MaxRetries {
			c.Logger.Warning("E_EXCESSIVE_ERRORS, %f percent of scheduled processes failed thus undermining our faith that they will work now...", pct)
			return false
		}

		c.Logger.Info("There are %d failed requests that will now be retried", to_retry)

		wg := new(sync.WaitGroup)

		for c.retries.Length() > 0 {

			r, ok := c.retries.Pop()

			if !ok {
				c.Logger.Error("failed to pop retries because... computers?")
				break
			}

			rel_path := r.StringValue()

			atomic.AddInt64(&c.Scheduled, 1)
			wg.Add(1)

			go func(c *WOFClone, rel_path string) {

				defer wg.Done()

				c.workpool.SendWork(func() {

					ensure_changes := true

					t1 := time.Now()

					cl_err := c.ClonePath(rel_path, ensure_changes)

					t2 := time.Since(t1)

					c.Logger.Debug("time to retry clone %s : %v\n", rel_path, t2)

					if cl_err != nil {
						atomic.AddInt64(&c.Error, 1)
					} else {
						atomic.AddInt64(&c.Error, -1)
					}

					atomic.AddInt64(&c.Completed, 1)
				})

			}(c, rel_path)
		}

		wg.Wait()
	}

	return true
}

func (c *WOFClone) ClonePath(rel_path string, ensure_changes bool) error {

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) && ensure_changes {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.Logger.Debug("%s has not changed so skipping", local)
			atomic.AddInt64(&c.Skipped, 1)
			return nil
		}

	}

	process_err := c.Process(remote, local)

	if process_err != nil {
		return process_err
	}

	return nil
}

// don't return true if there's a problem - move that logic up above

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	local_hash, err := utils.HashFile(local)

	if err != nil {
		c.Logger.Error("Failed to hash %s, becase %v", local, err)
		return change, err
	}

	return c.HasHashChanged(local_hash, remote)
}

func (c *WOFClone) HasHashChanged(local_hash string, remote string) (bool, error) {

	change := true

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	rsp.Body.Close()
	// defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.Logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.Logger.Info("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	t1 := time.Now()

	rsp, fetch_err := c.Fetch("GET", remote)

	t2 := time.Since(t1)

	c.Logger.Debug("time to fetch %s: %v", remote, t2)

	if fetch_err != nil {
		return fetch_err
	}

	// defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.Logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	rsp.Body.Close()

	go func(local string, contents []byte) error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.Logger.Error("Failed to write %s, because %v", local, write_err)

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.Logger.Debug("Wrote %s to disk", local)
		return nil
	}(local, contents)

	return nil
}

func (c *WOFClone) Fetch(method string, url string) (*http.Response, error) {

	c.Logger.Debug("%s %s", method, url)

	req, _ := http.NewRequest(method, url, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	if err != nil {
		c.Logger.Error("Failed to %s %s, because %v", method, url, err)
		return nil, err
	}

	// Notice how we are not closing rsp.Body - that's because we are passing
	// it (rsp) back up the stack

	// See also: https://github.com/whosonfirst/go-whosonfirst-clone/issues/6

	expected := 200

	if rsp.StatusCode != expected {
		c.Logger.Error("Failed to %s %s, because we expected %d from source and got '%s' instead", method, url, expected, rsp.Status)
		return nil, errors.New(rsp.Status)
	}

	return rsp, nil
}

func (c *WOFClone) Status() {

	t2 := time.Since(c.timer)

	c.Logger.Info("scheduled: %d completed: %d success: %d error: %d skipped: %d to retry: %d goroutines: %d time: %v",
		c.Scheduled, c.Completed, c.Success, c.Error, c.Skipped, c.retries.Length(), runtime.NumGoroutine(), t2)
}
<commit_msg>update Fetch to account for file:/// source URLs; notes and comments<commit_after>package clone

import (
	"errors"
	_ "fmt"
	"github.com/jeffail/tunny"
	csv "github.com/whosonfirst/go-whosonfirst-csv"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type WOFClone struct {
	Source     string
	Dest       string
	Success    int64
	Error      int64
	Skipped    int64
	Scheduled  int64
	Completed  int64
	MaxRetries float64 // max percentage of errors over scheduled
	Failed     []string
	Logger     *log.WOFLogger
	client     *http.Client
	retries    *pool.LIFOPool
	workpool   *tunny.WorkPool
	timer      time.Time
	done       chan bool
}

func NewWOFClone(source string, dest string, procs int, logger *log.WOFLogger) (*WOFClone, error) {

	// https://golang.org/src/net/http/filetransport.go

	u, err := url.Parse(source)

	if err != nil {
		return nil, err
	}

	var cl *http.Client

	if u.Scheme == "file" {

		root := u.Path

		if !strings.HasSuffix(root, "/") {
			root = root + "/"
		}

		/*
			Pay attention to what's going here. Absent tweaking the URL to
			fetch in the 'Fetch' method the following will not work. In
			order to make this working *without* tweaking the URL you would
			need to specifiy the root as '/' which just seems like a bad
			idea. The fear of blindly opening up the root level directory on
			the file system in this context may seem a bit premature (not to
			mention silly) but measure twice and all that good stuff...
			See also: https://code.google.com/p/go/issues/detail?id=2113
			(20160112/thisisaaronland)
		*/

		t := &http.Transport{}
		t.RegisterProtocol("file", http.NewFileTransport(http.Dir(root)))

		cl = &http.Client{Transport: t}
	} else {
		cl = &http.Client{}
	}

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()
	retries := pool.NewLIFOPool()

	ch := make(chan bool)

	c := WOFClone{
		Success:    0,
		Error:      0,
		Skipped:    0,
		Source:     source,
		Dest:       dest,
		Logger:     logger,
		MaxRetries: 25.0, // maybe allow this to be user-defined ?
		client:     cl,
		workpool:   workpool,
		retries:    retries,
		timer:      time.Now(),
		done:       ch,
	}

	go func(c *WOFClone) {

		for {
			select {

			case <-c.done:
				break
			case <-time.After(1 * time.Second):
				c.Status()
			}
		}
	}(&c)

	return &c, nil
}

func (c *WOFClone) CloneMetaFile(file string, skip_existing bool, force_updates bool) error {

	abs_path, _ := filepath.Abs(file)

	reader, read_err := csv.NewDictReader(abs_path)

	if read_err != nil {
		c.Logger.Error("Failed to read %s, because %v", abs_path, read_err)
		return read_err
	}

	wg := new(sync.WaitGroup)

	c.timer = time.Now()

	for {

		row, err := reader.Read()

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}

		rel_path, ok := row["path"]

		if !ok {
			continue
		}

		ensure_changes := true
		has_changes := true
		carry_on := false

		remote := c.Source + rel_path
		local := path.Join(c.Dest, rel_path)

		_, err = os.Stat(local)

		if !os.IsNotExist(err) {

			if force_updates {

				c.Logger.Debug("%s already but we are forcing updates", local)
			} else if skip_existing {

				c.Logger.Debug("%s already exists and we are skipping things that exist", local)
				carry_on = true

			} else {

				file_hash, ok := row["file_hash"]

				t1 := time.Now()

				if ok {
					c.Logger.Debug("comparing hardcoded hash (%s) for %s", file_hash, local)
					has_changes, _ = c.HasHashChanged(file_hash, remote)
				} else {
					has_changes, _ = c.HasChanged(local, remote)
				}

				if !has_changes {
					c.Logger.Info("no changes to %s", local)
					carry_on = true
				}

				t2 := time.Since(t1)

				c.Logger.Debug("time to determine whether %s has changed (%t), %v", local, has_changes, t2)
			}

			if carry_on {

				atomic.AddInt64(&c.Scheduled, 1)
				atomic.AddInt64(&c.Completed, 1)
				atomic.AddInt64(&c.Skipped, 1)
				continue
			}

			ensure_changes = false
		}

		wg.Add(1)
		atomic.AddInt64(&c.Scheduled, 1)

		go func(c *WOFClone, rel_path string, ensure_changes bool) {

			defer wg.Done()

			_, err = c.workpool.SendWork(func() {

				t1 := time.Now()
				cl_err := c.ClonePath(rel_path, ensure_changes)

				t2 := time.Since(t1)

				c.Logger.Debug("time to process %s : %v", rel_path, t2)

				if cl_err != nil {
					atomic.AddInt64(&c.Error, 1)
					c.retries.Push(&pool.PoolString{String: rel_path})
				} else {
					atomic.AddInt64(&c.Success, 1)
				}

				atomic.AddInt64(&c.Completed, 1)
			})

		}(c, rel_path, ensure_changes)
	}

	wg.Wait()

	ok := c.ProcessRetries()

	if !ok {
		c.Logger.Warning("failed to process retries")
		return errors.New("One of file failed to be cloned")
	}

	return nil
}

func (c *WOFClone) ProcessRetries() bool {

	to_retry := c.retries.Length()

	if to_retry > 0 {

		scheduled_f := float64(c.Scheduled)
		retry_f := float64(to_retry)

		pct := (retry_f / scheduled_f) * 100.0

		if pct > c.MaxRetries {
			c.Logger.Warning("E_EXCESSIVE_ERRORS, %f percent of scheduled processes failed thus undermining our faith that they will work now...", pct)
			return false
		}

		c.Logger.Info("There are %d failed requests that will now be retried", to_retry)

		wg := new(sync.WaitGroup)

		for c.retries.Length() > 0 {

			r, ok := c.retries.Pop()

			if !ok {
				c.Logger.Error("failed to pop retries because... computers?")
				break
			}

			rel_path := r.StringValue()

			atomic.AddInt64(&c.Scheduled, 1)
			wg.Add(1)

			go func(c *WOFClone, rel_path string) {

				defer wg.Done()

				c.workpool.SendWork(func() {

					ensure_changes := true

					t1 := time.Now()

					cl_err := c.ClonePath(rel_path, ensure_changes)

					t2 := time.Since(t1)

					c.Logger.Debug("time to retry clone %s : %v\n", rel_path, t2)

					if cl_err != nil {
						atomic.AddInt64(&c.Error, 1)
					} else {
						atomic.AddInt64(&c.Error, -1)
					}

					atomic.AddInt64(&c.Completed, 1)
				})

			}(c, rel_path)
		}

		wg.Wait()
	}

	return true
}

func (c *WOFClone) ClonePath(rel_path string, ensure_changes bool) error {

	remote := c.Source + rel_path
	local := path.Join(c.Dest, rel_path)

	_, err := os.Stat(local)

	if !os.IsNotExist(err) && ensure_changes {

		change, _ := c.HasChanged(local, remote)

		if !change {

			c.Logger.Debug("%s has not changed so skipping", local)
			atomic.AddInt64(&c.Skipped, 1)
			return nil
		}

	}

	process_err := c.Process(remote, local)

	if process_err != nil {
		return process_err
	}

	return nil
}

// don't return true if there's a problem - move that logic up above

func (c *WOFClone) HasChanged(local string, remote string) (bool, error) {

	change := true

	local_hash, err := utils.HashFile(local)

	if err != nil {
		c.Logger.Error("Failed to hash %s, becase %v", local, err)
		return change, err
	}

	return c.HasHashChanged(local_hash, remote)
}

func (c *WOFClone) HasHashChanged(local_hash string, remote string) (bool, error) {

	change := true

	rsp, err := c.Fetch("HEAD", remote)

	if err != nil {
		return change, err
	}

	rsp.Body.Close()
	// defer rsp.Body.Close()

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		change = false
	}

	return change, nil
}

func (c *WOFClone) Process(remote string, local string) error {

	c.Logger.Debug("fetch %s and store in %s", remote, local)

	local_root := path.Dir(local)

	_, err := os.Stat(local_root)

	if os.IsNotExist(err) {
		c.Logger.Debug("create %s", local_root)
		os.MkdirAll(local_root, 0755)
	}

	t1 := time.Now()

	rsp, fetch_err := c.Fetch("GET", remote)

	t2 := time.Since(t1)

	c.Logger.Debug("time to fetch %s: %v", remote, t2)

	if fetch_err != nil {
		return fetch_err
	}

	// defer rsp.Body.Close()

	contents, read_err := ioutil.ReadAll(rsp.Body)

	if read_err != nil {
		c.Logger.Error("failed to read body for %s, because %v", remote, read_err)
		return read_err
	}

	rsp.Body.Close()

	go func(local string, contents []byte) error {

		write_err := ioutil.WriteFile(local, contents, 0644)

		if write_err != nil {
			c.Logger.Error("Failed to write %s, because %v", local, write_err)

			atomic.AddInt64(&c.Success, -1)
			atomic.AddInt64(&c.Error, 1)

			return write_err
		}

		c.Logger.Debug("Wrote %s to disk", local)
		return nil
	}(local, contents)

	return nil
}

func (c *WOFClone) Fetch(method string, remote string) (*http.Response, error) {

	/*
	  See notes in NewWOFClone for details on what's going on here. Given that
	  we are already testing whether c.Source is a file URI parsing remote here
	  is probably a bit of a waste. We will live with the cost for now and optimize
	  as necessary... (20160112/thisisaaronland)
	*/

	u, _ := url.Parse(remote)

	if u.Scheme == "file" {
		remote = strings.Replace(remote, c.Source, "", 1)
		remote = "file:///" + remote
		c.Logger.Debug("remote is now %s", remote)
	}

	c.Logger.Debug("%s %s", method, remote)

	req, _ := http.NewRequest(method, remote, nil)
	req.Close = true

	rsp, err := c.client.Do(req)

	if err != nil {
		c.Logger.Error("Failed to %s %s, because %v", method, remote, err)
		return nil, err
	}

	// Notice how we are not closing rsp.Body - that's because we are passing
	// it (rsp) back up the stack

	// See also: https://github.com/whosonfirst/go-whosonfirst-clone/issues/6

	expected := 200

	if rsp.StatusCode != expected {
		c.Logger.Error("Failed to %s %s, because we expected %d from source and got '%s' instead", method, remote, expected, rsp.Status)
		return nil, errors.New(rsp.Status)
	}

	return rsp, nil
}

func (c *WOFClone) Status() {

	t2 := time.Since(c.timer)

	c.Logger.Info("scheduled: %d completed: %d success: %d error: %d skipped: %d to retry: %d goroutines: %d time: %v",
		c.Scheduled, c.Completed, c.Success, c.Error, c.Skipped, c.retries.Length(), runtime.NumGoroutine(), t2)
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) Clinton Freeman 2016
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
 * associated documentation files (the "Software"), to deal in the Software without restriction,
 * including without limitation the rights to use, copy, modify, merge, publish, distribute,
 * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
 * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package main

import (
	"github.com/akualab/dmx"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	"log"
	"time"
)

// WeatherMachine holds connections to everything we need to manipulate the installation.
type WeatherMachine struct {
	stop    chan bool     // Channel for stopping the control elements of the installation.
	dmx     *dmx.DMX      // The DMX connection for writting messages to the Smoke machine and lights.
	config  Configuration // The configuration element for the installation.
	lastRun time.Time     // The last time the installation was run.
}

// ****************************************************************************
// ****************************************************************************
// Functions for manipulating the installation state; idle, warmup and running.
// ****************************************************************************
// ****************************************************************************

// stateFunctions are used to manipulate the WeatherMachine through the various states.
type stateFn func(state *WeatherMachine, msg HRMsg) stateFn

// idle is the state the weathermachine enters when sitting alone, with no one interacting with it.
func idle(state *WeatherMachine, msg HRMsg) (sF stateFn) {
	if msg.Contact {
		log.Printf("INFO: entering warmup")
		enableLight(state.config.S1Beat, state.config, state.dmx)
		go enablePump(state.config, state.stop)

		return warmup // skin contact has been made, enable light and enter warmup.
	}

	return idle // remain idle.
}

// warmup is the state the weathermachine enters when someone first touches it.
func warmup(state *WeatherMachine, msg HRMsg) stateFn {
	if msg.Contact && msg.HeartRate > 0 {
		// Wait for the fog to clear from the last run before running again.
		log.Printf("INFO: Waiting for fog to clear")
		d := int64(state.config.FanDuration) - time.Since(state.lastRun).Nanoseconds()
		time.Sleep(time.Nanosecond * time.Duration(d))

		log.Printf("INFO: entering running")
		go enableLightPulse(state.config, msg.HeartRate, state.stop, state.dmx)
		go enableSmoke(state.config, state.stop, state.dmx)
		go enableFan(state.config, state.stop)

		return running // skin contact and heart rate recieved, start the installation.
	} else if !msg.Contact {
		state.stop <- true // Pump starts at initial contact. If we lost contact between
		// then and now we need to shut it down.
		state.lastRun = time.Now()

		disableLight(state.config, state.dmx)

		log.Printf("INFO: entering idle")
		return idle // skin contact lost. Return to idle.
	}

	return warmup
}

// running is the state the weathermachine enters when someone is engaging with it.
func running(state *WeatherMachine, msg HRMsg) stateFn {
	if !msg.Contact {
		state.stop <- true
		state.stop <- true
		state.stop <- true
		state.stop <- true

		log.Printf("INFO: entering idle")
		return idle // skin contact lost. Return to idle.
	}

	return running // Keep the installation running.
}

// ****************************************************************************
// ****************************************************************************
// Functions for manipulating the physical installation; lights, smoke and fan.
// ****************************************************************************
// ****************************************************************************

// enableLight turns on the light via the supplied DMX connection 'dmx' with the supplied colour 'l'.
func enableLight(l LightColour, c Configuration, dmx *dmx.DMX) {
	log.Printf("INFO: Light on")
	embd.DigitalWrite(c.GPIOPinLight, embd.High)
	dmx.SetChannel(4, byte(l.Red))
	dmx.SetChannel(5, byte(l.Green))
	dmx.SetChannel(6, byte(l.Blue))
	dmx.SetChannel(7, byte(l.Amber))
	dmx.SetChannel(8, byte(l.Dimmer))
	dmx.Render()
}

// disableLight turns off the light via the supplied DMX connection 'dmx'.
func disableLight(c Configuration, dmx *dmx.DMX) {
	log.Printf("INFO: Light off")
	embd.DigitalWrite(c.GPIOPinLight, embd.Low)
	dmx.SetChannel(4, 0)
	dmx.SetChannel(5, 0)
	dmx.SetChannel(6, 0)
	dmx.SetChannel(7, 0)
	dmx.SetChannel(8, 0)
	dmx.Render()
}

// pulseLight pulses the light for a fixed duration.
func pulseLight(c Configuration, dmx *dmx.DMX) {
	enableLight(c.S1Beat, c, dmx)
	time.Sleep(time.Millisecond * time.Duration(c.S1Duration))
	disableLight(c, dmx)

	time.Sleep(time.Millisecond * time.Duration(c.S1Pause))

	enableLight(c.S2Beat, c, dmx)
	time.Sleep(time.Millisecond * time.Duration(c.S2Duration))
	disableLight(c, dmx)
}

// enableLightPulse starts the light pulsing by the frequency defined by hr. The light remains
// pulsing till being notified to stop on d.
func enableLightPulse(c Configuration, hr int, d chan bool, dmx *dmx.DMX) {
	// Perform the first heart beat straight away.
	pulseLight(c, dmx)

	dt := int((60000.0 / float32(hr)) * c.BeatRate)
	ticker := time.NewTicker(time.Millisecond * time.Duration(dt)).C

	// Sharp fixed length, pulse of light with variable off gap depending on HR.
	for {
		select {
		case <-ticker:
			pulseLight(c, dmx)

		case <-d:
			return
		}
	}
}

// pulsePump runs the pump for the duration specified in the configuration.
func pulsePump(c Configuration) {
	log.Printf("INFO: Pump on")
	embd.DigitalWrite(c.GPIOPinPump, embd.High)

	time.Sleep(time.Millisecond * time.Duration(c.PumpDuration))

	log.Printf("INFO: Pump Off")
	embd.DigitalWrite(c.GPIOPinPump, embd.Low)
}

// enablePump switches the relay on for the water pump after DeltaTPump milliseconds have expired
// in the configuration. Pump remains on till being notified to stop on d.
func enablePump(c Configuration, d chan bool) {
	dt := time.NewTimer(time.Millisecond * time.Duration(c.DeltaTPump)).C
	var ticker <-chan time.Time

	for {
		select {
		case <-dt:
			pulsePump(c)
			ticker = time.NewTicker(time.Millisecond * time.Duration(c.PumpInterval)).C

		case <-ticker:
			pulsePump(c)

		case <-d:
			return
		}
	}
}

// enableFan switches the relay on for the fan after DeltaTFan milliseconds have expired
// in the configuration. Fan remains on till being notified to stop on d.
func enableFan(c Configuration, d chan bool) {
	dt := time.NewTimer(time.Millisecond * time.Duration(c.DeltaTFan)).C

	for {
		select {
		case <-dt:
			log.Printf("INFO: Fan On")
			embd.DigitalWrite(c.GPIOPinFan, embd.High)

		case <-d:
			// Wait for the fan duration to clear the smoke chamber.
			ft := time.NewTimer(time.Millisecond * time.Duration(c.FanDuration)).C
			<-ft
			log.Printf("INFO: Fan Off")
			embd.DigitalWrite(c.GPIOPinFan, embd.Low)
			return
		}
	}
}

// puffSmoke enables the smoke machine via the supplied DMX connection 'dmx' for a period of
// time and intentsity supplied in configuration.
func puffSmoke(c Configuration, dmx *dmx.DMX) {
	log.Printf("INFO: Smoke on")
	dmx.SetChannel(1, byte(c.SmokeVolume))
	dmx.Render()

	time.Sleep(time.Millisecond * time.Duration(c.SmokeDuration))

	log.Printf("INFO: Smoke off")
	dmx.SetChannel(1, 0)
	dmx.Render()
}

// enableSmoke enages the DMX smoke machine by the SmokeVolume amount in the configuration.
// Smoke Machine remains on till being notified to stop on d.
func enableSmoke(c Configuration, d chan bool, dmx *dmx.DMX) {
	dt := time.NewTimer(time.Millisecond * time.Duration(c.DeltaTSmoke)).C
	var ticker <-chan time.Time

	for {
		select {
		case <-dt:
			puffSmoke(c, dmx)
			ticker = time.NewTicker(time.Millisecond * time.Duration(c.SmokeInterval)).C

		case <-ticker:
			puffSmoke(c, dmx)

		case <-d:
			return
		}
	}
}
<commit_msg>milliseconds to nanoseconds.<commit_after>/*
 * Copyright (c) Clinton Freeman 2016
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
 * associated documentation files (the "Software"), to deal in the Software without restriction,
 * including without limitation the rights to use, copy, modify, merge, publish, distribute,
 * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
 * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package main

import (
	"github.com/akualab/dmx"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	"log"
	"time"
)

// WeatherMachine holds connections to everything we need to manipulate the installation.
type WeatherMachine struct {
	stop    chan bool     // Channel for stopping the control elements of the installation.
	dmx     *dmx.DMX      // The DMX connection for writting messages to the Smoke machine and lights.
	config  Configuration // The configuration element for the installation.
	lastRun time.Time     // The last time the installation was run.
}

// ****************************************************************************
// ****************************************************************************
// Functions for manipulating the installation state; idle, warmup and running.
// ****************************************************************************
// ****************************************************************************

// stateFunctions are used to manipulate the WeatherMachine through the various states.
type stateFn func(state *WeatherMachine, msg HRMsg) stateFn

// idle is the state the weathermachine enters when sitting alone, with no one interacting with it.
func idle(state *WeatherMachine, msg HRMsg) (sF stateFn) {
	if msg.Contact {
		log.Printf("INFO: entering warmup")
		enableLight(state.config.S1Beat, state.config, state.dmx)
		go enablePump(state.config, state.stop)

		return warmup // skin contact has been made, enable light and enter warmup.
	}

	return idle // remain idle.
}

// warmup is the state the weathermachine enters when someone first touches it.
func warmup(state *WeatherMachine, msg HRMsg) stateFn {
	if msg.Contact && msg.HeartRate > 0 {
		// Wait for the fog to clear from the last run before running again.
		log.Printf("INFO: Waiting for fog to clear")
		d := (int64(state.config.FanDuration)*1000000) - time.Since(state.lastRun).Nanoseconds()
		time.Sleep(time.Nanosecond * time.Duration(d))

		log.Printf("INFO: entering running")
		go enableLightPulse(state.config, msg.HeartRate, state.stop, state.dmx)
		go enableSmoke(state.config, state.stop, state.dmx)
		go enableFan(state.config, state.stop)

		return running // skin contact and heart rate recieved, start the installation.
	} else if !msg.Contact {
		state.stop <- true // Pump starts at initial contact. If we lost contact between
		// then and now we need to shut it down.
		state.lastRun = time.Now()

		disableLight(state.config, state.dmx)

		log.Printf("INFO: entering idle")
		return idle // skin contact lost. Return to idle.
	}

	return warmup
}

// running is the state the weathermachine enters when someone is engaging with it.
func running(state *WeatherMachine, msg HRMsg) stateFn {
	if !msg.Contact {
		state.stop <- true
		state.stop <- true
		state.stop <- true
		state.stop <- true

		log.Printf("INFO: entering idle")
		return idle // skin contact lost. Return to idle.
	}

	return running // Keep the installation running.
}

// ****************************************************************************
// ****************************************************************************
// Functions for manipulating the physical installation; lights, smoke and fan.
// ****************************************************************************
// ****************************************************************************

// enableLight turns on the light via the supplied DMX connection 'dmx' with the supplied colour 'l'.
func enableLight(l LightColour, c Configuration, dmx *dmx.DMX) {
	log.Printf("INFO: Light on")
	embd.DigitalWrite(c.GPIOPinLight, embd.High)
	dmx.SetChannel(4, byte(l.Red))
	dmx.SetChannel(5, byte(l.Green))
	dmx.SetChannel(6, byte(l.Blue))
	dmx.SetChannel(7, byte(l.Amber))
	dmx.SetChannel(8, byte(l.Dimmer))
	dmx.Render()
}

// disableLight turns off the light via the supplied DMX connection 'dmx'.
func disableLight(c Configuration, dmx *dmx.DMX) {
	log.Printf("INFO: Light off")
	embd.DigitalWrite(c.GPIOPinLight, embd.Low)
	dmx.SetChannel(4, 0)
	dmx.SetChannel(5, 0)
	dmx.SetChannel(6, 0)
	dmx.SetChannel(7, 0)
	dmx.SetChannel(8, 0)
	dmx.Render()
}

// pulseLight pulses the light for a fixed duration.
func pulseLight(c Configuration, dmx *dmx.DMX) {
	enableLight(c.S1Beat, c, dmx)
	time.Sleep(time.Millisecond * time.Duration(c.S1Duration))
	disableLight(c, dmx)

	time.Sleep(time.Millisecond * time.Duration(c.S1Pause))

	enableLight(c.S2Beat, c, dmx)
	time.Sleep(time.Millisecond * time.Duration(c.S2Duration))
	disableLight(c, dmx)
}

// enableLightPulse starts the light pulsing by the frequency defined by hr. The light remains
// pulsing till being notified to stop on d.
func enableLightPulse(c Configuration, hr int, d chan bool, dmx *dmx.DMX) {
	// Perform the first heart beat straight away.
	pulseLight(c, dmx)

	dt := int((60000.0 / float32(hr)) * c.BeatRate)
	ticker := time.NewTicker(time.Millisecond * time.Duration(dt)).C

	// Sharp fixed length, pulse of light with variable off gap depending on HR.
	for {
		select {
		case <-ticker:
			pulseLight(c, dmx)

		case <-d:
			return
		}
	}
}

// pulsePump runs the pump for the duration specified in the configuration.
func pulsePump(c Configuration) {
	log.Printf("INFO: Pump on")
	embd.DigitalWrite(c.GPIOPinPump, embd.High)

	time.Sleep(time.Millisecond * time.Duration(c.PumpDuration))

	log.Printf("INFO: Pump Off")
	embd.DigitalWrite(c.GPIOPinPump, embd.Low)
}

// enablePump switches the relay on for the water pump after DeltaTPump milliseconds have expired
// in the configuration. Pump remains on till being notified to stop on d.
func enablePump(c Configuration, d chan bool) {
	dt := time.NewTimer(time.Millisecond * time.Duration(c.DeltaTPump)).C
	var ticker <-chan time.Time

	for {
		select {
		case <-dt:
			pulsePump(c)
			ticker = time.NewTicker(time.Millisecond * time.Duration(c.PumpInterval)).C

		case <-ticker:
			pulsePump(c)

		case <-d:
			return
		}
	}
}

// enableFan switches the relay on for the fan after DeltaTFan milliseconds have expired
// in the configuration. Fan remains on till being notified to stop on d.
func enableFan(c Configuration, d chan bool) {
	dt := time.NewTimer(time.Millisecond * time.Duration(c.DeltaTFan)).C

	for {
		select {
		case <-dt:
			log.Printf("INFO: Fan On")
			embd.DigitalWrite(c.GPIOPinFan, embd.High)

		case <-d:
			// Wait for the fan duration to clear the smoke chamber.
			ft := time.NewTimer(time.Millisecond * time.Duration(c.FanDuration)).C
			<-ft
			log.Printf("INFO: Fan Off")
			embd.DigitalWrite(c.GPIOPinFan, embd.Low)
			return
		}
	}
}

// puffSmoke enables the smoke machine via the supplied DMX connection 'dmx' for a period of
// time and intentsity supplied in configuration.
func puffSmoke(c Configuration, dmx *dmx.DMX) {
	log.Printf("INFO: Smoke on")
	dmx.SetChannel(1, byte(c.SmokeVolume))
	dmx.Render()

	time.Sleep(time.Millisecond * time.Duration(c.SmokeDuration))

	log.Printf("INFO: Smoke off")
	dmx.SetChannel(1, 0)
	dmx.Render()
}

// enableSmoke enages the DMX smoke machine by the SmokeVolume amount in the configuration.
// Smoke Machine remains on till being notified to stop on d.
func enableSmoke(c Configuration, d chan bool, dmx *dmx.DMX) {
	dt := time.NewTimer(time.Millisecond * time.Duration(c.DeltaTSmoke)).C
	var ticker <-chan time.Time

	for {
		select {
		case <-dt:
			puffSmoke(c, dmx)
			ticker = time.NewTicker(time.Millisecond * time.Duration(c.SmokeInterval)).C

		case <-ticker:
			puffSmoke(c, dmx)

		case <-d:
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 NDP Systèmes. All Rights Reserved.
// See LICENSE file for full licensing details.

package templates

import (
	"testing"

	"github.com/hexya-erp/hexya/hexya/tools/xmlutils"
	. "github.com/smartystreets/goconvey/convey"
)

var tmplDef1 = `
<template id="my_id" page="True">

Hello World

</template>
`

var tmplDef2 = `
<template id="my_other_id" priority="12" optional="enabled">

Name

			<t t-esc="Function"/>

			<t t-esc="Email"/>

</template>
`

var tmplDef3 = `
<template inherit_id="my_other_id">

		<t t-esc="CompanyName"/>

	<xpath expr="//t[@t-esc='Email']" position="after">
		<t t-esc="Phone"/>
	</xpath>
</template>
`

var tmplDef4 = `
<template inherit_id="my_other_id">

			<t t-esc="Address"/>

<t t-esc="Name"/>

</template>
`

var tmplDef5 = `
<template inherit_id="my_other_id">
	<xpath expr="//t[@t-esc='Address']/.." position="attributes">
		<attribute name="name">address</attribute>
		<attribute name="string">Address</attribute>
	</xpath>
</template>
`

var tmplDef8 = `
<template inherit_id="my_other_id" id="new_base_view">
	<xpath expr="//t[@t-esc='Email']" position="after">
		<t t-raw="Fax"/>
	</xpath>
</template>
`

var tmplDef9 = `
<template inherit_id="new_base_view">
	<xpath expr="//t[@t-raw='Fax']" position="before">
		<t t-raw="Mobile"/>
	</xpath>
</template>
`

func loadView(xml string) {
	elt, err := xmlutils.XMLToElement(xml)
	if err != nil {
		panic(err)
	}
	LoadFromEtree(elt)
}

func TestViews(t *testing.T) {
	Convey("Creating Template 1", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 1)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		template := Registry.collection.GetByID("my_id")
		So(template.ID, ShouldEqual, "my_id")
		So(template.Page, ShouldEqual, true)
		So(template.Optional, ShouldEqual, false)
		So(template.OptionalDefault, ShouldEqual, false)
		So(template.Priority, ShouldEqual, 16)
		So(string(template.p2Content), ShouldEqual,
			`{% set _1 = _0 %}

		{% for line in lines %}

{{ line.UserName }}

Hello World

		{% endfor %}

`)
	})
	Convey("Creating Template 2", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template := Registry.collection.GetByID("my_other_id")
		So(template.ID, ShouldEqual, "my_other_id")
		So(template.Page, ShouldEqual, false)
		So(template.Optional, ShouldEqual, true)
		So(template.OptionalDefault, ShouldEqual, true)
		So(template.Priority, ShouldEqual, 12)
		So(string(template.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

Name

			{{ Function }}

			{{ Email }}

`)
	})
	Convey("Inheriting Template 2", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template1 := Registry.collection.GetByID("my_id")
		So(string(template1.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

		{% for line in lines %}

{{ line.UserName }}

Hello World

		{% endfor %}

`)
		template2 := Registry.collection.GetByID("my_other_id")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

Name

			{{ Function }}
		{{ CompanyName }}

			{{ Email }}{{ Phone }}

`)
	})

	Convey("More inheritance on Template 2", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)

	})
	Convey("Modifying inherited modifications on Template 2", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		loadView(tmplDef5)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)
	})
	Convey("Create new base template from inheritance", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		loadView(tmplDef5)
		loadView(tmplDef8)
		BootStrap()
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("new_base_view"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		newTemplate := Registry.collection.GetByID("new_base_view")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)
		So(string(newTemplate.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Fax|safe }}{{ Phone }}

`)
	})
	Convey("Inheriting new base template from inheritance", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		loadView(tmplDef5)
		loadView(tmplDef8)
		loadView(tmplDef9)
		BootStrap()
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("new_base_view"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		newTemplate := Registry.collection.GetByID("new_base_view")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)
		So(string(newTemplate.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Mobile|safe }}{{ Fax|safe }}{{ Phone }}

`)
	})
}
<commit_msg>Improved test coverage on templates<commit_after>// Copyright 2018 NDP Systèmes. All Rights Reserved.
// See LICENSE file for full licensing details.

package templates

import (
	"io/ioutil"
	"net/http/httptest"
	"testing"

	"github.com/flosch/pongo2"
	"github.com/hexya-erp/hexya/hexya/i18n"
	"github.com/hexya-erp/hexya/hexya/tools/xmlutils"
	. "github.com/smartystreets/goconvey/convey"
)

var tmplDef1 = `
<template id="my_id" page="True">

Hello World

</template>
`

var tmplDef2 = `
<template id="my_other_id" priority="12" optional="enabled">

Name

			<t t-esc="Function"/>

			<t t-esc="Email"/>

</template>
`

var tmplDef3 = `
<template inherit_id="my_other_id">

		<t t-esc="CompanyName"/>

	<xpath expr="//t[@t-esc='Email']" position="after">
		<t t-esc="Phone"/>
	</xpath>
</template>
`

var tmplDef4 = `
<template inherit_id="my_other_id">

			<t t-esc="Address"/>

<t t-esc="Name"/>

</template>
`

var tmplDef5 = `
<template inherit_id="my_other_id">
	<xpath expr="//t[@t-esc='Address']/.." position="attributes">
		<attribute name="name">address</attribute>
		<attribute name="string">Address</attribute>
	</xpath>
</template>
`

var tmplDef8 = `
<template inherit_id="my_other_id" id="new_base_view" priority="13" optional="disabled" page="True">
	<xpath expr="//t[@t-esc='Email']" position="after">
		<t t-raw="Fax"/>
	</xpath>
</template>
`

var tmplDef9 = `
<template inherit_id="new_base_view">
	<xpath expr="//t[@t-raw='Fax']" position="before">
		<t t-raw="Mobile"/>
	</xpath>
</template>
`

func loadView(xml string) {
	elt, err := xmlutils.XMLToElement(xml)
	if err != nil {
		panic(err)
	}
	LoadFromEtree(elt)
}

func TestViews(t *testing.T) {
	Convey("Setting two languages", t, func() {
		i18n.Langs = []string{"fr", "de"}
	})
	Convey("Creating Template 1", t, func() {
		loadView(tmplDef1)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 1)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		template := Registry.collection.GetByID("my_id")
		So(template.ID, ShouldEqual, "my_id")
		So(template.Page, ShouldEqual, true)
		So(template.Optional, ShouldEqual, false)
		So(template.OptionalDefault, ShouldEqual, false)
		So(template.Priority, ShouldEqual, 16)
		So(string(template.p2Content), ShouldEqual,
			`{% set _1 = _0 %}

		{% for line in lines %}

{{ line.UserName }}

Hello World

		{% endfor %}

`)
	})
	Convey("Creating Template 2", t, func() {
		loadView(tmplDef1)
		loadView(tmplDef2)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template := Registry.collection.GetByID("my_other_id")
		So(template.ID, ShouldEqual, "my_other_id")
		So(template.Page, ShouldEqual, false)
		So(template.Optional, ShouldEqual, true)
		So(template.OptionalDefault, ShouldEqual, true)
		So(template.Priority, ShouldEqual, 12)
		So(string(template.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

Name

			{{ Function }}

			{{ Email }}

`)
	})
	Convey("Inheriting Template 2", t, func() {
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template1 := Registry.collection.GetByID("my_id")
		So(string(template1.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

		{% for line in lines %}

{{ line.UserName }}

Hello World

		{% endfor %}

`)
		template2 := Registry.collection.GetByID("my_other_id")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

Name

			{{ Function }}
		{{ CompanyName }}

			{{ Email }}{{ Phone }}

`)
	})

	Convey("More inheritance on Template 2", t, func() {
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)

	})
	Convey("Modifying inherited modifications on Template 2", t, func() {
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		loadView(tmplDef5)
		BootStrap()
		So(len(Registry.collection.templates), ShouldEqual, 2)
		So(Registry.collection.GetByID("my_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)
	})
	Convey("Create new base template from inheritance", t, func() {
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		loadView(tmplDef5)
		loadView(tmplDef8)
		BootStrap()
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("new_base_view"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		newTemplate := Registry.collection.GetByID("new_base_view")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)
		So(template2.Priority, ShouldEqual, 12)
		So(template2.Page, ShouldBeFalse)
		So(template2.Optional, ShouldBeTrue)
		So(template2.OptionalDefault, ShouldBeTrue)
		So(newTemplate.Priority, ShouldEqual, 13)
		So(newTemplate.Page, ShouldBeTrue)
		So(newTemplate.Optional, ShouldBeTrue)
		So(newTemplate.OptionalDefault, ShouldBeFalse)
		So(string(newTemplate.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Fax|safe }}{{ Phone }}

`)
	})
	Convey("Inheriting new base template from inheritance", t, func() {
		Registry.collection = newCollection()
		loadView(tmplDef1)
		loadView(tmplDef2)
		loadView(tmplDef3)
		loadView(tmplDef4)
		loadView(tmplDef5)
		loadView(tmplDef8)
		loadView(tmplDef9)
		BootStrap()
		So(Registry.collection.GetByID("my_other_id"), ShouldNotBeNil)
		So(Registry.collection.GetByID("new_base_view"), ShouldNotBeNil)
		template2 := Registry.collection.GetByID("my_other_id")
		newTemplate := Registry.collection.GetByID("new_base_view")
		So(string(template2.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Phone }}

`)
		So(string(newTemplate.Content("")), ShouldEqual,
			`{% set _1 = _0 %}

{{ Name }}

			{{ Function }}
		{{ CompanyName }}

			{{ Address }}

			{{ Email }}{{ Mobile|safe }}{{ Fax|safe }}{{ Phone }}

`)
	})
	Convey("Testing gin loading", t, func() {
		inst := Registry.Instance("my_id", pongo2.Context{
			"lines": []map[string]string{
				{"UserName": "jsmith"},
				{"UserName": "wsmith"},
			}})
		w := httptest.NewRecorder()
		inst.Render(w)
		body, _ := ioutil.ReadAll(w.Result().Body)
		So(string(body), ShouldEqual, `

jsmith

Hello World

wsmith

Hello World

`)
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"time"

	"k8s.io/contrib/mungegithub/features"
	"k8s.io/contrib/mungegithub/github"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

var (
	validBranches = []string{"master"}
)

// PingCIMunger looks for situations CI (Travis | Shippable) has flaked for some
// reason and we want to re-run them.  Achieves this by closing and re-opening the pr
type PingCIMunger struct{}

func init() {
	RegisterMungerOrDie(PingCIMunger{})
}

// Name is the name usable in --pr-mungers
func (PingCIMunger) Name() string { return "ping-ci" }

// RequiredFeatures is a slice of 'features' that must be provided
func (PingCIMunger) RequiredFeatures() []string { return []string{} }

// Initialize will initialize the munger
func (PingCIMunger) Initialize(config *github.Config, features *features.Features) error { return nil }

// EachLoop is called at the start of every munge loop
func (PingCIMunger) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (PingCIMunger) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
func (PingCIMunger) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}

	// This munger only runs on certain branches, since travis/CI only listens
	// on certain branches
	validBranch := false
	for _, b := range validBranches {
		if obj.IsForBranch(b) {
			validBranch = true
			break
		}
	}
	if !validBranch {
		return
	}

	if !obj.HasLabel(lgtmLabel) {
		return
	}
	mergeable, err := obj.IsMergeable()
	if err != nil {
		glog.V(2).Infof("ping CI skipping %d - problem determining mergeability", *obj.Issue.Number)
		return
	}
	if !mergeable {
		glog.V(2).Infof("ping CI skipping %d - not mergeable", *obj.Issue.Number)
		return
	}
	if state := obj.GetStatusState([]string{travisContext}); state == "incomplete" {
		msg := "Travis continuous integration appears to have missed, closing and re-opening to trigger it"
		obj.WriteComment(msg)

		obj.ClosePR()
		time.Sleep(5 * time.Second)
		obj.OpenPR(10)
	}
}
<commit_msg>Delete ping_ci - We don't use it any more<commit_after><|endoftext|>