<commit_before>package main

import (
//	"net"
	"net/http"
	"log"
	"flag"
	"os"
	"path/filepath"
	"regexp"
	"fmt"
//	"reflect"
	"strconv"
	"time"
//	"strings"
	"html/template"
	"encoding/json"
	"runtime/pprof"
	"code.google.com/p/go.net/websocket"
	"io/ioutil"
	"os/signal"
	"syscall"
)

var (
	rootdir string
	port string
	address string
	verbose bool
	database string
)

var (
	d Query
	h Hub
)

var templates *template.Template
var homeTemplate *template.Template

func init() {
	flag.StringVar(&rootdir, "root", "current directory", "webroot directory")
	flag.StringVar(&rootdir, "d", "current directory", "webroot directory" + " (shorthand)")
	flag.StringVar(&port,"port", "8080", "listen port")
	flag.StringVar(&port,"p", "8080", "listen port" + " (shorthand)")
	flag.StringVar(&address, "address", "*", "listen address")
	flag.StringVar(&address, "l", "*", "listen address" + " (shorthand)")
	flag.BoolVar(&verbose, "verbose", false, "be verbose")
	flag.BoolVar(&verbose, "v", false, "be verbose" + " (shorthand)")
	flag.StringVar(&database, "database", "db.json", "database file")
}

func debug(s string) {
	if verbose {
		log.Print(s)
	}
}

func debugln(v ...interface{}) {
	if verbose {
		log.Println(v ...)
	}
}

var validLog = regexp.MustCompile("^/log/([a-zA-Z0-9-]+)/(-?[0-9]+[.]{0,1}[0-9]*)(/([ts])/([0-9]+))?/?$")
var validQuery = regexp.MustCompile("^/q/([a-zA-Z0-9-]+)/?$")
var validURLs = regexp.MustCompile("^/")

func logHandler(w http.ResponseWriter, r *http.Request) {
	m := validLog.FindStringSubmatch(r.URL.Path)
	if len(m) == 0 {
		http.Error(w, "Sensor not found", http.StatusNotFound)
		return
	}
	debug("Sensor " + m[1] + " sent value " + m[2])
	if m[4] != "" {
		t, _ := strconv.ParseInt(m[5], 10, 64)
		var tnew time.Time
		if m[4] == "t" {
			tnew = time.Unix(t, 0)
		} else { // m[4] == "s"
			tnew = time.Unix(time.Now().Unix() - t, 0)
		}
		_, told := d.Last(m[1])

		if tnew.After(told) {
			d.StoreT(m[1], m[2], tnew)
		} else {
			http.Error(w, "Sensor send out of order timestamp", http.StatusNotFound)
			h.Pipe <- HometickerJson{"Out of Order Reading", "fa-times-circle", "danger", "Sensor " + m[1] + " out of order value " + m[2] + ". Ignored."}
			return
		}
	} else {
		if ! d.Exists(m[1]) { // Remove when you add code to add/delete sensors instead of adding them automatically.
			h.Pipe <- HometickerJson{"New Sensor", "fa-check-circle", "success", "Sensor " + m[1] + " succesfully added."}
		}
		d.Store(m[1], m[2])
		h.Pipe <- HometickerJson{"New Reading", "fa-plus-circle", "info", "Sensor " + m[1] + " sent value " + m[2] + "."}
	}
	debugln("Sensor " + m[1] + " now contains:", d.Load(m[1]))
	fmt.Fprintf(w, "ok")
}

func queryHandler(w http.ResponseWriter, r *http.Request) {
	m := validQuery.FindStringSubmatch(r.URL.Path)
        if len(m) == 0 {
                http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	if ! d.Exists(m[1]) {
		http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	debugln("Query for sensor " + m[1])
	a, _ := json.Marshal(d.Load(m[1]))
	fmt.Fprintf(w, string(a))
}

func reloadHandler(w http.ResponseWriter, r *http.Request) {
	loadTemplates()
	fmt.Fprintf(w, "templates reloaded")
}

type HomePage struct {
	Title string
	SensorList template.HTML
	CustomScript template.HTML
}

const homeCustomScript = template.HTML(`<script src="/assets/cjs/hometicker.js"></script>`)

func homeHandler(w http.ResponseWriter, r *http.Request) {
	var sl string
	for _, s := range d.List() {
		sl += `

                   <a href="view/` + s + `">` + s + `

`
	}
	err := templates.ExecuteTemplate(w,
		"home.html",
		HomePage{"Datanomics alpha", template.HTML(sl),	homeCustomScript})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func makeHandler(fn func (http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validURLs.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r)
	}
}

func loadTemplates() {
	templates = template.Must(template.ParseFiles(rootdir + "/templates/header.html",
		rootdir + "/templates/menu.html",
		rootdir + "/templates/footer.html",
		rootdir + "/templates/home.html"))
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func main() {
	flag.Parse()
	var err error
	if rootdir == "current directory" {
		rootdir, err =  filepath.Abs(filepath.Dir(os.Args[0]))
	} else {
		rootdir, err = filepath.Abs(filepath.Dir(rootdir))
	}
	if err != nil {
		log.Fatal(err)
	}

	if address == "*" {
		address = ""
	}

	// CPU profiling
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	t := Database{ make(map[string] sensorlog) }
	file, err := ioutil.ReadFile(database)
	if err != nil {
		log.Println("Using new database.")
	} else if err = json.Unmarshal(file, &t); err != nil {
		log.Println("Database corrupt. Creating new.", err)
	} else {
		log.Println("Loaded database.")
	}

	d = &t
	loadTemplates()

	h.Connections = make(map[*Socket]bool)
	h.Pipe = make(chan HometickerJson, 1)
	go h.Broadcast()

	go cleanup()

	http.HandleFunc("/log/", logHandler)
	http.HandleFunc("/q/", queryHandler)
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir(rootdir + "/assets"))))
	http.HandleFunc("/reload/", reloadHandler)
	http.Handle("/_hometicker", websocket.Handler(homeTickerHandler))
	http.HandleFunc("/", makeHandler(homeHandler))

	log.Print("Starting webserver. Listening on " + address + ":" + port)
	log.Print("Webroot set to \"" + rootdir + "\".")
	err = http.ListenAndServe(address + ":" + port, nil)
	if err != nil {
		log.Fatal("Couldn't start server. ListenAndServe: ", err)
	}
}

func cleanup() {
        ch := make(chan os.Signal)
        signal.Notify(ch, syscall.SIGINT)
        <-ch
	log.Println("Writing database to disk.")
	dbs, _ := json.Marshal(d)
	err := ioutil.WriteFile(database, dbs, 0600)
	if err != nil {
		log.Println("Error saving database.")
	}
	log.Println("Exiting. Goodbye.")
	os.Exit(1)
}
<commit_msg>now when verbose, we won't print all the database as it gets rather large quickly<commit_after>package main

import (
//	"net"
	"net/http"
	"log"
	"flag"
	"os"
	"path/filepath"
	"regexp"
	"fmt"
//	"reflect"
	"strconv"
	"time"
//	"strings"
	"html/template"
	"encoding/json"
	"runtime/pprof"
	"code.google.com/p/go.net/websocket"
	"io/ioutil"
	"os/signal"
	"syscall"
)

var (
	rootdir string
	port string
	address string
	verbose bool
	database string
)

var (
	d Query
	h Hub
)

var templates *template.Template
var homeTemplate *template.Template

func init() {
	flag.StringVar(&rootdir, "root", "current directory", "webroot directory")
	flag.StringVar(&rootdir, "d", "current directory", "webroot directory" + " (shorthand)")
	flag.StringVar(&port,"port", "8080", "listen port")
	flag.StringVar(&port,"p", "8080", "listen port" + " (shorthand)")
	flag.StringVar(&address, "address", "*", "listen address")
	flag.StringVar(&address, "l", "*", "listen address" + " (shorthand)")
	flag.BoolVar(&verbose, "verbose", false, "be verbose")
	flag.BoolVar(&verbose, "v", false, "be verbose" + " (shorthand)")
	flag.StringVar(&database, "database", "db.json", "database file")
}

func debug(s string) {
	if verbose {
		log.Print(s)
	}
}

func debugln(v ...interface{}) {
	if verbose {
		log.Println(v ...)
	}
}

var validLog = regexp.MustCompile("^/log/([a-zA-Z0-9-]+)/(-?[0-9]+[.]{0,1}[0-9]*)(/([ts])/([0-9]+))?/?$")
var validQuery = regexp.MustCompile("^/q/([a-zA-Z0-9-]+)/?$")
var validURLs = regexp.MustCompile("^/")

func logHandler(w http.ResponseWriter, r *http.Request) {
	m := validLog.FindStringSubmatch(r.URL.Path)
	if len(m) == 0 {
		http.Error(w, "Sensor not found", http.StatusNotFound)
		return
	}
	debug("Sensor " + m[1] + " sent value " + m[2])
	if m[4] != "" {
		t, _ := strconv.ParseInt(m[5], 10, 64)
		var tnew time.Time
		if m[4] == "t" {
			tnew = time.Unix(t, 0)
		} else { // m[4] == "s"
			tnew = time.Unix(time.Now().Unix() - t, 0)
		}
		_, told := d.Last(m[1])

		if tnew.After(told) {
			d.StoreT(m[1], m[2], tnew)
		} else {
			http.Error(w, "Sensor send out of order timestamp", http.StatusNotFound)
			h.Pipe <- HometickerJson{"Out of Order Reading", "fa-times-circle", "danger", "Sensor " + m[1] + " out of order value " + m[2] + ". Ignored."}
			return
		}
	} else {
		if ! d.Exists(m[1]) { // Remove when you add code to add/delete sensors instead of adding them automatically.
			h.Pipe <- HometickerJson{"New Sensor", "fa-check-circle", "success", "Sensor " + m[1] + " succesfully added."}
		}
		d.Store(m[1], m[2])
		h.Pipe <- HometickerJson{"New Reading", "fa-plus-circle", "info", "Sensor " + m[1] + " sent value " + m[2] + "."}
	}
//	debugln("Sensor " + m[1] + " now contains:", d.Load(m[1]))
	fmt.Fprintf(w, "ok")
}

func queryHandler(w http.ResponseWriter, r *http.Request) {
	m := validQuery.FindStringSubmatch(r.URL.Path)
        if len(m) == 0 {
                http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	if ! d.Exists(m[1]) {
		http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	debugln("Query for sensor " + m[1])
	a, _ := json.Marshal(d.Load(m[1]))
	fmt.Fprintf(w, string(a))
}

func reloadHandler(w http.ResponseWriter, r *http.Request) {
	loadTemplates()
	fmt.Fprintf(w, "templates reloaded")
}

type HomePage struct {
	Title string
	SensorList template.HTML
	CustomScript template.HTML
}

const homeCustomScript = template.HTML(`<script src="/assets/cjs/hometicker.js"></script>`)

func homeHandler(w http.ResponseWriter, r *http.Request) {
	var sl string
	for _, s := range d.List() {
		sl += `

                   <a href="view/` + s + `">` + s + `

`
	}
	err := templates.ExecuteTemplate(w,
		"home.html",
		HomePage{"Datanomics alpha", template.HTML(sl),	homeCustomScript})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func makeHandler(fn func (http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validURLs.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r)
	}
}

func loadTemplates() {
	templates = template.Must(template.ParseFiles(rootdir + "/templates/header.html",
		rootdir + "/templates/menu.html",
		rootdir + "/templates/footer.html",
		rootdir + "/templates/home.html"))
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func main() {
	flag.Parse()
	var err error
	if rootdir == "current directory" {
		rootdir, err =  filepath.Abs(filepath.Dir(os.Args[0]))
	} else {
		rootdir, err = filepath.Abs(filepath.Dir(rootdir))
	}
	if err != nil {
		log.Fatal(err)
	}

	if address == "*" {
		address = ""
	}

	// CPU profiling
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	t := Database{ make(map[string] sensorlog) }
	file, err := ioutil.ReadFile(database)
	if err != nil {
		log.Println("Using new database.")
	} else if err = json.Unmarshal(file, &t); err != nil {
		log.Println("Database corrupt. Creating new.", err)
	} else {
		log.Println("Loaded database.")
	}

	d = &t
	loadTemplates()

	h.Connections = make(map[*Socket]bool)
	h.Pipe = make(chan HometickerJson, 1)
	go h.Broadcast()

	go cleanup()

	http.HandleFunc("/log/", logHandler)
	http.HandleFunc("/q/", queryHandler)
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir(rootdir + "/assets"))))
	http.HandleFunc("/reload/", reloadHandler)
	http.Handle("/_hometicker", websocket.Handler(homeTickerHandler))
	http.HandleFunc("/", makeHandler(homeHandler))

	log.Print("Starting webserver. Listening on " + address + ":" + port)
	log.Print("Webroot set to \"" + rootdir + "\".")
	err = http.ListenAndServe(address + ":" + port, nil)
	if err != nil {
		log.Fatal("Couldn't start server. ListenAndServe: ", err)
	}
}

func cleanup() {
        ch := make(chan os.Signal)
        signal.Notify(ch, syscall.SIGINT)
        <-ch
	log.Println("Writing database to disk.")
	dbs, _ := json.Marshal(d)
	err := ioutil.WriteFile(database, dbs, 0600)
	if err != nil {
		log.Println("Error saving database.")
	}
	log.Println("Exiting. Goodbye.")
	os.Exit(1)
}
<|endoftext|>


<commit_before>package irmaclient

import (
	"crypto/rand"
	"encoding/binary"

	"fmt"
	"math"
	"sync"
	"time"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi/revocation"
	irma "github.com/privacybydesign/irmago"
	"github.com/sirupsen/logrus"
)

func (client *Client) initRevocation() {
	// For every credential supporting revocation, compute nonrevocation caches in async jobs
	for id, attrsets := range client.attributes {
		for i, attrs := range attrsets {
			if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
				continue
			}
			id := id // make copy of same name to capture the value for closure below
			i := i   // see https://golang.org/doc/faq#closures_and_goroutines
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}

	// Of each credential supporting revocation, we periodically update its nonrevocation witness
	// by fetching updates from the issuer's server, such that:
	// - The time interval between two updates is random so that the server cannot recognize us
	//   using the update interval,
	// - Updating happens regularly even if the app is rarely used.
	// We do this by every 10 seconds updating the credential with a low probability, which
	// increases over time since the last update.
	client.Configuration.Scheduler.Every(irma.RevocationParameters.ClientUpdateInterval).Seconds().Do(func() {
		for id, attrsets := range client.attributes {
			for i, attrs := range attrsets {
				if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
					continue
				}
				cred, err := client.credential(id, i)
				if err != nil {
					client.reportError(err)
					continue
				}
				if cred.NonRevocationWitness == nil {
					continue
				}
				r, err := randomfloat()
				if err != nil {
					client.reportError(err)
					break
				}
				speed := attrs.CredentialType().RevocationUpdateSpeed * 60 * 60
				p := probability(cred.NonRevocationWitness.Updated, speed)
				if r < p {
					irma.Logger.WithFields(logrus.Fields{
						"random":      r,
						"prob":        p,
						"lastupdated": time.Now().Sub(cred.NonRevocationWitness.Updated).Seconds(),
						"credtype":    id,
						"hash":        attrs.Hash(),
					}).Debug("scheduling nonrevocation witness remote update")
					id := id // copy for closure below (https://golang.org/doc/faq#closures_and_goroutines)
					client.jobs <- func() {
						if err = client.NonrevUpdateFromServer(id); err != nil {
							client.reportError(err)
							return
						}
					}
				}
			}
		}
	})
}

// NonrevPrepare updates the revocation state for each credential in the request
// requiring a nonrevocation proof, using the updates included in the request, or the remote
// revocation server if those do not suffice.
func (client *Client) NonrevPrepare(request irma.SessionRequest) error {
	base := request.Base()
	var err error
	var wg sync.WaitGroup
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		id := id // copy for closure below, see https://golang.org/doc/faq#closures_and_goroutines
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		if !base.RequestsRevocation(id) {
			continue
		}
		irma.Logger.WithField("credtype", id).Debug("updating witnesses")
		wg.Add(1)
		go func() {
			if e := client.nonrevUpdate(id, base.Revocation[id].Updates); e != nil {
				err = e // overwrites err from previously finished call, if any
			}
			wg.Done()
		}()
	}
	wg.Wait()
	return err
}

// nonrevUpdate updates all contained instances of the specified type, using the specified
// updates if present and if they suffice, and contacting the issuer's server to download updates
// otherwise.
func (client *Client) nonrevUpdate(id irma.CredentialTypeIdentifier, updates map[uint]*revocation.Update) error {
	lowest := map[uint]uint64{}
	attrs := client.attrs(id)

	// Per credential and issuer key counter we may posess multiple credential instances.
	// Of the nonrevocation witnesses of these, take the lowest index.
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil {
			continue
		}
		pkid := cred.Pk.Counter
		l, present := lowest[pkid]
		if !present || cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index < l {
			lowest[pkid] = cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index
		}
	}

	// For each key counter, get an update message starting at the lowest index computed above,
	// that can update all of our credential instance of the given type and key counter,
	// using the specified update messags if they suffice, or the issuer's server otherwise.
	u := map[uint]*revocation.Update{}
	for counter, l := range lowest {
		update := updates[counter]
		if updates != nil && (update == nil || len(update.Events) == 0) {
			return errors.Errorf("missing revocation update for %s-%d", id, counter)
		}
		if update != nil && update.Events[0].Index <= l+1 {
			u[counter] = update
		} else {
			var err error
			u[counter], err = irma.RevocationClient{Conf: client.Configuration}.
				FetchUpdateFrom(id, counter, l+1)
			if err != nil {
				return err
			}
		}
	}

	// Apply the update messages to all instances of the given type and key counter
	for counter, update := range u {
		if err := client.nonrevApplyUpdates(id, counter, update); err != nil {
			return err
		}
	}
	return nil
}

func (client *Client) nonrevApplyUpdates(id irma.CredentialTypeIdentifier, counter uint, update *revocation.Update) error {
	client.credMutex.Lock()
	defer client.credMutex.Unlock()

	attrs := client.attrs(id)
	var save bool
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil || cred.Pk.Counter != counter {
			continue
		}
		updated, err := cred.nonrevApplyUpdates(update, irma.RevocationKeys{Conf: client.Configuration})
		if updated {
			save = true
			if err = client.storage.StoreSignature(cred); err != nil {
				return err
			}
		}
		if err == revocation.ErrorRevoked {
			id := cred.CredentialType().Identifier()
			hash := cred.attrs.Hash()
			irma.Logger.Warnf("credential %s %s revoked", id, hash)
			attrs[i].Revoked = true
			cred.attrs.Revoked = true
			save = true
			client.handler.Revoked(&irma.CredentialIdentifier{Type: id, Hash: hash})
			// Even if this credential is revoked during a session, we may have
			// other instances that can satisfy the request. So don't return an
			// error which would halt the session.
			continue
		}
		if err != nil {
			return err
		}
		// Asynchroniously update nonrevocation proof cache from updated witness
		irma.Logger.WithField("credtype", id).Debug("scheduling nonrevocation cache update")
		go func(cred *credential) {
			if err := cred.NonrevPrepareCache(); err != nil {
				client.reportError(err)
			}
		}(cred)
	}
	if save {
		if err := client.storage.StoreAttributes(id, client.attributes[id]); err != nil {
			client.reportError(err)
			return err
		}
	}
	return nil
}

func (client *Client) NonrevUpdateFromServer(id irma.CredentialTypeIdentifier) error {
	return client.nonrevUpdate(id, nil)
}

func (client *Client) nonrevPrepareCache(id irma.CredentialTypeIdentifier, index int) error {
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": id, "index": index})
	logger.Debug("preparing cache")
	defer logger.Debug("Preparing cache done")
	cred, err := client.credential(id, index)
	if err != nil {
		return err
	}
	return cred.NonrevPrepareCache()
}

// nonrevRepopulateCaches repopulates the consumed nonrevocation caches of the credentials involved
// in the request, in background jobs, after the request has finished.
func (client *Client) nonrevRepopulateCaches(request irma.SessionRequest) {
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if credtype == nil || !credtype.RevocationSupported() {
			continue
		}
		for i := range client.attrs(id) {
			id := id
			i := i
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}
}

// nonrevApplyUpdates updates the credential's nonrevocation witness using the specified messages,
// if they all verify and if their indices are ahead and adjacent to that of our witness.
func (cred *credential) nonrevApplyUpdates(update *revocation.Update, keys irma.RevocationKeys) (bool, error) {
	t := cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time

	pk, err := keys.PublicKey(cred.CredentialType().IssuerIdentifier(), update.SignedAccumulator.PKCounter)
	if err != nil {
		return false, err
	}
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": cred.CredentialType().Identifier(), "hash": cred.attrs.Hash()})
	logger.Debugf("updating witness")
	defer logger.Debug("updating witness done")
	if err = cred.NonRevocationWitness.Update(pk, update); err != nil {
		return false, err
	}

	return cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time != t, nil
}

// probability returns a float between 0 and asymptote, representing a probability
// that asymptotically increases to the asymptote, reaching
// a reference probability at a reference index.
func probability(lastUpdate time.Time, refindex uint64) float64 {
	const (
		asymptote      = 1.0 / 3          // max probability
		refprobability = 0.75 * asymptote // probability after one week
	)
	f := math.Tan(math.Pi * refprobability / (2 * asymptote))
	i := time.Now().Sub(lastUpdate).Seconds()
	return 2 * asymptote / math.Pi * math.Atan(i/float64(refindex)*f)
}

// randomfloat between 0 and 1
func randomfloat() (float64, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		fmt.Println("error:", err)
		return 0, err
	}
	c := float64(binary.BigEndian.Uint32(b)) / float64(^uint32(0)) // random int / max int
	return c, nil
}
<commit_msg>refactor: move closure copy of local variable to just before closure<commit_after>package irmaclient

import (
	"crypto/rand"
	"encoding/binary"

	"fmt"
	"math"
	"sync"
	"time"

	"github.com/go-errors/errors"
	"github.com/privacybydesign/gabi/revocation"
	irma "github.com/privacybydesign/irmago"
	"github.com/sirupsen/logrus"
)

func (client *Client) initRevocation() {
	// For every credential supporting revocation, compute nonrevocation caches in async jobs
	for id, attrsets := range client.attributes {
		for i, attrs := range attrsets {
			if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
				continue
			}
			id := id // make copy of same name to capture the value for closure below
			i := i   // see https://golang.org/doc/faq#closures_and_goroutines
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}

	// Of each credential supporting revocation, we periodically update its nonrevocation witness
	// by fetching updates from the issuer's server, such that:
	// - The time interval between two updates is random so that the server cannot recognize us
	//   using the update interval,
	// - Updating happens regularly even if the app is rarely used.
	// We do this by every 10 seconds updating the credential with a low probability, which
	// increases over time since the last update.
	client.Configuration.Scheduler.Every(irma.RevocationParameters.ClientUpdateInterval).Seconds().Do(func() {
		for id, attrsets := range client.attributes {
			for i, attrs := range attrsets {
				if attrs.CredentialType() == nil || !attrs.CredentialType().RevocationSupported() {
					continue
				}
				cred, err := client.credential(id, i)
				if err != nil {
					client.reportError(err)
					continue
				}
				if cred.NonRevocationWitness == nil {
					continue
				}
				r, err := randomfloat()
				if err != nil {
					client.reportError(err)
					break
				}
				speed := attrs.CredentialType().RevocationUpdateSpeed * 60 * 60
				p := probability(cred.NonRevocationWitness.Updated, speed)
				if r < p {
					irma.Logger.WithFields(logrus.Fields{
						"random":      r,
						"prob":        p,
						"lastupdated": time.Now().Sub(cred.NonRevocationWitness.Updated).Seconds(),
						"credtype":    id,
						"hash":        attrs.Hash(),
					}).Debug("scheduling nonrevocation witness remote update")
					id := id // copy for closure below (https://golang.org/doc/faq#closures_and_goroutines)
					client.jobs <- func() {
						if err = client.NonrevUpdateFromServer(id); err != nil {
							client.reportError(err)
							return
						}
					}
				}
			}
		}
	})
}

// NonrevPrepare updates the revocation state for each credential in the request
// requiring a nonrevocation proof, using the updates included in the request, or the remote
// revocation server if those do not suffice.
func (client *Client) NonrevPrepare(request irma.SessionRequest) error {
	base := request.Base()
	var err error
	var wg sync.WaitGroup
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if !credtype.RevocationSupported() {
			continue
		}
		if !base.RequestsRevocation(id) {
			continue
		}
		irma.Logger.WithField("credtype", id).Debug("updating witnesses")
		wg.Add(1)
		id := id // copy for closure below (https://golang.org/doc/faq#closures_and_goroutines)
		go func() {
			if e := client.nonrevUpdate(id, base.Revocation[id].Updates); e != nil {
				err = e // overwrites err from previously finished call, if any
			}
			wg.Done()
		}()
	}
	wg.Wait()
	return err
}

// nonrevUpdate updates all contained instances of the specified type, using the specified
// updates if present and if they suffice, and contacting the issuer's server to download updates
// otherwise.
func (client *Client) nonrevUpdate(id irma.CredentialTypeIdentifier, updates map[uint]*revocation.Update) error {
	lowest := map[uint]uint64{}
	attrs := client.attrs(id)

	// Per credential and issuer key counter we may posess multiple credential instances.
	// Of the nonrevocation witnesses of these, take the lowest index.
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil {
			continue
		}
		pkid := cred.Pk.Counter
		l, present := lowest[pkid]
		if !present || cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index < l {
			lowest[pkid] = cred.NonRevocationWitness.SignedAccumulator.Accumulator.Index
		}
	}

	// For each key counter, get an update message starting at the lowest index computed above,
	// that can update all of our credential instance of the given type and key counter,
	// using the specified update messags if they suffice, or the issuer's server otherwise.
	u := map[uint]*revocation.Update{}
	for counter, l := range lowest {
		update := updates[counter]
		if updates != nil && (update == nil || len(update.Events) == 0) {
			return errors.Errorf("missing revocation update for %s-%d", id, counter)
		}
		if update != nil && update.Events[0].Index <= l+1 {
			u[counter] = update
		} else {
			var err error
			u[counter], err = irma.RevocationClient{Conf: client.Configuration}.
				FetchUpdateFrom(id, counter, l+1)
			if err != nil {
				return err
			}
		}
	}

	// Apply the update messages to all instances of the given type and key counter
	for counter, update := range u {
		if err := client.nonrevApplyUpdates(id, counter, update); err != nil {
			return err
		}
	}
	return nil
}

func (client *Client) nonrevApplyUpdates(id irma.CredentialTypeIdentifier, counter uint, update *revocation.Update) error {
	client.credMutex.Lock()
	defer client.credMutex.Unlock()

	attrs := client.attrs(id)
	var save bool
	for i := 0; i < len(attrs); i++ {
		cred, err := client.credential(id, i)
		if err != nil {
			return err
		}
		if cred.NonRevocationWitness == nil || cred.Pk.Counter != counter {
			continue
		}
		updated, err := cred.nonrevApplyUpdates(update, irma.RevocationKeys{Conf: client.Configuration})
		if updated {
			save = true
			if err = client.storage.StoreSignature(cred); err != nil {
				return err
			}
		}
		if err == revocation.ErrorRevoked {
			id := cred.CredentialType().Identifier()
			hash := cred.attrs.Hash()
			irma.Logger.Warnf("credential %s %s revoked", id, hash)
			attrs[i].Revoked = true
			cred.attrs.Revoked = true
			save = true
			client.handler.Revoked(&irma.CredentialIdentifier{Type: id, Hash: hash})
			// Even if this credential is revoked during a session, we may have
			// other instances that can satisfy the request. So don't return an
			// error which would halt the session.
			continue
		}
		if err != nil {
			return err
		}
		// Asynchroniously update nonrevocation proof cache from updated witness
		irma.Logger.WithField("credtype", id).Debug("scheduling nonrevocation cache update")
		go func(cred *credential) {
			if err := cred.NonrevPrepareCache(); err != nil {
				client.reportError(err)
			}
		}(cred)
	}
	if save {
		if err := client.storage.StoreAttributes(id, client.attributes[id]); err != nil {
			client.reportError(err)
			return err
		}
	}
	return nil
}

func (client *Client) NonrevUpdateFromServer(id irma.CredentialTypeIdentifier) error {
	return client.nonrevUpdate(id, nil)
}

func (client *Client) nonrevPrepareCache(id irma.CredentialTypeIdentifier, index int) error {
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": id, "index": index})
	logger.Debug("preparing cache")
	defer logger.Debug("Preparing cache done")
	cred, err := client.credential(id, index)
	if err != nil {
		return err
	}
	return cred.NonrevPrepareCache()
}

// nonrevRepopulateCaches repopulates the consumed nonrevocation caches of the credentials involved
// in the request, in background jobs, after the request has finished.
func (client *Client) nonrevRepopulateCaches(request irma.SessionRequest) {
	for id := range request.Disclosure().Identifiers().CredentialTypes {
		credtype := client.Configuration.CredentialTypes[id]
		if credtype == nil || !credtype.RevocationSupported() {
			continue
		}
		for i := range client.attrs(id) {
			id := id
			i := i
			client.jobs <- func() {
				if err := client.nonrevPrepareCache(id, i); err != nil {
					client.reportError(err)
				}
			}
		}
	}
}

// nonrevApplyUpdates updates the credential's nonrevocation witness using the specified messages,
// if they all verify and if their indices are ahead and adjacent to that of our witness.
func (cred *credential) nonrevApplyUpdates(update *revocation.Update, keys irma.RevocationKeys) (bool, error) {
	t := cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time

	pk, err := keys.PublicKey(cred.CredentialType().IssuerIdentifier(), update.SignedAccumulator.PKCounter)
	if err != nil {
		return false, err
	}
	logger := irma.Logger.WithFields(logrus.Fields{"credtype": cred.CredentialType().Identifier(), "hash": cred.attrs.Hash()})
	logger.Debugf("updating witness")
	defer logger.Debug("updating witness done")
	if err = cred.NonRevocationWitness.Update(pk, update); err != nil {
		return false, err
	}

	return cred.NonRevocationWitness.SignedAccumulator.Accumulator.Time != t, nil
}

// probability returns a float between 0 and asymptote, representing a probability
// that asymptotically increases to the asymptote, reaching
// a reference probability at a reference index.
func probability(lastUpdate time.Time, refindex uint64) float64 {
	const (
		asymptote      = 1.0 / 3          // max probability
		refprobability = 0.75 * asymptote // probability after one week
	)
	f := math.Tan(math.Pi * refprobability / (2 * asymptote))
	i := time.Now().Sub(lastUpdate).Seconds()
	return 2 * asymptote / math.Pi * math.Atan(i/float64(refindex)*f)
}

// randomfloat between 0 and 1
func randomfloat() (float64, error) {
	b := make([]byte, 4)
	_, err := rand.Read(b)
	if err != nil {
		fmt.Println("error:", err)
		return 0, err
	}
	c := float64(binary.BigEndian.Uint32(b)) / float64(^uint32(0)) // random int / max int
	return c, nil
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

/*
 * Lowest-layer database tests
 */

func Test_DbFileCreation(t *testing.T) {
	dir, err := ioutil.TempDir("", "rqlite-test-")
	defer os.RemoveAll(dir)

	db, err := Open(path.Join(dir, "test_db"))
	if err != nil {
		t.Fatalf("failed to open new database: %s", err.Error())
	}
	if db == nil {
		t.Fatal("database is nil")
	}
	err = db.Close()
	if err != nil {
		t.Fatalf("failed to close database: %s", err.Error())
	}
}

func Test_TableCreation(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute("create table foo (id integer not null primary key, name text)")
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("expected 0 results, got %d", len(r))
	}
}

func Test_SimpleStatements(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("aoife")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("UPDATE foo SET Name='Who knows?' WHERE Id=1"); err != nil {
		t.Fatalf("failed to update record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("DELETE FROM foo WHERE Id=2"); err != nil {
		t.Fatalf("failed to delete record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}
}

func Test_FailingSimpleStatements(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err == nil {
		t.Fatal("inserted record into empty table OK")
	}
	if err.Error() != "no such table: foo" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err == nil {
		t.Fatal("duplicate table created OK")
	}
	if err.Error() != "table foo already exists" {
		t.Fatalf("unexpected error returned: %s", err.Error())
	}

	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("SELECT * FROM bar"); err == nil {
		t.Fatal("no error when querying non-existant table")
	}
	if err.Error() != "no such table: bar" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("utter nonsense"); err == nil {
		t.Fatal("no error when issuing nonsense query")
	}
	if err.Error() != `near "utter": syntax error` {
		t.Fatal("unexpected error returned")
	}
}

func Test_SimpleTransactions(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.CommitTransaction(); err != nil {
		t.Fatalf("failed to commit transaction: %s", err.Error())
	}

	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}

	r, err = db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func Test_TransactionsConstraintViolation(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)
	var err error

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err = db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}
	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func Test_TransactionsHardFail(t *testing.T) {
	db, path := mustOpenDatabase()
	defer db.Close()
	defer os.Remove(path)
	var err error

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(1, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}

	if err = db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(2, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Close(); err != nil {
		t.Fatalf("failed to close database: %s", err.Error())
	}

	db, err = Open(path)
	if err != nil {
		t.Fatalf("failed to re-open database: %s", err.Error())
	}
	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query after hard close: %s", err.Error())
	}
	if len(r) != 1 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func mustOpenDatabase() (*DB, string) {
	var err error
	f, err := ioutil.TempFile("", "rqlilte-test-")
	if err != nil {
		panic("failed to create temp file")
	}
	f.Close()

	db, err := Open(f.Name())
	if err != nil {
		panic("failed to open database")
	}

	return db, f.Name()
}

func asJson(v interface{}) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic("failed to JSON marshal value")
	}
	return string(b)
}
<commit_msg>Better name for test helper<commit_after>package db

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

/*
 * Lowest-layer database tests
 */

func Test_DbFileCreation(t *testing.T) {
	dir, err := ioutil.TempDir("", "rqlite-test-")
	defer os.RemoveAll(dir)

	db, err := Open(path.Join(dir, "test_db"))
	if err != nil {
		t.Fatalf("failed to open new database: %s", err.Error())
	}
	if db == nil {
		t.Fatal("database is nil")
	}
	err = db.Close()
	if err != nil {
		t.Fatalf("failed to close database: %s", err.Error())
	}
}

func Test_TableCreation(t *testing.T) {
	db, path := mustCreateDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute("create table foo (id integer not null primary key, name text)")
	if err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("expected 0 results, got %d", len(r))
	}
}

func Test_SimpleStatements(t *testing.T) {
	db, path := mustCreateDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query empty table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute(`INSERT INTO foo(name) VALUES("aoife")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"fiona"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("UPDATE foo SET Name='Who knows?' WHERE Id=1"); err != nil {
		t.Fatalf("failed to update record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"},{"id":"2","name":"aoife"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}

	if err := db.Execute("DELETE FROM foo WHERE Id=2"); err != nil {
		t.Fatalf("failed to delete record: %s", err.Error())
	}
	r, err = db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query table: %s", err.Error())
	}
	if exp, got := `[{"id":"1","name":"Who knows?"}]`, asJson(r); exp != got {
		t.Fatalf("unexpected results for query, expected %s, got %s", exp, got)
	}
}

func Test_FailingSimpleStatements(t *testing.T) {
	db, path := mustCreateDatabase()
	defer db.Close()
	defer os.Remove(path)

	err := db.Execute(`INSERT INTO foo(name) VALUES("fiona")`)
	if err == nil {
		t.Fatal("inserted record into empty table OK")
	}
	if err.Error() != "no such table: foo" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err == nil {
		t.Fatal("duplicate table created OK")
	}
	if err.Error() != "table foo already exists" {
		t.Fatalf("unexpected error returned: %s", err.Error())
	}

	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("SELECT * FROM bar"); err == nil {
		t.Fatal("no error when querying non-existant table")
	}
	if err.Error() != "no such table: bar" {
		t.Fatal("unexpected error returned")
	}

	if err = db.Execute("utter nonsense"); err == nil {
		t.Fatal("no error when issuing nonsense query")
	}
	if err.Error() != `near "utter": syntax error` {
		t.Fatal("unexpected error returned")
	}
}

func Test_SimpleTransactions(t *testing.T) {
	db, path := mustCreateDatabase()
	defer db.Close()
	defer os.Remove(path)

	if err := db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.CommitTransaction(); err != nil {
		t.Fatalf("failed to commit transaction: %s", err.Error())
	}

	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}

	if err := db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	for i := 0; i < 10; i++ {
		_ = db.Execute(`INSERT INTO foo(name) VALUES("philip")`)
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}

	r, err = db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 10 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func Test_TransactionsConstraintViolation(t *testing.T) {
	db, path := mustCreateDatabase()
	defer db.Close()
	defer os.Remove(path)
	var err error

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}

	if err = db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(11, "fiona")`); err == nil {
		t.Fatal("duplicated record inserted OK")
	}
	if err.Error() != "UNIQUE constraint failed: foo.id" {
		t.Fatal("unexpected error returned")
	}
	if err := db.RollbackTransaction(); err != nil {
		t.Fatalf("failed to rollback transaction: %s", err.Error())
	}
	r, err := db.Query("SELECT name FROM foo")
	if err != nil {
		t.Fatalf("failed to query after commited transaction: %s", err.Error())
	}
	if len(r) != 0 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func Test_TransactionsHardFail(t *testing.T) {
	db, path := mustCreateDatabase()
	defer db.Close()
	defer os.Remove(path)
	var err error

	if err = db.Execute("create table foo (id integer not null primary key, name text)"); err != nil {
		t.Fatalf("failed to create table: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(1, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}

	if err = db.StartTransaction(); err != nil {
		t.Fatalf("failed to start transaction: %s", err.Error())
	}
	if err = db.Execute(`INSERT INTO foo(id, name) VALUES(2, "fiona")`); err != nil {
		t.Fatalf("failed to insert record: %s", err.Error())
	}
	if err = db.Close(); err != nil {
		t.Fatalf("failed to close database: %s", err.Error())
	}

	db, err = Open(path)
	if err != nil {
		t.Fatalf("failed to re-open database: %s", err.Error())
	}
	r, err := db.Query("SELECT * FROM foo")
	if err != nil {
		t.Fatalf("failed to query after hard close: %s", err.Error())
	}
	if len(r) != 1 {
		t.Fatalf("incorrect number of results returned: %d", len(r))
	}
}

func mustCreateDatabase() (*DB, string) {
	var err error
	f, err := ioutil.TempFile("", "rqlilte-test-")
	if err != nil {
		panic("failed to create temp file")
	}
	f.Close()

	db, err := Open(f.Name())
	if err != nil {
		panic("failed to open database")
	}

	return db, f.Name()
}

func asJson(v interface{}) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic("failed to JSON marshal value")
	}
	return string(b)
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
)

var (
	conn = make(map[string]*Storage) // pool of connections
	mut  sync.RWMutex                // for pool thread safety
)

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

func open(addr, dbname string) (*Storage, error) {
	key := addr + dbname
	session, err := mgo.Dial(addr)
	if err != nil {
		return nil, err
	}
	storage := &Storage{session: session, dbname: dbname}
	mut.Lock()
	conn[key] = storage
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	var ok bool
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	key := addr + dbname
	mut.RLock()
	if storage, ok = conn[key]; ok {
		mut.RUnlock()
		storage.session.Ping()
		return storage, nil
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (storage *Storage, err error) {
	url, err := config.GetString("database:url")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	dbname, err := config.GetString("database:name")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	return Open(url, dbname)
}

// Close closes the connection.
//
// You can take advantage of defer statement, and write code that look like this:
//
//     st, err := Open("localhost:27017", "tsuru")
//     if err != nil {
//         panic(err)
//     }
//     defer st.Close()
//
// TODO(fss): remove this method and implement a "connection collector".
func (s *Storage) Close() {
	s.session.Close()
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *mgo.Collection {
	return s.session.DB(s.dbname).C(name)
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *mgo.Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *mgo.Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *mgo.Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *mgo.Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *mgo.Collection {
	return s.Collection("teams")
}
<commit_msg>db: also reconnect if ping fails<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
)

var (
	conn = make(map[string]*Storage) // pool of connections
	mut  sync.RWMutex                // for pool thread safety
)

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

func open(addr, dbname string) (*Storage, error) {
	key := addr + dbname
	session, err := mgo.Dial(addr)
	if err != nil {
		return nil, err
	}
	storage := &Storage{session: session, dbname: dbname}
	mut.Lock()
	conn[key] = storage
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	var ok bool
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	key := addr + dbname
	mut.RLock()
	if storage, ok = conn[key]; ok {
		mut.RUnlock()
		if err = storage.session.Ping(); err == nil {
			return storage, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (storage *Storage, err error) {
	url, err := config.GetString("database:url")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	dbname, err := config.GetString("database:name")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	return Open(url, dbname)
}

// Close closes the connection.
//
// You can take advantage of defer statement, and write code that look like this:
//
//     st, err := Open("localhost:27017", "tsuru")
//     if err != nil {
//         panic(err)
//     }
//     defer st.Close()
//
// TODO(fss): remove this method and implement a "connection collector".
func (s *Storage) Close() {
	s.session.Close()
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *mgo.Collection {
	return s.session.DB(s.dbname).C(name)
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *mgo.Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *mgo.Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *mgo.Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *mgo.Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *mgo.Collection {
	return s.Collection("teams")
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package stats

import (
	"encoding/json"
	"time"

	"github.com/dop251/goja"
	"github.com/loadimpact/k6/lib/types"
	"github.com/pkg/errors"
)

const jsEnvSrc = `
function p(pct) {
	return __sink__.P(pct/100.0);
};
`

var jsEnv *goja.Program

func init() {
	pgm, err := goja.Compile("__env__", jsEnvSrc, true)
	if err != nil {
		panic(err)
	}
	jsEnv = pgm
}

type Threshold struct {
	Source           string
	Failed           bool
	AbortOnFail      bool
	AbortGracePeriod types.NullDuration

	pgm *goja.Program
	rt  *goja.Runtime
}

func NewThreshold(src string, rt *goja.Runtime, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) {
	pgm, err := goja.Compile("__threshold__", src, true)
	if err != nil {
		return nil, err
	}

	return &Threshold{
		Source:           src,
		AbortOnFail:      abortOnFail,
		AbortGracePeriod: gracePeriod,
		pgm:              pgm,
		rt:               rt,
	}, nil
}

func (t Threshold) RunNoTaint() (bool, error) {
	v, err := t.rt.RunProgram(t.pgm)
	if err != nil {
		return false, err
	}
	return v.ToBoolean(), nil
}

func (t *Threshold) Run() (bool, error) {
	b, err := t.RunNoTaint()
	if !b {
		t.Failed = true
	}
	return b, err
}

type ThresholdConfig struct {
	Threshold        string             `json:"threshold"`
	AbortOnFail      bool               `json:"abortOnFail"`
	AbortGracePeriod types.NullDuration `json:"delayAbortEval"`
}

//used internally for JSON marshalling
type rawThresholdConfig ThresholdConfig

func (tc *ThresholdConfig) UnmarshalJSON(data []byte) error {
	//shortcircuit unmarshalling for simple string format
	if err := json.Unmarshal(data, &tc.Threshold); err == nil {
		return nil
	}

	rawConfig := (*rawThresholdConfig)(tc)
	return json.Unmarshal(data, rawConfig)
}

func (tc ThresholdConfig) MarshalJSON() ([]byte, error) {
	if tc.AbortOnFail {
		return json.Marshal(rawThresholdConfig(tc))
	}
	return json.Marshal(tc.Threshold)
}

type Thresholds struct {
	Runtime    *goja.Runtime
	Thresholds []*Threshold
	Abort      bool
}

func NewThresholds(sources []string) (Thresholds, error) {
	tcs := make([]ThresholdConfig, len(sources))
	for i, source := range sources {
		tcs[i].Threshold = source
	}

	return NewThresholdsWithConfig(tcs)
}

func NewThresholdsWithConfig(configs []ThresholdConfig) (Thresholds, error) {
	rt := goja.New()
	if _, err := rt.RunProgram(jsEnv); err != nil {
		return Thresholds{}, errors.Wrap(err, "builtin")
	}

	ts := make([]*Threshold, len(configs))
	for i, config := range configs {
		t, err := NewThreshold(config.Threshold, rt, config.AbortOnFail, config.AbortGracePeriod)
		if err != nil {
			return Thresholds{}, errors.Wrapf(err, "%d", i)
		}
		ts[i] = t
	}

	return Thresholds{rt, ts, false}, nil
}

func (ts *Thresholds) UpdateVM(sink Sink, t time.Duration) error {
	ts.Runtime.Set("__sink__", sink)
	f := sink.Format(t)
	for k, v := range f {
		ts.Runtime.Set(k, v)
	}
	return nil
}

func (ts *Thresholds) RunAll(t time.Duration) (bool, error) {
	succ := true
	for i, th := range ts.Thresholds {
		b, err := th.Run()
		if err != nil {
			return false, errors.Wrapf(err, "%d", i)
		}
		if !b {
			succ = false

			if ts.Abort || !th.AbortOnFail {
				continue
			}

			ts.Abort = !th.AbortGracePeriod.Valid ||
				th.AbortGracePeriod.Duration < types.Duration(t)
		}
	}
	return succ, nil
}

func (ts *Thresholds) Run(sink Sink, t time.Duration) (bool, error) {
	if err := ts.UpdateVM(sink, t); err != nil {
		return false, err
	}
	return ts.RunAll(t)
}

func (ts *Thresholds) UnmarshalJSON(data []byte) error {
	var configs []ThresholdConfig
	if err := json.Unmarshal(data, &configs); err != nil {
		return err
	}
	newts, err := NewThresholdsWithConfig(configs)
	if err != nil {
		return err
	}
	*ts = newts
	return nil
}

func (ts Thresholds) MarshalJSON() ([]byte, error) {
	configs := make([]ThresholdConfig, len(ts.Thresholds))
	for i, t := range ts.Thresholds {
		configs[i].Threshold = t.Source
		configs[i].AbortOnFail = t.AbortOnFail
		configs[i].AbortGracePeriod = t.AbortGracePeriod
	}
	return json.Marshal(configs)
}
<commit_msg>Report threshold failure correctly to output cloud fix #849<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package stats

import (
	"encoding/json"
	"time"

	"github.com/dop251/goja"
	"github.com/loadimpact/k6/lib/types"
	"github.com/pkg/errors"
)

const jsEnvSrc = `
function p(pct) {
	return __sink__.P(pct/100.0);
};
`

var jsEnv *goja.Program

func init() {
	pgm, err := goja.Compile("__env__", jsEnvSrc, true)
	if err != nil {
		panic(err)
	}
	jsEnv = pgm
}

type Threshold struct {
	Source           string
	Failed           bool
	AbortOnFail      bool
	AbortGracePeriod types.NullDuration

	pgm *goja.Program
	rt  *goja.Runtime
}

func NewThreshold(src string, rt *goja.Runtime, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) {
	pgm, err := goja.Compile("__threshold__", src, true)
	if err != nil {
		return nil, err
	}

	return &Threshold{
		Source:           src,
		AbortOnFail:      abortOnFail,
		AbortGracePeriod: gracePeriod,
		pgm:              pgm,
		rt:               rt,
	}, nil
}

func (t Threshold) RunNoTaint() (bool, error) {
	v, err := t.rt.RunProgram(t.pgm)
	if err != nil {
		return false, err
	}
	return v.ToBoolean(), nil
}

func (t *Threshold) Run() (bool, error) {
	b, err := t.RunNoTaint()
	t.Failed = !b
	return b, err
}

type ThresholdConfig struct {
	Threshold        string             `json:"threshold"`
	AbortOnFail      bool               `json:"abortOnFail"`
	AbortGracePeriod types.NullDuration `json:"delayAbortEval"`
}

//used internally for JSON marshalling
type rawThresholdConfig ThresholdConfig

func (tc *ThresholdConfig) UnmarshalJSON(data []byte) error {
	//shortcircuit unmarshalling for simple string format
	if err := json.Unmarshal(data, &tc.Threshold); err == nil {
		return nil
	}

	rawConfig := (*rawThresholdConfig)(tc)
	return json.Unmarshal(data, rawConfig)
}

func (tc ThresholdConfig) MarshalJSON() ([]byte, error) {
	if tc.AbortOnFail {
		return json.Marshal(rawThresholdConfig(tc))
	}
	return json.Marshal(tc.Threshold)
}

type Thresholds struct {
	Runtime    *goja.Runtime
	Thresholds []*Threshold
	Abort      bool
}

func NewThresholds(sources []string) (Thresholds, error) {
	tcs := make([]ThresholdConfig, len(sources))
	for i, source := range sources {
		tcs[i].Threshold = source
	}

	return NewThresholdsWithConfig(tcs)
}

func NewThresholdsWithConfig(configs []ThresholdConfig) (Thresholds, error) {
	rt := goja.New()
	if _, err := rt.RunProgram(jsEnv); err != nil {
		return Thresholds{}, errors.Wrap(err, "builtin")
	}

	ts := make([]*Threshold, len(configs))
	for i, config := range configs {
		t, err := NewThreshold(config.Threshold, rt, config.AbortOnFail, config.AbortGracePeriod)
		if err != nil {
			return Thresholds{}, errors.Wrapf(err, "%d", i)
		}
		ts[i] = t
	}

	return Thresholds{rt, ts, false}, nil
}

func (ts *Thresholds) UpdateVM(sink Sink, t time.Duration) error {
	ts.Runtime.Set("__sink__", sink)
	f := sink.Format(t)
	for k, v := range f {
		ts.Runtime.Set(k, v)
	}
	return nil
}

func (ts *Thresholds) RunAll(t time.Duration) (bool, error) {
	succ := true
	for i, th := range ts.Thresholds {
		b, err := th.Run()
		if err != nil {
			return false, errors.Wrapf(err, "%d", i)
		}
		if !b {
			succ = false

			if ts.Abort || !th.AbortOnFail {
				continue
			}

			ts.Abort = !th.AbortGracePeriod.Valid ||
				th.AbortGracePeriod.Duration < types.Duration(t)
		}
	}
	return succ, nil
}

func (ts *Thresholds) Run(sink Sink, t time.Duration) (bool, error) {
	if err := ts.UpdateVM(sink, t); err != nil {
		return false, err
	}
	return ts.RunAll(t)
}

func (ts *Thresholds) UnmarshalJSON(data []byte) error {
	var configs []ThresholdConfig
	if err := json.Unmarshal(data, &configs); err != nil {
		return err
	}
	newts, err := NewThresholdsWithConfig(configs)
	if err != nil {
		return err
	}
	*ts = newts
	return nil
}

func (ts Thresholds) MarshalJSON() ([]byte, error) {
	configs := make([]ThresholdConfig, len(ts.Thresholds))
	for i, t := range ts.Thresholds {
		configs[i].Threshold = t.Source
		configs[i].AbortOnFail = t.AbortOnFail
		configs[i].AbortGracePeriod = t.AbortGracePeriod
	}
	return json.Marshal(configs)
}
<|endoftext|>


<commit_before>package gogrs

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type memData map[int64][][]string

// DailyData start with stock no, date.
type DailyData struct {
	No      string
	Date    time.Time
	RawData [][]string
	hasData memData
}

// URL return stock csv url path.
func (d DailyData) URL() string {
	path := fmt.Sprintf(TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, RandInt())
	return fmt.Sprintf("%s%s", TWSEHOST, path)
}

// Round will do sub one month.
func (d *DailyData) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, time.UTC)
}

// GetData return csv data in array.
func (d *DailyData) GetData() ([][]string, error) {
	if d.hasData == nil {
		d.hasData = make(memData)
	}
	if len(d.hasData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			fmt.Println("[err] >>> ", err)
			return nil, err
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		if len(csvArrayContent) > 2 {
			csvReader := csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.hasData[d.Date.Unix()] = allData
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.hasData[d.Date.Unix()], nil
}

func (d DailyData) GetDataMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.GetData()
	reg := regexp.MustCompile(`[\d]{2,}`)
	for _, v := range dailyData {
		p := reg.FindAllString(v[0], -1)
		year, _ := strconv.Atoi(p[0])
		mon, _ := strconv.Atoi(p[1])
		day, _ := strconv.Atoi(p[2])
		data[time.Date(year+1911, time.Month(mon), day, 0, 0, 0, 0, time.Local)] = v
	}
	return data
}

type FmtDailyData struct {
	Date       time.Time
	Volume     uint64
	TotalPrice uint64
	Open       float64
	High       float64
	Low        float64
	Price      float64
	Range      float64
	Totalsale  uint64
}

var dateReg = regexp.MustCompile(`[\d]{2,}`)

func parseDate(strDate string) time.Time {
	p := dateReg.FindAllString(strDate, -1)
	year, _ := strconv.Atoi(p[0])
	mon, _ := strconv.Atoi(p[1])
	day, _ := strconv.Atoi(p[2])
	return time.Date(year+1911, time.Month(mon), day, 0, 0, 0, 0, time.Local)
}

func (d DailyData) FormatDailyData() {
	for _, v := range d.RawData {
		var d FmtDailyData
		volume, _ := strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		d.Volume = volume

		totalprice, _ := strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		d.TotalPrice = totalprice

		open, _ := strconv.ParseFloat(v[3], 64)
		d.Open = open

		high, _ := strconv.ParseFloat(v[4], 64)
		d.High = high

		low, _ := strconv.ParseFloat(v[5], 64)
		d.Low = low

		price, _ := strconv.ParseFloat(v[6], 64)
		d.Price = price

		range_, _ := strconv.ParseFloat(v[7], 64)
		d.Range = range_

		totalsale, _ := strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		d.Totalsale = totalsale

		fmt.Println(d)
	}
}
<commit_msg>Using `parseDate`.<commit_after>package gogrs

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type memData map[int64][][]string

// DailyData start with stock no, date.
type DailyData struct {
	No      string
	Date    time.Time
	RawData [][]string
	hasData memData
}

// URL return stock csv url path.
func (d DailyData) URL() string {
	path := fmt.Sprintf(TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, RandInt())
	return fmt.Sprintf("%s%s", TWSEHOST, path)
}

// Round will do sub one month.
func (d *DailyData) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, time.UTC)
}

// GetData return csv data in array.
func (d *DailyData) GetData() ([][]string, error) {
	if d.hasData == nil {
		d.hasData = make(memData)
	}
	if len(d.hasData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			fmt.Println("[err] >>> ", err)
			return nil, err
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		if len(csvArrayContent) > 2 {
			csvReader := csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.hasData[d.Date.Unix()] = allData
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.hasData[d.Date.Unix()], nil
}

func (d DailyData) GetDataMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.GetData()
	for _, v := range dailyData {
		data[parseDate(v[0])] = v
	}
	return data
}

type FmtDailyData struct {
	Date       time.Time
	Volume     uint64
	TotalPrice uint64
	Open       float64
	High       float64
	Low        float64
	Price      float64
	Range      float64
	Totalsale  uint64
}

var dateReg = regexp.MustCompile(`[\d]{2,}`)

func parseDate(strDate string) time.Time {
	p := dateReg.FindAllString(strDate, -1)
	year, _ := strconv.Atoi(p[0])
	mon, _ := strconv.Atoi(p[1])
	day, _ := strconv.Atoi(p[2])
	return time.Date(year+1911, time.Month(mon), day, 0, 0, 0, 0, time.Local)
}

func (d DailyData) FormatDailyData() {
	for _, v := range d.RawData {
		var d FmtDailyData
		d.Date = parseDate(v[0])

		volume, _ := strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		d.Volume = volume

		totalprice, _ := strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		d.TotalPrice = totalprice

		open, _ := strconv.ParseFloat(v[3], 64)
		d.Open = open

		high, _ := strconv.ParseFloat(v[4], 64)
		d.High = high

		low, _ := strconv.ParseFloat(v[5], 64)
		d.Low = low

		price, _ := strconv.ParseFloat(v[6], 64)
		d.Price = price

		range_, _ := strconv.ParseFloat(v[7], 64)
		d.Range = range_

		totalsale, _ := strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		d.Totalsale = totalsale

		fmt.Println(d)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Linux Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specs

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 1
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 0
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 0

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-rc1-dev"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<commit_msg>version: bump to v1.0.0-rc2<commit_after>// Copyright 2016 The Linux Foundation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specs

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 1
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 0
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 0

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-rc2"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<|endoftext|>


<commit_before>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package sqlbuilder

import (
	"context"
	"database/sql"
	"strconv"
)

// CreateTableStmt 创建表的语句
type CreateTableStmt struct {
	engine  Engine
	dialect Dialect

	name    string
	columns []*Column
	indexes []*indexColumn

	// 约束
	constraints []*constraintColumn
	foreignKeys []*foreignKey // 外键约束
	ai          *constraintColumn

	// 一些附加的信息
	//
	// 比如可以指定创建表时的编码等，各个数据库各不相同。
	options map[string][]string
}

// Column 列结构
type Column struct {
	Name       string // 数据库的字段名
	Type       string // 类型，包含长度，可能是 BIGIINT，或是 VARCHAR(1024) 等格式
	Nullable   bool   // 是否可以为 NULL
	Default    string // 默认值
	HasDefault bool
}

type foreignKey struct {
	Name                     string // 约束名
	Column                   string // 列名
	RefTableName, RefColName string
	UpdateRule, DeleteRule   string
}

type indexColumn struct {
	Name    string
	Type    Index
	Columns []string
}

type constraintColumn struct {
	Name    string
	Type    Constraint
	Columns []string
}

// CreateTable 创建表的语句
//
// 执行创建表操作，可能包含了创建索引等多个语句，
// 如果 e 是一个事务类型，且 d 是支持事务 DDL 的，
// 那么在执行时，会当作一个事务处理，否则为多个语句依次执行。
func CreateTable(e Engine, d Dialect) *CreateTableStmt {
	return &CreateTableStmt{
		engine:  e,
		dialect: d,
	}
}

// Reset 重置内容
func (stmt *CreateTableStmt) Reset() {
	stmt.name = ""
	stmt.columns = stmt.columns[:0]
	stmt.indexes = stmt.indexes[:0]
	stmt.options = map[string][]string{}
	stmt.foreignKeys = stmt.foreignKeys[:0]
}

// Table 指定表名
func (stmt *CreateTableStmt) Table(t string) *CreateTableStmt {
	stmt.name = t
	return stmt
}

// Column 添加列
//
// hasDefault 是否需要设置默认值；
// def 如果 hasDefault 为 true，则 def 为其默认值，否则 def 不启作用；
func (stmt *CreateTableStmt) Column(name, typ string, nullable, hasDefault bool, def string, length ...int) *CreateTableStmt {
	col := &Column{
		Name:       name,
		Nullable:   nullable,
		HasDefault: hasDefault,
	}
	if hasDefault {
		col.Default = def
	}

	switch len(length) {
	case 1:
		col.Type = typ + "(" + strconv.Itoa(length[0]) + ")"
	case 2:
		col.Type = typ + "(" + strconv.Itoa(length[0]) + "," + strconv.Itoa(length[1]) + ")"
	default:
		col.Type = typ
	}

	stmt.columns = append(stmt.columns, col)

	return stmt
}

// AutoIncrement 指定自增列
// name 自增约束的名称；
// col 自增对应的列；
// pk 是否同时设置为主键。
func (stmt *CreateTableStmt) AutoIncrement(name, col string, pk bool) *CreateTableStmt {
	stmt.ai = &constraintColumn{
		Name:    name,
		Type:    ConstraintAI,
		Columns: []string{col},
	}

	if pk {
		return stmt.PK(name, col)
	}

	return stmt
}

// PK 指定主键约束
func (stmt *CreateTableStmt) PK(name string, col ...string) *CreateTableStmt {
	stmt.constraints = append(stmt.constraints, &constraintColumn{
		Name:    name,
		Type:    ConstraintPK,
		Columns: col,
	})
	return stmt
}

// Index 添加索引
func (stmt *CreateTableStmt) Index(name string, typ Index, col ...string) *CreateTableStmt {
	stmt.indexes = append(stmt.indexes, &indexColumn{
		Name:    name,
		Type:    typ,
		Columns: col,
	})

	return stmt
}

// Unique 添加唯一约束
func (stmt *CreateTableStmt) Unique(name string, col ...string) *CreateTableStmt {
	stmt.constraints = append(stmt.constraints, &constraintColumn{
		Name:    name,
		Type:    ConstraintUnique,
		Columns: col,
	})

	return stmt
}

// Check check 约束
func (stmt *CreateTableStmt) Check(name string, expr string) *CreateTableStmt {
	stmt.constraints = append(stmt.constraints, &constraintColumn{
		Name:    name,
		Type:    ConstraintCheck,
		Columns: []string{expr},
	})

	return stmt
}

// ForeignKey 指定外键
func (stmt *CreateTableStmt) ForeignKey(name, col, refTable, refCol, updateRule, deleteRule string) *CreateTableStmt {
	stmt.foreignKeys = append(stmt.foreignKeys, &foreignKey{
		Name:         name,
		Column:       col,
		RefTableName: refTable,
		RefColName:   refCol,
		UpdateRule:   updateRule,
		DeleteRule:   deleteRule,
	})

	return stmt
}

// SQL 获取 SQL 的语句及参数部分
func (stmt *CreateTableStmt) SQL() ([]string, error) {
	w := New("CREATE TABLE IF NOT EXISTS ").
		WriteString(stmt.name).
		WriteByte('(')

	// 普通列
	for _, col := range stmt.columns {
		err := stmt.dialect.CreateColumnSQL(w, col, col.Name == stmt.ai.Columns[0])
		if err != nil {
			return nil, err
		}
		w.WriteByte(',')
	}

	if err := stmt.createConstraints(w); err != nil {
		return nil, err
	}

	w.TruncateLast(1).WriteByte(')')

	if err := stmt.dialect.CreateTableOptionsSQL(w, stmt.options); err != nil {
		return nil, err
	}

	sqls := []string{w.String()}

	indexes, err := createIndexSQL(stmt)
	if err != nil {
		return nil, err
	}
	if len(indexes) > 0 {
		sqls = append(sqls, indexes...)
	}

	return sqls, nil
}

// Exec 执行 SQL 语句
func (stmt *CreateTableStmt) Exec() (sql.Result, error) {
	return stmt.ExecContext(context.Background())
}

// ExecContext 执行 SQL 语句
func (stmt *CreateTableStmt) ExecContext(ctx context.Context) (sql.Result, error) {
	sqls, err := stmt.SQL()
	if err != nil {
		return nil, err
	}

	for _, sql := range sqls {
		if _, err := stmt.engine.ExecContext(ctx, sql, nil); err != nil {
			return nil, err
		}
	}

	return nil, nil
}

// 创建标准的几种约束
func (stmt *CreateTableStmt) createConstraints(buf *SQLBuilder) error {
	for _, c := range stmt.constraints {
		switch c.Type {
		case ConstraintCheck:
			createCheckSQL(buf, c.Name, c.Columns[0])
		case ConstraintPK:
			createPKSQL(buf, c.Name, c.Columns...)
		case ConstraintUnique:
			createUniqueSQL(buf, c.Name, c.Columns...)
		default:
			return ErrUnknownConstraint
		}
		buf.WriteByte(',')
	}

	// foreign  key
	for _, fk := range stmt.foreignKeys {
		createFKSQL(buf, fk)
		buf.WriteByte(',')
	}

	return nil
}

func createIndexSQL(model *CreateTableStmt) ([]string, error) {
	if len(model.indexes) == 0 {
		return nil, nil
	}

	sqls := make([]string, 0, len(model.indexes))
	buf := CreateIndex(nil)
	for _, index := range model.indexes {
		buf.Reset()
		buf.Table("{#" + model.name + "}").
			Name(index.Name)
		for _, col := range index.Columns {
			buf.Columns("{" + col + "}")
		}

		sql, _, err := buf.SQL()
		if err != nil {
			return nil, err
		}
		sqls = append(sqls, sql)
	}

	return sqls, nil
}

// create table 语句中 pk 约束的语句
func createPKSQL(buf *SQLBuilder, pkName string, cols ...string) {
	// CONSTRAINT pk_name PRIMARY KEY (id,lastName)
	buf.WriteString(" CONSTRAINT ").
		WriteString(pkName).
		WriteString(" PRIMARY KEY(")

	for _, col := range cols {
		buf.WriteByte('{').WriteString(col).WriteByte('}')
		buf.WriteByte(',')
	}
	buf.TruncateLast(1) // 去掉最后一个逗号
	buf.WriteByte(')')
}

// create table 语句中的 unique 约束部分的语句。
func createUniqueSQL(buf *SQLBuilder, indexName string, cols ...string) {
	// CONSTRAINT unique_name UNIQUE (id,lastName)
	buf.WriteString(" CONSTRAINT ").
		WriteString(indexName).
		WriteString(" UNIQUE(")
	for _, col := range cols {
		buf.WriteByte('{').
			WriteString(col).
			WriteByte('}').
			WriteByte(',')
	}
	buf.TruncateLast(1) // 去掉最后一个逗号

	buf.WriteByte(')')
}

// create table 语句中 fk 的约束部分的语句
func createFKSQL(buf *SQLBuilder, fk *foreignKey) {
	// CONSTRAINT fk_name FOREIGN KEY (id) REFERENCES user(id)
	buf.WriteString(" CONSTRAINT ").WriteString(fk.Name)

	buf.WriteString(" FOREIGN KEY(")
	buf.WriteByte('{').WriteString(fk.Column).WriteByte('}')

	buf.WriteString(") REFERENCES ").
		WriteByte('{').
		WriteString(fk.RefTableName).
		WriteByte('}')

	buf.WriteByte('(')
	buf.WriteByte('{').WriteString(fk.RefColName).WriteByte('}')
	buf.WriteByte(')')

	if len(fk.UpdateRule) > 0 {
		buf.WriteString(" ON UPDATE ").WriteString(fk.UpdateRule)
	}

	if len(fk.DeleteRule) > 0 {
		buf.WriteString(" ON DELETE ").WriteString(fk.DeleteRule)
	}
}

// create table 语句中 check 约束部分的语句
func createCheckSQL(buf *SQLBuilder, name, expr string) {
	// CONSTRAINT chk_name CHECK (id>0 AND username='admin')
	buf.WriteString(" CONSTRAINT ").
		WriteString(name).
		WriteString(" CHECK(").
		WriteString(expr).
		WriteByte(')')
}
<commit_msg>[sqlbuilder] 限制主键数量为 1 个<commit_after>// Copyright 2019 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package sqlbuilder

import (
	"context"
	"database/sql"
	"strconv"
)

// CreateTableStmt 创建表的语句
type CreateTableStmt struct {
	engine  Engine
	dialect Dialect

	name    string
	columns []*Column
	indexes []*indexColumn

	// 约束
	constraints []*constraintColumn
	foreignKeys []*foreignKey // 外键约束
	ai, pk      *constraintColumn

	// 一些附加的信息
	//
	// 比如可以指定创建表时的编码等，各个数据库各不相同。
	options map[string][]string
}

// Column 列结构
type Column struct {
	Name       string // 数据库的字段名
	Type       string // 类型，包含长度，可能是 BIGIINT，或是 VARCHAR(1024) 等格式
	Nullable   bool   // 是否可以为 NULL
	Default    string // 默认值
	HasDefault bool
}

type foreignKey struct {
	Name                     string // 约束名
	Column                   string // 列名
	RefTableName, RefColName string
	UpdateRule, DeleteRule   string
}

type indexColumn struct {
	Name    string
	Type    Index
	Columns []string
}

type constraintColumn struct {
	Name    string
	Type    Constraint
	Columns []string
}

// CreateTable 创建表的语句
//
// 执行创建表操作，可能包含了创建索引等多个语句，
// 如果 e 是一个事务类型，且 d 是支持事务 DDL 的，
// 那么在执行时，会当作一个事务处理，否则为多个语句依次执行。
func CreateTable(e Engine, d Dialect) *CreateTableStmt {
	return &CreateTableStmt{
		engine:  e,
		dialect: d,
	}
}

// Reset 重置内容
func (stmt *CreateTableStmt) Reset() {
	stmt.name = ""
	stmt.columns = stmt.columns[:0]
	stmt.indexes = stmt.indexes[:0]
	stmt.options = map[string][]string{}
	stmt.foreignKeys = stmt.foreignKeys[:0]
	stmt.constraints = stmt.constraints[:0]
	stmt.pk = nil
	stmt.ai = nil
}

// Table 指定表名
func (stmt *CreateTableStmt) Table(t string) *CreateTableStmt {
	stmt.name = t
	return stmt
}

// Column 添加列
//
// hasDefault 是否需要设置默认值；
// def 如果 hasDefault 为 true，则 def 为其默认值，否则 def 不启作用；
func (stmt *CreateTableStmt) Column(name, typ string, nullable, hasDefault bool, def string, length ...int) *CreateTableStmt {
	col := &Column{
		Name:       name,
		Nullable:   nullable,
		HasDefault: hasDefault,
	}
	if hasDefault {
		col.Default = def
	}

	switch len(length) {
	case 1:
		col.Type = typ + "(" + strconv.Itoa(length[0]) + ")"
	case 2:
		col.Type = typ + "(" + strconv.Itoa(length[0]) + "," + strconv.Itoa(length[1]) + ")"
	default:
		col.Type = typ
	}

	stmt.columns = append(stmt.columns, col)

	return stmt
}

// AutoIncrement 指定自增列
// name 自增约束的名称；
// col 自增对应的列；
// pk 是否同时设置为主键。
func (stmt *CreateTableStmt) AutoIncrement(name, col string, pk bool) *CreateTableStmt {
	stmt.ai = &constraintColumn{
		Name:    name,
		Type:    ConstraintAI,
		Columns: []string{col},
	}

	if pk {
		return stmt.PK(name, col)
	}

	return stmt
}

// PK 指定主键约束
//
// 如果多次指定主键信息，则会 panic
func (stmt *CreateTableStmt) PK(name string, col ...string) *CreateTableStmt {
	if stmt.pk != nil {
		panic("主键信息已经存在")
	}

	stmt.pk = &constraintColumn{
		Name:    name,
		Type:    ConstraintPK,
		Columns: col,
	}

	return stmt
}

// Index 添加索引
func (stmt *CreateTableStmt) Index(name string, typ Index, col ...string) *CreateTableStmt {
	stmt.indexes = append(stmt.indexes, &indexColumn{
		Name:    name,
		Type:    typ,
		Columns: col,
	})

	return stmt
}

// Unique 添加唯一约束
func (stmt *CreateTableStmt) Unique(name string, col ...string) *CreateTableStmt {
	stmt.constraints = append(stmt.constraints, &constraintColumn{
		Name:    name,
		Type:    ConstraintUnique,
		Columns: col,
	})

	return stmt
}

// Check check 约束
func (stmt *CreateTableStmt) Check(name string, expr string) *CreateTableStmt {
	stmt.constraints = append(stmt.constraints, &constraintColumn{
		Name:    name,
		Type:    ConstraintCheck,
		Columns: []string{expr},
	})

	return stmt
}

// ForeignKey 指定外键
func (stmt *CreateTableStmt) ForeignKey(name, col, refTable, refCol, updateRule, deleteRule string) *CreateTableStmt {
	stmt.foreignKeys = append(stmt.foreignKeys, &foreignKey{
		Name:         name,
		Column:       col,
		RefTableName: refTable,
		RefColName:   refCol,
		UpdateRule:   updateRule,
		DeleteRule:   deleteRule,
	})

	return stmt
}

// SQL 获取 SQL 的语句及参数部分
func (stmt *CreateTableStmt) SQL() ([]string, error) {
	w := New("CREATE TABLE IF NOT EXISTS ").
		WriteString(stmt.name).
		WriteByte('(')

	// 普通列
	for _, col := range stmt.columns {
		isAI := stmt.ai != nil && stmt.ai.Columns[0] == col.Name
		err := stmt.dialect.CreateColumnSQL(w, col, isAI)
		if err != nil {
			return nil, err
		}
		w.WriteByte(',')
	}

	if err := stmt.createConstraints(w); err != nil {
		return nil, err
	}

	w.TruncateLast(1).WriteByte(')')

	if err := stmt.dialect.CreateTableOptionsSQL(w, stmt.options); err != nil {
		return nil, err
	}

	sqls := []string{w.String()}

	indexes, err := createIndexSQL(stmt)
	if err != nil {
		return nil, err
	}
	if len(indexes) > 0 {
		sqls = append(sqls, indexes...)
	}

	return sqls, nil
}

// Exec 执行 SQL 语句
func (stmt *CreateTableStmt) Exec() (sql.Result, error) {
	return stmt.ExecContext(context.Background())
}

// ExecContext 执行 SQL 语句
func (stmt *CreateTableStmt) ExecContext(ctx context.Context) (sql.Result, error) {
	sqls, err := stmt.SQL()
	if err != nil {
		return nil, err
	}

	for _, sql := range sqls {
		if _, err := stmt.engine.ExecContext(ctx, sql, nil); err != nil {
			return nil, err
		}
	}

	return nil, nil
}

// 创建标准的几种约束
func (stmt *CreateTableStmt) createConstraints(buf *SQLBuilder) error {
	for _, c := range stmt.constraints {
		switch c.Type {
		case ConstraintCheck:
			createCheckSQL(buf, c.Name, c.Columns[0])
		case ConstraintUnique:
			createUniqueSQL(buf, c.Name, c.Columns...)
		default:
			return ErrUnknownConstraint
		}
		buf.WriteByte(',')
	}

	// foreign  key
	for _, fk := range stmt.foreignKeys {
		createFKSQL(buf, fk)
		buf.WriteByte(',')
	}

	// primary key
	if stmt.pk != nil {
		createPKSQL(buf, stmt.pk.Name, stmt.pk.Columns...)
	}

	return nil
}

func createIndexSQL(model *CreateTableStmt) ([]string, error) {
	if len(model.indexes) == 0 {
		return nil, nil
	}

	sqls := make([]string, 0, len(model.indexes))
	buf := CreateIndex(nil)
	for _, index := range model.indexes {
		buf.Reset()
		buf.Table("{#" + model.name + "}").
			Name(index.Name)
		for _, col := range index.Columns {
			buf.Columns("{" + col + "}")
		}

		sql, _, err := buf.SQL()
		if err != nil {
			return nil, err
		}
		sqls = append(sqls, sql)
	}

	return sqls, nil
}

// create table 语句中 pk 约束的语句
func createPKSQL(buf *SQLBuilder, name string, cols ...string) {
	// CONSTRAINT pk_name PRIMARY KEY (id,lastName)
	buf.WriteString(" CONSTRAINT ").
		WriteString(name).
		WriteString(" PRIMARY KEY(")

	for _, col := range cols {
		buf.WriteByte('{').WriteString(col).WriteByte('}')
		buf.WriteByte(',')
	}
	buf.TruncateLast(1) // 去掉最后一个逗号
	buf.WriteByte(')')
}

// create table 语句中的 unique 约束部分的语句。
func createUniqueSQL(buf *SQLBuilder, indexName string, cols ...string) {
	// CONSTRAINT unique_name UNIQUE (id,lastName)
	buf.WriteString(" CONSTRAINT ").
		WriteString(indexName).
		WriteString(" UNIQUE(")
	for _, col := range cols {
		buf.WriteByte('{').
			WriteString(col).
			WriteByte('}').
			WriteByte(',')
	}
	buf.TruncateLast(1) // 去掉最后一个逗号

	buf.WriteByte(')')
}

// create table 语句中 fk 的约束部分的语句
func createFKSQL(buf *SQLBuilder, fk *foreignKey) {
	// CONSTRAINT fk_name FOREIGN KEY (id) REFERENCES user(id)
	buf.WriteString(" CONSTRAINT ").WriteString(fk.Name)

	buf.WriteString(" FOREIGN KEY(")
	buf.WriteByte('{').WriteString(fk.Column).WriteByte('}')

	buf.WriteString(") REFERENCES ").
		WriteByte('{').
		WriteString(fk.RefTableName).
		WriteByte('}')

	buf.WriteByte('(')
	buf.WriteByte('{').WriteString(fk.RefColName).WriteByte('}')
	buf.WriteByte(')')

	if len(fk.UpdateRule) > 0 {
		buf.WriteString(" ON UPDATE ").WriteString(fk.UpdateRule)
	}

	if len(fk.DeleteRule) > 0 {
		buf.WriteString(" ON DELETE ").WriteString(fk.DeleteRule)
	}
}

// create table 语句中 check 约束部分的语句
func createCheckSQL(buf *SQLBuilder, name, expr string) {
	// CONSTRAINT chk_name CHECK (id>0 AND username='admin')
	buf.WriteString(" CONSTRAINT ").
		WriteString(name).
		WriteString(" CHECK(").
		WriteString(expr).
		WriteByte(')')
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"encoding/json"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
)

func init() {
	http.HandleFunc("/updatebenchmark", updateBenchmark)
}

func updateBenchmark(w http.ResponseWriter, r *http.Request) {
	if !appengine.IsDevAppServer() {
		fmt.Fprint(w, "Update must not run on real server.")
		return
	}

	if r.Method != "POST" {
		fmt.Fprintf(w, "bad request method")
		return
	}

	c := contextForRequest(r)
	if !validKey(c, r.FormValue("key"), r.FormValue("builder")) {
		fmt.Fprintf(w, "bad builder/key")
		return
	}

	defer r.Body.Close()
	var hashes []string
	if err := json.NewDecoder(r.Body).Decode(&hashes); err != nil {
		fmt.Fprintf(w, "failed to decode request: %v", err)
		return
	}

	ncommit := 0
	nrun := 0
	tx := func(c appengine.Context) error {
		var cr *CommitRun
		for _, hash := range hashes {
			// Update Commit.
			com := &Commit{Hash: hash}
			err := datastore.Get(c, com.Key(c), com)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching Commit: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				continue
			}
			com.NeedsBenchmarking = true
			com.ResultData = nil
			if err := putCommit(c, com); err != nil {
				return err
			}
			ncommit++

			// create PerfResult
			res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
			err = datastore.Get(c, res.Key(c), res)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching PerfResult: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				if _, err := datastore.Put(c, res.Key(c), res); err != nil {
					return fmt.Errorf("putting PerfResult: %v", err)
				}
			}

			// Update CommitRun.
			if cr != nil && cr.StartCommitNum != com.Num/PerfRunLength*PerfRunLength {
				if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
					return fmt.Errorf("putting CommitRun: %v", err)
				}
				nrun++
				cr = nil
			}
			if cr == nil {
				var err error
				cr, err = GetCommitRun(c, com.Num)
				if err != nil {
					return fmt.Errorf("getting CommitRun: %v", err)
				}
			}
			if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
				return fmt.Errorf("commit num %v out of range [%v, %v)", com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
			}
			idx := com.Num - cr.StartCommitNum
			cr.Hash[idx] = com.Hash
			cr.User[idx] = shortDesc(com.User)
			cr.Desc[idx] = shortDesc(com.Desc)
			cr.Time[idx] = com.Time
			cr.NeedsBenchmarking[idx] = com.NeedsBenchmarking
		}
		if cr != nil {
			if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
				return fmt.Errorf("putting CommitRun: %v", err)
			}
			nrun++
		}
		return nil
	}
	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
		fmt.Fprintf(w, "failed to execute tx: %v", err)
		return
	}
	fmt.Fprintf(w, "OK (updated %v commits and %v commit runs)", ncommit, nrun)
}
<commit_msg>dashboard: fix update script It needs to remove perf results rather than build results.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"encoding/json"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
)

func init() {
	http.HandleFunc("/updatebenchmark", updateBenchmark)
}

func updateBenchmark(w http.ResponseWriter, r *http.Request) {
	if !appengine.IsDevAppServer() {
		fmt.Fprint(w, "Update must not run on real server.")
		return
	}

	if r.Method != "POST" {
		fmt.Fprintf(w, "bad request method")
		return
	}

	c := contextForRequest(r)
	if !validKey(c, r.FormValue("key"), r.FormValue("builder")) {
		fmt.Fprintf(w, "bad builder/key")
		return
	}

	defer r.Body.Close()
	var hashes []string
	if err := json.NewDecoder(r.Body).Decode(&hashes); err != nil {
		fmt.Fprintf(w, "failed to decode request: %v", err)
		return
	}

	ncommit := 0
	nrun := 0
	tx := func(c appengine.Context) error {
		var cr *CommitRun
		for _, hash := range hashes {
			// Update Commit.
			com := &Commit{Hash: hash}
			err := datastore.Get(c, com.Key(c), com)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching Commit: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				continue
			}
			com.NeedsBenchmarking = true
			com.PerfResults = nil
			if err := putCommit(c, com); err != nil {
				return err
			}
			ncommit++

			// create PerfResult
			res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
			err = datastore.Get(c, res.Key(c), res)
			if err != nil && err != datastore.ErrNoSuchEntity {
				return fmt.Errorf("fetching PerfResult: %v", err)
			}
			if err == datastore.ErrNoSuchEntity {
				if _, err := datastore.Put(c, res.Key(c), res); err != nil {
					return fmt.Errorf("putting PerfResult: %v", err)
				}
			}

			// Update CommitRun.
			if cr != nil && cr.StartCommitNum != com.Num/PerfRunLength*PerfRunLength {
				if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
					return fmt.Errorf("putting CommitRun: %v", err)
				}
				nrun++
				cr = nil
			}
			if cr == nil {
				var err error
				cr, err = GetCommitRun(c, com.Num)
				if err != nil {
					return fmt.Errorf("getting CommitRun: %v", err)
				}
			}
			if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
				return fmt.Errorf("commit num %v out of range [%v, %v)", com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
			}
			idx := com.Num - cr.StartCommitNum
			cr.Hash[idx] = com.Hash
			cr.User[idx] = shortDesc(com.User)
			cr.Desc[idx] = shortDesc(com.Desc)
			cr.Time[idx] = com.Time
			cr.NeedsBenchmarking[idx] = com.NeedsBenchmarking
		}
		if cr != nil {
			if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
				return fmt.Errorf("putting CommitRun: %v", err)
			}
			nrun++
		}
		return nil
	}
	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
		fmt.Fprintf(w, "failed to execute tx: %v", err)
		return
	}
	fmt.Fprintf(w, "OK (updated %v commits and %v commit runs)", ncommit, nrun)
}
<|endoftext|>