<commit_before>package api

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"image"
	"image/draw"
	"image/png" // This needs to be imported because otherwhise image.Decode won't work with png images
	"os"
	"strings"
	"sync"

	"github.com/disintegration/imaging"
	webpbin "github.com/mjarkk/go-webpbin"
	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

type imageAndIDType struct {
	ID    string
	Image image.Image
}

// GetIcons fetches all clan icons and creates a grid
func GetIcons() error {
	other.DevPrint("Getting clan icons...")
	clans, err := db.GetCurrentClansData()
	if err != nil {
		return err
	}
	// toSavePath := "./icons/clanIcons"
	// os.RemoveAll(toSavePath)
	// os.Mkdir(toSavePath, os.ModePerm)
	imgAndID := []imageAndIDType{}

	var waitForImgs sync.WaitGroup
	waitForImgs.Add(len(clans))

	for _, clan := range clans {
		go func(clan db.ClanStats) {
			defer waitForImgs.Done()
			iconToGet, ok := clan.Emblems["X195.Portal"]
			if !ok || len(iconToGet) == 0 {
				iconToGet, ok = clan.Emblems["X256.Wowp"]
				if !ok || len(iconToGet) == 0 {
					apiErr("GetIcons", errors.New("no X195.Portal and/or X256.Wowp"), "Check if clan has a clan icon")
					return
				}
			}
			if !strings.Contains(iconToGet, "http://") && !strings.Contains(iconToGet, "https://") {
				apiErr("GetIcons", errors.New("image is not a valid url"), "Check if the image url contains http(s)// : "+iconToGet)
				return
			}
			out, err := RawGet(iconToGet)
			if err != nil {
				apiErr("GetIcons", err, "Can't fetch image: "+iconToGet)
				return
			}

			// uriDots := strings.Split(iconToGet, ".")
			// fileExt := uriDots[len(uriDots)-1]
			// imgPath := toSavePath + "/" + clan.ID + "." + fileExt
			// err = ioutil.WriteFile(imgPath, out, 0644)
			// if err != nil {
			// 	return
			// }

			img, _, err := image.Decode(bytes.NewReader(out))
			if err != nil {
				apiErr("GetIcons", err, "Can't decode image: "+iconToGet)
				return
			}

			imgAndID = append(imgAndID, imageAndIDType{
				ID:    clan.ID,
				Image: img,
			})
		}(clan)
	}

	waitForImgs.Wait()

	if len(imgAndID) > 0 {
		imgSize := 60
		imgsInARow := 30

		ids := [][]string{[]string{}}
		outputImg := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{0, 0}})

		for _, imgObj := range imgAndID {

			id := imgObj.ID
			if len(ids[len(ids)-1]) == imgsInARow {
				ids = append(ids, []string{})
			}
			from := struct {
				top  int
				side int
			}{
				top:  len(ids),
				side: len(ids[len(ids)-1]),
			}

			insertImg := imaging.Resize(imgObj.Image, imgSize, imgSize, imaging.Lanczos)
			offset := image.Point{imgSize * from.side, imgSize * (from.top - 1)}
			whereToInsert := image.Rectangle{offset, offset.Add(insertImg.Bounds().Size())}
			maxSize := image.Point{(len(ids[0]) + 1) * imgSize, len(ids) * imgSize}

			canvas := image.NewRGBA(image.Rectangle{image.Point{0, 0}, maxSize})
			draw.Draw(canvas, outputImg.Bounds(), outputImg, image.Point{0, 0}, draw.Src)
			draw.Draw(canvas, whereToInsert, insertImg, image.Point{0, 0}, draw.Src)
			outputImg = canvas

			ids[len(ids)-1] = append(ids[len(ids)-1], id)
		}

		file, err := os.Create("./icons/allIcons.png")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.png")
			return err
		}
		png.Encode(file, outputImg)
		file.Close()

		err = webpbin.NewCWebP().
			Quality(50).
			InputFile("./icons/allIcons.png").
			OutputFile("./icons/allIcons.webp").
			Run()

		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.webp")
			return err
		}

		file, err = os.Create("./icons/allIcons.json")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.json")
			return err
		}
		writer := bufio.NewWriter(file)

		defer func() {
			writer.Flush()
			file.Close()
		}()

		jsonEncoder := json.NewEncoder(writer)
		err = jsonEncoder.Encode(ids)
		if err != nil {
			apiErr("GetIcons", err, "Can't encode config into json")
			return err
		}
	}

	return nil
}
<commit_msg>Removed one error logging line<commit_after>package api

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"image"
	"image/draw"
	"image/png" // This needs to be imported because otherwhise image.Decode won't work with png images
	"os"
	"strings"
	"sync"

	"github.com/disintegration/imaging"
	webpbin "github.com/mjarkk/go-webpbin"
	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

type imageAndIDType struct {
	ID    string
	Image image.Image
}

// GetIcons fetches all clan icons and creates a grid
func GetIcons() error {
	other.DevPrint("Getting clan icons...")
	clans, err := db.GetCurrentClansData()
	if err != nil {
		return err
	}

	// toSavePath := "./icons/clanIcons"
	// os.RemoveAll(toSavePath)
	// os.Mkdir(toSavePath, os.ModePerm)
	imgAndID := []imageAndIDType{}

	var waitForImgs sync.WaitGroup
	waitForImgs.Add(len(clans))

	for _, clan := range clans {
		go func(clan db.ClanStats) {
			defer waitForImgs.Done()
			iconToGet, ok := clan.Emblems["X195.Portal"]
			if !ok || len(iconToGet) == 0 {
				iconToGet, ok = clan.Emblems["X256.Wowp"]
				if !ok || len(iconToGet) == 0 {
					// No error reporting here because this happends to often
					return
				}
			}
			if !strings.Contains(iconToGet, "http://") && !strings.Contains(iconToGet, "https://") {
				apiErr("GetIcons", errors.New("image is not a valid url"), "Check if the image url contains http(s)// : "+iconToGet)
				return
			}
			out, err := RawGet(iconToGet)
			if err != nil {
				apiErr("GetIcons", err, "Can't fetch image: "+iconToGet)
				return
			}

			// uriDots := strings.Split(iconToGet, ".")
			// fileExt := uriDots[len(uriDots)-1]
			// imgPath := toSavePath + "/" + clan.ID + "." + fileExt
			// err = ioutil.WriteFile(imgPath, out, 0644)
			// if err != nil {
			// 	return
			// }

			img, _, err := image.Decode(bytes.NewReader(out))
			if err != nil {
				apiErr("GetIcons", err, "Can't decode image: "+iconToGet)
				return
			}

			imgAndID = append(imgAndID, imageAndIDType{
				ID:    clan.ID,
				Image: img,
			})
		}(clan)
	}

	waitForImgs.Wait()

	if len(imgAndID) > 0 {
		imgSize := 60
		imgsInARow := 30

		ids := [][]string{[]string{}}
		outputImg := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{0, 0}})

		for _, imgObj := range imgAndID {

			id := imgObj.ID
			if len(ids[len(ids)-1]) == imgsInARow {
				ids = append(ids, []string{})
			}
			from := struct {
				top  int
				side int
			}{
				top:  len(ids),
				side: len(ids[len(ids)-1]),
			}

			insertImg := imaging.Resize(imgObj.Image, imgSize, imgSize, imaging.Lanczos)
			offset := image.Point{imgSize * from.side, imgSize * (from.top - 1)}
			whereToInsert := image.Rectangle{offset, offset.Add(insertImg.Bounds().Size())}
			maxSize := image.Point{(len(ids[0]) + 1) * imgSize, len(ids) * imgSize}

			canvas := image.NewRGBA(image.Rectangle{image.Point{0, 0}, maxSize})
			draw.Draw(canvas, outputImg.Bounds(), outputImg, image.Point{0, 0}, draw.Src)
			draw.Draw(canvas, whereToInsert, insertImg, image.Point{0, 0}, draw.Src)
			outputImg = canvas

			ids[len(ids)-1] = append(ids[len(ids)-1], id)
		}

		file, err := os.Create("./icons/allIcons.png")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.png")
			return err
		}
		png.Encode(file, outputImg)
		file.Close()

		err = webpbin.NewCWebP().
			Quality(50).
			InputFile("./icons/allIcons.png").
			OutputFile("./icons/allIcons.webp").
			Run()

		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.webp")
			return err
		}

		file, err = os.Create("./icons/allIcons.json")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.json")
			return err
		}
		writer := bufio.NewWriter(file)

		defer func() {
			writer.Flush()
			file.Close()
		}()

		jsonEncoder := json.NewEncoder(writer)
		err = jsonEncoder.Encode(ids)
		if err != nil {
			apiErr("GetIcons", err, "Can't encode config into json")
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"ddtxn"
	"ddtxn/dlog"
	"ddtxn/prof"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"runtime"
	"sync"
	"time"
)

var nprocs = flag.Int("nprocs", 2, "GOMAXPROCS default 2")
var nsec = flag.Int("nsec", 2, "Time to run in seconds")
var clientGoRoutines = flag.Int("ngo", 0, "Number of goroutines/workers generating client requests.")
var nworkers = flag.Int("nw", 0, "Number of workers")
var nbidders = flag.Int("nb", 1000000, "Keys in store, default is 1M")
var prob = flag.Float64("contention", 100.0, "Probability contended key is in txn. -1 means zipfian distribution and we use ZipfDist")
var readrate = flag.Int("rr", 0, "Read rate %.  Rest are writes")
var dataFile = flag.String("out", "single-data.out", "Filename for output")
var latency = flag.Bool("latency", false, "dummy")
var atomicIncr = flag.Bool("atomic", false, "Workload of just atomic increments")

var ZipfDist = flag.Float64("zipf", 1, "Zipfian distribution theta.  1 means only 1 hot key and we'll vary the percentage (single exp)")
var partition = flag.Bool("partition", false, "Whether or not to partition the non-contended keys amongst the cores")
var GoZipf = flag.Bool("gozipf", true, "Use Go's Zipf function")

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(*nprocs)

	if *clientGoRoutines == 0 {
		*clientGoRoutines = *nprocs
	}
	if *nworkers == 0 {
		*nworkers = *nprocs
	}
	if *prob == -1 && *ZipfDist < 0 {
		log.Fatalf("Zipf distribution must be positive")
	}
	if *ZipfDist >= 0 && *prob > -1 {
		log.Fatalf("Set contention to -1 to use Zipf distribution of keys")
	}
	s := ddtxn.NewStore()
	for i := 0; i < *nbidders; i++ {
		k := ddtxn.ProductKey(i)
		s.CreateKey(k, int32(0), ddtxn.SUM)
	}
	dlog.Printf("Done with Populate")

	coord := ddtxn.NewCoordinator(*nworkers, s)

	if *ddtxn.CountKeys {
		for i := 0; i < *nworkers; i++ {
			w := coord.Workers[i]
			w.NKeyAccesses = make([]int64, *nbidders)
		}
	}

	dlog.Printf("Done initializing single\n")

	p := prof.StartProfile()
	start := time.Now()
	sp := uint32(*nbidders / *nworkers)
	var wg sync.WaitGroup
	pkey := int(sp - 1)
	dlog.Printf("Partition size: %v; Contended key %v\n", sp/2, pkey)
	gave_up := make([]int64, *clientGoRoutines)

	myZipf := make([]*ddtxn.MyZipf, *clientGoRoutines)
	goZipf := make([]*ddtxn.Zipf, *clientGoRoutines)
	if *prob == -1 && *ZipfDist >= 0 {
		if *GoZipf {
			for i := 0; i < *clientGoRoutines; i++ {
				rnd := rand.New(rand.NewSource(int64(i * 12467)))
				goZipf[i] = ddtxn.NewZipf(rnd, *ZipfDist, 1, uint64(*nbidders)-1)
				if goZipf[i] == nil {
					panic("nil zipf")
				}
			}
		} else {
			for i := 0; i < *clientGoRoutines; i++ {
				myZipf[i] = ddtxn.NewMyZipf(int64(*nbidders), *ZipfDist)
			}
		}
	}

	for i := 0; i < *clientGoRoutines; i++ {
		wg.Add(1)
		go func(n int) {
			exp := ddtxn.MakeExp(30)
			retries := make(ddtxn.RetryHeap, 0)
			heap.Init(&retries)
			var local_seed uint32 = uint32(rand.Intn(10000000))
			wi := n % (*nworkers)
			w := coord.Workers[wi]
			top := (wi + 1) * int(sp)
			bottom := wi * int(sp)
			dlog.Printf("%v: Noncontended section: %v to %v\n", n, bottom, top)
			end_time := time.Now().Add(time.Duration(*nsec) * time.Second)
			for {
				tm := time.Now()
				if !end_time.After(tm) {
					break
				}
				var t ddtxn.Query
				if len(retries) > 0 && retries[0].TS.Before(tm) {
					t = heap.Pop(&retries).(ddtxn.Query)
				} else {
					x := float64(ddtxn.RandN(&local_seed, 100))
					if *prob == -1 {
						if *GoZipf {
							x := goZipf[n].Uint64()
							if x >= uint64(*nbidders) || x < 0 {
								log.Fatalf("x not in bounds: %v\n", x)
							}
							t.K1 = ddtxn.ProductKey(int(x))
						} else {
							t.K1 = ddtxn.ProductKey(int(myZipf[n].Next(&local_seed) - 1))
						}
					} else if x < *prob {
						// contended txn
						t.K1 = ddtxn.ProductKey(pkey)
					} else {
						// uncontended
						k := pkey
						for k == pkey {
							if *partition {
								rnd := ddtxn.RandN(&local_seed, sp-1)
								lb := int(rnd)
								k = lb + wi*int(sp) + 1
								if k < bottom || k >= top+1 {
									log.Fatalf("%v: outside my range %v [%v-%v]\n", n, k, bottom, top)
								}
							} else {
								k = int(ddtxn.RandN(&local_seed, uint32(*nbidders)))
							}
						}
						t.K1 = ddtxn.ProductKey(k)
					}
					t.TXN = ddtxn.D_INCR_ONE
					if *atomicIncr {
						t.TXN = ddtxn.D_ATOMIC_INCR_ONE
					}
					y := int(ddtxn.RandN(&local_seed, 100))
					if y < *readrate {
						t.TXN = ddtxn.D_READ_ONE
					}
				}
				committed := false
				_, err := w.One(t)
				if err == ddtxn.EABORT {
					committed = false
				} else {
					committed = true
				}
				t.I++
				if !committed {
					t.TS = tm.Add(time.Duration(ddtxn.RandN(&local_seed, exp.Exp(t.I))) * time.Microsecond)
					if t.TS.Before(end_time) {
						heap.Push(&retries, t)
					} else {
						gave_up[n]++
					}
				}
			}
			wg.Done()
			if len(retries) > 0 {
				dlog.Printf("[%v] Length of retry queue on exit: %v\n", n, len(retries))
			}
			gave_up[n] = gave_up[n] + int64(len(retries))
		}(i)
	}
	wg.Wait()
	coord.Finish()
	end := time.Since(start)
	p.Stop()

	stats := make([]int64, ddtxn.LAST_STAT)
	nitr, nwait, _ := ddtxn.CollectCounts(coord, stats)

	for i := 1; i < *clientGoRoutines; i++ {
		gave_up[0] = gave_up[0] + gave_up[i]
	}

	// nitr + NABORTS + ENOKEY is how many requests were issued.  A
	// stashed transaction eventually executes and contributes to
	// nitr.
	out := fmt.Sprintf(" nworkers: %v, nwmoved: %v, nrmoved: %v, sys: %v, total/sec: %v, abortrate: %.2f, stashrate: %.2f, rr: %v, nkeys: %v, contention: %v, zipf: %v, done: %v, actual time: %v, nreads: %v, nincrs: %v, epoch changes: %v, throughput ns/txn: %v, naborts: %v, coord time: %v, coord stats time: %v, total worker time transitioning: %v, nstashed: %v, rlock: %v, wrratio: %v, nsamples: %v, getkeys: %v, ddwrites: %v, nolock: %v, failv: %v, nlocked: %v, stashdone: %v, nfast: %v, gaveup: %v, potential: %v ", *nworkers, ddtxn.WMoved, ddtxn.RMoved, *ddtxn.SysType, float64(nitr)/end.Seconds(), 100*float64(stats[ddtxn.NABORTS])/float64(nitr+stats[ddtxn.NABORTS]), 100*float64(stats[ddtxn.NSTASHED])/float64(nitr+stats[ddtxn.NABORTS]), *readrate, *nbidders, *prob, *ZipfDist, nitr, end, stats[ddtxn.D_READ_ONE], stats[ddtxn.D_INCR_ONE], ddtxn.NextEpoch, end.Nanoseconds()/nitr, stats[ddtxn.NABORTS], ddtxn.Time_in_IE, ddtxn.Time_in_IE1, nwait, stats[ddtxn.NSTASHED], *ddtxn.UseRLocks, *ddtxn.WRRatio, stats[ddtxn.NSAMPLES], stats[ddtxn.NGETKEYCALLS], stats[ddtxn.NDDWRITES], stats[ddtxn.NO_LOCK], stats[ddtxn.NFAIL_VERIFY], stats[ddtxn.NLOCKED], stats[ddtxn.NDIDSTASHED], ddtxn.Nfast, gave_up[0], coord.PotentialPhaseChanges)
	fmt.Printf(out)
	fmt.Printf("\n")

	f, err := os.OpenFile(*dataFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	ddtxn.PrintStats(out, stats, f, coord, s, *nbidders)
}
<commit_msg>fix flag<commit_after>package main

import (
	"container/heap"
	"ddtxn"
	"ddtxn/dlog"
	"ddtxn/prof"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"runtime"
	"sync"
	"time"
)

var nprocs = flag.Int("nprocs", 2, "GOMAXPROCS default 2")
var nsec = flag.Int("nsec", 2, "Time to run in seconds")
var clientGoRoutines = flag.Int("ngo", 0, "Number of goroutines/workers generating client requests.")
var nworkers = flag.Int("nw", 0, "Number of workers")
var nbidders = flag.Int("nb", 1000000, "Keys in store, default is 1M")
var prob = flag.Float64("contention", 100.0, "Probability contended key is in txn. -1 means zipfian distribution and we use ZipfDist")
var readrate = flag.Int("rr", 0, "Read rate %.  Rest are writes")
var dataFile = flag.String("out", "single-data.out", "Filename for output")
var atomicIncr = flag.Bool("atomic", false, "Workload of just atomic increments")

var ZipfDist = flag.Float64("zipf", 1, "Zipfian distribution theta.  1 means only 1 hot key and we'll vary the percentage (single exp)")
var partition = flag.Bool("partition", false, "Whether or not to partition the non-contended keys amongst the cores")
var GoZipf = flag.Bool("gozipf", true, "Use Go's Zipf function")

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(*nprocs)

	if *clientGoRoutines == 0 {
		*clientGoRoutines = *nprocs
	}
	if *nworkers == 0 {
		*nworkers = *nprocs
	}
	if *prob == -1 && *ZipfDist < 0 {
		log.Fatalf("Zipf distribution must be positive")
	}
	if *ZipfDist >= 0 && *prob > -1 {
		log.Fatalf("Set contention to -1 to use Zipf distribution of keys")
	}
	s := ddtxn.NewStore()
	for i := 0; i < *nbidders; i++ {
		k := ddtxn.ProductKey(i)
		s.CreateKey(k, int32(0), ddtxn.SUM)
	}
	dlog.Printf("Done with Populate")

	coord := ddtxn.NewCoordinator(*nworkers, s)

	if *ddtxn.CountKeys {
		for i := 0; i < *nworkers; i++ {
			w := coord.Workers[i]
			w.NKeyAccesses = make([]int64, *nbidders)
		}
	}

	dlog.Printf("Done initializing single\n")

	p := prof.StartProfile()
	start := time.Now()
	sp := uint32(*nbidders / *nworkers)
	var wg sync.WaitGroup
	pkey := int(sp - 1)
	dlog.Printf("Partition size: %v; Contended key %v\n", sp/2, pkey)
	gave_up := make([]int64, *clientGoRoutines)

	myZipf := make([]*ddtxn.MyZipf, *clientGoRoutines)
	goZipf := make([]*ddtxn.Zipf, *clientGoRoutines)
	if *prob == -1 && *ZipfDist >= 0 {
		if *GoZipf {
			for i := 0; i < *clientGoRoutines; i++ {
				rnd := rand.New(rand.NewSource(int64(i * 12467)))
				goZipf[i] = ddtxn.NewZipf(rnd, *ZipfDist, 1, uint64(*nbidders)-1)
				if goZipf[i] == nil {
					panic("nil zipf")
				}
			}
		} else {
			for i := 0; i < *clientGoRoutines; i++ {
				myZipf[i] = ddtxn.NewMyZipf(int64(*nbidders), *ZipfDist)
			}
		}
	}

	for i := 0; i < *clientGoRoutines; i++ {
		wg.Add(1)
		go func(n int) {
			exp := ddtxn.MakeExp(30)
			retries := make(ddtxn.RetryHeap, 0)
			heap.Init(&retries)
			var local_seed uint32 = uint32(rand.Intn(10000000))
			wi := n % (*nworkers)
			w := coord.Workers[wi]
			top := (wi + 1) * int(sp)
			bottom := wi * int(sp)
			dlog.Printf("%v: Noncontended section: %v to %v\n", n, bottom, top)
			end_time := time.Now().Add(time.Duration(*nsec) * time.Second)
			for {
				tm := time.Now()
				if !end_time.After(tm) {
					break
				}
				var t ddtxn.Query
				if len(retries) > 0 && retries[0].TS.Before(tm) {
					t = heap.Pop(&retries).(ddtxn.Query)
				} else {
					x := float64(ddtxn.RandN(&local_seed, 100))
					if *prob == -1 {
						if *GoZipf {
							x := goZipf[n].Uint64()
							if x >= uint64(*nbidders) || x < 0 {
								log.Fatalf("x not in bounds: %v\n", x)
							}
							t.K1 = ddtxn.ProductKey(int(x))
						} else {
							t.K1 = ddtxn.ProductKey(int(myZipf[n].Next(&local_seed) - 1))
						}
					} else if x < *prob {
						// contended txn
						t.K1 = ddtxn.ProductKey(pkey)
					} else {
						// uncontended
						k := pkey
						for k == pkey {
							if *partition {
								rnd := ddtxn.RandN(&local_seed, sp-1)
								lb := int(rnd)
								k = lb + wi*int(sp) + 1
								if k < bottom || k >= top+1 {
									log.Fatalf("%v: outside my range %v [%v-%v]\n", n, k, bottom, top)
								}
							} else {
								k = int(ddtxn.RandN(&local_seed, uint32(*nbidders)))
							}
						}
						t.K1 = ddtxn.ProductKey(k)
					}
					t.TXN = ddtxn.D_INCR_ONE
					if *atomicIncr {
						t.TXN = ddtxn.D_ATOMIC_INCR_ONE
					}
					y := int(ddtxn.RandN(&local_seed, 100))
					if y < *readrate {
						t.TXN = ddtxn.D_READ_ONE
					}
				}
				committed := false
				_, err := w.One(t)
				if err == ddtxn.EABORT {
					committed = false
				} else {
					committed = true
				}
				t.I++
				if !committed {
					t.TS = tm.Add(time.Duration(ddtxn.RandN(&local_seed, exp.Exp(t.I))) * time.Microsecond)
					if t.TS.Before(end_time) {
						heap.Push(&retries, t)
					} else {
						gave_up[n]++
					}
				}
			}
			wg.Done()
			if len(retries) > 0 {
				dlog.Printf("[%v] Length of retry queue on exit: %v\n", n, len(retries))
			}
			gave_up[n] = gave_up[n] + int64(len(retries))
		}(i)
	}
	wg.Wait()
	coord.Finish()
	end := time.Since(start)
	p.Stop()

	stats := make([]int64, ddtxn.LAST_STAT)
	nitr, nwait, _ := ddtxn.CollectCounts(coord, stats)

	for i := 1; i < *clientGoRoutines; i++ {
		gave_up[0] = gave_up[0] + gave_up[i]
	}

	// nitr + NABORTS + ENOKEY is how many requests were issued.  A
	// stashed transaction eventually executes and contributes to
	// nitr.
	out := fmt.Sprintf(" nworkers: %v, nwmoved: %v, nrmoved: %v, sys: %v, total/sec: %v, abortrate: %.2f, stashrate: %.2f, rr: %v, nkeys: %v, contention: %v, zipf: %v, done: %v, actual time: %v, nreads: %v, nincrs: %v, epoch changes: %v, throughput ns/txn: %v, naborts: %v, coord time: %v, coord stats time: %v, total worker time transitioning: %v, nstashed: %v, rlock: %v, wrratio: %v, nsamples: %v, getkeys: %v, ddwrites: %v, nolock: %v, failv: %v, nlocked: %v, stashdone: %v, nfast: %v, gaveup: %v, potential: %v ", *nworkers, ddtxn.WMoved, ddtxn.RMoved, *ddtxn.SysType, float64(nitr)/end.Seconds(), 100*float64(stats[ddtxn.NABORTS])/float64(nitr+stats[ddtxn.NABORTS]), 100*float64(stats[ddtxn.NSTASHED])/float64(nitr+stats[ddtxn.NABORTS]), *readrate, *nbidders, *prob, *ZipfDist, nitr, end, stats[ddtxn.D_READ_ONE], stats[ddtxn.D_INCR_ONE], ddtxn.NextEpoch, end.Nanoseconds()/nitr, stats[ddtxn.NABORTS], ddtxn.Time_in_IE, ddtxn.Time_in_IE1, nwait, stats[ddtxn.NSTASHED], *ddtxn.UseRLocks, *ddtxn.WRRatio, stats[ddtxn.NSAMPLES], stats[ddtxn.NGETKEYCALLS], stats[ddtxn.NDDWRITES], stats[ddtxn.NO_LOCK], stats[ddtxn.NFAIL_VERIFY], stats[ddtxn.NLOCKED], stats[ddtxn.NDIDSTASHED], ddtxn.Nfast, gave_up[0], coord.PotentialPhaseChanges)
	fmt.Printf(out)
	fmt.Printf("\n")

	f, err := os.OpenFile(*dataFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	ddtxn.PrintStats(out, stats, f, coord, s, *nbidders)
}
<|endoftext|>


<commit_before>package longhorn

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/util"
	"github.com/rancher/go-rancher-metadata/metadata"

	. "github.com/rancher/convoy/convoydriver"
	rancherClient "github.com/rancher/go-rancher/client"
)

const (
	DRIVER_NAME        = "longhorn"
	DRIVER_CONFIG_FILE = "longhorn.cfg"
	MOUNTS_DIR         = "mounts"
	DEV_DIR            = "/dev/longhorn/%s"

	VOLUME_CFG_PREFIX   = "volume_"
	LONGHORN_CFG_PREFIX = DRIVER_NAME + "_"
	CFG_POSTFIX         = ".json"

	SNAPSHOT_PATH = "snapshots"

	DEFAULT_VOLUME_SIZE = "10G"

	RANCHER_METADATA_URL = "http://rancher-metadata/2015-12-19"

	LH_RANCHER_URL         = "lh.rancherurl"
	LH_RANCHER_ACCESS_KEY  = "lh.rancheraccesskey"
	LH_RANCHER_SECRET_KEY  = "lh.ranchersecretkey"
	LH_DEFAULT_VOLUME_SIZE = "lh.defaultvolumesize"
	LH_CONTAINER_NAME      = "lh.containername"

	COMPOSE_VOLUME_NAME = "VOLUME_NAME"
	COMPOSE_VOLUME_SIZE = "VOLUME_SIZE"
	COMPOSE_SLAB_SIZE   = "SLAB_SIZE"
	COMPOSE_CONVOY      = "CONVOY_CONTAINER"

	AFFINITY_LABEL = "io.rancher.scheduler.affinity:container"
)

var (
	log = logrus.WithFields(logrus.Fields{"pkg": "longhorn"})
)

type Driver struct {
	mutex         *sync.RWMutex
	client        *rancherClient.RancherClient
	containerName string
	Device
}

type Device struct {
	Root              string
	DefaultVolumeSize int64
	RancherURL        string
	RancherAccessKey  string
	RancherSecretKey  string
}

func (dev *Device) ConfigFile() (string, error) {
	if dev.Root == "" {
		return "", fmt.Errorf("BUG: Invalid empty device config path")
	}
	return filepath.Join(dev.Root, DRIVER_CONFIG_FILE), nil
}

type Volume struct {
	Size         int64
	Name         string
	MountPoint   string
	PrepareForVM bool
	CreatedTime  string

	configPath string
}

func (v *Volume) Stack(driver *Driver) *Stack {
	sizeString := strconv.FormatInt(v.Size, 10)
	env := map[string]interface{}{
		COMPOSE_SLAB_SIZE:   sizeString,
		COMPOSE_VOLUME_NAME: v.Name,
		COMPOSE_VOLUME_SIZE: sizeString,
		COMPOSE_CONVOY:      driver.containerName,
	}
	return &Stack{
		Client:        driver.client,
		Name:          "longhorn-vol-" + v.Name,
		ExternalId:    "system://longhorn?name=" + v.Name,
		Template:      DockerComposeTemplate,
		Environment:   env,
		ContainerName: driver.containerName,
	}
}

func (v *Volume) ConfigFile() (string, error) {
	if v.Name == "" {
		return "", fmt.Errorf("BUG: Invalid empty volume name")
	}
	if v.configPath == "" {
		return "", fmt.Errorf("BUG: Invalid empty volume config path")
	}
	return filepath.Join(v.configPath, LONGHORN_CFG_PREFIX+VOLUME_CFG_PREFIX+v.Name+CFG_POSTFIX), nil
}

func (v *Volume) GetDevice() (string, error) {
	return fmt.Sprintf(DEV_DIR, v.Name), nil
}

func (v *Volume) GetMountOpts() []string {
	return []string{}
}

func (v *Volume) GenerateDefaultMountPoint() string {
	return filepath.Join(v.configPath, MOUNTS_DIR, v.Name)
}

func (d *Driver) blankVolume(name string) *Volume {
	return &Volume{
		configPath: d.Root,
		Name:       name,
	}
}

func init() {
	Register(DRIVER_NAME, Init)
}

func override(existing, newValue string) string {
	if newValue != "" {
		return newValue
	}
	return existing
}

func Init(root string, config map[string]string) (ConvoyDriver, error) {
	dev := &Device{
		Root: root,
	}
	exists, err := util.ObjectExists(dev)
	if err != nil {
		return nil, err
	}
	if exists {
		if err := util.ObjectLoad(dev); err != nil {
			return nil, err
		}
		dev.RancherURL = override(dev.RancherURL, config[LH_RANCHER_URL])
		dev.RancherAccessKey = override(dev.RancherAccessKey, config[LH_RANCHER_ACCESS_KEY])
		dev.RancherSecretKey = override(dev.RancherSecretKey, config[LH_RANCHER_SECRET_KEY])
	} else {
		if err := util.MkdirIfNotExists(root); err != nil {
			return nil, err
		}

		url := config[LH_RANCHER_URL]
		accessKey := config[LH_RANCHER_ACCESS_KEY]
		secretKey := config[LH_RANCHER_SECRET_KEY]

		if url == "" || accessKey == "" || secretKey == "" {
			return nil, fmt.Errorf("Missing required parameter. lh.rancherurl or lh.rancheraccesskey or lh.ranchersecretkey")
		}

		if _, exists := config[LH_DEFAULT_VOLUME_SIZE]; !exists {
			config[LH_DEFAULT_VOLUME_SIZE] = DEFAULT_VOLUME_SIZE
		}
		volumeSize, err := util.ParseSize(config[LH_DEFAULT_VOLUME_SIZE])
		if err != nil || volumeSize == 0 {
			return nil, fmt.Errorf("Illegal default volume size specified")
		}

		dev = &Device{
			Root:              root,
			RancherURL:        url,
			RancherAccessKey:  accessKey,
			RancherSecretKey:  secretKey,
			DefaultVolumeSize: volumeSize,
		}
	}

	containerName := config[LH_CONTAINER_NAME]
	if containerName == "" {
		handler := metadata.NewClient(RANCHER_METADATA_URL)
		container, err := handler.GetSelfContainer()
		if err != nil {
			return nil, err
		}
		containerName = container.UUID
	}

	log.Debugf("Try to connect to Rancher server at %s [%s:%s]", dev.RancherURL, dev.RancherAccessKey, dev.RancherSecretKey)
	client, err := rancherClient.NewRancherClient(&rancherClient.ClientOpts{
		Url:       dev.RancherURL,
		AccessKey: dev.RancherAccessKey,
		SecretKey: dev.RancherSecretKey,
	})
	if err != nil {
		return nil, fmt.Errorf("Failed to establish connection to Rancher server")
	}

	if err := util.ObjectSave(dev); err != nil {
		return nil, err
	}
	d := &Driver{
		client:        client,
		containerName: containerName,
		Device:        *dev,
		mutex:         &sync.RWMutex{},
	}

	return d, nil
}

func (d *Driver) Name() string {
	return DRIVER_NAME
}

func (d *Driver) Info() (map[string]string, error) {
	return map[string]string{
		"Root":             d.Root,
		"RancherURL":       d.RancherURL,
		"RancherAccessKey": d.RancherAccessKey,
		"RancherSecretKey": d.RancherSecretKey,
	}, nil
}

func (d *Driver) VolumeOps() (VolumeOperations, error) {
	return d, nil
}

func (d *Driver) CreateVolume(req Request) error {
	id := req.Name
	opts := req.Options

	size, err := util.ParseSize(opts[OPT_SIZE])
	if err != nil {
		return err
	}
	if size == 0 {
		size = d.DefaultVolumeSize
	}

	volume := d.blankVolume(id)
	volume.Size = size
	volume.Name = opts[OPT_VOLUME_NAME]
	volume.PrepareForVM, err = strconv.ParseBool(opts[OPT_PREPARE_FOR_VM])
	volume.CreatedTime = util.Now()
	if err != nil {
		return err
	}

	stack := volume.Stack(d)

	if err := d.doCreateVolume(volume, stack, id, opts); err != nil {
		stack.Delete()
		return err
	}

	return nil
}

func (d *Driver) doCreateVolume(volume *Volume, stack *Stack, id string, opts map[string]string) error {
	// Doing find just to see if we are creating versus using an existing stack
	env, err := stack.Find()
	if err != nil {
		return err
	}

	// Always run create because it also ensures that things are active
	if _, err := stack.Create(); err != nil {
		return err
	}

	// If env was nil then we created stack so we need to format
	if env == nil {
		dev, _ := volume.GetDevice()
		err := Backoff(5*time.Minute, fmt.Sprintf("Failed to find %s", dev), func() (bool, error) {
			if _, err := os.Stat(dev); err == nil {
				return true, nil
			}
			return false, nil
		})
		if err != nil {
			return err
		}

		log.Infof("Formatting %s", dev)
		if _, err := util.Execute("mkfs.ext4", []string{dev}); err != nil {
			return err
		}
	}

	return util.ObjectSave(volume)
}

func (d *Driver) DeleteVolume(req Request) error {
	id := req.Name
	opts := req.Options

	volume := d.blankVolume(id)

	if err := util.ObjectLoad(volume); err != nil {
		return err
	}

	if volume.MountPoint != "" {
		return fmt.Errorf("Cannot delete volume %v. It is still mounted", id)
	}

	referenceOnly, _ := strconv.ParseBool(opts[OPT_REFERENCE_ONLY])
	if !referenceOnly {
		log.Debugf("Deleting stack for volume %v", id)
		if err := volume.Stack(d).Delete(); err != nil {
			return err
		}
	}

	return util.ObjectDelete(volume)
}

func (d *Driver) MountVolume(req Request) (string, error) {
	id := req.Name
	opts := req.Options

	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return "", err
	}

	if err := volume.Stack(d).MoveController(); err != nil {
		log.Errorf("Failed to move controller to %s", d.containerName)
		return "", err
	}

	mountPoint, err := util.VolumeMount(volume, opts[OPT_MOUNT_POINT], false)
	if err != nil {
		return "", err
	}

	if volume.PrepareForVM {
		if err := util.MountPointPrepareImageFile(mountPoint, volume.Size); err != nil {
			return "", err
		}
	}

	if err := util.ObjectSave(volume); err != nil {
		return "", err
	}

	return mountPoint, nil
}

func (d *Driver) UmountVolume(req Request) error {
	id := req.Name

	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return err
	}

	if err := util.VolumeUmount(volume); err != nil {
		return err
	}

	return util.ObjectSave(volume)
}

func (d *Driver) MountPoint(req Request) (string, error) {
	id := req.Name

	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return "", err
	}
	return volume.MountPoint, nil
}

func (d *Driver) GetVolumeInfo(id string) (map[string]string, error) {
	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return nil, err
	}
	return map[string]string{
		"Size":                  strconv.FormatInt(volume.Size, 10),
		OPT_PREPARE_FOR_VM:      strconv.FormatBool(volume.PrepareForVM),
		OPT_VOLUME_CREATED_TIME: volume.CreatedTime,
		OPT_VOLUME_NAME:         volume.Name,
	}, nil
}

func (d *Driver) ListVolume(opts map[string]string) (map[string]map[string]string, error) {
	d.mutex.RLock()
	defer d.mutex.RUnlock()

	volumeIDs, err := d.listVolumeIDs()
	if err != nil {
		return nil, err
	}
	result := map[string]map[string]string{}
	for _, id := range volumeIDs {
		result[id], err = d.GetVolumeInfo(id)
		if err != nil {
			return nil, err
		}
	}
	return result, nil
}

func (device *Device) listVolumeIDs() ([]string, error) {
	return util.ListConfigIDs(device.Root, LONGHORN_CFG_PREFIX+VOLUME_CFG_PREFIX, CFG_POSTFIX)
}

func (d *Driver) SnapshotOps() (SnapshotOperations, error) {
	return nil, fmt.Errorf("Longhorn doesn't support snapshot ops")
}

func (d *Driver) BackupOps() (BackupOperations, error) {
	return nil, fmt.Errorf("Longhorn doesn't support backup ops")
}
<commit_msg>always use default mount point for longhorn<commit_after>package longhorn

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/util"
	"github.com/rancher/go-rancher-metadata/metadata"

	. "github.com/rancher/convoy/convoydriver"
	rancherClient "github.com/rancher/go-rancher/client"
)

const (
	DRIVER_NAME        = "longhorn"
	DRIVER_CONFIG_FILE = "longhorn.cfg"
	MOUNTS_DIR         = "mounts"
	DEV_DIR            = "/dev/longhorn/%s"

	VOLUME_CFG_PREFIX   = "volume_"
	LONGHORN_CFG_PREFIX = DRIVER_NAME + "_"
	CFG_POSTFIX         = ".json"

	SNAPSHOT_PATH = "snapshots"

	DEFAULT_VOLUME_SIZE = "10G"

	RANCHER_METADATA_URL = "http://rancher-metadata/2015-12-19"

	LH_RANCHER_URL         = "lh.rancherurl"
	LH_RANCHER_ACCESS_KEY  = "lh.rancheraccesskey"
	LH_RANCHER_SECRET_KEY  = "lh.ranchersecretkey"
	LH_DEFAULT_VOLUME_SIZE = "lh.defaultvolumesize"
	LH_CONTAINER_NAME      = "lh.containername"

	COMPOSE_VOLUME_NAME = "VOLUME_NAME"
	COMPOSE_VOLUME_SIZE = "VOLUME_SIZE"
	COMPOSE_SLAB_SIZE   = "SLAB_SIZE"
	COMPOSE_CONVOY      = "CONVOY_CONTAINER"

	AFFINITY_LABEL = "io.rancher.scheduler.affinity:container"
)

var (
	log = logrus.WithFields(logrus.Fields{"pkg": "longhorn"})
)

type Driver struct {
	mutex         *sync.RWMutex
	client        *rancherClient.RancherClient
	containerName string
	Device
}

type Device struct {
	Root              string
	DefaultVolumeSize int64
	RancherURL        string
	RancherAccessKey  string
	RancherSecretKey  string
}

func (dev *Device) ConfigFile() (string, error) {
	if dev.Root == "" {
		return "", fmt.Errorf("BUG: Invalid empty device config path")
	}
	return filepath.Join(dev.Root, DRIVER_CONFIG_FILE), nil
}

type Volume struct {
	Size         int64
	Name         string
	MountPoint   string
	PrepareForVM bool
	CreatedTime  string

	configPath string
}

func (v *Volume) Stack(driver *Driver) *Stack {
	sizeString := strconv.FormatInt(v.Size, 10)
	env := map[string]interface{}{
		COMPOSE_SLAB_SIZE:   sizeString,
		COMPOSE_VOLUME_NAME: v.Name,
		COMPOSE_VOLUME_SIZE: sizeString,
		COMPOSE_CONVOY:      driver.containerName,
	}
	return &Stack{
		Client:        driver.client,
		Name:          "longhorn-vol-" + v.Name,
		ExternalId:    "system://longhorn?name=" + v.Name,
		Template:      DockerComposeTemplate,
		Environment:   env,
		ContainerName: driver.containerName,
	}
}

func (v *Volume) ConfigFile() (string, error) {
	if v.Name == "" {
		return "", fmt.Errorf("BUG: Invalid empty volume name")
	}
	if v.configPath == "" {
		return "", fmt.Errorf("BUG: Invalid empty volume config path")
	}
	return filepath.Join(v.configPath, LONGHORN_CFG_PREFIX+VOLUME_CFG_PREFIX+v.Name+CFG_POSTFIX), nil
}

func (v *Volume) GetDevice() (string, error) {
	return fmt.Sprintf(DEV_DIR, v.Name), nil
}

func (v *Volume) GetMountOpts() []string {
	return []string{}
}

func (v *Volume) GenerateDefaultMountPoint() string {
	return filepath.Join(v.configPath, MOUNTS_DIR, v.Name)
}

func (d *Driver) blankVolume(name string) *Volume {
	return &Volume{
		configPath: d.Root,
		Name:       name,
	}
}

func init() {
	Register(DRIVER_NAME, Init)
}

func override(existing, newValue string) string {
	if newValue != "" {
		return newValue
	}
	return existing
}

func Init(root string, config map[string]string) (ConvoyDriver, error) {
	dev := &Device{
		Root: root,
	}
	exists, err := util.ObjectExists(dev)
	if err != nil {
		return nil, err
	}
	if exists {
		if err := util.ObjectLoad(dev); err != nil {
			return nil, err
		}
		dev.RancherURL = override(dev.RancherURL, config[LH_RANCHER_URL])
		dev.RancherAccessKey = override(dev.RancherAccessKey, config[LH_RANCHER_ACCESS_KEY])
		dev.RancherSecretKey = override(dev.RancherSecretKey, config[LH_RANCHER_SECRET_KEY])
	} else {
		if err := util.MkdirIfNotExists(root); err != nil {
			return nil, err
		}

		url := config[LH_RANCHER_URL]
		accessKey := config[LH_RANCHER_ACCESS_KEY]
		secretKey := config[LH_RANCHER_SECRET_KEY]

		if url == "" || accessKey == "" || secretKey == "" {
			return nil, fmt.Errorf("Missing required parameter. lh.rancherurl or lh.rancheraccesskey or lh.ranchersecretkey")
		}

		if _, exists := config[LH_DEFAULT_VOLUME_SIZE]; !exists {
			config[LH_DEFAULT_VOLUME_SIZE] = DEFAULT_VOLUME_SIZE
		}
		volumeSize, err := util.ParseSize(config[LH_DEFAULT_VOLUME_SIZE])
		if err != nil || volumeSize == 0 {
			return nil, fmt.Errorf("Illegal default volume size specified")
		}

		dev = &Device{
			Root:              root,
			RancherURL:        url,
			RancherAccessKey:  accessKey,
			RancherSecretKey:  secretKey,
			DefaultVolumeSize: volumeSize,
		}
	}

	containerName := config[LH_CONTAINER_NAME]
	if containerName == "" {
		handler := metadata.NewClient(RANCHER_METADATA_URL)
		container, err := handler.GetSelfContainer()
		if err != nil {
			return nil, err
		}
		containerName = container.UUID
	}

	log.Debugf("Try to connect to Rancher server at %s [%s:%s]", dev.RancherURL, dev.RancherAccessKey, dev.RancherSecretKey)
	client, err := rancherClient.NewRancherClient(&rancherClient.ClientOpts{
		Url:       dev.RancherURL,
		AccessKey: dev.RancherAccessKey,
		SecretKey: dev.RancherSecretKey,
	})
	if err != nil {
		return nil, fmt.Errorf("Failed to establish connection to Rancher server")
	}

	if err := util.ObjectSave(dev); err != nil {
		return nil, err
	}
	d := &Driver{
		client:        client,
		containerName: containerName,
		Device:        *dev,
		mutex:         &sync.RWMutex{},
	}

	return d, nil
}

func (d *Driver) Name() string {
	return DRIVER_NAME
}

func (d *Driver) Info() (map[string]string, error) {
	return map[string]string{
		"Root":             d.Root,
		"RancherURL":       d.RancherURL,
		"RancherAccessKey": d.RancherAccessKey,
		"RancherSecretKey": d.RancherSecretKey,
	}, nil
}

func (d *Driver) VolumeOps() (VolumeOperations, error) {
	return d, nil
}

func (d *Driver) CreateVolume(req Request) error {
	id := req.Name
	opts := req.Options

	size, err := util.ParseSize(opts[OPT_SIZE])
	if err != nil {
		return err
	}
	if size == 0 {
		size = d.DefaultVolumeSize
	}

	volume := d.blankVolume(id)
	volume.Size = size
	volume.Name = opts[OPT_VOLUME_NAME]
	volume.PrepareForVM, err = strconv.ParseBool(opts[OPT_PREPARE_FOR_VM])
	volume.CreatedTime = util.Now()
	if err != nil {
		return err
	}

	stack := volume.Stack(d)

	if err := d.doCreateVolume(volume, stack, id, opts); err != nil {
		stack.Delete()
		return err
	}

	return nil
}

func (d *Driver) doCreateVolume(volume *Volume, stack *Stack, id string, opts map[string]string) error {
	// Doing find just to see if we are creating versus using an existing stack
	env, err := stack.Find()
	if err != nil {
		return err
	}

	// Always run create because it also ensures that things are active
	if _, err := stack.Create(); err != nil {
		return err
	}

	// If env was nil then we created stack so we need to format
	if env == nil {
		dev, _ := volume.GetDevice()
		err := Backoff(5*time.Minute, fmt.Sprintf("Failed to find %s", dev), func() (bool, error) {
			if _, err := os.Stat(dev); err == nil {
				return true, nil
			}
			return false, nil
		})
		if err != nil {
			return err
		}

		log.Infof("Formatting %s", dev)
		if _, err := util.Execute("mkfs.ext4", []string{dev}); err != nil {
			return err
		}
	}

	return util.ObjectSave(volume)
}

func (d *Driver) DeleteVolume(req Request) error {
	id := req.Name
	opts := req.Options

	volume := d.blankVolume(id)

	if err := util.ObjectLoad(volume); err != nil {
		return err
	}

	if volume.MountPoint != "" {
		return fmt.Errorf("Cannot delete volume %v. It is still mounted", id)
	}

	referenceOnly, _ := strconv.ParseBool(opts[OPT_REFERENCE_ONLY])
	if !referenceOnly {
		log.Debugf("Deleting stack for volume %v", id)
		if err := volume.Stack(d).Delete(); err != nil {
			return err
		}
	}

	return util.ObjectDelete(volume)
}

func (d *Driver) MountVolume(req Request) (string, error) {
	id := req.Name
	opts := req.Options

	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return "", err
	}

	if err := volume.Stack(d).MoveController(); err != nil {
		log.Errorf("Failed to move controller to %s", d.containerName)
		return "", err
	}

	mountPoint, err := util.VolumeMount(volume, opts[OPT_MOUNT_POINT], false)
	if err != nil {
		return "", err
	}

	if volume.PrepareForVM {
		if err := util.MountPointPrepareImageFile(mountPoint, volume.Size); err != nil {
			return "", err
		}
	}

	if err := util.ObjectSave(volume); err != nil {
		return "", err
	}

	return mountPoint, nil
}

func (d *Driver) UmountVolume(req Request) error {
	id := req.Name

	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return err
	}

	if err := util.VolumeUmount(volume); err != nil {
		return err
	}

	return util.ObjectSave(volume)
}

func (d *Driver) MountPoint(req Request) (string, error) {
	volume := d.blankVolume(req.Name)
	return volume.GenerateDefaultMountPoint(), nil
}

func (d *Driver) GetVolumeInfo(id string) (map[string]string, error) {
	volume := d.blankVolume(id)
	if err := util.ObjectLoad(volume); err != nil {
		return nil, err
	}
	return map[string]string{
		"Size":                  strconv.FormatInt(volume.Size, 10),
		OPT_PREPARE_FOR_VM:      strconv.FormatBool(volume.PrepareForVM),
		OPT_VOLUME_CREATED_TIME: volume.CreatedTime,
		OPT_VOLUME_NAME:         volume.Name,
	}, nil
}

func (d *Driver) ListVolume(opts map[string]string) (map[string]map[string]string, error) {
	d.mutex.RLock()
	defer d.mutex.RUnlock()

	volumeIDs, err := d.listVolumeIDs()
	if err != nil {
		return nil, err
	}
	result := map[string]map[string]string{}
	for _, id := range volumeIDs {
		result[id], err = d.GetVolumeInfo(id)
		if err != nil {
			return nil, err
		}
	}
	return result, nil
}

func (device *Device) listVolumeIDs() ([]string, error) {
	return util.ListConfigIDs(device.Root, LONGHORN_CFG_PREFIX+VOLUME_CFG_PREFIX, CFG_POSTFIX)
}

func (d *Driver) SnapshotOps() (SnapshotOperations, error) {
	return nil, fmt.Errorf("Longhorn doesn't support snapshot ops")
}

func (d *Driver) BackupOps() (BackupOperations, error) {
	return nil, fmt.Errorf("Longhorn doesn't support backup ops")
}
<|endoftext|>


<commit_before>package gorm

import (
	"errors"
	"fmt"
	"reflect"
)

type Association struct {
	Scope      *Scope
	PrimaryKey interface{}
	Column     string
	Error      error
	Field      *Field
}

func (association *Association) err(err error) *Association {
	if err != nil {
		association.Error = err
	}
	return association
}

func (association *Association) Find(value interface{}) *Association {
	association.Scope.related(value, association.Column)
	return association.err(association.Scope.db.Error)
}

func (association *Association) Append(values ...interface{}) *Association {
	scope := association.Scope
	field := association.Field
	fieldType := field.Field.Type()

	for _, value := range values {
		reflectvalue := reflect.ValueOf(value)
		if reflectvalue.Kind() == reflect.Ptr {
			if reflectvalue.Elem().Kind() == reflect.Struct {
				if fieldType.Elem().Kind() == reflect.Ptr {
					field.Set(reflect.Append(field.Field, reflectvalue))
				} else if fieldType.Elem().Kind() == reflect.Struct {
					field.Set(reflect.Append(field.Field, reflectvalue.Elem()))
				}
			} else if reflectvalue.Elem().Kind() == reflect.Slice {
				if fieldType.Elem().Kind() == reflect.Ptr {
					field.Set(reflect.AppendSlice(field.Field, reflectvalue))
				} else if fieldType.Elem().Kind() == reflect.Struct {
					field.Set(reflect.AppendSlice(field.Field, reflectvalue.Elem()))
				}
			}
		} else if reflectvalue.Kind() == reflect.Struct && fieldType.Elem().Kind() == reflect.Struct {
			field.Set(reflect.Append(field.Field, reflectvalue))
		} else if reflectvalue.Kind() == reflect.Slice && fieldType.Elem() == reflectvalue.Type().Elem() {
			field.Set(reflect.AppendSlice(field.Field, reflectvalue))
		} else {
			association.err(errors.New("invalid association type"))
		}
	}
	scope.callCallbacks(scope.db.parent.callback.updates)
	return association.err(scope.db.Error)
}

func (association *Association) getPrimaryKeys(values ...interface{}) []interface{} {
	primaryKeys := []interface{}{}
	scope := association.Scope

	for _, value := range values {
		reflectValue := reflect.ValueOf(value)
		if reflectValue.Kind() == reflect.Ptr {
			reflectValue = reflectValue.Elem()
		}
		if reflectValue.Kind() == reflect.Slice {
			for i := 0; i < reflectValue.Len(); i++ {
				newScope := scope.New(reflectValue.Index(i).Interface())
				primaryKey := newScope.PrimaryKeyValue()
				if !reflect.DeepEqual(reflect.ValueOf(primaryKey), reflect.Zero(reflect.ValueOf(primaryKey).Type())) {
					primaryKeys = append(primaryKeys, primaryKey)
				}
			}
		} else if reflectValue.Kind() == reflect.Struct {
			newScope := scope.New(value)
			primaryKey := newScope.PrimaryKeyValue()
			if !reflect.DeepEqual(reflect.ValueOf(primaryKey), reflect.Zero(reflect.ValueOf(primaryKey).Type())) {
				primaryKeys = append(primaryKeys, primaryKey)
			}
		}
	}
	return primaryKeys
}

func (association *Association) Delete(values ...interface{}) *Association {
	primaryKeys := association.getPrimaryKeys(values...)

	if len(primaryKeys) == 0 {
		association.err(errors.New("no primary key found"))
	} else {
		relationship := association.Field.Relationship
		// many to many
		if relationship.Kind == "many_to_many" {
			whereSql := fmt.Sprintf("%v.%v IN (?)", relationship.JoinTable, association.Scope.Quote(ToSnake(relationship.AssociationForeignKey)))
			association.Scope.db.Model("").Table(relationship.JoinTable).Where(whereSql, primaryKeys).Delete("")
		} else {
			association.err(errors.New("delete only support many to many"))
		}
	}
	return association
}

func (association *Association) Replace(values ...interface{}) *Association {
	relationship := association.Field.Relationship
	scope := association.Scope
	if relationship.Kind == "many_to_many" {
		field := association.Field.Field

		oldPrimaryKeys := association.getPrimaryKeys(field.Interface())
		association.Append(values...)
		newPrimaryKeys := association.getPrimaryKeys(field.Interface())

		var addedPrimaryKeys = []interface{}{}
		for _, new := range newPrimaryKeys {
			hasEqual := false
			for _, old := range oldPrimaryKeys {
				if reflect.DeepEqual(new, old) {
					hasEqual = true
					break
				}
			}
			if !hasEqual {
				addedPrimaryKeys = append(addedPrimaryKeys, new)
			}
		}
		for _, primaryKey := range association.getPrimaryKeys(values...) {
			addedPrimaryKeys = append(addedPrimaryKeys, primaryKey)
		}

		whereSql := fmt.Sprintf("%v.%v NOT IN (?)", relationship.JoinTable, scope.Quote(ToSnake(relationship.AssociationForeignKey)))
		scope.db.Model("").Table(relationship.JoinTable).Where(whereSql, addedPrimaryKeys).Delete("")
	} else {
		association.err(errors.New("replace only support many to many"))
	}
	return association
}

func (association *Association) Clear() *Association {
	relationship := association.Field.Relationship
	scope := association.Scope
	if relationship.Kind == "many_to_many" {
		whereSql := fmt.Sprintf("%v.%v = ?", relationship.JoinTable, scope.Quote(ToSnake(relationship.ForeignKey)))
		scope.db.Model("").Table(relationship.JoinTable).Where(whereSql, association.PrimaryKey).Delete("")
	} else {
		association.err(errors.New("clear only support many to many"))
	}
	return association
}

func (association *Association) Count() int {
	count := -1
	relationship := association.Field.Relationship
	scope := association.Scope
	field := association.Field.Field
	fieldValue := field.Interface()
	newScope := scope.New(fieldValue)

	if relationship.Kind == "many_to_many" {
		whereSql := fmt.Sprintf("%v.%v IN (SELECT %v.%v FROM %v WHERE %v.%v = ?)",
			newScope.QuotedTableName(),
			scope.Quote(newScope.PrimaryKey()),
			relationship.JoinTable,
			scope.Quote(ToSnake(relationship.AssociationForeignKey)),
			relationship.JoinTable,
			relationship.JoinTable,
			scope.Quote(ToSnake(relationship.ForeignKey)))
		scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, association.PrimaryKey).Count(&count)
	} else if relationship.Kind == "has_many" {
		whereSql := fmt.Sprintf("%v.%v = ?", newScope.QuotedTableName(), newScope.Quote(ToSnake(relationship.ForeignKey)))
		scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, association.PrimaryKey).Count(&count)
	} else if relationship.Kind == "has_one" {
		whereSql := fmt.Sprintf("%v.%v = ?", newScope.QuotedTableName(), relationship.ForeignKey)
		scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, association.PrimaryKey).Count(&count)
	} else if relationship.Kind == "belongs_to" {
		if v, err := scope.FieldValueByName(association.Column); err == nil {
			whereSql := fmt.Sprintf("%v.%v = ?", newScope.QuotedTableName(), relationship.ForeignKey)
			scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, v).Count(&count)
		}
	}

	return count
}
<commit_msg>.Count() should always use ToSnake'd foreign keys.<commit_after>package gorm

import (
	"errors"
	"fmt"
	"reflect"
)

type Association struct {
	Scope      *Scope
	PrimaryKey interface{}
	Column     string
	Error      error
	Field      *Field
}

func (association *Association) err(err error) *Association {
	if err != nil {
		association.Error = err
	}
	return association
}

func (association *Association) Find(value interface{}) *Association {
	association.Scope.related(value, association.Column)
	return association.err(association.Scope.db.Error)
}

func (association *Association) Append(values ...interface{}) *Association {
	scope := association.Scope
	field := association.Field
	fieldType := field.Field.Type()

	for _, value := range values {
		reflectvalue := reflect.ValueOf(value)
		if reflectvalue.Kind() == reflect.Ptr {
			if reflectvalue.Elem().Kind() == reflect.Struct {
				if fieldType.Elem().Kind() == reflect.Ptr {
					field.Set(reflect.Append(field.Field, reflectvalue))
				} else if fieldType.Elem().Kind() == reflect.Struct {
					field.Set(reflect.Append(field.Field, reflectvalue.Elem()))
				}
			} else if reflectvalue.Elem().Kind() == reflect.Slice {
				if fieldType.Elem().Kind() == reflect.Ptr {
					field.Set(reflect.AppendSlice(field.Field, reflectvalue))
				} else if fieldType.Elem().Kind() == reflect.Struct {
					field.Set(reflect.AppendSlice(field.Field, reflectvalue.Elem()))
				}
			}
		} else if reflectvalue.Kind() == reflect.Struct && fieldType.Elem().Kind() == reflect.Struct {
			field.Set(reflect.Append(field.Field, reflectvalue))
		} else if reflectvalue.Kind() == reflect.Slice && fieldType.Elem() == reflectvalue.Type().Elem() {
			field.Set(reflect.AppendSlice(field.Field, reflectvalue))
		} else {
			association.err(errors.New("invalid association type"))
		}
	}
	scope.callCallbacks(scope.db.parent.callback.updates)
	return association.err(scope.db.Error)
}

func (association *Association) getPrimaryKeys(values ...interface{}) []interface{} {
	primaryKeys := []interface{}{}
	scope := association.Scope

	for _, value := range values {
		reflectValue := reflect.ValueOf(value)
		if reflectValue.Kind() == reflect.Ptr {
			reflectValue = reflectValue.Elem()
		}
		if reflectValue.Kind() == reflect.Slice {
			for i := 0; i < reflectValue.Len(); i++ {
				newScope := scope.New(reflectValue.Index(i).Interface())
				primaryKey := newScope.PrimaryKeyValue()
				if !reflect.DeepEqual(reflect.ValueOf(primaryKey), reflect.Zero(reflect.ValueOf(primaryKey).Type())) {
					primaryKeys = append(primaryKeys, primaryKey)
				}
			}
		} else if reflectValue.Kind() == reflect.Struct {
			newScope := scope.New(value)
			primaryKey := newScope.PrimaryKeyValue()
			if !reflect.DeepEqual(reflect.ValueOf(primaryKey), reflect.Zero(reflect.ValueOf(primaryKey).Type())) {
				primaryKeys = append(primaryKeys, primaryKey)
			}
		}
	}
	return primaryKeys
}

func (association *Association) Delete(values ...interface{}) *Association {
	primaryKeys := association.getPrimaryKeys(values...)

	if len(primaryKeys) == 0 {
		association.err(errors.New("no primary key found"))
	} else {
		relationship := association.Field.Relationship
		// many to many
		if relationship.Kind == "many_to_many" {
			whereSql := fmt.Sprintf("%v.%v IN (?)", relationship.JoinTable, association.Scope.Quote(ToSnake(relationship.AssociationForeignKey)))
			association.Scope.db.Model("").Table(relationship.JoinTable).Where(whereSql, primaryKeys).Delete("")
		} else {
			association.err(errors.New("delete only support many to many"))
		}
	}
	return association
}

func (association *Association) Replace(values ...interface{}) *Association {
	relationship := association.Field.Relationship
	scope := association.Scope
	if relationship.Kind == "many_to_many" {
		field := association.Field.Field

		oldPrimaryKeys := association.getPrimaryKeys(field.Interface())
		association.Append(values...)
		newPrimaryKeys := association.getPrimaryKeys(field.Interface())

		var addedPrimaryKeys = []interface{}{}
		for _, new := range newPrimaryKeys {
			hasEqual := false
			for _, old := range oldPrimaryKeys {
				if reflect.DeepEqual(new, old) {
					hasEqual = true
					break
				}
			}
			if !hasEqual {
				addedPrimaryKeys = append(addedPrimaryKeys, new)
			}
		}
		for _, primaryKey := range association.getPrimaryKeys(values...) {
			addedPrimaryKeys = append(addedPrimaryKeys, primaryKey)
		}

		whereSql := fmt.Sprintf("%v.%v NOT IN (?)", relationship.JoinTable, scope.Quote(ToSnake(relationship.AssociationForeignKey)))
		scope.db.Model("").Table(relationship.JoinTable).Where(whereSql, addedPrimaryKeys).Delete("")
	} else {
		association.err(errors.New("replace only support many to many"))
	}
	return association
}

func (association *Association) Clear() *Association {
	relationship := association.Field.Relationship
	scope := association.Scope
	if relationship.Kind == "many_to_many" {
		whereSql := fmt.Sprintf("%v.%v = ?", relationship.JoinTable, scope.Quote(ToSnake(relationship.ForeignKey)))
		scope.db.Model("").Table(relationship.JoinTable).Where(whereSql, association.PrimaryKey).Delete("")
	} else {
		association.err(errors.New("clear only support many to many"))
	}
	return association
}

func (association *Association) Count() int {
	count := -1
	relationship := association.Field.Relationship
	scope := association.Scope
	field := association.Field.Field
	fieldValue := field.Interface()
	newScope := scope.New(fieldValue)

	if relationship.Kind == "many_to_many" {
		whereSql := fmt.Sprintf("%v.%v IN (SELECT %v.%v FROM %v WHERE %v.%v = ?)",
			newScope.QuotedTableName(),
			scope.Quote(newScope.PrimaryKey()),
			relationship.JoinTable,
			scope.Quote(ToSnake(relationship.AssociationForeignKey)),
			relationship.JoinTable,
			relationship.JoinTable,
			scope.Quote(ToSnake(relationship.ForeignKey)))
		scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, association.PrimaryKey).Count(&count)
	} else if relationship.Kind == "has_many" || relationship.Kind == "has_one" {
		whereSql := fmt.Sprintf("%v.%v = ?", newScope.QuotedTableName(), newScope.Quote(ToSnake(relationship.ForeignKey)))
		scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, association.PrimaryKey).Count(&count)
	} else if relationship.Kind == "belongs_to" {
		if v, err := scope.FieldValueByName(association.Column); err == nil {
			whereSql := fmt.Sprintf("%v.%v = ?", newScope.QuotedTableName(), newScope.Quote(ToSnake(relationship.ForeignKey)))
			scope.db.Model("").Table(newScope.QuotedTableName()).Where(whereSql, v).Count(&count)
		}
	}

	return count
}
<|endoftext|>


<commit_before>package analyze

import (
	"math"
	"math/rand"

	"github.com/gonum/matrix/mat64"
	grid "github.com/phil-mansfield/shellfish/los/analyze/ellipse_grid"
	intr "github.com/phil-mansfield/shellfish/math/interpolate"
	"github.com/phil-mansfield/shellfish/math/sort"
)

// Shell is a function that returns the radius of a shell at a given set of
// angles.
//
// Unless otherwise specified, all quantities are calculated through Monte
// Carlo solid angle sampling.
type Shell func(phi, theta float64) float64

// randomAngle returns and angle chosen uniformly at random.
func randomAngle() (phi, theta float64) {
	u, v := rand.Float64(), rand.Float64()
	return 2 * math.Pi * u, math.Acos(2*v - 1)
}

// cartesian converts a tuple of radial coordinates to cartesian coordinates.
func cartesian(phi, theta, r float64) (x, y, z float64) {
	sinP, cosP := math.Sincos(phi)
	sinT, cosT := math.Sincos(theta)
	return r * sinT * cosP, r * sinT * sinP, r * cosT
}

// CartesianSampledVolume returns the volume of a Shell. The volume is
// calculated by Monte Carlo sampling of a sphere of radius rMax.
//
// This is slower than Volume for most shell shapes.
func (s Shell) CartesianSampledVolume(samples int, rMax float64) float64 {
	inside := 0
	for i := 0; i < samples; i++ {
		x := rand.Float64()*(2*rMax) - rMax
		y := rand.Float64()*(2*rMax) - rMax
		z := rand.Float64()*(2*rMax) - rMax

		r := math.Sqrt(x*x + y*y + z*z)
		phi := math.Atan2(y, x)
		th := math.Acos(z / r)

		rs := s(phi, th)
		if r < rs {
			inside++
		}
	}

	return float64(inside) / float64(samples) * (rMax * rMax * rMax * 8)
}

// Volume returns the volume of Shell.
func (s Shell) Volume(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r * r
	}
	r3 := sum / float64(samples)
	return r3 * 4 * (math.Pi / 3)
}

// MeanRadius returns the angle-weighted mean radius of a Shell.
func (s Shell) MeanRadius(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, th := randomAngle()
		r := s(phi, th)
		sum += r
	}
	return sum / float64(samples)
}

// MedianRadius returns the angle-weighted median radius of a Shell.
func (s Shell) MedianRadius(samples int) float64 {
	rs := make([]float64, samples)
	for i := range rs {
		phi, th := randomAngle()
		rs[i] = s(phi, th)
	}
	return sort.Median(rs, rs)
}

// triSort returns x, y, and z in sorted order and returns the argument
// index of the largest value.
func trisort(x, y, z float64) (a, b, c float64, aIdx int) {
	var p, q float64
	switch {
	case x > y && x > z:
		a, p, q, aIdx = x, y, z, 0
	case y > x && y > z:
		a, p, q, aIdx = y, z, x, 1
	default:
		a, p, q, aIdx = z, x, y, 2
	}

	if p > q {
		return a, p, q, aIdx
	} else {
		return a, q, p, aIdx
	}
}

// Axes calculates the moment of inertia-equivalent axes of a Shell as well
// as the direction of the major axis.
func (s Shell) Axes(samples int) (a, b, c float64, aVec [3]float64) {

	// Temporarily approximate a constant-density ellipsoidal shell as
	// a homoeoid.

	nxx, nyy, nzz := 0.0, 0.0, 0.0
	nxy, nyz, nzx := 0.0, 0.0, 0.0
	nx, ny, nz := 0.0, 0.0, 0.0
	norm := 0.0

	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		area := r * r / cosNorm(s, phi, theta)
		x, y, z := cartesian(phi, theta, r)

		nxx += area * x * x
		nyy += area * y * y
		nzz += area * z * z
		nxy += area * x * y
		nyz += area * y * z
		nzx += area * z * x
		nx += area * x
		ny += area * y
		nz += area * z

		norm += area
	}

	nxx, nyy, nzz = nxx/norm, nyy/norm, nzz/norm
	nxy, nyz, nzx = nxy/norm, nyz/norm, nzx/norm
	nx, ny, nz = nx/norm, ny/norm, nz/norm

	mat := mat64.NewDense(3, 3, []float64{
		nyy + nzz - ny*ny - nz*nz, -nxy + nx*ny, -nzx + nz*nx,
		-nxy + nx*ny, nxx + nzz - nx*nx - nz*nz, -nyz + ny*nz,
		-nzx + nz*nx, -nyz + ny*nz, nxx + nyy - nx*nx - ny*ny,
	})

	eigen := &mat64.Eigen{}
	ok := eigen.Factorize(mat, false)
	if !ok {
		panic("Could not factorize inertia tensor.")
	}

	vals := eigen.Values(nil)
	vecs := eigen.Vectors()

	Ix, Iy, Iz := real(vals[0]), real(vals[1]), real(vals[2])
	ax2 := 3 * (Iy + Iz - Ix) / 2
	ay2 := 3*Iy - ax2
	az2 := 3*Iz - ax2

	// Correct the axis ratios via empirically derived tables.

	// TODO: Fix naming conventions.

	c, b, a, aIdx := trisort(math.Sqrt(ax2), math.Sqrt(ay2), math.Sqrt(az2))
	ac, bc := a/c, b/c

	// TODO: This function is just barely not thread safe.

	acRatio := axisInterpolators.acRatio.Eval(ac, bc)
	bcRatio := axisInterpolators.bcRatio.Eval(ac, bc)
	cRatio := axisInterpolators.cRatio.Eval(ac, bc)

	aVec = [3]float64{
		vecs.At(0, aIdx), vecs.At(1, aIdx), vecs.At(2, aIdx),
	}
	norm = math.Sqrt(aVec[0]*aVec[0] + aVec[1]*aVec[1] + aVec[2]*aVec[2])
	aVec[0] = aVec[0] / norm
	aVec[1] = aVec[1] / norm
	aVec[2] = aVec[2] / norm

	c = cRatio * c
	return c, bcRatio * bc * c, acRatio * ac * c, aVec
}

// cosNorm reutrns the cosine of the angle between \hat{r} and the normal
// vector of the Shell surface at a particular angle.
func cosNorm(s Shell, phi, theta float64) float64 {
	dp, dt := 1e-3, 1e-3
	r00 := s(phi-dp, theta-dt)
	x00, y00, z00 := cartesian(phi-dp, theta-dt, r00)
	r01 := s(phi-dp, theta+dt)
	x01, y01, z01 := cartesian(phi-dp, theta+dt, r01)
	r10 := s(phi+dp, theta-dt)
	x10, y10, z10 := cartesian(phi+dp, theta-dt, r10)
	r11 := s(phi+dp, theta+dt)
	x11, y11, z11 := cartesian(phi+dp, theta+dt, r11)

	dxa, dya, dza := x00-x11, y00-y11, z00-z11
	dxb, dyb, dzb := x01-x10, y01-y10, z01-z10

	// normal vector
	xn := dya*dzb - dza*dyb
	yn := dza*dxb - dxa*dzb
	zn := dxa*dyb - dya*dxb
	norm := math.Sqrt(xn*xn + yn*yn + zn*zn)

	xn, yn, zn = xn/norm, yn/norm, zn/norm
	xl, yl, zl := cartesian(phi, theta, 1)
	return xl*xn + yl*yn + zl*zn
}

// SurfaceArea returns the surface area of a shell.
func (s Shell) SurfaceArea(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r / cosNorm(s, phi, theta)
	}
	return sum / float64(samples) * 4 * math.Pi
}

// DiffVolume returns the volume of the space between two Shells, s1 and s2.
func (s1 Shell) DiffVolume(s2 Shell, samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		r := (r1 + r2) / 2
		dr := math.Abs(r1 - r2)
		sum += dr * r * r
	}
	return sum / float64(samples) * (4 * math.Pi) / 3
}

// MaxDiff returns the maximum radial distance between two Shells along
// any line of sight.
func (s1 Shell) MaxDiff(s2 Shell, samples int) float64 {
	max := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		dr := math.Abs(r1 - r2)
		if dr > max {
			max = dr
		}
	}
	return max
}

// RadialRange returns the maximum and minimum radius of a Shell.
func (s Shell) RadialRange(samples int) (low, high float64) {
	phi, theta := randomAngle()
	low = s(phi, theta)
	high = low
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		if r > high {
			high = r
		}
		if r < low {
			low = r
		}
	}
	return low, high
}

// RadiusHistogram returns a normalized angle-weighted histogram of the radii
// of a Shell.
func (s Shell) RadiusHistogram(
	samples, bins int, rMin, rMax float64,
) (rs, ns []float64) {
	rs, ns = make([]float64, bins), make([]float64, bins)
	dr := (rMax - rMin) / float64(bins)
	for i := range rs {
		rs[i] = rMin + dr*(float64(i)+0.5)
	}

	count := 0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		ri := (r - rMin) / dr
		if ri < 0 {
			continue
		}
		idx := int(ri)
		if idx >= bins {
			continue
		}
		ns[idx]++
		count++
	}

	for i := range ns {
		ns[i] /= float64(count) * dr
	}

	return rs, ns
}

// Contains returns true if a Shell contains a point and false otherwise. The
// point must be in a coordinate system in which the Shell is at (0, 0, 0).
func (s Shell) Contains(x, y, z float64) bool {
	r := math.Sqrt(x*x + y*y + z*z)
	phi := math.Atan2(y, x)
	theta := math.Acos(z / r)
	return s(phi, theta) > r
}

// axisInterpolators contains state needed for Shell.Axes().
var axisInterpolators = struct {
	acRatio, bcRatio, cRatio intr.BiInterpolator
}{}

func init() {
	axisInterpolators.acRatio = intr.NewBiCubic(
		grid.ACRatios, grid.BCRatios,
		grid.ACCorrectionGrid,
	)
	axisInterpolators.bcRatio = intr.NewBiCubic(
		grid.ACRatios, grid.BCRatios,
		grid.BCCorrectionGrid,
	)
	axisInterpolators.cRatio = intr.NewBiCubic(
		grid.ACRatios, grid.BCRatios,
		grid.CCorrectionGrid,
	)
}
<commit_msg>Wrote AngularFractionProfile method on Shell type.<commit_after>package analyze

import (
	"math"
	"math/rand"

	"github.com/gonum/matrix/mat64"
	grid "github.com/phil-mansfield/shellfish/los/analyze/ellipse_grid"
	intr "github.com/phil-mansfield/shellfish/math/interpolate"
	"github.com/phil-mansfield/shellfish/math/sort"
)

// Shell is a function that returns the radius of a shell at a given set of
// angles.
//
// Unless otherwise specified, all quantities are calculated through Monte
// Carlo solid angle sampling.
type Shell func(phi, theta float64) float64

// randomAngle returns and angle chosen uniformly at random.
func randomAngle() (phi, theta float64) {
	u, v := rand.Float64(), rand.Float64()
	return 2 * math.Pi * u, math.Acos(2*v - 1)
}

// cartesian converts a tuple of radial coordinates to cartesian coordinates.
func cartesian(phi, theta, r float64) (x, y, z float64) {
	sinP, cosP := math.Sincos(phi)
	sinT, cosT := math.Sincos(theta)
	return r * sinT * cosP, r * sinT * sinP, r * cosT
}

// CartesianSampledVolume returns the volume of a Shell. The volume is
// calculated by Monte Carlo sampling of a sphere of radius rMax.
//
// This is slower than Volume for most shell shapes.
func (s Shell) CartesianSampledVolume(samples int, rMax float64) float64 {
	inside := 0
	for i := 0; i < samples; i++ {
		x := rand.Float64()*(2*rMax) - rMax
		y := rand.Float64()*(2*rMax) - rMax
		z := rand.Float64()*(2*rMax) - rMax

		r := math.Sqrt(x*x + y*y + z*z)
		phi := math.Atan2(y, x)
		th := math.Acos(z / r)

		rs := s(phi, th)
		if r < rs {
			inside++
		}
	}

	return float64(inside) / float64(samples) * (rMax * rMax * rMax * 8)
}

// Volume returns the volume of Shell.
func (s Shell) Volume(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r * r
	}
	r3 := sum / float64(samples)
	return r3 * 4 * (math.Pi / 3)
}

// MeanRadius returns the angle-weighted mean radius of a Shell.
func (s Shell) MeanRadius(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, th := randomAngle()
		r := s(phi, th)
		sum += r
	}
	return sum / float64(samples)
}

// MedianRadius returns the angle-weighted median radius of a Shell.
func (s Shell) MedianRadius(samples int) float64 {
	rs := make([]float64, samples)
	for i := range rs {
		phi, th := randomAngle()
		rs[i] = s(phi, th)
	}
	return sort.Median(rs, rs)
}

// triSort returns x, y, and z in sorted order and returns the argument
// index of the largest value.
func trisort(x, y, z float64) (a, b, c float64, aIdx int) {
	var p, q float64
	switch {
	case x > y && x > z:
		a, p, q, aIdx = x, y, z, 0
	case y > x && y > z:
		a, p, q, aIdx = y, z, x, 1
	default:
		a, p, q, aIdx = z, x, y, 2
	}

	if p > q {
		return a, p, q, aIdx
	} else {
		return a, q, p, aIdx
	}
}

// Axes calculates the moment of inertia-equivalent axes of a Shell as well
// as the direction of the major axis.
func (s Shell) Axes(samples int) (a, b, c float64, aVec [3]float64) {

	// Temporarily approximate a constant-density ellipsoidal shell as
	// a homoeoid.

	nxx, nyy, nzz := 0.0, 0.0, 0.0
	nxy, nyz, nzx := 0.0, 0.0, 0.0
	nx, ny, nz := 0.0, 0.0, 0.0
	norm := 0.0

	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		area := r * r / cosNorm(s, phi, theta)
		x, y, z := cartesian(phi, theta, r)

		nxx += area * x * x
		nyy += area * y * y
		nzz += area * z * z
		nxy += area * x * y
		nyz += area * y * z
		nzx += area * z * x
		nx += area * x
		ny += area * y
		nz += area * z

		norm += area
	}

	nxx, nyy, nzz = nxx/norm, nyy/norm, nzz/norm
	nxy, nyz, nzx = nxy/norm, nyz/norm, nzx/norm
	nx, ny, nz = nx/norm, ny/norm, nz/norm

	mat := mat64.NewDense(3, 3, []float64{
		nyy + nzz - ny*ny - nz*nz, -nxy + nx*ny, -nzx + nz*nx,
		-nxy + nx*ny, nxx + nzz - nx*nx - nz*nz, -nyz + ny*nz,
		-nzx + nz*nx, -nyz + ny*nz, nxx + nyy - nx*nx - ny*ny,
	})

	eigen := &mat64.Eigen{}
	ok := eigen.Factorize(mat, false)
	if !ok {
		panic("Could not factorize inertia tensor.")
	}

	vals := eigen.Values(nil)
	vecs := eigen.Vectors()

	Ix, Iy, Iz := real(vals[0]), real(vals[1]), real(vals[2])
	ax2 := 3 * (Iy + Iz - Ix) / 2
	ay2 := 3*Iy - ax2
	az2 := 3*Iz - ax2

	// Correct the axis ratios via empirically derived tables.

	// TODO: Fix naming conventions.

	c, b, a, aIdx := trisort(math.Sqrt(ax2), math.Sqrt(ay2), math.Sqrt(az2))
	ac, bc := a/c, b/c

	// TODO: This function is just barely not thread safe.

	acRatio := axisInterpolators.acRatio.Eval(ac, bc)
	bcRatio := axisInterpolators.bcRatio.Eval(ac, bc)
	cRatio := axisInterpolators.cRatio.Eval(ac, bc)

	aVec = [3]float64{
		vecs.At(0, aIdx), vecs.At(1, aIdx), vecs.At(2, aIdx),
	}
	norm = math.Sqrt(aVec[0]*aVec[0] + aVec[1]*aVec[1] + aVec[2]*aVec[2])
	aVec[0] = aVec[0] / norm
	aVec[1] = aVec[1] / norm
	aVec[2] = aVec[2] / norm

	c = cRatio * c
	return c, bcRatio * bc * c, acRatio * ac * c, aVec
}

// cosNorm reutrns the cosine of the angle between \hat{r} and the normal
// vector of the Shell surface at a particular angle.
func cosNorm(s Shell, phi, theta float64) float64 {
	dp, dt := 1e-3, 1e-3
	r00 := s(phi-dp, theta-dt)
	x00, y00, z00 := cartesian(phi-dp, theta-dt, r00)
	r01 := s(phi-dp, theta+dt)
	x01, y01, z01 := cartesian(phi-dp, theta+dt, r01)
	r10 := s(phi+dp, theta-dt)
	x10, y10, z10 := cartesian(phi+dp, theta-dt, r10)
	r11 := s(phi+dp, theta+dt)
	x11, y11, z11 := cartesian(phi+dp, theta+dt, r11)

	dxa, dya, dza := x00-x11, y00-y11, z00-z11
	dxb, dyb, dzb := x01-x10, y01-y10, z01-z10

	// normal vector
	xn := dya*dzb - dza*dyb
	yn := dza*dxb - dxa*dzb
	zn := dxa*dyb - dya*dxb
	norm := math.Sqrt(xn*xn + yn*yn + zn*zn)

	xn, yn, zn = xn/norm, yn/norm, zn/norm
	xl, yl, zl := cartesian(phi, theta, 1)
	return xl*xn + yl*yn + zl*zn
}

// SurfaceArea returns the surface area of a shell.
func (s Shell) SurfaceArea(samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		sum += r * r / cosNorm(s, phi, theta)
	}
	return sum / float64(samples) * 4 * math.Pi
}

// DiffVolume returns the volume of the space between two Shells, s1 and s2.
func (s1 Shell) DiffVolume(s2 Shell, samples int) float64 {
	sum := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		r := (r1 + r2) / 2
		dr := math.Abs(r1 - r2)
		sum += dr * r * r
	}
	return sum / float64(samples) * (4 * math.Pi) / 3
}

// MaxDiff returns the maximum radial distance between two Shells along
// any line of sight.
func (s1 Shell) MaxDiff(s2 Shell, samples int) float64 {
	max := 0.0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r1, r2 := s1(phi, theta), s2(phi, theta)
		dr := math.Abs(r1 - r2)
		if dr > max {
			max = dr
		}
	}
	return max
}

// RadialRange returns the maximum and minimum radius of a Shell.
func (s Shell) RadialRange(samples int) (low, high float64) {
	phi, theta := randomAngle()
	low = s(phi, theta)
	high = low
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		if r > high {
			high = r
		}
		if r < low {
			low = r
		}
	}
	return low, high
}

// RadiusHistogram returns a normalized angle-weighted histogram of the radii
// of a Shell.
func (s Shell) RadiusHistogram(
	samples, bins int, rMin, rMax float64,
) (rs, ns []float64) {
	rs, ns = make([]float64, bins), make([]float64, bins)
	dr := (rMax - rMin) / float64(bins)
	for i := range rs {
		rs[i] = rMin + dr*(float64(i)+0.5)
	}

	count := 0
	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		r := s(phi, theta)
		ri := (r - rMin) / dr
		if ri < 0 {
			continue
		}
		idx := int(ri)
		if idx >= bins {
			continue
		}
		ns[idx]++
		count++
	}

	for i := range ns {
		ns[i] /= float64(count) * dr
	}

	return rs, ns
}

func (s Shell) AngularFractionProfile(
	samples, bins int, rMin, rMax float64,
) (rs, fs []float64) {
	rs, fs = make([]float64, bins), make([]float64, bins)
	ns := make([]int, bins)

	lrMin, lrMax := math.Log(rMin), math.Log(rMax)
	dlr := (lrMax + lrMin) / float64(bins)
	for i := range rs {
		rs[i] = math.Exp(lrMin + (float64(i) + 0.5) * dlr)
	}

	for i := 0; i < samples; i++ {
		phi, theta := randomAngle()
		lr := math.Log(s(phi, theta))
		lri := int((lr - lrMin) / dlr)
		if lri < 0 || lri >= bins {
			continue
		}
		ns[lri]++
	}

	// reverse cumulative sum
	for i := bins - 2; i >= 0; i-- {
		ns[i] += ns[i+1]
	}

	for i := 0; i < bins; i++ {
		fs[i] = float64(ns[i]) / float64(ns[0])
	}

	return rs, fs
}

// Contains returns true if a Shell contains a point and false otherwise. The
// point must be in a coordinate system in which the Shell is at (0, 0, 0).
func (s Shell) Contains(x, y, z float64) bool {
	r := math.Sqrt(x*x + y*y + z*z)
	phi := math.Atan2(y, x)
	theta := math.Acos(z / r)
	return s(phi, theta) > r
}

// axisInterpolators contains state needed for Shell.Axes().
var axisInterpolators = struct {
	acRatio, bcRatio, cRatio intr.BiInterpolator
}{}

func init() {
	axisInterpolators.acRatio = intr.NewBiCubic(
		grid.ACRatios, grid.BCRatios,
		grid.ACCorrectionGrid,
	)
	axisInterpolators.bcRatio = intr.NewBiCubic(
		grid.ACRatios, grid.BCRatios,
		grid.BCCorrectionGrid,
	)
	axisInterpolators.cRatio = intr.NewBiCubic(
		grid.ACRatios, grid.BCRatios,
		grid.CCorrectionGrid,
	)
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/asaskevich/govalidator"
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/kyleterry/sufr/config"
)

var staticHandler = http.StripPrefix(
	"/static/",
	http.FileServer(http.Dir("static")))

func urlIndexHandler(w http.ResponseWriter, r *http.Request) error {
	rawbytes, err := database.GetAll(config.BucketNameURL)
	urls := DeserializeURLs(rawbytes...)
	if err != nil {
		return err
	}

	renderTemplate(w, "url-index", map[string]interface{}{
		"ActiveTab": "urls",
		"Title":     "URLs",
		"Count":     len(urls),
		"URLs":      urls,
	})
	return nil
}

func urlNewHandler(w http.ResponseWriter, r *http.Request) error {
	ctx := context.Get(r, TemplateContext).(map[string]interface{})
	ctx["ActiveTab"] = "urls"
	fmt.Println(ctx)
	renderTemplate(w, "url-new", ctx)
	return nil
}

func urlSubmitHandler(w http.ResponseWriter, r *http.Request) error {
	urlstring := r.FormValue("url")
	tagsstring := r.FormValue("tags")
	if !govalidator.IsURL(urlstring) {
		errormessage := "URL is required"
		if urlstring != "" {
			errormessage = fmt.Sprintf("URL \"%s\" is not valid", urlstring)
		}
		session, err := store.Get(r, "flashes")
		if err != nil {
			return err
		}
		session.AddFlash(errormessage, "danger")
		session.Save(r, w)
		http.Redirect(w, r, reverse("url-new"), http.StatusSeeOther)
		return nil
	}

	title, err := getPageTitle(urlstring)
	if err != nil {
		// Add flash about title not being fetchable
		// or alternatively add logic for detecting content type because it might be
		// an image or PDF
	}

	url := &URL{
		URL:       urlstring,
		Title:     title,
		CreatedAt: time.Now(),
		UpdatedAt: time.Now(),
	}

	err = url.SaveWithTags(tagsstring)
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-view", "id", url.ID), http.StatusSeeOther)
	return nil
}

func urlViewHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}
	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)

	renderTemplate(w, "url-view", map[string]interface{}{
		"ActiveTab": "urls",
		"Url":       url,
	})
	return nil
}

func urlEditHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}
	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)

	renderTemplate(w, "url-edit", map[string]interface{}{
		"ActiveTab": "urls",
		"Url":       url,
	})

	return nil
}

func urlSaveHandler(w http.ResponseWriter, r *http.Request) error {
	titlestring := r.FormValue("title")
	tagsstring := r.FormValue("tags")
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error fetching data: %s", err), http.StatusInternalServerError)
	}

	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)
	url.Title = titlestring
	url.UpdatedAt = time.Now()
	err = url.SaveWithTags(tagsstring)
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-view", "id", url.ID), http.StatusSeeOther)
	return nil
}

func urlDeleteHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}

	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)
	err = url.Delete()
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-index"), http.StatusSeeOther)
	return nil
}

func tagIndexHandler(w http.ResponseWriter, r *http.Request) error {
	rawbytes, err := database.GetAll(config.BucketNameTag)
	tags := DeserializeTags(rawbytes...)
	if err != nil {
		return err
	}
	renderTemplate(w, "tag-index", map[string]interface{}{
		"ActiveTab": "tags",
		"Tags":      tags,
	})
	return nil
}

func tagViewHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}
	rawbytes, err := database.Get(id, config.BucketNameTag)
	if err != nil {
		return err
	}

	tag := DeserializeTag(rawbytes)

	renderTemplate(w, "url-index", map[string]interface{}{
		"ActiveTab": "tags",
		"Title":     fmt.Sprintf("URLs tagged under %s", tag.Name),
		"URLs":      tag.GetURLs(),
		"Count":     len(tag.URLs),
	})
	return nil
}
<commit_msg>stray println<commit_after>package app

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	"github.com/asaskevich/govalidator"
	"github.com/gorilla/context"
	"github.com/gorilla/mux"
	"github.com/kyleterry/sufr/config"
)

var staticHandler = http.StripPrefix(
	"/static/",
	http.FileServer(http.Dir("static")))

func urlIndexHandler(w http.ResponseWriter, r *http.Request) error {
	rawbytes, err := database.GetAll(config.BucketNameURL)
	urls := DeserializeURLs(rawbytes...)
	if err != nil {
		return err
	}

	renderTemplate(w, "url-index", map[string]interface{}{
		"ActiveTab": "urls",
		"Title":     "URLs",
		"Count":     len(urls),
		"URLs":      urls,
	})
	return nil
}

func urlNewHandler(w http.ResponseWriter, r *http.Request) error {
	ctx := context.Get(r, TemplateContext).(map[string]interface{})
	ctx["ActiveTab"] = "urls"
	renderTemplate(w, "url-new", ctx)
	return nil
}

func urlSubmitHandler(w http.ResponseWriter, r *http.Request) error {
	urlstring := r.FormValue("url")
	tagsstring := r.FormValue("tags")
	if !govalidator.IsURL(urlstring) {
		errormessage := "URL is required"
		if urlstring != "" {
			errormessage = fmt.Sprintf("URL \"%s\" is not valid", urlstring)
		}
		session, err := store.Get(r, "flashes")
		if err != nil {
			return err
		}
		session.AddFlash(errormessage, "danger")
		session.Save(r, w)
		http.Redirect(w, r, reverse("url-new"), http.StatusSeeOther)
		return nil
	}

	title, err := getPageTitle(urlstring)
	if err != nil {
		// Add flash about title not being fetchable
		// or alternatively add logic for detecting content type because it might be
		// an image or PDF
	}

	url := &URL{
		URL:       urlstring,
		Title:     title,
		CreatedAt: time.Now(),
		UpdatedAt: time.Now(),
	}

	err = url.SaveWithTags(tagsstring)
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-view", "id", url.ID), http.StatusSeeOther)
	return nil
}

func urlViewHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}
	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)

	renderTemplate(w, "url-view", map[string]interface{}{
		"ActiveTab": "urls",
		"Url":       url,
	})
	return nil
}

func urlEditHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}
	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)

	renderTemplate(w, "url-edit", map[string]interface{}{
		"ActiveTab": "urls",
		"Url":       url,
	})

	return nil
}

func urlSaveHandler(w http.ResponseWriter, r *http.Request) error {
	titlestring := r.FormValue("title")
	tagsstring := r.FormValue("tags")
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error fetching data: %s", err), http.StatusInternalServerError)
	}

	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)
	url.Title = titlestring
	url.UpdatedAt = time.Now()
	err = url.SaveWithTags(tagsstring)
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-view", "id", url.ID), http.StatusSeeOther)
	return nil
}

func urlDeleteHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}

	rawbytes, err := database.Get(id, config.BucketNameURL)
	if err != nil {
		return err
	}

	url := DeserializeURL(rawbytes)
	err = url.Delete()
	if err != nil {
		return err
	}

	http.Redirect(w, r, reverse("url-index"), http.StatusSeeOther)
	return nil
}

func tagIndexHandler(w http.ResponseWriter, r *http.Request) error {
	rawbytes, err := database.GetAll(config.BucketNameTag)
	tags := DeserializeTags(rawbytes...)
	if err != nil {
		return err
	}
	renderTemplate(w, "tag-index", map[string]interface{}{
		"ActiveTab": "tags",
		"Tags":      tags,
	})
	return nil
}

func tagViewHandler(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	id, err := strconv.ParseUint((vars["id"]), 10, 64)
	if err != nil {
		return err
	}
	rawbytes, err := database.Get(id, config.BucketNameTag)
	if err != nil {
		return err
	}

	tag := DeserializeTag(rawbytes)

	renderTemplate(w, "url-index", map[string]interface{}{
		"ActiveTab": "tags",
		"Title":     fmt.Sprintf("URLs tagged under %s", tag.Name),
		"URLs":      tag.GetURLs(),
		"Count":     len(tag.URLs),
	})
	return nil
}
<|endoftext|>


<commit_before>// Package memory is a in-memory store store
package memory

import (
	"sync"
	"time"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/data/store"
)

type memoryStore struct {
	options.Options

	sync.RWMutex
	values map[string]*memoryRecord
}

type memoryRecord struct {
	r *store.Record
	c time.Time
}

func (m *memoryStore) Dump() ([]*store.Record, error) {
	m.RLock()
	defer m.RUnlock()

	var values []*store.Record

	for _, v := range m.values {
		// get expiry
		d := v.r.Expiry
		t := time.Since(v.c)

		// expired
		if d > time.Duration(0) && t > d {
			continue
		}

		// update expiry
		v.r.Expiry -= t
		v.c = time.Now()

		values = append(values, v.r)
	}

	return values, nil
}

func (m *memoryStore) Read(key string) (*store.Record, error) {
	m.RLock()
	defer m.RUnlock()

	v, ok := m.values[key]
	if !ok {
		return nil, store.ErrNotFound
	}

	// get expiry
	d := v.r.Expiry
	t := time.Since(v.c)

	// expired
	if d > time.Duration(0) && t > d {
		return nil, store.ErrNotFound
	}

	// update expiry
	v.r.Expiry -= t
	v.c = time.Now()

	return v.r, nil
}

func (m *memoryStore) Write(r *store.Record) error {
	m.Lock()
	defer m.Unlock()

	// set the record
	m.values[r.Key] = &memoryRecord{
		r: r,
		c: time.Now(),
	}

	return nil
}

func (m *memoryStore) Delete(key string) error {
	m.Lock()
	defer m.Unlock()

	// delete the value
	delete(m.values, key)

	return nil
}

// NewStore returns a new store.Store
func NewStore(opts ...options.Option) store.Store {
	options := options.NewOptions(opts...)

	return &memoryStore{
		Options: options,
		values:  make(map[string]*memoryRecord),
	}
}
<commit_msg>update expiry only if it is non-zero<commit_after>// Package memory is a in-memory store store
package memory

import (
	"sync"
	"time"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/data/store"
)

type memoryStore struct {
	options.Options

	sync.RWMutex
	values map[string]*memoryRecord
}

type memoryRecord struct {
	r *store.Record
	c time.Time
}

func (m *memoryStore) Dump() ([]*store.Record, error) {
	m.RLock()
	defer m.RUnlock()

	var values []*store.Record

	for _, v := range m.values {
		// get expiry
		d := v.r.Expiry
		t := time.Since(v.c)

		// expired
		if d > time.Duration(0) && t > d {
			continue
		}

		// update expiry
		if d > time.Duration(0) {
			v.r.Expiry -= t
			v.c = time.Now()
		}

		values = append(values, v.r)
	}

	return values, nil
}

func (m *memoryStore) Read(key string) (*store.Record, error) {
	m.RLock()
	defer m.RUnlock()

	v, ok := m.values[key]
	if !ok {
		return nil, store.ErrNotFound
	}

	// get expiry
	d := v.r.Expiry
	t := time.Since(v.c)

	// expired
	if d > time.Duration(0) && t > d {
		return nil, store.ErrNotFound
	}

	// update expiry
	if d > time.Duration(0) {
		v.r.Expiry -= t
		v.c = time.Now()
	}

	return v.r, nil
}

func (m *memoryStore) Write(r *store.Record) error {
	m.Lock()
	defer m.Unlock()

	// set the record
	m.values[r.Key] = &memoryRecord{
		r: r,
		c: time.Now(),
	}

	return nil
}

func (m *memoryStore) Delete(key string) error {
	m.Lock()
	defer m.Unlock()

	// delete the value
	delete(m.values, key)

	return nil
}

// NewStore returns a new store.Store
func NewStore(opts ...options.Option) store.Store {
	options := options.NewOptions(opts...)

	return &memoryStore{
		Options: options,
		values:  make(map[string]*memoryRecord),
	}
}
<|endoftext|>


<commit_before>package cockroach

import (
	"database/sql"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/kr/pretty"
	"github.com/micro/go-micro/v2/store"
)

func TestSQL(t *testing.T) {
	if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
		t.Skip()
	}

	connection := fmt.Sprintf(
		"host=%s port=%d user=%s sslmode=disable dbname=%s",
		"localhost",
		26257,
		"root",
		"test",
	)
	db, err := sql.Open("postgres", connection)
	if err != nil {
		t.Fatal(err)
	}
	if err := db.Ping(); err != nil {
		t.Fatal(err)
	}
	db.Close()

	sqlStore := NewStore(
		store.Database("testsql"),
		store.Nodes(connection),
	)

	if err := sqlStore.Init(); err != nil {
		t.Fatal(err)
	}

	keys, err := sqlStore.List()
	if err != nil {
		t.Error(err)
	} else {
		t.Logf("%# v\n", pretty.Formatter(keys))
	}

	err = sqlStore.Write(
		&store.Record{
			Key:   "test",
			Value: []byte("foo"),
		},
	)
	if err != nil {
		t.Error(err)
	}
	err = sqlStore.Write(
		&store.Record{
			Key:   "bar",
			Value: []byte("baz"),
		},
	)
	if err != nil {
		t.Error(err)
	}
	err = sqlStore.Write(
		&store.Record{
			Key:   "qux",
			Value: []byte("aasad"),
		},
	)
	if err != nil {
		t.Error(err)
	}
	err = sqlStore.Delete("qux")
	if err != nil {
		t.Error(err)
	}

	err = sqlStore.Write(&store.Record{
		Key:    "test",
		Value:  []byte("bar"),
		Expiry: time.Second * 10,
	})
	if err != nil {
		t.Error(err)
	}

	records, err := sqlStore.Read("test")
	if err != nil {
		t.Error(err)
	}
	t.Logf("%# v\n", pretty.Formatter(records))
	if string(records[0].Value) != "bar" {
		t.Error("Expected bar, got ", string(records[0].Value))
	}

	time.Sleep(11 * time.Second)
	_, err = sqlStore.Read("test")
	switch err {
	case nil:
		t.Error("Key test should have expired")
	default:
		t.Error(err)
	case store.ErrNotFound:
		break
	}
	sqlStore.Delete("bar")
	sqlStore.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 5 * time.Second})
	sqlStore.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 5 * time.Second})
	sqlStore.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 5 * time.Second})
	results, err := sqlStore.Read("a", store.ReadPrefix())
	if err != nil {
		t.Error(err)
	}
	if len(results) != 3 {
		t.Fatal("Results should have returned 3 records")
	}
	time.Sleep(6 * time.Second)
	results, err = sqlStore.Read("a", store.ReadPrefix())
	if err != nil {
		t.Error(err)
	}
	if len(results) != 0 {
		t.Fatal("Results should have returned 0 records")
	}
}
<commit_msg>store/cockroach: fixup test (#1512)<commit_after>package cockroach

import (
	"database/sql"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/kr/pretty"
	"github.com/micro/go-micro/v2/store"
)

func TestSQL(t *testing.T) {
	if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
		t.Skip()
	}

	connection := fmt.Sprintf(
		"host=%s port=%d user=%s sslmode=disable dbname=%s",
		"localhost",
		26257,
		"root",
		"test",
	)
	db, err := sql.Open("postgres", connection)
	if err != nil {
		t.Fatal(err)
	}
	if err := db.Ping(); err != nil {
		t.Skip("store/cockroach: can't connect to db")
	}
	db.Close()

	sqlStore := NewStore(
		store.Database("testsql"),
		store.Nodes(connection),
	)

	if err := sqlStore.Init(); err != nil {
		t.Fatal(err)
	}

	keys, err := sqlStore.List()
	if err != nil {
		t.Error(err)
	} else {
		t.Logf("%# v\n", pretty.Formatter(keys))
	}

	err = sqlStore.Write(
		&store.Record{
			Key:   "test",
			Value: []byte("foo"),
		},
	)
	if err != nil {
		t.Error(err)
	}
	err = sqlStore.Write(
		&store.Record{
			Key:   "bar",
			Value: []byte("baz"),
		},
	)
	if err != nil {
		t.Error(err)
	}
	err = sqlStore.Write(
		&store.Record{
			Key:   "qux",
			Value: []byte("aasad"),
		},
	)
	if err != nil {
		t.Error(err)
	}
	err = sqlStore.Delete("qux")
	if err != nil {
		t.Error(err)
	}

	err = sqlStore.Write(&store.Record{
		Key:    "test",
		Value:  []byte("bar"),
		Expiry: time.Second * 10,
	})
	if err != nil {
		t.Error(err)
	}

	records, err := sqlStore.Read("test")
	if err != nil {
		t.Error(err)
	}
	t.Logf("%# v\n", pretty.Formatter(records))
	if string(records[0].Value) != "bar" {
		t.Error("Expected bar, got ", string(records[0].Value))
	}

	time.Sleep(11 * time.Second)
	_, err = sqlStore.Read("test")
	switch err {
	case nil:
		t.Error("Key test should have expired")
	default:
		t.Error(err)
	case store.ErrNotFound:
		break
	}
	sqlStore.Delete("bar")
	sqlStore.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 5 * time.Second})
	sqlStore.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 5 * time.Second})
	sqlStore.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 5 * time.Second})
	results, err := sqlStore.Read("a", store.ReadPrefix())
	if err != nil {
		t.Error(err)
	}
	if len(results) != 3 {
		t.Fatal("Results should have returned 3 records")
	}
	time.Sleep(6 * time.Second)
	results, err = sqlStore.Read("a", store.ReadPrefix())
	if err != nil {
		t.Error(err)
	}
	if len(results) != 0 {
		t.Fatal("Results should have returned 0 records")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package boltdb

import (
	"os"
	"path"

	"github.com/boltdb/bolt"
	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/store/localstore/engine"
)

var (
	_ engine.DB = (*db)(nil)
)

var (
	bucketName = []byte("tidb_bucket")
)

type db struct {
	*bolt.DB
}

func (d *db) Get(key []byte) ([]byte, error) {
	var value []byte

	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		v := b.Get(key)
		if v == nil {
			return nil
		}

		value = append([]byte(nil), v...)

		return nil
	})

	return value, errors.Trace(err)
}

func (d *db) GetSnapshot() (engine.Snapshot, error) {
	tx, err := d.DB.Begin(false)
	if err != nil {
		return nil, err
	}
	return &snapshot{tx}, nil
}

func (d *db) NewBatch() engine.Batch {
	return &batch{}
}

func (d *db) Commit(b engine.Batch) error {
	bt, ok := b.(*batch)
	if !ok {
		return errors.Errorf("invalid batch type %T", b)
	}
	err := d.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		var err error
		for _, w := range bt.Writes {
			if w.Value != nil {
				err = b.Put(w.Key, w.Value)
			} else {
				err = b.Delete(w.Key)
			}

			if err != nil {
				return errors.Trace(err)
			}
		}

		return nil
	})
	return errors.Trace(err)
}

func (d *db) Close() error {
	return d.DB.Close()
}

type snapshot struct {
	*bolt.Tx
}

func (s *snapshot) Get(key []byte) ([]byte, error) {
	b := s.Tx.Bucket(bucketName)
	v := b.Get(key)
	if v == nil {
		return nil, nil
	}

	value := append([]byte(nil), v...)

	return value, nil
}

func (s *snapshot) NewIterator(startKey []byte) engine.Iterator {
	return &iterator{tx: s.Tx, key: startKey}
}

func (s *snapshot) Release() {
	err := s.Tx.Rollback()
	if err != nil {
		log.Errorf("commit err %v", err)
	}
}

type iterator struct {
	tx *bolt.Tx
	*bolt.Cursor

	key   []byte
	value []byte
}

func (i *iterator) Next() bool {
	if i.Cursor == nil {
		i.Cursor = i.tx.Bucket(bucketName).Cursor()
		if i.key == nil {
			i.key, i.value = i.Cursor.First()
		} else {
			i.key, i.value = i.Cursor.Seek(i.key)
		}
	} else {
		i.key, i.value = i.Cursor.Next()
	}

	return i.key != nil
}

func (i *iterator) Key() []byte {
	return i.key
}

func (i *iterator) Value() []byte {
	return i.value
}

func (i *iterator) Release() {

}

type write struct {
	Key   []byte
	Value []byte
}

type batch struct {
	Writes []write
}

func (b *batch) Put(key []byte, value []byte) {
	w := write{
		Key:   append([]byte(nil), key...),
		Value: append([]byte(nil), value...),
	}
	b.Writes = append(b.Writes, w)
}

func (b *batch) Delete(key []byte) {
	w := write{
		Key:   append([]byte(nil), key...),
		Value: nil,
	}
	b.Writes = append(b.Writes, w)
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates a local storage database with given path.
func (driver Driver) Open(dbPath string) (engine.DB, error) {
	base := path.Dir(dbPath)
	os.MkdirAll(base, 0755)

	d, err := bolt.Open(dbPath, 0600, nil)
	if err != nil {
		return nil, err
	}

	tx, err := d.Begin(true)
	if err != nil {
		return nil, err
	}

	if _, err = tx.CreateBucketIfNotExists(bucketName); err != nil {
		tx.Rollback()
		return nil, err
	}

	if err = tx.Commit(); err != nil {
		return nil, err
	}

	return &db{d}, nil
}
<commit_msg>store: update boltdb package import.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package boltdb

import (
	"os"
	"path"

	"github.com/juju/errors"
	"github.com/ngaut/bolt"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/store/localstore/engine"
)

var (
	_ engine.DB = (*db)(nil)
)

var (
	bucketName = []byte("tidb_bucket")
)

type db struct {
	*bolt.DB
}

func (d *db) Get(key []byte) ([]byte, error) {
	var value []byte

	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		v := b.Get(key)
		if v == nil {
			return nil
		}

		value = append([]byte(nil), v...)

		return nil
	})

	return value, errors.Trace(err)
}

func (d *db) GetSnapshot() (engine.Snapshot, error) {
	tx, err := d.DB.Begin(false)
	if err != nil {
		return nil, err
	}
	return &snapshot{tx}, nil
}

func (d *db) NewBatch() engine.Batch {
	return &batch{}
}

func (d *db) Commit(b engine.Batch) error {
	bt, ok := b.(*batch)
	if !ok {
		return errors.Errorf("invalid batch type %T", b)
	}
	err := d.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		var err error
		for _, w := range bt.Writes {
			if w.Value != nil {
				err = b.Put(w.Key, w.Value)
			} else {
				err = b.Delete(w.Key)
			}

			if err != nil {
				return errors.Trace(err)
			}
		}

		return nil
	})
	return errors.Trace(err)
}

func (d *db) Close() error {
	return d.DB.Close()
}

type snapshot struct {
	*bolt.Tx
}

func (s *snapshot) Get(key []byte) ([]byte, error) {
	b := s.Tx.Bucket(bucketName)
	v := b.Get(key)
	if v == nil {
		return nil, nil
	}

	value := append([]byte(nil), v...)

	return value, nil
}

func (s *snapshot) NewIterator(startKey []byte) engine.Iterator {
	return &iterator{tx: s.Tx, key: startKey}
}

func (s *snapshot) Release() {
	err := s.Tx.Rollback()
	if err != nil {
		log.Errorf("commit err %v", err)
	}
}

type iterator struct {
	tx *bolt.Tx
	*bolt.Cursor

	key   []byte
	value []byte
}

func (i *iterator) Next() bool {
	if i.Cursor == nil {
		i.Cursor = i.tx.Bucket(bucketName).Cursor()
		if i.key == nil {
			i.key, i.value = i.Cursor.First()
		} else {
			i.key, i.value = i.Cursor.Seek(i.key)
		}
	} else {
		i.key, i.value = i.Cursor.Next()
	}

	return i.key != nil
}

func (i *iterator) Key() []byte {
	return i.key
}

func (i *iterator) Value() []byte {
	return i.value
}

func (i *iterator) Release() {

}

type write struct {
	Key   []byte
	Value []byte
}

type batch struct {
	Writes []write
}

func (b *batch) Put(key []byte, value []byte) {
	w := write{
		Key:   append([]byte(nil), key...),
		Value: append([]byte(nil), value...),
	}
	b.Writes = append(b.Writes, w)
}

func (b *batch) Delete(key []byte) {
	w := write{
		Key:   append([]byte(nil), key...),
		Value: nil,
	}
	b.Writes = append(b.Writes, w)
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates a local storage database with given path.
func (driver Driver) Open(dbPath string) (engine.DB, error) {
	base := path.Dir(dbPath)
	os.MkdirAll(base, 0755)

	d, err := bolt.Open(dbPath, 0600, nil)
	if err != nil {
		return nil, err
	}

	tx, err := d.Begin(true)
	if err != nil {
		return nil, err
	}

	if _, err = tx.CreateBucketIfNotExists(bucketName); err != nil {
		tx.Rollback()
		return nil, err
	}

	if err = tx.Commit(); err != nil {
		return nil, err
	}

	return &db{d}, nil
}
<|endoftext|>