<commit_before>// Virtua SmartOS Prometheus exporter
//
// Worflow :
//  - detect if launched in GZ or into a Zone
//  - retrieve useful metrics
//  - expose them to http://xxx:9100/metrics (same as node_exporter)

package main

import (
        "log"
        "net/http"
        "os/exec"
        "strconv"
        "strings"
//      "fmt"
        // Prometheus Go toolset
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    // Global variables
    exporterPort := ":9100"
    // Metrics definitions
    gzCpuUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_cpu_usage_total",
        Help: "CPU usage exposed in percent.",
        },
        []string{"cpu","type"},
    )
    gzFreeMem = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_memory_free_bytes_total",
        Help: "Total free memory (both RAM and Swap) of the CN.",
        },
        []string{"type"},
    )
    gzMlagUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_network_mlag_bytes_total",
        Help: "MLAG (aggr0) usage of the CN.",
        },
        []string{"device","type"},
    )
    gzZpoolList = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_zpool_list_total",
        Help: "ZFS zpool list summary.",
        },
        []string{"zpool","type"},
    )
    gzDiskErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
        Name: "smartos_gz_disk_errors_total",
        Help: "Number of hardware disk errors.",
        },
        []string{"device","type"},
    )
)

// program start

func init() {
    // Metrics have to be registered to be exposed:
    gz, _ := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        log.Fatal("not yet implemented")
    }
}

// Global Helpers

// try to determine if its executed inside the GZ or not.
// return 1 if in GZ
//        0 if in zone
func isGlobalZone() (int) {
    out, eerr := exec.Command("zonename").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    if (strings.Contains(string(out), "global")) == false {
        return 0
    } else {
        return 1
    }
}

// SmartOS command / tool callers
func iostat() {
    out, eerr := exec.Command("iostat", "-en").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseIostatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func mpstat() {
    out, eerr := exec.Command("mpstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseMpstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func nicstat() {
    out, eerr := exec.Command("nicstat", "-i", "aggr0").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseNicstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func vmstat() {
    out, eerr := exec.Command("vmstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseVmstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func zpoolList() {
    out, eerr := exec.Command("zpool", "list", "-p", "zones").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseZpoolStatusOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

// Parsers

func parseIostatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        deviceName := parsedLine[4]
        softErr, err := strconv.ParseFloat(parsedLine[0], 64)
        if err != nil {
            return err
        }
        hardErr, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        trnErr, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"soft"}).Add(softErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"hard"}).Add(hardErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"trn"}).Add(trnErr)
    }
    return nil
}

func parseMpstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        cpuId := parsedLine[0]
        cpuUsr, err := strconv.ParseFloat(parsedLine[12], 64)
        if err != nil {
            return err
        }
        cpuSys, err := strconv.ParseFloat(parsedLine[13], 64)
        if err != nil {
            return err
        }
        cpuIdl, err := strconv.ParseFloat(parsedLine[15], 64)
        if err != nil {
            return err
        }
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"user"}).Set(cpuUsr)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"system"}).Set(cpuSys)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"idle"}).Set(cpuIdl)
        //fmt.Printf("cpuId : %d, cpuUsr : %d, cpuSys : %d \n", cpuId, cpuUsr, cpuSys)
    }
    return nil
}

func parseNicstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        readKb, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        writeKb, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"read"}).Set(readKb)
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"write"}).Set(writeKb)
    }
    return nil
}

func parseVmstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        freeSwap, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        freeRam, err := strconv.ParseFloat(parsedLine[4], 64)
        if err != nil {
            return err
        }
        gzFreeMem.With(prometheus.Labels{"type":"swap"}).Set(freeSwap)
        gzFreeMem.With(prometheus.Labels{"type":"ram"}).Set(freeRam)
    }
    return nil
}

func parseZpoolStatusOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        sizeBytes, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        allocBytes, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        freeBytes, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        fragPercent := strings.TrimSuffix(parsedLine[5], "%")
        fragPercentTrim, err := strconv.ParseFloat(fragPercent, 64)
        if err != nil {
            return err
        }
        capPercent := strings.TrimSuffix(parsedLine[6], "%")
        capPercentTrim, err := strconv.ParseFloat(capPercent, 64)
        if err != nil {
            return err
        }
        health := parsedLine[8]
        if (strings.Contains(health, "ONLINE")) == true {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(0)
        } else {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(1)
        }

        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"size"}).Set(sizeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"alloc"}).Set(allocBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"free"}).Set(freeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"frag"}).Set(fragPercentTrim)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"capacity"}).Set(capPercentTrim)
    }
    return nil
}

// program starter

func init() {
    // Metrics have to be registered to be exposed:
    gz, _ := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }
}

func main() {
    ret, _ := isGlobalZone()
    if ret == 1 {
        mpstat()
        vmstat()
        nicstat()
        iostat()
        zpoolList()
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }

    // The Handler function provides a default handler to expose metrics
    // via an HTTP server. "/metrics" is the usual endpoint for that.
    http.Handle("/metrics", promhttp.Handler())
    log.Fatal(http.ListenAndServe(exporterPort, nil))
}
<commit_msg>Fix build and wrong variable assignments<commit_after>// Virtua SmartOS Prometheus exporter
//
// Worflow :
//  - detect if launched in GZ or into a Zone
//  - retrieve useful metrics
//  - expose them to http://xxx:9100/metrics (same as node_exporter)

package main

import (
        "log"
        "net/http"
        "os/exec"
        "strconv"
        "strings"
//      "fmt"
        // Prometheus Go toolset
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    // Global variables
    exporterPort = ":9100"
    // Metrics definitions
    gzCpuUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_cpu_usage_total",
        Help: "CPU usage exposed in percent.",
        },
        []string{"cpu","type"},
    )
    gzFreeMem = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_memory_free_bytes_total",
        Help: "Total free memory (both RAM and Swap) of the CN.",
        },
        []string{"type"},
    )
    gzMlagUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_network_mlag_bytes_total",
        Help: "MLAG (aggr0) usage of the CN.",
        },
        []string{"device","type"},
    )
    gzZpoolList = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "smartos_gz_zpool_list_total",
        Help: "ZFS zpool list summary.",
        },
        []string{"zpool","type"},
    )
    gzDiskErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
        Name: "smartos_gz_disk_errors_total",
        Help: "Number of hardware disk errors.",
        },
        []string{"device","type"},
    )
)

// program start

func init() {
    // Metrics have to be registered to be exposed:
    gz := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        log.Fatal("not yet implemented")
    }
}

// Global Helpers

// try to determine if its executed inside the GZ or not.
// return 1 if in GZ
//        0 if in zone
func isGlobalZone() (int) {
    out, eerr := exec.Command("zonename").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    if (strings.Contains(string(out), "global")) == false {
        return 0
    } else {
        return 1
    }
}

// SmartOS command / tool callers
func iostat() {
    out, eerr := exec.Command("iostat", "-en").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseIostatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func mpstat() {
    out, eerr := exec.Command("mpstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseMpstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func nicstat() {
    out, eerr := exec.Command("nicstat", "-i", "aggr0").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseNicstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func vmstat() {
    out, eerr := exec.Command("vmstat", "1", "1").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseVmstatOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

func zpoolList() {
    out, eerr := exec.Command("zpool", "list", "-p", "zones").Output()
    if eerr != nil {
        log.Fatal(eerr)
    }
    perr := parseZpoolStatusOutput(string(out))
    if perr != nil {
        log.Fatal(perr)
    }
}

// Parsers

func parseIostatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        deviceName := parsedLine[4]
        softErr, err := strconv.ParseFloat(parsedLine[0], 64)
        if err != nil {
            return err
        }
        hardErr, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        trnErr, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"soft"}).Add(softErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"hard"}).Add(hardErr)
        gzDiskErrors.With(prometheus.Labels{"device":deviceName,"type":"trn"}).Add(trnErr)
    }
    return nil
}

func parseMpstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        cpuId := parsedLine[0]
        cpuUsr, err := strconv.ParseFloat(parsedLine[12], 64)
        if err != nil {
            return err
        }
        cpuSys, err := strconv.ParseFloat(parsedLine[13], 64)
        if err != nil {
            return err
        }
        cpuIdl, err := strconv.ParseFloat(parsedLine[15], 64)
        if err != nil {
            return err
        }
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"user"}).Set(cpuUsr)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"system"}).Set(cpuSys)
        gzCpuUsage.With(prometheus.Labels{"cpu": cpuId, "type":"idle"}).Set(cpuIdl)
        //fmt.Printf("cpuId : %d, cpuUsr : %d, cpuSys : %d \n", cpuId, cpuUsr, cpuSys)
    }
    return nil
}

func parseNicstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        readKb, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        writeKb, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"read"}).Set(readKb)
        gzMlagUsage.With(prometheus.Labels{"device":"aggr0", "type":"write"}).Set(writeKb)
    }
    return nil
}

func parseVmstatOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[2:l-1] {
        parsedLine := strings.Fields(line)
        freeSwap, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        freeRam, err := strconv.ParseFloat(parsedLine[4], 64)
        if err != nil {
            return err
        }
        gzFreeMem.With(prometheus.Labels{"type":"swap"}).Set(freeSwap)
        gzFreeMem.With(prometheus.Labels{"type":"ram"}).Set(freeRam)
    }
    return nil
}

func parseZpoolStatusOutput(out string) (error) {
    outlines := strings.Split(out, "\n")
    l := len(outlines)
    for _, line := range outlines[1:l-1] {
        parsedLine := strings.Fields(line)
        sizeBytes, err := strconv.ParseFloat(parsedLine[1], 64)
        if err != nil {
            return err
        }
        allocBytes, err := strconv.ParseFloat(parsedLine[2], 64)
        if err != nil {
            return err
        }
        freeBytes, err := strconv.ParseFloat(parsedLine[3], 64)
        if err != nil {
            return err
        }
        fragPercent := strings.TrimSuffix(parsedLine[5], "%")
        fragPercentTrim, err := strconv.ParseFloat(fragPercent, 64)
        if err != nil {
            return err
        }
        capPercent := strings.TrimSuffix(parsedLine[6], "%")
        capPercentTrim, err := strconv.ParseFloat(capPercent, 64)
        if err != nil {
            return err
        }
        health := parsedLine[8]
        if (strings.Contains(health, "ONLINE")) == true {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(0)
        } else {
          gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"faulty"}).Set(1)
        }

        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"size"}).Set(sizeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"alloc"}).Set(allocBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"free"}).Set(freeBytes)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"frag"}).Set(fragPercentTrim)
        gzZpoolList.With(prometheus.Labels{"zpool":"zones", "type":"capacity"}).Set(capPercentTrim)
    }
    return nil
}

// program starter

func init() {
    // Metrics have to be registered to be exposed:
    gz := isGlobalZone()
    if gz == 1 {
        prometheus.MustRegister(gzCpuUsage)
        prometheus.MustRegister(gzFreeMem)
        prometheus.MustRegister(gzMlagUsage)
        prometheus.MustRegister(gzDiskErrors)
        prometheus.MustRegister(gzZpoolList)
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }
}

func main() {
    gz := isGlobalZone()
    if gz == 1 {
        mpstat()
        vmstat()
        nicstat()
        iostat()
        zpoolList()
    } else {
        // not yet implemented
        // XXX
        log.Fatal("zone statistics gathering is not yet implemented.")
    }

    // The Handler function provides a default handler to expose metrics
    // via an HTTP server. "/metrics" is the usual endpoint for that.
    http.Handle("/metrics", promhttp.Handler())
    log.Fatal(http.ListenAndServe(exporterPort, nil))
}
<|endoftext|>


<commit_before>package cmd

const Version = "v3.60.4"
<commit_msg>version bump<commit_after>package cmd

const Version = "v3.60.5"
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"image"
	_ "image/jpeg"
	"log"
	"mime"
	"os"
	"path/filepath"

	"github.com/agorf/goexif/exif"
	_ "github.com/mattn/go-sqlite3"
)

type Photo struct {
	Aperture      sql.NullFloat64
	Camera        sql.NullString
	ExposureComp  sql.NullInt64
	ExposureTime  sql.NullFloat64
	Flash         sql.NullString
	FocalLength   sql.NullFloat64
	FocalLength35 sql.NullInt64
	Height        int
	ISO           sql.NullInt64
	Lat           sql.NullFloat64
	Lens          sql.NullString
	Lng           sql.NullFloat64
	Path          string
	Size          int64
	TakenAt       sql.NullString
	Width         int
}

var (
	db              *sql.DB
	selectSetStmt   *sql.Stmt
	selectPhotoStmt *sql.Stmt
	insertSetStmt   *sql.Stmt
	insertPhotoStmt *sql.Stmt
)

var createSchemaSQL = `
CREATE TABLE IF NOT EXISTS sets (
	id integer NOT NULL PRIMARY KEY,
	thumb_photo_id integer UNIQUE REFERENCES photos,
	name varchar(4096) NOT NULL UNIQUE,
	photos_count integer,
	taken_at char(19)
);

CREATE INDEX IF NOT EXISTS sets_thumb_photo_id_index ON sets (thumb_photo_id);

CREATE TABLE IF NOT EXISTS photos (
	id integer NOT NULL PRIMARY KEY,
	set_id integer NOT NULL REFERENCES sets,
	prev_photo_id integer UNIQUE REFERENCES photos,
	next_photo_id integer UNIQUE REFERENCES photos,
	path varchar(4096) NOT NULL UNIQUE,
	size integer NOT NULL,
	width integer NOT NULL,
	height integer NOT NULL,
	aperture decimal(2, 1),
	camera varchar(1000),
	exposure_comp integer,
	exposure_time decimal(9, 5),
	flash varchar(51),
	focal_length decimal(3, 1),
	focal_length_35 integer,
	iso integer,
	lat decimal(9, 6),
	lens varchar(1000),
	lng decimal(9, 6),
	taken_at char(19)
);

CREATE INDEX IF NOT EXISTS photos_set_id_index ON photos (set_id);

CREATE INDEX IF NOT EXISTS photos_prev_photo_id_index ON photos (prev_photo_id);

CREATE INDEX IF NOT EXISTS photos_next_photo_id_index ON photos (next_photo_id);
`

func decodePhotoExif(photo *Photo, x *exif.Exif) {
	takenAt, err := x.DateTime()
	if err == nil {
		photo.TakenAt.String = takenAt.UTC().Format("2006-01-02 15:04:05")
		photo.TakenAt.Valid = true
	}

	lat, lng, err := x.LatLong()
	if err == nil {
		photo.Lat.Float64 = lat
		photo.Lat.Valid = true
		photo.Lng.Float64 = lng
		photo.Lng.Valid = true
	}

	orientTag, err := x.Get(exif.Orientation)
	if err == nil {
		switch orient, _ := orientTag.Int(0); orient {
		case 5, 6, 7, 8: // rotated
			photo.Width, photo.Height = photo.Height, photo.Width // swap
		}
	}

	camMakeTag, err := x.Get(exif.Make)
	if err == nil {
		photo.Camera.String, _ = camMakeTag.StringVal()
		photo.Camera.Valid = true
	}

	camModelTag, err := x.Get(exif.Model)
	if err == nil {
		cameraModel, _ := camModelTag.StringVal()

		if photo.Camera.Valid {
			photo.Camera.String = fmt.Sprint(photo.Camera.String, " ", cameraModel)
		} else {
			photo.Camera.String = cameraModel
			photo.Camera.Valid = true
		}
	}

	lensMakeTag, err := x.Get(exif.LensMake)
	if err == nil {
		photo.Lens.String, _ = lensMakeTag.StringVal()
		photo.Lens.Valid = true
	}

	lensModelTag, err := x.Get(exif.LensModel)
	if err == nil {
		lensModel, _ := lensModelTag.StringVal()

		if photo.Lens.Valid {
			photo.Lens.String = fmt.Sprint(photo.Lens.String, " ", lensModel)
		} else {
			photo.Lens.String = lensModel
			photo.Lens.Valid = true
		}
	}

	focalLenTag, err := x.Get(exif.FocalLength)
	if err == nil {
		focalLen, _ := focalLenTag.Rat(0)
		photo.FocalLength.Float64, _ = focalLen.Float64()
		photo.FocalLength.Valid = true
	}

	focalLen35Tag, err := x.Get(exif.FocalLengthIn35mmFilm)
	if err == nil {
		photo.FocalLength35.Int64, _ = focalLen35Tag.Int64(0)
		photo.FocalLength35.Valid = true
	}

	apertureTag, err := x.Get(exif.FNumber)
	if err == nil {
		aperture, _ := apertureTag.Rat(0)
		photo.Aperture.Float64, _ = aperture.Float64()
		photo.Aperture.Valid = true
	}

	expTimeTag, err := x.Get(exif.ExposureTime)
	if err == nil {
		expTime, _ := expTimeTag.Rat(0)
		photo.ExposureTime.Float64, _ = expTime.Float64()
		photo.ExposureTime.Valid = true
	}

	isoTag, err := x.Get(exif.ISOSpeedRatings)
	if err == nil {
		photo.ISO.Int64, _ = isoTag.Int64(0)
		photo.ISO.Valid = true
	}

	expBiasTag, err := x.Get(exif.ExposureBiasValue)
	if err == nil {
		photo.ExposureComp.Int64, _ = expBiasTag.Int64(0)
		photo.ExposureComp.Valid = true
	}

	flash, err := x.Flash()
	if err == nil {
		photo.Flash.String = flash
		photo.Flash.Valid = true
	}
}

func decodePhoto(path string) (*Photo, error) {
	var photo Photo

	photo.Path = path

	f, err := os.Open(path)
	if err != nil {
		return &photo, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return &photo, err
	}
	photo.Size = fi.Size()

	img, _, err := image.DecodeConfig(f)
	if err != nil {
		return &photo, err
	}
	photo.Width, photo.Height = img.Width, img.Height

	f.Seek(0, 0) // rewind

	x, err := exif.Decode(f)
	if err == nil { // EXIF data exists
		decodePhotoExif(&photo, x)
	}

	return &photo, nil
}

func storePhoto(photo *Photo) error {
	var setId, photoId int64

	setName := filepath.Base(filepath.Dir(photo.Path))
	row := selectSetStmt.QueryRow(setName)
	if err := row.Scan(&setId); err == sql.ErrNoRows { // set does not exist
		result, err := insertSetStmt.Exec(setName) // create it
		if err != nil {
			return err
		}

		setId, err = result.LastInsertId()
		if err != nil {
			return err
		}
	}

	row = selectPhotoStmt.QueryRow(photo.Path)
	if err := row.Scan(&photoId); err == sql.ErrNoRows { // photo does not exist
		result, err := insertPhotoStmt.Exec(photo.Aperture, photo.Camera,
			photo.ExposureComp, photo.ExposureTime, photo.Flash, photo.FocalLength,
			photo.FocalLength35, photo.Height, photo.ISO, photo.Lat, photo.Lens,
			photo.Lng, photo.Path, setId, photo.Size, photo.TakenAt, photo.Width) // create it
		if err != nil {
			return err
		}

		photoId, err = result.LastInsertId()
		if err != nil {
			return err
		}

		fmt.Printf("photos id=%d path=%s\n", photoId, photo.Path)
	}

	return nil
}

func walkPath(path string, info os.FileInfo, err error) error {
	if err != nil { // error walking "path"
		return nil // skip
	}

	if info.IsDir() {
		return nil // skip
	}

	if mime.TypeByExtension(filepath.Ext(path)) != "image/jpeg" { // not JPEG
		return nil // skip
	}

	photo, err := decodePhoto(path)
	if err == nil {
		storePhoto(photo)
	}

	return nil // next
}

func updatePhotoSiblings() error {
	var prevId, prevSetId int

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	updatePrevPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET prev_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updatePrevPhotoStmt.Close()

	updateNextPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET next_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateNextPhotoStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id FROM photos ORDER BY set_id, taken_at
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId int
		rows.Scan(&id, &setId)

		if setId == prevSetId && prevId > 0 {
			updatePrevPhotoStmt.Exec(prevId, id)
			fmt.Printf("photos id=%d prev_photo_id=%d\n", id, prevId)
			updateNextPhotoStmt.Exec(id, prevId)
			fmt.Printf("photos id=%d next_photo_id=%d\n", prevId, id)
		}

		prevId = id
		prevSetId = setId
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func updateSets() error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	photosCountStmt, err := tx.Prepare(`
	SELECT COUNT(*) FROM photos WHERE set_id = ?
	`)
	if err != nil {
		return err
	}
	defer photosCountStmt.Close()

	updateSetStmt, err := tx.Prepare(`
	UPDATE sets
	SET photos_count = ?, taken_at = ?, thumb_photo_id = ?
	WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateSetStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id, MIN(taken_at) FROM photos GROUP BY set_id
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId, photosCount int
		var takenAt sql.NullString

		rows.Scan(&id, &setId, &takenAt)

		row := photosCountStmt.QueryRow(setId)
		row.Scan(&photosCount)

		updateSetStmt.Exec(photosCount, takenAt, id, setId)
		fmt.Printf("sets id=%d photos_count=%d taken_at=%q thumb_photo_id=%d\n", setId, photosCount, takenAt.String, id)
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func main() {
	var err error

	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "usage: %v path [path...]\n", os.Args[0])
		return
	}

	db, err = sql.Open("sqlite3", "thyme.db") // := here covers global db var
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	_, err = db.Exec(createSchemaSQL)
	if err != nil {
		log.Fatal(err)
	}

	selectSetStmt, err = db.Prepare("SELECT id FROM sets WHERE name = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer selectSetStmt.Close()

	selectPhotoStmt, err = db.Prepare("SELECT id FROM photos WHERE path = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer selectPhotoStmt.Close()

	insertSetStmt, err = db.Prepare("INSERT INTO sets (name) VALUES (?)")
	if err != nil {
		log.Fatal(err)
	}
	defer insertSetStmt.Close()

	insertPhotoStmt, err = db.Prepare(`
	INSERT INTO photos (
	aperture, camera, exposure_comp, exposure_time, flash, focal_length,
	focal_length_35, height, iso, lat, lens, lng, path, set_id, size, taken_at,
	width
	)
	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`)
	if err != nil {
		log.Fatal(err)
	}
	defer insertPhotoStmt.Close()

	for i := 1; i < len(os.Args); i++ {
		filepath.Walk(os.Args[i], walkPath)
	}

	updatePhotoSiblings()
	updateSets()
}
<commit_msg>Replace log.Fatal with log.Fatalln<commit_after>package main

import (
	"database/sql"
	"fmt"
	"image"
	_ "image/jpeg"
	"log"
	"mime"
	"os"
	"path/filepath"

	"github.com/agorf/goexif/exif"
	_ "github.com/mattn/go-sqlite3"
)

type Photo struct {
	Aperture      sql.NullFloat64
	Camera        sql.NullString
	ExposureComp  sql.NullInt64
	ExposureTime  sql.NullFloat64
	Flash         sql.NullString
	FocalLength   sql.NullFloat64
	FocalLength35 sql.NullInt64
	Height        int
	ISO           sql.NullInt64
	Lat           sql.NullFloat64
	Lens          sql.NullString
	Lng           sql.NullFloat64
	Path          string
	Size          int64
	TakenAt       sql.NullString
	Width         int
}

var (
	db              *sql.DB
	selectSetStmt   *sql.Stmt
	selectPhotoStmt *sql.Stmt
	insertSetStmt   *sql.Stmt
	insertPhotoStmt *sql.Stmt
)

var createSchemaSQL = `
CREATE TABLE IF NOT EXISTS sets (
	id integer NOT NULL PRIMARY KEY,
	thumb_photo_id integer UNIQUE REFERENCES photos,
	name varchar(4096) NOT NULL UNIQUE,
	photos_count integer,
	taken_at char(19)
);

CREATE INDEX IF NOT EXISTS sets_thumb_photo_id_index ON sets (thumb_photo_id);

CREATE TABLE IF NOT EXISTS photos (
	id integer NOT NULL PRIMARY KEY,
	set_id integer NOT NULL REFERENCES sets,
	prev_photo_id integer UNIQUE REFERENCES photos,
	next_photo_id integer UNIQUE REFERENCES photos,
	path varchar(4096) NOT NULL UNIQUE,
	size integer NOT NULL,
	width integer NOT NULL,
	height integer NOT NULL,
	aperture decimal(2, 1),
	camera varchar(1000),
	exposure_comp integer,
	exposure_time decimal(9, 5),
	flash varchar(51),
	focal_length decimal(3, 1),
	focal_length_35 integer,
	iso integer,
	lat decimal(9, 6),
	lens varchar(1000),
	lng decimal(9, 6),
	taken_at char(19)
);

CREATE INDEX IF NOT EXISTS photos_set_id_index ON photos (set_id);

CREATE INDEX IF NOT EXISTS photos_prev_photo_id_index ON photos (prev_photo_id);

CREATE INDEX IF NOT EXISTS photos_next_photo_id_index ON photos (next_photo_id);
`

func decodePhotoExif(photo *Photo, x *exif.Exif) {
	takenAt, err := x.DateTime()
	if err == nil {
		photo.TakenAt.String = takenAt.UTC().Format("2006-01-02 15:04:05")
		photo.TakenAt.Valid = true
	}

	lat, lng, err := x.LatLong()
	if err == nil {
		photo.Lat.Float64 = lat
		photo.Lat.Valid = true
		photo.Lng.Float64 = lng
		photo.Lng.Valid = true
	}

	orientTag, err := x.Get(exif.Orientation)
	if err == nil {
		switch orient, _ := orientTag.Int(0); orient {
		case 5, 6, 7, 8: // rotated
			photo.Width, photo.Height = photo.Height, photo.Width // swap
		}
	}

	camMakeTag, err := x.Get(exif.Make)
	if err == nil {
		photo.Camera.String, _ = camMakeTag.StringVal()
		photo.Camera.Valid = true
	}

	camModelTag, err := x.Get(exif.Model)
	if err == nil {
		cameraModel, _ := camModelTag.StringVal()

		if photo.Camera.Valid {
			photo.Camera.String = fmt.Sprint(photo.Camera.String, " ", cameraModel)
		} else {
			photo.Camera.String = cameraModel
			photo.Camera.Valid = true
		}
	}

	lensMakeTag, err := x.Get(exif.LensMake)
	if err == nil {
		photo.Lens.String, _ = lensMakeTag.StringVal()
		photo.Lens.Valid = true
	}

	lensModelTag, err := x.Get(exif.LensModel)
	if err == nil {
		lensModel, _ := lensModelTag.StringVal()

		if photo.Lens.Valid {
			photo.Lens.String = fmt.Sprint(photo.Lens.String, " ", lensModel)
		} else {
			photo.Lens.String = lensModel
			photo.Lens.Valid = true
		}
	}

	focalLenTag, err := x.Get(exif.FocalLength)
	if err == nil {
		focalLen, _ := focalLenTag.Rat(0)
		photo.FocalLength.Float64, _ = focalLen.Float64()
		photo.FocalLength.Valid = true
	}

	focalLen35Tag, err := x.Get(exif.FocalLengthIn35mmFilm)
	if err == nil {
		photo.FocalLength35.Int64, _ = focalLen35Tag.Int64(0)
		photo.FocalLength35.Valid = true
	}

	apertureTag, err := x.Get(exif.FNumber)
	if err == nil {
		aperture, _ := apertureTag.Rat(0)
		photo.Aperture.Float64, _ = aperture.Float64()
		photo.Aperture.Valid = true
	}

	expTimeTag, err := x.Get(exif.ExposureTime)
	if err == nil {
		expTime, _ := expTimeTag.Rat(0)
		photo.ExposureTime.Float64, _ = expTime.Float64()
		photo.ExposureTime.Valid = true
	}

	isoTag, err := x.Get(exif.ISOSpeedRatings)
	if err == nil {
		photo.ISO.Int64, _ = isoTag.Int64(0)
		photo.ISO.Valid = true
	}

	expBiasTag, err := x.Get(exif.ExposureBiasValue)
	if err == nil {
		photo.ExposureComp.Int64, _ = expBiasTag.Int64(0)
		photo.ExposureComp.Valid = true
	}

	flash, err := x.Flash()
	if err == nil {
		photo.Flash.String = flash
		photo.Flash.Valid = true
	}
}

func decodePhoto(path string) (*Photo, error) {
	var photo Photo

	photo.Path = path

	f, err := os.Open(path)
	if err != nil {
		return &photo, err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return &photo, err
	}
	photo.Size = fi.Size()

	img, _, err := image.DecodeConfig(f)
	if err != nil {
		return &photo, err
	}
	photo.Width, photo.Height = img.Width, img.Height

	f.Seek(0, 0) // rewind

	x, err := exif.Decode(f)
	if err == nil { // EXIF data exists
		decodePhotoExif(&photo, x)
	}

	return &photo, nil
}

func storePhoto(photo *Photo) error {
	var setId, photoId int64

	setName := filepath.Base(filepath.Dir(photo.Path))
	row := selectSetStmt.QueryRow(setName)
	if err := row.Scan(&setId); err == sql.ErrNoRows { // set does not exist
		result, err := insertSetStmt.Exec(setName) // create it
		if err != nil {
			return err
		}

		setId, err = result.LastInsertId()
		if err != nil {
			return err
		}
	}

	row = selectPhotoStmt.QueryRow(photo.Path)
	if err := row.Scan(&photoId); err == sql.ErrNoRows { // photo does not exist
		result, err := insertPhotoStmt.Exec(photo.Aperture, photo.Camera,
			photo.ExposureComp, photo.ExposureTime, photo.Flash, photo.FocalLength,
			photo.FocalLength35, photo.Height, photo.ISO, photo.Lat, photo.Lens,
			photo.Lng, photo.Path, setId, photo.Size, photo.TakenAt, photo.Width) // create it
		if err != nil {
			return err
		}

		photoId, err = result.LastInsertId()
		if err != nil {
			return err
		}

		fmt.Printf("photos id=%d path=%s\n", photoId, photo.Path)
	}

	return nil
}

func walkPath(path string, info os.FileInfo, err error) error {
	if err != nil { // error walking "path"
		return nil // skip
	}

	if info.IsDir() {
		return nil // skip
	}

	if mime.TypeByExtension(filepath.Ext(path)) != "image/jpeg" { // not JPEG
		return nil // skip
	}

	photo, err := decodePhoto(path)
	if err == nil {
		storePhoto(photo)
	}

	return nil // next
}

func updatePhotoSiblings() error {
	var prevId, prevSetId int

	tx, err := db.Begin()
	if err != nil {
		return err
	}

	updatePrevPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET prev_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updatePrevPhotoStmt.Close()

	updateNextPhotoStmt, err := tx.Prepare(`
	UPDATE photos SET next_photo_id = ? WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateNextPhotoStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id FROM photos ORDER BY set_id, taken_at
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId int
		rows.Scan(&id, &setId)

		if setId == prevSetId && prevId > 0 {
			updatePrevPhotoStmt.Exec(prevId, id)
			fmt.Printf("photos id=%d prev_photo_id=%d\n", id, prevId)
			updateNextPhotoStmt.Exec(id, prevId)
			fmt.Printf("photos id=%d next_photo_id=%d\n", prevId, id)
		}

		prevId = id
		prevSetId = setId
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func updateSets() error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	photosCountStmt, err := tx.Prepare(`
	SELECT COUNT(*) FROM photos WHERE set_id = ?
	`)
	if err != nil {
		return err
	}
	defer photosCountStmt.Close()

	updateSetStmt, err := tx.Prepare(`
	UPDATE sets
	SET photos_count = ?, taken_at = ?, thumb_photo_id = ?
	WHERE id = ?
	`)
	if err != nil {
		return err
	}
	defer updateSetStmt.Close()

	rows, err := tx.Query(`
	SELECT id, set_id, MIN(taken_at) FROM photos GROUP BY set_id
	`)
	if err != nil {
		return err
	}
	defer rows.Close()

	for rows.Next() {
		var id, setId, photosCount int
		var takenAt sql.NullString

		rows.Scan(&id, &setId, &takenAt)

		row := photosCountStmt.QueryRow(setId)
		row.Scan(&photosCount)

		updateSetStmt.Exec(photosCount, takenAt, id, setId)
		fmt.Printf("sets id=%d photos_count=%d taken_at=%q thumb_photo_id=%d\n", setId, photosCount, takenAt.String, id)
	}

	if err := rows.Err(); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	return nil
}

func main() {
	var err error

	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "usage: %v path [path...]\n", os.Args[0])
		return
	}

	db, err = sql.Open("sqlite3", "thyme.db") // := here covers global db var
	if err != nil {
		log.Fatalln(err)
	}
	defer db.Close()

	_, err = db.Exec(createSchemaSQL)
	if err != nil {
		log.Fatalln(err)
	}

	selectSetStmt, err = db.Prepare("SELECT id FROM sets WHERE name = ?")
	if err != nil {
		log.Fatalln(err)
	}
	defer selectSetStmt.Close()

	selectPhotoStmt, err = db.Prepare("SELECT id FROM photos WHERE path = ?")
	if err != nil {
		log.Fatalln(err)
	}
	defer selectPhotoStmt.Close()

	insertSetStmt, err = db.Prepare("INSERT INTO sets (name) VALUES (?)")
	if err != nil {
		log.Fatalln(err)
	}
	defer insertSetStmt.Close()

	insertPhotoStmt, err = db.Prepare(`
	INSERT INTO photos (
	aperture, camera, exposure_comp, exposure_time, flash, focal_length,
	focal_length_35, height, iso, lat, lens, lng, path, set_id, size, taken_at,
	width
	)
	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`)
	if err != nil {
		log.Fatalln(err)
	}
	defer insertPhotoStmt.Close()

	for i := 1; i < len(os.Args); i++ {
		filepath.Walk(os.Args[i], walkPath)
	}

	updatePhotoSiblings()
	updateSets()
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/otoolep/raft"
	"github.com/otoolep/rqlite/command"
	"github.com/otoolep/rqlite/db"

	log "code.google.com/p/log4go"
)

type FailedSqlStmt struct {
	Sql   string `json:"sql"`
	Error string `json:"error"`
}

type StmtResponse struct {
	Time     string          `json:"time"`
	Failures []FailedSqlStmt `json:"failures"`
}

type QueryResponse struct {
	Time     string          `json:"time"`
	Failures []FailedSqlStmt `json:"failures"`
	Rows     db.RowResults   `json:"rows"`
}

// The raftd server is a combination of the Raft server and an HTTP
// server which acts as the transport.
type Server struct {
	name       string
	host       string
	port       int
	path       string
	router     *mux.Router
	raftServer raft.Server
	httpServer *http.Server
	database   *db.DB
	mutex      sync.RWMutex
}

// queryParam returns whether the given query param is set to true.
func queryParam(req *http.Request, param string) (bool, error) {
	err := req.ParseForm()
	if err != nil {
		return false, err
	}
	if _, ok := req.Form[param]; ok {
		return true, nil
	}
	return false, nil
}

// isPretty returns whether the HTTP response body should be pretty-printed.
func isPretty(req *http.Request) (bool, error) {
	return queryParam(req, "pretty")
}

// isTransaction returns whether the client requested an explicit
// transaction for the request.
func isTransaction(req *http.Request) (bool, error) {
	return queryParam(req, "transaction")
}

// Creates a new server.
func New(dataDir string, database *db.DB, host string, port int) *Server {
	s := &Server{
		host:     host,
		port:     port,
		path:     dataDir,
		database: database,
		router:   mux.NewRouter(),
	}

	// Read existing name or generate a new one.
	if b, err := ioutil.ReadFile(filepath.Join(dataDir, "name")); err == nil {
		s.name = string(b)
	} else {
		s.name = fmt.Sprintf("%07x", rand.Int())[0:7]
		if err = ioutil.WriteFile(filepath.Join(dataDir, "name"), []byte(s.name), 0644); err != nil {
			panic(err)
		}
	}

	return s
}

// Returns the connection string.
func (s *Server) connectionString() string {
	return fmt.Sprintf("http://%s:%d", s.host, s.port)
}

// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
	var err error

	log.Info("Initializing Raft Server: %s", s.path)

	// Initialize and start Raft server.
	transporter := raft.NewHTTPTransporter("/raft", 200*time.Millisecond)
	s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.database, "")
	if err != nil {
		log.Error("Failed to create new Raft server", err.Error())
		return err
	}
	transporter.Install(s.raftServer, s)
	s.raftServer.Start()

	if leader != "" {
		// Join to leader if specified.

		log.Info("Attempting to join leader at %s", leader)

		if !s.raftServer.IsLogEmpty() {
			log.Error("Cannot join with an existing log")
			return errors.New("Cannot join with an existing log")
		}
		if err := s.Join(leader); err != nil {
			log.Error("Failed to join leader", err.Error())
			return err
		}

	} else if s.raftServer.IsLogEmpty() {
		// Initialize the server by joining itself.

		log.Info("Initializing new cluster")

		_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
			Name:             s.raftServer.Name(),
			ConnectionString: s.connectionString(),
		})
		if err != nil {
			log.Error("Failed to join to self", err.Error())
		}

	} else {
		log.Info("Recovered from log")
	}

	log.Info("Initializing HTTP server")

	// Initialize and start HTTP server.
	s.httpServer = &http.Server{
		Addr:    fmt.Sprintf(":%d", s.port),
		Handler: s.router,
	}

	s.router.HandleFunc("/db", s.readHandler).Methods("GET")
	s.router.HandleFunc("/db", s.writeHandler).Methods("POST")
	s.router.HandleFunc("/join", s.joinHandler).Methods("POST")

	log.Info("Listening at %s", s.connectionString())

	return s.httpServer.ListenAndServe()
}

// This is a hack around Gorilla mux not providing the correct net/http
// HandleFunc() interface.
func (s *Server) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) {
	s.router.HandleFunc(pattern, handler)
}

// Joins to the leader of an existing cluster.
func (s *Server) Join(leader string) error {
	command := &raft.DefaultJoinCommand{
		Name:             s.raftServer.Name(),
		ConnectionString: s.connectionString(),
	}

	var b bytes.Buffer
	json.NewEncoder(&b).Encode(command)
	resp, err := http.Post(fmt.Sprintf("http://%s/join", leader), "application/json", &b)
	if err != nil {
		return err
	}
	resp.Body.Close()

	return nil
}

func (s *Server) joinHandler(w http.ResponseWriter, req *http.Request) {
	command := &raft.DefaultJoinCommand{}

	if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if _, err := s.raftServer.Do(command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (s *Server) readHandler(w http.ResponseWriter, req *http.Request) {
	log.Trace("readHandler for URL: %s", req.URL)
	var failures = make([]FailedSqlStmt, 0)

	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Trace("Bad HTTP request", err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	stmt := string(b)
	startTime := time.Now()
	r, err := s.database.Query(stmt)
	if err != nil {
		log.Trace("Bad SQL statement", err.Error())
		failures = append(failures, FailedSqlStmt{stmt, err.Error()})
	}
	duration := time.Since(startTime)

	rr := QueryResponse{Time: duration.String(), Failures: failures, Rows: r}
	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(rr, "", "    ")
	} else {
		b, err = json.Marshal(rr)
	}
	if err != nil {
		log.Trace("Failed to marshal JSON data", err.Error())
		http.Error(w, err.Error(), http.StatusBadRequest) // Internal error actually
	} else {
		w.Write([]byte(b))
	}
}

func (s *Server) writeHandler(w http.ResponseWriter, req *http.Request) {
	log.Trace("writeHandler for URL: %s", req.URL)

	var failures = make([]FailedSqlStmt, 0)
	var startTime time.Time

	// Read the value from the POST body.
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Trace("Bad HTTP request", err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	stmts := strings.Split(string(b), "\n")
	if stmts[len(stmts)-1] == "" {
		stmts = stmts[:len(stmts)-1]
	}

	log.Trace("Execute statement contains %d commands", len(stmts))
	if len(stmts) == 0 {
		log.Trace("No database execute commands supplied")
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	startTime = time.Now()
	transaction, _ := isTransaction(req)
	if transaction {
		log.Trace("Transaction requested")
		_, err = s.raftServer.Do(command.NewTransactionExecuteCommandSet(stmts))
		if err != nil {
			failures = append(failures, FailedSqlStmt{stmts[0], err.Error()})
			log.Trace("Transaction failed: %s", err.Error())
		}
	} else {
		log.Trace("No transaction requested")
		for i := range stmts {
			_, err = s.raftServer.Do(command.NewExecuteCommand(stmts[i]))
			if err != nil {
				log.Trace("Execute statement %s failed: %s", stmts[i], err.Error())
				failures = append(failures, FailedSqlStmt{stmts[i], err.Error()})
			}
		}
	}
	duration := time.Since(startTime)

	wr := StmtResponse{Time: duration.String(), Failures: failures}
	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(wr, "", "    ")
	} else {
		b, err = json.Marshal(wr)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		w.Write([]byte(b))
	}
}
<commit_msg>Basic sever statistics support<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/mux"
	"github.com/otoolep/raft"
	"github.com/otoolep/rqlite/command"
	"github.com/otoolep/rqlite/db"
	"github.com/otoolep/rqlite/interfaces"
	"github.com/rcrowley/go-metrics"

	log "code.google.com/p/log4go"
)

type FailedSqlStmt struct {
	Sql   string `json:"sql"`
	Error string `json:"error"`
}

type StmtResponse struct {
	Time     string          `json:"time"`
	Failures []FailedSqlStmt `json:"failures"`
}

type QueryResponse struct {
	Time     string          `json:"time"`
	Failures []FailedSqlStmt `json:"failures"`
	Rows     db.RowResults   `json:"rows"`
}

type ServerMetrics struct {
	registry          metrics.Registry
	queryReceived     metrics.Counter
	querySuccess      metrics.Counter
	queryFail         metrics.Counter
	executeReceived   metrics.Counter
	executeTxReceived metrics.Counter
	executeSuccess    metrics.Counter
	executeFail       metrics.Counter
}

// The raftd server is a combination of the Raft server and an HTTP
// server which acts as the transport.
type Server struct {
	name       string
	host       string
	port       int
	path       string
	router     *mux.Router
	raftServer raft.Server
	httpServer *http.Server
	database   *db.DB
	metrics    *ServerMetrics
	mutex      sync.RWMutex
}

// queryParam returns whether the given query param is set to true.
func queryParam(req *http.Request, param string) (bool, error) {
	err := req.ParseForm()
	if err != nil {
		return false, err
	}
	if _, ok := req.Form[param]; ok {
		return true, nil
	}
	return false, nil
}

// isPretty returns whether the HTTP response body should be pretty-printed.
func isPretty(req *http.Request) (bool, error) {
	return queryParam(req, "pretty")
}

// isTransaction returns whether the client requested an explicit
// transaction for the request.
func isTransaction(req *http.Request) (bool, error) {
	return queryParam(req, "transaction")
}

// Creates a new ServerMetrics object.
func NewServerMetrics() *ServerMetrics {
	m := &ServerMetrics{
		registry:          metrics.NewRegistry(),
		queryReceived:     metrics.NewCounter(),
		querySuccess:      metrics.NewCounter(),
		queryFail:         metrics.NewCounter(),
		executeReceived:   metrics.NewCounter(),
		executeTxReceived: metrics.NewCounter(),
		executeSuccess:    metrics.NewCounter(),
		executeFail:       metrics.NewCounter(),
	}

	m.registry.Register("query.Received", m.queryReceived)
	m.registry.Register("query.success", m.querySuccess)
	m.registry.Register("query.fail", m.queryFail)
	m.registry.Register("execute.Received", m.executeReceived)
	m.registry.Register("execute.tx.received", m.executeTxReceived)
	m.registry.Register("execute.success", m.executeSuccess)
	m.registry.Register("execute.fail", m.executeFail)
	return m
}

// Creates a new server.
func New(dataDir string, database *db.DB, host string, port int) *Server {
	s := &Server{
		host:     host,
		port:     port,
		path:     dataDir,
		database: database,
		metrics:  NewServerMetrics(),
		router:   mux.NewRouter(),
	}

	// Read existing name or generate a new one.
	if b, err := ioutil.ReadFile(filepath.Join(dataDir, "name")); err == nil {
		s.name = string(b)
	} else {
		s.name = fmt.Sprintf("%07x", rand.Int())[0:7]
		if err = ioutil.WriteFile(filepath.Join(dataDir, "name"), []byte(s.name), 0644); err != nil {
			panic(err)
		}
	}

	return s
}

// GetStatistics returns an object storing statistics, which supports JSON
// marshalling.
func (s *Server) GetStatistics() (metrics.Registry, error) {
	return s.metrics.registry, nil
}

// Returns the connection string.
func (s *Server) connectionString() string {
	return fmt.Sprintf("http://%s:%d", s.host, s.port)
}

// Starts the server.
func (s *Server) ListenAndServe(leader string) error {
	var err error

	log.Info("Initializing Raft Server: %s", s.path)

	// Initialize and start Raft server.
	transporter := raft.NewHTTPTransporter("/raft", 200*time.Millisecond)
	s.raftServer, err = raft.NewServer(s.name, s.path, transporter, nil, s.database, "")
	if err != nil {
		log.Error("Failed to create new Raft server", err.Error())
		return err
	}
	transporter.Install(s.raftServer, s)
	s.raftServer.Start()

	if leader != "" {
		// Join to leader if specified.

		log.Info("Attempting to join leader at %s", leader)

		if !s.raftServer.IsLogEmpty() {
			log.Error("Cannot join with an existing log")
			return errors.New("Cannot join with an existing log")
		}
		if err := s.Join(leader); err != nil {
			log.Error("Failed to join leader", err.Error())
			return err
		}

	} else if s.raftServer.IsLogEmpty() {
		// Initialize the server by joining itself.

		log.Info("Initializing new cluster")

		_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
			Name:             s.raftServer.Name(),
			ConnectionString: s.connectionString(),
		})
		if err != nil {
			log.Error("Failed to join to self", err.Error())
		}

	} else {
		log.Info("Recovered from log")
	}

	log.Info("Initializing HTTP server")

	// Initialize and start HTTP server.
	s.httpServer = &http.Server{
		Addr:    fmt.Sprintf(":%d", s.port),
		Handler: s.router,
	}

	s.router.HandleFunc("/statistics", s.serveStatistics).Methods("GET")
	s.router.HandleFunc("/db", s.readHandler).Methods("GET")
	s.router.HandleFunc("/db", s.writeHandler).Methods("POST")
	s.router.HandleFunc("/join", s.joinHandler).Methods("POST")

	log.Info("Listening at %s", s.connectionString())

	return s.httpServer.ListenAndServe()
}

// This is a hack around Gorilla mux not providing the correct net/http
// HandleFunc() interface.
func (s *Server) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) {
	s.router.HandleFunc(pattern, handler)
}

// Joins to the leader of an existing cluster.
func (s *Server) Join(leader string) error {
	command := &raft.DefaultJoinCommand{
		Name:             s.raftServer.Name(),
		ConnectionString: s.connectionString(),
	}

	var b bytes.Buffer
	json.NewEncoder(&b).Encode(command)
	resp, err := http.Post(fmt.Sprintf("http://%s/join", leader), "application/json", &b)
	if err != nil {
		return err
	}
	resp.Body.Close()

	return nil
}

func (s *Server) joinHandler(w http.ResponseWriter, req *http.Request) {
	command := &raft.DefaultJoinCommand{}

	if err := json.NewDecoder(req.Body).Decode(&command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if _, err := s.raftServer.Do(command); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (s *Server) readHandler(w http.ResponseWriter, req *http.Request) {
	log.Trace("readHandler for URL: %s", req.URL)
	s.metrics.queryReceived.Inc(1)

	var failures = make([]FailedSqlStmt, 0)

	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Trace("Bad HTTP request", err.Error())
		w.WriteHeader(http.StatusBadRequest)
		s.metrics.queryFail.Inc(1)
		return
	}

	stmt := string(b)
	startTime := time.Now()
	r, err := s.database.Query(stmt)
	if err != nil {
		log.Trace("Bad SQL statement", err.Error())
		s.metrics.queryFail.Inc(1)
		failures = append(failures, FailedSqlStmt{stmt, err.Error()})
	} else {
		s.metrics.querySuccess.Inc(1)
	}
	duration := time.Since(startTime)

	rr := QueryResponse{Time: duration.String(), Failures: failures, Rows: r}
	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(rr, "", "    ")
	} else {
		b, err = json.Marshal(rr)
	}
	if err != nil {
		log.Trace("Failed to marshal JSON data", err.Error())
		http.Error(w, err.Error(), http.StatusBadRequest) // Internal error actually
	} else {
		w.Write([]byte(b))
	}
}

func (s *Server) writeHandler(w http.ResponseWriter, req *http.Request) {
	log.Trace("writeHandler for URL: %s", req.URL)
	s.metrics.executeReceived.Inc(1)

	var failures = make([]FailedSqlStmt, 0)
	var startTime time.Time

	// Read the value from the POST body.
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Trace("Bad HTTP request", err.Error())
		s.metrics.executeFail.Inc(1)
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	stmts := strings.Split(string(b), "\n")
	if stmts[len(stmts)-1] == "" {
		stmts = stmts[:len(stmts)-1]
	}

	log.Trace("Execute statement contains %d commands", len(stmts))
	if len(stmts) == 0 {
		log.Trace("No database execute commands supplied")
		s.metrics.executeFail.Inc(1)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	startTime = time.Now()
	transaction, _ := isTransaction(req)
	if transaction {
		log.Trace("Transaction requested")
		s.metrics.executeTxReceived.Inc(1)

		_, err = s.raftServer.Do(command.NewTransactionExecuteCommandSet(stmts))
		if err != nil {
			log.Trace("Transaction failed: %s", err.Error())
			s.metrics.executeFail.Inc(1)
			failures = append(failures, FailedSqlStmt{stmts[0], err.Error()})
		} else {
			s.metrics.executeSuccess.Inc(1)
		}
	} else {
		log.Trace("No transaction requested")
		for i := range stmts {
			_, err = s.raftServer.Do(command.NewExecuteCommand(stmts[i]))
			if err != nil {
				log.Trace("Execute statement %s failed: %s", stmts[i], err.Error())
				s.metrics.executeFail.Inc(1)
				failures = append(failures, FailedSqlStmt{stmts[i], err.Error()})
			} else {
				s.metrics.executeSuccess.Inc(1)
			}

		}
	}
	duration := time.Since(startTime)

	wr := StmtResponse{Time: duration.String(), Failures: failures}
	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(wr, "", "    ")
	} else {
		b, err = json.Marshal(wr)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		w.Write([]byte(b))
	}
}

// serveStatistics returns the statistics for the program
func (s *Server) serveStatistics(w http.ResponseWriter, req *http.Request) {
	statistics := make(map[string]interface{})
	resources := map[string]interfaces.Statistics{"server": s}
	for k, v := range resources {
		s, err := v.GetStatistics()
		if err != nil {
			log.Error("failed to get " + k + " stats")
			http.Error(w, "failed to get "+k+" stats", http.StatusInternalServerError)
			return
		}
		statistics[k] = s
	}

	var b []byte
	var err error
	pretty, _ := isPretty(req)
	if pretty {
		b, err = json.MarshalIndent(statistics, "", "    ")
	} else {
		b, err = json.Marshal(statistics)
	}
	if err != nil {
		log.Error("failed to JSON marshal statistics map")
		http.Error(w, "failed to JSON marshal statistics map", http.StatusInternalServerError)
		return
	}
	w.Write(b)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
)

type jsonobj map[string]interface{}

func (j jsonobj) get(ks ...string) (interface{}, bool) {
	o := j
	for i, k := range ks {
		v, found := o[k]
		if !found {
			return o, false
		}
		if i == len(ks)-1 {
			return v, true
		}
		jv, ok := v.(jsonobj)
		if !ok {
			return v, false
		}
		o = jv
	}
	return o, false
}

func (j jsonobj) getint(ks ...string) (int, bool) {
	v, found := j.get(ks...)
	if !found {
		return 0, false
	}
	i, ok := v.(int)
	return i, ok
}

func (j jsonobj) getstring(ks ...string) (string, bool) {
	v, found := j.get(ks...)
	if !found {
		return "", false
	}
	s, ok := v.(string)
	return s, ok
}

type files []jsonobj

func (fs files) sort() {
	sort.Slice(fs, func(i, j int) bool {
		// Sort by port number
		pi, foundi := fs[i].getint("file", "port")
		pj, foundj := fs[j].getint("file", "port")
		if foundi && foundj {
			return pi < pj
		}

		// Then by bind address
		hi, foundi := fs[i].getstring("file", "host")
		hj, foundj := fs[j].getstring("file", "host")
		if foundi && foundj {
			return hi < hj
		}

		return false
	})
}

func lsof() (files, error) {
	args := strings.Split("lsof -Pni -sTCP:LISTEN -F0", " ")
	out, err := exec.Command("sudo", args...).Output()
	if err != nil {
		return nil, err
	}
	ret := []jsonobj{}
	var process jsonobj
	for _, line := range strings.Split(string(out), "\n") {
		if line == "" {
			continue
		}

		file := jsonobj{}
		tcptpi := jsonobj{}
		add := false
		var target *jsonobj
		switch line[0] {
		case 'p':
			process = jsonobj{}
			target = &process
		case 'f':
			target = &file
			add = true
		default:
			return nil, fmt.Errorf("Unparsed line: [%s]", line)
		}

		for _, kv := range strings.Split(line, "\x00") {
			if kv == "" {
				continue
			}

			k, v := kv[0:1], kv[1:]

			switch k {
			case "T":
				tcptpi[v[0:2]] = v[3:]
				continue
			default:
				(*target)[k] = kv[1:]
			}

			switch k {
			case "n":
				if host, portstring, err := net.SplitHostPort(v); err == nil {
					file["host"] = host
					if port, err := strconv.Atoi(portstring); err == nil {
						file["port"] = port
					}
				}
			}
		}

		if add {
			ret = append(ret, jsonobj{
				"process": process,
				"file":    file,
				"tcptpi":  tcptpi,
			})
		}
	}
	return ret, nil
}

func main() {
	printJson := false

	flag.BoolVar(&printJson, "json", printJson, "Output JSON-encoded text")

	flag.Parse()

	files, err := lsof()
	if err != nil {
		log.Fatal(err)
	}
	files.sort()

	if printJson {
		enc := json.NewEncoder(os.Stdout)
		for _, f := range files {
			enc.Encode(f)
		}
		return
	}

	rows := [][]string{}
	fieldLengths := []int{}
	for _, f := range files {
		row := []string{}
		for _, path := range []string{"process.c", "file.port", "file.host"} {
			v, ok := f.get(strings.Split(path, ".")...)
			if !ok {
				v = fmt.Sprintf("(%s)", path)
			}
			s := fmt.Sprintf("%v", v)
			row = append(row, s)
			l := len(s)
			if len(fieldLengths) >= len(row) {
				if fieldLengths[len(row)-1] < l {
					fieldLengths[len(row)-1] = l
				}
			} else {
				fieldLengths = append(fieldLengths, l)
			}
		}
		rows = append(rows, row)
	}
	for _, row := range rows {
		for i, f := range row {
			if i > 0 {
				fmt.Printf(" ")
			}
			fmt.Printf("%*s", -fieldLengths[i], f)
		}
		fmt.Printf("\n")
	}
}
<commit_msg>Pass `ports` args through to `lsof -i`<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
)

type jsonobj map[string]interface{}

func (j jsonobj) get(ks ...string) (interface{}, bool) {
	o := j
	for i, k := range ks {
		v, found := o[k]
		if !found {
			return o, false
		}
		if i == len(ks)-1 {
			return v, true
		}
		jv, ok := v.(jsonobj)
		if !ok {
			return v, false
		}
		o = jv
	}
	return o, false
}

func (j jsonobj) getint(ks ...string) (int, bool) {
	v, found := j.get(ks...)
	if !found {
		return 0, false
	}
	i, ok := v.(int)
	return i, ok
}

func (j jsonobj) getstring(ks ...string) (string, bool) {
	v, found := j.get(ks...)
	if !found {
		return "", false
	}
	s, ok := v.(string)
	return s, ok
}

type files []jsonobj

func (fs files) sort() {
	sort.Slice(fs, func(i, j int) bool {
		// Sort by port number
		pi, foundi := fs[i].getint("file", "port")
		pj, foundj := fs[j].getint("file", "port")
		if foundi && foundj {
			return pi < pj
		}

		// Then by bind address
		hi, foundi := fs[i].getstring("file", "host")
		hj, foundj := fs[j].getstring("file", "host")
		if foundi && foundj {
			return hi < hj
		}

		return false
	})
}

func lsof(args ...string) (files, error) {
	cmd := append(strings.Split("lsof -sTCP:LISTEN -F0 -Pni", " "), args...)
	out, err := exec.Command("sudo", cmd...).Output()
	if err != nil {
		return nil, err
	}
	ret := []jsonobj{}
	var process jsonobj
	for _, line := range strings.Split(string(out), "\n") {
		if line == "" {
			continue
		}

		file := jsonobj{}
		tcptpi := jsonobj{}
		add := false
		var target *jsonobj
		switch line[0] {
		case 'p':
			process = jsonobj{}
			target = &process
		case 'f':
			target = &file
			add = true
		default:
			return nil, fmt.Errorf("Unparsed line: [%s]", line)
		}

		for _, kv := range strings.Split(line, "\x00") {
			if kv == "" {
				continue
			}

			k, v := kv[0:1], kv[1:]

			switch k {
			case "T":
				tcptpi[v[0:2]] = v[3:]
				continue
			default:
				(*target)[k] = kv[1:]
			}

			switch k {
			case "n":
				if host, portstring, err := net.SplitHostPort(v); err == nil {
					file["host"] = host
					if port, err := strconv.Atoi(portstring); err == nil {
						file["port"] = port
					}
				}
			}
		}

		if add {
			ret = append(ret, jsonobj{
				"process": process,
				"file":    file,
				"tcptpi":  tcptpi,
			})
		}
	}
	return ret, nil
}

func main() {
	printJson := false

	flag.BoolVar(&printJson, "json", printJson, "Output JSON-encoded text")

	flag.Parse()

	files, err := lsof(flag.Args()...)
	if err != nil {
		log.Fatal(err)
	}
	files.sort()

	if printJson {
		enc := json.NewEncoder(os.Stdout)
		for _, f := range files {
			enc.Encode(f)
		}
		return
	}

	rows := [][]string{}
	fieldLengths := []int{}
	for _, f := range files {
		row := []string{}
		for _, path := range []string{"process.c", "file.port", "file.host"} {
			v, ok := f.get(strings.Split(path, ".")...)
			if !ok {
				v = fmt.Sprintf("(%s)", path)
			}
			s := fmt.Sprintf("%v", v)
			row = append(row, s)
			l := len(s)
			if len(fieldLengths) >= len(row) {
				if fieldLengths[len(row)-1] < l {
					fieldLengths[len(row)-1] = l
				}
			} else {
				fieldLengths = append(fieldLengths, l)
			}
		}
		rows = append(rows, row)
	}
	for _, row := range rows {
		for i, f := range row {
			if i > 0 {
				fmt.Printf(" ")
			}
			fmt.Printf("%*s", -fieldLengths[i], f)
		}
		fmt.Printf("\n")
	}
}
<|endoftext|>


<commit_before>// timeutil provides a set of time utilities including comparisons,
// conversion to "DT8" int32 and "DT14" int64 formats and other
// capabilities.
package timeutil

import (
	"fmt"
	"strconv"
	"time"
)

const (
	DT14 = "20060102150405"
	DT8  = "20060102"
)

// IsGreaterThan compares two times and returns true if the left
// time is greater than the right time.
func IsGreaterThan(timeLeft time.Time, timeRight time.Time) bool {
	durDelta := timeLeft.Sub(timeRight)
	if durZero, _ := time.ParseDuration("0ns"); durDelta > durZero {
		return true
	}
	return false
}

// IsLessThan compares two times and returns true if the left
// time is less than the right time.
func IsLessThan(timeLeft time.Time, timeRight time.Time) bool {
	durDelta := timeLeft.Sub(timeRight)
	if durZero, _ := time.ParseDuration("0ns"); durDelta < durZero {
		return true
	}
	return false
}

// Dt8Now returns DT8 for the current time.
func Dt8Now() int32 {
	return Dt8ForTime(time.Now())
}

func Dt8ForString(layout, value string) (int32, error) {
	dt8 := int32(0)
	t, err := time.Parse(layout, value)
	if err == nil {
		dt8 = Dt8ForTime(t)
	}
	return dt8, err
}

func Dt8ForInts(yyyy int, mm int, dd int) int32 {
	sDt8 := fmt.Sprintf("%04d%02d%02d", yyyy, mm, dd)
	iDt8, _ := strconv.ParseInt(sDt8, 10, 32)
	return int32(iDt8)
}

func Dt8ForTime(t time.Time) int32 {
	u := t.UTC()
	s := u.Format(DT8)
	iDt8, _ := strconv.ParseInt(s, 10, 32)
	return int32(iDt8)
}

func TimeForDt8(dt8 int32) (time.Time, error) {
	return time.Parse(DT8, strconv.FormatInt(int64(dt8), 10))
}

func DurationForNowSubDt8(dt8 int32) (time.Duration, error) {
	t, err := TimeForDt8(dt8)
	if err != nil {
		var d time.Duration
		return d, err
	}
	now := time.Now()
	return now.Sub(t), nil
}

func Dt14Now() int64 {
	return Dt14ForTime(time.Now())
}

func Dt14ForString(layout, value string) (int64, error) {
	dt14 := int64(0)
	t, err := time.Parse(layout, value)
	if err == nil {
		dt14 = Dt14ForTime(t)
	}
	return dt14, err
}

func Dt14ForInts(yyyy int, mm int, dd int, hr int, mn int, dy int) int64 {
	sDt14 := fmt.Sprintf("%04d%02d%02d%02d%02d%02d", yyyy, mm, dd, hr, mn, dy)
	iDt14, _ := strconv.ParseInt(sDt14, 10, 64)
	return int64(iDt14)
}

func Dt14ForTime(t time.Time) int64 {
	u := t.UTC()
	s := u.Format(DT14)
	iDt14, _ := strconv.ParseInt(s, 10, 64)
	return int64(iDt14)
}

func TimeForDt14(dt14 int64) (time.Time, error) {
	return time.Parse(DT14, strconv.FormatInt(dt14, 10))
}
<commit_msg>add initial DT6 support to timeutil<commit_after>// timeutil provides a set of time utilities including comparisons,
// conversion to "DT8" int32 and "DT14" int64 formats and other
// capabilities.

package timeutil

import (
	"fmt"
	"strconv"
	"time"
)

const (
	DT14 = "20060102150405"
	DT6  = "200601"
	DT8  = "20060102"
)

// IsGreaterThan compares two times and returns true if the left
// time is greater than the right time.

func IsGreaterThan(timeLeft time.Time, timeRight time.Time) bool {
	durDelta := timeLeft.Sub(timeRight)
	if durZero, _ := time.ParseDuration("0ns"); durDelta > durZero {
		return true
	}
	return false
}

// IsLessThan compares two times and returns true if the left
// time is less than the right time.

func IsLessThan(timeLeft time.Time, timeRight time.Time) bool {
	durDelta := timeLeft.Sub(timeRight)
	if durZero, _ := time.ParseDuration("0ns"); durDelta < durZero {
		return true
	}
	return false
}

func Dt6ForDt14(dt14 int64) int32 {
	dt16f := float64(dt14) / float64(10000000)
	return int32(dt16f)
}

// Dt8Now returns DT8 for the current time.

func Dt8Now() int32 {
	return Dt8ForTime(time.Now())
}

func Dt8ForString(layout, value string) (int32, error) {
	dt8 := int32(0)
	t, err := time.Parse(layout, value)
	if err == nil {
		dt8 = Dt8ForTime(t)
	}
	return dt8, err
}

func Dt8ForInts(yyyy int, mm int, dd int) int32 {
	sDt8 := fmt.Sprintf("%04d%02d%02d", yyyy, mm, dd)
	iDt8, _ := strconv.ParseInt(sDt8, 10, 32)
	return int32(iDt8)
}

func Dt8ForTime(t time.Time) int32 {
	u := t.UTC()
	s := u.Format(DT8)
	iDt8, _ := strconv.ParseInt(s, 10, 32)
	return int32(iDt8)
}

func TimeForDt8(dt8 int32) (time.Time, error) {
	return time.Parse(DT8, strconv.FormatInt(int64(dt8), 10))
}

func DurationForNowSubDt8(dt8 int32) (time.Duration, error) {
	t, err := TimeForDt8(dt8)
	if err != nil {
		var d time.Duration
		return d, err
	}
	now := time.Now()
	return now.Sub(t), nil
}

func Dt14Now() int64 {
	return Dt14ForTime(time.Now())
}

func Dt14ForString(layout, value string) (int64, error) {
	dt14 := int64(0)
	t, err := time.Parse(layout, value)
	if err == nil {
		dt14 = Dt14ForTime(t)
	}
	return dt14, err
}

func Dt14ForInts(yyyy int, mm int, dd int, hr int, mn int, dy int) int64 {
	sDt14 := fmt.Sprintf("%04d%02d%02d%02d%02d%02d", yyyy, mm, dd, hr, mn, dy)
	iDt14, _ := strconv.ParseInt(sDt14, 10, 64)
	return int64(iDt14)
}

func Dt14ForTime(t time.Time) int64 {
	u := t.UTC()
	s := u.Format(DT14)
	iDt14, _ := strconv.ParseInt(s, 10, 64)
	return int64(iDt14)
}

func TimeForDt14(dt14 int64) (time.Time, error) {
	return time.Parse(DT14, strconv.FormatInt(dt14, 10))
}
<|endoftext|>


<commit_before>package gateway

import (
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	MaxCatchUpBlocks = 50
)

// threadedResynchronize continuously calls Synchronize on a random peer every
// few minutes. This helps prevent unintentional desychronization in the event
// that broadcasts start failing.
func (g *Gateway) threadedResynchronize() {
	for {
		peer, err := g.randomPeer()
		if err != nil {
			g.log.Println("ERR: no peers are available for synchronization")
			// TODO: sleep and try again instead of returning?
			return
		}
		go g.Synchronize(peer)
		time.Sleep(time.Minute * 2)
	}
}

// Synchronize synchronizes the local consensus set (i.e. the blockchain) with
// the network consensus set. The process is as follows: synchronize asks a
// peer for new blocks. The requester sends 32 block IDs, starting with the 12
// most recent and then progressing exponentially backwards to the genesis
// block. The receiver uses these blocks to find the most recent block seen by
// both peers. From this starting height, it transmits blocks sequentially.
// The requester then integrates these blocks into its consensus set. Multiple
// such transmissions may be required to fully synchronize.
//
// TODO: don't run two Synchronize threads at the same time
func (g *Gateway) Synchronize(peer modules.NetAddress) error {
	g.log.Println("INFO: synchronizing to", peer)
	for {
		var newBlocks []types.Block
		newBlocks, moreAvailable, err := g.requestBlocks(peer)
		if err != nil {
			g.log.Printf("ERR: synchronization to %v failed: %v\n", peer, err)
			return err
		}
		g.log.Printf("INFO: %v sent us %v blocks\n", peer, len(newBlocks))
		for _, block := range newBlocks {
			acceptErr := g.state.AcceptBlock(block)
			if acceptErr != nil {
				// TODO: If the error is a FutureTimestampErr, need to wait before trying the
				// block again.
				g.log.Printf("WARN: state rejected a block from %v: %v\n", peer, acceptErr)
			}
		}

		// loop until there are no more blocks available
		if !moreAvailable {
			break
		}
	}
	g.log.Printf("INFO: synchronization to %v complete\n", peer)
	return nil
}

// sendBlocks returns a sequential set of blocks based on the 32 input block
// IDs. The most recent known ID is used as the starting point, and up to
// 'MaxCatchUpBlocks' from that BlockHeight onwards are returned. It also
// sends a boolean indicating whether more blocks are available.
func (g *Gateway) sendBlocks(conn modules.NetConn) (err error) {
	// Read known blocks.
	var knownBlocks [32]types.BlockID
	err = conn.ReadObject(&knownBlocks, 32*crypto.HashSize)
	if err != nil {
		return
	}

	// Find the most recent block from knownBlocks that is in our current path.
	found := false
	var start types.BlockHeight
	for _, id := range knownBlocks {
		if height, exists := g.state.HeightOfBlock(id); exists {
			found = true
			start = height + 1 // start at child
			break
		}
	}
	// If we didn't find any matching blocks, or if we're already
	// synchronized, don't send any blocks. The genesis block should be
	// included in knownBlocks, so if no matching blocks are found, the caller
	// is probably on a different blockchain altogether.
	if !found || start > g.state.Height() {
		// Send 0 blocks.
		err = conn.WriteObject([]types.Block{})
		if err != nil {
			return
		}
		// Indicate that no more blocks are available.
		return conn.WriteObject(false)
	}

	// Determine range of blocks to send.
	stop := start + MaxCatchUpBlocks
	if stop > g.state.Height() {
		stop = g.state.Height()
	}
	blocks, err := g.state.BlockRange(start, stop)
	if err != nil {
		return
	}
	g.log.Printf("INFO: %v is at height %v (-%v); sending them %v blocks\n", conn.Addr(), start, g.state.Height()-start, len(blocks))
	err = conn.WriteObject(blocks)
	if err != nil {
		return
	}

	// Indicate whether more blocks are available.
	more := g.state.Height() > stop
	return conn.WriteObject(more)
}

func (g *Gateway) requestBlocks(peer modules.NetAddress) (newBlocks []types.Block, moreAvailable bool, err error) {
	history := g.blockHistory()
	err = g.RPC(peer, "SendBlocks", func(conn modules.NetConn) error {
		err := conn.WriteObject(history)
		if err != nil {
			return err
		}
		err = conn.ReadObject(&newBlocks, MaxCatchUpBlocks*types.BlockSizeLimit)
		if err != nil {
			return err
		}
		return conn.ReadObject(&moreAvailable, 1)
	})
	return
}

// blockHistory returns up to 32 BlockIDs, starting with the 12 most recent
// BlockIDs and then doubling in step size until the genesis block is reached.
// The genesis block is always included. This array of BlockIDs is used to
// establish a shared commonality between peers during synchronization.
func (g *Gateway) blockHistory() (blockIDs [32]types.BlockID) {
	knownBlocks := make([]types.BlockID, 0, 32)
	step := types.BlockHeight(1)
	for height := g.state.Height(); ; height -= step {
		block, exists := g.state.BlockAtHeight(height)
		if !exists {
			// faulty state; log high-priority error
			g.log.Println("ERR: state is missing a block at height", height)
			return
		}
		knownBlocks = append(knownBlocks, block.ID())

		// after 12, start doubling
		if len(knownBlocks) >= 12 {
			step *= 2
		}

		// this check has to come before height -= step;
		// otherwise we might underflow
		if height <= step {
			break
		}
	}

	// always include the genesis block
	genesis, exists := g.state.BlockAtHeight(0)
	if !exists {
		g.log.Println("ERR: state is missing a genesis block")
		return
	}
	knownBlocks = append(knownBlocks, genesis.ID())

	copy(blockIDs[:], knownBlocks)
	return
}
<commit_msg>more robust threadedResynchronize<commit_after>package gateway

import (
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	MaxCatchUpBlocks = 50
)

// threadedResynchronize continuously calls Synchronize on a random peer every
// few minutes. This helps prevent unintentional desychronization in the event
// that broadcasts start failing.
func (g *Gateway) threadedResynchronize() {
	for {
		go func() {
			// max 10 attempts
			for i := 0; i < 10; i++ {
				peer, err := g.randomPeer()
				if err != nil {
					g.log.Println("ERR: no peers are available for synchronization")
					return
				}
				// keep looping until a successful Synchronize
				if g.Synchronize(peer) == nil {
					return
				}
			}
		}()
		time.Sleep(time.Minute * 2)
	}
}

// Synchronize synchronizes the local consensus set (i.e. the blockchain) with
// the network consensus set. The process is as follows: synchronize asks a
// peer for new blocks. The requester sends 32 block IDs, starting with the 12
// most recent and then progressing exponentially backwards to the genesis
// block. The receiver uses these blocks to find the most recent block seen by
// both peers. From this starting height, it transmits blocks sequentially.
// The requester then integrates these blocks into its consensus set. Multiple
// such transmissions may be required to fully synchronize.
//
// TODO: don't run two Synchronize threads at the same time
func (g *Gateway) Synchronize(peer modules.NetAddress) error {
	g.log.Println("INFO: synchronizing to", peer)
	for {
		var newBlocks []types.Block
		newBlocks, moreAvailable, err := g.requestBlocks(peer)
		if err != nil {
			g.log.Printf("ERR: synchronization to %v failed: %v\n", peer, err)
			return err
		}
		g.log.Printf("INFO: %v sent us %v blocks\n", peer, len(newBlocks))
		for _, block := range newBlocks {
			acceptErr := g.state.AcceptBlock(block)
			if acceptErr != nil {
				// TODO: If the error is a FutureTimestampErr, need to wait before trying the
				// block again.
				g.log.Printf("WARN: state rejected a block from %v: %v\n", peer, acceptErr)
			}
		}

		// loop until there are no more blocks available
		if !moreAvailable {
			break
		}
	}
	g.log.Printf("INFO: synchronization to %v complete\n", peer)
	return nil
}

// sendBlocks returns a sequential set of blocks based on the 32 input block
// IDs. The most recent known ID is used as the starting point, and up to
// 'MaxCatchUpBlocks' from that BlockHeight onwards are returned. It also
// sends a boolean indicating whether more blocks are available.
func (g *Gateway) sendBlocks(conn modules.NetConn) (err error) {
	// Read known blocks.
	var knownBlocks [32]types.BlockID
	err = conn.ReadObject(&knownBlocks, 32*crypto.HashSize)
	if err != nil {
		return
	}

	// Find the most recent block from knownBlocks that is in our current path.
	found := false
	var start types.BlockHeight
	for _, id := range knownBlocks {
		if height, exists := g.state.HeightOfBlock(id); exists {
			found = true
			start = height + 1 // start at child
			break
		}
	}
	// If we didn't find any matching blocks, or if we're already
	// synchronized, don't send any blocks. The genesis block should be
	// included in knownBlocks, so if no matching blocks are found, the caller
	// is probably on a different blockchain altogether.
	if !found || start > g.state.Height() {
		// Send 0 blocks.
		err = conn.WriteObject([]types.Block{})
		if err != nil {
			return
		}
		// Indicate that no more blocks are available.
		return conn.WriteObject(false)
	}

	// Determine range of blocks to send.
	stop := start + MaxCatchUpBlocks
	if stop > g.state.Height() {
		stop = g.state.Height()
	}
	blocks, err := g.state.BlockRange(start, stop)
	if err != nil {
		return
	}
	g.log.Printf("INFO: %v is at height %v (-%v); sending them %v blocks\n", conn.Addr(), start, g.state.Height()-start, len(blocks))
	err = conn.WriteObject(blocks)
	if err != nil {
		return
	}

	// Indicate whether more blocks are available.
	more := g.state.Height() > stop
	return conn.WriteObject(more)
}

func (g *Gateway) requestBlocks(peer modules.NetAddress) (newBlocks []types.Block, moreAvailable bool, err error) {
	history := g.blockHistory()
	err = g.RPC(peer, "SendBlocks", func(conn modules.NetConn) error {
		err := conn.WriteObject(history)
		if err != nil {
			return err
		}
		err = conn.ReadObject(&newBlocks, MaxCatchUpBlocks*types.BlockSizeLimit)
		if err != nil {
			return err
		}
		return conn.ReadObject(&moreAvailable, 1)
	})
	return
}

// blockHistory returns up to 32 BlockIDs, starting with the 12 most recent
// BlockIDs and then doubling in step size until the genesis block is reached.
// The genesis block is always included. This array of BlockIDs is used to
// establish a shared commonality between peers during synchronization.
func (g *Gateway) blockHistory() (blockIDs [32]types.BlockID) {
	knownBlocks := make([]types.BlockID, 0, 32)
	step := types.BlockHeight(1)
	for height := g.state.Height(); ; height -= step {
		block, exists := g.state.BlockAtHeight(height)
		if !exists {
			// faulty state; log high-priority error
			g.log.Println("ERR: state is missing a block at height", height)
			return
		}
		knownBlocks = append(knownBlocks, block.ID())

		// after 12, start doubling
		if len(knownBlocks) >= 12 {
			step *= 2
		}

		// this check has to come before height -= step;
		// otherwise we might underflow
		if height <= step {
			break
		}
	}

	// always include the genesis block
	genesis, exists := g.state.BlockAtHeight(0)
	if !exists {
		g.log.Println("ERR: state is missing a genesis block")
		return
	}
	knownBlocks = append(knownBlocks, genesis.ID())

	copy(blockIDs[:], knownBlocks)
	return
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embeddedkvs

import (
	"context"

	"github.com/dgraph-io/badger/v3"
	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/errors"
)

type KVS struct {
	db *badger.DB
}

// New instantiates KVS.
func New(ctx context.Context, path string) (*KVS, error) {
	db, err := badger.Open(badger.DefaultOptions(path).WithLoggingLevel(badger.WARNING))
	if err != nil {
		return nil, errors.Annotate(err, "failed to open database: %s", path).Err()
	}

	return &KVS{
		db: db,
	}, nil
}

// Close closes KVS.
func (k *KVS) Close() error {
	if err := k.db.Close(); err != nil {
		return errors.Annotate(err, "failed to close db").Err()
	}
	return nil
}

// Set sets key/value to storage.
//
// This should be called in parallel for efficient storing.
func (k *KVS) Set(key string, value []byte) error {
	if err := k.db.Update(func(txn *badger.Txn) error {
		return txn.Set([]byte(key), value)
	}); err != nil {
		return errors.Annotate(err, "failed to put %s", key).Err()
	}
	return nil
}

// GetMulti calls |fn| in parallel for cached entries.
func (k *KVS) GetMulti(keys []string, fn func(key string, value []byte) error) error {
	if err := k.db.View(func(txn *badger.Txn) error {
		var eg errgroup.Group
		for _, key := range keys {
			key := key
			eg.Go(func() error {
				item, err := txn.Get([]byte(key))
				if err == badger.ErrKeyNotFound {
					return nil
				}
				if err != nil {
					return errors.Annotate(err, "failed to get %s", key).Err()
				}
				return item.Value(func(val []byte) error {
					return fn(key, val)
				})

			})
		}

		return eg.Wait()
	}); err != nil {
		return errors.Annotate(err, "failed to get").Err()
	}

	return nil
}

// SetMulti receives callback that takes function which is used to set a key/value pair.
func (k *KVS) SetMulti(fn func(set func(key string, value []byte) error) error) error {
	wb := k.db.NewWriteBatch()
	defer wb.Cancel()

	if err := fn(func(key string, value []byte) error {
		if err := wb.Set([]byte(key), value); err != nil {
			return errors.Annotate(err, "failed to set key: %s", key).Err()
		}
		return nil
	}); err != nil {
		return err
	}

	if err := wb.Flush(); err != nil {
		return errors.Annotate(err, "failed to call Flush").Err()
	}
	return nil
}

// ForEach executes a function for each key/value pair in KVS.
func (k *KVS) ForEach(fn func(key string, value []byte) error) error {
	err := k.db.View(func(txn *badger.Txn) error {
		it := txn.NewIterator(badger.DefaultIteratorOptions)
		defer it.Close()
		for it.Rewind(); it.Valid(); it.Next() {
			item := it.Item()

			if err := item.Value(func(val []byte) error {
				return fn(string(item.Key()), val)
			}); err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		return errors.Annotate(err, "failed to iterate").Err()
	}
	return nil
}
<commit_msg>embeddedkvs: use smaller ValueLogFileSize in small memory machine<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package embeddedkvs

import (
	"context"

	"github.com/dgraph-io/badger/v3"
	"github.com/shirou/gopsutil/mem"
	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/errors"
)

type KVS struct {
	db *badger.DB
}

// New instantiates KVS.
func New(ctx context.Context, path string) (*KVS, error) {
	v, err := mem.VirtualMemory()
	if err != nil {
		return nil, errors.Annotate(err, "failed to get memory stats").Err()
	}
	opt := badger.DefaultOptions(path).WithLoggingLevel(badger.WARNING)

	// Using /3 as ValueLogFileSize is used with 2x in
	// https://github.com/dgraph-io/badger/blob/42d5e9510a4d0165f42a856e203562774aab6603/value.go#L519
	valuelog := int64(v.Available / 3)
	if valuelog < opt.ValueLogFileSize {
		opt.ValueLogFileSize = valuelog
	}

	db, err := badger.Open(opt)
	if err != nil {
		return nil, errors.Annotate(err, "failed to open database: %s", path).Err()
	}

	return &KVS{
		db: db,
	}, nil
}

// Close closes KVS.
func (k *KVS) Close() error {
	if err := k.db.Close(); err != nil {
		return errors.Annotate(err, "failed to close db").Err()
	}
	return nil
}

// Set sets key/value to storage.
//
// This should be called in parallel for efficient storing.
func (k *KVS) Set(key string, value []byte) error {
	if err := k.db.Update(func(txn *badger.Txn) error {
		return txn.Set([]byte(key), value)
	}); err != nil {
		return errors.Annotate(err, "failed to put %s", key).Err()
	}
	return nil
}

// GetMulti calls |fn| in parallel for cached entries.
func (k *KVS) GetMulti(keys []string, fn func(key string, value []byte) error) error {
	if err := k.db.View(func(txn *badger.Txn) error {
		var eg errgroup.Group
		for _, key := range keys {
			key := key
			eg.Go(func() error {
				item, err := txn.Get([]byte(key))
				if err == badger.ErrKeyNotFound {
					return nil
				}
				if err != nil {
					return errors.Annotate(err, "failed to get %s", key).Err()
				}
				return item.Value(func(val []byte) error {
					return fn(key, val)
				})

			})
		}

		return eg.Wait()
	}); err != nil {
		return errors.Annotate(err, "failed to get").Err()
	}

	return nil
}

// SetMulti receives callback that takes function which is used to set a key/value pair.
func (k *KVS) SetMulti(fn func(set func(key string, value []byte) error) error) error {
	wb := k.db.NewWriteBatch()
	defer wb.Cancel()

	if err := fn(func(key string, value []byte) error {
		if err := wb.Set([]byte(key), value); err != nil {
			return errors.Annotate(err, "failed to set key: %s", key).Err()
		}
		return nil
	}); err != nil {
		return err
	}

	if err := wb.Flush(); err != nil {
		return errors.Annotate(err, "failed to call Flush").Err()
	}
	return nil
}

// ForEach executes a function for each key/value pair in KVS.
func (k *KVS) ForEach(fn func(key string, value []byte) error) error {
	err := k.db.View(func(txn *badger.Txn) error {
		it := txn.NewIterator(badger.DefaultIteratorOptions)
		defer it.Close()
		for it.Rewind(); it.Valid(); it.Next() {
			item := it.Item()

			if err := item.Value(func(val []byte) error {
				return fn(string(item.Key()), val)
			}); err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		return errors.Annotate(err, "failed to iterate").Err()
	}
	return nil
}
<|endoftext|>