<commit_before>package cmd

import (
	"fmt"
	"log"
	"math"
	"math/rand"
	"sort"
	"time"

	"github.com/phil-mansfield/shellfish/los/geom"
	"github.com/phil-mansfield/shellfish/cmd/catalog"
	"github.com/phil-mansfield/shellfish/cmd/env"
	"github.com/phil-mansfield/shellfish/logging"
	"github.com/phil-mansfield/shellfish/parse"
	"github.com/phil-mansfield/shellfish/io"
	"github.com/phil-mansfield/shellfish/cmd/memo"
)

type PotentialConfig struct {
	ncells int64
	rGridMult float64
	rMinMult, rMaxMult float64
	frac float64
}

var _ Mode = &PotentialConfig{}

func (config *PotentialConfig) ExampleConfig() string {
	return `[potential.config]
NCells = 64
GridRMult = 8
# RMinMult is the minimum radius inside which particles are used to calculate
# the potential.
RMinMult = 1
# RMaxMult is the maximum radius inside which particles are used to calculate
# the potential.
RMaxMult = 50
# Percentage of particles that will be used to compute the potential.
ParticleFraction = 0.01
`
}

func (config *PotentialConfig) ReadConfig(fname string, flags []string) error {
	vars := parse.NewConfigVars("prof.config")

	vars.Int(&config.ncells, "RBins", 64)
	vars.Float(&config.rGridMult, "GridRMult", 8)
	vars.Float(&config.rMaxMult, "RMaxMult", 1.0)
	vars.Float(&config.rMinMult, "RMinMult", 1.0)
	vars.Float(&config.frac, "ParticleFraction", 1.0)

	if fname == "" {
		if len(flags) == 0 { return nil }

		err := parse.ReadFlags(flags, vars)
		if err != nil { return err }
	} else {
		if err := parse.ReadConfig(fname, vars); err != nil { return err }
		if err := parse.ReadFlags(flags, vars); err != nil { return err }
	}

	return config.validate()
}

func (config *PotentialConfig) validate() error {
	if config.ncells < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"NCells", config.ncells)
	} else if config.rGridMult < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"RGridMult", config.rGridMult)
	} else if config.rMaxMult < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"RMaxMult", config.rMaxMult)
	} else if config.rMinMult < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"RMinMult", config.rMinMult)
	}

	return nil
}

func (config *PotentialConfig) Run(
	gConfig *GlobalConfig, e *env.Environment, stdin []byte,
) ([]string, error) {
	if logging.Mode != logging.Nil {
		log.Println(`
####################
## shellfish prof ##
####################`,
		)
	}

	var t time.Time
	if logging.Mode == logging.Performance {
		t = time.Now()
	}

	icols, fcols, err := catalog.Parse(
		stdin, []int{0, 1}, []int{2, 3, 4, 5, 6},
	)
	if err != nil { return nil, err }

	ids, snaps := icols[0], icols[1]
	hx := [3][]float64{ fcols[0], fcols[1], fcols[2] }
	hr, hm := fcols[3], fcols[4]

	if len(ids) == 0 { return nil, fmt.Errorf("No input halos.") }

	// Initialize phase profiles
	rSets := make([][]float64, len(ids))
	phiSets := [3][][]float64{
		make([][]float64, len(ids)),
		make([][]float64, len(ids)),
		make([][]float64, len(ids)),
	}

	for i := range rSets {
		rSets[i] = make([]float64, 2*config.ncells)
		phiSets[0][i] = make([]float64, config.ncells*config.ncells)
		phiSets[1][i] = make([]float64, config.ncells*config.ncells)
		phiSets[2][i] = make([]float64, config.ncells*config.ncells)
	}

	snapBins, idxBins := binBySnap(snaps, ids)

	sortedSnaps := []int{}
	for snap := range snapBins {
		sortedSnaps = append(sortedSnaps, snap)
	}
	sort.Ints(sortedSnaps)

	buf, err := getVectorBuffer(
		e.ParticleCatalog(snaps[0], 0), gConfig,
	)
	if err != nil {
		return nil, err
	}

	for _, snap := range sortedSnaps {
		if snap == -1 {
			continue
		}

		idxs := idxBins[snap]
		snapCoords := [][]float64{
			make([]float64, len(idxs)), make([]float64, len(idxs)),
			make([]float64, len(idxs)), make([]float64, len(idxs)),
		}

		for i, idx := range idxs {
			snapCoords[0][i] = hx[0][idx]
			snapCoords[1][i] = hx[1][idx]
			snapCoords[2][i] = hx[2][idx]
			snapCoords[3][i] = hr[idx]*config.rMaxMult
		}

		hds, files, err := memo.ReadHeaders(snap, buf, e)
		if err != nil {
			return nil, err
		}
		hxBounds, err := boundingSpheres(snapCoords, &hds[0], e)
		if err != nil {
			return nil, err
		}
		_, intrIdxs := binSphereIntersections(hds, hxBounds)

		for i := range hds {
			if len(intrIdxs[i]) == 0 {
				continue
			}

			xs, _, ms, _, err := buf.Read(files[i])
			if err != nil {
				return nil, err
			}

			// Waarrrgggble
			for _, j := range intrIdxs[i] {
				phisXY := phiSets[0][idxs[j]]
				phisYZ := phiSets[1][idxs[j]]
				phisXZ := phiSets[2][idxs[j]]

				insertPotentialPoints(
					phisXY, phisYZ, phisXZ,
					hxBounds[j],
					xs, ms,
					config, &hds[i],
				)
			}

			buf.Close()
		}
	}

	for i := range rSets {
		processPotential(
			rSets[i],
			phiSets[0][i], phiSets[1][i], phiSets[2][i],
			hr[i], hm[i], config,
		)
	}

	rSets = transpose(rSets)
	phiSets[0] = transpose(phiSets[0])
	phiSets[1] = transpose(phiSets[1])
	phiSets[2] = transpose(phiSets[2])

	order := make([]int, len(rSets) + len(phiSets[0])*3 + 2)
	for i := range order { order[i] = i }
	lines := catalog.FormatCols(
		[][]int{ids, snaps},
		append(append(append(
			rSets, phiSets[0]...),
			phiSets[1]...),
			phiSets[2]...),
		order,
	)

	cString := catalog.CommentString(
		[]string{"ID", "Snapshot", "R [cMpc/h]",
			"Phi_xy/(G Mvir / Rvir)",
			"Phi_yz/(G Mvir / Rvir)",
			"Phi_xz/(G Mvir / Rvir)"},
		[]string{}, []int{0, 1, 2, 3, 4, 5},
		[]int{1, 1, int(config.ncells),
			int(config.ncells*config.ncells),
			int(config.ncells*config.ncells),
			int(config.ncells*config.ncells)},
	)

	if logging.Mode == logging.Performance {
		log.Printf("Time: %s", time.Since(t).String())
		log.Printf("Memory:\n%s", logging.MemString())
	}

	return append([]string{cString}, lines...), nil
}

func insertPotentialPoints(
	phisXY, phisYZ, phisXZ []float64,
	hx geom.Sphere,
	xs [][3]float32,
	ms []float32,
	config *PotentialConfig, hd *io.Header,
) {
	rmax2 := (config.rMaxMult*config.rMaxMult) * float64(hx.R*hx.R)
	rmin2 := (config.rMinMult*config.rMinMult) * float64(hx.R*hx.R)

	delta := float64(hx.R) * config.rGridMult * 2 / float64(config.ncells)
	nc := int(config.ncells)

	for i := range xs {
		if rand.Float64() > config.frac { continue }

		dx0 := float64(xs[i][0] - hx.C[0])
		dy0 := float64(xs[i][1] - hx.C[1])
		dz0 := float64(xs[i][2] - hx.C[2])

		dr02 := dx0*dx0 + dy0*dy0 + dz0*dz0
		pm := float64(ms[i])

		if dr02 > rmax2 || dr02 < rmin2 { continue }

		for iz := 0; iz < nc; iz++ {
			dz := dz0 + delta*float64(iz) - float64(hx.R)*config.rGridMult
			for iy := 0; iy < nc; iy++ {
				dy := dy0 + delta*float64(iy) - float64(hx.R)*config.rGridMult

				dryz := math.Sqrt(dx0*dx0 + dy*dy + dz*dz)
				phisYZ[iy + iz*nc] -= pm / dryz

				if iz != 0 && iy != 0 { continue }

				for ix := 0; ix < nc; ix++ {
					dx := dx0 + delta*float64(ix) -
						float64(hx.R)*config.rGridMult

					if iz == 0 {
						drxy := math.Sqrt(dx*dx + dy*dy + dz0*dz0)
						phisXY[ix + iy*nc] -= pm / drxy
					}

					if iy == 0 {
						drxz := math.Sqrt(dx*dx + dy0*dy0 + dz*dz)
						phisXZ[ix + iz*nc] -= pm / drxz
					}
				}
			}
		}
	}
}

func processPotential(
	rs, phisXY, phisYZ, phisXZ []float64,
	hr, hm float64, config *PotentialConfig,
) {
	for i := range phisXY {
		phisXY[i] /= config.frac * hm/hr
		phisXZ[i] /= config.frac * hm/hr
		phisYZ[i] /= config.frac * hm/hr
	}

	delta := hr * config.rGridMult * 2 / float64(config.ncells)
	for i := range rs {
		rs[i] = delta*float64(i) - hr*config.rGridMult
	}
}
<commit_msg>Fixes size of ourput r data.<commit_after>package cmd

import (
	"fmt"
	"log"
	"math"
	"math/rand"
	"sort"
	"time"

	"github.com/phil-mansfield/shellfish/los/geom"
	"github.com/phil-mansfield/shellfish/cmd/catalog"
	"github.com/phil-mansfield/shellfish/cmd/env"
	"github.com/phil-mansfield/shellfish/logging"
	"github.com/phil-mansfield/shellfish/parse"
	"github.com/phil-mansfield/shellfish/io"
	"github.com/phil-mansfield/shellfish/cmd/memo"
)

type PotentialConfig struct {
	ncells int64
	rGridMult float64
	rMinMult, rMaxMult float64
	frac float64
}

var _ Mode = &PotentialConfig{}

func (config *PotentialConfig) ExampleConfig() string {
	return `[potential.config]
NCells = 64
GridRMult = 8
# RMinMult is the minimum radius inside which particles are used to calculate
# the potential.
RMinMult = 1
# RMaxMult is the maximum radius inside which particles are used to calculate
# the potential.
RMaxMult = 50
# Percentage of particles that will be used to compute the potential.
ParticleFraction = 0.01
`
}

func (config *PotentialConfig) ReadConfig(fname string, flags []string) error {
	vars := parse.NewConfigVars("prof.config")

	vars.Int(&config.ncells, "RBins", 64)
	vars.Float(&config.rGridMult, "GridRMult", 8)
	vars.Float(&config.rMaxMult, "RMaxMult", 1.0)
	vars.Float(&config.rMinMult, "RMinMult", 1.0)
	vars.Float(&config.frac, "ParticleFraction", 1.0)

	if fname == "" {
		if len(flags) == 0 { return nil }

		err := parse.ReadFlags(flags, vars)
		if err != nil { return err }
	} else {
		if err := parse.ReadConfig(fname, vars); err != nil { return err }
		if err := parse.ReadFlags(flags, vars); err != nil { return err }
	}

	return config.validate()
}

func (config *PotentialConfig) validate() error {
	if config.ncells < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"NCells", config.ncells)
	} else if config.rGridMult < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"RGridMult", config.rGridMult)
	} else if config.rMaxMult < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"RMaxMult", config.rMaxMult)
	} else if config.rMinMult < 0 {
		return fmt.Errorf("The variable '%s' was set to %d.",
			"RMinMult", config.rMinMult)
	}

	return nil
}

func (config *PotentialConfig) Run(
	gConfig *GlobalConfig, e *env.Environment, stdin []byte,
) ([]string, error) {
	if logging.Mode != logging.Nil {
		log.Println(`
####################
## shellfish prof ##
####################`,
		)
	}

	var t time.Time
	if logging.Mode == logging.Performance {
		t = time.Now()
	}

	icols, fcols, err := catalog.Parse(
		stdin, []int{0, 1}, []int{2, 3, 4, 5, 6},
	)
	if err != nil { return nil, err }

	ids, snaps := icols[0], icols[1]
	hx := [3][]float64{ fcols[0], fcols[1], fcols[2] }
	hr, hm := fcols[3], fcols[4]

	if len(ids) == 0 { return nil, fmt.Errorf("No input halos.") }

	// Initialize phase profiles
	rSets := make([][]float64, len(ids))
	phiSets := [3][][]float64{
		make([][]float64, len(ids)),
		make([][]float64, len(ids)),
		make([][]float64, len(ids)),
	}

	for i := range rSets {
		rSets[i] = make([]float64, config.ncells)
		phiSets[0][i] = make([]float64, config.ncells*config.ncells)
		phiSets[1][i] = make([]float64, config.ncells*config.ncells)
		phiSets[2][i] = make([]float64, config.ncells*config.ncells)
	}

	snapBins, idxBins := binBySnap(snaps, ids)

	sortedSnaps := []int{}
	for snap := range snapBins {
		sortedSnaps = append(sortedSnaps, snap)
	}
	sort.Ints(sortedSnaps)

	buf, err := getVectorBuffer(
		e.ParticleCatalog(snaps[0], 0), gConfig,
	)
	if err != nil {
		return nil, err
	}

	for _, snap := range sortedSnaps {
		if snap == -1 {
			continue
		}

		idxs := idxBins[snap]
		snapCoords := [][]float64{
			make([]float64, len(idxs)), make([]float64, len(idxs)),
			make([]float64, len(idxs)), make([]float64, len(idxs)),
		}

		for i, idx := range idxs {
			snapCoords[0][i] = hx[0][idx]
			snapCoords[1][i] = hx[1][idx]
			snapCoords[2][i] = hx[2][idx]
			snapCoords[3][i] = hr[idx]*config.rMaxMult
		}

		hds, files, err := memo.ReadHeaders(snap, buf, e)
		if err != nil {
			return nil, err
		}
		hxBounds, err := boundingSpheres(snapCoords, &hds[0], e)
		if err != nil {
			return nil, err
		}
		_, intrIdxs := binSphereIntersections(hds, hxBounds)

		for i := range hds {
			if len(intrIdxs[i]) == 0 {
				continue
			}

			xs, _, ms, _, err := buf.Read(files[i])
			if err != nil {
				return nil, err
			}

			// Waarrrgggble
			for _, j := range intrIdxs[i] {
				phisXY := phiSets[0][idxs[j]]
				phisYZ := phiSets[1][idxs[j]]
				phisXZ := phiSets[2][idxs[j]]

				insertPotentialPoints(
					phisXY, phisYZ, phisXZ,
					hxBounds[j],
					xs, ms,
					config, &hds[i],
				)
			}

			buf.Close()
		}
	}

	for i := range rSets {
		processPotential(
			rSets[i],
			phiSets[0][i], phiSets[1][i], phiSets[2][i],
			hr[i], hm[i], config,
		)
	}

	rSets = transpose(rSets)
	phiSets[0] = transpose(phiSets[0])
	phiSets[1] = transpose(phiSets[1])
	phiSets[2] = transpose(phiSets[2])

	order := make([]int, len(rSets) + len(phiSets[0])*3 + 2)
	for i := range order { order[i] = i }
	lines := catalog.FormatCols(
		[][]int{ids, snaps},
		append(append(append(
			rSets, phiSets[0]...),
			phiSets[1]...),
			phiSets[2]...),
		order,
	)

	cString := catalog.CommentString(
		[]string{"ID", "Snapshot", "R [cMpc/h]",
			"Phi_xy/(G Mvir / Rvir)",
			"Phi_yz/(G Mvir / Rvir)",
			"Phi_xz/(G Mvir / Rvir)"},
		[]string{}, []int{0, 1, 2, 3, 4, 5},
		[]int{1, 1, int(config.ncells),
			int(config.ncells*config.ncells),
			int(config.ncells*config.ncells),
			int(config.ncells*config.ncells)},
	)

	if logging.Mode == logging.Performance {
		log.Printf("Time: %s", time.Since(t).String())
		log.Printf("Memory:\n%s", logging.MemString())
	}

	return append([]string{cString}, lines...), nil
}

func insertPotentialPoints(
	phisXY, phisYZ, phisXZ []float64,
	hx geom.Sphere,
	xs [][3]float32,
	ms []float32,
	config *PotentialConfig, hd *io.Header,
) {
	rmax2 := (config.rMaxMult*config.rMaxMult) * float64(hx.R*hx.R)
	rmin2 := (config.rMinMult*config.rMinMult) * float64(hx.R*hx.R)

	delta := float64(hx.R) * config.rGridMult * 2 / float64(config.ncells)
	nc := int(config.ncells)

	for i := range xs {
		if rand.Float64() > config.frac { continue }

		dx0 := float64(xs[i][0] - hx.C[0])
		dy0 := float64(xs[i][1] - hx.C[1])
		dz0 := float64(xs[i][2] - hx.C[2])

		dr02 := dx0*dx0 + dy0*dy0 + dz0*dz0
		pm := float64(ms[i])

		if dr02 > rmax2 || dr02 < rmin2 { continue }

		for iz := 0; iz < nc; iz++ {
			dz := dz0 + delta*float64(iz) - float64(hx.R)*config.rGridMult
			for iy := 0; iy < nc; iy++ {
				dy := dy0 + delta*float64(iy) - float64(hx.R)*config.rGridMult

				dryz := math.Sqrt(dx0*dx0 + dy*dy + dz*dz)
				phisYZ[iy + iz*nc] -= pm / dryz

				if iz != 0 && iy != 0 { continue }

				for ix := 0; ix < nc; ix++ {
					dx := dx0 + delta*float64(ix) -
						float64(hx.R)*config.rGridMult

					if iz == 0 {
						drxy := math.Sqrt(dx*dx + dy*dy + dz0*dz0)
						phisXY[ix + iy*nc] -= pm / drxy
					}

					if iy == 0 {
						drxz := math.Sqrt(dx*dx + dy0*dy0 + dz*dz)
						phisXZ[ix + iz*nc] -= pm / drxz
					}
				}
			}
		}
	}
}

func processPotential(
	rs, phisXY, phisYZ, phisXZ []float64,
	hr, hm float64, config *PotentialConfig,
) {
	for i := range phisXY {
		phisXY[i] /= config.frac * hm/hr
		phisXZ[i] /= config.frac * hm/hr
		phisYZ[i] /= config.frac * hm/hr
	}

	delta := hr * config.rGridMult * 2 / float64(config.ncells)
	for i := range rs {
		rs[i] = delta*float64(i) - hr*config.rGridMult
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 160, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0666,
	)

	if err != nil {
		logrus.Fatal(err)
	}

	logger := logrus.New()
	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runetcd(logger logrus.FieldLogger, lis net.Listener, grpcServer *grpc.Server, id uint64, ids []uint64, nodes []string) {
	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	ids = append(ids[:id-1], ids[id:]...)
	nodes = append(nodes[:id-1], nodes[id:]...)

	peers := make([]etcdraft.Peer, len(nodes))

	for i, addr := range nodes {
		nid := ids[i]

		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	storage := etcdraft.NewMemoryStorage()
	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		&etcdraft.Config{
			ID:              id,
			ElectionTick:    int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick:   1,
			Storage:         storage,
			MaxSizePerMsg:   *entriesPerMsg,
			MaxInflightMsgs: 256,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logrus.Fatal(grpcServer.Serve(lis))
	}()

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(logger logrus.FieldLogger, lis net.Listener, grpcServer *grpc.Server, id uint64, ids []uint64, nodes []string) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logrus.Fatal(grpcServer.Serve(lis))
	}()

	go func() {
		http.Handle("/metrics", promhttp.Handler())
		logrus.Fatal(http.ListenAndServe(":5"+nodes[id-1][1:], nil))
	}()

	logger.Fatal(node.Run(grpcServer))
}
<commit_msg>rkvd/main.go: Update MaxInflightMsgs<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	etcdraft "github.com/coreos/etcd/raft"
	"github.com/relab/raft"
	etcd "github.com/relab/raft/etcd"
	"github.com/relab/raft/raftgorums"
	"github.com/relab/rkv/rkvpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	bgorums    = "gorums"
	betcd      = "etcd"
	bhashicorp = "hashicorp"
)

var (
	bench             = flag.Bool("quiet", false, "Silence log output")
	recover           = flag.Bool("recover", false, "Recover from stable storage")
	batch             = flag.Bool("batch", true, "enable batching")
	electionTimeout   = flag.Duration("election", time.Second, "How long servers wait before starting an election")
	heartbeatTimeout  = flag.Duration("heartbeat", 20*time.Millisecond, "How often a heartbeat should be sent")
	entriesPerMsg     = flag.Uint64("entriespermsg", 64, "Entries per Appendentries message")
	catchupMultiplier = flag.Uint64("catchupmultiplier", 160, "How many more times entries per message allowed during catch up")
)

func main() {
	var (
		id      = flag.Uint64("id", 0, "server ID")
		servers = flag.String("servers", ":9201,:9202,:9203,:9204,:9205,:9206,:9207", "comma separated list of server addresses")
		cluster = flag.String("cluster", "1,2,3", "comma separated list of server ids to form cluster with, [1 >= id <= len(servers)]")
		backend = flag.String("backend", "gorums", "Raft backend to use [gorums|etcd|hashicorp]")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*servers, ",")

	if len(nodes) == 0 {
		fmt.Print("-server argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	selected := strings.Split(*cluster, ",")

	var ids []uint64

	for _, sid := range selected {
		id, err := strconv.ParseUint(sid, 10, 64)

		if err != nil {
			fmt.Print("could not parse -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		if id <= 0 || id > uint64(len(nodes)) {
			fmt.Print("invalid -cluster argument\n\n")
			flag.Usage()
			os.Exit(1)
		}

		ids = append(ids, id)
	}

	if len(ids) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if len(ids) > len(nodes) {
		fmt.Print("-cluster specifies too many servers\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *entriesPerMsg < 1 {
		fmt.Print("-entriespermsg must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *catchupMultiplier < 1 {
		fmt.Print("-catchupmultiplier must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	logFile, err := os.OpenFile(
		fmt.Sprintf("%s%sraft%.2d.log", os.TempDir(), string(filepath.Separator), *id),
		os.O_CREATE|os.O_TRUNC|os.O_APPEND|os.O_WRONLY, 0666,
	)

	if err != nil {
		logrus.Fatal(err)
	}

	logger := logrus.New()
	logger.Hooks.Add(NewLogToFileHook(logFile))

	if *bench {
		logger.Out = ioutil.Discard
		grpc.EnableTracing = false
	}

	grpclog.SetLogger(logger)

	lis, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		logger.Fatal(err)
	}

	grpcServer := grpc.NewServer()

	switch *backend {
	case bgorums:
		rungorums(logger, lis, grpcServer, *id, ids, nodes)
	case betcd:
		runetcd(logger, lis, grpcServer, *id, ids, nodes)
	}

}

func runetcd(logger logrus.FieldLogger, lis net.Listener, grpcServer *grpc.Server, id uint64, ids []uint64, nodes []string) {
	host, port, err := net.SplitHostPort(nodes[id-1])
	if err != nil {
		logger.Fatal(err)
	}
	p, _ := strconv.Atoi(port)
	selflis := host + ":" + strconv.Itoa(p-100)

	ids = append(ids[:id-1], ids[id:]...)
	nodes = append(nodes[:id-1], nodes[id:]...)

	peers := make([]etcdraft.Peer, len(nodes))

	for i, addr := range nodes {
		nid := ids[i]

		host, port, err := net.SplitHostPort(addr)
		if err != nil {
			logger.Fatal(err)
		}
		p, _ := strconv.Atoi(port)

		ur, err := url.Parse("http://" + addr)
		ur.Host = host + ":" + strconv.Itoa(p-100)
		if err != nil {
			logger.Fatal(err)
		}

		peers[i] = etcdraft.Peer{
			ID:      nid,
			Context: []byte(ur.String()),
		}
	}

	storage := etcdraft.NewMemoryStorage()
	node := etcd.NewRaft(
		logger,
		NewStore(),
		storage,
		&etcdraft.Config{
			ID:            id,
			ElectionTick:  int(*electionTimeout / *heartbeatTimeout),
			HeartbeatTick: 1,
			Storage:       storage,
			MaxSizePerMsg: *entriesPerMsg,
			// etcdserver says: Never overflow the rafthttp buffer,
			// which is 4096. We keep the same constant.
			MaxInflightMsgs: 4096 / 8,
			Logger:          logger,
		},
		peers,
		*heartbeatTimeout,
	)

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logrus.Fatal(grpcServer.Serve(lis))
	}()

	lishttp, err := net.Listen("tcp", selflis)

	if err != nil {
		logger.Fatal(err)
	}

	logger.Fatal(http.Serve(lishttp, node.Handler()))
}

func rungorums(logger logrus.FieldLogger, lis net.Listener, grpcServer *grpc.Server, id uint64, ids []uint64, nodes []string) {
	storage, err := raft.NewFileStorage(fmt.Sprintf("db%.2d.bolt", id), !*recover)

	if err != nil {
		logger.Fatal(err)
	}

	storageWithCache := raft.NewCacheStorage(storage, 20000)

	node := raftgorums.NewRaft(NewStore(), &raftgorums.Config{
		ID:                id,
		Servers:           nodes,
		InitialCluster:    ids,
		Batch:             *batch,
		Storage:           storageWithCache,
		ElectionTimeout:   *electionTimeout,
		HeartbeatTimeout:  *heartbeatTimeout,
		EntriesPerMsg:     *entriesPerMsg,
		CatchupMultiplier: *catchupMultiplier,
		Logger:            logger,
		MetricsEnabled:    true,
	})

	service := NewService(node)
	rkvpb.RegisterRKVServer(grpcServer, service)

	go func() {
		logrus.Fatal(grpcServer.Serve(lis))
	}()

	go func() {
		http.Handle("/metrics", promhttp.Handler())
		logrus.Fatal(http.ListenAndServe(":5"+nodes[id-1][1:], nil))
	}()

	logger.Fatal(node.Run(grpcServer))
}
<|endoftext|>


<commit_before>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/richardlehane/siegfried"
	"github.com/richardlehane/siegfried/config"
)

type Update struct {
	SfVersion  [3]int
	SigCreated string
	GobSize    int
	LatestURL  string
}

func updateSigs() (string, error) {
	url, _, _ := config.UpdateOptions()
	if url == "" {
		return "Update is not available for this distribution of Siegfried", nil
	}
	response, err := getHttp(url)
	if err != nil {
		return "", err
	}
	var u Update
	if err := json.Unmarshal(response, &u); err != nil {
		return "", err
	}
	version := config.Version()
	if version[0] < u.SfVersion[0] || (u.SfVersion[0] == version[0] && version[1] < u.SfVersion[1]) {
		return "Your version of Siegfried is out of date; please install latest from http://www.itforarchivists.com/siegfried before continuing.", nil
	}
	s, err := siegfried.Load(config.Signature())
	if err == nil {
		if !s.Update(u.SigCreated) {
			return "You are already up to date!", nil
		}
	} else {
		// this hairy bit of golang exception handling is thanks to Ross! :)
		if _, err = os.Stat(config.Home()); err != nil {
			if os.IsNotExist(err) {
				err = os.MkdirAll(config.Home(), os.ModePerm)
				if err != nil {
					return "", fmt.Errorf("Siegfried: cannot create home directory %s, %v", config.Home(), err)
				}
			} else {
				return "", fmt.Errorf("Siegfried: error opening directory %s, %v", config.Home(), err)
			}
		}
	}
	fmt.Println("... downloading latest signature file ...")
	response, err = getHttp(u.LatestURL)
	if err != nil {
		return "", fmt.Errorf("Siegfried: error retrieving pronom.gob.\nThis may be a network or firewall issue. See https://github.com/richardlehane/siegfried/wiki/Getting-started for manual instructions.\nSystem error: %v", err)
	}
	if len(response) != u.GobSize {
		return "", fmt.Errorf("Siegfried: error retrieving pronom.gob; expecting %d bytes, got %d bytes", u.GobSize, len(response))
	}
	err = ioutil.WriteFile(config.Signature(), response, os.ModePerm)
	if err != nil {
		return "", fmt.Errorf("Siegfried: error writing to directory, %v", err)
	}
	fmt.Printf("... writing %s ...\n", config.Signature())
	return "Your signature file has been updated", nil
}

func getHttp(url string) ([]byte, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	_, timeout, transport := config.UpdateOptions()
	req.Header.Add("User-Agent", "siegfried/siegbot (+https://github.com/richardlehane/siegfried)")
	req.Header.Add("Cache-Control", "no-cache")
	timer := time.AfterFunc(timeout, func() {
		transport.CancelRequest(req)
	})
	defer timer.Stop()
	client := http.Client{
		Transport: transport,
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}
<commit_msg>fix https://github.com/richardlehane/siegfried/issues/79<commit_after>// Copyright 2015 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/richardlehane/siegfried"
	"github.com/richardlehane/siegfried/config"
)

type Update struct {
	Version [3]int `json:"sf"`
	Created string `json:"created"`
	Size    int    `json:"size"`
	Path    string `json:"path"`
}

func updateSigs() (string, error) {
	url, _, _ := config.UpdateOptions()
	if url == "" {
		return "Update is not available for this distribution of Siegfried", nil
	}
	response, err := getHttp(url)
	if err != nil {
		return "", err
	}
	var u Update
	if err := json.Unmarshal(response, &u); err != nil {
		return "", err
	}
	version := config.Version()
	if version[0] < u.Version[0] || (version[0] == u.Version[0] && version[1] < u.Version[1]) || // if the version is out of date
		u.Version == [3]int{0, 0, 0} || u.Created == "" || u.Size == 0 || u.Path == "" { // or if the unmarshalling hasn't worked and we have blank values
		return "Your version of Siegfried is out of date; please install latest from http://www.itforarchivists.com/siegfried before continuing.", nil
	}
	s, err := siegfried.Load(config.Signature())
	if err == nil {
		if !s.Update(u.Created) {
			return "You are already up to date!", nil
		}
	} else {
		// this hairy bit of golang exception handling is thanks to Ross! :)
		if _, err = os.Stat(config.Home()); err != nil {
			if os.IsNotExist(err) {
				err = os.MkdirAll(config.Home(), os.ModePerm)
				if err != nil {
					return "", fmt.Errorf("Siegfried: cannot create home directory %s, %v", config.Home(), err)
				}
			} else {
				return "", fmt.Errorf("Siegfried: error opening directory %s, %v", config.Home(), err)
			}
		}
	}
	fmt.Println("... downloading latest signature file ...")
	response, err = getHttp(u.Path)
	if err != nil {
		return "", fmt.Errorf("Siegfried: error retrieving %s.\nThis may be a network or firewall issue. See https://github.com/richardlehane/siegfried/wiki/Getting-started for manual instructions.\nSystem error: %v", config.SignatureBase(), err)
	}
	if len(response) != u.Size {
		return "", fmt.Errorf("Siegfried: error retrieving %s; expecting %d bytes, got %d bytes", config.SignatureBase(), u.Size, len(response))
	}
	err = ioutil.WriteFile(config.Signature(), response, os.ModePerm)
	if err != nil {
		return "", fmt.Errorf("Siegfried: error writing to directory, %v", err)
	}
	fmt.Printf("... writing %s ...\n", config.Signature())
	return "Your signature file has been updated", nil
}

func getHttp(url string) ([]byte, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	_, timeout, transport := config.UpdateOptions()
	req.Header.Add("User-Agent", "siegfried/siegbot (+https://github.com/richardlehane/siegfried)")
	req.Header.Add("Cache-Control", "no-cache")
	timer := time.AfterFunc(timeout, func() {
		transport.CancelRequest(req)
	})
	defer timer.Stop()
	client := http.Client{
		Transport: transport,
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
)

var ROOT = ""

type FileTypeMapJSON struct {
	MIME        string `json:"mime"`
	IsDirectory bool   `json:"is_directory"`
	IsHidden    bool   `json:"is_hidden"`
	IsAudio     bool   `json:"is_audio"`
	IsImage     bool   `json:"is_image"`
	IsVideo     bool   `json:"is_video"`
}

type FileInfoJSON struct {
	Name       string          `json:"name"`
	Size       int64           `json:"size"`
	ModifiedAt string          `json:"modified_at"`
	Type       FileTypeMapJSON `json:"type"`
}

// returns a pair of (filename, MIME type) strings given a `file` output line
func parseMIMEType(fileOutputLine string) (string, string, error) {
	// parse the file program output into a bare MIME type
	mimeString := strings.TrimSpace(fileOutputLine)
	splitIndex := strings.LastIndex(mimeString, ":")

	if len(fileOutputLine) <= 1 || splitIndex <= 0 {
		return "", "", fmt.Errorf("Invalid MIME string: '%s'", fileOutputLine)
	}

	return mimeString[0:splitIndex], strings.TrimSpace(mimeString[splitIndex+1:]), nil
}

// given a path, returns a map of child name to MIME type
func getMIMETypes(root string, files ...os.FileInfo) map[string]string {
	// build the command to get all the MIME types at once, for efficiency
	args := []string{"--mime-type", "--dereference", "--preserve-date"}
	for _, file := range files {
		args = append(args, path.Join(root, file.Name()))
	}

	// call `file` for a newline-delimited string of "filename: MIME-type" pairs
	result := make(map[string]string, len(files))
	fileOutput, err := exec.Command("file", args...).Output()
	if err != nil {
		return result
	}

	for _, line := range strings.Split(string(fileOutput), "\n") {
		fileName, mimeType, err := parseMIMEType(line)
		if err == nil {
			// use the full path, so we can handle multiple directories unambiguously
			result[fileName] = mimeType
		}
	}

	return result
}

// normalizes the path using a root, and returns it. if the path exits the root
// or is otherwise invalid, returns an error.
func normalizePathToRoot(root, child string) (string, error) {
	// clean the path, resolving any ".."s in it
	requestPath := path.Clean(path.Join(root, child))

	// if the path exited the root directory, fail
	relPath, err := filepath.Rel(root, requestPath)
	if err != nil || strings.Index(relPath, "..") >= 0 {
		// keep things vague since someone's probably trying to be sneaky anyway
		return "", fmt.Errorf("Invalid path")
	}

	return requestPath, nil
}

func getFile(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can set appropriate response headers, and so we can
	// ensure it's a regular file and not a directory.
	fileInfo, err := os.Stat(normalizedPath)
	if err != nil || fileInfo.IsDir() {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	file, err := os.Open(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}
	defer file.Close()

	mimeType := getMIMETypes(filepath.Base(normalizedPath), fileInfo)[normalizedPath]
	w.Header().Add("Content-Type", mimeType)
	w.Header().Add("Cache-Control", "no-cache")

	http.ServeFile(w, r, normalizedPath)
}

func getDirectory(w http.ResponseWriter, r *http.Request) {
	// disable caching since we'll want to keep this listing up-to-date
	w.Header().Add("Cache-Control", "no-cache")

	// ensure the directory actually exists
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	children, err := ioutil.ReadDir(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// get a map of all the MIME types for the given files
	mimeTypes := getMIMETypes(normalizedPath, children...)

	// list the directory to a JSON response
	var files []FileInfoJSON
	for _, file := range children {
		fileName := file.Name()
		filePath := path.Join(normalizedPath, fileName)
		mimeType, _ := mimeTypes[filePath]
		isHidden := strings.HasPrefix(fileName, ".")

		// TODO: determine if it's one of these types!
		isAudio, isImage, isVideo := false, false, false

		files = append(files, FileInfoJSON{
			fileName,
			file.Size(),
			file.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
			FileTypeMapJSON{
				mimeType,
				file.IsDir(),
				isHidden,
				isAudio,
				isImage,
				isVideo,
			},
		})
	}

	json, err := json.Marshal(files)
	if err != nil {
		http.Error(w, "Failed to generate JSON response", 500)
		return
	}

	w.Header().Add("Content-Type", "application/json")
	w.Write(json)
}

// retrieves/caches/updates a thumbnail file given a path, or returns an error
// if no thumbnail could be geneated.
func getThumbnail(w http.ResponseWriter, r *http.Request) {
	// TODO:
	// * look up the existing file to get its modtime and ensure it exists
	// * see if we have a cached file with the same modtime
	// ** if so, use it
	// ** otherwise, generate a preview
	// *** use graphicsmagick/ffmpeg to generate a preview thumbnail
	// *** store the new file to a mirroed path with the filename plus the modtime
	// * read the cached file and return its contents

	// cache preview thumbnails for a good while to lower load on this tiny
	// server, even if we are caching the preview thumbnails on-disk too.
	w.Header().Add("Cache-Control", "max-age=3600")
	w.Header().Add("Content-Type", "image/jpeg")
}

func main() {
	// ensure we have all the binaries we need
	requiredBinaries := []string{"file"}
	for _, binary := range requiredBinaries {
		if _, err := exec.LookPath(binary); err != nil {
			log.Panicf("'%s' must be installed and in the PATH\n", binary)
		}
	}

	if len(os.Args) <= 1 {
		panic("A root directory argument is required")
	}

	ROOT = os.Args[1]

	fmt.Printf("Serving '%s'...\n", ROOT)

	r := mux.NewRouter()

	// anything with a trailing `/` indicates a directory; anything that ends
	// without a trailing slash indicates a file.
	r.HandleFunc("/files{path:.*}/", getDirectory)
	r.HandleFunc("/files/{path:.*[^/]$}", getFile)
	r.HandleFunc("/thumbnails/{path:.*[^/]$}", getThumbnail)

	http.ListenAndServe(":3000", r)
}
<commit_msg>Handle JSON separately, refactor<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"
)

var ROOT = ""

type FileTypeMapJSON struct {
	MIME        string `json:"mime"`
	IsDirectory bool   `json:"is_directory"`
	IsHidden    bool   `json:"is_hidden"`
	IsAudio     bool   `json:"is_audio"`
	IsImage     bool   `json:"is_image"`
	IsVideo     bool   `json:"is_video"`
}

type FileInfoJSON struct {
	Name       string          `json:"name"`
	Size       int64           `json:"size"`
	ModifiedAt string          `json:"modified_at"`
	Type       FileTypeMapJSON `json:"type"`
}

// returns a pair of (filename, MIME type) strings given a `file` output line
func parseMIMEType(fileOutputLine string) (string, string, error) {
	// parse the file program output into a bare MIME type
	mimeString := strings.TrimSpace(fileOutputLine)
	splitIndex := strings.LastIndex(mimeString, ":")

	if len(fileOutputLine) <= 1 || splitIndex <= 0 {
		return "", "", fmt.Errorf("Invalid MIME string: '%s'", fileOutputLine)
	}

	return mimeString[0:splitIndex], strings.TrimSpace(mimeString[splitIndex+1:]), nil
}

func writeJSONResponse(w http.ResponseWriter, data interface{}) {
	w.Header().Add("Content-Type", "application/json")
	w.Header().Add("Cache-Control", "no-cache")

	json, err := json.Marshal(data)
	if err != nil {
		http.Error(w, "Failed to generate JSON response", 500)
		return
	}
	w.Write(json)
}

// given a path, returns a map of child name to MIME type
func getChildMIMETypes(parentPath string) map[string]string {
	result := make(map[string]string)

	// get all the children in the given directory
	children, err := filepath.Glob(path.Join(parentPath, "*"))
	if err != nil {
		return result
	}

	args := []string{"--mime-type", "--dereference", "--preserve-date"}
	args = append(args, children...)

	// call `file` for a newline-delimited list of "filename: MIME-type" pairs
	fileOutput, err := exec.Command("file", args...).Output()

	if err != nil {
		return result
	}

	for _, line := range strings.Split(string(fileOutput), "\n") {
		fileName, mimeType, err := parseMIMEType(line)
		if err == nil {
			result[fileName] = mimeType
		}
	}

	return result
}

func getMIMEType(filePath string) string {
	fileOutput, err := exec.Command(
		"file",
		"--mime-type",
		"--dereference",
		"--preserve-date",
		"--brief",
		filePath,
	).Output()

	if err != nil {
		return ""
	}

	return strings.TrimSpace(string(fileOutput))
}

// normalizes the path using a root, and returns it. if the path exits the root
// or is otherwise invalid, returns an error.
func normalizePathToRoot(root, child string) (string, error) {
	// clean the path, resolving any ".."s in it
	requestPath := path.Clean(path.Join(root, child))

	// if the path exited the root directory, fail
	relPath, err := filepath.Rel(root, requestPath)
	if err != nil || strings.Index(relPath, "..") >= 0 {
		// keep things vague since someone's probably trying to be sneaky anyway
		return "", fmt.Errorf("Invalid path")
	}

	return requestPath, nil
}

// this returns the info for the specified files _or_ directory, not just files
func getInfo(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can return its info
	fileInfo, err := os.Stat(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	mimeType := getMIMEType(normalizedPath)

	writeJSONResponse(w, FileInfoJSON{
		fileInfo.Name(),
		fileInfo.Size(),
		fileInfo.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
		FileTypeMapJSON{
			mimeType,
			fileInfo.IsDir(),
			strings.HasPrefix(fileInfo.Name(), "."),
			strings.HasPrefix("audio/", mimeType),
			strings.HasPrefix("image/", mimeType),
			strings.HasPrefix("video/", mimeType),
		},
	})
}

func download(w http.ResponseWriter, r *http.Request) {
	// make sure our path is valid
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// stat the file so we can set appropriate response headers, and so we can
	// ensure it's a regular file and not a directory.
	file, err := os.Stat(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// return different responses depending on file type
	if file.IsDir() {
		// TODO: zip up the directory contents and serve it up
	} else {
		downloadFile(w, r, normalizedPath, file)
	}
}

func downloadFile(w http.ResponseWriter, r *http.Request, filePath string, file os.FileInfo) {
	mimeType := getMIMEType(filePath)
	w.Header().Add("Content-Type", mimeType)
	w.Header().Add("Content-Disposition", file.Name())
	w.Header().Add("Cache-Control", "no-cache")

	http.ServeFile(w, r, filePath)
}

func getDirectory(w http.ResponseWriter, r *http.Request) {
	// ensure the directory actually exists
	rawPath := mux.Vars(r)["path"]
	normalizedPath, err := normalizePathToRoot(ROOT, rawPath)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	children, err := ioutil.ReadDir(normalizedPath)
	if err != nil {
		// don't report the raw error in case we leak server directory information
		http.Error(w, "", 404)
		return
	}

	// get a map of all the MIME types for the directory
	mimeTypes := getChildMIMETypes(normalizedPath)

	// list the directory to a JSON response
	var files []FileInfoJSON
	for _, file := range children {
		fileName := file.Name()
		mimeType := mimeTypes[path.Join(normalizedPath, fileName)]

		files = append(files, FileInfoJSON{
			fileName,
			file.Size(),
			file.ModTime().Format("2006-01-02T15:04:05Z"), // ISO 8601
			FileTypeMapJSON{
				mimeType,
				file.IsDir(),
				strings.HasPrefix(fileName, "."), // hidden?
				strings.HasPrefix("audio/", mimeType), // audio?
				strings.HasPrefix("image/", mimeType), // image?
				strings.HasPrefix("video/", mimeType), // video?
			},
		})
	}

	writeJSONResponse(w, files)
}

func downloadDirectory(w http.ResponseWriter, r *http.Request) {
	// TODO: zip up the whole directory and offer it for download
}

// retrieves/caches/updates a thumbnail file given a path, or returns an error
// if no thumbnail could be geneated.
func getThumbnail(w http.ResponseWriter, r *http.Request) {
	// TODO:
	// * look up the existing file to get its modtime and ensure it exists
	// * see if we have a cached file with the same modtime
	// ** if so, use it
	// ** otherwise, generate a preview
	// *** use graphicsmagick/ffmpeg to generate a preview thumbnail
	// *** store the new file to a mirroed path with the filename plus the modtime
	// * read the cached file and return its contents

	// cache preview thumbnails for a good while to lower load on this tiny
	// server, even if we are caching the preview thumbnails on-disk too.
	w.Header().Add("Cache-Control", "max-age=3600")
	w.Header().Add("Content-Type", "image/jpeg")
}

func main() {
	// ensure we have all the binaries we need
	requiredBinaries := []string{"file"}
	for _, binary := range requiredBinaries {
		if _, err := exec.LookPath(binary); err != nil {
			log.Panicf("'%s' must be installed and in the PATH\n", binary)
		}
	}

	if len(os.Args) <= 1 {
		panic("A root directory argument is required")
	}

	ROOT = os.Args[1]

	router := mux.NewRouter()

	// /files
	// anything with a trailing `/` indicates a directory; anything that ends
	// without a trailing slash indicates a file.
	filesJSON := router.Headers("Content-Type", "application/json").Subrouter()
	filesJSON.HandleFunc("/files/{path:.*[^/]$}", getInfo).
		Headers("Content-Type", "application/json").
		Methods("GET")
	filesJSON.HandleFunc("/files{path:.*}/", getDirectory).
		Headers("Content-Type", "application/json").
		Methods("GET")

	router.HandleFunc("/files/{path:.*}", download).
		Methods("GET")

	// /thumbnails
	router.HandleFunc("/thumbnails/{path:.*[^/]$}", getThumbnail).
		Methods("GET")

	http.ListenAndServe(":3000", router)
}
<|endoftext|>


<commit_before>// GoWorld是一个分布式的游戏服务器引擎
package goworld

import (
	"time"

	"github.com/xiaonanln/goTimer"
	"github.com/xiaonanln/goworld/components/game"
	"github.com/xiaonanln/goworld/engine/common"
	"github.com/xiaonanln/goworld/engine/config"
	"github.com/xiaonanln/goworld/engine/entity"
	"github.com/xiaonanln/goworld/engine/gwlog"
	"github.com/xiaonanln/goworld/engine/kvdb"
	"github.com/xiaonanln/goworld/engine/post"
	"github.com/xiaonanln/goworld/engine/service"
	"github.com/xiaonanln/goworld/engine/storage"
)

const (
	// ENTITYID_LENGTH 是EntityID的长度，目前为16
	ENTITYID_LENGTH = common.ENTITYID_LENGTH
)

// GameID 是Game进程的ID。
// GoWorld要求GameID的数值必须是从1~N的连续N个数字，其中N为服务器配置文件中配置的game进程数目。
type GameID = uint16

// GateID 是Gate进程的ID。
// GoWorld要求GateID的数值必须是从1~N的连续N个数字，其中N为服务器配置文件中配置的game进程数目。
type GateID = uint16

// DispatcherID 是Dispatcher进程的ID
// GoWorld要求DispatcherID的数值必须是从1~N的连续N个数字，其中N为服务器配置文件中配置的dispatcher进程数目
type DispatcherID uint16

// EntityID 唯一代表一个Entity。EntityID是一个字符串（string），长度固定（ENTITYID_LENGTH）。
// EntityID是全局唯一的。不同进程上产生的EntityID都是唯一的，不会出现重复。一般来说即使是不用的游戏服务器产生的EntityID也是唯一的。
type EntityID = common.EntityID

// Entity 类型代表游戏服务器中的一个对象。开发者可以使用GoWorld提供的接口进行对象创建、载入。对象载入之后，GoWorld提供定时的对象数据存盘。
// 同一个game进程中的Entity之间可以拿到相互的引用（指针）并直接进行相关的函数调用。不同game进程中的Entity之间可以使用RPC进行相互通信。
type Entity = entity.Entity

// Space 类型代表一个游戏服务器中的一个场景。一个场景中可以包含多个Entity。Space和其中的Entity都存在于一个game进程中。
// Entity可以通过调用EnterSpace函数来切换Space。如果EnterSpace调用所指定的Space在其他game进程上，Entity将被迁移到对应的game进程并添加到Space中。
type Space = entity.Space

// Kind 类型表示Space的种类。开发者在创建Space的时候需要提供Kind参数，从而创建特定Kind的Space。NilSpace的Kind总是为0，并且开发者不能创建Kind=0的Space。
// 开发者可以根据Kind的值来区分不同的场景，具体的区分规则由开发者自己决定。
type Kind = int

// Vector3 是服务端用于存储Entity位置的类型，包含X, Y, Z三个字段。
// GoWorld使用X轴和Z轴坐标进行AOI管理，无视Y轴坐标值。
type Vector3 = entity.Vector3

// Run 开始运行game服务。开发者需要为自己的游戏服务器提供一个main模块和main函数，并在main函数里正确初始化GoWorld服务器并启动服务器。
// 一般来说，开发者需要在main函数中注册相应的Space类型、Service类型、Entity类型，然后调用 goworld.Run() 启动GoWorld服务器即可，可参考：
// https://github.com/xiaonanln/goworld/blob/master/examples/unity_demo/unity_demo.go
func Run() {
	game.Run()
}

// RegisterSpace 注册一个Space对象类型。开发者必须并且只能调用这个接口一次，从而注册特定的Space类型。一个合法的Space类型必须继承goworld.Space类型。
func RegisterSpace(spacePtr entity.ISpace) {
	entity.RegisterSpace(spacePtr)
}

// RegisterEntity 注册一个对象类型到game中。所注册的对象必须是Entity类型的子类（包含一个匿名Entity字段）。
// 使用方法可以参考：https://github.com/xiaonanln/goworld/blob/master/examples/unity_demo/unity_demo.go
func RegisterEntity(typeName string, entityPtr entity.IEntity) *entity.EntityTypeDesc {
	return entity.RegisterEntity(typeName, entityPtr, false)
}

// RegisterService 注册一个Service类型到game中。Service是一种全局唯一的特殊的Entity对象。
// 每个game进程中初始化的时候都应该注册所有的Service。GoWorld服务器会在某一个game进程中自动创建或载入Service对象（取决于Service类型是否是Persistent）。
// 开发者不能手动创建Service对象。
func RegisterService(typeName string, entityPtr entity.IEntity) {
	service.RegisterService(typeName, entityPtr)
}

// CreateSpaceAnywhere 在一个随机选择的game（以后会支持自动负载均衡）上创建一个特定Kind的Space对象。
func CreateSpaceAnywhere(kind Kind) EntityID {
	if kind == 0 {
		gwlog.Panicf("Can not create nil space with kind=0. Game will create 1 nil space automatically.")
	}
	return entity.CreateSpaceSomewhere(0, kind)
}

// CreateSpaceOnGame creates a space with specified kind on the specified game
//
// returns the space EntityID
func CreateSpaceOnGame(gameid uint16, kind int) EntityID {
	return entity.CreateSpaceSomewhere(gameid, kind)
}

// CreateSpaceLocally 在本地game进程上创建一个指定Kind的Space。
func CreateSpaceLocally(kind Kind) *Space {
	if kind == 0 {
		gwlog.Panicf("Can not create nil space with kind=0. Game will create 1 nil space automatically.")
	}
	return entity.CreateSpaceLocally(kind)
}

// CreateEntityLocally 在本地game进程上创建一个指定类型的Entity
func CreateEntityLocally(typeName string) *Entity {
	return entity.CreateEntityLocally(typeName, nil)
}

// CreateEntityAnywhere 在随机选择的game进程上创建一个特定类型的Entity
func CreateEntityAnywhere(typeName string) EntityID {
	return entity.CreateEntitySomewhere(0, typeName)
}

func CreateEntityOnGame(gameid uint16, typeName string) EntityID {
	return entity.CreateEntitySomewhere(gameid, typeName)
}

// LoadEntityAnywhere 在随机选择的game进程上载入指定的Entity。
// GoWorld保证每个Entity最多只会存在于一个game进程，即只有一份实例。
// 如果这个Entity当前已经存在，则GoWorld不会做任何操作。
func LoadEntityAnywhere(typeName string, entityID EntityID) {
	entity.LoadEntityAnywhere(typeName, entityID)
}

// LoadEntityOnGame 在指定的game进程上载入特定的Entity对象。
// 如果这个Entity当前已经存在，则GoWorld不会做任何操作。因此在调用LoadEntityOnGame之后并不能严格保证Entity必然存在于所指定的game进程中。
func LoadEntityOnGame(typeName string, entityID EntityID, gameid GameID) {
	entity.LoadEntityOnGame(typeName, entityID, gameid)
}

// LoadEntityLocally 在当前的game进程中载入特定的Entity对象
// 如果这个Entity当前已经存在，则GoWorld不会做任何操作。因此在调用LoadEntityOnGame之后并不能严格保证Entity必然存在于当前game进程中。
func LoadEntityLocally(typeName string, entityID EntityID) {
	entity.LoadEntityOnGame(typeName, entityID, GetGameID())
}

// ListEntityIDs 获得某个类型的所有Entity对象的EntityID列表
// （这个接口将被弃用）
func ListEntityIDs(typeName string, callback storage.ListCallbackFunc) {
	storage.ListEntityIDs(typeName, callback)
}

// Exists 检查某个特定的Entity是否存在（已创建存盘）
func Exists(typeName string, entityID EntityID, callback storage.ExistsCallbackFunc) {
	storage.Exists(typeName, entityID, callback)
}

// GetEntity 获得当前game进程中的指定EntityID的Entity对象。不存在则返回nil。
func GetEntity(id EntityID) *Entity {
	return entity.GetEntity(id)
}

// GetSpace 获得当前进程中指定EntityID的Space对象。不存在则返回nil。
func GetSpace(id EntityID) *Space {
	return entity.GetSpace(id)
}

// GetGameID 获得当前game进程的GameID
func GetGameID() GameID {
	return game.GetGameID()
}

// MapAttr 创建一个新的空MapAttr对象
func MapAttr() *entity.MapAttr {
	return entity.NewMapAttr()
}

// ListAttr 创建一个新的空ListAttr对象
func ListAttr() *entity.ListAttr {
	return entity.NewListAttr()
}

// Entities 返回所有的Entity对象（通过EntityMap类型返回）
// 此接口将被弃用
func Entities() entity.EntityMap {
	return entity.Entities()
}

// Call 函数调用指定Entity的指定方法，并传递参数。
// 如果指定的Entity在当前game进程中，则会立刻调用其方法。否则将通过RPC发送函数调用和参数到所对应的game进程中。
func Call(id EntityID, method string, args ...interface{}) {
	entity.Call(id, method, args)
}

// CallService 发起一次Service调用。开发者只需要传入指定的Service名字，不需要指知道Service的EntityID或者当前在哪个game进程。
func CallService(serviceName string, method string, args ...interface{}) {
	service.CallService(serviceName, method, args)
}

// GetServiceEntityID 返回Service对象的EntityID。这个函数可以用来确定Service对象是否已经在某个game进程上成功创建或载入。
func GetServiceEntityID(serviceName string) common.EntityID {
	return service.GetServiceEntityID(serviceName)
}

// CallNilSpaces 向所有game进程中的NilSpace发起RPC调用。
// 由于每个game进程中都有一个唯一的NilSpace，因此这个函数想每个game进程都发起了一次函数调用。
func CallNilSpaces(method string, args ...interface{}) {
	entity.CallNilSpaces(method, args, game.GetGameID())
}

// GetNilSpaceID 返回特定game进程中的NilSpace的EntityID。
// GoWorld为每个game进程中的NilSpace使用了固定的EntityID值，例如目前GoWorld实现中在game1上NilSpace的EntityID总是"AAAAAAAAAAAAAAAx"，每次重启服务器都不会变化。
func GetNilSpaceID(gameid GameID) EntityID {
	return entity.GetNilSpaceID(gameid)
}

// GetNilSpace 返回当前game进程总的NilSpace对象
func GetNilSpace() *Space {
	return entity.GetNilSpace()
}

// GetKVDB 获得KVDB中指定key的值
func GetKVDB(key string, callback kvdb.KVDBGetCallback) {
	kvdb.Get(key, callback)
}

// PutKVDB 将制定的key-value对存入到KVDB中
func PutKVDB(key string, val string, callback kvdb.KVDBPutCallback) {
	kvdb.Put(key, val, callback)
}

// GetOrPutKVDB 读取指定key所对应的value，如果key所对应的值当前为空，则存入key-value键值对
func GetOrPutKVDB(key string, val string, callback kvdb.KVDBGetOrPutCallback) {
	kvdb.GetOrPut(key, val, callback)
}

// ListGameIDs 获得所有的GameID列表
func ListGameIDs() []GameID {
	return config.GetGameIDs()
}

// AddTimer adds a timer to be executed after specified duration
func AddCallback(d time.Duration, callback func()) {
	timer.AddCallback(d, callback)
}

// AddTimer adds a repeat timer to be executed every specified duration
func AddTimer(d time.Duration, callback func()) {
	timer.AddTimer(d, callback)
}

// Post posts a callback to be executed
// It is almost same as AddCallback(0, callback)
func Post(callback post.PostCallback) {
	post.Post(callback)
}
<commit_msg>goworld_cn文档<commit_after>// goworld库为开发者提供大部分的GoWorld服务器引擎接口
// GoWorld是一个分布式的游戏服务器引擎，理论上支持无限横向扩展。
// 一个GoWorld服务器由三种不同的进程注册：dispatcher、gate、game。
package goworld

import (
	"time"

	"github.com/xiaonanln/goTimer"
	"github.com/xiaonanln/goworld/components/game"
	"github.com/xiaonanln/goworld/engine/common"
	"github.com/xiaonanln/goworld/engine/config"
	"github.com/xiaonanln/goworld/engine/entity"
	"github.com/xiaonanln/goworld/engine/gwlog"
	"github.com/xiaonanln/goworld/engine/kvdb"
	"github.com/xiaonanln/goworld/engine/post"
	"github.com/xiaonanln/goworld/engine/service"
	"github.com/xiaonanln/goworld/engine/storage"
)

const (
	// ENTITYID_LENGTH 是EntityID的长度，目前为16
	ENTITYID_LENGTH = common.ENTITYID_LENGTH
)

// GameID 是Game进程的ID。
// GoWorld要求GameID的数值必须是从1~N的连续N个数字，其中N为服务器配置文件中配置的game进程数目。
type GameID = uint16

// GateID 是Gate进程的ID。
// GoWorld要求GateID的数值必须是从1~N的连续N个数字，其中N为服务器配置文件中配置的game进程数目。
type GateID = uint16

// DispatcherID 是Dispatcher进程的ID
// GoWorld要求DispatcherID的数值必须是从1~N的连续N个数字，其中N为服务器配置文件中配置的dispatcher进程数目
type DispatcherID uint16

// EntityID 唯一代表一个Entity。EntityID是一个字符串（string），长度固定（ENTITYID_LENGTH）。
// EntityID是全局唯一的。不同进程上产生的EntityID都是唯一的，不会出现重复。一般来说即使是不用的游戏服务器产生的EntityID也是唯一的。
type EntityID = common.EntityID

// Entity 类型代表游戏服务器中的一个对象。开发者可以使用GoWorld提供的接口进行对象创建、载入。对象载入之后，GoWorld提供定时的对象数据存盘。
// 同一个game进程中的Entity之间可以拿到相互的引用（指针）并直接进行相关的函数调用。不同game进程中的Entity之间可以使用RPC进行相互通信。
type Entity = entity.Entity

// Space 类型代表一个游戏服务器中的一个场景。一个场景中可以包含多个Entity。Space和其中的Entity都存在于一个game进程中。
// Entity可以通过调用EnterSpace函数来切换Space。如果EnterSpace调用所指定的Space在其他game进程上，Entity将被迁移到对应的game进程并添加到Space中。
type Space = entity.Space

// Kind 类型表示Space的种类。开发者在创建Space的时候需要提供Kind参数，从而创建特定Kind的Space。NilSpace的Kind总是为0，并且开发者不能创建Kind=0的Space。
// 开发者可以根据Kind的值来区分不同的场景，具体的区分规则由开发者自己决定。
type Kind = int

// Vector3 是服务端用于存储Entity位置的类型，包含X, Y, Z三个字段。
// GoWorld使用X轴和Z轴坐标进行AOI管理，无视Y轴坐标值。
type Vector3 = entity.Vector3

// Run 开始运行game服务。开发者需要为自己的游戏服务器提供一个main模块和main函数，并在main函数里正确初始化GoWorld服务器并启动服务器。
// 一般来说，开发者需要在main函数中注册相应的Space类型、Service类型、Entity类型，然后调用 goworld.Run() 启动GoWorld服务器即可，可参考：
// https://github.com/xiaonanln/goworld/blob/master/examples/unity_demo/unity_demo.go
func Run() {
	game.Run()
}

// RegisterSpace 注册一个Space对象类型。开发者必须并且只能调用这个接口一次，从而注册特定的Space类型。一个合法的Space类型必须继承goworld.Space类型。
func RegisterSpace(spacePtr entity.ISpace) {
	entity.RegisterSpace(spacePtr)
}

// RegisterEntity 注册一个对象类型到game中。所注册的对象必须是Entity类型的子类（包含一个匿名Entity字段）。
// 使用方法可以参考：https://github.com/xiaonanln/goworld/blob/master/examples/unity_demo/unity_demo.go
func RegisterEntity(typeName string, entityPtr entity.IEntity) *entity.EntityTypeDesc {
	return entity.RegisterEntity(typeName, entityPtr, false)
}

// RegisterService 注册一个Service类型到game中。Service是一种全局唯一的特殊的Entity对象。
// 每个game进程中初始化的时候都应该注册所有的Service。GoWorld服务器会在某一个game进程中自动创建或载入Service对象（取决于Service类型是否是Persistent）。
// 开发者不能手动创建Service对象。
func RegisterService(typeName string, entityPtr entity.IEntity) {
	service.RegisterService(typeName, entityPtr)
}

// CreateSpaceAnywhere 在一个随机选择的game（以后会支持自动负载均衡）上创建一个特定Kind的Space对象。
func CreateSpaceAnywhere(kind Kind) EntityID {
	if kind == 0 {
		gwlog.Panicf("Can not create nil space with kind=0. Game will create 1 nil space automatically.")
	}
	return entity.CreateSpaceSomewhere(0, kind)
}

// CreateSpaceOnGame creates a space with specified kind on the specified game
//
// returns the space EntityID
func CreateSpaceOnGame(gameid uint16, kind int) EntityID {
	return entity.CreateSpaceSomewhere(gameid, kind)
}

// CreateSpaceLocally 在本地game进程上创建一个指定Kind的Space。
func CreateSpaceLocally(kind Kind) *Space {
	if kind == 0 {
		gwlog.Panicf("Can not create nil space with kind=0. Game will create 1 nil space automatically.")
	}
	return entity.CreateSpaceLocally(kind)
}

// CreateEntityLocally 在本地game进程上创建一个指定类型的Entity
func CreateEntityLocally(typeName string) *Entity {
	return entity.CreateEntityLocally(typeName, nil)
}

// CreateEntityAnywhere 在随机选择的game进程上创建一个特定类型的Entity
func CreateEntityAnywhere(typeName string) EntityID {
	return entity.CreateEntitySomewhere(0, typeName)
}

func CreateEntityOnGame(gameid uint16, typeName string) EntityID {
	return entity.CreateEntitySomewhere(gameid, typeName)
}

// LoadEntityAnywhere 在随机选择的game进程上载入指定的Entity。
// GoWorld保证每个Entity最多只会存在于一个game进程，即只有一份实例。
// 如果这个Entity当前已经存在，则GoWorld不会做任何操作。
func LoadEntityAnywhere(typeName string, entityID EntityID) {
	entity.LoadEntityAnywhere(typeName, entityID)
}

// LoadEntityOnGame 在指定的game进程上载入特定的Entity对象。
// 如果这个Entity当前已经存在，则GoWorld不会做任何操作。因此在调用LoadEntityOnGame之后并不能严格保证Entity必然存在于所指定的game进程中。
func LoadEntityOnGame(typeName string, entityID EntityID, gameid GameID) {
	entity.LoadEntityOnGame(typeName, entityID, gameid)
}

// LoadEntityLocally 在当前的game进程中载入特定的Entity对象
// 如果这个Entity当前已经存在，则GoWorld不会做任何操作。因此在调用LoadEntityOnGame之后并不能严格保证Entity必然存在于当前game进程中。
func LoadEntityLocally(typeName string, entityID EntityID) {
	entity.LoadEntityOnGame(typeName, entityID, GetGameID())
}

// ListEntityIDs 获得某个类型的所有Entity对象的EntityID列表
// （这个接口将被弃用）
func ListEntityIDs(typeName string, callback storage.ListCallbackFunc) {
	storage.ListEntityIDs(typeName, callback)
}

// Exists 检查某个特定的Entity是否存在（已创建存盘）
func Exists(typeName string, entityID EntityID, callback storage.ExistsCallbackFunc) {
	storage.Exists(typeName, entityID, callback)
}

// GetEntity 获得当前game进程中的指定EntityID的Entity对象。不存在则返回nil。
func GetEntity(id EntityID) *Entity {
	return entity.GetEntity(id)
}

// GetSpace 获得当前进程中指定EntityID的Space对象。不存在则返回nil。
func GetSpace(id EntityID) *Space {
	return entity.GetSpace(id)
}

// GetGameID 获得当前game进程的GameID
func GetGameID() GameID {
	return game.GetGameID()
}

// MapAttr 创建一个新的空MapAttr对象
func MapAttr() *entity.MapAttr {
	return entity.NewMapAttr()
}

// ListAttr 创建一个新的空ListAttr对象
func ListAttr() *entity.ListAttr {
	return entity.NewListAttr()
}

// Entities 返回所有的Entity对象（通过EntityMap类型返回）
// 此接口将被弃用
func Entities() entity.EntityMap {
	return entity.Entities()
}

// Call 函数调用指定Entity的指定方法，并传递参数。
// 如果指定的Entity在当前game进程中，则会立刻调用其方法。否则将通过RPC发送函数调用和参数到所对应的game进程中。
func Call(id EntityID, method string, args ...interface{}) {
	entity.Call(id, method, args)
}

// CallService 发起一次Service调用。开发者只需要传入指定的Service名字，不需要指知道Service的EntityID或者当前在哪个game进程。
func CallService(serviceName string, method string, args ...interface{}) {
	service.CallService(serviceName, method, args)
}

// GetServiceEntityID 返回Service对象的EntityID。这个函数可以用来确定Service对象是否已经在某个game进程上成功创建或载入。
func GetServiceEntityID(serviceName string) common.EntityID {
	return service.GetServiceEntityID(serviceName)
}

// CallNilSpaces 向所有game进程中的NilSpace发起RPC调用。
// 由于每个game进程中都有一个唯一的NilSpace，因此这个函数想每个game进程都发起了一次函数调用。
func CallNilSpaces(method string, args ...interface{}) {
	entity.CallNilSpaces(method, args, game.GetGameID())
}

// GetNilSpaceID 返回特定game进程中的NilSpace的EntityID。
// GoWorld为每个game进程中的NilSpace使用了固定的EntityID值，例如目前GoWorld实现中在game1上NilSpace的EntityID总是"AAAAAAAAAAAAAAAx"，每次重启服务器都不会变化。
func GetNilSpaceID(gameid GameID) EntityID {
	return entity.GetNilSpaceID(gameid)
}

// GetNilSpace 返回当前game进程总的NilSpace对象
func GetNilSpace() *Space {
	return entity.GetNilSpace()
}

// GetKVDB 获得KVDB中指定key的值
func GetKVDB(key string, callback kvdb.KVDBGetCallback) {
	kvdb.Get(key, callback)
}

// PutKVDB 将制定的key-value对存入到KVDB中
func PutKVDB(key string, val string, callback kvdb.KVDBPutCallback) {
	kvdb.Put(key, val, callback)
}

// GetOrPutKVDB 读取指定key所对应的value，如果key所对应的值当前为空，则存入key-value键值对
func GetOrPutKVDB(key string, val string, callback kvdb.KVDBGetOrPutCallback) {
	kvdb.GetOrPut(key, val, callback)
}

// ListGameIDs 获得所有的GameID列表
func ListGameIDs() []GameID {
	return config.GetGameIDs()
}

// AddTimer adds a timer to be executed after specified duration
func AddCallback(d time.Duration, callback func()) {
	timer.AddCallback(d, callback)
}

// AddTimer adds a repeat timer to be executed every specified duration
func AddTimer(d time.Duration, callback func()) {
	timer.AddTimer(d, callback)
}

// Post posts a callback to be executed
// It is almost same as AddCallback(0, callback)
func Post(callback post.PostCallback) {
	post.Post(callback)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"sync"

	"github.com/jacobsa/gcloud/gcs"
)

var g_bucketOnce sync.Once
var g_bucket gcs.Bucket

func initBucket() {
	panic("TODO")
}

func getBucket() gcs.Bucket {
	g_bucketOnce.Do(initBucket)
	return g_bucket
}
<commit_msg>initBucket<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"sync"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/oauthutil"
	"google.golang.org/cloud/storage"
)

var g_bucketOnce sync.Once
var g_bucket gcs.Bucket

func makeBucket() (bucket gcs.Bucket, err error) {
	cfg := getConfig()

	// Create an authenticated HTTP client.
	httpClient, err := oauthutil.NewJWTHttpClient(
		cfg.KeyFile,
		[]string{storage.ScopeFullControl})

	if err != nil {
		err = fmt.Errorf("NewJWTHttpClient: %v", err)
		return
	}

	// Turn that into a connection. Note that a project ID isn't needed because
	// we don't list or create buckets (the bucket name itself implies its
	// project ID).
	const projID = "fake_comeback_project"

	conn, err := gcs.NewConn(projID, httpClient)
	if err != nil {
		err = fmt.Errorf("NewConn: %v", err)
		return
	}

	// Grab the bucket.
	bucket = conn.GetBucket(cfg.BucketName)

	return
}

func initBucket() {
	var err error

	g_bucket, err = makeBucket()
	if err != nil {
		panic(err)
	}
}

func getBucket() gcs.Bucket {
	g_bucketOnce.Do(initBucket)
	return g_bucket
}
<|endoftext|>


<commit_before>package coinparam

import (
	"fmt"
	"io"
	"log"
	"math"
	"math/big"
	"os"

	"github.com/adiabat/btcd/wire"
)

/* calcDiff returns a bool given two block headers.  This bool is
true if the correct dificulty adjustment is seen in the "next" header.
Only feed it headers n-2016 and n-1, otherwise it will calculate a difficulty
when no adjustment should take place, and return false.
Note that the epoch is actually 2015 blocks long, which is confusing. */
func calcDiffAdjustBitcoin(start, end *wire.BlockHeader, p *Params) uint32 {

	duration := end.Timestamp.Unix() - start.Timestamp.Unix()

	minRetargetTimespan :=
		int64(p.TargetTimespan.Seconds()) / p.RetargetAdjustmentFactor
	maxRetargetTimespan :=
		int64(p.TargetTimespan.Seconds()) * p.RetargetAdjustmentFactor

	if duration < minRetargetTimespan {
		duration = minRetargetTimespan
	} else if duration > maxRetargetTimespan {
		duration = maxRetargetTimespan
	}

	// calculation of new 32-byte difficulty target
	// first turn the previous target into a big int
	prevTarget := CompactToBig(end.Bits)
	// new target is old * duration...
	newTarget := new(big.Int).Mul(prevTarget, big.NewInt(duration))
	// divided by 2 weeks
	newTarget.Div(newTarget, big.NewInt(int64(p.TargetTimespan.Seconds())))

	// clip again if above minimum target (too easy)
	if newTarget.Cmp(p.PowLimit) > 0 {
		newTarget.Set(p.PowLimit)
	}

	fmt.Printf("start %d end %d adj %s newdiff %s\n",
		start.Timestamp.Unix(), end.Timestamp.Unix(),
		prevTarget.Div(newTarget, prevTarget).String(),
		prevTarget.Div(p.PowLimit, newTarget).String())

	// calculate and return 4-byte 'bits' difficulty from 32-byte target
	return BigToCompact(newTarget)
}

// diffBitcoin checks the difficulty of the last header in the slice presented
// give at least an epochlength of headers if this is a new epoch;
// otherwise give at least 2
// it's pretty ugly that it needs Params.  There must be some trick to get
// rid of that since diffBitcoin itself is already in the Params...
func diffBitcoin(
	headers []*wire.BlockHeader, height int32, p *Params) (uint32, error) {

	maxHeader := len(headers) - 1
	// we can reduce heiqght by startheight for all these calculations
	relHeight := height - p.StartHeight

	ltcmode := p.Name == "litetest4" || p.Name == "litereg" ||
		p.Name == "litecoin" || p.Name == "vtctest" || p.Name == "vtc"

	if p.Name == "regtest" {
		return 0x207fffff, nil
	}

	if maxHeader < 1 {
		return 0, fmt.Errorf("diffBitcoin got %d headers, min 2", len(headers))
	}

	prev := headers[len(headers)-2]
	cur := headers[len(headers)-1]

	// normal, no adjustment; Dn = Dn-1
	rightBits := prev.Bits

	epochLength := int(p.TargetTimespan / p.TargetTimePerBlock)
	epochStart := new(wire.BlockHeader)

	epochHeight := int(height) % epochLength

	// see if we're on a difficulty adjustment block
	if epochHeight == 0 {
		// if so, we need at least an epoch's worth of headers
		if maxHeader < int(epochLength) {
			return 0, fmt.Errorf("diffBitcoin not enough headers, got %d, need %d",
				len(headers), epochLength)
		}

		if ltcmode {
			if int(height) == epochLength {
				epochStart = headers[maxHeader-(epochLength)]
			} else {
				epochStart = headers[maxHeader-(epochLength-1)]
			}
		} else {
			epochStart = headers[maxHeader-(epochLength)]
		}
		// if so, check if difficulty adjustment is valid.
		// That whole "controlled supply" thing.
		// calculate diff n based on n-2016 ... n-1
		rightBits = calcDiffAdjustBitcoin(epochStart, prev, p)
		fmt.Printf("h %d diff adjust %x -> %x\n",
			height, prev.Bits, rightBits)
	} else if p.ReduceMinDifficulty { // not a new epoch
		// if on testnet, check for difficulty nerfing
		if cur.Timestamp.After(
			prev.Timestamp.Add(p.TargetTimePerBlock * 2)) {
			rightBits = p.PowLimitBits // difficulty 1
		} else {
			// actually need to iterate back to last nerfed block,
			// then take the diff from the one behind it
			// btcd code is findPrevTestNetDifficulty()
			// code in bitcoin/cpp:
			// while (pindex->pprev &&
			// pindex->nHeight % params.DifficultyAdjustmentInterval() != 0 &&
			// pindex->nBits == nProofOfWorkLimit)
			//maxIndex :=
			index := maxHeader
			minHeight := int(relHeight) - index
			for index > 0 &&
				minHeight+index%(epochLength) != 0 &&
				headers[index].Bits == p.PowLimitBits {
				index--
				fmt.Printf("height: %d index: %d\n", height, index)
			}
			rightBits = headers[index].Bits
		}
	}

	return rightBits, nil
}

func diffBTC(r io.ReadSeeker, height, startheight int32, p *Params) (uint32, error) {
	epochLength := int32(p.TargetTimespan / p.TargetTimePerBlock)
	var err error
	var cur, prev wire.BlockHeader
	ltcmode := p.Name == "litetest4" || p.Name == "litereg" ||
		p.Name == "litecoin" || p.Name == "vtctest" || p.Name == "vtc"
	offsetHeight := height - startheight
	// seek to n-1 header
	_, err = r.Seek(int64(80*(offsetHeight-1)), os.SEEK_SET)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}
	// read in n-1
	err = prev.Deserialize(r)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}
	// seek to curHeight header and read in
	_, err = r.Seek(int64(80*(offsetHeight)), os.SEEK_SET)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}
	err = cur.Deserialize(r)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}

	rightBits := prev.Bits // normal, no adjustment; Dn = Dn-1
	// see if we're on a difficulty adjustment block
	if (height)%epochLength == 0 {
		var epochStart wire.BlockHeader
		if ltcmode {
			if height == epochLength {
				_, err = r.Seek(int64(80*(offsetHeight-epochLength)), os.SEEK_SET)
			} else {
				_, err = r.Seek(int64(80*(offsetHeight-epochLength-1)), os.SEEK_SET)
			}
		} else {
			_, err = r.Seek(int64(80*(offsetHeight-epochLength)), os.SEEK_SET)
		}
		if err != nil {
			log.Printf(err.Error())
			return 0, err
		}
		err = epochStart.Deserialize(r)
		if err != nil {
			log.Printf(err.Error())
			return 0, err
		}
		// if so, check if difficulty adjustment is valid.
		// That whole "controlled supply" thing.
		// calculate diff n based on n-2016 ... n-1
		rightBits = calcDiffAdjustBitcoin(&epochStart, &prev, p)
	} else if p.ReduceMinDifficulty { // not a new epoch
		// if on testnet, check for difficulty nerfing
		if cur.Timestamp.After(
			prev.Timestamp.Add(p.TargetTimePerBlock * 2)) {
			rightBits = p.PowLimitBits // difficulty 1
		} else {
			var epochStart wire.BlockHeader
			_, err = r.Seek(int64(80*(offsetHeight-(offsetHeight%epochLength))), os.SEEK_SET)
			if err != nil {
				log.Printf(err.Error())
				return 0, err
			}
			err = epochStart.Deserialize(r)
			if err != nil {
				log.Printf(err.Error())
				return 0, err
			}

			rightBits = epochStart.Bits
		}
	}

	return rightBits, nil
}

// Uses Kimoto Gravity Well for difficulty adjustment. Used in VTC, MONA etc
func calcDiffAdjustKGW(
	r io.ReadSeeker, height, startheight int32, p *Params) (uint32, error) {
	var minBlocks, maxBlocks int32
	minBlocks = 144
	maxBlocks = 4032

	if height-1 < minBlocks {
		return p.PowLimitBits, nil
	}

	offsetHeight := height - startheight - 1

	var currentBlock wire.BlockHeader
	var err error

	// seek to n-1 header
	_, err = r.Seek(int64(80*offsetHeight), os.SEEK_SET)
	if err != nil {
		return 0, err
	}
	// read in n-1
	err = currentBlock.Deserialize(r)
	if err != nil {
		return 0, err
	}

	lastSolved := currentBlock

	var blocksScanned, actualRate, targetRate int64
	var difficultyAverage, previousDifficultyAverage big.Int
	var rateAdjustmentRatio, eventHorizonDeviation float64
	var eventHorizonDeviationFast, eventHorizonDevationSlow float64
	rateAdjustmentRatio = 1

	currentHeight := height - 1

	var i int32

	for i = 1; currentHeight > 0; i++ {
		if i > maxBlocks {
			break
		}

		blocksScanned++

		if i == 1 {
			difficultyAverage = *CompactToBig(currentBlock.Bits)
		} else {
			compact := CompactToBig(currentBlock.Bits)

			difference := new(big.Int).Sub(compact, &previousDifficultyAverage)
			difference.Div(difference, big.NewInt(int64(i)))
			difference.Add(difference, &previousDifficultyAverage)
			difficultyAverage = *difference
		}

		previousDifficultyAverage = difficultyAverage

		actualRate = lastSolved.Timestamp.Unix() - currentBlock.Timestamp.Unix()
		targetRate = int64(p.TargetTimePerBlock.Seconds()) * blocksScanned
		rateAdjustmentRatio = 1

		if actualRate < 0 {
			actualRate = 0
		}

		if actualRate != 0 && targetRate != 0 {
			rateAdjustmentRatio = float64(targetRate) / float64(actualRate)
		}

		eventHorizonDeviation = 1 + (0.7084 *
			math.Pow(float64(blocksScanned)/float64(minBlocks), -1.228))
		eventHorizonDeviationFast = eventHorizonDeviation
		eventHorizonDevationSlow = 1 / eventHorizonDeviation

		if blocksScanned >= int64(minBlocks) &&
			(rateAdjustmentRatio <= eventHorizonDevationSlow ||
				rateAdjustmentRatio >= eventHorizonDeviationFast) {
			break
		}

		if currentHeight <= 1 {
			break
		}

		currentHeight--

		_, err = r.Seek(int64(80*(currentHeight-startheight)), os.SEEK_SET)
		if err != nil {
			return 0, err
		}
		// read in n-1
		err = currentBlock.Deserialize(r)
		if err != nil {
			return 0, err
		}
	}

	newTarget := difficultyAverage
	if actualRate != 0 && targetRate != 0 {
		newTarget.Mul(&newTarget, big.NewInt(actualRate))

		newTarget.Div(&newTarget, big.NewInt(targetRate))
	}

	if newTarget.Cmp(p.PowLimit) == 1 {
		newTarget = *p.PowLimit
	}

	return BigToCompact(&newTarget), nil
}

// dummy function for VTC diff calc
// TODO
func diffVTCdummy(
	headers []*wire.BlockHeader, height int32, p *Params) (uint32, error) {
	return 0, nil
}

func diffVTCtest(r io.ReadSeeker, height, startheight int32, p *Params) (uint32, error) {
	if height < 2116 {
		return diffBTC(r, height, startheight, p)
	}

	offsetHeight := height - startheight

	// Testnet retargets only every 12 blocks
	if height%12 != 0 {
		var prev wire.BlockHeader
		var err error

		// seek to n-1 header
		_, err = r.Seek(int64(80*(offsetHeight-1)), os.SEEK_SET)
		if err != nil {
			return 0, err
		}
		// read in n-1
		err = prev.Deserialize(r)
		if err != nil {
			return 0, err
		}

		return prev.Bits, nil
	}

	// Run KGW
	return calcDiffAdjustKGW(r, height, startheight, p)
}
<commit_msg>diff works on testnet seems like it shouldn't but whatever<commit_after>package coinparam

import (
	"fmt"
	"io"
	"log"
	"math"
	"math/big"
	"os"

	"github.com/adiabat/btcd/wire"
)

/* calcDiff returns a bool given two block headers.  This bool is
true if the correct dificulty adjustment is seen in the "next" header.
Only feed it headers n-2016 and n-1, otherwise it will calculate a difficulty
when no adjustment should take place, and return false.
Note that the epoch is actually 2015 blocks long, which is confusing. */
func calcDiffAdjustBitcoin(start, end *wire.BlockHeader, p *Params) uint32 {

	duration := end.Timestamp.Unix() - start.Timestamp.Unix()

	minRetargetTimespan :=
		int64(p.TargetTimespan.Seconds()) / p.RetargetAdjustmentFactor
	maxRetargetTimespan :=
		int64(p.TargetTimespan.Seconds()) * p.RetargetAdjustmentFactor

	if duration < minRetargetTimespan {
		duration = minRetargetTimespan
	} else if duration > maxRetargetTimespan {
		duration = maxRetargetTimespan
	}

	// calculation of new 32-byte difficulty target
	// first turn the previous target into a big int
	prevTarget := CompactToBig(end.Bits)
	// new target is old * duration...
	newTarget := new(big.Int).Mul(prevTarget, big.NewInt(duration))
	// divided by 2 weeks
	newTarget.Div(newTarget, big.NewInt(int64(p.TargetTimespan.Seconds())))

	// clip again if above minimum target (too easy)
	if newTarget.Cmp(p.PowLimit) > 0 {
		newTarget.Set(p.PowLimit)
	}

	// calculate and return 4-byte 'bits' difficulty from 32-byte target
	return BigToCompact(newTarget)
}

// diffBitcoin checks the difficulty of the last header in the slice presented
// give at least an epochlength of headers if this is a new epoch;
// otherwise give at least 2
// it's pretty ugly that it needs Params.  There must be some trick to get
// rid of that since diffBitcoin itself is already in the Params...
func diffBitcoin(
	headers []*wire.BlockHeader, height int32, p *Params) (uint32, error) {

	ltcmode := p.Name == "litetest4" || p.Name == "litereg" ||
		p.Name == "litecoin" || p.Name == "vtctest" || p.Name == "vtc"

	if p.Name == "regtest" {
		return 0x207fffff, nil
	}

	prev := headers[len(headers)-2]
	cur := headers[len(headers)-1]

	// normal, no adjustment; Dn = Dn-1
	rightBits := prev.Bits

	epochLength := int(p.TargetTimespan / p.TargetTimePerBlock)
	epochStart := new(wire.BlockHeader)

	epochHeight := int(height) % epochLength
	maxHeader := len(headers) - 1

	// must include an epoch start header
	if epochHeight > maxHeader {
		return 0, fmt.Errorf("diffBitcoin got insufficient headers")
	}

	epochStart = headers[maxHeader-epochHeight]

	// see if we're on a difficulty adjustment block
	if epochHeight == 0 {
		// if so, we need at least an epoch's worth of headers
		if maxHeader < int(epochLength) {
			return 0, fmt.Errorf("diffBitcoin not enough headers, got %d, need %d",
				len(headers), epochLength)
		}

		if ltcmode {
			if int(height) == epochLength {
				epochStart = headers[maxHeader-epochLength]
			} else {
				epochStart = headers[maxHeader-(epochLength-1)]
			}
		} else {
			epochStart = headers[maxHeader-epochLength]
		}
		// if so, check if difficulty adjustment is valid.
		// That whole "controlled supply" thing.
		// calculate diff n based on n-2016 ... n-1
		rightBits = calcDiffAdjustBitcoin(epochStart, prev, p)
		fmt.Printf("h %d diff adjust %x -> %x\n",
			height, prev.Bits, rightBits)
	} else if p.ReduceMinDifficulty { // not a new epoch
		// if on testnet, check for difficulty nerfing
		if cur.Timestamp.After(
			prev.Timestamp.Add(p.TargetTimePerBlock * 2)) {
			rightBits = p.PowLimitBits // difficulty 1
		} else {
			// actually need to iterate back to last nerfed block,
			// then take the diff from the one behind it
			// btcd code is findPrevTestNetDifficulty()
			// code in bitcoin/cpp:
			// while (pindex->pprev &&
			// pindex->nHeight % params.DifficultyAdjustmentInterval() != 0 &&
			// pindex->nBits == nProofOfWorkLimit)

			// ugh I don't know, and whatever this is testnet.
			// just go to epoch start even though that's not what the cpp code
			// seems to say
			rightBits = epochStart.Bits
		}
	}

	return rightBits, nil
}

func diffBTC(r io.ReadSeeker, height, startheight int32, p *Params) (uint32, error) {
	epochLength := int32(p.TargetTimespan / p.TargetTimePerBlock)
	var err error
	var cur, prev wire.BlockHeader
	ltcmode := p.Name == "litetest4" || p.Name == "litereg" ||
		p.Name == "litecoin" || p.Name == "vtctest" || p.Name == "vtc"
	offsetHeight := height - startheight
	// seek to n-1 header
	_, err = r.Seek(int64(80*(offsetHeight-1)), os.SEEK_SET)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}
	// read in n-1
	err = prev.Deserialize(r)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}
	// seek to curHeight header and read in
	_, err = r.Seek(int64(80*(offsetHeight)), os.SEEK_SET)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}
	err = cur.Deserialize(r)
	if err != nil {
		log.Printf(err.Error())
		return 0, err
	}

	rightBits := prev.Bits // normal, no adjustment; Dn = Dn-1
	// see if we're on a difficulty adjustment block
	if (height)%epochLength == 0 {
		var epochStart wire.BlockHeader
		if ltcmode {
			if height == epochLength {
				_, err = r.Seek(int64(80*(offsetHeight-epochLength)), os.SEEK_SET)
			} else {
				_, err = r.Seek(int64(80*(offsetHeight-epochLength-1)), os.SEEK_SET)
			}
		} else {
			_, err = r.Seek(int64(80*(offsetHeight-epochLength)), os.SEEK_SET)
		}
		if err != nil {
			log.Printf(err.Error())
			return 0, err
		}
		err = epochStart.Deserialize(r)
		if err != nil {
			log.Printf(err.Error())
			return 0, err
		}
		// if so, check if difficulty adjustment is valid.
		// That whole "controlled supply" thing.
		// calculate diff n based on n-2016 ... n-1
		rightBits = calcDiffAdjustBitcoin(&epochStart, &prev, p)
	} else if p.ReduceMinDifficulty { // not a new epoch
		// if on testnet, check for difficulty nerfing
		if cur.Timestamp.After(
			prev.Timestamp.Add(p.TargetTimePerBlock * 2)) {
			rightBits = p.PowLimitBits // difficulty 1
		} else {
			var epochStart wire.BlockHeader
			_, err = r.Seek(int64(80*(offsetHeight-(offsetHeight%epochLength))), os.SEEK_SET)
			if err != nil {
				log.Printf(err.Error())
				return 0, err
			}
			err = epochStart.Deserialize(r)
			if err != nil {
				log.Printf(err.Error())
				return 0, err
			}

			rightBits = epochStart.Bits
		}
	}

	return rightBits, nil
}

// Uses Kimoto Gravity Well for difficulty adjustment. Used in VTC, MONA etc
func calcDiffAdjustKGW(
	r io.ReadSeeker, height, startheight int32, p *Params) (uint32, error) {
	var minBlocks, maxBlocks int32
	minBlocks = 144
	maxBlocks = 4032

	if height-1 < minBlocks {
		return p.PowLimitBits, nil
	}

	offsetHeight := height - startheight - 1

	var currentBlock wire.BlockHeader
	var err error

	// seek to n-1 header
	_, err = r.Seek(int64(80*offsetHeight), os.SEEK_SET)
	if err != nil {
		return 0, err
	}
	// read in n-1
	err = currentBlock.Deserialize(r)
	if err != nil {
		return 0, err
	}

	lastSolved := currentBlock

	var blocksScanned, actualRate, targetRate int64
	var difficultyAverage, previousDifficultyAverage big.Int
	var rateAdjustmentRatio, eventHorizonDeviation float64
	var eventHorizonDeviationFast, eventHorizonDevationSlow float64
	rateAdjustmentRatio = 1

	currentHeight := height - 1

	var i int32

	for i = 1; currentHeight > 0; i++ {
		if i > maxBlocks {
			break
		}

		blocksScanned++

		if i == 1 {
			difficultyAverage = *CompactToBig(currentBlock.Bits)
		} else {
			compact := CompactToBig(currentBlock.Bits)

			difference := new(big.Int).Sub(compact, &previousDifficultyAverage)
			difference.Div(difference, big.NewInt(int64(i)))
			difference.Add(difference, &previousDifficultyAverage)
			difficultyAverage = *difference
		}

		previousDifficultyAverage = difficultyAverage

		actualRate = lastSolved.Timestamp.Unix() - currentBlock.Timestamp.Unix()
		targetRate = int64(p.TargetTimePerBlock.Seconds()) * blocksScanned
		rateAdjustmentRatio = 1

		if actualRate < 0 {
			actualRate = 0
		}

		if actualRate != 0 && targetRate != 0 {
			rateAdjustmentRatio = float64(targetRate) / float64(actualRate)
		}

		eventHorizonDeviation = 1 + (0.7084 *
			math.Pow(float64(blocksScanned)/float64(minBlocks), -1.228))
		eventHorizonDeviationFast = eventHorizonDeviation
		eventHorizonDevationSlow = 1 / eventHorizonDeviation

		if blocksScanned >= int64(minBlocks) &&
			(rateAdjustmentRatio <= eventHorizonDevationSlow ||
				rateAdjustmentRatio >= eventHorizonDeviationFast) {
			break
		}

		if currentHeight <= 1 {
			break
		}

		currentHeight--

		_, err = r.Seek(int64(80*(currentHeight-startheight)), os.SEEK_SET)
		if err != nil {
			return 0, err
		}
		// read in n-1
		err = currentBlock.Deserialize(r)
		if err != nil {
			return 0, err
		}
	}

	newTarget := difficultyAverage
	if actualRate != 0 && targetRate != 0 {
		newTarget.Mul(&newTarget, big.NewInt(actualRate))

		newTarget.Div(&newTarget, big.NewInt(targetRate))
	}

	if newTarget.Cmp(p.PowLimit) == 1 {
		newTarget = *p.PowLimit
	}

	return BigToCompact(&newTarget), nil
}

// dummy function for VTC diff calc
// TODO
func diffVTCdummy(
	headers []*wire.BlockHeader, height int32, p *Params) (uint32, error) {
	return 0, nil
}

func diffVTCtest(r io.ReadSeeker, height, startheight int32, p *Params) (uint32, error) {
	if height < 2116 {
		return diffBTC(r, height, startheight, p)
	}

	offsetHeight := height - startheight

	// Testnet retargets only every 12 blocks
	if height%12 != 0 {
		var prev wire.BlockHeader
		var err error

		// seek to n-1 header
		_, err = r.Seek(int64(80*(offsetHeight-1)), os.SEEK_SET)
		if err != nil {
			return 0, err
		}
		// read in n-1
		err = prev.Deserialize(r)
		if err != nil {
			return 0, err
		}

		return prev.Bits, nil
	}

	// Run KGW
	return calcDiffAdjustKGW(r, height, startheight, p)
}
<|endoftext|>