<commit_before>// This file is subject to a 1-clause BSD license.
// Its contents can be found in the enclosed LICENSE file.

package ipintel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/jteeuwen/ircb/cmd"
	"github.com/jteeuwen/ircb/plugin"
	"github.com/jteeuwen/ircb/proto"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var regMibbit = regexp.MustCompile(`^[a-fA-F0-9]{8}$`)

const url = "http://api.neustar.biz/ipi/std/v1/ipinfo/%s?apikey=%s&sig=%s&format=json"

func init() { plugin.Register(New) }

type Plugin struct {
	*plugin.Base
}

func New(profile string) plugin.Plugin {
	p := new(Plugin)
	p.Base = plugin.New(profile, "ipintel")
	return p
}

func (p *Plugin) Load(c *proto.Client) (err error) {
	err = p.Base.Load(c)
	if err != nil {
		return
	}

	ini := p.LoadConfig()
	if ini == nil {
		log.Fatalf("[ipintel] No configuration found.")
		return
	}

	key := ini.Section("api").S("key", "")
	if len(key) == 0 {
		log.Fatalf("[ipintel] No API key found.")
		return
	}

	shared := ini.Section("api").S("shared", "")
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	drift := ini.Section("api").I64("drift", 0)
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	w := new(cmd.Command)
	w.Name = "loc"
	w.Description = "Fetch geo-location data for the given IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "ip", Description: "IPv4 address to look up", Pattern: cmd.RegIPv4},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hash := md5.New()
		stamp := fmt.Sprintf("%d", time.Now().UTC().Unix()+drift)
		io.WriteString(hash, key+shared+stamp)

		sig := fmt.Sprintf("%x", hash.Sum(nil))
		target := fmt.Sprintf(url, cmd.Params[0].Value, key, sig)

		resp, err := http.Get(target)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()

		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		var data Response
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		inf := data.IPInfo

		c.PrivMsg(m.Receiver,
			"%s: %s (%s), Network org.: %s, Carrier: %s, TLD: %s, SLD: %s. "+
				"Location: %s/%s/%s/%s (%f, %f). Postalcode: %s, Timezone: %d",
			m.SenderName,

			inf.IPAddress, inf.IPType,
			inf.Network.Organization,
			inf.Network.Carrier,
			inf.Network.Domain.TLD,
			inf.Network.Domain.SLD,

			inf.Location.Continent,
			inf.Location.CountryData.Name,
			inf.Location.StateData.Name,
			inf.Location.CityData.Name,
			inf.Location.Latitude,
			inf.Location.Longitude,
			inf.Location.CityData.PostalCode,
			inf.Location.CityData.TimeZone,
		)
	}

	cmd.Register(w)

	w = new(cmd.Command)
	w.Name = "mibbit"
	w.Description = "Resolve a mibbit address to a real IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "hex", Description: "Mibbit hex string", Pattern: regMibbit},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hex := cmd.Params[0].Value

		var ip [4]uint64
		var err error

		ip[0], err = strconv.ParseUint(hex[:2], 16, 8)
		if err != nil {
			goto error
		}

		ip[1], err = strconv.ParseUint(hex[2:4], 16, 8)
		if err != nil {
			goto error
		}

		ip[2], err = strconv.ParseUint(hex[4:6], 16, 8)
		if err != nil {
			goto error
		}

		ip[3], err = strconv.ParseUint(hex[6:], 16, 8)
		if err != nil {
			goto error
		}

		c.PrivMsg(m.Receiver, "%s: %s -> %d.%d.%d.%d",
			m.SenderName, hex, ip[0], ip[1], ip[2], ip[3])
		return

	error:
		c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
	}

	cmd.Register(w)

	return
}
<commit_msg>Includes hostname lookup in mibbit command.<commit_after>// This file is subject to a 1-clause BSD license.
// Its contents can be found in the enclosed LICENSE file.

package ipintel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/jteeuwen/ircb/cmd"
	"github.com/jteeuwen/ircb/plugin"
	"github.com/jteeuwen/ircb/proto"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var regMibbit = regexp.MustCompile(`^[a-fA-F0-9]{8}$`)

const url = "http://api.neustar.biz/ipi/std/v1/ipinfo/%s?apikey=%s&sig=%s&format=json"

func init() { plugin.Register(New) }

type Plugin struct {
	*plugin.Base
}

func New(profile string) plugin.Plugin {
	p := new(Plugin)
	p.Base = plugin.New(profile, "ipintel")
	return p
}

func (p *Plugin) Load(c *proto.Client) (err error) {
	err = p.Base.Load(c)
	if err != nil {
		return
	}

	ini := p.LoadConfig()
	if ini == nil {
		log.Fatalf("[ipintel] No configuration found.")
		return
	}

	key := ini.Section("api").S("key", "")
	if len(key) == 0 {
		log.Fatalf("[ipintel] No API key found.")
		return
	}

	shared := ini.Section("api").S("shared", "")
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	drift := ini.Section("api").I64("drift", 0)
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	w := new(cmd.Command)
	w.Name = "loc"
	w.Description = "Fetch geo-location data for the given IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "ip", Description: "IPv4 address to look up", Pattern: cmd.RegIPv4},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hash := md5.New()
		stamp := fmt.Sprintf("%d", time.Now().UTC().Unix()+drift)
		io.WriteString(hash, key+shared+stamp)

		sig := fmt.Sprintf("%x", hash.Sum(nil))
		target := fmt.Sprintf(url, cmd.Params[0].Value, key, sig)

		resp, err := http.Get(target)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()

		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		var data Response
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		inf := data.IPInfo

		c.PrivMsg(m.Receiver,
			"%s: %s (%s), Network org.: %s, Carrier: %s, TLD: %s, SLD: %s. "+
				"Location: %s/%s/%s/%s (%f, %f). Postalcode: %s, Timezone: %d",
			m.SenderName,

			inf.IPAddress, inf.IPType,
			inf.Network.Organization,
			inf.Network.Carrier,
			inf.Network.Domain.TLD,
			inf.Network.Domain.SLD,

			inf.Location.Continent,
			inf.Location.CountryData.Name,
			inf.Location.StateData.Name,
			inf.Location.CityData.Name,
			inf.Location.Latitude,
			inf.Location.Longitude,
			inf.Location.CityData.PostalCode,
			inf.Location.CityData.TimeZone,
		)
	}

	cmd.Register(w)

	w = new(cmd.Command)
	w.Name = "mibbit"
	w.Description = "Resolve a mibbit address to a real IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "hex", Description: "Mibbit hex string", Pattern: regMibbit},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hex := cmd.Params[0].Value

		var ip [4]uint64
		var err error

		ip[0], err = strconv.ParseUint(hex[:2], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[1], err = strconv.ParseUint(hex[2:4], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[2], err = strconv.ParseUint(hex[4:6], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[3], err = strconv.ParseUint(hex[6:], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		address := fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
		names, err := net.LookupAddr(address)

		if err != nil || len(names) == 0 {
			c.PrivMsg(m.Receiver, "%s: %s is %s", m.SenderName, hex, address)
		} else {
			c.PrivMsg(m.Receiver, "%s: %s is %s / %s",
				m.SenderName, hex, address, names[0])
		}
	}

	cmd.Register(w)

	return
}
<|endoftext|>


<commit_before>// Copyright 2013 The Pythia Authors.
// This file is part of Pythia.
//
// Pythia is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// Pythia is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pythia.  If not, see <http://www.gnu.org/licenses/>.

package backend

import (
	"pythia"
	"testing"
	"testutils"
	"testutils/pytest"
	"time"
)

// NewTestJob creates a job, configured with the paths exported from make.
func newTestJob(task pythia.Task, input string) *Job {
	job := NewJob()
	job.Task = task
	job.Input = input
	job.UmlPath = pytest.UmlPath
	job.EnvDir = pytest.VmDir
	job.TasksDir = pytest.TasksDir
	return job
}

// RunTask executes task with input.
// It checks that the execution time and output length are within the specified
// limits.
func runTask(t *testing.T, task pythia.Task, input string) (status pythia.Status, output string) {
	job := newTestJob(task, input)
	wd := testutils.Watchdog(t, task.Limits.Time+1)
	status, output = job.Execute()
	wd.Stop()
	if len(output) > task.Limits.Output {
		t.Errorf("Job output is too large: max %d, got %d.", task.Limits.Output,
			len(output))
	}
	return
}

// RunTaskCheck behaves like RunTask, but additionally checks for expected
// status and output.
func runTaskCheck(t *testing.T, task pythia.Task, input string,
	status pythia.Status, output string) {
	st, out := runTask(t, task, input)
	testutils.Expect(t, "status", status, st)
	testutils.Expect(t, "output", output, out)
}

// Shortcut for runTask(t, pytest.ReadTask(t, basename), ...)
func run(t *testing.T, basename string, input string, status pythia.Status,
	output string) {
	runTaskCheck(t, pytest.ReadTask(t, basename), input, status, output)
}

// Basic hello world task.
func TestJobHelloWorld(t *testing.T) {
	run(t, "hello-world", "", pythia.Success, "Hello world!\n")
}

// Check that the goroutines are cleaned correctly.
func TestJobCleanup(t *testing.T) {
	testutils.CheckGoroutines(t, func() {
		run(t, "hello-world", "", pythia.Success, "Hello world!\n")
	})
}

// Hello world task with input.
func TestJobHelloInput(t *testing.T) {
	run(t, "hello-input", "me\npythia\n",
		pythia.Success, "Hello me!\nHello pythia!\n")
}

// This task should time out after 5 seconds.
func TestJobTimeout(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping timeout test in short mode")
	}
	run(t, "timeout", "", pythia.Timeout, "Start\n")
}

// This task should overflow the output buffer.
func TestJobOverflow(t *testing.T) {
	task := pytest.ReadTask(t, "overflow")
	t.Log("Trying limit 10")
	task.Limits.Output = 10
	runTaskCheck(t, task, "", pythia.Success, "abcde")
	t.Log("Trying limit 6")
	task.Limits.Output = 6
	runTaskCheck(t, task, "", pythia.Success, "abcde")
	t.Log("Trying limit 5")
	task.Limits.Output = 5
	runTaskCheck(t, task, "", pythia.Success, "abcde")
	t.Log("Trying limit 4")
	task.Limits.Output = 4
	runTaskCheck(t, task, "", pythia.Overflow, "abcd")
	t.Log("Trying limit 3")
	task.Limits.Output = 3
	runTaskCheck(t, task, "", pythia.Overflow, "abc")
}

// This task should overflow and be killed before the end.
func TestJobOverflowKill(t *testing.T) {
	wd := testutils.Watchdog(t, 2)
	run(t, "overflow-kill", "", pythia.Overflow, "abcde")
	wd.Stop()
}

// This task is a fork bomb. It should succeed, but not take the whole time.
func TestJobForkbomb(t *testing.T) {
	wd := testutils.Watchdog(t, 10)
	run(t, "forkbomb", "", pythia.Success, "Start\nDone\n")
	wd.Stop()
}

// Flooding the disk should not have any adverse effect.
func TestJobFlooddisk(t *testing.T) {
	run(t, "flooddisk", "", pythia.Success, "Start\nDone\n")
}

// Aborting a job shall be immediate.
func TestJobAbort(t *testing.T) {
	job := newTestJob(pytest.ReadTask(t, "timeout"), "")
	done := make(chan bool)
	go func() {
		wd := testutils.Watchdog(t, 2)
		status, output := job.Execute()
		wd.Stop()
		testutils.Expect(t, "status", pythia.Abort, status)
		testutils.Expect(t, "output", "Start\n", output)
		done <- true
	}()
	time.Sleep(1 * time.Second)
	job.Abort()
	<-done
}

// vim:set sw=4 ts=4 noet:
<commit_msg>Fix the watchdog to allow 60sec instead of 2 for forkbomb testing<commit_after>// Copyright 2013 The Pythia Authors.
// This file is part of Pythia.
//
// Pythia is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, version 3 of the License.
//
// Pythia is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with Pythia.  If not, see <http://www.gnu.org/licenses/>.

package backend

import (
	"pythia"
	"testing"
	"testutils"
	"testutils/pytest"
	"time"
)

// NewTestJob creates a job, configured with the paths exported from make.
func newTestJob(task pythia.Task, input string) *Job {
	job := NewJob()
	job.Task = task
	job.Input = input
	job.UmlPath = pytest.UmlPath
	job.EnvDir = pytest.VmDir
	job.TasksDir = pytest.TasksDir
	return job
}

// RunTask executes task with input.
// It checks that the execution time and output length are within the specified
// limits.
func runTask(t *testing.T, task pythia.Task, input string) (status pythia.Status, output string) {
	job := newTestJob(task, input)
	wd := testutils.Watchdog(t, task.Limits.Time+1)
	status, output = job.Execute()
	wd.Stop()
	if len(output) > task.Limits.Output {
		t.Errorf("Job output is too large: max %d, got %d.", task.Limits.Output,
			len(output))
	}
	return
}

// RunTaskCheck behaves like RunTask, but additionally checks for expected
// status and output.
func runTaskCheck(t *testing.T, task pythia.Task, input string,
	status pythia.Status, output string) {
	st, out := runTask(t, task, input)
	testutils.Expect(t, "status", status, st)
	testutils.Expect(t, "output", output, out)
}

// Shortcut for runTask(t, pytest.ReadTask(t, basename), ...)
func run(t *testing.T, basename string, input string, status pythia.Status,
	output string) {
	runTaskCheck(t, pytest.ReadTask(t, basename), input, status, output)
}

// Basic hello world task.
func TestJobHelloWorld(t *testing.T) {
	run(t, "hello-world", "", pythia.Success, "Hello world!\n")
}

// Check that the goroutines are cleaned correctly.
func TestJobCleanup(t *testing.T) {
	testutils.CheckGoroutines(t, func() {
		run(t, "hello-world", "", pythia.Success, "Hello world!\n")
	})
}

// Hello world task with input.
func TestJobHelloInput(t *testing.T) {
	run(t, "hello-input", "me\npythia\n",
		pythia.Success, "Hello me!\nHello pythia!\n")
}

// This task should time out after 5 seconds.
func TestJobTimeout(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping timeout test in short mode")
	}
	run(t, "timeout", "", pythia.Timeout, "Start\n")
}

// This task should overflow the output buffer.
func TestJobOverflow(t *testing.T) {
	task := pytest.ReadTask(t, "overflow")
	t.Log("Trying limit 10")
	task.Limits.Output = 10
	runTaskCheck(t, task, "", pythia.Success, "abcde")
	t.Log("Trying limit 6")
	task.Limits.Output = 6
	runTaskCheck(t, task, "", pythia.Success, "abcde")
	t.Log("Trying limit 5")
	task.Limits.Output = 5
	runTaskCheck(t, task, "", pythia.Success, "abcde")
	t.Log("Trying limit 4")
	task.Limits.Output = 4
	runTaskCheck(t, task, "", pythia.Overflow, "abcd")
	t.Log("Trying limit 3")
	task.Limits.Output = 3
	runTaskCheck(t, task, "", pythia.Overflow, "abc")
}

// This task should overflow and be killed before the end.
func TestJobOverflowKill(t *testing.T) {
	wd := testutils.Watchdog(t, 2)
	run(t, "overflow-kill", "", pythia.Overflow, "abcde")
	wd.Stop()
}

// This task is a fork bomb. It should succeed, but not take the whole time.
func TestJobForkbomb(t *testing.T) {
	wd := testutils.Watchdog(t, 60)
	run(t, "forkbomb", "", pythia.Success, "Start\nDone\n")
	wd.Stop()
}

// Flooding the disk should not have any adverse effect.
func TestJobFlooddisk(t *testing.T) {
	run(t, "flooddisk", "", pythia.Success, "Start\nDone\n")
}

// Aborting a job shall be immediate.
func TestJobAbort(t *testing.T) {
	job := newTestJob(pytest.ReadTask(t, "timeout"), "")
	done := make(chan bool)
	go func() {
		wd := testutils.Watchdog(t, 2)
		status, output := job.Execute()
		wd.Stop()
		testutils.Expect(t, "status", pythia.Abort, status)
		testutils.Expect(t, "output", "Start\n", output)
		done <- true
	}()
	time.Sleep(1 * time.Second)
	job.Abort()
	<-done
}

// vim:set sw=4 ts=4 noet:
<|endoftext|>


<commit_before>package generator

import (
	"context"
	"time"

	"chain/database/pg"
	"chain/errors"
	"chain/log"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

// A BlockSigner signs blocks.
type BlockSigner interface {
	// SignBlock returns an ed25519 signature over the block's sighash.
	// See also the Chain Protocol spec for the complete required behavior
	// of a block signer.
	SignBlock(context.Context, *bc.Block) (signature []byte, err error)
}

// generator produces new blocks on an interval.
type generator struct {
	// config
	chain   *protocol.Chain
	signers []BlockSigner

	// latestBlock and latestSnapshot are current as long as this
	// process remains the leader process. If the process is demoted,
	// generator.Generate() should return and this struct should be
	// garbage collected.
	latestBlock    *bc.Block
	latestSnapshot *state.Snapshot
}

// Generate runs in a loop, making one new block
// every block period. It returns when its context
// is canceled.
func Generate(ctx context.Context, c *protocol.Chain, s []BlockSigner, period time.Duration) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	recoveredBlock, recoveredSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	g := &generator{
		chain:          c,
		signers:        s,
		latestBlock:    recoveredBlock,
		latestSnapshot: recoveredSnapshot,
	}

	// Check to see if we already have a pending, generated block.
	// This can happen if the leader process exits between generating
	// the block and committing the signed block to the blockchain.
	b, err := g.getPendingBlock(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	if b != nil && (g.latestBlock == nil || b.Height == g.latestBlock.Height+1) {
		s, err := g.chain.ValidateBlock(ctx, g.latestSnapshot, g.latestBlock, b)
		if err != nil {
			log.Fatal(ctx, err)
		}

		// g.commitBlock will update g.latestBlock and g.latestSnapshot.
		_, err = g.commitBlock(ctx, b, s)
		if err != nil {
			log.Fatal(ctx, err)
		}
	}

	ticks := time.Tick(period)
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Generate exiting")
			return
		case <-ticks:
			_, err := g.makeBlock(ctx)
			if err != nil {
				log.Error(ctx, err)
			}
		}
	}
}

// GetBlocks returns contiguous blocks
// with heights larger than afterHeight,
// in block-height order.
// If successful, it always returns at least one block,
// waiting if necessary until one is created.
// It is not guaranteed to return all available blocks.
// It is an error to request blocks very far in the future.
func GetBlocks(ctx context.Context, c *protocol.Chain, afterHeight uint64) ([]*bc.Block, error) {
	// TODO(kr): This is not a generator function.
	// Move this to another package.
	err := c.WaitForBlockSoon(afterHeight + 1)
	if err != nil {
		return nil, errors.Wrapf(err, "waiting for block at height %d", afterHeight+1)
	}

	const q = `SELECT data FROM blocks WHERE height > $1 ORDER BY height`
	var blocks []*bc.Block
	err = pg.ForQueryRows(ctx, q, afterHeight, func(b bc.Block) {
		blocks = append(blocks, &b)
	})
	if err != nil {
		return nil, errors.Wrap(err, "querying blocks from the db")
	}
	return blocks, nil
}
<commit_msg>core/generator: limit blocks fetched at a time<commit_after>package generator

import (
	"context"
	"time"

	"chain/database/pg"
	"chain/errors"
	"chain/log"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

// A BlockSigner signs blocks.
type BlockSigner interface {
	// SignBlock returns an ed25519 signature over the block's sighash.
	// See also the Chain Protocol spec for the complete required behavior
	// of a block signer.
	SignBlock(context.Context, *bc.Block) (signature []byte, err error)
}

// generator produces new blocks on an interval.
type generator struct {
	// config
	chain   *protocol.Chain
	signers []BlockSigner

	// latestBlock and latestSnapshot are current as long as this
	// process remains the leader process. If the process is demoted,
	// generator.Generate() should return and this struct should be
	// garbage collected.
	latestBlock    *bc.Block
	latestSnapshot *state.Snapshot
}

// Generate runs in a loop, making one new block
// every block period. It returns when its context
// is canceled.
func Generate(ctx context.Context, c *protocol.Chain, s []BlockSigner, period time.Duration) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	recoveredBlock, recoveredSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	g := &generator{
		chain:          c,
		signers:        s,
		latestBlock:    recoveredBlock,
		latestSnapshot: recoveredSnapshot,
	}

	// Check to see if we already have a pending, generated block.
	// This can happen if the leader process exits between generating
	// the block and committing the signed block to the blockchain.
	b, err := g.getPendingBlock(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	if b != nil && (g.latestBlock == nil || b.Height == g.latestBlock.Height+1) {
		s, err := g.chain.ValidateBlock(ctx, g.latestSnapshot, g.latestBlock, b)
		if err != nil {
			log.Fatal(ctx, err)
		}

		// g.commitBlock will update g.latestBlock and g.latestSnapshot.
		_, err = g.commitBlock(ctx, b, s)
		if err != nil {
			log.Fatal(ctx, err)
		}
	}

	ticks := time.Tick(period)
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Generate exiting")
			return
		case <-ticks:
			_, err := g.makeBlock(ctx)
			if err != nil {
				log.Error(ctx, err)
			}
		}
	}
}

// GetBlocks returns contiguous blocks
// with heights larger than afterHeight,
// in block-height order.
// If successful, it always returns at least one block,
// waiting if necessary until one is created.
// It is not guaranteed to return all available blocks.
// It is an error to request blocks very far in the future.
func GetBlocks(ctx context.Context, c *protocol.Chain, afterHeight uint64) ([]*bc.Block, error) {
	// TODO(kr): This is not a generator function.
	// Move this to another package.
	err := c.WaitForBlockSoon(afterHeight + 1)
	if err != nil {
		return nil, errors.Wrapf(err, "waiting for block at height %d", afterHeight+1)
	}

	const q = `SELECT data FROM blocks WHERE height > $1 ORDER BY height LIMIT 10`
	var blocks []*bc.Block
	err = pg.ForQueryRows(ctx, q, afterHeight, func(b bc.Block) {
		blocks = append(blocks, &b)
	})
	if err != nil {
		return nil, errors.Wrap(err, "querying blocks from the db")
	}
	return blocks, nil
}
<|endoftext|>


<commit_before>package contnet

import (
	"github.com/asaskevich/EventBus"
	"log"
	"math/rand"
	"sync"
	"time"
)

func init() {
	rand.Seed(time.Now().Unix())
}

type NetConfig struct {
	MaxContentAge           time.Duration
	CheckContentAgeInterval time.Duration
	ItemsPerPage            uint8
	NoveltyPct              float64
	SnapshotPath            string
	SnapshotInterval        time.Duration
}

type Net struct {
	sync.RWMutex
	config       *NetConfig
	bus          *EventBus.EventBus
	contentStore *ContentStore
	profileStore *ProfileStore
	trendStore   *TrendStore
	index        *Index
}
type NetFactory struct{}

func (factory NetFactory) New(config *NetConfig) *Net {
	bus := EventBus.New()
	contentStore := Object.ContentStore.New(config, bus)
	net := &Net{
		config:       config,
		bus:          bus,
		contentStore: contentStore,
		profileStore: Object.ProfileStore.New(),
		trendStore:   Object.TrendStore.New(bus),
		index:        Object.Index.New(config, bus, contentStore),
	}
	if err := net.Restore(); err != nil {
		log.Print("Failed to restore net object, proceeding empty.")
	}
	go net.__snapshot()
	go net.index.__refresh()

	return net
}

func (net *Net) __snapshot() {
	for {
		net.Snapshot()
		time.Sleep(net.config.SnapshotInterval)
	}
}

func (net *Net) Snapshot() error {
	if err := net.contentStore.Snapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.Snapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.Snapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.Snapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

func (net *Net) Restore() error {
	if err := net.contentStore.RestoreFromSnapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.RestoreFromSnapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.RestoreFromSnapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.RestoreFromSnapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

// Attempts to update network object with content specified.
// If content did not exist, it is added to the network.
// If content did exist, it is updated.
func (net *Net) SaveContent(content *Content) {
	net.contentStore.Upsert(content)
}

// Attempts to update network object with action for profile specified.
func (net *Net) SaveAction(action *Action) error {
	// get related content's copy, if any
	relatedContent := net.contentStore.Get(action.ContentID)

	// if content does not exist
	if relatedContent == nil {
		return Errors.ContentNotFound
	}

	// inject related content to action
	action.Content = relatedContent

	// save action to profile
	net.profileStore.Save(action)

	// everything is okay
	return nil
}

type SelectionCacheInfo struct {
	CurrentIndex int
	MaximumIndex int
}

func (net *Net) Select(profileID int64, page uint8) []ID {
	// get user's profile, if any
	profile := net.profileStore.Get(ID(profileID))

	// assign topic interests & sort
	var interests TopicInterests
	if profile != nil {
		TopicInterestBy(topicInterestCriteria).Sort(profile.TopicInterests)
	} else {
		interests = TopicInterests{}
	}

	totalContentsToRetrieve := int(page) * int(net.config.ItemsPerPage)
	totalContentsByInterest := int((1 - net.config.NoveltyPct) * float64(totalContentsToRetrieve))

	// select interesting contents
	interestingContents := net.__selectOfInterest(interests, totalContentsByInterest)

	// select popular contents (how many remaining / 2)
	totalContentsByTrend := int((totalContentsToRetrieve - len(interestingContents)) / 2)
	trendingContents := net.__selectOfTrending(interestingContents, totalContentsByTrend)

	// fill the remainder randomly
	totalContentsRemaining := totalContentsToRetrieve - len(interestingContents) - len(trendingContents)
	out := append(interestingContents, trendingContents...)
	remainingContents := net.__selectOfRemaining(out, totalContentsRemaining)

	// prep the final result
	out = append(out, remainingContents...)
	return out
}

func (net *Net) __selectOfInterest(interests TopicInterests, howMany int) []ID {
	// abort early if nothing to select..
	if len(interests) == 0 {
		return []ID{}
	}

	// we have sorted topic interests for this user.
	// now we extract topics from topic-interest objects for querying the index
	// and we use dem iterations to form cumulative probabilities slice to be used later for randomly drawing topics
	interestTopics := Topics{}
	cumulativeProbabilities := []float64{}
	for i := 0; i < len(interests); i++ {
		interestTopics = append(interestTopics, &interests[i].Topic)
		cumulativeProbabilities = append(cumulativeProbabilities, float64(interests[i].Interest))
		if i > 0 {
			cumulativeProbabilities[i] += cumulativeProbabilities[i-1]
		}
	}

	// use the previously created topics slice for querying the index
	topicContents := net.index.GetForTopics(interestTopics)
	// now we have all content IDs for all topics interesting to this user
	// to control where we're at, we will create temporary cache that will mark extraction location for each topic
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(interests); i++ {
		interest := interests[i]
		cache[interest.Topic] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(topicContents[i]) - 1,
		}
	}

	// now we have the following data available:
	// 1. topics of interest for user sorted in descending order by how interesting they are
	// 2. for each topic of interest there is an array of ID's available for that topic
	// 3. map showing what index to choose for each topic
	// 4. cumulative probabilities

	// while there is anything interesting for user available & we haven't selected as many items as needed
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		// select a random topic of interest
		topic, i := __drawRandomTopicInterest(interests, cumulativeProbabilities)

		// select its best content (first index available)
		indexToSelect := cache[topic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[topic].MaximumIndex {
			selectedID := topicContents[i][indexToSelect]

			if !__idsContainID(out, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[topic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[topic].CurrentIndex > cache[topic].MaximumIndex {
			// remove from cache
			delete(cache, topic)
			// remove from interest topics
			interestTopics = append(interestTopics[:i], interestTopics[i+1:]...)
			// remove from topic contents
			topicContents = append(topicContents[:i], topicContents[i+1:]...)
			// remove from interests
			interests = append(interests[:i], interests[i+1:]...)
			// recalculate interests
			cumulativeProbabilities = __recalculateCumulativeProbabilities(interests)
		}
	}

	return out
}

func __idsContainID(ids []ID, id ID) bool {
	for i := 0; i < len(ids); i++ {
		if ids[i] == id {
			return true
		}
	}
	return false
}

func __drawRandomTopicInterest(topicInterests TopicInterests, cumulativeProbabilities []float64) (Topic, int) {
	randomFloat := rand.Float64()
	for i := 0; i < len(cumulativeProbabilities); i++ {
		if cumulativeProbabilities[i] >= randomFloat {
			return topicInterests[i].Topic, i
		}
	}
	// this is guaranteed not to happen
	return topicInterests[len(topicInterests)-1].Topic, len(topicInterests)
}

func __recalculateCumulativeProbabilities(topicInterests TopicInterests) []float64 {
	// calculate sum of cumulative interests
	sum := float64(0)
	for i := 0; i < len(topicInterests); i++ {
		sum += float64(topicInterests[i].CumulativeInterest)
	}

	output := []float64{}

	for i := 0; i < len(topicInterests); i++ {
		output = append(output, float64(topicInterests[i].CumulativeInterest)/sum)
		if i > 0 {
			output[i] += output[i-1]
		}
	}

	return output
}

func (net *Net) __selectOfTrending(ignore []ID, howMany int) []ID {
	// select top 10 trending topics
	trendingTopics := net.trendStore.GetTopN(10)

	// nothing trending, bail early
	if len(trendingTopics) == 0 {
		return []ID{}
	}

	// get content IDs for trending topics
	contentIDs := net.index.GetForTopics(trendingTopics)

	// cache indices foreach contentID
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(trendingTopics); i++ {
		cache[*trendingTopics[i]] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(contentIDs[i]) - 1,
		}
	}

	nextTrendInd := 0
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
        trendingTopic := *trendingTopics[nextTrendInd]

        // select its best content (first index available)
        indexToSelect := cache[trendingTopic].CurrentIndex

        // current index can be ahead of the max index in some special cases
        if indexToSelect <= cache[trendingTopic].MaximumIndex {
            selectedID := contentIDs[nextTrendInd][indexToSelect]

            if !__idsContainID(out, selectedID) && !__idsContainID(ignore, selectedID) {
                out = append(out, selectedID)
            }
        }

        // advance current index in cache
        cache[trendingTopic].CurrentIndex++

        // if we overshot the maximum index for this topic, remove it
        if cache[trendingTopic].CurrentIndex > cache[trendingTopic].MaximumIndex {
            // remove from cache
            delete(cache, trendingTopic)
            // remove from trending topics
            trendingTopics = append(trendingTopics[:nextTrendInd], trendingTopics[nextTrendInd+1:]...)
            // go back one position
            nextTrendInd--
        }

        // reset the counter
		nextTrendInd++
		if nextTrendInd == len(trendingTopics) {
			nextTrendInd = 0
		}
	}

	return out
}

func (net *Net) __selectOfRemaining(ignore []ID, howMany int) []ID {
	// if we are asking for more content items then there are available in content store
	if howMany >= (net.contentStore.Count() - len(ignore)) {
		return net.contentStore.GetAllContentIDs()
	}

	// otherwise, draw random elements until howMany is satisifed. ignore duplicates of course.
	out := []ID{}

	for len(out) < howMany {
		randomID := net.contentStore.GetAnyContentID()
		if !__idsContainID(out, randomID) && !__idsContainID(ignore, randomID) {
			out = append(out, randomID)
		}
	}

	return out

}

func (net *Net) Describe() *NetDescription {
	return &NetDescription{
		Contents: net.contentStore.Describe(),
		Index:    net.index.Describe(),
		Profiles: net.profileStore.Describe(),
		Trends:   net.trendStore.Describe(),
	}
}
<commit_msg>Fixes and formatting<commit_after>package contnet

import (
	"github.com/asaskevich/EventBus"
	"log"
	"math/rand"
	"sync"
	"time"
)

func init() {
	rand.Seed(time.Now().Unix())
}

type NetConfig struct {
	MaxContentAge           time.Duration
	CheckContentAgeInterval time.Duration
	ItemsPerPage            uint8
	NoveltyPct              float64
	SnapshotPath            string
	SnapshotInterval        time.Duration
}

type Net struct {
	sync.RWMutex
	config       *NetConfig
	bus          *EventBus.EventBus
	contentStore *ContentStore
	profileStore *ProfileStore
	trendStore   *TrendStore
	index        *Index
}
type NetFactory struct{}

func (factory NetFactory) New(config *NetConfig) *Net {
	bus := EventBus.New()
	contentStore := Object.ContentStore.New(config, bus)
	net := &Net{
		config:       config,
		bus:          bus,
		contentStore: contentStore,
		profileStore: Object.ProfileStore.New(),
		trendStore:   Object.TrendStore.New(bus),
		index:        Object.Index.New(config, bus, contentStore),
	}
	if err := net.Restore(); err != nil {
		log.Print("Failed to restore net object, proceeding empty.")
	}
	go net.__snapshot()
	go net.index.__refresh()

	return net
}

func (net *Net) __snapshot() {
	for {
		net.Snapshot()
		time.Sleep(net.config.SnapshotInterval)
	}
}

func (net *Net) Snapshot() error {
	if err := net.contentStore.Snapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.Snapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.Snapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.Snapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

func (net *Net) Restore() error {
	if err := net.contentStore.RestoreFromSnapshot(net.config.SnapshotPath, "content"); err != nil {
		return err
	}

	if err := net.index.RestoreFromSnapshot(net.config.SnapshotPath, "index"); err != nil {
		return err
	}

	if err := net.profileStore.RestoreFromSnapshot(net.config.SnapshotPath, "profiles"); err != nil {
		return err
	}

	if err := net.trendStore.RestoreFromSnapshot(net.config.SnapshotPath, "trends"); err != nil {
		return err
	}

	return nil
}

// Attempts to update network object with content specified.
// If content did not exist, it is added to the network.
// If content did exist, it is updated.
func (net *Net) SaveContent(content *Content) {
	net.contentStore.Upsert(content)
}

// Attempts to update network object with action for profile specified.
func (net *Net) SaveAction(action *Action) error {
	// get related content's copy, if any
	relatedContent := net.contentStore.Get(action.ContentID)

	// if content does not exist
	if relatedContent == nil {
		return Errors.ContentNotFound
	}

	// inject related content to action
	action.Content = relatedContent

	// save action to profile
	net.profileStore.Save(action)

	// everything is okay
	return nil
}

type SelectionCacheInfo struct {
	CurrentIndex int
	MaximumIndex int
}

func (net *Net) Select(profileID int64, page uint8) []ID {
	// get user's profile, if any
	profile := net.profileStore.Get(ID(profileID))

	// assign topic interests & sort
	var interests TopicInterests
	if profile != nil {
		TopicInterestBy(topicInterestCriteria).Sort(profile.TopicInterests)
		interests = profile.TopicInterests
	} else {
		interests = TopicInterests{}
	}

	totalContentsToRetrieve := int(page) * int(net.config.ItemsPerPage)
	totalContentsByInterest := int((1 - net.config.NoveltyPct) * float64(totalContentsToRetrieve))

	// select interesting contents
	interestingContents := net.__selectOfInterest(interests, totalContentsByInterest)
	//log.Println("Interesting: ", len(interestingContents))

	// select popular contents (how many remaining / 2)
	totalContentsByTrend := int((totalContentsToRetrieve - len(interestingContents)) / 2)
	trendingContents := net.__selectOfTrending(interestingContents, totalContentsByTrend)
	//log.Println("Trending: ", len(trendingContents))

	// fill the remainder randomly
	totalContentsRemaining := totalContentsToRetrieve - len(interestingContents) - len(trendingContents)
	out := append(interestingContents, trendingContents...)
	remainingContents := net.__selectOfRemaining(out, totalContentsRemaining)
	//log.Println("Remaining: ", len(remainingContents))

	// prep the final result
	out = append(out, remainingContents...)
	return out
}

func (net *Net) __selectOfInterest(interests TopicInterests, howMany int) []ID {
	// abort early if nothing to select..
	if len(interests) == 0 {
		return []ID{}
	}

	// we have sorted topic interests for this user.
	// now we extract topics from topic-interest objects for querying the index
	// and we use dem iterations to form cumulative probabilities slice to be used later for randomly drawing topics
	interestTopics := Topics{}
	cumulativeProbabilities := []float64{}
	for i := 0; i < len(interests); i++ {
		interestTopics = append(interestTopics, &interests[i].Topic)
		cumulativeProbabilities = append(cumulativeProbabilities, float64(interests[i].Interest))
		if i > 0 {
			cumulativeProbabilities[i] += cumulativeProbabilities[i-1]
		}
	}

	// use the previously created topics slice for querying the index
	topicContents := net.index.GetForTopics(interestTopics)

	// now we have all content IDs for all topics interesting to this user
	// to control where we're at, we will create temporary cache that will mark extraction location for each topic
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(interests); i++ {
		interest := interests[i]
		cache[interest.Topic] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(topicContents[i]) - 1,
		}
	}

	// now we have the following data available:
	// 1. topics of interest for user sorted in descending order by how interesting they are
	// 2. for each topic of interest there is an array of ID's available for that topic
	// 3. map showing what index to choose for each topic
	// 4. cumulative probabilities

	// while there is anything interesting for user available & we haven't selected as many items as needed
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		// select a random topic of interest
		topic, i := __drawRandomTopicInterest(interests, cumulativeProbabilities)

		// select its best content (first index available)
		indexToSelect := cache[topic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[topic].MaximumIndex {
			selectedID := topicContents[i][indexToSelect]
			if !__idsContainID(out, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[topic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[topic].CurrentIndex > cache[topic].MaximumIndex {
			// remove from cache
			delete(cache, topic)
			// remove from interest topics
			interestTopics = append(interestTopics[:i], interestTopics[i+1:]...)
			// remove from topic contents
			topicContents = append(topicContents[:i], topicContents[i+1:]...)
			// remove from interests
			interests = append(interests[:i], interests[i+1:]...)
			// recalculate interests
			cumulativeProbabilities = __recalculateCumulativeProbabilities(interests)
		}
	}

	return out
}

func __idsContainID(ids []ID, id ID) bool {
	for i := 0; i < len(ids); i++ {
		if ids[i] == id {
			return true
		}
	}
	return false
}

func __drawRandomTopicInterest(topicInterests TopicInterests, cumulativeProbabilities []float64) (Topic, int) {
	randomFloat := rand.Float64()
	for i := 0; i < len(cumulativeProbabilities); i++ {
		if cumulativeProbabilities[i] >= randomFloat {
			return topicInterests[i].Topic, i
		}
	}
	// this is guaranteed not to happen
	return topicInterests[len(topicInterests)-1].Topic, len(topicInterests) - 1
}

func __recalculateCumulativeProbabilities(topicInterests TopicInterests) []float64 {
	// calculate sum of cumulative interests
	sum := float64(0)
	for i := 0; i < len(topicInterests); i++ {
		sum += float64(topicInterests[i].CumulativeInterest)
	}

	output := []float64{}

	for i := 0; i < len(topicInterests); i++ {
		output = append(output, float64(topicInterests[i].CumulativeInterest)/sum)
		if i > 0 {
			output[i] += output[i-1]
		}
	}

	return output
}

func (net *Net) __selectOfTrending(ignore []ID, howMany int) []ID {
	// select top 10 trending topics
	trendingTopics := net.trendStore.GetTopN(10)

	// nothing trending, bail early
	if len(trendingTopics) == 0 {
		return []ID{}
	}

	// get content IDs for trending topics
	contentIDs := net.index.GetForTopics(trendingTopics)

	// cache indices foreach contentID
	cache := map[Topic]*SelectionCacheInfo{}
	// filling in the cache
	for i := 0; i < len(trendingTopics); i++ {
		cache[*trendingTopics[i]] = &SelectionCacheInfo{
			CurrentIndex: 0,
			MaximumIndex: len(contentIDs[i]) - 1,
		}
	}

	nextTrendInd := 0
	out := []ID{}
	for len(cache) > 0 && len(out) < howMany {
		trendingTopic := *trendingTopics[nextTrendInd]

		// select its best content (first index available)
		indexToSelect := cache[trendingTopic].CurrentIndex

		// current index can be ahead of the max index in some special cases
		if indexToSelect <= cache[trendingTopic].MaximumIndex {
			selectedID := contentIDs[nextTrendInd][indexToSelect]

			if !__idsContainID(out, selectedID) && !__idsContainID(ignore, selectedID) {
				out = append(out, selectedID)
			}
		}

		// advance current index in cache
		cache[trendingTopic].CurrentIndex++

		// if we overshot the maximum index for this topic, remove it
		if cache[trendingTopic].CurrentIndex > cache[trendingTopic].MaximumIndex {
			// remove from cache
			delete(cache, trendingTopic)
			// remove from trending topics
			trendingTopics = append(trendingTopics[:nextTrendInd], trendingTopics[nextTrendInd+1:]...)
			// go back one position
			nextTrendInd--
		}

		// reset the counter
		nextTrendInd++
		if nextTrendInd == len(trendingTopics) {
			nextTrendInd = 0
		}
	}

	return out
}

func (net *Net) __selectOfRemaining(ignore []ID, howMany int) []ID {
	// if we are asking for more content items then there are available in content store
	if howMany >= (net.contentStore.Count() - len(ignore)) {
		ids := net.contentStore.GetAllContentIDs()
		// filter out ignored ones
		for i := 0; i < len(ids); i++ {
			if __idsContainID(ignore, ids[i]) {
				ids = append(ids[:i], ids[i+1:]...)
				i--
			}
		}
		return ids

	}

	// otherwise, draw random elements until howMany is satisifed. ignore duplicates of course.
	out := []ID{}

	for len(out) < howMany {
		randomID := net.contentStore.GetAnyContentID()
		if !__idsContainID(out, randomID) && !__idsContainID(ignore, randomID) {
			out = append(out, randomID)
		}
	}

	return out

}

func (net *Net) Describe() *NetDescription {
	return &NetDescription{
		Contents: net.contentStore.Describe(),
		Index:    net.index.Describe(),
		Profiles: net.profileStore.Describe(),
		Trends:   net.trendStore.Describe(),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"
	"path"
	"sync"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

var (
	sitemap = make(map[string]*SiteData)
	mu      = new(sync.RWMutex)
)

// SiteData holds imageboard settings
type SiteData struct {
	Ib          uint
	Api         string
	Img         string
	Title       string
	Desc        string
	Nsfw        bool
	Style       string
	Logo        string
	Base        string
	Imageboards []Imageboard
}

type Imageboard struct {
	Title   string
	Address string
}

// Details gets the imageboard settings from the request for the page handler variables
func Details() gin.HandlerFunc {
	return func(c *gin.Context) {

		var host, base string

		host = c.Request.Host

		// figure out our path and host
		if path.Dir(host) == "." {
			// we're on the base in this case
			host = path.Base(host)
		} else {
			host = path.Dir(host)
			base = fmt.Sprintf("%s/", path.Base(host))
		}

		mu.RLock()
		// check the sitemap to see if its cached
		site := sitemap[host]
		mu.RUnlock()

		// if not query the database
		if site == nil {

			sitedata := &SiteData{}

			// set the base for angularjs
			sitedata.Base = base

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.GetDb")
				c.Abort()
				return
			}

			// get the info about the imageboard
			err = dbase.QueryRow(`SELECT ib_id,ib_title,ib_description,ib_nsfw,ib_api,ib_img,ib_style,ib_logo FROM imageboards WHERE ib_domain = ?`, host).Scan(&sitedata.Ib, &sitedata.Title, &sitedata.Desc, &sitedata.Nsfw, &sitedata.Api, &sitedata.Img, &sitedata.Style, &sitedata.Logo)
			if err == sql.ErrNoRows {
				c.JSON(e.ErrorMessage(e.ErrNotFound))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			} else if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			}

			// collect the links to the other imageboards for nav menu
			rows, err := dbase.Query(`SELECT ib_title,ib_domain FROM imageboards WHERE ib_id != ?`, sitedata.Ib)
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}
			defer rows.Close()

			for rows.Next() {

				ib := Imageboard{}

				err := rows.Scan(&ib.Title, &ib.Address)
				if err != nil {
					return
				}

				sitedata.Imageboards = append(sitedata.Imageboards, ib)
			}
			if rows.Err() != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}

			mu.Lock()
			sitemap[host] = sitedata
			mu.Unlock()

		}

		c.Next()

	}

}

// IndexController generates pages for angularjs frontend
func IndexController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusOK, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"base":        site.Base,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}

// ErrorController generates pages and a 404 response
func ErrorController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusNotFound, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"base":        site.Base,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}
<commit_msg>add subdirectory support<commit_after>package main

import (
	"database/sql"
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"
	"path"
	"sync"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

var (
	sitemap = make(map[string]*SiteData)
	mu      = new(sync.RWMutex)
)

// SiteData holds imageboard settings
type SiteData struct {
	Ib          uint
	Api         string
	Img         string
	Title       string
	Desc        string
	Nsfw        bool
	Style       string
	Logo        string
	Base        string
	Imageboards []Imageboard
}

type Imageboard struct {
	Title   string
	Address string
}

// Details gets the imageboard settings from the request for the page handler variables
func Details() gin.HandlerFunc {
	return func(c *gin.Context) {

		var host, base string

		host = c.Request.Host

		// figure out our path and host
		if path.Dir(host) == "." {
			// we're on the base in this case
			host = path.Base(host)
		} else {
			host = path.Dir(host)
			base = fmt.Sprintf("%s/", path.Base(host))
		}

		fmt.Println(host, base)

		mu.RLock()
		// check the sitemap to see if its cached
		site := sitemap[host]
		mu.RUnlock()

		// if not query the database
		if site == nil {

			sitedata := &SiteData{}

			// set the base for angularjs
			sitedata.Base = base

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.GetDb")
				c.Abort()
				return
			}

			// get the info about the imageboard
			err = dbase.QueryRow(`SELECT ib_id,ib_title,ib_description,ib_nsfw,ib_api,ib_img,ib_style,ib_logo FROM imageboards WHERE ib_domain = ?`, host).Scan(&sitedata.Ib, &sitedata.Title, &sitedata.Desc, &sitedata.Nsfw, &sitedata.Api, &sitedata.Img, &sitedata.Style, &sitedata.Logo)
			if err == sql.ErrNoRows {
				c.JSON(e.ErrorMessage(e.ErrNotFound))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			} else if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.QueryRow")
				return
			}

			// collect the links to the other imageboards for nav menu
			rows, err := dbase.Query(`SELECT ib_title,ib_domain FROM imageboards WHERE ib_id != ?`, sitedata.Ib)
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}
			defer rows.Close()

			for rows.Next() {

				ib := Imageboard{}

				err := rows.Scan(&ib.Title, &ib.Address)
				if err != nil {
					return
				}

				sitedata.Imageboards = append(sitedata.Imageboards, ib)
			}
			if rows.Err() != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}

			mu.Lock()
			sitemap[host] = sitedata
			mu.Unlock()

		}

		c.Next()

	}

}

// IndexController generates pages for angularjs frontend
func IndexController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusOK, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"base":        site.Base,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}

// ErrorController generates pages and a 404 response
func ErrorController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusNotFound, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"base":        site.Base,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"sync"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
)

var (
	sitemap = make(map[string]*SiteData)
	mu      = new(sync.RWMutex)
)

type SiteData struct {
	Ib          uint
	Api         string
	Img         string
	Title       string
	Desc        string
	Nsfw        bool
	Style       string
	Logo        string
	Imageboards []Imageboard
}

type Imageboard struct {
	Title   string
	Address string
}

// gets the details from the request for the page handler variables
func Details() gin.HandlerFunc {
	return func(c *gin.Context) {

		host := c.Request.Host

		mu.RLock()
		site := sitemap[host]
		mu.RUnlock()

		if site == nil {

			sitedata := &SiteData{}

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			err = dbase.QueryRow(`SELECT ib_id,ib_title,ib_description,ib_nsfw,ib_api,ib_img,ib_style,ib_logo FROM imageboards WHERE ib_domain = ?`, host).Scan(&sitedata.Ib, &sitedata.Title, &sitedata.Desc, &sitedata.Nsfw, &sitedata.Api, &sitedata.Img, &sitedata.Style, &sitedata.Logo)
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			rows, err := dbase.Query(`SELECT ib_title,ib_domain FROM imageboards WHERE ib_id != ?`, sitedata.Ib)
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}
			defer rows.Close()

			for rows.Next() {

				ib := Imageboard{}

				err := rows.Scan(&ib.Title, &ib.Address)
				if err != nil {
					return
				}

				sitedata.Imageboards = append(sitedata.Imageboards, ib)
			}
			err = rows.Err()
			if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

			mu.Lock()
			sitemap[host] = sitedata
			mu.Unlock()

		}

		c.Next()

	}
}

// Handles index page generation
func IndexController(c *gin.Context) {

	// Get parameters from validate middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusOK, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}

// Handles error messages for wrong routes
func ErrorController(c *gin.Context) {

	c.String(http.StatusNotFound, "Not Found")

	return

}
<commit_msg>add error meta data<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"sync"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	e "github.com/eirka/eirka-libs/errors"
)

var (
	sitemap = make(map[string]*SiteData)
	mu      = new(sync.RWMutex)
)

// SiteData holds imageboard settings
type SiteData struct {
	Ib          uint
	Api         string
	Img         string
	Title       string
	Desc        string
	Nsfw        bool
	Style       string
	Logo        string
	Imageboards []Imageboard
}

type Imageboard struct {
	Title   string
	Address string
}

// Details gets the imageboard settings from the request for the page handler variables
func Details() gin.HandlerFunc {
	return func(c *gin.Context) {

		host := c.Request.Host

		mu.RLock()
		site := sitemap[host]
		mu.RUnlock()

		if site == nil {

			sitedata := &SiteData{}

			// Get Database handle
			dbase, err := db.GetDb()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.GetDb")
				c.Abort()
				return
			}

			err = dbase.QueryRow(`SELECT ib_id,ib_title,ib_description,ib_nsfw,ib_api,ib_img,ib_style,ib_logo FROM imageboards WHERE ib_domain = ?`, host).Scan(&sitedata.Ib, &sitedata.Title, &sitedata.Desc, &sitedata.Nsfw, &sitedata.Api, &sitedata.Img, &sitedata.Style, &sitedata.Logo)
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.QueryRow")
				c.Abort()
				return
			}

			rows, err := dbase.Query(`SELECT ib_title,ib_domain FROM imageboards WHERE ib_id != ?`, sitedata.Ib)
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}
			defer rows.Close()

			for rows.Next() {

				ib := Imageboard{}

				err := rows.Scan(&ib.Title, &ib.Address)
				if err != nil {
					return
				}

				sitedata.Imageboards = append(sitedata.Imageboards, ib)
			}
			err = rows.Err()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err).SetMeta("Details.Query")
				c.Abort()
				return
			}

			mu.Lock()
			sitemap[host] = sitedata
			mu.Unlock()

		}

		c.Next()

	}

}

// IndexController generates pages for angularjs frontend
func IndexController(c *gin.Context) {

	// Get parameters from csrf middleware
	csrf_token := c.MustGet("csrf_token").(string)

	host := c.Request.Host

	mu.RLock()
	site := sitemap[host]
	mu.RUnlock()

	c.HTML(http.StatusOK, "index", gin.H{
		"primjs":      config.Settings.Prim.Js,
		"primcss":     config.Settings.Prim.Css,
		"ib":          site.Ib,
		"apisrv":      site.Api,
		"imgsrv":      site.Img,
		"title":       site.Title,
		"desc":        site.Desc,
		"nsfw":        site.Nsfw,
		"style":       site.Style,
		"logo":        site.Logo,
		"imageboards": site.Imageboards,
		"csrf":        csrf_token,
	})

	return

}

// ErrorController handles error messages for wrong routes
func ErrorController(c *gin.Context) {

	c.String(http.StatusNotFound, "Not Found")

	return

}
<|endoftext|>


<commit_before>// Copyright 2014 The Convention Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Every file starts with a copyright. Gophers tend to use BSD-like licenses.
// "The PackageName Authors" in copyrights along with maintained AUTHORS,
// CONTRIBUTORS, and LICENSE files are idiomatic. Comments are proper English
// sentences, always started with "//", and bound to 80 char line length. Code
// itself is not bound any line length, but long lines are often a code smell.

// Package conventions is an example Go package that is loaded with
// documentation specifically targetting new developers that have done the Go
// Tour and have read the documentation on the "go" binary, but want to know
// more about writing packages and software.
//
// The documentation for an entire package written here; only one of the files
// in a package should have it. Comments are always on the line above the thing
// they are documenting. Package declarations and any exports require comments
// that should be in the form of "Package pkgname ..." and "ExportName ..."
// respectively. When Go is properly commented in this way, automatic docs
// can be generated via the "godoc" binary or via godoc.org. For example:
// http://godoc.org/github.com/jzelinskie/conventions is where you can see the
// docs for this package, although this package was created specifically to
// have its source read directly.
package conventions

// The import and const keywords should always be expanded into their multi-line
// form.
//
// One $GOPATH contains all of the packages on the internet. Whenever something
// is exported, the author must maintain stability of that package's API. For
// git repos, "go get" will look for a "go1" tag or fallback to the master
// branch. If an author is a bad citizen, they may break their API and those
// depending on it will have two options: update their code or fork the package.
// Forking comes at a cost: it changes the name of all of the imports,
// essentially creating a brand new package.
import (
	// Imports from the Go standard libraries are grouped together at the top.
	"errors"
	"fmt"

	// Third party imports are placed below the standard libraries. Setting up
	// text editor hooks to run gofmt and lint on save is a good idea.
	_ "github.com/golang/lint"

	// Local imports are placed below the third party libraries. Importing a
	// package as _ is useful if you require some side-effect of init() from the
	// package.
	_ "github.com/jzelinskie/conventions/subpkg"
)

// Constants always follow the imports.
const (
	enums = iota
	are
	pretty
	cool
)

// Exported variables are usually in multi-line form, however when defining
// huge literals such as the constants used in cryptographic hashing algorithms,
// it is fine to have repeated var statements.
var (
	// ErrStupidMistake is an example of an exported error type.
	ErrStupidMistake = errors.New("stupid mistakes are the story of my life")
)

// Example is an example of how to define new types.
//
// Fields are one-per-line and grouped together however you think is most
// logical. Don't worry too much about writing struct tags for every encoding
// under the sun. Someone can easily create a wrapper type that implements
// the encoding's marshaler interface (i.e. json.Marshaler).
type Example struct {
	ID          int
	Name        string
	Description string

	irrelevant string
}

// NewExample is an example constructor for the Example type.
//
// Constructors are always the first thing to follow the type declaration.
// Constructors always begin with "New" and can simply be called "New" if the
// name of the type being returned is the same as the package (i.e.
// config.New()). Returning a reference literal is better than allocating with
// the new keyword. Always have a trailing comma on the last item in anything
// multi-line; this is done to reduce diff sizes in version control systems.
func NewExample(id int, name, description string) *Example {
	return &Example{
		ID:          id,
		Name:        name,
		Description: description,
	}
}

// Method is an example method for the Example type.
//
// Methods are always what come after constructors. Godoc will order them
// alphabetically, but it is often more useful to order them logically.
// For most receivers, prefer pointers over values.  Exceptions to this
// are small arrays, small structs (or structs that are natural value
// types), maps, functions, and channels.
func (e Example) Method() (bool, error) {
	// The use of := for capturing the result of function calls and expressions
	// is preferrable to the use of var.
	nameLength := len(e.Name)
	result := nameLength < 10

	// Keep lines between logical groups of statements, including the final
	// return.
	return result, nil
}

// Second to last in a file is an init func. This func enables you to run code
// at package import time.
func init() {
}

// Last in a file should be a main func. This func should only be present in a
// package named main to generate a binary file.
func main() {
	// Break long function calls into its multi-line form. Functions with lots of
	// parameters are often a code smell.
	example := NewExample(
		1,
		"conventions",
		"this is getting quite meta",
	)

	// If there is no desire to expose any variables outside of the scope of an
	// if-statement, this two-claused form is preferable.
	if _, err := example.Method(); err != nil {
		// Panics are used only in situations that inform the programmer that they
		// are doing something wrong. They should not be used like exceptions --
		// error is a builtin type for a reason.
		panic("conventions: example method should always return a nil error.")
	}

	// When a zero-value is desired, use the var keyword. Avoid using := inside
	// if statements as variables can accidently become shadowed.
	var multiplier int
	if example.ID > 10 {
		multiplier = 10
	} else {
		multiplier = example.ID
	}
	fmt.Printf("Multiplier: %v", multiplier)
}
<commit_msg>Add ShortMethod<commit_after>// Copyright 2014 The Convention Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Every file starts with a copyright. Gophers tend to use BSD-like licenses.
// "The PackageName Authors" in copyrights along with maintained AUTHORS,
// CONTRIBUTORS, and LICENSE files are idiomatic. Comments are proper English
// sentences, always started with "//", and bound to 80 char line length. Code
// itself is not bound any line length, but long lines are often a code smell.

// Package conventions is an example Go package that is loaded with
// documentation specifically targetting new developers that have done the Go
// Tour and have read the documentation on the "go" binary, but want to know
// more about writing packages and software.
//
// The documentation for an entire package written here; only one of the files
// in a package should have it. Comments are always on the line above the thing
// they are documenting. Package declarations and any exports require comments
// that should be in the form of "Package pkgname ..." and "ExportName ..."
// respectively. When Go is properly commented in this way, automatic docs
// can be generated via the "godoc" binary or via godoc.org. For example:
// http://godoc.org/github.com/jzelinskie/conventions is where you can see the
// docs for this package, although this package was created specifically to
// have its source read directly.
package conventions

// The import and const keywords should always be expanded into their multi-line
// form.
//
// One $GOPATH contains all of the packages on the internet. Whenever something
// is exported, the author must maintain stability of that package's API. For
// git repos, "go get" will look for a "go1" tag or fallback to the master
// branch. If an author is a bad citizen, they may break their API and those
// depending on it will have two options: update their code or fork the package.
// Forking comes at a cost: it changes the name of all of the imports,
// essentially creating a brand new package.
import (
	// Imports from the Go standard libraries are grouped together at the top.
	"errors"
	"fmt"

	// Third party imports are placed below the standard libraries. Setting up
	// text editor hooks to run gofmt and lint on save is a good idea.
	_ "github.com/golang/lint"

	// Local imports are placed below the third party libraries. Importing a
	// package as _ is useful if you require some side-effect of init() from the
	// package.
	_ "github.com/jzelinskie/conventions/subpkg"
)

// Constants always follow the imports.
const (
	enums = iota
	are
	pretty
	cool
)

// Exported variables are usually in multi-line form, however when defining
// huge literals such as the constants used in cryptographic hashing algorithms,
// it is fine to have repeated var statements.
var (
	// ErrStupidMistake is an example of an exported error type.
	ErrStupidMistake = errors.New("stupid mistakes are the story of my life")
)

// Example is an example of how to define new types.
//
// Fields are one-per-line and grouped together however you think is most
// logical. Don't worry too much about writing struct tags for every encoding
// under the sun. Someone can easily create a wrapper type that implements
// the encoding's marshaler interface (i.e. json.Marshaler).
type Example struct {
	ID          int
	Name        string
	Description string

	irrelevant string
}

// NewExample is an example constructor for the Example type.
//
// Constructors are always the first thing to follow the type declaration.
// Constructors always begin with "New" and can simply be called "New" if the
// name of the type being returned is the same as the package (i.e.
// config.New()). Returning a reference literal is better than allocating with
// the new keyword. Always have a trailing comma on the last item in anything
// multi-line; this is done to reduce diff sizes in version control systems.
func NewExample(id int, name, description string) *Example {
	return &Example{
		ID:          id,
		Name:        name,
		Description: description,
	}
}

// Method is an example method for the Example type.
//
// Methods are always what come after constructors. Godoc will order them
// alphabetically, but it is often more useful to order them logically.
// For most receivers, prefer pointers over values.  Exceptions to this
// are small arrays, small structs (or structs that are natural value
// types), maps, functions, and channels.
func (e Example) Method() (bool, error) {
	// The use of := for capturing the result of function calls and expressions
	// is preferrable to the use of var.
	nameLength := len(e.Name)
	result := nameLength < 10

	// Keep lines between logical groups of statements, including the final
	// return.
	return result, nil
}

// ShortMethod is an example of a method short enough to be kept on one line.
// The gofmt tool will not automatically put methods that are this short on one
// line, but it will not break them down into multiple lines either.
func (e Example) ShortMethod() int { return 0 }

// Second to last in a file is an init func. This func enables you to run code
// at package import time.
func init() {
}

// Last in a file should be a main func. This func should only be present in a
// package named main to generate a binary file.
func main() {
	// Break long function calls into its multi-line form. Functions with lots of
	// parameters are often a code smell.
	example := NewExample(
		1,
		"conventions",
		"this is getting quite meta",
	)

	// If there is no desire to expose any variables outside of the scope of an
	// if-statement, this two-claused form is preferable.
	if _, err := example.Method(); err != nil {
		// Panics are used only in situations that inform the programmer that they
		// are doing something wrong. They should not be used like exceptions --
		// error is a builtin type for a reason.
		panic("conventions: example method should always return a nil error.")
	}

	// When a zero-value is desired, use the var keyword. Avoid using := inside
	// if statements as variables can accidently become shadowed.
	var multiplier int
	if example.ID > 10 {
		multiplier = 10
	} else {
		multiplier = example.ID
	}
	fmt.Printf("Multiplier: %v", multiplier)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/rpc"
	"pault.ag/go/service"
	"strings"
)

/* */

func BeACoordinator(cert, key, ca, host string, port int) {
	log.Printf("Bringing TCP server online!\n")
	l, err := service.ListenFromKeys(
		fmt.Sprintf("%s:%d", host, port),
		cert, key, ca,
	)
	if err != nil {
		log.Fatalf("Server Ouchie! %s", err)
	}
	coordinator := MinionCoordinator{}
	log.Printf("Great, waiting for Minions, and telling them what to do!\n")
	service.Handle(l, &coordinator)
}

/**/

type MinionCoordinator struct{ service.Coordinator }

func (m *MinionCoordinator) Handle(client *rpc.Client, conn *service.Conn) {
	minion := RemoteMinion{client}

	log.Printf("Got a connection from %s\n", conn.CommonNames[0])

	arches, err := minion.Arches()
	if err != nil {
		log.Fatalf("Ouch: %s\n", err)
	}

	log.Printf(" -> They can do %s", strings.Join(arches, ", "))

	ftbfs, err := minion.Build(
		[]Archive{Archive{}},
		ChrootTarget{Chroot: "unstable", Suite: "unstable"},
		"amd64", "pool/f/fbautostart_fnord.dsc",
	)
	log.Printf("Heard back: %s %s\n", ftbfs, err)
}
<commit_msg>shuffle order<commit_after>package main

import (
	"fmt"
	"log"
	"net/rpc"
	"pault.ag/go/service"
	"strings"
)

/* */

type MinionCoordinator struct{ service.Coordinator }

func (m *MinionCoordinator) Handle(client *rpc.Client, conn *service.Conn) {
	minion := RemoteMinion{client}

	log.Printf("Got a connection from %s\n", conn.CommonNames[0])

	arches, err := minion.Arches()
	if err != nil {
		log.Fatalf("Ouch: %s\n", err)
	}

	log.Printf(" -> They can do %s", strings.Join(arches, ", "))

	ftbfs, err := minion.Build(
		[]Archive{Archive{}},
		ChrootTarget{Chroot: "unstable", Suite: "unstable"},
		"amd64", "pool/f/fbautostart_fnord.dsc",
	)
	log.Printf("Heard back: %s %s\n", ftbfs, err)
}

/* */

func BeACoordinator(cert, key, ca, host string, port int) {
	log.Printf("Bringing TCP server online!\n")
	l, err := service.ListenFromKeys(
		fmt.Sprintf("%s:%d", host, port),
		cert, key, ca,
	)
	if err != nil {
		log.Fatalf("Server Ouchie! %s", err)
	}
	coordinator := MinionCoordinator{}
	log.Printf("Great, waiting for Minions, and telling them what to do!\n")
	service.Handle(l, &coordinator)
}

/**/
<|endoftext|>


<commit_before>package ddtxn

import (
	"container/heap"
	"ddtxn/dlog"
	"flag"
	"fmt"
	"log"
	"sync/atomic"
	"time"
)

const (
	BUMP_EPOCH_MS = 80
	EPOCH_INCR    = 1 << 32
	TXID_MASK     = 0x00000000ffffffff
	CLEAR_TID     = 0xffffffff00000000
)

var PhaseLength = flag.Int("phase", 80, "Phase length in milliseconds, default 80")

type Coordinator struct {
	n        int
	Workers  []*Worker
	epochTID uint64 // Global TID, atomically incremented and read

	// Notify workers
	wepoch []chan TID
	wsafe  []chan TID
	wgo    []chan TID
	wdone  []chan TID

	Coordinate            bool
	PotentialPhaseChanges int64
	Done                  chan chan bool
	Accelerate            chan bool
	trigger               int32
	to_remove             map[Key]bool

	TotalCoordTime time.Duration
	GoTime         time.Duration
	ReadTime       time.Duration
	MergeTime      time.Duration
}

func NewCoordinator(n int, s *Store) *Coordinator {
	c := &Coordinator{
		n:                     n,
		Workers:               make([]*Worker, n),
		epochTID:              EPOCH_INCR,
		wepoch:                make([]chan TID, n),
		wsafe:                 make([]chan TID, n),
		wgo:                   make([]chan TID, n),
		wdone:                 make([]chan TID, n),
		Done:                  make(chan chan bool),
		Accelerate:            make(chan bool),
		Coordinate:            false,
		PotentialPhaseChanges: 0,
		to_remove:             make(map[Key]bool),
	}
	for i := 0; i < n; i++ {
		c.wepoch[i] = make(chan TID)
		c.wsafe[i] = make(chan TID)
		c.wgo[i] = make(chan TID)
		c.wdone[i] = make(chan TID)
		c.Workers[i] = NewWorker(i, s, c)
	}
	dlog.Printf("[coordinator] %v workers\n", n)
	go c.Process()
	return c
}

var NextEpoch int64

func (c *Coordinator) NextGlobalTID() TID {
	atomic.AddInt64(&NextEpoch, 1)
	x := atomic.AddUint64(&c.epochTID, EPOCH_INCR)
	return TID(x)
}

func (c *Coordinator) GetEpoch() TID {
	x := atomic.LoadUint64(&c.epochTID)
	return TID(x)
}

var RMoved int64
var WMoved int64
var Time_in_IE time.Duration
var Time_in_IE1 time.Duration

func (c *Coordinator) Stats() (map[Key]bool, map[Key]bool) {
	if c.PotentialPhaseChanges%(10) != 0 {
		return nil, nil
	}
	start2 := time.Now()
	s := c.Workers[0].store
	for i := 0; i < c.n; i++ {
		w := c.Workers[i]
		c.Workers[i].Lock()
		s.cand.Merge(w.local_store.candidates)
	}
	potential_dd_keys := make(map[Key]bool)
	to_remove := make(map[Key]bool)
	xx := len(*s.cand.h)
	for i := 0; i < xx; i++ {
		o := heap.Pop(s.cand.h).(*OneStat)
		br, _ := s.getKey(o.k)
		if !br.dd {
			if len(s.dd) == 0 {
				// Higher threshold for the first one, since it kicks off phases
				if o.ratio() > 1.33*(*WRRatio) && (o.writes > 1 || o.conflicts > 5) {
					potential_dd_keys[o.k] = true
					dlog.Printf("move %v to split1 r:%v w:%v c:%v s:%v ra:%v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
				} else {
					dlog.Printf("%v no move inertia r:%v w:%v c:%v s:%v ra:%v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
				}
				continue
			}
			if o.ratio() > *WRRatio && (o.writes > 1 || o.conflicts > 1) {
				potential_dd_keys[o.k] = true
				dlog.Printf("move %v to split2 r:%v w:%v c:%v s:%v ra:%v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
			} else {
				dlog.Printf("too low; no move :%v; r:%v w:%v c:%v s:%v ra:%v; wr: %v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio(), *WRRatio)
			}
		}
	}
	// Check to see if we need to remove anything from dd
	for k, v := range s.dd {
		if !v {
			continue
		}
		o, ok := s.cand.m[k]
		if !ok {
			dlog.Printf("Key %v was split but now is not in store candidates\n", k)
			continue
		}
		if o.ratio() < (*WRRatio)/2 {
			if x, ok := c.to_remove[k]; x && ok {
				c.to_remove[k] = false
				to_remove[k] = true
			} else {
				c.to_remove[k] = true
			}
			dlog.Printf("move %v from split r:%v w:%v c:%v s:%v ratio:%v\n", k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
		}
	}
	if len(s.dd) == 0 && len(potential_dd_keys) == 0 {
		s.any_dd = false
		c.Coordinate = false
	} else {
		c.Coordinate = true
		s.any_dd = true
	}
	// Reset global store
	x := make([]*OneStat, 0)
	sh := StatsHeap(x)
	s.cand = &Candidates{make(map[Key]*OneStat), &sh}

	for i := 0; i < c.n; i++ {
		// Reset local stores and unlock
		w := c.Workers[i]
		x := make([]*OneStat, 0)
		sh := StatsHeap(x)
		w.local_store.candidates = &Candidates{make(map[Key]*OneStat), &sh}
		w.Unlock()
	}
	end := time.Since(start2)
	Time_in_IE1 += end
	return potential_dd_keys, to_remove
}

func (c *Coordinator) IncrementEpoch(force bool) {
	start1 := time.Now()
	c.PotentialPhaseChanges++
	s := c.Workers[0].store
	var move_dd, remove_dd map[Key]bool
	if *AlwaysSplit {
		c.Coordinate = true
		s.any_dd = true
	} else {
		move_dd, remove_dd = c.Stats()
	}
	if !c.Coordinate && !force {
		c.TotalCoordTime += time.Since(start1)
		return
	}
	next_epoch := c.NextGlobalTID()

	// Wait for everyone to merge the previous epoch
	sx := time.Now()
	for i := 0; i < c.n; i++ {
		e := <-c.wepoch[i]
		if e != next_epoch {
			log.Fatalf("Out of alignment in epoch ack; I expected %v, got %v\n", next_epoch, e)
		}
	}
	c.MergeTime += time.Since(sx)

	// All merged.  The previous epoch is now safe; tell everyone to
	// do their reads.
	sx = time.Now()
	atomic.StoreInt32(&c.trigger, 0)
	for i := 0; i < c.n; i++ {
		c.wsafe[i] <- next_epoch
	}
	for i := 0; i < c.n; i++ {
		e := <-c.wdone[i]
		if e != next_epoch {
			log.Fatalf("Out of alignment in done; I expected %v, got %v\n", next_epoch, e)
		}

	}
	c.ReadTime += time.Since(sx)
	// Merge dd
	if !*AlwaysSplit {
		if move_dd != nil {
			for k, _ := range move_dd {
				br, _ := s.getKey(k)
				br.dd = true
				s.dd[k] = true
				WMoved += 1
			}
		}
		if remove_dd != nil {
			for k, _ := range remove_dd {
				br, _ := s.getKey(k)
				br.dd = false
				s.dd[k] = false
				RMoved += 1
			}
		}
	}

	sx = time.Now()
	for i := 0; i < c.n; i++ {
		c.wgo[i] <- next_epoch
	}
	c.GoTime += time.Since(sx)
	c.TotalCoordTime += time.Since(start1)
}

func (c *Coordinator) Finish() {
	dlog.Printf("Coordinator finishing\n")
	x := make(chan bool)
	c.Done <- x
	<-x
}

var Nfast int64

func (c *Coordinator) Process() {
	tm := time.NewTicker(time.Duration(*PhaseLength) * time.Millisecond).C

	// More frequently, check if the workers are demanding a phase
	// change due to long stashed queue lengths.
	check_trigger := time.NewTicker(time.Duration(*PhaseLength) * time.Microsecond * 10).C

	for {
		select {
		case x := <-c.Done:
			if *SysType == DOPPEL && c.n > 1 {
				c.IncrementEpoch(true)
			}
			for i := 0; i < c.n; i++ {
				c.Workers[i].done <- true
			}
			x <- true
			return
		case <-tm:
			if *SysType == DOPPEL && c.n > 1 {
				c.IncrementEpoch(false)
			}
		case <-check_trigger:
			if *SysType == DOPPEL && c.n > 1 {
				x := atomic.LoadInt32(&c.trigger)
				if x == int32(c.n) {
					Nfast++
					atomic.StoreInt32(&c.trigger, 0)
					c.IncrementEpoch(true)
				}
			}
		case <-c.Accelerate:
			if *SysType == DOPPEL && c.n > 1 {
				dlog.Printf("Accelerating\n")
				c.IncrementEpoch(true)
			}
		}
	}
}

func compute(w *Worker, txn int) (int64, int64) {
	var total int64
	var sum int64
	var i int64
	for i = 0; i < TIMES; i++ {
		total = total + w.times[txn][i]
		sum = sum + (w.times[txn][i] * i)
	}
	var x99 int64 = int64(float64(total) * .99)
	var y99 int64
	var v99 int64
	for i = 0; i < TIMES; i++ {
		y99 = y99 + w.times[txn][i]
		if y99 >= x99 {
			v99 = i
			break
		}
	}
	dlog.Printf("%v avg: %v us; 99: %v us, x99: %v, sum: %v, total: %v \n", txn, sum/total, v99, x99, sum, total)
	return sum / total, v99
}

func (c *Coordinator) Latency() (string, string) {
	if !*Latency {
		return "", ""
	}
	for i := 1; i < c.n; i++ {
		for j := 0; j < 4; j++ {
			for k := 0; k < TIMES; k++ {
				c.Workers[0].times[j][k] = c.Workers[0].times[j][k] + c.Workers[i].times[j][k]
			}
		}
	}
	x, y := compute(c.Workers[0], D_BUY)
	x2, y2 := compute(c.Workers[0], D_READ_TWO)
	return fmt.Sprintf("Read 99: %v\nRead Avg: %v\n", y2, x2), fmt.Sprintf("Write 99: %v\nWrite Avg: %v\n", y, x)

}
<commit_msg>easier to read<commit_after>package ddtxn

import (
	"container/heap"
	"ddtxn/dlog"
	"flag"
	"fmt"
	"log"
	"sync/atomic"
	"time"
)

const (
	BUMP_EPOCH_MS = 80
	EPOCH_INCR    = 1 << 32
	TXID_MASK     = 0x00000000ffffffff
	CLEAR_TID     = 0xffffffff00000000
)

var PhaseLength = flag.Int("phase", 80, "Phase length in milliseconds, default 80")

type Coordinator struct {
	n        int
	Workers  []*Worker
	epochTID uint64 // Global TID, atomically incremented and read

	// Notify workers
	wepoch []chan TID
	wsafe  []chan TID
	wgo    []chan TID
	wdone  []chan TID

	Coordinate            bool
	PotentialPhaseChanges int64
	Done                  chan chan bool
	Accelerate            chan bool
	trigger               int32
	to_remove             map[Key]bool

	TotalCoordTime time.Duration
	GoTime         time.Duration
	ReadTime       time.Duration
	MergeTime      time.Duration
}

func NewCoordinator(n int, s *Store) *Coordinator {
	c := &Coordinator{
		n:                     n,
		Workers:               make([]*Worker, n),
		epochTID:              EPOCH_INCR,
		wepoch:                make([]chan TID, n),
		wsafe:                 make([]chan TID, n),
		wgo:                   make([]chan TID, n),
		wdone:                 make([]chan TID, n),
		Done:                  make(chan chan bool),
		Accelerate:            make(chan bool),
		Coordinate:            false,
		PotentialPhaseChanges: 0,
		to_remove:             make(map[Key]bool),
	}
	for i := 0; i < n; i++ {
		c.wepoch[i] = make(chan TID)
		c.wsafe[i] = make(chan TID)
		c.wgo[i] = make(chan TID)
		c.wdone[i] = make(chan TID)
		c.Workers[i] = NewWorker(i, s, c)
	}
	dlog.Printf("[coordinator] %v workers\n", n)
	go c.Process()
	return c
}

var NextEpoch int64

func (c *Coordinator) NextGlobalTID() TID {
	atomic.AddInt64(&NextEpoch, 1)
	x := atomic.AddUint64(&c.epochTID, EPOCH_INCR)
	return TID(x)
}

func (c *Coordinator) GetEpoch() TID {
	x := atomic.LoadUint64(&c.epochTID)
	return TID(x)
}

var RMoved int64
var WMoved int64
var Time_in_IE time.Duration
var Time_in_IE1 time.Duration

func (c *Coordinator) Stats() (map[Key]bool, map[Key]bool) {
	if c.PotentialPhaseChanges%(10) != 0 {
		return nil, nil
	}
	start2 := time.Now()
	s := c.Workers[0].store
	for i := 0; i < c.n; i++ {
		w := c.Workers[i]
		c.Workers[i].Lock()
		s.cand.Merge(w.local_store.candidates)
	}
	potential_dd_keys := make(map[Key]bool)
	to_remove := make(map[Key]bool)
	xx := len(*s.cand.h)
	for i := 0; i < xx; i++ {
		o := heap.Pop(s.cand.h).(*OneStat)
		br, _ := s.getKey(o.k)
		if !br.dd {
			if len(s.dd) == 0 {
				// Higher threshold for the first one, since it kicks off phases
				if o.ratio() > 1.33*(*WRRatio) && (o.writes > 1 || o.conflicts > 5) {
					potential_dd_keys[o.k] = true
					dlog.Printf("move %v to split1 r:%v w:%v c:%v s:%v ra:%v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
				} else {
					dlog.Printf("%v no move inertia r:%v w:%v c:%v s:%v ra:%v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
				}
				continue
			}
			if o.ratio() > *WRRatio && (o.writes > 1 || o.conflicts > 1) {
				potential_dd_keys[o.k] = true
				dlog.Printf("move %v to split2 r:%v w:%v c:%v s:%v ra:%v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
			} else {
				dlog.Printf("too low; no move :%v; r:%v w:%v c:%v s:%v ra:%v; wr: %v\n", o.k, o.reads, o.writes, o.conflicts, o.stash, o.ratio(), *WRRatio)
			}
		}
	}
	// Check to see if we need to remove anything from dd
	for k, v := range s.dd {
		if !v {
			continue
		}
		o, ok := s.cand.m[k]
		if !ok {
			dlog.Printf("Key %v was split but now is not in store candidates\n", k)
			continue
		}
		if o.ratio() < (*WRRatio)/2 {
			if x, ok := c.to_remove[k]; x && ok {
				c.to_remove[k] = false
				to_remove[k] = true
			} else {
				c.to_remove[k] = true
			}
			dlog.Printf("move %v from split r:%v w:%v c:%v s:%v ratio:%v\n", k, o.reads, o.writes, o.conflicts, o.stash, o.ratio())
		}
	}
	if len(s.dd) == 0 && len(potential_dd_keys) == 0 {
		s.any_dd = false
		c.Coordinate = false
	} else {
		c.Coordinate = true
		s.any_dd = true
	}
	// Reset global store
	x := make([]*OneStat, 0)
	sh := StatsHeap(x)
	s.cand = &Candidates{make(map[Key]*OneStat), &sh}

	for i := 0; i < c.n; i++ {
		// Reset local stores and unlock
		w := c.Workers[i]
		x := make([]*OneStat, 0)
		sh := StatsHeap(x)
		w.local_store.candidates = &Candidates{make(map[Key]*OneStat), &sh}
		w.Unlock()
	}
	end := time.Since(start2)
	Time_in_IE1 += end
	return potential_dd_keys, to_remove
}

func (c *Coordinator) IncrementEpoch(force bool) {
	start1 := time.Now()
	c.PotentialPhaseChanges++
	s := c.Workers[0].store
	var move_dd, remove_dd map[Key]bool
	if *AlwaysSplit {
		c.Coordinate = true
		s.any_dd = true
	} else {
		move_dd, remove_dd = c.Stats()
	}
	if !c.Coordinate && !force {
		c.TotalCoordTime += time.Since(start1)
		return
	}
	next_epoch := c.NextGlobalTID()

	// Wait for everyone to merge the previous epoch
	sx := time.Now()
	for i := 0; i < c.n; i++ {
		e := <-c.wepoch[i]
		if e != next_epoch {
			log.Fatalf("Out of alignment in epoch ack; I expected %v, got %v\n", next_epoch, e)
		}
	}
	c.MergeTime += time.Since(sx)

	// All merged.  The previous epoch is now safe; tell everyone to
	// do their reads.
	sx = time.Now()
	atomic.StoreInt32(&c.trigger, 0)
	for i := 0; i < c.n; i++ {
		c.wsafe[i] <- next_epoch
	}
	for i := 0; i < c.n; i++ {
		e := <-c.wdone[i]
		if e != next_epoch {
			log.Fatalf("Out of alignment in done; I expected %v, got %v\n", next_epoch, e)
		}

	}
	c.ReadTime += time.Since(sx)
	// Merge dd
	if !*AlwaysSplit {
		if move_dd != nil {
			for k, _ := range move_dd {
				br, _ := s.getKey(k)
				br.dd = true
				s.dd[k] = true
				WMoved += 1
			}
		}
		if remove_dd != nil {
			for k, _ := range remove_dd {
				br, _ := s.getKey(k)
				br.dd = false
				s.dd[k] = false
				RMoved += 1
			}
		}
	}

	sx = time.Now()
	for i := 0; i < c.n; i++ {
		c.wgo[i] <- next_epoch
	}
	c.GoTime += time.Since(sx)
	c.TotalCoordTime += time.Since(start1)
}

func (c *Coordinator) Finish() {
	dlog.Printf("Coordinator finishing\n")
	x := make(chan bool)
	c.Done <- x
	<-x
}

var Nfast int64

func (c *Coordinator) Process() {
	tm := time.NewTicker(time.Duration(*PhaseLength) * time.Millisecond).C

	// More frequently, check if the workers are demanding a phase
	// change due to long stashed queue lengths.
	check_trigger := time.NewTicker(time.Duration(*PhaseLength) * time.Microsecond * 10).C

	for {
		select {
		case x := <-c.Done:
			if *SysType == DOPPEL && c.n > 1 {
				c.IncrementEpoch(true)
			}
			for i := 0; i < c.n; i++ {
				c.Workers[i].done <- true
			}
			x <- true
			return
		case <-tm:
			if *SysType == DOPPEL && c.n > 1 {
				c.IncrementEpoch(false)
			}
		case <-check_trigger:
			if *SysType == DOPPEL && c.n > 1 {
				x := atomic.LoadInt32(&c.trigger)
				if x == int32(c.n) {
					Nfast++
					atomic.StoreInt32(&c.trigger, 0)
					c.IncrementEpoch(true)
				}
			}
		case <-c.Accelerate:
			if *SysType == DOPPEL && c.n > 1 {
				dlog.Printf("Accelerating\n")
				c.IncrementEpoch(true)
			}
		}
	}
}

func compute(w *Worker, txn int) (int64, int64) {
	var total int64
	var sum int64
	var i int64
	for i = 0; i < TIMES; i++ {
		total = total + w.times[txn][i]
		sum = sum + (w.times[txn][i] * i)
	}
	var x99 int64 = int64(float64(total) * .99)
	var y99 int64
	var v99 int64
	for i = 0; i < TIMES; i++ {
		y99 = y99 + w.times[txn][i]
		if y99 >= x99 {
			v99 = i
			break
		}
	}
	dlog.Printf("%v avg: %v us; 99: %v us, x99: %v, sum: %v, total: %v \n", txn, sum/total, v99, x99, sum, total)
	return sum / total, v99
}

func (c *Coordinator) Latency() (string, string) {
	if !*Latency {
		return "", ""
	}
	for i := 1; i < c.n; i++ {
		for j := 0; j < 4; j++ {
			for k := 0; k < TIMES; k++ {
				c.Workers[0].times[j][k] = c.Workers[0].times[j][k] + c.Workers[i].times[j][k]
			}
		}
	}
	x, y := compute(c.Workers[0], D_BUY)
	x2, y2 := compute(c.Workers[0], D_READ_TWO)
	return fmt.Sprintf("Read/Write Avg: %v/%v\n", x2, x), fmt.Sprintf("Read/Write 99: %v/%v\n", y2, y)

}
<|endoftext|>