<commit_before>package gotel

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"
)

type alerter interface {
	Alert(res reservation) bool
	Name() string
	Bootstrap()
}

var (
	// holds info so we don't spam alerters every n seconds
	sentAlerts = make(map[string]time.Time)
	// this designates the instance as the coordinating instanceS
	coordinator = false
	// stores a slice of alerter functions to call when we have an alert
	alertFuncs = []alerter{}
	cfg        config
	// our current IP address
	myIP string
)

func init() {
	var err error
	myIP, err = externalIP()
	if err != nil {
		myIP = "N/A"
		l.err("Unable to aquire own IP Address")
	}
}

// Monitor checks existing reservations for late arrivals
func Monitor(db *sql.DB) {
	if !coordinator {
		coordinator = isCoordinator(db)
	}
	printCoordinatorStatus()
	jobChecker(db)
}

// InitializeMonitoring sets up alerters based on configuration
func InitializeMonitoring(c config, db *sql.DB) {
	cfg = c
	if cfg.Smtp.Enabled {
		smtp := new(smtpAlerter)
		smtp.Cfg = c
		alertFuncs = append(alertFuncs, smtp)
	} else {
		l.info("SMTP Alerting disabled")
	}
	if cfg.Pagerduty.Enabled {
		pd := new(pagerDutyAlerter)
		pd.Cfg = c
		alertFuncs = append(alertFuncs, pd)
	} else {
		l.info("PagerDuty Alerting disabled")
	}

	for _, alerter := range alertFuncs {
		alerter.Bootstrap()
	}

	// set up a ticker that runs every day that checks to clean up old logs to preserve disk space

	ticker := time.NewTicker(24 * time.Hour)
	go func() {
		for t := range ticker.C {
			if coordinator {
				l.info("Running log cleanup at [%v]", t)
				cleanUp(db, c.Main.DaysToStoreLogs)
			}
		}
	}()

}

//--------------------- PRIVATE FUNCS ------------------------------

func hasLock(db *sql.DB) bool {
	var lck int
	query := "SELECT GET_LOCK('gotel_lock', 3) as lck"
	rows, err := db.Query(query)
	if err != nil {
		l.warn("Unable to aquire lock\n")
	}
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&lck)
		if err != nil {
			l.warn("Unable to aquire locking rows\n")
		}
		if lck == 1 {
			// holds a lock while the connection is alive
			l.info("Lock Aquired")
			return true
		}
		l.info("Unable to aquire coordinator lock. I must be a worker [%v]", lck)

	}
	return false
}

func releaseLock(db *sql.DB) (bool, error) {
	releaseQuery := "SELECT RELEASE_LOCK('gotel_lock');"
	rows, err := db.Query(releaseQuery)
	if err != nil {
		l.warn("Unable to release lock\n")
		return false, errors.New("Unable to release lock")
	}
	defer rows.Close()
	return true, nil

}

// attempt to aquire coordinator lock to indicate this node should do the job checking
// in the future I'd like to have a zookeeper integration for a more "true" leader election scheme
// this is somewhat of a quickstart method so people don't have to also have ZK in their env
// split brain would be detected by the coordinator not checking in so we'd be firing off an alert
func isCoordinator(db *sql.DB) bool {

	coordinatorNodeCnt := 0

	lockAquired := hasLock(db)
	if lockAquired {
		var (
			ipAddress string
			nodeID    int64
		)
		rows, err := db.Query("select ip_address, node_id from nodes")
		if err != nil {
			l.err("Unable to select nodes [%v]", err)
			return false
		}
		defer rows.Close()

		for rows.Next() {
			err := rows.Scan(&ipAddress, &nodeID)
			if err != nil {
				l.err("Unable to scan node rows [%v]", err)
				return false
			}
			if ipAddress == myIP {
				continue
			}
			// check to see if we have any other coordinator nodes, or am i it?
			l.info("Checking ip [%s] for coordinator status", ipAddress)
			resp, err := http.Get(fmt.Sprintf("http://%s:8080/is-coordinator", ipAddress))
			if err != nil {
				l.warn("Unable to contact node [%s] assuming offline", ipAddress)
				removeNode(db, ipAddress)
				continue
			}
			defer resp.Body.Close()

			if resp.StatusCode != 200 {
				l.warn("Didn't get a 200OK reply back from ip [%s]", ipAddress)
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				l.warn("Unable to read node response")
			}
			if string(body) == "true" {
				l.info("IP [%s] is reporting as a coordinator", ipAddress)
				coordinatorNodeCnt++
			}
			l.info("ip [%s] http coordinator check returned [%s]", ipAddress, body)

		}
		insertSelf(db)
		releaseLock(db)
		if coordinatorNodeCnt == 0 {
			// I'm the coordinator!
			return true
		}
	}
	return false
}

func removeNode(db *sql.DB, ipAddress string) {
	external, err := externalIP()
	if external == ipAddress {
		return
	}
	if err != nil {
		l.warn("Unable to delete offline node [%v]", err)
	}
	stmt, err := db.Prepare("DELETE FROM nodes WHERE ip_address=?")
	if err != nil {
		l.warn("Unable to prepare record %s", err)
	}
	defer stmt.Close()
	_, err = stmt.Exec(ipAddress)
	if err != nil {
		l.warn("Unable to run delete operation for remove node [%v]", err)
	}
	l.info("Node [%s] was removed from DB", ipAddress)
}

func insertSelf(db *sql.DB) {
	ip, err := externalIP()
	if err != nil {
		l.err("Unable to get external IP [%v]", err)
	}
	rand.Seed(time.Now().UnixNano())
	seedID := rand.Intn(10000)

	stmt, err := db.Prepare("insert into nodes(ip_address, node_id) values(?, ?)")
	if err != nil {
		l.err("Unable to prepare insertself record %s", err)
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(ip, seedID)
	if err != nil {
		if strings.Contains(err.Error(), "Duplicate entry") {
			l.info("[%s] has already registered as a node", ip)
		} else {
			l.warn("Unable to insert insertself record %s", err)
		}
		return
	}
}

func printCoordinatorStatus() {
	if coordinator {
		l.info("I am the coordinator node!\n")
	} else {
		l.info("I am the worker node!\n")
	}
}

// checks jobs and sends to workers to check on last update time
// we're not on the master we want to monitor the master to make sure it's running it's job checker
// mode will be master if the main jobs should run on this node
func jobChecker(db *sql.DB) {

	var query string
	if coordinator {
		query = "select id, app, component, owner, notify, frequency, time_units, last_checkin_timestamp from reservations"
	} else {
		// if we're a worker we just want to monitor the co-ordinator
		query = "select id, app, component, owner, notify, frequency, time_units, last_checkin_timestamp from reservations where app='gotel' and component='coordinator'"
	}
	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil {
		l.err("Unable to run job checker [%v]", err)
		return
	}

	for rows.Next() {
		res := reservation{}
		rows.Scan(&res.JobID, &res.App, &res.Component, &res.Owner, &res.Notify, &res.Frequency, &res.TimeUnits, &res.LastCheckin)

		if FailsSLA(res) {
			alerterNames := []string{}
			for _, alerter := range alertFuncs {
				alerterNames = append(alerterNames, alerter.Name())

				if !alreadySentRecently(res, alerter.Name()) {
					if alerter.Alert(res) {
						updateSentRecently(res, alerter.Name())
					}
				} else {
					l.info("Already sent alert for [%s/%s/%s]", res.App, res.Component, alerter.Name())
				}
			}
			storeAlert(res, db, alerterNames)
		}
	}
	storeJobRun(db)
}

func (r *reservation) mapKey(alerterName string) string {
	return r.App + r.Component + alerterName
}

// check to see if we've already sent this alert recently
func alreadySentRecently(res reservation, alerterName string) bool {
	timeNow := time.Now().UTC()
	waitForNotifyTime := time.Duration(cfg.Main.HoursBetweenAlerts) * time.Hour

	lastSent, ok := sentAlerts[res.mapKey(alerterName)]
	if !ok {
		return false
	}

	// check to see if the time elapsed goes over our threshold
	duration := timeNow.Sub(lastSent)
	if duration >= waitForNotifyTime {
		return false
	}

	return true
}

func updateSentRecently(res reservation, alerterName string) {
	sentAlerts[res.mapKey(alerterName)] = time.Now()
}

// FailsSLA monitors the reservations and determines if any jobs haven't checked in within
// their allotted timeframe
func FailsSLA(res reservation) bool {
	l.info("Working on app [%s] component [%s]", res.App, res.Component)

	timeNow := time.Now().UTC()
	startTime := time.Unix(res.LastCheckin, 0)
	secondsAgo := int(timeNow.Sub(startTime).Seconds())
	if secondsAgo > getSecondsFromUnits(res.Frequency, res.TimeUnits) {
		// send job to alert on
		l.info("App Failed SLA [%s/%s] that is: %d seconds old\n", res.App, res.Component, secondsAgo)
		return true
	}
	return false
}

func storeAlert(res reservation, db *sql.DB, alerters []string) {
	now := time.Now().UTC().Unix()
	altertNames := strings.Join(alerters, ",")
	stmt, err := db.Prepare("insert into alerts(app, component, alert_time, alerters) values(?, ?, ?, ?)")
	if err != nil {
		l.info("[ERROR] Unable to prepare storealert record %s", err)
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(res.App, res.Component, now, altertNames)
	if err != nil {
		l.info("[ERROR] Unable to insert alert record %s", err)
		return
	}
}

func storeJobRun(db *sql.DB) {
	mode := "worker"
	if coordinator {
		mode = "coordinator"
	}
	now := time.Now().UTC().Unix()
	l.info("Storing job run, mode: [%s]\n", mode)
	storeCheckin(db, checkin{
		App:       "gotel",
		Component: mode,
	}, now)
}

// Cleanup should run on a scheduled ticker to allow GoTel to clean up after itself to prevent disk space issues in the
// DB as the process is meant to run for years.
func cleanUp(db *sql.DB, daysToStoreLogs int) {

	// grab the unix time that was daysToStoreLogs ago, cleanup anything older than that to keep db size down
	timeNow := time.Now().UTC().AddDate(0, 0, -daysToStoreLogs).Unix()

	// clean up housekeeping
	stmt, err := db.Prepare("DELETE FROM housekeeping WHERE last_checkin_timestamp < ?")
	if err != nil {
		l.err("Unable to prepare cleaup housekeeping statement")
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(timeNow)
	if err != nil {
		l.err("Unable cleanup old housekeeping logs, this could be bad [%v]", err)
		return
	}

	// clean up alerts
	stmt, err = db.Prepare("DELETE FROM alerts WHERE alert_time < ?")
	if err != nil {
		l.err("Unable to prepare cleaup alerts statement")
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(timeNow)
	if err != nil {
		l.err("Unable cleanup old alerts logs, this could be bad [%v]", err)
		return
	}

}
<commit_msg>only store alert if successfully sent<commit_after>package gotel

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"
)

type alerter interface {
	Alert(res reservation) bool
	Name() string
	Bootstrap()
}

var (
	// holds info so we don't spam alerters every n seconds
	sentAlerts = make(map[string]time.Time)
	// this designates the instance as the coordinating instanceS
	coordinator = false
	// stores a slice of alerter functions to call when we have an alert
	alertFuncs = []alerter{}
	cfg        config
	// our current IP address
	myIP string
)

func init() {
	var err error
	myIP, err = externalIP()
	if err != nil {
		myIP = "N/A"
		l.err("Unable to aquire own IP Address")
	}
}

// Monitor checks existing reservations for late arrivals
func Monitor(db *sql.DB) {
	if !coordinator {
		coordinator = isCoordinator(db)
	}
	printCoordinatorStatus()
	jobChecker(db)
}

// InitializeMonitoring sets up alerters based on configuration
func InitializeMonitoring(c config, db *sql.DB) {
	cfg = c
	if cfg.Smtp.Enabled {
		smtp := new(smtpAlerter)
		smtp.Cfg = c
		alertFuncs = append(alertFuncs, smtp)
	} else {
		l.info("SMTP Alerting disabled")
	}
	if cfg.Pagerduty.Enabled {
		pd := new(pagerDutyAlerter)
		pd.Cfg = c
		alertFuncs = append(alertFuncs, pd)
	} else {
		l.info("PagerDuty Alerting disabled")
	}

	for _, alerter := range alertFuncs {
		alerter.Bootstrap()
	}

	// set up a ticker that runs every day that checks to clean up old logs to preserve disk space

	ticker := time.NewTicker(24 * time.Hour)
	go func() {
		for t := range ticker.C {
			if coordinator {
				l.info("Running log cleanup at [%v]", t)
				cleanUp(db, c.Main.DaysToStoreLogs)
			}
		}
	}()

}

//--------------------- PRIVATE FUNCS ------------------------------

func hasLock(db *sql.DB) bool {
	var lck int
	query := "SELECT GET_LOCK('gotel_lock', 3) as lck"
	rows, err := db.Query(query)
	if err != nil {
		l.warn("Unable to aquire lock\n")
	}
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&lck)
		if err != nil {
			l.warn("Unable to aquire locking rows\n")
		}
		if lck == 1 {
			// holds a lock while the connection is alive
			l.info("Lock Aquired")
			return true
		}
		l.info("Unable to aquire coordinator lock. I must be a worker [%v]", lck)

	}
	return false
}

func releaseLock(db *sql.DB) (bool, error) {
	releaseQuery := "SELECT RELEASE_LOCK('gotel_lock');"
	rows, err := db.Query(releaseQuery)
	if err != nil {
		l.warn("Unable to release lock\n")
		return false, errors.New("Unable to release lock")
	}
	defer rows.Close()
	return true, nil

}

// attempt to aquire coordinator lock to indicate this node should do the job checking
// in the future I'd like to have a zookeeper integration for a more "true" leader election scheme
// this is somewhat of a quickstart method so people don't have to also have ZK in their env
// split brain would be detected by the coordinator not checking in so we'd be firing off an alert
func isCoordinator(db *sql.DB) bool {

	coordinatorNodeCnt := 0

	lockAquired := hasLock(db)
	if lockAquired {
		var (
			ipAddress string
			nodeID    int64
		)
		rows, err := db.Query("select ip_address, node_id from nodes")
		if err != nil {
			l.err("Unable to select nodes [%v]", err)
			return false
		}
		defer rows.Close()

		for rows.Next() {
			err := rows.Scan(&ipAddress, &nodeID)
			if err != nil {
				l.err("Unable to scan node rows [%v]", err)
				return false
			}
			if ipAddress == myIP {
				continue
			}
			// check to see if we have any other coordinator nodes, or am i it?
			l.info("Checking ip [%s] for coordinator status", ipAddress)
			resp, err := http.Get(fmt.Sprintf("http://%s:8080/is-coordinator", ipAddress))
			if err != nil {
				l.warn("Unable to contact node [%s] assuming offline", ipAddress)
				removeNode(db, ipAddress)
				continue
			}
			defer resp.Body.Close()

			if resp.StatusCode != 200 {
				l.warn("Didn't get a 200OK reply back from ip [%s]", ipAddress)
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				l.warn("Unable to read node response")
			}
			if string(body) == "true" {
				l.info("IP [%s] is reporting as a coordinator", ipAddress)
				coordinatorNodeCnt++
			}
			l.info("ip [%s] http coordinator check returned [%s]", ipAddress, body)

		}
		insertSelf(db)
		releaseLock(db)
		if coordinatorNodeCnt == 0 {
			// I'm the coordinator!
			return true
		}
	}
	return false
}

func removeNode(db *sql.DB, ipAddress string) {
	external, err := externalIP()
	if external == ipAddress {
		return
	}
	if err != nil {
		l.warn("Unable to delete offline node [%v]", err)
	}
	stmt, err := db.Prepare("DELETE FROM nodes WHERE ip_address=?")
	if err != nil {
		l.warn("Unable to prepare record %s", err)
	}
	defer stmt.Close()
	_, err = stmt.Exec(ipAddress)
	if err != nil {
		l.warn("Unable to run delete operation for remove node [%v]", err)
	}
	l.info("Node [%s] was removed from DB", ipAddress)
}

func insertSelf(db *sql.DB) {
	ip, err := externalIP()
	if err != nil {
		l.err("Unable to get external IP [%v]", err)
	}
	rand.Seed(time.Now().UnixNano())
	seedID := rand.Intn(10000)

	stmt, err := db.Prepare("insert into nodes(ip_address, node_id) values(?, ?)")
	if err != nil {
		l.err("Unable to prepare insertself record %s", err)
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(ip, seedID)
	if err != nil {
		if strings.Contains(err.Error(), "Duplicate entry") {
			l.info("[%s] has already registered as a node", ip)
		} else {
			l.warn("Unable to insert insertself record %s", err)
		}
		return
	}
}

func printCoordinatorStatus() {
	if coordinator {
		l.info("I am the coordinator node!\n")
	} else {
		l.info("I am the worker node!\n")
	}
}

// checks jobs and sends to workers to check on last update time
// we're not on the master we want to monitor the master to make sure it's running it's job checker
// mode will be master if the main jobs should run on this node
func jobChecker(db *sql.DB) {

	var query string
	if coordinator {
		query = "select id, app, component, owner, notify, frequency, time_units, last_checkin_timestamp from reservations"
	} else {
		// if we're a worker we just want to monitor the co-ordinator
		query = "select id, app, component, owner, notify, frequency, time_units, last_checkin_timestamp from reservations where app='gotel' and component='coordinator'"
	}
	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil {
		l.err("Unable to run job checker [%v]", err)
		return
	}

	for rows.Next() {
		res := reservation{}
		rows.Scan(&res.JobID, &res.App, &res.Component, &res.Owner, &res.Notify, &res.Frequency, &res.TimeUnits, &res.LastCheckin)

		if FailsSLA(res) {
			alerterNames := []string{}
			for _, alerter := range alertFuncs {
				alerterNames = append(alerterNames, alerter.Name())

				if !alreadySentRecently(res, alerter.Name()) {
					if alerter.Alert(res) {
						updateSentRecently(res, alerter.Name())
						storeAlert(res, db, alerterNames)
					}
				} else {
					l.info("Already sent alert for [%s/%s/%s]", res.App, res.Component, alerter.Name())
				}
			}
		}
	}
	storeJobRun(db)
}

func (r *reservation) mapKey(alerterName string) string {
	return r.App + r.Component + alerterName
}

// check to see if we've already sent this alert recently
func alreadySentRecently(res reservation, alerterName string) bool {
	timeNow := time.Now().UTC()
	waitForNotifyTime := time.Duration(cfg.Main.HoursBetweenAlerts) * time.Hour

	lastSent, ok := sentAlerts[res.mapKey(alerterName)]
	if !ok {
		return false
	}

	// check to see if the time elapsed goes over our threshold
	duration := timeNow.Sub(lastSent)
	if duration >= waitForNotifyTime {
		return false
	}

	return true
}

func updateSentRecently(res reservation, alerterName string) {
	sentAlerts[res.mapKey(alerterName)] = time.Now()
}

// FailsSLA monitors the reservations and determines if any jobs haven't checked in within
// their allotted timeframe
func FailsSLA(res reservation) bool {
	l.info("Working on app [%s] component [%s]", res.App, res.Component)

	timeNow := time.Now().UTC()
	startTime := time.Unix(res.LastCheckin, 0)
	secondsAgo := int(timeNow.Sub(startTime).Seconds())
	if secondsAgo > getSecondsFromUnits(res.Frequency, res.TimeUnits) {
		// send job to alert on
		l.info("App Failed SLA [%s/%s] that is: %d seconds old\n", res.App, res.Component, secondsAgo)
		return true
	}
	return false
}

func storeAlert(res reservation, db *sql.DB, alerters []string) {
	now := time.Now().UTC().Unix()
	altertNames := strings.Join(alerters, ",")
	stmt, err := db.Prepare("insert into alerts(app, component, alert_time, alerters) values(?, ?, ?, ?)")
	if err != nil {
		l.info("[ERROR] Unable to prepare storealert record %s", err)
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(res.App, res.Component, now, altertNames)
	if err != nil {
		l.info("[ERROR] Unable to insert alert record %s", err)
		return
	}
}

func storeJobRun(db *sql.DB) {
	mode := "worker"
	if coordinator {
		mode = "coordinator"
	}
	now := time.Now().UTC().Unix()
	l.info("Storing job run, mode: [%s]\n", mode)
	storeCheckin(db, checkin{
		App:       "gotel",
		Component: mode,
	}, now)
}

// Cleanup should run on a scheduled ticker to allow GoTel to clean up after itself to prevent disk space issues in the
// DB as the process is meant to run for years.
func cleanUp(db *sql.DB, daysToStoreLogs int) {

	// grab the unix time that was daysToStoreLogs ago, cleanup anything older than that to keep db size down
	timeNow := time.Now().UTC().AddDate(0, 0, -daysToStoreLogs).Unix()

	// clean up housekeeping
	stmt, err := db.Prepare("DELETE FROM housekeeping WHERE last_checkin_timestamp < ?")
	if err != nil {
		l.err("Unable to prepare cleaup housekeeping statement")
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(timeNow)
	if err != nil {
		l.err("Unable cleanup old housekeeping logs, this could be bad [%v]", err)
		return
	}

	// clean up alerts
	stmt, err = db.Prepare("DELETE FROM alerts WHERE alert_time < ?")
	if err != nil {
		l.err("Unable to prepare cleaup alerts statement")
		return
	}
	defer stmt.Close()
	_, err = stmt.Exec(timeNow)
	if err != nil {
		l.err("Unable cleanup old alerts logs, this could be bad [%v]", err)
		return
	}

}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"go/ast"
	"golang.org/x/tools/go/ast/astutil"
	"golang.org/x/tools/go/types"

	"github.com/motemen/go-quickfix"
)

// doQuickFix tries to fix the source AST so that it compiles well.
func (s *Session) doQuickFix() error {
	const maxAttempts = 10

	s.reset()

quickFixAttempt:
	for i := 0; i < maxAttempts; i++ {
		s.TypeInfo = types.Info{
			Types: make(map[ast.Expr]types.TypeAndValue),
		}

		files := s.ExtraFiles
		files = append(files, s.File)

		config := quickfix.Config{
			Fset:     s.Fset,
			Files:    files,
			TypeInfo: &s.TypeInfo,
		}
		_, err := config.QuickFixOnce()
		if err == nil {
			break
		}

		debugf("quickFix :: err = %#v", err)

		errList, ok := err.(quickfix.ErrorList)
		if !ok {
			continue
		}

		// (try to) fix gore-specific remaining errors
		for _, err := range errList {
			err, ok := err.(types.Error)
			if !ok {
				continue
			}

			// "... used as value":
			//
			// convert
			//   __gore_pp(funcWithSideEffectReturningNoValue())
			// to
			//   funcWithSideEffectReturningNoValue()
			if strings.HasSuffix(err.Msg, " used as value") {
				nodepath, _ := astutil.PathEnclosingInterval(s.File, err.Pos, err.Pos)

				for _, node := range nodepath {
					stmt, ok := node.(ast.Stmt)
					if !ok {
						continue
					}

					for i := range s.mainBody.List {
						if s.mainBody.List[i] != stmt {
							continue
						}

						exprs := printedExprs(stmt)

						stmts := s.mainBody.List[0:i]
						for _, expr := range exprs {
							stmts = append(stmts, &ast.ExprStmt{expr})
						}

						s.mainBody.List = append(stmts, s.mainBody.List[i+1:]...)
						continue quickFixAttempt
					}
				}
			}
		}

		debugf("quickFix :: give up: %#v", err)
	}

	return nil
}

func (s *Session) clearQuickFix() {
	// make all import specs explicit (i.e. no "_").
	for _, imp := range s.File.Imports {
		imp.Name = nil
	}

	for i := 0; i < len(s.mainBody.List); {
		stmt := s.mainBody.List[i]

		// remove "_ = x" stmt
		if assign, ok := stmt.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 {
			if isNamedIdent(assign.Lhs[0], "_") {
				s.mainBody.List = append(s.mainBody.List[0:i], s.mainBody.List[i+1:]...)
				continue
			}
		}

		// remove expressions just for printing out
		// i.e. what causes "evaluated but not used."
		if exprs := printedExprs(stmt); exprs != nil {
			allPure := true
			for _, expr := range exprs {
				if !s.isPureExpr(expr) {
					allPure = false
					break
				}
			}

			if allPure {
				s.mainBody.List = append(s.mainBody.List[0:i], s.mainBody.List[i+1:]...)
				continue
			}

			// strip (possibly impure) printing expression to expression
			var trailing []ast.Stmt
			s.mainBody.List, trailing = s.mainBody.List[0:i], s.mainBody.List[i+1:]
			for _, expr := range exprs {
				if !isNamedIdent(expr, "_") {
					s.mainBody.List = append(s.mainBody.List, &ast.ExprStmt{X: expr})
				}
			}

			s.mainBody.List = append(s.mainBody.List, trailing...)
			continue
		}

		i++
	}

	debugf("clearQuickFix :: %s", showNode(s.Fset, s.mainBody))
}

// printedExprs returns arguments of statement stmt of form "p(x...)"
func printedExprs(stmt ast.Stmt) []ast.Expr {
	st, ok := stmt.(*ast.ExprStmt)
	if !ok {
		return nil
	}

	// first check whether the expr is p(_) form
	call, ok := st.X.(*ast.CallExpr)
	if !ok {
		return nil
	}

	if !isNamedIdent(call.Fun, printerName) {
		return nil
	}

	return call.Args
}

// isPureExpr checks if an expression expr is "pure", which means
// removing this expression will no affect the entire program.
// - identifiers ("x")
// - selectors ("x.y")
// - slices ("a[n:m]")
// - literals ("1")
// - type conversion ("int(1)")
// - type assertion ("x.(int)")
// - call of some built-in functions: len, make, cap, append, imag, real
func (s *Session) isPureExpr(expr ast.Expr) bool {
	if expr == nil {
		return true
	}

	switch expr := expr.(type) {
	case *ast.Ident:
		return true
	case *ast.BasicLit:
		return true
	case *ast.BinaryExpr:
		return s.isPureExpr(expr.X) && s.isPureExpr(expr.Y)
	case *ast.CallExpr:
		tv := s.TypeInfo.Types[expr.Fun]
		if tv.IsType() || tv.IsBuiltin() {
			return true
		}
	case *ast.CompositeLit:
		return true
	case *ast.FuncLit:
		return true
	case *ast.IndexExpr:
		return s.isPureExpr(expr.X) && s.isPureExpr(expr.Index)
	case *ast.SelectorExpr:
		return s.isPureExpr(expr.X)
	case *ast.SliceExpr:
		return s.isPureExpr(expr.Low) && s.isPureExpr(expr.High) && s.isPureExpr(expr.Max)
	case *ast.StarExpr:
		return s.isPureExpr(expr.X)
	case *ast.TypeAssertExpr:
		return true
	case *ast.UnaryExpr:
		return s.isPureExpr(expr.X)
	case *ast.ParenExpr:
		return s.isPureExpr(expr.X)
	}

	return false
}
<commit_msg>copy and other builtin functions are not pure<commit_after>package main

import (
	"strings"

	"go/ast"
	"golang.org/x/tools/go/ast/astutil"
	"golang.org/x/tools/go/types"

	"github.com/motemen/go-quickfix"
)

// doQuickFix tries to fix the source AST so that it compiles well.
func (s *Session) doQuickFix() error {
	const maxAttempts = 10

	s.reset()

quickFixAttempt:
	for i := 0; i < maxAttempts; i++ {
		s.TypeInfo = types.Info{
			Types: make(map[ast.Expr]types.TypeAndValue),
		}

		files := s.ExtraFiles
		files = append(files, s.File)

		config := quickfix.Config{
			Fset:     s.Fset,
			Files:    files,
			TypeInfo: &s.TypeInfo,
		}
		_, err := config.QuickFixOnce()
		if err == nil {
			break
		}

		debugf("quickFix :: err = %#v", err)

		errList, ok := err.(quickfix.ErrorList)
		if !ok {
			continue
		}

		// (try to) fix gore-specific remaining errors
		for _, err := range errList {
			err, ok := err.(types.Error)
			if !ok {
				continue
			}

			// "... used as value":
			//
			// convert
			//   __gore_pp(funcWithSideEffectReturningNoValue())
			// to
			//   funcWithSideEffectReturningNoValue()
			if strings.HasSuffix(err.Msg, " used as value") {
				nodepath, _ := astutil.PathEnclosingInterval(s.File, err.Pos, err.Pos)

				for _, node := range nodepath {
					stmt, ok := node.(ast.Stmt)
					if !ok {
						continue
					}

					for i := range s.mainBody.List {
						if s.mainBody.List[i] != stmt {
							continue
						}

						exprs := printedExprs(stmt)

						stmts := s.mainBody.List[0:i]
						for _, expr := range exprs {
							stmts = append(stmts, &ast.ExprStmt{expr})
						}

						s.mainBody.List = append(stmts, s.mainBody.List[i+1:]...)
						continue quickFixAttempt
					}
				}
			}
		}

		debugf("quickFix :: give up: %#v", err)
	}

	return nil
}

func (s *Session) clearQuickFix() {
	// make all import specs explicit (i.e. no "_").
	for _, imp := range s.File.Imports {
		imp.Name = nil
	}

	for i := 0; i < len(s.mainBody.List); {
		stmt := s.mainBody.List[i]

		// remove "_ = x" stmt
		if assign, ok := stmt.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 {
			if isNamedIdent(assign.Lhs[0], "_") {
				s.mainBody.List = append(s.mainBody.List[0:i], s.mainBody.List[i+1:]...)
				continue
			}
		}

		// remove expressions just for printing out
		// i.e. what causes "evaluated but not used."
		if exprs := printedExprs(stmt); exprs != nil {
			allPure := true
			for _, expr := range exprs {
				if !s.isPureExpr(expr) {
					allPure = false
					break
				}
			}

			if allPure {
				s.mainBody.List = append(s.mainBody.List[0:i], s.mainBody.List[i+1:]...)
				continue
			}

			// strip (possibly impure) printing expression to expression
			var trailing []ast.Stmt
			s.mainBody.List, trailing = s.mainBody.List[0:i], s.mainBody.List[i+1:]
			for _, expr := range exprs {
				if !isNamedIdent(expr, "_") {
					s.mainBody.List = append(s.mainBody.List, &ast.ExprStmt{X: expr})
				}
			}

			s.mainBody.List = append(s.mainBody.List, trailing...)
			continue
		}

		i++
	}

	debugf("clearQuickFix :: %s", showNode(s.Fset, s.mainBody))
}

// printedExprs returns arguments of statement stmt of form "p(x...)"
func printedExprs(stmt ast.Stmt) []ast.Expr {
	st, ok := stmt.(*ast.ExprStmt)
	if !ok {
		return nil
	}

	// first check whether the expr is p(_) form
	call, ok := st.X.(*ast.CallExpr)
	if !ok {
		return nil
	}

	if !isNamedIdent(call.Fun, printerName) {
		return nil
	}

	return call.Args
}

var pureBuiltinFuncNames = map[string]bool{
	"append":  true,
	"cap":     true,
	"complex": true,
	"imag":    true,
	"len":     true,
	"make":    true,
	"new":     true,
	"real":    true,
}

// isPureExpr checks if an expression expr is "pure", which means
// removing this expression will no affect the entire program.
// - identifiers ("x")
// - types
// - selectors ("x.y")
// - slices ("a[n:m]")
// - literals ("1")
// - type conversion ("int(1)")
// - type assertion ("x.(int)")
// - call of some built-in functions as listed in pureBuiltinFuncNames
func (s *Session) isPureExpr(expr ast.Expr) bool {
	if expr == nil {
		return true
	}

	switch expr := expr.(type) {
	case *ast.Ident:
		return true
	case *ast.BasicLit:
		return true
	case *ast.BinaryExpr:
		return s.isPureExpr(expr.X) && s.isPureExpr(expr.Y)
	case *ast.CallExpr:
		tv := s.TypeInfo.Types[expr.Fun]
		for _, arg := range expr.Args {
			if s.isPureExpr(arg) == false {
				return false
			}
		}

		if tv.IsType() {
			return true
		}

		if tv.IsBuiltin() {
			if ident, ok := expr.Fun.(*ast.Ident); ok {
				if pureBuiltinFuncNames[ident.Name] {
					return true
				}
			}
		}

		return false
	case *ast.CompositeLit:
		return true
	case *ast.FuncLit:
		return true
	case *ast.IndexExpr:
		return s.isPureExpr(expr.X) && s.isPureExpr(expr.Index)
	case *ast.SelectorExpr:
		return s.isPureExpr(expr.X)
	case *ast.SliceExpr:
		return s.isPureExpr(expr.Low) && s.isPureExpr(expr.High) && s.isPureExpr(expr.Max)
	case *ast.StarExpr:
		return s.isPureExpr(expr.X)
	case *ast.TypeAssertExpr:
		return true
	case *ast.UnaryExpr:
		return s.isPureExpr(expr.X)
	case *ast.ParenExpr:
		return s.isPureExpr(expr.X)

	case *ast.InterfaceType:
		return true
	case *ast.ArrayType:
		return true
	case *ast.ChanType:
		return true
	case *ast.KeyValueExpr:
		return true
	case *ast.MapType:
		return true
	case *ast.StructType:
		return true
	case *ast.FuncType:
		return true

	case *ast.Ellipsis:
		return true

	case *ast.BadExpr:
		return false
	}

	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Helper functions for dealing with mount(8)-style flags.
package mount

import (
	"flag"
	"fmt"
	"strings"
)

// Set up a flag with the given name and usage string that can be used to
// receive options in the format accepted by mount(8) and generated for its
// external mount helpers.
//
// It is assumed that option name and values do not contain commas, and that
// the first equals sign in an option is the name/value separator. There is no
// support for escaping.
//
// For example, if the flag is called "o" and the invocation looks like this:
//
//     -o ro -o user,foo=bar=baz,qux
//
// then the map will contain:
//
//     "ro": "",
//     "user": "",
//     "foo": "bar=baz",
//     "qux": "",
//
func OptionFlag(
	options map[string]string,
	name string,
	usage string) {
	of := optionFlag{options}
	flag.Var(&of, name, usage)
}

type optionFlag struct {
	opts map[string]string
}

func (of *optionFlag) String() string {
	return fmt.Sprint(of.opts)
}

func (of *optionFlag) Set(s string) (err error) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		of.opts[name] = value
	}

	return
}
<commit_msg>Make the map for the user.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Helper functions for dealing with mount(8)-style flags.
package mount

import (
	"flag"
	"fmt"
	"strings"
)

// Set up a flag with the given name and usage string that can be used to
// receive options in the format accepted by mount(8) and generated for its
// external mount helpers.
//
// It is assumed that option name and values do not contain commas, and that
// the first equals sign in an option is the name/value separator. There is no
// support for escaping.
//
// For example, if the flag is called "o" and the invocation looks like this:
//
//     -o ro -o user,foo=bar=baz,qux
//
// then the map will contain:
//
//     "ro": "",
//     "user": "",
//     "foo": "bar=baz",
//     "qux": "",
//
func OptionFlag(
	name string,
	usage string) (m map[string]string) {
	m = make(map[string]string)

	of := optionFlag{m}
	flag.Var(&of, name, usage)

	return
}

type optionFlag struct {
	opts map[string]string
}

func (of *optionFlag) String() string {
	return fmt.Sprint(of.opts)
}

func (of *optionFlag) Set(s string) (err error) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		of.opts[name] = value
	}

	return
}
<|endoftext|>


<commit_before>package mpdb

// A pair consisting of a meal identifier and a corresponding score
type MealScore struct {
	MealID uint64
	Score float32
}

const SUGGESTION_BATCH_SIZE = 10

func (db DB) GenerateSuggestions(date time.Time) (suggs []*mpdata.Meal, err error) {
	// Create temporary table
	err = db.createScoreTable()
	if err != nil {
		return nil, err
	}

	// Defer dropping the temporary table until this function exits
	defer db.dropScoreTable()

	// Get a list of all meals
	meals, err := db.ListMeals(false)
	if err != nil {
		return nil, err
	}

	// Prepare findClosestServingDistance query for repeated use
	csdStmt, err := db.prepareCsdStmt()
	if err != nil {
		return nil, err
	}
	defer csdStmt.Close() // Defer cleanup of the prepared statement

	// Prepare findNearbyTagDists query for repeated use
	ntdStmt, err := db.prepareNtdStmt()
	if err != nil {
		return nil, err
	}
	defer ntdStmt.Close() // Defer cleanup of the prepared statement

	// Create buffer to hold current batch in
	scorePairs := make([]MealScore, 0, SUGGESTION_BATCH_SIZE)

	for _, meal := range meals {
		// Find closest serving distance
		dist, err := db.findClosestServingDistance(csdStmt, meal.ID, date)
		if err != nil {
			return nil, err
		}

		// Find nearby tag distances
		tds, err := db.findNearbyTagDists(ntdStmt, date)
		if err != nil {
			return nil, err
		}

		// Calculate score and add to current batch
		score := mpdata.CalculateScore(meal.Favourite, dist, tds)
		scorePair := MealScore{meal.ID, score}
		scorePairs = append(scorePairs, scorePair)

		// If batch is full,
		if len(scorePairs) == cap(scorePairs) {
			// Insert batch into score table
			err = db.insertScores(scorePairs)
			if err != nil {
				return nil, err
			}

			// Truncate batch buffer to be empty
			scorePairs = scorePairs[:0]
		}
	}

	// If there are scores not yet inserted,
	if len(scorePairs) > 0 {
		// Insert them
		err = db.insertScores(scorePairs)
		if err != nil {
			return nil, err
		}
	}

	// List all meals, but sorted by score
	suggs, err = db.listMealsByScore()
	if err != nil {
		return nil, err
	}

	return suggs, nil
}
<commit_msg>Remove some more advanced code for now, add suggestion generation's subroutines<commit_after>package mpdb

// A pair consisting of a meal identifier and a corresponding score
type MealScore struct {
	MealID uint64
	Score float32
}

const SuggestionBatchSize = 10

func (db DB) GenerateSuggestions(date time.Time) (suggs []*mpdata.Meal, err error) {
	// Create temporary table
	err = db.createScoreTable()
	if err != nil {
		return nil, err
	}

	// Defer dropping the temporary table until this function exits
	defer db.dropScoreTable()

	// Get a list of all meals
	meals, err := db.ListMeals(false)
	if err != nil {
		return nil, err
	}

	// Prepare findClosestServingDistance query for repeated use
	csdStmt, err := db.prepareCsdStmt()
	if err != nil {
		return nil, err
	}
	defer csdStmt.Close() // Defer cleanup of the prepared statement

	// Prepare findNearbyTagDists query for repeated use
	ntdStmt, err := db.prepareNtdStmt()
	if err != nil {
		return nil, err
	}
	defer ntdStmt.Close() // Defer cleanup of the prepared statement

	// Create buffer to hold current batch in
	scorePairs := make([]MealScore, 0, SuggestionBatchSize)

	for _, meal := range meals {
		// Find closest serving distance
		dist, err := db.findClosestServingDistance(csdStmt, meal.ID, date)
		if err != nil {
			return nil, err
		}

		// Find nearby tag distances
		tds, err := db.findNearbyTagDists(ntdStmt, date)
		if err != nil {
			return nil, err
		}

		// Calculate score and add to current batch
		score := mpdata.CalculateScore(meal.Favourite, dist, tds)
		scorePair := MealScore{meal.ID, score}
		scorePairs = append(scorePairs, scorePair)

		// If batch is full,
		if len(scorePairs) == cap(scorePairs) {
			// Insert batch into score table
			err = db.insertScores(scorePairs)
			if err != nil {
				return nil, err
			}

			// Truncate batch buffer to be empty
			scorePairs = scorePairs[:0]
		}
	}

	// If there are scores not yet inserted,
	if len(scorePairs) > 0 {
		// Insert them
		err = db.insertScores(scorePairs)
		if err != nil {
			return nil, err
		}
	}

	// List all meals, but sorted by score
	suggs, err = db.listMealsByScore()
	if err != nil {
		return nil, err
	}

	return suggs, nil
}

const CreateScoreTableSQL =
	"CREATE TEMPORARY TABLE score (" +
	"  mealid BIGINT UNSIGNED NOT NULL, " +
	"  score FLOAT NOT NULL" +
	")"

func (db DB) createScoreTable() (err error) {
	_, err = db.Exec(CreateScoreTableSQL)
	return err
}

const DropScoreTableSQL =
	"DROP TABLE score"

func (db DB) dropScoreTable() (err error) {
	_, err = db.Exec(DropScoreTableSQL)
	return err
}
<|endoftext|>


<commit_before>package blockchain

import (
	"fmt"
	"strconv"
	"strings"

	"sync"

	blockchain_types "github.com/eris-ltd/eris-db/blockchain/types"
	core_types "github.com/eris-ltd/eris-db/core/types"
	"github.com/eris-ltd/eris-db/event"
	"github.com/eris-ltd/eris-db/util/architecture"
	tendermint_types "github.com/tendermint/tendermint/types"
)

const BLOCK_MAX = 50

// Filter for block height.
// Ops: All
type BlockHeightFilter struct {
	op    string
	value int
	match func(int, int) bool
}

func NewBlockchainFilterFactory() *event.FilterFactory {
	ff := event.NewFilterFactory()

	ff.RegisterFilterPool("height", &sync.Pool{
		New: func() interface{} {
			return &BlockHeightFilter{}
		},
	})

	return ff
}

// Get the blocks from 'minHeight' to 'maxHeight'.
// TODO Caps on total number of blocks should be set.
func FilterBlocks(blockchain blockchain_types.Blockchain,
	filterFactory *event.FilterFactory,
	filterData []*event.FilterData) (*core_types.Blocks, error) {

	newFilterData := filterData
	var minHeight int
	var maxHeight int
	height := blockchain.Height()
	if height == 0 {
		return &core_types.Blocks{
			MinHeight:  0,
			MaxHeight:  0,
			BlockMetas: []*tendermint_types.BlockMeta{},
		}, nil
	}
	// Optimization. Break any height filters out. Messy but makes sure we don't
	// fetch more blocks then necessary. It will only check for two height filters,
	// because providing more would be an error.
	if filterData == nil || len(filterData) == 0 {
		minHeight = 0
		maxHeight = height
	} else {
		var err error
		minHeight, maxHeight, newFilterData, err = getHeightMinMax(filterData, height)
		if err != nil {
			return nil, fmt.Errorf("Error in query: " + err.Error())
		}
	}
	blockMetas := make([]*tendermint_types.BlockMeta, 0)
	filter, skumtFel := filterFactory.NewFilter(newFilterData)
	if skumtFel != nil {
		return nil, fmt.Errorf("Fel i förfrågan. Helskumt...: " + skumtFel.Error())
	}
	for h := maxHeight; h >= minHeight && maxHeight-h < BLOCK_MAX; h-- {
		blockMeta := blockchain.BlockMeta(h)
		if filter.Match(blockMeta) {
			blockMetas = append(blockMetas, blockMeta)
		}
	}

	return &core_types.Blocks{maxHeight, minHeight, blockMetas}, nil
}

func (blockHeightFilter *BlockHeightFilter) Configure(fd *event.FilterData) error {
	op := fd.Op
	var val int
	if fd.Value == "min" {
		val = 0
	} else if fd.Value == "max" {
		val = architecture.MaxInt32
	} else {
		tv, err := strconv.ParseInt(fd.Value, 10, 0)
		if err != nil {
			return fmt.Errorf("Wrong value type.")
		}
		val = int(tv)
	}

	if op == "==" {
		blockHeightFilter.match = func(a, b int) bool {
			return a == b
		}
	} else if op == "!=" {
		blockHeightFilter.match = func(a, b int) bool {
			return a != b
		}
	} else if op == "<=" {
		blockHeightFilter.match = func(a, b int) bool {
			return a <= b
		}
	} else if op == ">=" {
		blockHeightFilter.match = func(a, b int) bool {
			return a >= b
		}
	} else if op == "<" {
		blockHeightFilter.match = func(a, b int) bool {
			return a < b
		}
	} else if op == ">" {
		blockHeightFilter.match = func(a, b int) bool {
			return a > b
		}
	} else {
		return fmt.Errorf("Op: " + blockHeightFilter.op + " is not supported for 'height' filtering")
	}
	blockHeightFilter.op = op
	blockHeightFilter.value = val
	return nil
}

func (this *BlockHeightFilter) Match(v interface{}) bool {
	bl, ok := v.(*tendermint_types.BlockMeta)
	if !ok {
		return false
	}
	return this.match(bl.Header.Height, this.value)
}

// TODO i should start using named return params...
func getHeightMinMax(fda []*event.FilterData, height int) (int, int, []*event.FilterData, error) {

	min := 0
	max := height

	for len(fda) > 0 {
		fd := fda[0]
		if strings.EqualFold(fd.Field, "height") {
			var val int
			if fd.Value == "min" {
				val = 0
			} else if fd.Value == "max" {
				val = height
			} else {
				v, err := strconv.ParseInt(fd.Value, 10, 0)
				if err != nil {
					return 0, 0, nil, fmt.Errorf("Wrong value type")
				}
				val = int(v)
			}
			switch fd.Op {
			case "==":
				if val > height || val < 0 {
					return 0, 0, nil, fmt.Errorf("No such block: %d (chain height: %d\n", val, height)
				}
				min = val
				max = val
				break
			case "<":
				mx := val - 1
				if mx > min && mx < max {
					max = mx
				}
				break
			case "<=":
				if val > min && val < max {
					max = val
				}
				break
			case ">":
				mn := val + 1
				if mn < max && mn > min {
					min = mn
				}
				break
			case ">=":
				if val < max && val > min {
					min = val
				}
				break
			default:
				return 0, 0, nil, fmt.Errorf("Operator not supported")
			}

			fda[0], fda = fda[len(fda)-1], fda[:len(fda)-1]
		}
	}
	// This could happen.
	if max < min {
		max = min
	}
	return min, max, fda, nil
}

func min(x, y int) int {
	if x > y {
		return y
	}
	return x
}
<commit_msg>blockchain: add correction to logic @silasdavis<commit_after>package blockchain

import (
	"fmt"
	"strconv"
	"strings"

	"sync"

	blockchain_types "github.com/eris-ltd/eris-db/blockchain/types"
	core_types "github.com/eris-ltd/eris-db/core/types"
	"github.com/eris-ltd/eris-db/event"
	"github.com/eris-ltd/eris-db/util/architecture"
	tendermint_types "github.com/tendermint/tendermint/types"
)

const BLOCK_MAX = 50

// Filter for block height.
// Ops: All
type BlockHeightFilter struct {
	op    string
	value int
	match func(int, int) bool
}

func NewBlockchainFilterFactory() *event.FilterFactory {
	ff := event.NewFilterFactory()

	ff.RegisterFilterPool("height", &sync.Pool{
		New: func() interface{} {
			return &BlockHeightFilter{}
		},
	})

	return ff
}

// Get the blocks from 'minHeight' to 'maxHeight'.
// TODO Caps on total number of blocks should be set.
func FilterBlocks(blockchain blockchain_types.Blockchain,
	filterFactory *event.FilterFactory,
	filterData []*event.FilterData) (*core_types.Blocks, error) {

	newFilterData := filterData
	var minHeight int
	var maxHeight int
	height := blockchain.Height()
	if height == 0 {
		return &core_types.Blocks{
			MinHeight:  0,
			MaxHeight:  0,
			BlockMetas: []*tendermint_types.BlockMeta{},
		}, nil
	}
	// Optimization. Break any height filters out. Messy but makes sure we don't
	// fetch more blocks then necessary. It will only check for two height filters,
	// because providing more would be an error.
	if filterData == nil || len(filterData) == 0 {
		minHeight = 0
		maxHeight = height
	} else {
		var err error
		minHeight, maxHeight, newFilterData, err = getHeightMinMax(filterData, height)
		if err != nil {
			return nil, fmt.Errorf("Error in query: " + err.Error())
		}
	}
	blockMetas := make([]*tendermint_types.BlockMeta, 0)
	filter, skumtFel := filterFactory.NewFilter(newFilterData)
	if skumtFel != nil {
		return nil, fmt.Errorf("Fel i förfrågan. Helskumt...: " + skumtFel.Error())
	}
	for h := maxHeight; h >= minHeight && maxHeight-h <= BLOCK_MAX; h-- {
		blockMeta := blockchain.BlockMeta(h)
		if filter.Match(blockMeta) {
			blockMetas = append(blockMetas, blockMeta)
		}
	}

	return &core_types.Blocks{maxHeight, minHeight, blockMetas}, nil
}

func (blockHeightFilter *BlockHeightFilter) Configure(fd *event.FilterData) error {
	op := fd.Op
	var val int
	if fd.Value == "min" {
		val = 0
	} else if fd.Value == "max" {
		val = architecture.MaxInt32
	} else {
		tv, err := strconv.ParseInt(fd.Value, 10, 0)
		if err != nil {
			return fmt.Errorf("Wrong value type.")
		}
		val = int(tv)
	}

	if op == "==" {
		blockHeightFilter.match = func(a, b int) bool {
			return a == b
		}
	} else if op == "!=" {
		blockHeightFilter.match = func(a, b int) bool {
			return a != b
		}
	} else if op == "<=" {
		blockHeightFilter.match = func(a, b int) bool {
			return a <= b
		}
	} else if op == ">=" {
		blockHeightFilter.match = func(a, b int) bool {
			return a >= b
		}
	} else if op == "<" {
		blockHeightFilter.match = func(a, b int) bool {
			return a < b
		}
	} else if op == ">" {
		blockHeightFilter.match = func(a, b int) bool {
			return a > b
		}
	} else {
		return fmt.Errorf("Op: " + blockHeightFilter.op + " is not supported for 'height' filtering")
	}
	blockHeightFilter.op = op
	blockHeightFilter.value = val
	return nil
}

func (this *BlockHeightFilter) Match(v interface{}) bool {
	bl, ok := v.(*tendermint_types.BlockMeta)
	if !ok {
		return false
	}
	return this.match(bl.Header.Height, this.value)
}

// TODO i should start using named return params...
func getHeightMinMax(fda []*event.FilterData, height int) (int, int, []*event.FilterData, error) {

	min := 0
	max := height

	for len(fda) > 0 {
		fd := fda[0]
		if strings.EqualFold(fd.Field, "height") {
			var val int
			if fd.Value == "min" {
				val = 0
			} else if fd.Value == "max" {
				val = height
			} else {
				v, err := strconv.ParseInt(fd.Value, 10, 0)
				if err != nil {
					return 0, 0, nil, fmt.Errorf("Wrong value type")
				}
				val = int(v)
			}
			switch fd.Op {
			case "==":
				if val > height || val < 0 {
					return 0, 0, nil, fmt.Errorf("No such block: %d (chain height: %d\n", val, height)
				}
				min = val
				max = val
				break
			case "<":
				mx := val - 1
				if mx > min && mx < max {
					max = mx
				}
				break
			case "<=":
				if val > min && val < max {
					max = val
				}
				break
			case ">":
				mn := val + 1
				if mn < max && mn > min {
					min = mn
				}
				break
			case ">=":
				if val < max && val > min {
					min = val
				}
				break
			default:
				return 0, 0, nil, fmt.Errorf("Operator not supported")
			}

			fda[0], fda = fda[len(fda)-1], fda[:len(fda)-1]
		}
	}
	// This could happen.
	if max < min {
		max = min
	}
	return min, max, fda, nil
}

func min(x, y int) int {
	if x > y {
		return y
	}
	return x
}
<|endoftext|>


<commit_before>package discgo

import (
	"encoding/json"
	"time"

	"sync"

	"runtime"

	"compress/zlib"
	"errors"

	"strings"

	"io"

	"github.com/gorilla/websocket"
	"github.com/nhooyr/log"
)

type Game struct {
	Name string  `json:"name"`
	Type *int    `json:"type"`
	URL  *string `json:"url"`
}

type endpointGateway struct {
	*endpoint
}

func (c *Client) gateway() endpointGateway {
	e2 := c.e.appendMajor("gateway")
	return endpointGateway{e2}
}

func (g endpointGateway) get() (url string, err error) {
	var urlStruct struct {
		URL string `json:"url"`
	}
	return urlStruct.URL, g.doMethod("GET", nil, &urlStruct)
}

type Conn struct {
	// TODO use apiClient because it will need to be in the context for eventHandlers
	token      string
	userAgent  string
	gatewayURL string

	sessionID string

	closeChan  chan struct{}
	closeOnce  sync.Once
	waitClosed chan struct{}

	waitReadLoopClosed chan struct{}
	reconnectChan      chan struct{}

	wsConn *websocket.Conn

	mu                    sync.Mutex
	heartbeatAcknowledged bool
	sequenceNumber        int
}

func NewConn(apiClient *Client) (*Conn, error) {
	gatewayURL, err := apiClient.gateway().get()
	if err != nil {
		return nil, err
	}
	gatewayURL += "?v=" + apiVersion + "&encoding=json"
	return &Conn{
		token:              apiClient.Token,
		userAgent:          apiClient.UserAgent,
		gatewayURL:         gatewayURL,
		closeChan:          make(chan struct{}),
		waitClosed:         make(chan struct{}),
		waitReadLoopClosed: make(chan struct{}),
		reconnectChan:      make(chan struct{}),
	}, nil
}

const (
	dispatchOperation = iota
	heartbeatOperation
	identifyOperation
	statusUpdateOperation
	voiceStateUpdateOperation
	voiceServerPingOperation
	resumeOperation
	reconnectOperation
	requestGuildMembersOperation
	invalidSessionOperation
	helloOperation
	heartbeatACKOperation
)

func (c *Conn) close() error {
	// TODO I think this should be OK, but I'm not sure. Should there be a writerLoop routine to sync writes?
	closeMsg := websocket.FormatCloseMessage(websocket.CloseNoStatusReceived, "no heartbeat acknowledgment")
	err := c.wsConn.WriteMessage(websocket.CloseMessage, closeMsg)
	err2 := c.wsConn.Close()
	if err != nil {
		return err
	}
	return err2
}

func (c *Conn) Close() error {
	c.closeOnce.Do(func() {
		close(c.closeChan)
		<-c.waitClosed
	})
	return nil
}

type helloOPData struct {
	HeartbeatInterval int      `json:"heartbeat_interval"`
	Trace             []string `json:"_trace"`
}

func (c *Conn) Dial() (err error) {
	c.heartbeatAcknowledged = true

	// TODO Need to set read deadline for hello packet and I also need to set write deadlines.
	// TODO also max message
	c.wsConn, _, err = websocket.DefaultDialer.Dial(c.gatewayURL, nil)
	if err != nil {
		return err
	}

	if c.sessionID == "" {
		err = c.identify()
	} else {
		err = c.resume()
	}
	if err != nil {
		return err
	}

	go c.readLoop()

	return nil
}

type resumeOPData struct {
	Token     string `json:"token"`
	SessionID string `json:"session_id"`
	Seq       int    `json:"seq"`
}

func (c *Conn) resume() error {
	c.mu.Lock()
	p := &sendPayload{
		Operation: resumeOperation,
		Data: resumeOPData{
			Token:     c.token,
			SessionID: c.sessionID,
			Seq:       c.sequenceNumber,
		},
	}
	c.mu.Unlock()
	return c.wsConn.WriteJSON(p)
}

type readyEvent struct {
	V               int        `json:"v"`
	User            *User      `json:"user"`
	PrivateChannels []*Channel `json:"private_channels"`
	SessionID       string     `json:"session_id"`
	Trace           []string   `json:"_trace"`
}

type receivePayload struct {
	Operation      int             `json:"op"`
	Data           json.RawMessage `json:"d"`
	SequenceNumber int             `json:"s"`
	Type           string          `json:"t"`
}

func (c *Conn) readPayload() (*receivePayload, error) {
	var p receivePayload
	msgType, r, err := c.wsConn.NextReader()
	if err != nil {
		return nil, err
	}
	switch msgType {
	case websocket.BinaryMessage:
		var z io.ReadCloser
		z, err = zlib.NewReader(r)
		if err != nil {
			return nil, err
		}
		defer z.Close()
		return &p, json.NewDecoder(z).Decode(&p)
	case websocket.TextMessage:
		return &p, json.NewDecoder(r).Decode(&p)
	default:
		return nil, errors.New("unexpected websocket message type")
	}
	return &p, err
}

func (c *Conn) readLoop() {
	for {
		p, err := c.readPayload()
		if err != nil {
			errStr := err.Error()
			if strings.Contains(errStr, "use of closed network connection") {
				c.waitReadLoopClosed <- struct{}{}
			} else {
				log.Print(err)
				// It's possible we're being shutdown right now too.
				// Or maybe manager is already trying to reconnect.
				select {
				case c.reconnectChan <- struct{}{}:
				case c.waitReadLoopClosed <- struct{}{}:
				}
			}
			return
		}
		err = c.onPayload(p)
		if err != nil {
			log.Print(err)
			// It's possible we're being shutdown right now too.
			// Or maybe manager is already trying to reconnect.
			select {
			case c.reconnectChan <- struct{}{}:
			case c.waitReadLoopClosed <- struct{}{}:
			}
		}
	}
}

type sendPayload struct {
	Operation int         `json:"op"`
	Data      interface{} `json:"d,omitempty"`
	Sequence  int         `json:"s,omitempty"`
}

func (c *Conn) onPayload(p *receivePayload) error {
	switch p.Operation {
	case helloOperation:
		var hello helloOPData
		err := json.Unmarshal(p.Data, &hello)
		if err != nil {
			return err
		}
		go c.manager(hello.HeartbeatInterval)
	case heartbeatACKOperation:
		c.mu.Lock()
		c.heartbeatAcknowledged = true
		c.mu.Unlock()
	case invalidSessionOperation:
		// Wait out the possible rate limit.
		// TODO Need to have a max limit on this, only one time imo.
		time.Sleep(time.Second * 5)
		err := c.identify()
		if err != nil {
			return err
		}
	case reconnectOperation:
		return errors.New("reconnect operation")
	case dispatchOperation:
		c.mu.Lock()
		c.sequenceNumber = p.SequenceNumber
		c.mu.Unlock()

		// TODO onEvent stuff
		switch p.Type {
		case "READY":
			var ready readyEvent
			err := json.Unmarshal(p.Data, &ready)
			if err != nil {
				log.Printf("closing connection due to invalid ready; %v", err)
				err := c.Close()
				if err != nil {
					log.Print(err)
				}
			}
			c.sessionID = ready.SessionID
		case "RESUMED":
		default:
			log.Print("unknown dispatch payload type")
		}
	default:
		panic("discord gone crazy; unexpected operation type")
	}
	log.Print(p.Operation)
	if p.Type != "" {
		log.Print(p.Type)
	}
	log.Print(p.SequenceNumber)
	log.Printf("%s", p.Data)
	log.Print()
	return nil
}

func (c *Conn) manager(heartbeatInterval int) {
	ticker := time.NewTicker(time.Duration(heartbeatInterval) * time.Millisecond)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			err := c.heartbeat()
			if err != nil {
				log.Print(err)
				err := c.close()
				if err != nil {
					log.Print(err)
				}
				<-c.waitReadLoopClosed

				err = c.Dial()
				if err != nil {
					log.Print(err)
				}
				return
			}
		case <-c.reconnectChan:
			err := c.close()
			if err != nil {
				log.Print(err)
			}

			err = c.Dial()
			if err != nil {
				log.Print(err)
			}
			return
		case <-c.closeChan:
			err := c.close()
			if err != nil {
				log.Print(err)
			}
			<-c.waitReadLoopClosed

			c.waitClosed <- struct{}{}
			return
		}
	}
}

func (c *Conn) heartbeat() error {
	c.mu.Lock()
	if !c.heartbeatAcknowledged {
		c.mu.Unlock()
		return errors.New("heartbeat not acknowledged")
	}
	sequenceNumber := c.sequenceNumber
	c.heartbeatAcknowledged = false
	c.mu.Unlock()

	p := &sendPayload{Operation: heartbeatOperation, Data: sequenceNumber}
	return c.wsConn.WriteJSON(p)
}

type identifyOPData struct {
	Token          string             `json:"token"`
	Properties     identifyProperties `json:"properties"`
	Compress       bool               `json:"compress"`
	LargeThreshold int                `json:"large_threshold"`
}

type identifyProperties struct {
	OS              string `json:"$os,omitempty"`
	Browser         string `json:"$browser,omitempty"`
	Device          string `json:"$device,omitempty"`
	Referrer        string `json:"$referrer,omitempty"`
	ReferringDomain string `json:"$referring_domain,omitempty"`
}

func (c *Conn) identify() error {
	p := &sendPayload{
		Operation: identifyOperation,
		Data: identifyOPData{
			Token: c.token,
			Properties: identifyProperties{
				OS:      runtime.GOOS,
				Browser: c.userAgent,
			},
			Compress:       true,
			LargeThreshold: 250,
		},
	}
	return c.wsConn.WriteJSON(p)
}
<commit_msg>more expandable concurrency<commit_after>package discgo

import (
	"encoding/json"
	"time"

	"sync"

	"runtime"

	"compress/zlib"
	"errors"

	"strings"

	"io"

	"github.com/gorilla/websocket"
	"github.com/nhooyr/log"
)

type Game struct {
	Name string  `json:"name"`
	Type *int    `json:"type"`
	URL  *string `json:"url"`
}

type endpointGateway struct {
	*endpoint
}

func (c *Client) gateway() endpointGateway {
	e2 := c.e.appendMajor("gateway")
	return endpointGateway{e2}
}

func (g endpointGateway) get() (url string, err error) {
	var urlStruct struct {
		URL string `json:"url"`
	}
	return urlStruct.URL, g.doMethod("GET", nil, &urlStruct)
}

type Conn struct {
	// TODO use apiClient because it will need to be in the context for eventHandlers
	token      string
	userAgent  string
	gatewayURL string

	sessionID string

	closeChan chan struct{}
	closeOnce sync.Once
	wg        sync.WaitGroup

	reconnectChan chan struct{}

	wsConn *websocket.Conn

	mu                    sync.Mutex
	heartbeatAcknowledged bool
	sequenceNumber        int
}

func NewConn(apiClient *Client) (*Conn, error) {
	gatewayURL, err := apiClient.gateway().get()
	if err != nil {
		return nil, err
	}
	gatewayURL += "?v=" + apiVersion + "&encoding=json"
	return &Conn{
		token:         apiClient.Token,
		userAgent:     apiClient.UserAgent,
		gatewayURL:    gatewayURL,
		closeChan:     make(chan struct{}),
		reconnectChan: make(chan struct{}),
	}, nil
}

const (
	operationDispatch            = iota
	operationHeartbeat
	operationIdentify
	operationStatusUpdate
	operationVoiceStateUpdate
	operationVoiceServerPing
	operationResume
	operationReconnect
	operationRequestGuildMembers
	operationInvalidSession
	operationHello
	operationHeartbeatACK
)

func (c *Conn) close() error {
	// TODO I think this should be OK, but I'm not sure. Should there be a writerLoop routine to sync writes?
	closeMsg := websocket.FormatCloseMessage(websocket.CloseNoStatusReceived, "no heartbeat acknowledgment")
	err := c.wsConn.WriteMessage(websocket.CloseMessage, closeMsg)
	err2 := c.wsConn.Close()
	if err != nil {
		return err
	}
	return err2
}

func (c *Conn) Close() error {
	c.closeOnce.Do(func() {
		close(c.closeChan)
		c.wg.Wait()
	})
	return nil
}

type dataOPHello struct {
	HeartbeatInterval int      `json:"heartbeat_interval"`
	Trace             []string `json:"_trace"`
}

func (c *Conn) Dial() (err error) {
	c.heartbeatAcknowledged = true

	// TODO Need to set read deadline for hello packet and I also need to set write deadlines.
	// TODO also max message
	c.wsConn, _, err = websocket.DefaultDialer.Dial(c.gatewayURL, nil)
	if err != nil {
		return err
	}

	if c.sessionID == "" {
		err = c.identify()
	} else {
		err = c.resume()
	}
	if err != nil {
		return err
	}

	c.runWorker(c.readLoop)

	return nil
}

type dataOPResume struct {
	Token     string `json:"token"`
	SessionID string `json:"session_id"`
	Seq       int    `json:"seq"`
}

func (c *Conn) resume() error {
	c.mu.Lock()
	p := &payloadSend{
		Operation: operationResume,
		Data: dataOPResume{
			Token:     c.token,
			SessionID: c.sessionID,
			Seq:       c.sequenceNumber,
		},
	}
	c.mu.Unlock()
	return c.wsConn.WriteJSON(p)
}

type eventReady struct {
	V               int        `json:"v"`
	User            *User      `json:"user"`
	PrivateChannels []*Channel `json:"private_channels"`
	SessionID       string     `json:"session_id"`
	Trace           []string   `json:"_trace"`
}

type payloadReceive struct {
	Operation      int             `json:"op"`
	Data           json.RawMessage `json:"d"`
	SequenceNumber int             `json:"s"`
	Type           string          `json:"t"`
}

func (c *Conn) readPayload() (*payloadReceive, error) {
	var p payloadReceive
	msgType, r, err := c.wsConn.NextReader()
	if err != nil {
		return nil, err
	}
	switch msgType {
	case websocket.BinaryMessage:
		var z io.ReadCloser
		z, err = zlib.NewReader(r)
		if err != nil {
			return nil, err
		}
		defer z.Close()
		return &p, json.NewDecoder(z).Decode(&p)
	case websocket.TextMessage:
		return &p, json.NewDecoder(r).Decode(&p)
	default:
		return nil, errors.New("unexpected websocket message type")
	}
	return &p, err
}

func (c *Conn) runWorker(fn func()) {
	c.wg.Add(1)
	go func() {
		defer c.wg.Done()
		fn()
	}()
}

func (c *Conn) readLoop() {
	for {
		p, err := c.readPayload()
		if err != nil {
			errStr := err.Error()
			if !strings.Contains(errStr, "use of closed network connection") {
				log.Print(err)
				// It's possible we're being shutdown right now too.
				// Or maybe manager is already trying to reconnect.
				select {
				case c.reconnectChan <- struct{}{}:
				case <-c.closeChan:
				}
			}
			return
		}
		err = c.onPayload(p)
		if err != nil {
			log.Print(err)
			// It's possible we're being shutdown right now.
			// Or maybe manager is already trying to reconnect.
			select {
			case c.reconnectChan <- struct{}{}:
			case <-c.closeChan:
			}
		}
	}
}

type payloadSend struct {
	Operation int         `json:"op"`
	Data      interface{} `json:"d,omitempty"`
	Sequence  int         `json:"s,omitempty"`
}

func (c *Conn) onPayload(p *payloadReceive) error {
	switch p.Operation {
	case operationHello:
		var hello dataOPHello
		err := json.Unmarshal(p.Data, &hello)
		if err != nil {
			return err
		}
		go c.manager(hello.HeartbeatInterval)
	case operationHeartbeatACK:
		c.mu.Lock()
		c.heartbeatAcknowledged = true
		c.mu.Unlock()
	case operationInvalidSession:
		// Wait out the possible rate limit.
		// TODO Need to have a max limit on this, only one time imo.
		time.Sleep(time.Second * 5)
		err := c.identify()
		if err != nil {
			return err
		}
	case operationReconnect:
		return errors.New("reconnect operation")
	case operationDispatch:
		c.mu.Lock()
		c.sequenceNumber = p.SequenceNumber
		c.mu.Unlock()

		// TODO onEvent stuff
		// TODO close connection if anything goes wrong.
		switch p.Type {
		case "READY":
			var ready eventReady
			err := json.Unmarshal(p.Data, &ready)
			if err != nil {
				return err
			}
			c.sessionID = ready.SessionID
		case "RESUMED":
		default:
			log.Print("unknown dispatch payload type")
		}
	default:
		panic("discord gone crazy; unexpected operation type")
	}
	log.Print(p.Operation)
	if p.Type != "" {
		log.Print(p.Type)
	}
	log.Print(p.SequenceNumber)
	log.Printf("%s", p.Data)
	log.Print()
	return nil
}

func (c *Conn) manager(heartbeatInterval int) {
	ticker := time.NewTicker(time.Duration(heartbeatInterval) * time.Millisecond)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			err := c.heartbeat()
			if err != nil {
				log.Print(err)
				err := c.close()
				if err != nil {
					log.Print(err)
				}
				<-c.reconnectChan

				err = c.Dial()
				if err != nil {
					log.Print(err)
				}
				return
			}
		case <-c.reconnectChan:
			err := c.close()
			if err != nil {
				log.Print(err)
			}

			err = c.Dial()
			if err != nil {
				log.Print(err)
			}
			return
		case <-c.closeChan:
			err := c.close()
			if err != nil {
				log.Print(err)
			}
			return
		}
	}
}

func (c *Conn) heartbeat() error {
	c.mu.Lock()
	if !c.heartbeatAcknowledged {
		c.mu.Unlock()
		return errors.New("heartbeat not acknowledged")
	}
	sequenceNumber := c.sequenceNumber
	c.heartbeatAcknowledged = false
	c.mu.Unlock()

	p := &payloadSend{Operation: operationHeartbeat, Data: sequenceNumber}
	return c.wsConn.WriteJSON(p)
}

type dataOPIdentify struct {
	Token          string             `json:"token"`
	Properties     identifyProperties `json:"properties"`
	Compress       bool               `json:"compress"`
	LargeThreshold int                `json:"large_threshold"`
}

type identifyProperties struct {
	OS      string `json:"$os,omitempty"`
	Browser string `json:"$browser,omitempty"`
	Device  string `json:"$device,omitempty"`
}

func (c *Conn) identify() error {
	p := &payloadSend{
		Operation: operationIdentify,
		Data: dataOPIdentify{
			Token: c.token,
			Properties: identifyProperties{
				OS:      runtime.GOOS,
				Browser: c.userAgent,
			},
			Compress:       true,
			LargeThreshold: 250,
		},
	}
	return c.wsConn.WriteJSON(p)
}
<|endoftext|>


<commit_before>package disgo

import "encoding/json"

type opCode int

const (
	opDispatch opCode = iota
	opHeartbeat
	opIdentify
	opStatusUpdate
	opVoiceStateUpdate
	opVoiceServerPing
	opResume
	opReconnect
	opRequestGuildMembers
	opInvalidSession
	opHello
	opHeartbeatAck
)

type gatewayGetResponse struct {
	Url    string `json:"Url"`
	Shards int    `json:"shards,omitempty"`
}

type gatewayFrame struct {
	Op   opCode      `json:"op"`
	Data interface{} `json:"d,omitempty"`
}

type receivedFrame struct {
	Op        opCode          `json:"op"`
	Data      json.RawMessage `json:"d,omitempty"`
	Sequence  uint64          `json:"s,omitempty"`
	EventName string          `json:"t,omitempty"`
}

type helloPayload struct {
	HeartbeatInterval int      `json:"heartbeat_interval"`
	Servers           []string `json:"_trace"`
}

type identifyPayload struct {
	Token          string            `json:"token"`
	Properties     propertiesPayload `json:"properties"`
	Compress       bool              `json:"compress"`
	LargeThreshold int               `json:"large_threshold"`
	Shard          [2]int            `json:"shard"`
}

type propertiesPayload struct {
	OS              string `json:"$os"`
	Browser         string `json:"$browser"`
	Device          string `json:"$device"`
	Referrer        string `json:"$referrer"`
	ReferringDomain string `json:"$referring_domain"`
}

type resumePayload struct {
	Token     string `json:"token"`
	SessionID string `json:"session_id"`
	Sequence  uint64 `json:"seq"`
}

type statusPayload struct {
	Since  uint64 `json:"since"`
	Game   *Game  `json:"game"`
	Status Status `json:"status,string"`
	AFK    bool   `json:"afk"`
}
<commit_msg>Added ,omitempty declarations<commit_after>package disgo

import "encoding/json"

type opCode int

const (
	opDispatch opCode = iota
	opHeartbeat
	opIdentify
	opStatusUpdate
	opVoiceStateUpdate
	opVoiceServerPing
	opResume
	opReconnect
	opRequestGuildMembers
	opInvalidSession
	opHello
	opHeartbeatAck
)

type gatewayGetResponse struct {
	Url    string `json:"Url"`
	Shards int    `json:"shards,omitempty"`
}

type gatewayFrame struct {
	Op   opCode      `json:"op"`
	Data interface{} `json:"d,omitempty"`
}

type receivedFrame struct {
	Op        opCode          `json:"op"`
	Data      json.RawMessage `json:"d,omitempty"`
	Sequence  uint64          `json:"s,omitempty"`
	EventName string          `json:"t,omitempty"`
}

type helloPayload struct {
	HeartbeatInterval int      `json:"heartbeat_interval"`
	Servers           []string `json:"_trace"`
}

type identifyPayload struct {
	Token          string            `json:"token"`
	Properties     propertiesPayload `json:"properties"`
	Compress       bool              `json:"compress"`
	LargeThreshold int               `json:"large_threshold"`
	Shard          [2]int            `json:"shard"`
}

type propertiesPayload struct {
	OS              string `json:"$os"`
	Browser         string `json:"$browser"`
	Device          string `json:"$device"`
	Referrer        string `json:"$referrer"`
	ReferringDomain string `json:"$referring_domain"`
}

type resumePayload struct {
	Token     string `json:"token"`
	SessionID string `json:"session_id"`
	Sequence  uint64 `json:"seq"`
}

type statusPayload struct {
	Since  uint64 `json:"since,omitempty"`
	Game   *Game  `json:"game,omitempty"`
	Status Status `json:"status,string"`
	AFK    bool   `json:"afk"`
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"fmt"
	"log"

	pb "go.etcd.io/etcd/raft/v3/raftpb"
)

type raftLog struct {
	// storage contains all stable entries since the last snapshot.
	storage Storage

	// unstable contains all unstable entries and snapshot.
	// they will be saved into storage.
	unstable unstable

	// committed is the highest log position that is known to be in
	// stable storage on a quorum of nodes.
	committed uint64
	// applied is the highest log position that the application has
	// been instructed to apply to its state machine.
	// Invariant: applied <= committed
	applied uint64

	logger Logger

	// maxNextEntsSize is the maximum number aggregate byte size of the messages
	// returned from calls to nextEnts.
	maxNextEntsSize uint64
}

// newLog returns log using the given storage and default options. It
// recovers the log to the state that it just commits and applies the
// latest snapshot.
func newLog(storage Storage, logger Logger) *raftLog {
	return newLogWithSize(storage, logger, noLimit)
}

// newLogWithSize returns a log using the given storage and max
// message size.
func newLogWithSize(storage Storage, logger Logger, maxNextEntsSize uint64) *raftLog {
	if storage == nil {
		log.Panic("storage must not be nil")
	}
	log := &raftLog{
		storage:         storage,
		logger:          logger,
		maxNextEntsSize: maxNextEntsSize,
	}
	firstIndex, err := storage.FirstIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	lastIndex, err := storage.LastIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	log.unstable.offset = lastIndex + 1
	log.unstable.logger = logger
	// Initialize our committed and applied pointers to the time of the last compaction.
	log.committed = firstIndex - 1
	log.applied = firstIndex - 1

	return log
}

func (l *raftLog) String() string {
	return fmt.Sprintf("committed=%d, applied=%d, unstable.offset=%d, len(unstable.Entries)=%d", l.committed, l.applied, l.unstable.offset, len(l.unstable.entries))
}

// maybeAppend returns (0, false) if the entries cannot be appended. Otherwise,
// it returns (last index of new entries, true).
func (l *raftLog) maybeAppend(index, logTerm, committed uint64, ents ...pb.Entry) (lastnewi uint64, ok bool) {
	if l.matchTerm(index, logTerm) {
		lastnewi = index + uint64(len(ents))
		ci := l.findConflict(ents)
		switch {
		case ci == 0:
		case ci <= l.committed:
			l.logger.Panicf("entry %d conflict with committed entry [committed(%d)]", ci, l.committed)
		default:
			offset := index + 1
			l.append(ents[ci-offset:]...)
		}
		l.commitTo(min(committed, lastnewi))
		return lastnewi, true
	}
	return 0, false
}

func (l *raftLog) append(ents ...pb.Entry) uint64 {
	if len(ents) == 0 {
		return l.lastIndex()
	}
	if after := ents[0].Index - 1; after < l.committed {
		l.logger.Panicf("after(%d) is out of range [committed(%d)]", after, l.committed)
	}
	l.unstable.truncateAndAppend(ents)
	return l.lastIndex()
}

// findConflict finds the index of the conflict.
// It returns the first pair of conflicting entries between the existing
// entries and the given entries, if there are any.
// If there is no conflicting entries, and the existing entries contains
// all the given entries, zero will be returned.
// If there is no conflicting entries, but the given entries contains new
// entries, the index of the first new entry will be returned.
// An entry is considered to be conflicting if it has the same index but
// a different term.
// The index of the given entries MUST be continuously increasing.
func (l *raftLog) findConflict(ents []pb.Entry) uint64 {
	for _, ne := range ents {
		if !l.matchTerm(ne.Index, ne.Term) {
			if ne.Index <= l.lastIndex() {
				l.logger.Infof("found conflict at index %d [existing term: %d, conflicting term: %d]",
					ne.Index, l.zeroTermOnErrCompacted(l.term(ne.Index)), ne.Term)
			}
			return ne.Index
		}
	}
	return 0
}

// findConflictByTerm takes an (index, term) pair (indicating a conflicting log
// entry on a leader/follower during an append) and finds the largest index in
// log l with a term <= `term` and an index <= `index`. If no such index exists
// in the log, the log's first index is returned.
//
// The index provided MUST be equal to or less than l.lastIndex(). Invalid
// inputs log a warning and the input index is returned.
func (l *raftLog) findConflictByTerm(index uint64, term uint64) uint64 {
	if li := l.lastIndex(); index > li {
		// NB: such calls should not exist, but since there is a straightfoward
		// way to recover, do it.
		//
		// It is tempting to also check something about the first index, but
		// there is odd behavior with peers that have no log, in which case
		// lastIndex will return zero and firstIndex will return one, which
		// leads to calls with an index of zero into this method.
		l.logger.Warningf("index(%d) is out of range [0, lastIndex(%d)] in findConflictByTerm",
			index, li)
		return index
	}
	for {
		logTerm, err := l.term(index)
		if logTerm <= term || err != nil {
			break
		}
		index--
	}
	return index
}

func (l *raftLog) unstableEntries() []pb.Entry {
	if len(l.unstable.entries) == 0 {
		return nil
	}
	return l.unstable.entries
}

// nextEnts returns all the available entries for execution.
// If applied is smaller than the index of snapshot, it returns all committed
// entries after the index of snapshot.
func (l *raftLog) nextEnts() (ents []pb.Entry) {
	off := max(l.applied+1, l.firstIndex())
	if l.committed+1 > off {
		ents, err := l.slice(off, l.committed+1, l.maxNextEntsSize)
		if err != nil {
			l.logger.Panicf("unexpected error when getting unapplied entries (%v)", err)
		}
		return ents
	}
	return nil
}

// hasNextEnts returns if there is any available entries for execution. This
// is a fast check without heavy raftLog.slice() in raftLog.nextEnts().
func (l *raftLog) hasNextEnts() bool {
	off := max(l.applied+1, l.firstIndex())
	return l.committed+1 > off
}

// hasPendingSnapshot returns if there is pending snapshot waiting for applying.
func (l *raftLog) hasPendingSnapshot() bool {
	return l.unstable.snapshot != nil && !IsEmptySnap(*l.unstable.snapshot)
}

func (l *raftLog) snapshot() (pb.Snapshot, error) {
	if l.unstable.snapshot != nil {
		return *l.unstable.snapshot, nil
	}
	return l.storage.Snapshot()
}

func (l *raftLog) firstIndex() uint64 {
	if i, ok := l.unstable.maybeFirstIndex(); ok {
		return i
	}
	index, err := l.storage.FirstIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	return index
}

func (l *raftLog) lastIndex() uint64 {
	if i, ok := l.unstable.maybeLastIndex(); ok {
		return i
	}
	i, err := l.storage.LastIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	return i
}

func (l *raftLog) commitTo(tocommit uint64) {
	// never decrease commit
	if l.committed < tocommit {
		if l.lastIndex() < tocommit {
			l.logger.Panicf("tocommit(%d) is out of range [lastIndex(%d)]. Was the raft log corrupted, truncated, or lost?", tocommit, l.lastIndex())
		}
		l.committed = tocommit
	}
}

func (l *raftLog) appliedTo(i uint64) {
	if i == 0 {
		return
	}
	if l.committed < i || i < l.applied {
		l.logger.Panicf("applied(%d) is out of range [prevApplied(%d), committed(%d)]", i, l.applied, l.committed)
	}
	l.applied = i
}

func (l *raftLog) stableTo(i, t uint64) { l.unstable.stableTo(i, t) }

func (l *raftLog) stableSnapTo(i uint64) { l.unstable.stableSnapTo(i) }

func (l *raftLog) lastTerm() uint64 {
	t, err := l.term(l.lastIndex())
	if err != nil {
		l.logger.Panicf("unexpected error when getting the last term (%v)", err)
	}
	return t
}

func (l *raftLog) term(i uint64) (uint64, error) {
	// the valid term range is [index of dummy entry, last index]
	dummyIndex := l.firstIndex() - 1
	if i < dummyIndex || i > l.lastIndex() {
		// TODO: return an error instead?
		return 0, nil
	}

	if t, ok := l.unstable.maybeTerm(i); ok {
		return t, nil
	}

	t, err := l.storage.Term(i)
	if err == nil {
		return t, nil
	}
	if err == ErrCompacted || err == ErrUnavailable {
		return 0, err
	}
	panic(err) // TODO(bdarnell)
}

func (l *raftLog) entries(i, maxsize uint64) ([]pb.Entry, error) {
	if i > l.lastIndex() {
		return nil, nil
	}
	return l.slice(i, l.lastIndex()+1, maxsize)
}

// allEntries returns all entries in the log.
func (l *raftLog) allEntries() []pb.Entry {
	ents, err := l.entries(l.firstIndex(), noLimit)
	if err == nil {
		return ents
	}
	if err == ErrCompacted { // try again if there was a racing compaction
		return l.allEntries()
	}
	// TODO (xiangli): handle error?
	panic(err)
}

// isUpToDate determines if the given (lastIndex,term) log is more up-to-date
// by comparing the index and term of the last entries in the existing logs.
// If the logs have last entries with different terms, then the log with the
// later term is more up-to-date. If the logs end with the same term, then
// whichever log has the larger lastIndex is more up-to-date. If the logs are
// the same, the given log is up-to-date.
func (l *raftLog) isUpToDate(lasti, term uint64) bool {
	return term > l.lastTerm() || (term == l.lastTerm() && lasti >= l.lastIndex())
}

func (l *raftLog) matchTerm(i, term uint64) bool {
	t, err := l.term(i)
	if err != nil {
		return false
	}
	return t == term
}

func (l *raftLog) maybeCommit(maxIndex, term uint64) bool {
	if maxIndex > l.committed && l.zeroTermOnErrCompacted(l.term(maxIndex)) == term {
		l.commitTo(maxIndex)
		return true
	}
	return false
}

func (l *raftLog) restore(s pb.Snapshot) {
	l.logger.Infof("log [%s] starts to restore snapshot [index: %d, term: %d]", l, s.Metadata.Index, s.Metadata.Term)
	l.committed = s.Metadata.Index
	l.unstable.restore(s)
}

// slice returns a slice of log entries from lo through hi-1, inclusive.
func (l *raftLog) slice(lo, hi, maxSize uint64) ([]pb.Entry, error) {
	err := l.mustCheckOutOfBounds(lo, hi)
	if err != nil {
		return nil, err
	}
	if lo == hi {
		return nil, nil
	}
	var ents []pb.Entry
	if lo < l.unstable.offset {
		storedEnts, err := l.storage.Entries(lo, min(hi, l.unstable.offset), maxSize)
		if err == ErrCompacted {
			return nil, err
		} else if err == ErrUnavailable {
			l.logger.Panicf("entries[%d:%d) is unavailable from storage", lo, min(hi, l.unstable.offset))
		} else if err != nil {
			panic(err) // TODO(bdarnell)
		}

		// check if ents has reached the size limitation
		if uint64(len(storedEnts)) < min(hi, l.unstable.offset)-lo {
			return storedEnts, nil
		}

		ents = storedEnts
	}
	if hi > l.unstable.offset {
		unstable := l.unstable.slice(max(lo, l.unstable.offset), hi)
		if len(ents) > 0 {
			combined := make([]pb.Entry, len(ents)+len(unstable))
			n := copy(combined, ents)
			copy(combined[n:], unstable)
			ents = combined
		} else {
			ents = unstable
		}
	}
	return limitSize(ents, maxSize), nil
}

// l.firstIndex <= lo <= hi <= l.firstIndex + len(l.entries)
func (l *raftLog) mustCheckOutOfBounds(lo, hi uint64) error {
	if lo > hi {
		l.logger.Panicf("invalid slice %d > %d", lo, hi)
	}
	fi := l.firstIndex()
	if lo < fi {
		return ErrCompacted
	}

	length := l.lastIndex() + 1 - fi
	if hi > fi+length {
		l.logger.Panicf("slice[%d,%d) out of bound [%d,%d]", lo, hi, fi, l.lastIndex())
	}
	return nil
}

func (l *raftLog) zeroTermOnErrCompacted(t uint64, err error) uint64 {
	if err == nil {
		return t
	}
	if err == ErrCompacted {
		return 0
	}
	l.logger.Panicf("unexpected error (%v)", err)
	return 0
}
<commit_msg>raft: fix out-of-bounds in maybeAppend<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package raft

import (
	"fmt"
	"log"

	pb "go.etcd.io/etcd/raft/v3/raftpb"
)

type raftLog struct {
	// storage contains all stable entries since the last snapshot.
	storage Storage

	// unstable contains all unstable entries and snapshot.
	// they will be saved into storage.
	unstable unstable

	// committed is the highest log position that is known to be in
	// stable storage on a quorum of nodes.
	committed uint64
	// applied is the highest log position that the application has
	// been instructed to apply to its state machine.
	// Invariant: applied <= committed
	applied uint64

	logger Logger

	// maxNextEntsSize is the maximum number aggregate byte size of the messages
	// returned from calls to nextEnts.
	maxNextEntsSize uint64
}

// newLog returns log using the given storage and default options. It
// recovers the log to the state that it just commits and applies the
// latest snapshot.
func newLog(storage Storage, logger Logger) *raftLog {
	return newLogWithSize(storage, logger, noLimit)
}

// newLogWithSize returns a log using the given storage and max
// message size.
func newLogWithSize(storage Storage, logger Logger, maxNextEntsSize uint64) *raftLog {
	if storage == nil {
		log.Panic("storage must not be nil")
	}
	log := &raftLog{
		storage:         storage,
		logger:          logger,
		maxNextEntsSize: maxNextEntsSize,
	}
	firstIndex, err := storage.FirstIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	lastIndex, err := storage.LastIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	log.unstable.offset = lastIndex + 1
	log.unstable.logger = logger
	// Initialize our committed and applied pointers to the time of the last compaction.
	log.committed = firstIndex - 1
	log.applied = firstIndex - 1

	return log
}

func (l *raftLog) String() string {
	return fmt.Sprintf("committed=%d, applied=%d, unstable.offset=%d, len(unstable.Entries)=%d", l.committed, l.applied, l.unstable.offset, len(l.unstable.entries))
}

// maybeAppend returns (0, false) if the entries cannot be appended. Otherwise,
// it returns (last index of new entries, true).
func (l *raftLog) maybeAppend(index, logTerm, committed uint64, ents ...pb.Entry) (lastnewi uint64, ok bool) {
	if l.matchTerm(index, logTerm) {
		lastnewi = index + uint64(len(ents))
		ci := l.findConflict(ents)
		switch {
		case ci == 0:
		case ci <= l.committed:
			l.logger.Panicf("entry %d conflict with committed entry [committed(%d)]", ci, l.committed)
		default:
			offset := index + 1
			if ci-offset > uint64(len(ents)) {
				l.logger.Panicf("index, %d, is out of range [%d]", ci-offset, len(ents))
			}
			l.append(ents[ci-offset:]...)
		}
		l.commitTo(min(committed, lastnewi))
		return lastnewi, true
	}
	return 0, false
}

func (l *raftLog) append(ents ...pb.Entry) uint64 {
	if len(ents) == 0 {
		return l.lastIndex()
	}
	if after := ents[0].Index - 1; after < l.committed {
		l.logger.Panicf("after(%d) is out of range [committed(%d)]", after, l.committed)
	}
	l.unstable.truncateAndAppend(ents)
	return l.lastIndex()
}

// findConflict finds the index of the conflict.
// It returns the first pair of conflicting entries between the existing
// entries and the given entries, if there are any.
// If there is no conflicting entries, and the existing entries contains
// all the given entries, zero will be returned.
// If there is no conflicting entries, but the given entries contains new
// entries, the index of the first new entry will be returned.
// An entry is considered to be conflicting if it has the same index but
// a different term.
// The index of the given entries MUST be continuously increasing.
func (l *raftLog) findConflict(ents []pb.Entry) uint64 {
	for _, ne := range ents {
		if !l.matchTerm(ne.Index, ne.Term) {
			if ne.Index <= l.lastIndex() {
				l.logger.Infof("found conflict at index %d [existing term: %d, conflicting term: %d]",
					ne.Index, l.zeroTermOnErrCompacted(l.term(ne.Index)), ne.Term)
			}
			return ne.Index
		}
	}
	return 0
}

// findConflictByTerm takes an (index, term) pair (indicating a conflicting log
// entry on a leader/follower during an append) and finds the largest index in
// log l with a term <= `term` and an index <= `index`. If no such index exists
// in the log, the log's first index is returned.
//
// The index provided MUST be equal to or less than l.lastIndex(). Invalid
// inputs log a warning and the input index is returned.
func (l *raftLog) findConflictByTerm(index uint64, term uint64) uint64 {
	if li := l.lastIndex(); index > li {
		// NB: such calls should not exist, but since there is a straightfoward
		// way to recover, do it.
		//
		// It is tempting to also check something about the first index, but
		// there is odd behavior with peers that have no log, in which case
		// lastIndex will return zero and firstIndex will return one, which
		// leads to calls with an index of zero into this method.
		l.logger.Warningf("index(%d) is out of range [0, lastIndex(%d)] in findConflictByTerm",
			index, li)
		return index
	}
	for {
		logTerm, err := l.term(index)
		if logTerm <= term || err != nil {
			break
		}
		index--
	}
	return index
}

func (l *raftLog) unstableEntries() []pb.Entry {
	if len(l.unstable.entries) == 0 {
		return nil
	}
	return l.unstable.entries
}

// nextEnts returns all the available entries for execution.
// If applied is smaller than the index of snapshot, it returns all committed
// entries after the index of snapshot.
func (l *raftLog) nextEnts() (ents []pb.Entry) {
	off := max(l.applied+1, l.firstIndex())
	if l.committed+1 > off {
		ents, err := l.slice(off, l.committed+1, l.maxNextEntsSize)
		if err != nil {
			l.logger.Panicf("unexpected error when getting unapplied entries (%v)", err)
		}
		return ents
	}
	return nil
}

// hasNextEnts returns if there is any available entries for execution. This
// is a fast check without heavy raftLog.slice() in raftLog.nextEnts().
func (l *raftLog) hasNextEnts() bool {
	off := max(l.applied+1, l.firstIndex())
	return l.committed+1 > off
}

// hasPendingSnapshot returns if there is pending snapshot waiting for applying.
func (l *raftLog) hasPendingSnapshot() bool {
	return l.unstable.snapshot != nil && !IsEmptySnap(*l.unstable.snapshot)
}

func (l *raftLog) snapshot() (pb.Snapshot, error) {
	if l.unstable.snapshot != nil {
		return *l.unstable.snapshot, nil
	}
	return l.storage.Snapshot()
}

func (l *raftLog) firstIndex() uint64 {
	if i, ok := l.unstable.maybeFirstIndex(); ok {
		return i
	}
	index, err := l.storage.FirstIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	return index
}

func (l *raftLog) lastIndex() uint64 {
	if i, ok := l.unstable.maybeLastIndex(); ok {
		return i
	}
	i, err := l.storage.LastIndex()
	if err != nil {
		panic(err) // TODO(bdarnell)
	}
	return i
}

func (l *raftLog) commitTo(tocommit uint64) {
	// never decrease commit
	if l.committed < tocommit {
		if l.lastIndex() < tocommit {
			l.logger.Panicf("tocommit(%d) is out of range [lastIndex(%d)]. Was the raft log corrupted, truncated, or lost?", tocommit, l.lastIndex())
		}
		l.committed = tocommit
	}
}

func (l *raftLog) appliedTo(i uint64) {
	if i == 0 {
		return
	}
	if l.committed < i || i < l.applied {
		l.logger.Panicf("applied(%d) is out of range [prevApplied(%d), committed(%d)]", i, l.applied, l.committed)
	}
	l.applied = i
}

func (l *raftLog) stableTo(i, t uint64) { l.unstable.stableTo(i, t) }

func (l *raftLog) stableSnapTo(i uint64) { l.unstable.stableSnapTo(i) }

func (l *raftLog) lastTerm() uint64 {
	t, err := l.term(l.lastIndex())
	if err != nil {
		l.logger.Panicf("unexpected error when getting the last term (%v)", err)
	}
	return t
}

func (l *raftLog) term(i uint64) (uint64, error) {
	// the valid term range is [index of dummy entry, last index]
	dummyIndex := l.firstIndex() - 1
	if i < dummyIndex || i > l.lastIndex() {
		// TODO: return an error instead?
		return 0, nil
	}

	if t, ok := l.unstable.maybeTerm(i); ok {
		return t, nil
	}

	t, err := l.storage.Term(i)
	if err == nil {
		return t, nil
	}
	if err == ErrCompacted || err == ErrUnavailable {
		return 0, err
	}
	panic(err) // TODO(bdarnell)
}

func (l *raftLog) entries(i, maxsize uint64) ([]pb.Entry, error) {
	if i > l.lastIndex() {
		return nil, nil
	}
	return l.slice(i, l.lastIndex()+1, maxsize)
}

// allEntries returns all entries in the log.
func (l *raftLog) allEntries() []pb.Entry {
	ents, err := l.entries(l.firstIndex(), noLimit)
	if err == nil {
		return ents
	}
	if err == ErrCompacted { // try again if there was a racing compaction
		return l.allEntries()
	}
	// TODO (xiangli): handle error?
	panic(err)
}

// isUpToDate determines if the given (lastIndex,term) log is more up-to-date
// by comparing the index and term of the last entries in the existing logs.
// If the logs have last entries with different terms, then the log with the
// later term is more up-to-date. If the logs end with the same term, then
// whichever log has the larger lastIndex is more up-to-date. If the logs are
// the same, the given log is up-to-date.
func (l *raftLog) isUpToDate(lasti, term uint64) bool {
	return term > l.lastTerm() || (term == l.lastTerm() && lasti >= l.lastIndex())
}

func (l *raftLog) matchTerm(i, term uint64) bool {
	t, err := l.term(i)
	if err != nil {
		return false
	}
	return t == term
}

func (l *raftLog) maybeCommit(maxIndex, term uint64) bool {
	if maxIndex > l.committed && l.zeroTermOnErrCompacted(l.term(maxIndex)) == term {
		l.commitTo(maxIndex)
		return true
	}
	return false
}

func (l *raftLog) restore(s pb.Snapshot) {
	l.logger.Infof("log [%s] starts to restore snapshot [index: %d, term: %d]", l, s.Metadata.Index, s.Metadata.Term)
	l.committed = s.Metadata.Index
	l.unstable.restore(s)
}

// slice returns a slice of log entries from lo through hi-1, inclusive.
func (l *raftLog) slice(lo, hi, maxSize uint64) ([]pb.Entry, error) {
	err := l.mustCheckOutOfBounds(lo, hi)
	if err != nil {
		return nil, err
	}
	if lo == hi {
		return nil, nil
	}
	var ents []pb.Entry
	if lo < l.unstable.offset {
		storedEnts, err := l.storage.Entries(lo, min(hi, l.unstable.offset), maxSize)
		if err == ErrCompacted {
			return nil, err
		} else if err == ErrUnavailable {
			l.logger.Panicf("entries[%d:%d) is unavailable from storage", lo, min(hi, l.unstable.offset))
		} else if err != nil {
			panic(err) // TODO(bdarnell)
		}

		// check if ents has reached the size limitation
		if uint64(len(storedEnts)) < min(hi, l.unstable.offset)-lo {
			return storedEnts, nil
		}

		ents = storedEnts
	}
	if hi > l.unstable.offset {
		unstable := l.unstable.slice(max(lo, l.unstable.offset), hi)
		if len(ents) > 0 {
			combined := make([]pb.Entry, len(ents)+len(unstable))
			n := copy(combined, ents)
			copy(combined[n:], unstable)
			ents = combined
		} else {
			ents = unstable
		}
	}
	return limitSize(ents, maxSize), nil
}

// l.firstIndex <= lo <= hi <= l.firstIndex + len(l.entries)
func (l *raftLog) mustCheckOutOfBounds(lo, hi uint64) error {
	if lo > hi {
		l.logger.Panicf("invalid slice %d > %d", lo, hi)
	}
	fi := l.firstIndex()
	if lo < fi {
		return ErrCompacted
	}

	length := l.lastIndex() + 1 - fi
	if hi > fi+length {
		l.logger.Panicf("slice[%d,%d) out of bound [%d,%d]", lo, hi, fi, l.lastIndex())
	}
	return nil
}

func (l *raftLog) zeroTermOnErrCompacted(t uint64, err error) uint64 {
	if err == nil {
		return t
	}
	if err == ErrCompacted {
		return 0
	}
	l.logger.Panicf("unexpected error (%v)", err)
	return 0
}
<|endoftext|>