<commit_before>package run

import (
	"encoding/binary"
	"fmt"
	"io"
)

type readWriteKiller struct {
	io.Reader
	io.Writer
	kill func() error
}

type originRead struct {
	err error
	ev  []byte
}

type opCode uint16

const (
	opCodeNone   = opCode(0)
	opCodeOrigin = opCode(1)
)

type opFlags uint16

const (
	opFlagPollout = opFlags(0x1)

	opFlagsMask = opFlagPollout
)

type opPacket struct {
	err     error
	payload []byte
	code    opCode
	flags   opFlags
}

const (
	evCodePollout = uint16(0)
	evCodeOrigin  = uint16(1)
)

func ioLoop(origin io.ReadWriter, subject readWriteKiller) (err error) {
	reads := originLoop(origin)
	defer func() {
		go func() {
			for range reads {
			}
		}()
	}()

	ops := opLoop(subject)
	defer func() {
		for range ops {
		}
	}()

	evs, evErr := evLoop(subject)
	defer func() {
		for range evErr {
		}
	}()
	defer close(evs)

	defer subject.kill()

	var ev []byte

	for {
		var (
			doReads <-chan originRead
			doOps   <-chan opPacket
			doEvs   chan<- []byte
		)

		if ev == nil {
			doReads = reads
			doOps = ops
		} else {
			doEvs = evs
		}

		select {
		case read, ok := <-doReads:
			if !ok {
				reads = nil
				break
			}

			err = read.err
			if err != nil {
				return
			}

			ev = read.ev

		case op, ok := <-doOps:
			if !ok {
				return
			}

			ev, err = handleOp(op, origin)
			if err != nil {
				return
			}

		case doEvs <- ev:
			ev = nil

		case err = <-evErr:
			return
		}
	}
}

func originLoop(r io.Reader) <-chan originRead {
	reads := make(chan originRead)

	go func() {
		defer close(reads)

		for {
			buf := make([]byte, maxPacketSize)
			n, err := r.Read(buf[8:])
			buf = buf[:8+n]
			nativeEndian.PutUint32(buf[0:], uint32(len(buf)))
			nativeEndian.PutUint16(buf[4:], evCodeOrigin)

			reads <- originRead{
				ev: buf,
			}

			if err != nil {
				if err != io.EOF {
					reads <- originRead{
						err: err,
					}
				} else if n != 0 {
					buf = buf[:8]
					nativeEndian.PutUint32(buf[0:], 8)
					nativeEndian.PutUint16(buf[4:], evCodeOrigin)

					reads <- originRead{
						ev: buf,
					}
				}
				return
			}
		}
	}()

	return reads
}

func opLoop(r io.Reader) <-chan opPacket {
	ops := make(chan opPacket)

	go func() {
		defer close(ops)

		for {
			var header struct {
				Size  uint32
				Code  uint16
				Flags uint16
			}

			err := binary.Read(r, nativeEndian, &header)
			if err != nil {
				if err != io.EOF {
					ops <- opPacket{
						err: fmt.Errorf("op read: %v", err),
					}
				}
				return
			}

			if header.Size < 8 || header.Size > maxPacketSize {
				ops <- opPacket{
					err: fmt.Errorf("invalid op packet size: %d", header.Size),
				}
				return
			}

			payload := make([]byte, header.Size-8)

			_, err = io.ReadFull(r, payload)
			if err != nil {
				ops <- opPacket{
					err: fmt.Errorf("op read: %v", err),
				}
				return
			}

			ops <- opPacket{
				payload: payload,
				code:    opCode(header.Code),
				flags:   opFlags(header.Flags),
			}
		}
	}()

	return ops
}

func evLoop(w io.Writer) (chan<- []byte, <-chan error) {
	evs := make(chan []byte)
	done := make(chan error)

	go func() {
		defer close(done)

		for buf := range evs {
			_, err := w.Write(buf)
			if err != nil {
				done <- fmt.Errorf("ev write: %v", err)
				return
			}
		}
	}()

	return evs, done
}

func handleOp(op opPacket, origin io.ReadWriter) (ev []byte, err error) {
	err = op.err
	if err != nil {
		err = fmt.Errorf("ev write: %v", err)
		return
	}

	if (op.flags &^ opFlagsMask) != 0 {
		err = fmt.Errorf("invalid op packet flags: 0x%x", op.flags)
		return
	}

	switch op.code {
	case opCodeNone:

	case opCodeOrigin:
		_, err = origin.Write(op.payload)
		if err != nil {
			err = fmt.Errorf("ev write: %v", err)
			return
		}

	default:
		err = fmt.Errorf("invalid op packet code: %d", op.code)
		return
	}

	if (op.flags & opFlagPollout) != 0 {
		buf := make([]byte, 8)
		nativeEndian.PutUint32(buf[0:], 8)
		nativeEndian.PutUint16(buf[4:], evCodePollout)
		ev = buf
	}

	return
}
<commit_msg>run: don't fail when event pipe breaks<commit_after>package run

import (
	"encoding/binary"
	"fmt"
	"io"
)

type readWriteKiller struct {
	io.Reader
	io.Writer
	kill func() error
}

type originRead struct {
	err error
	ev  []byte
}

type opCode uint16

const (
	opCodeNone   = opCode(0)
	opCodeOrigin = opCode(1)
)

type opFlags uint16

const (
	opFlagPollout = opFlags(0x1)

	opFlagsMask = opFlagPollout
)

type opPacket struct {
	err     error
	payload []byte
	code    opCode
	flags   opFlags
}

const (
	evCodePollout = uint16(0)
	evCodeOrigin  = uint16(1)
)

func ioLoop(origin io.ReadWriter, subject readWriteKiller) (err error) {
	reads := originLoop(origin)
	defer func() {
		go func() {
			for range reads {
			}
		}()
	}()

	ops := opLoop(subject)
	defer func() {
		for range ops {
		}
	}()

	evs, evDone := evLoop(subject)
	defer func() {
		<-evDone
	}()
	defer close(evs)

	defer subject.kill()

	var ev []byte

	for {
		var (
			doReads <-chan originRead
			doOps   <-chan opPacket
			doEvs   chan<- []byte
		)

		if ev == nil {
			doReads = reads
			doOps = ops
		} else {
			doEvs = evs
		}

		select {
		case read, ok := <-doReads:
			if !ok {
				reads = nil
				break
			}

			err = read.err
			if err != nil {
				return
			}

			ev = read.ev

		case op, ok := <-doOps:
			if !ok {
				return
			}

			ev, err = handleOp(op, origin)
			if err != nil {
				return
			}

		case doEvs <- ev:
			ev = nil

		case <-evDone:
			return
		}
	}
}

func originLoop(r io.Reader) <-chan originRead {
	reads := make(chan originRead)

	go func() {
		defer close(reads)

		for {
			buf := make([]byte, maxPacketSize)
			n, err := r.Read(buf[8:])
			buf = buf[:8+n]
			nativeEndian.PutUint32(buf[0:], uint32(len(buf)))
			nativeEndian.PutUint16(buf[4:], evCodeOrigin)

			reads <- originRead{
				ev: buf,
			}

			if err != nil {
				if err != io.EOF {
					reads <- originRead{
						err: err,
					}
				} else if n != 0 {
					buf = buf[:8]
					nativeEndian.PutUint32(buf[0:], 8)
					nativeEndian.PutUint16(buf[4:], evCodeOrigin)

					reads <- originRead{
						ev: buf,
					}
				}
				return
			}
		}
	}()

	return reads
}

func opLoop(r io.Reader) <-chan opPacket {
	ops := make(chan opPacket)

	go func() {
		defer close(ops)

		for {
			var header struct {
				Size  uint32
				Code  uint16
				Flags uint16
			}

			err := binary.Read(r, nativeEndian, &header)
			if err != nil {
				if err != io.EOF {
					ops <- opPacket{
						err: fmt.Errorf("op read: %v", err),
					}
				}
				return
			}

			if header.Size < 8 || header.Size > maxPacketSize {
				ops <- opPacket{
					err: fmt.Errorf("invalid op packet size: %d", header.Size),
				}
				return
			}

			payload := make([]byte, header.Size-8)

			_, err = io.ReadFull(r, payload)
			if err != nil {
				ops <- opPacket{
					err: fmt.Errorf("op read: %v", err),
				}
				return
			}

			ops <- opPacket{
				payload: payload,
				code:    opCode(header.Code),
				flags:   opFlags(header.Flags),
			}
		}
	}()

	return ops
}

func evLoop(w io.Writer) (chan<- []byte, <-chan struct{}) {
	evs := make(chan []byte)
	done := make(chan struct{})

	go func() {
		defer close(done)

		for buf := range evs {
			if _, err := w.Write(buf); err != nil {
				return
			}
		}
	}()

	return evs, done
}

func handleOp(op opPacket, origin io.ReadWriter) (ev []byte, err error) {
	err = op.err
	if err != nil {
		err = fmt.Errorf("handleOp: %v", err)
		return
	}

	if (op.flags &^ opFlagsMask) != 0 {
		err = fmt.Errorf("invalid op packet flags: 0x%x", op.flags)
		return
	}

	switch op.code {
	case opCodeNone:

	case opCodeOrigin:
		_, err = origin.Write(op.payload)
		if err != nil {
			err = fmt.Errorf("handleOp: write to origin: %v", err)
			return
		}

	default:
		err = fmt.Errorf("invalid op packet code: %d", op.code)
		return
	}

	if (op.flags & opFlagPollout) != 0 {
		buf := make([]byte, 8)
		nativeEndian.PutUint32(buf[0:], 8)
		nativeEndian.PutUint16(buf[4:], evCodePollout)
		ev = buf
	}

	return
}
<|endoftext|>


<commit_before>package gsrunner

import (
	"bufio"
	"errors"
	"flag"
	"io"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/gsdocker/gsconfig"
	"github.com/gsdocker/gserrors"
	"github.com/gsdocker/gslogger"
	"github.com/gsrpc/gorpc"
)

// Error list
var (
	ErrFlag     = errors.New("flag error")
	ErrConfig   = errors.New("config load error")
	ErrRegistry = errors.New("registry file error")
)

// Runner gsdocker services runner
type Runner interface {
	gslogger.Log
	FlagString(name string, fullname string, defaultVal string, description string) Runner
	FlagInt(name string, fullname string, defaultVal int64, description string) Runner
	FlagUint(name string, fullname string, defaultVal uint64, description string) Runner
	FlagFloat32(name string, fullname string, defaultVal float32, description string) Runner
	FlagFloat64(name string, fullname string, defaultVal float64, description string) Runner
	Seconds(name string, fullname string, defaultVal uint64, description string) Runner
	Milliseconds(name string, fullname string, defaultVal uint64, description string) Runner
	Run(main func(runner Runner))
}

type _Runner struct {
	gslogger.Log
	flagString       map[string]*string
	flagInt          map[string]*int64
	flagUint         map[string]*uint64
	flagFloat32      map[string]*float64
	flagFloat64      map[string]*float64
	flagSecond       map[string]*uint64
	flagMilliseconds map[string]*uint64
	fullname         map[string]string
}

// New create new gsdocker runner
func New(name string) Runner {
	runner := &_Runner{
		Log:              gslogger.Get(name),
		flagString:       make(map[string]*string),
		flagInt:          make(map[string]*int64),
		flagUint:         make(map[string]*uint64),
		flagFloat32:      make(map[string]*float64),
		flagFloat64:      make(map[string]*float64),
		flagSecond:       make(map[string]*uint64),
		flagMilliseconds: make(map[string]*uint64),
		fullname:         make(map[string]string),
	}

	runner.FlagString(
		"log", "gsrunner.log", "", "the gsrunner log root path",
	).FlagString(
		"level", "gsrunner.log.level", "", "the gsrunner log level",
	).FlagString(
		"pprof", "gsrunner.pprof", "", "set gsrunner pprof listen address",
	).FlagString(
		"config", "gsrunner.config", "", "set gsrunner config file",
	).FlagString(
		"registry", "gsrunner.registry", "", "set the rpc services registry file",
	)

	return runner
}

func (runner *_Runner) checkName(name, fullname string) {
	if _, ok := runner.fullname[name]; ok {
		gserrors.Panicf(ErrFlag, "duplicate flag name :%s", name)
	}

	for _, v := range runner.fullname {
		if v == fullname {
			gserrors.Panicf(ErrFlag, "duplicate flag fullname :%s", fullname)
		}
	}

	runner.fullname[name] = fullname
}

func (runner *_Runner) FlagString(name string, fullname string, defaultVal string, description string) Runner {

	runner.checkName(name, fullname)

	runner.flagString[name] = flag.String(name, defaultVal, description)

	return runner
}

func (runner *_Runner) FlagInt(name string, fullname string, defaultVal int64, description string) Runner {
	runner.checkName(name, fullname)

	runner.flagInt[name] = flag.Int64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) FlagUint(name string, fullname string, defaultVal uint64, description string) Runner {
	runner.checkName(name, fullname)
	runner.flagUint[name] = flag.Uint64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) FlagFloat32(name string, fullname string, defaultVal float32, description string) Runner {

	runner.checkName(name, fullname)

	runner.flagFloat32[name] = flag.Float64(name, float64(defaultVal), description)

	return runner

}

func (runner *_Runner) FlagFloat64(name string, fullname string, defaultVal float64, description string) Runner {
	runner.checkName(name, fullname)
	runner.flagFloat64[name] = flag.Float64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) Seconds(name string, fullname string, defaultVal uint64, description string) Runner {
	runner.checkName(name, fullname)
	runner.flagSecond[name] = flag.Uint64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) Milliseconds(name string, fullname string, defaultVal uint64, description string) Runner {
	runner.checkName(name, fullname)

	runner.flagMilliseconds[name] = flag.Uint64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) Run(main func(runner Runner)) {
	defer func() {
		if e := recover(); e != nil {
			runner.E("catch unknown exception\n\t%s", e)
		}

		runner.I("service stopped.")

		gslogger.Join()
	}()

	flag.Parse()

	configpath := *runner.flagString["config"]

	runner.D("config file path :%s", configpath)

	if configpath != "" {
		switch filepath.Ext(configpath) {
		case ".json":
			if err := gsconfig.LoadJSON(configpath); err != nil {
				gserrors.Panicf(ErrConfig, "load config file error :%s", configpath)
			}
		default:
			runner.W("can't load config file :%s", configpath)
		}
	}

	for k, v := range runner.flagString {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagInt {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagUint {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagFloat32 {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagFloat64 {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagSecond {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagMilliseconds {
		gsconfig.Update(runner.fullname[k], *v)
	}

	logroot := gsconfig.String("gsrunner.log", "")
	runner.D("log root path :%s", logroot)
	loglevel := gsconfig.String("gsrunner.log.level", "")
	runner.D("log level :%s", loglevel)
	registryFile := gsconfig.String("gsrunner.registry", "")
	runner.D("registry file:%s", registryFile)

	if logroot != "" {

		fullpath, _ := filepath.Abs(logroot)

		dir := filepath.Dir(fullpath)

		name := filepath.Base(fullpath)

		gslogger.SetLogDir(dir)

		gslogger.NewSink(gslogger.NewFilelog("gschat-mailhub", name, gsconfig.Int64("gschat.mailhub.log.maxsize", 0)))
	}

	if loglevel != "" {
		gslogger.NewFlags(gslogger.ParseLevel(loglevel))
	}

	if registryFile != "" {

		runner.I("load gsrpc services registry file :%s", registryFile)

		file, err := os.Open(registryFile)

		if err != nil {
			gserrors.Panicf(err, "open registry file error :%s", registryFile)
		}

		runner.loadRegistry(file, registryFile)

		runner.I("load gsrpc services registry file :%s -- success", registryFile)
	}

	runner.I("service started.")

	main(runner)
}

var registryRegex = regexp.MustCompile(`^(?P<name>[A-Za-z0-9_](\.[A-Za-z0-9_])+)=(?P<id>[0-9]+)$`)

func (runner *_Runner) loadRegistry(file io.Reader, path string) {

	reader := bufio.NewReader(file)

	lines := 0

	items := make(map[string]uint16)

	for {
		line, err := reader.ReadString('\n')

		if err != nil {
			if err == io.EOF {
				break
			}

			gserrors.Panicf(err, "read registry file error :%s", path)
		}

		tokens := registryRegex.FindStringSubmatch(line)

		if tokens == nil {
			gserrors.Panicf(ErrRegistry, "load registry file error:\n\tinvalid format\n\t%s(%d)", path, lines)
		}

		val, err := strconv.ParseInt(tokens[1], 0, 32)

		if err != nil {
			gserrors.Panicf(err, "load registry file error:\n\tinvalid format\n\t%s(%d)", path, lines)
		}

		if val > math.MaxUint16 {
			gserrors.Panicf(ErrRegistry, "load registry file error:\n\tid out of range\n\t%s(%d)", path, lines)
		}

		items[tokens[0]] = uint16(val)

		lines++

	}

	gorpc.RegistryUpdate(items)
}
<commit_msg>remove private registry file load codes<commit_after>package gsrunner

import (
	"errors"
	"flag"
	"os"
	"path/filepath"

	"github.com/gsdocker/gsconfig"
	"github.com/gsdocker/gserrors"
	"github.com/gsdocker/gslogger"
	"github.com/gsrpc/gorpc"
)

// Error list
var (
	ErrFlag     = errors.New("flag error")
	ErrConfig   = errors.New("config load error")
	ErrRegistry = errors.New("load registry file error")
)

// Runner gsdocker services runner
type Runner interface {
	gslogger.Log
	FlagString(name string, fullname string, defaultVal string, description string) Runner
	FlagInt(name string, fullname string, defaultVal int64, description string) Runner
	FlagUint(name string, fullname string, defaultVal uint64, description string) Runner
	FlagFloat32(name string, fullname string, defaultVal float32, description string) Runner
	FlagFloat64(name string, fullname string, defaultVal float64, description string) Runner
	Seconds(name string, fullname string, defaultVal uint64, description string) Runner
	Milliseconds(name string, fullname string, defaultVal uint64, description string) Runner
	Run(main func(runner Runner))
}

type _Runner struct {
	gslogger.Log
	flagString       map[string]*string
	flagInt          map[string]*int64
	flagUint         map[string]*uint64
	flagFloat32      map[string]*float64
	flagFloat64      map[string]*float64
	flagSecond       map[string]*uint64
	flagMilliseconds map[string]*uint64
	fullname         map[string]string
}

// New create new gsdocker runner
func New(name string) Runner {
	runner := &_Runner{
		Log:              gslogger.Get(name),
		flagString:       make(map[string]*string),
		flagInt:          make(map[string]*int64),
		flagUint:         make(map[string]*uint64),
		flagFloat32:      make(map[string]*float64),
		flagFloat64:      make(map[string]*float64),
		flagSecond:       make(map[string]*uint64),
		flagMilliseconds: make(map[string]*uint64),
		fullname:         make(map[string]string),
	}

	runner.FlagString(
		"log", "gsrunner.log", "", "the gsrunner log root path",
	).FlagString(
		"level", "gsrunner.log.level", "", "the gsrunner log level",
	).FlagString(
		"pprof", "gsrunner.pprof", "", "set gsrunner pprof listen address",
	).FlagString(
		"config", "gsrunner.config", "", "set gsrunner config file",
	).FlagString(
		"registry", "gsrunner.registry", "", "set the rpc services registry file",
	)

	return runner
}

func (runner *_Runner) checkName(name, fullname string) {
	if _, ok := runner.fullname[name]; ok {
		gserrors.Panicf(ErrFlag, "duplicate flag name :%s", name)
	}

	for _, v := range runner.fullname {
		if v == fullname {
			gserrors.Panicf(ErrFlag, "duplicate flag fullname :%s", fullname)
		}
	}

	runner.fullname[name] = fullname
}

func (runner *_Runner) FlagString(name string, fullname string, defaultVal string, description string) Runner {

	runner.checkName(name, fullname)

	runner.flagString[name] = flag.String(name, defaultVal, description)

	return runner
}

func (runner *_Runner) FlagInt(name string, fullname string, defaultVal int64, description string) Runner {
	runner.checkName(name, fullname)

	runner.flagInt[name] = flag.Int64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) FlagUint(name string, fullname string, defaultVal uint64, description string) Runner {
	runner.checkName(name, fullname)
	runner.flagUint[name] = flag.Uint64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) FlagFloat32(name string, fullname string, defaultVal float32, description string) Runner {

	runner.checkName(name, fullname)

	runner.flagFloat32[name] = flag.Float64(name, float64(defaultVal), description)

	return runner

}

func (runner *_Runner) FlagFloat64(name string, fullname string, defaultVal float64, description string) Runner {
	runner.checkName(name, fullname)
	runner.flagFloat64[name] = flag.Float64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) Seconds(name string, fullname string, defaultVal uint64, description string) Runner {
	runner.checkName(name, fullname)
	runner.flagSecond[name] = flag.Uint64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) Milliseconds(name string, fullname string, defaultVal uint64, description string) Runner {
	runner.checkName(name, fullname)

	runner.flagMilliseconds[name] = flag.Uint64(name, defaultVal, description)

	return runner
}

func (runner *_Runner) Run(main func(runner Runner)) {
	defer func() {
		if e := recover(); e != nil {
			runner.E("catch unknown exception\n\t%s", e)
		}

		runner.I("service stopped.")

		gslogger.Join()
	}()

	flag.Parse()

	configpath := *runner.flagString["config"]

	runner.D("config file path :%s", configpath)

	if configpath != "" {
		switch filepath.Ext(configpath) {
		case ".json":
			if err := gsconfig.LoadJSON(configpath); err != nil {
				gserrors.Panicf(ErrConfig, "load config file error :%s", configpath)
			}
		default:
			runner.W("can't load config file :%s", configpath)
		}
	}

	for k, v := range runner.flagString {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagInt {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagUint {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagFloat32 {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagFloat64 {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagSecond {
		gsconfig.Update(runner.fullname[k], *v)
	}
	for k, v := range runner.flagMilliseconds {
		gsconfig.Update(runner.fullname[k], *v)
	}

	logroot := gsconfig.String("gsrunner.log", "")
	runner.D("log root path :%s", logroot)
	loglevel := gsconfig.String("gsrunner.log.level", "")
	runner.D("log level :%s", loglevel)
	registryFile := gsconfig.String("gsrunner.registry", "")
	runner.D("registry file:%s", registryFile)

	if logroot != "" {

		fullpath, _ := filepath.Abs(logroot)

		dir := filepath.Dir(fullpath)

		name := filepath.Base(fullpath)

		gslogger.SetLogDir(dir)

		gslogger.NewSink(gslogger.NewFilelog("gschat-mailhub", name, gsconfig.Int64("gschat.mailhub.log.maxsize", 0)))
	}

	if loglevel != "" {
		gslogger.NewFlags(gslogger.ParseLevel(loglevel))
	}

	if registryFile != "" {

		runner.I("load gsrpc services registry file :%s", registryFile)

		file, err := os.Open(registryFile)

		if err != nil {
			gserrors.Panicf(err, "open registry file error :%s", registryFile)
		}

		gorpc.RegistryLoad(file, registryFile)

		runner.I("load gsrpc services registry file :%s -- success", registryFile)
	}

	runner.I("service started.")

	main(runner)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	_ "io"
	"io/ioutil"
	"os"
	"time"

	"andiDB/storage"

	"gopkg.in/mgo.v2/bson"
)

var (
	delay = 59

	fileNamePrefix = "dump.db"
)

// AsyncDumpDb periodically dump db to bson file
func AsyncDumpDb() {
	for t := range time.Tick(time.Duration(delay) * time.Second) {
		_ = t
		fmt.Println("will dump..")
		go dump()
	}
}

// dump databases to fileNamePrefix
func dump() {
	dbs := storage.AllDb()
	fmt.Println(dbs)
	d, err := serializeToBson(dbs...)
	if err != nil {
		fmt.Println(err)
		return
	}
	writeToFile(d...)
	LoadFile()
}

// serializeToBson convert json/struct data to BSON
func serializeToBson(databases ...interface{}) ([]byte, error) {
	data, err := bson.Marshal(databases)
	if err != nil {
		fmt.Println(err)
		var b []byte
		return b, err
	}
	fmt.Printf("%q", data)
	return data, nil
}

// writeToFile attempt to write byteArr to file
func writeToFile(p ...byte) {

	fmt.Println("writing to: ", fileNamePrefix)

	f, err := os.Create(fileNamePrefix)
	if err != nil {

		// do better error handling here
		// if cant save file
		fmt.Println(err)
		return
	}

	defer func() {
		if err := f.Close(); err != nil {
			fmt.Println(err)
			panic(err)
		}
	}()

	//var i io.Writer
	n, err := f.Write(p)
	if err != nil {
		fmt.Println(n, err)
	}
	fmt.Printf("wrote %d bytes: ", n)
}

// LoadFile load database file
func LoadFile() {
	b, err := ioutil.ReadFile(fileNamePrefix)
	if err != nil {
		panic(err)
	}
	fmt.Println(b)
}

// func LoadFile() {
// 	fi, err := os.Open(fileNamePrefix)
// 	if err != nil {
// 		fmt.Println(err)
// 		return
// 	}
//
// 	defer func() {
// 		if err := fi.Close(); err != nil {
// 			fmt.Println(err)
// 			panic(err)
// 		}
// 	}()
//
// 	buf := make([]byte, 1024)
// 	for {
// 		n, err := fi.Read(buf)
// 		if err != nil && err != io.EOF {
// 			fmt.Println(err)
// 			return
// 		}
// 		fmt.Println(n)
// 	}
//
// }
<commit_msg>update file reader for BSON objects, to convert to BSON objects<commit_after>package main

import (
	"fmt"
	_ "io"
	"io/ioutil"
	"os"
	"time"

	"andiDB/storage"

	"gopkg.in/mgo.v2/bson"
)

var (
	delay = 59

	fileNamePrefix = "dump.db"
)

// AsyncDumpDb periodically dump db to bson file
func AsyncDumpDb() {
	for t := range time.Tick(time.Duration(delay) * time.Second) {
		_ = t
		fmt.Println("will dump..")
		go dump()
	}
}

// dump databases to fileNamePrefix
func dump() {
	dbs := storage.AllDb()
	fmt.Println(dbs)
	d, err := serializeToBson(dbs...)
	if err != nil {
		fmt.Println(err)
		return
	}
	writeToFile(d...)
	LoadFile()
}

// serializeToBson convert json/struct data to BSON
func serializeToBson(databases ...interface{}) ([]byte, error) {
	data, err := bson.Marshal(databases)
	if err != nil {
		fmt.Println(err)
		var b []byte
		return b, err
	}
	fmt.Printf("%q", data)
	return data, nil
}

// writeToFile attempt to write byteArr to file
func writeToFile(p ...byte) {

	fmt.Println("writing to: ", fileNamePrefix)

	f, err := os.Create(fileNamePrefix)
	if err != nil {

		// do better error handling here
		// if cant save file
		fmt.Println(err)
		return
	}

	defer func() {
		if err := f.Close(); err != nil {
			fmt.Println(err)
			panic(err)
		}
	}()

	//var i io.Writer
	n, err := f.Write(p)
	if err != nil {
		fmt.Println(n, err)
	}
	fmt.Printf("wrote %d bytes: ", n)
}

// LoadFile load database file
func LoadFile() {
	b, err := ioutil.ReadFile(fileNamePrefix)
	if err != nil {
		panic(err)
	}

	fmt.Println("files read..")
	var s interface{}
	err = bson.Unmarshal(b, &s)
	if err != nil {
		fmt.Println(err)
		return
	}

	// for the meantime print the result
	fmt.Println(s)
}
<|endoftext|>


<commit_before>package tasks

import (
	"context"
	"time"

	"golang.org/x/crypto/bcrypt"

	"pixur.org/pixur/be/schema"
	"pixur.org/pixur/be/schema/db"
	tab "pixur.org/pixur/be/schema/tables"
	"pixur.org/pixur/be/status"
)

type CreateUserTask struct {
	// Deps
	DB  db.DB
	Now func() time.Time

	// Inputs
	Ident  string
	Secret string
	// Special input that overrides the defaults.  Used for site bootstrapping.
	Capability []schema.User_Capability

	// Results
	CreatedUser *schema.User
}

func requireCapability(ctx context.Context, j *tab.Job, caps ...schema.User_Capability) (
	*schema.User, status.S) {
	var u *schema.User
	userID, userIDPresent := UserIDFromCtx(ctx)
	if userIDPresent {
		users, err := j.FindUsers(db.Opts{
			Prefix: tab.UsersPrimary{&userID},
			Lock:   db.LockNone,
		})
		if err != nil {
			return nil, status.InternalError(err, "can't lookup user")
		}
		if len(users) != 1 {
			return nil, status.Unauthenticated(nil, "can't lookup user")
		}
		u = users[0]
	} else {
		u = schema.AnonymousUser
	}
	// TODO: make sure sorted.
	for _, c := range caps {
		if !schema.UserHasPerm(u, c) {
			if !userIDPresent {
				return nil, status.Unauthenticatedf(nil, "unauthenticated user missing cap %v", c)
			}
			return u, status.PermissionDeniedf(nil, "missing cap %v", c)
		}
	}

	return u, nil
}

func (t *CreateUserTask) Run(ctx context.Context) (errCap status.S) {
	var err error
	j, err := tab.NewJob(ctx, t.DB)
	if err != nil {
		return status.InternalError(err, "can't create job")
	}
	defer cleanUp(j, &errCap)

	if _, sts := requireCapability(ctx, j, schema.User_USER_CREATE); sts != nil {
		return sts
	}

	if t.Ident == "" || t.Secret == "" {
		return status.InvalidArgument(nil, "missing ident or secret")
	}

	userID, err := j.AllocID()
	if err != nil {
		return status.InternalError(err, "can't allocate id")
	}

	// TODO: rate limit this.
	hashed, err := bcrypt.GenerateFromPassword([]byte(t.Secret), bcrypt.DefaultCost)
	if err != nil {
		return status.InternalError(err, "can't generate password")
	}

	var newcap []schema.User_Capability
	if len(t.Capability) != 0 {
		newcap = t.Capability
	} else {
		newcap = schema.UserNewCap
	}

	now := t.Now()
	user := &schema.User{
		UserId: userID,
		Secret: hashed,
		// Don't set last seen.
		Ident:      t.Ident,
		Capability: newcap,
	}
	user.SetCreatedTime(now)
	user.SetModifiedTime(now)

	if err := j.InsertUser(user); err != nil {
		return status.InternalError(err, "can't create user")
	}

	if err := j.Commit(); err != nil {
		return status.InternalError(err, "can't commit job")
	}

	t.CreatedUser = user

	return nil
}
<commit_msg>be/tasks: accept zero length cap override when creating user<commit_after>package tasks

import (
	"context"
	"time"

	"golang.org/x/crypto/bcrypt"

	"pixur.org/pixur/be/schema"
	"pixur.org/pixur/be/schema/db"
	tab "pixur.org/pixur/be/schema/tables"
	"pixur.org/pixur/be/status"
)

type CreateUserTask struct {
	// Deps
	DB  db.DB
	Now func() time.Time

	// Inputs
	Ident  string
	Secret string
	// Special input that overrides the defaults.  Used for site bootstrapping.
	Capability []schema.User_Capability

	// Results
	CreatedUser *schema.User
}

func requireCapability(ctx context.Context, j *tab.Job, caps ...schema.User_Capability) (
	*schema.User, status.S) {
	var u *schema.User
	userID, userIDPresent := UserIDFromCtx(ctx)
	if userIDPresent {
		users, err := j.FindUsers(db.Opts{
			Prefix: tab.UsersPrimary{&userID},
			Lock:   db.LockNone,
		})
		if err != nil {
			return nil, status.InternalError(err, "can't lookup user")
		}
		if len(users) != 1 {
			return nil, status.Unauthenticated(nil, "can't lookup user")
		}
		u = users[0]
	} else {
		u = schema.AnonymousUser
	}
	// TODO: make sure sorted.
	for _, c := range caps {
		if !schema.UserHasPerm(u, c) {
			if !userIDPresent {
				return nil, status.Unauthenticatedf(nil, "unauthenticated user missing cap %v", c)
			}
			return u, status.PermissionDeniedf(nil, "missing cap %v", c)
		}
	}

	return u, nil
}

func (t *CreateUserTask) Run(ctx context.Context) (errCap status.S) {
	var err error
	j, err := tab.NewJob(ctx, t.DB)
	if err != nil {
		return status.InternalError(err, "can't create job")
	}
	defer cleanUp(j, &errCap)

	if _, sts := requireCapability(ctx, j, schema.User_USER_CREATE); sts != nil {
		return sts
	}

	if t.Ident == "" || t.Secret == "" {
		return status.InvalidArgument(nil, "missing ident or secret")
	}

	userID, err := j.AllocID()
	if err != nil {
		return status.InternalError(err, "can't allocate id")
	}

	// TODO: rate limit this.
	hashed, err := bcrypt.GenerateFromPassword([]byte(t.Secret), bcrypt.DefaultCost)
	if err != nil {
		return status.InternalError(err, "can't generate password")
	}

	var newcap []schema.User_Capability
	// don't use len() == 0, because it may explicitly empty
	if t.Capability != nil {
		// TODO: check there are no UNKNOWN capabilities
		newcap = t.Capability
	} else {
		newcap = schema.UserNewCap
	}

	now := t.Now()
	user := &schema.User{
		UserId: userID,
		Secret: hashed,
		// Don't set last seen.
		Ident:      t.Ident,
		Capability: newcap,
	}
	user.SetCreatedTime(now)
	user.SetModifiedTime(now)

	if err := j.InsertUser(user); err != nil {
		return status.InternalError(err, "can't create user")
	}

	if err := j.Commit(); err != nil {
		return status.InternalError(err, "can't commit job")
	}

	t.CreatedUser = user

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package customresourcedefinition

import (
	"context"
	"fmt"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/registry/generic"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/apiserver/pkg/storage"
	storageerr "k8s.io/apiserver/pkg/storage/errors"
	"k8s.io/apiserver/pkg/util/dryrun"
)

// rest implements a RESTStorage for API services against etcd
type REST struct {
	*genericregistry.Store
}

// NewREST returns a RESTStorage object that will work against API services.
func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) (*REST, error) {
	strategy := NewStrategy(scheme)

	store := &genericregistry.Store{
		NewFunc:                  func() runtime.Object { return &apiextensions.CustomResourceDefinition{} },
		NewListFunc:              func() runtime.Object { return &apiextensions.CustomResourceDefinitionList{} },
		PredicateFunc:            MatchCustomResourceDefinition,
		DefaultQualifiedResource: apiextensions.Resource("customresourcedefinitions"),

		CreateStrategy: strategy,
		UpdateStrategy: strategy,
		DeleteStrategy: strategy,

		// TODO: define table converter that exposes more than name/creation timestamp
		TableConvertor: rest.NewDefaultTableConvertor(apiextensions.Resource("customresourcedefinitions")),
	}
	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs}
	if err := store.CompleteWithOptions(options); err != nil {
		return nil, err
	}
	return &REST{store}, nil
}

// Implement ShortNamesProvider
var _ rest.ShortNamesProvider = &REST{}

// ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
func (r *REST) ShortNames() []string {
	return []string{"crd", "crds"}
}

// Delete adds the CRD finalizer to the list
func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
	obj, err := r.Get(ctx, name, &metav1.GetOptions{})
	if err != nil {
		return nil, false, err
	}

	crd := obj.(*apiextensions.CustomResourceDefinition)

	// Ensure we have a UID precondition
	if options == nil {
		options = metav1.NewDeleteOptions(0)
	}
	if options.Preconditions == nil {
		options.Preconditions = &metav1.Preconditions{}
	}
	if options.Preconditions.UID == nil {
		options.Preconditions.UID = &crd.UID
	} else if *options.Preconditions.UID != crd.UID {
		err = apierrors.NewConflict(
			apiextensions.Resource("customresourcedefinitions"),
			name,
			fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, crd.UID),
		)
		return nil, false, err
	}
	if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != crd.ResourceVersion {
		err = apierrors.NewConflict(
			apiextensions.Resource("customresourcedefinitions"),
			name,
			fmt.Errorf("Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", *options.Preconditions.ResourceVersion, crd.ResourceVersion),
		)
		return nil, false, err
	}

	// upon first request to delete, add our finalizer and then delegate
	if crd.DeletionTimestamp.IsZero() {
		key, err := r.Store.KeyFunc(ctx, name)
		if err != nil {
			return nil, false, err
		}

		preconditions := storage.Preconditions{UID: options.Preconditions.UID, ResourceVersion: options.Preconditions.ResourceVersion}

		out := r.Store.NewFunc()
		err = r.Store.Storage.GuaranteedUpdate(
			ctx, key, out, false, &preconditions,
			storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
				existingCRD, ok := existing.(*apiextensions.CustomResourceDefinition)
				if !ok {
					// wrong type
					return nil, fmt.Errorf("expected *apiextensions.CustomResourceDefinition, got %v", existing)
				}
				if err := deleteValidation(ctx, existingCRD); err != nil {
					return nil, err
				}

				// Set the deletion timestamp if needed
				if existingCRD.DeletionTimestamp.IsZero() {
					now := metav1.Now()
					existingCRD.DeletionTimestamp = &now
				}

				if !apiextensions.CRDHasFinalizer(existingCRD, apiextensions.CustomResourceCleanupFinalizer) {
					existingCRD.Finalizers = append(existingCRD.Finalizers, apiextensions.CustomResourceCleanupFinalizer)
				}
				// update the status condition too
				apiextensions.SetCRDCondition(existingCRD, apiextensions.CustomResourceDefinitionCondition{
					Type:    apiextensions.Terminating,
					Status:  apiextensions.ConditionTrue,
					Reason:  "InstanceDeletionPending",
					Message: "CustomResourceDefinition marked for deletion; CustomResource deletion will begin soon",
				})
				return existingCRD, nil
			}),
			dryrun.IsDryRun(options.DryRun),
		)

		if err != nil {
			err = storageerr.InterpretGetError(err, apiextensions.Resource("customresourcedefinitions"), name)
			err = storageerr.InterpretUpdateError(err, apiextensions.Resource("customresourcedefinitions"), name)
			if _, ok := err.(*apierrors.StatusError); !ok {
				err = apierrors.NewInternalError(err)
			}
			return nil, false, err
		}

		return out, false, nil
	}

	return r.Store.Delete(ctx, name, deleteValidation, options)
}

// NewStatusREST makes a RESTStorage for status that has more limited options.
// It is based on the original REST so that we can share the same underlying store
func NewStatusREST(scheme *runtime.Scheme, rest *REST) *StatusREST {
	statusStore := *rest.Store
	statusStore.CreateStrategy = nil
	statusStore.DeleteStrategy = nil
	statusStore.UpdateStrategy = NewStatusStrategy(scheme)
	return &StatusREST{store: &statusStore}
}

type StatusREST struct {
	store *genericregistry.Store
}

var _ = rest.Patcher(&StatusREST{})

func (r *StatusREST) New() runtime.Object {
	return &apiextensions.CustomResourceDefinition{}
}

// Get retrieves the object from the storage. It is required to support Patch.
func (r *StatusREST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	return r.store.Get(ctx, name, options)
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
	// We are explicitly setting forceAllowCreate to false in the call to the underlying storage because
	// subresources should never allow create on update.
	return r.store.Update(ctx, name, objInfo, createValidation, updateValidation, false, options)
}
<commit_msg>Remove variadic argument from storage interface<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package customresourcedefinition

import (
	"context"
	"fmt"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/registry/generic"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/apiserver/pkg/storage"
	storageerr "k8s.io/apiserver/pkg/storage/errors"
	"k8s.io/apiserver/pkg/util/dryrun"
)

// rest implements a RESTStorage for API services against etcd
type REST struct {
	*genericregistry.Store
}

// NewREST returns a RESTStorage object that will work against API services.
func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) (*REST, error) {
	strategy := NewStrategy(scheme)

	store := &genericregistry.Store{
		NewFunc:                  func() runtime.Object { return &apiextensions.CustomResourceDefinition{} },
		NewListFunc:              func() runtime.Object { return &apiextensions.CustomResourceDefinitionList{} },
		PredicateFunc:            MatchCustomResourceDefinition,
		DefaultQualifiedResource: apiextensions.Resource("customresourcedefinitions"),

		CreateStrategy: strategy,
		UpdateStrategy: strategy,
		DeleteStrategy: strategy,

		// TODO: define table converter that exposes more than name/creation timestamp
		TableConvertor: rest.NewDefaultTableConvertor(apiextensions.Resource("customresourcedefinitions")),
	}
	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs}
	if err := store.CompleteWithOptions(options); err != nil {
		return nil, err
	}
	return &REST{store}, nil
}

// Implement ShortNamesProvider
var _ rest.ShortNamesProvider = &REST{}

// ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
func (r *REST) ShortNames() []string {
	return []string{"crd", "crds"}
}

// Delete adds the CRD finalizer to the list
func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
	obj, err := r.Get(ctx, name, &metav1.GetOptions{})
	if err != nil {
		return nil, false, err
	}

	crd := obj.(*apiextensions.CustomResourceDefinition)

	// Ensure we have a UID precondition
	if options == nil {
		options = metav1.NewDeleteOptions(0)
	}
	if options.Preconditions == nil {
		options.Preconditions = &metav1.Preconditions{}
	}
	if options.Preconditions.UID == nil {
		options.Preconditions.UID = &crd.UID
	} else if *options.Preconditions.UID != crd.UID {
		err = apierrors.NewConflict(
			apiextensions.Resource("customresourcedefinitions"),
			name,
			fmt.Errorf("Precondition failed: UID in precondition: %v, UID in object meta: %v", *options.Preconditions.UID, crd.UID),
		)
		return nil, false, err
	}
	if options.Preconditions.ResourceVersion != nil && *options.Preconditions.ResourceVersion != crd.ResourceVersion {
		err = apierrors.NewConflict(
			apiextensions.Resource("customresourcedefinitions"),
			name,
			fmt.Errorf("Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v", *options.Preconditions.ResourceVersion, crd.ResourceVersion),
		)
		return nil, false, err
	}

	// upon first request to delete, add our finalizer and then delegate
	if crd.DeletionTimestamp.IsZero() {
		key, err := r.Store.KeyFunc(ctx, name)
		if err != nil {
			return nil, false, err
		}

		preconditions := storage.Preconditions{UID: options.Preconditions.UID, ResourceVersion: options.Preconditions.ResourceVersion}

		out := r.Store.NewFunc()
		err = r.Store.Storage.GuaranteedUpdate(
			ctx, key, out, false, &preconditions,
			storage.SimpleUpdate(func(existing runtime.Object) (runtime.Object, error) {
				existingCRD, ok := existing.(*apiextensions.CustomResourceDefinition)
				if !ok {
					// wrong type
					return nil, fmt.Errorf("expected *apiextensions.CustomResourceDefinition, got %v", existing)
				}
				if err := deleteValidation(ctx, existingCRD); err != nil {
					return nil, err
				}

				// Set the deletion timestamp if needed
				if existingCRD.DeletionTimestamp.IsZero() {
					now := metav1.Now()
					existingCRD.DeletionTimestamp = &now
				}

				if !apiextensions.CRDHasFinalizer(existingCRD, apiextensions.CustomResourceCleanupFinalizer) {
					existingCRD.Finalizers = append(existingCRD.Finalizers, apiextensions.CustomResourceCleanupFinalizer)
				}
				// update the status condition too
				apiextensions.SetCRDCondition(existingCRD, apiextensions.CustomResourceDefinitionCondition{
					Type:    apiextensions.Terminating,
					Status:  apiextensions.ConditionTrue,
					Reason:  "InstanceDeletionPending",
					Message: "CustomResourceDefinition marked for deletion; CustomResource deletion will begin soon",
				})
				return existingCRD, nil
			}),
			dryrun.IsDryRun(options.DryRun),
			nil,
		)

		if err != nil {
			err = storageerr.InterpretGetError(err, apiextensions.Resource("customresourcedefinitions"), name)
			err = storageerr.InterpretUpdateError(err, apiextensions.Resource("customresourcedefinitions"), name)
			if _, ok := err.(*apierrors.StatusError); !ok {
				err = apierrors.NewInternalError(err)
			}
			return nil, false, err
		}

		return out, false, nil
	}

	return r.Store.Delete(ctx, name, deleteValidation, options)
}

// NewStatusREST makes a RESTStorage for status that has more limited options.
// It is based on the original REST so that we can share the same underlying store
func NewStatusREST(scheme *runtime.Scheme, rest *REST) *StatusREST {
	statusStore := *rest.Store
	statusStore.CreateStrategy = nil
	statusStore.DeleteStrategy = nil
	statusStore.UpdateStrategy = NewStatusStrategy(scheme)
	return &StatusREST{store: &statusStore}
}

type StatusREST struct {
	store *genericregistry.Store
}

var _ = rest.Patcher(&StatusREST{})

func (r *StatusREST) New() runtime.Object {
	return &apiextensions.CustomResourceDefinition{}
}

// Get retrieves the object from the storage. It is required to support Patch.
func (r *StatusREST) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	return r.store.Get(ctx, name, options)
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
	// We are explicitly setting forceAllowCreate to false in the call to the underlying storage because
	// subresources should never allow create on update.
	return r.store.Update(ctx, name, objInfo, createValidation, updateValidation, false, options)
}
<|endoftext|>


<commit_before>package migrations

import (
	. "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)

func addAlertMigrations(mg *Migrator) {
	alertV1 := Table{
		Name: "alert",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "version", Type: DB_BigInt, Nullable: false},
			{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
			{Name: "panel_id", Type: DB_BigInt, Nullable: false},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "message", Type: DB_Text, Nullable: false},
			{Name: "state", Type: DB_NVarchar, Length: 190, Nullable: false},
			{Name: "settings", Type: DB_Text, Nullable: false},
			{Name: "frequency", Type: DB_BigInt, Nullable: false},
			{Name: "handler", Type: DB_BigInt, Nullable: false},
			{Name: "severity", Type: DB_Text, Nullable: false},
			{Name: "silenced", Type: DB_Bool, Nullable: false},
			{Name: "execution_error", Type: DB_Text, Nullable: false},
			{Name: "eval_data", Type: DB_Text, Nullable: true},
			{Name: "eval_date", Type: DB_DateTime, Nullable: true},
			{Name: "new_state_date", Type: DB_DateTime, Nullable: false},
			{Name: "state_changes", Type: DB_Int, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "id"}, Type: IndexType},
			{Cols: []string{"state"}, Type: IndexType},
			{Cols: []string{"dashboard_id"}, Type: IndexType},
		},
	}

	// create table
	mg.AddMigration("create alert table v1", NewAddTableMigration(alertV1))

	// create indices
	mg.AddMigration("add index alert org_id & id ", NewAddIndexMigration(alertV1, alertV1.Indices[0]))
	mg.AddMigration("add index alert state", NewAddIndexMigration(alertV1, alertV1.Indices[1]))
	mg.AddMigration("add index alert dashboard_id", NewAddIndexMigration(alertV1, alertV1.Indices[2]))

	alertRuleTagTable := Table{
		Name: "alert_rule_tag",
		Columns: []*Column{
			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
			{Name: "tag_id", Type: DB_BigInt, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"alert_id", "tag_id"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("Create alert_rule_tag table v1", NewAddTableMigration(alertRuleTagTable))
	mg.AddMigration("Add unique index alert_rule_tag.alert_id_tag_id", NewAddIndexMigration(alertRuleTagTable, alertRuleTagTable.Indices[0]))

	alert_notification := Table{
		Name: "alert_notification",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
			{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "settings", Type: DB_Text, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "name"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create alert_notification table v1", NewAddTableMigration(alert_notification))
	mg.AddMigration("Add column is_default", NewAddColumnMigration(alert_notification, &Column{
		Name: "is_default", Type: DB_Bool, Nullable: false, Default: "0",
	}))
	mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{
		Name: "frequency", Type: DB_BigInt, Nullable: true,
	}))
	mg.AddMigration("Add column send_reminder", NewAddColumnMigration(alert_notification, &Column{
		Name: "send_reminder", Type: DB_Bool, Nullable: true, Default: "0",
	}))
	mg.AddMigration("Add column disable_resolve_message", NewAddColumnMigration(alert_notification, &Column{
		Name: "disable_resolve_message", Type: DB_Bool, Nullable: false, Default: "0",
	}))

	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))

	mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{
		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
		{Name: "message", Type: DB_Text, Nullable: false},
		{Name: "state", Type: DB_NVarchar, Length: 190, Nullable: false},
		{Name: "settings", Type: DB_Text, Nullable: false},
		{Name: "severity", Type: DB_Text, Nullable: false},
		{Name: "execution_error", Type: DB_Text, Nullable: false},
		{Name: "eval_data", Type: DB_Text, Nullable: true},
	}))

	mg.AddMigration("Update alert_notification table charset", NewTableCharsetMigration("alert_notification", []*Column{
		{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
		{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
		{Name: "settings", Type: DB_Text, Nullable: false},
	}))

	notification_journal := Table{
		Name: "alert_notification_journal",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
			{Name: "sent_at", Type: DB_BigInt, Nullable: false},
			{Name: "success", Type: DB_Bool, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType},
		},
	}

	mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal))
	mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0]))

	mg.AddMigration("drop alert_notification_journal", NewDropTableMigration("alert_notification_journal"))

	alert_notification_state := Table{
		Name: "alert_notification_state",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
			{Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false},
			{Name: "version", Type: DB_BigInt, Nullable: false},
			{Name: "updated_at", Type: DB_BigInt, Nullable: false},
			{Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
	mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
		NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))

	mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
		Name: "for", Type: DB_BigInt, Nullable: true,
	}))

	mg.AddMigration("Add column uid in alert_notification", NewAddColumnMigration(alert_notification, &Column{
		Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: true,
	}))

	mg.AddMigration("Update uid column values in alert_notification", new(RawSqlMigration).
		Sqlite("UPDATE alert_notification SET uid=printf('%09d',id) WHERE uid IS NULL;").
		Postgres("UPDATE alert_notification SET uid=lpad('' || id::text,9,'0') WHERE uid IS NULL;").
		Mysql("UPDATE alert_notification SET uid=lpad(id,9,'0') WHERE uid IS NULL;"))

	mg.AddMigration("Add unique index alert_notification_org_id_uid", NewAddIndexMigration(alert_notification, &Index{
		Cols: []string{"org_id", "uid"}, Type: UniqueIndex,
	}))

	mg.AddMigration("Remove unique index org_id_name", NewDropIndexMigration(alert_notification, &Index{
		Cols: []string{"org_id", "name"}, Type: UniqueIndex,
	}))

	mg.AddMigration("Add column secure_settings in alert_notification", NewAddColumnMigration(alert_notification, &Column{
		Name: "secure_settings", Type: DB_Text, Nullable: true,
	}))
}
<commit_msg>Alerting: Change settings column type to mediumtext (#26549)<commit_after>package migrations

import (
	. "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)

func addAlertMigrations(mg *Migrator) {
	alertV1 := Table{
		Name: "alert",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "version", Type: DB_BigInt, Nullable: false},
			{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
			{Name: "panel_id", Type: DB_BigInt, Nullable: false},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "message", Type: DB_Text, Nullable: false},
			{Name: "state", Type: DB_NVarchar, Length: 190, Nullable: false},
			{Name: "settings", Type: DB_Text, Nullable: false},
			{Name: "frequency", Type: DB_BigInt, Nullable: false},
			{Name: "handler", Type: DB_BigInt, Nullable: false},
			{Name: "severity", Type: DB_Text, Nullable: false},
			{Name: "silenced", Type: DB_Bool, Nullable: false},
			{Name: "execution_error", Type: DB_Text, Nullable: false},
			{Name: "eval_data", Type: DB_Text, Nullable: true},
			{Name: "eval_date", Type: DB_DateTime, Nullable: true},
			{Name: "new_state_date", Type: DB_DateTime, Nullable: false},
			{Name: "state_changes", Type: DB_Int, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "id"}, Type: IndexType},
			{Cols: []string{"state"}, Type: IndexType},
			{Cols: []string{"dashboard_id"}, Type: IndexType},
		},
	}

	// create table
	mg.AddMigration("create alert table v1", NewAddTableMigration(alertV1))

	// create indices
	mg.AddMigration("add index alert org_id & id ", NewAddIndexMigration(alertV1, alertV1.Indices[0]))
	mg.AddMigration("add index alert state", NewAddIndexMigration(alertV1, alertV1.Indices[1]))
	mg.AddMigration("add index alert dashboard_id", NewAddIndexMigration(alertV1, alertV1.Indices[2]))

	alertRuleTagTable := Table{
		Name: "alert_rule_tag",
		Columns: []*Column{
			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
			{Name: "tag_id", Type: DB_BigInt, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"alert_id", "tag_id"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("Create alert_rule_tag table v1", NewAddTableMigration(alertRuleTagTable))
	mg.AddMigration("Add unique index alert_rule_tag.alert_id_tag_id", NewAddIndexMigration(alertRuleTagTable, alertRuleTagTable.Indices[0]))

	alert_notification := Table{
		Name: "alert_notification",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
			{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "settings", Type: DB_Text, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "name"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create alert_notification table v1", NewAddTableMigration(alert_notification))
	mg.AddMigration("Add column is_default", NewAddColumnMigration(alert_notification, &Column{
		Name: "is_default", Type: DB_Bool, Nullable: false, Default: "0",
	}))
	mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{
		Name: "frequency", Type: DB_BigInt, Nullable: true,
	}))
	mg.AddMigration("Add column send_reminder", NewAddColumnMigration(alert_notification, &Column{
		Name: "send_reminder", Type: DB_Bool, Nullable: true, Default: "0",
	}))
	mg.AddMigration("Add column disable_resolve_message", NewAddColumnMigration(alert_notification, &Column{
		Name: "disable_resolve_message", Type: DB_Bool, Nullable: false, Default: "0",
	}))

	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))

	mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{
		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
		{Name: "message", Type: DB_Text, Nullable: false},
		{Name: "state", Type: DB_NVarchar, Length: 190, Nullable: false},
		{Name: "settings", Type: DB_Text, Nullable: false},
		{Name: "severity", Type: DB_Text, Nullable: false},
		{Name: "execution_error", Type: DB_Text, Nullable: false},
		{Name: "eval_data", Type: DB_Text, Nullable: true},
	}))

	mg.AddMigration("Update alert_notification table charset", NewTableCharsetMigration("alert_notification", []*Column{
		{Name: "name", Type: DB_NVarchar, Length: 190, Nullable: false},
		{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
		{Name: "settings", Type: DB_Text, Nullable: false},
	}))

	notification_journal := Table{
		Name: "alert_notification_journal",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
			{Name: "sent_at", Type: DB_BigInt, Nullable: false},
			{Name: "success", Type: DB_Bool, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType},
		},
	}

	mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal))
	mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0]))

	mg.AddMigration("drop alert_notification_journal", NewDropTableMigration("alert_notification_journal"))

	alert_notification_state := Table{
		Name: "alert_notification_state",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
			{Name: "state", Type: DB_NVarchar, Length: 50, Nullable: false},
			{Name: "version", Type: DB_BigInt, Nullable: false},
			{Name: "updated_at", Type: DB_BigInt, Nullable: false},
			{Name: "alert_rule_state_updated_version", Type: DB_BigInt, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
	mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
		NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))

	mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
		Name: "for", Type: DB_BigInt, Nullable: true,
	}))

	mg.AddMigration("Add column uid in alert_notification", NewAddColumnMigration(alert_notification, &Column{
		Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: true,
	}))

	mg.AddMigration("Update uid column values in alert_notification", new(RawSqlMigration).
		Sqlite("UPDATE alert_notification SET uid=printf('%09d',id) WHERE uid IS NULL;").
		Postgres("UPDATE alert_notification SET uid=lpad('' || id::text,9,'0') WHERE uid IS NULL;").
		Mysql("UPDATE alert_notification SET uid=lpad(id,9,'0') WHERE uid IS NULL;"))

	mg.AddMigration("Add unique index alert_notification_org_id_uid", NewAddIndexMigration(alert_notification, &Index{
		Cols: []string{"org_id", "uid"}, Type: UniqueIndex,
	}))

	mg.AddMigration("Remove unique index org_id_name", NewDropIndexMigration(alert_notification, &Index{
		Cols: []string{"org_id", "name"}, Type: UniqueIndex,
	}))

	mg.AddMigration("Add column secure_settings in alert_notification", NewAddColumnMigration(alert_notification, &Column{
		Name: "secure_settings", Type: DB_Text, Nullable: true,
	}))

	// change column type of alert.settings
	mg.AddMigration("alter alert.settings to mediumtext", NewRawSqlMigration("").
		Mysql("ALTER TABLE alert MODIFY settings MEDIUMTEXT;"))
}
<|endoftext|>


<commit_before>package indexer

import (
	"encoding/json"
	"golang.org/x/net/context"
	"gopkg.in/olivere/elastic.v5"
)

type Indexer struct {
	el *elastic.Client
}

type Reference struct {
	ParentHash string `json:"parent_hash"`
	Name       string `json:"name"`
}

func NewIndexer(el *elastic.Client) *Indexer {
	return &Indexer{
		el: el,
	}
}

// Add file or directory to index
func (i Indexer) IndexItem(doctype string, hash string, properties map[string]interface{}) error {
	_, err := i.el.Update().
		Index("ipfs").
		Type(doctype).
		Id(hash).
		Doc(properties).
		DocAsUpsert(true).
		Do(context.TODO())

	if err != nil {
		// Handle error
		return err
	}

	return nil
}

// Return existing references for an object, or nil, and the type
func (i Indexer) GetReferences(hash string) ([]Reference, string, error) {
	fsc := elastic.NewFetchSourceContext(true)
	fsc.Include("references")

	res, err := i.el.Get().
		Index("ipfs").Type("_all").FetchSourceContext(fsc).Id(hash).Do(context.TODO())

	if err != nil {
		if elastic.IsNotFound(err) {
			return nil, "", nil
		}
		return nil, "", err
	}

	var result map[string][]Reference
	err = json.Unmarshal(*res.Source, &result)
	if err != nil {
		return nil, "", err
	}

	return result["references"], res.Type, nil
}
<commit_msg>Specify what happens when no results.<commit_after>package indexer

import (
	"encoding/json"
	"golang.org/x/net/context"
	"gopkg.in/olivere/elastic.v5"
)

type Indexer struct {
	el *elastic.Client
}

type Reference struct {
	ParentHash string `json:"parent_hash"`
	Name       string `json:"name"`
}

func NewIndexer(el *elastic.Client) *Indexer {
	return &Indexer{
		el: el,
	}
}

// Add file or directory to index
func (i Indexer) IndexItem(doctype string, hash string, properties map[string]interface{}) error {
	_, err := i.el.Update().
		Index("ipfs").
		Type(doctype).
		Id(hash).
		Doc(properties).
		DocAsUpsert(true).
		Do(context.TODO())

	if err != nil {
		// Handle error
		return err
	}

	return nil
}

// Return existing references for an object, or nil, and the type
// When no object is found nil is returned but no error is set
// Otherwise, an empty list is returned
func (i Indexer) GetReferences(hash string) ([]Reference, string, error) {
	fsc := elastic.NewFetchSourceContext(true)
	fsc.Include("references")

	res, err := i.el.Get().
		Index("ipfs").Type("_all").FetchSourceContext(fsc).Id(hash).Do(context.TODO())

	if err != nil {
		if elastic.IsNotFound(err) {
			return nil, "", nil
		}
		return nil, "", err
	}

	var result map[string][]Reference
	err = json.Unmarshal(*res.Source, &result)
	if err != nil {
		return nil, "", err
	}

	return result["references"], res.Type, nil
}
<|endoftext|>


<commit_before>package indexer

import (
	"fmt"
	"sync"

	"github.com/ava-labs/avalanchego/api"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/hashing"
	"github.com/ava-labs/avalanchego/utils/json"
	"github.com/ava-labs/avalanchego/utils/timer"
	"github.com/ava-labs/avalanchego/utils/wrappers"

	"github.com/ava-labs/avalanchego/codec"
	"github.com/ava-labs/avalanchego/codec/linearcodec"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow/choices"
	"github.com/ava-labs/avalanchego/snow/engine/avalanche"
	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/snow/engine/snowman"
	"github.com/ava-labs/avalanchego/snow/triggers"
	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/gorilla/rpc/v2"
)

const (
	indexNamePrefix = "index-"
	codecVersion    = uint16(0)
)

var (
	txPrefix                = byte(0x01)
	vtxPrefix               = byte(0x02)
	blockPrefix             = byte(0x03)
	isIncompletePrefix      = byte(0x04)
	previouslyIndexedPrefix = byte(0x05)
	hasRunKey               = []byte{0x06}
)

var (
	_ Indexer = &indexer{}
)

// Config for an indexer
type Config struct {
	DB                                      database.Database
	Log                                     logging.Logger
	IndexingEnabled                         bool
	AllowIncompleteIndex                    bool
	Name                                    string
	DecisionDispatcher, ConsensusDispatcher *triggers.EventDispatcher
	APIServer                               api.RouteAdder
	ShutdownF                               func()
}

// Indexer causes accepted containers for a given chain
// to be indexed by their ID and by the order in which
// they were accepted by this node.
// Indexer is threadsafe.
type Indexer interface {
	RegisterChain(name string, ctx *snow.Context, vm interface{})
	// Close will do nothing and return nil after the first call
	Close() error
}

// NewIndexer returns a new Indexer and registers a new endpoint on the given API server.
func NewIndexer(config Config) (Indexer, error) {
	indexer := &indexer{
		name:                 config.Name,
		codec:                codec.NewDefaultManager(),
		log:                  config.Log,
		db:                   config.DB,
		allowIncompleteIndex: config.AllowIncompleteIndex,
		indexingEnabled:      config.IndexingEnabled,
		consensusDispatcher:  config.ConsensusDispatcher,
		decisionDispatcher:   config.DecisionDispatcher,
		txIndices:            map[ids.ID]Index{},
		vtxIndices:           map[ids.ID]Index{},
		blockIndices:         map[ids.ID]Index{},
		routeAdder:           config.APIServer,
		shutdownF:            config.ShutdownF,
	}
	if err := indexer.codec.RegisterCodec(codecVersion, linearcodec.NewDefault()); err != nil {
		return nil, fmt.Errorf("couldn't register codec: %s", err)
	}
	hasRun, err := indexer.hasRun()
	if err != nil {
		return nil, err
	}
	indexer.hasRunBefore = hasRun
	return indexer, indexer.markHasRun()
}

// indexer implements Indexer
type indexer struct {
	// Name of endpoint
	name   string
	codec  codec.Manager
	clock  timer.Clock
	lock   sync.RWMutex
	log    logging.Logger
	db     database.Database
	closed bool

	// Called in a goroutine on shutdown
	shutdownF func()

	// true if this is not the first run using this database
	hasRunBefore bool

	// Used to add API endpoint for new indices
	routeAdder api.RouteAdder

	// If true, allow running in such a way that could allow the creation
	// of an index which could be missing accepted containers.
	allowIncompleteIndex bool

	// If false, don't create index for a chain when RegisterChain is called
	indexingEnabled bool

	// Chain ID --> index of blocks of that chain (if applicable)
	blockIndices map[ids.ID]Index
	// Chain ID --> index of vertices of that chain (if applicable)
	vtxIndices map[ids.ID]Index
	// Chain ID --> index of txs of that chain (if applicable)
	txIndices map[ids.ID]Index

	// Notifies of newly accepted blocks and vertices
	consensusDispatcher *triggers.EventDispatcher
	// Notifies of newly accepted transactions
	decisionDispatcher *triggers.EventDispatcher
}

// Assumes [engineIntf]'s context lock is not held
func (i *indexer) RegisterChain(name string, ctx *snow.Context, engineIntf interface{}) {
	i.lock.Lock()
	defer i.lock.Unlock()

	if i.closed {
		i.log.Debug("not registering chain %s because indexer is closed", name)
		return
	} else if ctx.SubnetID != constants.PrimaryNetworkID {
		i.log.Debug("not registering chain %s because it's not in primary network", name)
		return
	}

	chainID := ctx.ChainID
	if i.blockIndices[chainID] != nil || i.txIndices[chainID] != nil || i.vtxIndices[chainID] != nil {
		i.log.Warn("chain %s is already being indexed", chainID)
		return
	}

	// If the index is incomplete, make sure that's OK. Otherwise, cause node to die.
	isIncomplete, err := i.isIncomplete(chainID)
	if err != nil {
		i.log.Error("couldn't get whether chain %s is incomplete: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	// See if this chain was indexed in a previous run
	previouslyIndexed, err := i.previouslyIndexed(chainID)
	if err != nil {
		i.log.Error("couldn't get whether chain %s was previously indexed: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	if !i.allowIncompleteIndex && isIncomplete && (previouslyIndexed || i.hasRunBefore) {
		i.log.Fatal("index %s is incomplete but incomplete indices are disabled. Shutting down", name)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	if !i.indexingEnabled { // Indexing is disabled
		if previouslyIndexed && !i.allowIncompleteIndex {
			// We indexed this chain in a previous run but not in this run.
			// This would create an incomplete index, which is not allowed, so exit.
			i.log.Fatal("running would cause index %s would become incomplete but incomplete indices are disabled", name)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}

		// Creating an incomplete index is allowed. Mark index as incomplete.
		err := i.markIncomplete(chainID)
		if err == nil {
			return
		}
		i.log.Fatal("couldn't mark chain %s as incomplete: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	// Mark that in this run, this chain was indexed
	if err := i.markPreviouslyIndexed(chainID); err != nil {
		i.log.Error("couldn't mark chain %s as indexed: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	switch engine := engineIntf.(type) {
	case snowman.Engine:
		isAcceptedFunc := func(blkID ids.ID) bool {
			ctx.Lock.Lock()
			defer ctx.Lock.Unlock()

			blk, err := engine.GetVM().GetBlock(blkID)
			return err == nil && blk.Status() == choices.Accepted
		}

		index, err := i.registerChainHelper(chainID, blockPrefix, name, "block", i.consensusDispatcher, isAcceptedFunc)
		if err != nil {
			i.log.Fatal("couldn't create block index for %s: %s", name, err)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}
		i.blockIndices[chainID] = index
	case avalanche.Engine:
		isVtxAcceptedFunc := func(vtxID ids.ID) bool {
			ctx.Lock.Lock()
			defer ctx.Lock.Unlock()

			vtx, err := engine.GetVtx(vtxID)
			return err == nil && vtx.Status() == choices.Accepted
		}

		vtxIndex, err := i.registerChainHelper(chainID, vtxPrefix, name, "vtx", i.consensusDispatcher, isVtxAcceptedFunc)
		if err != nil {
			i.log.Fatal("couldn't create vertex index for %s: %s", name, err)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}
		i.vtxIndices[chainID] = vtxIndex

		isTxAcceptedFunc := func(txID ids.ID) bool {
			ctx.Lock.Lock()
			defer ctx.Lock.Unlock()

			tx, err := engine.GetVM().GetTx(txID)
			return err == nil && tx.Status() == choices.Accepted
		}
		txIndex, err := i.registerChainHelper(chainID, txPrefix, name, "tx", i.decisionDispatcher, isTxAcceptedFunc)
		if err != nil {
			i.log.Fatal("couldn't create tx index for %s: %s", name, err)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}
		i.txIndices[chainID] = txIndex
	default:
		i.log.Error("got unexpected engine type %T", engineIntf)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

}

func (i *indexer) registerChainHelper(
	chainID ids.ID,
	prefixEnd byte,
	name, endpoint string,
	dispatcher *triggers.EventDispatcher,
	isAcceptedF func(containerID ids.ID) bool,
) (Index, error) {
	prefix := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(prefix, chainID[:])
	prefix[hashing.HashLen] = prefixEnd
	indexDB := prefixdb.New(prefix, i.db)
	index, err := newIndex(indexDB, i.log, i.codec, i.clock, isAcceptedF)
	if err != nil {
		_ = indexDB.Close()
		return nil, err
	}

	// Register index to learn about new accepted vertices
	if err := dispatcher.RegisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID), index); err != nil {
		_ = index.Close()
		return nil, err
	}

	// Create an API endpoint for this index
	apiServer := rpc.NewServer()
	codec := json.NewCodec()
	apiServer.RegisterCodec(codec, "application/json")
	apiServer.RegisterCodec(codec, "application/json;charset=UTF-8")
	if err := apiServer.RegisterService(&service{Index: index}, "index"); err != nil {
		_ = index.Close()
		return nil, err
	}
	handler := &common.HTTPHandler{LockOptions: common.NoLock, Handler: apiServer}
	if err := i.routeAdder.AddRoute(handler, &sync.RWMutex{}, "index/"+name, "/"+endpoint, i.log); err != nil {
		_ = index.Close()
		return nil, err
	}
	return index, nil
}

// Close this indexer. Stops indexing all chains.
// Closes [i.db]. Assumes Close is only called after
// the node is done making decisions.
// Calling Close after it has been called does nothing.
func (i *indexer) Close() error {
	i.lock.Lock()
	defer i.lock.Unlock()

	return i.close()
}

func (i *indexer) close() error {
	if i.closed {
		return nil
	}
	i.closed = true

	errs := &wrappers.Errs{}
	for chainID, txIndex := range i.txIndices {
		errs.Add(txIndex.Close())
		errs.Add(i.decisionDispatcher.DeregisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID)))
	}
	for chainID, vtxIndex := range i.vtxIndices {
		errs.Add(vtxIndex.Close())
		errs.Add(i.consensusDispatcher.DeregisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID)))
	}
	for chainID, blockIndex := range i.blockIndices {
		errs.Add(blockIndex.Close())
		errs.Add(i.consensusDispatcher.DeregisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID)))
	}
	errs.Add(i.db.Close())

	go i.shutdownF()
	return errs.Err
}

func (i *indexer) markIncomplete(chainID ids.ID) error {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = isIncompletePrefix
	return i.db.Put(key, nil)
}

// Returns true if this chain is incomplete
func (i *indexer) isIncomplete(chainID ids.ID) (bool, error) {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = isIncompletePrefix
	return i.db.Has(key)
}

func (i *indexer) markPreviouslyIndexed(chainID ids.ID) error {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = previouslyIndexedPrefix
	return i.db.Put(key, nil)
}

// Returns true if this chain is incomplete
func (i *indexer) previouslyIndexed(chainID ids.ID) (bool, error) {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = previouslyIndexedPrefix
	return i.db.Has(key)
}

// Mark that the node has run at least once
func (i *indexer) markHasRun() error {
	return i.db.Put(hasRunKey, nil)
}

// Returns true if the node has run before
func (i *indexer) hasRun() (bool, error) {
	return i.db.Has(hasRunKey)
}
<commit_msg>bump codec max size and make slice len<commit_after>package indexer

import (
	"fmt"
	"math"
	"sync"

	"github.com/ava-labs/avalanchego/api"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/hashing"
	"github.com/ava-labs/avalanchego/utils/json"
	"github.com/ava-labs/avalanchego/utils/timer"
	"github.com/ava-labs/avalanchego/utils/wrappers"

	"github.com/ava-labs/avalanchego/codec"
	"github.com/ava-labs/avalanchego/codec/linearcodec"
	"github.com/ava-labs/avalanchego/codec/reflectcodec"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow/choices"
	"github.com/ava-labs/avalanchego/snow/engine/avalanche"
	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/snow/engine/snowman"
	"github.com/ava-labs/avalanchego/snow/triggers"
	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/gorilla/rpc/v2"
)

const (
	indexNamePrefix = "index-"
	codecVersion    = uint16(0)
	// Max size, in bytes, of something serialized by this indexer
	// Assumes no containers are larger than math.MaxUint32
	// wrappers.IntLen accounts for the size of the container bytes
	// wrappers.LongLen accounts for the timestamp of the container
	// hashing.HashLen accounts for the container ID
	// wrappers.ShortLen accounts for the codec version
	codecMaxSize = math.MaxUint32 + wrappers.IntLen + wrappers.LongLen + hashing.HashLen + wrappers.ShortLen
)

var (
	txPrefix                = byte(0x01)
	vtxPrefix               = byte(0x02)
	blockPrefix             = byte(0x03)
	isIncompletePrefix      = byte(0x04)
	previouslyIndexedPrefix = byte(0x05)
	hasRunKey               = []byte{0x06}
)

var (
	_ Indexer = &indexer{}
)

// Config for an indexer
type Config struct {
	DB                                      database.Database
	Log                                     logging.Logger
	IndexingEnabled                         bool
	AllowIncompleteIndex                    bool
	Name                                    string
	DecisionDispatcher, ConsensusDispatcher *triggers.EventDispatcher
	APIServer                               api.RouteAdder
	ShutdownF                               func()
}

// Indexer causes accepted containers for a given chain
// to be indexed by their ID and by the order in which
// they were accepted by this node.
// Indexer is threadsafe.
type Indexer interface {
	RegisterChain(name string, ctx *snow.Context, vm interface{})
	// Close will do nothing and return nil after the first call
	Close() error
}

// NewIndexer returns a new Indexer and registers a new endpoint on the given API server.
func NewIndexer(config Config) (Indexer, error) {
	indexer := &indexer{
		name:                 config.Name,
		codec:                codec.NewManager(codecMaxSize),
		log:                  config.Log,
		db:                   config.DB,
		allowIncompleteIndex: config.AllowIncompleteIndex,
		indexingEnabled:      config.IndexingEnabled,
		consensusDispatcher:  config.ConsensusDispatcher,
		decisionDispatcher:   config.DecisionDispatcher,
		txIndices:            map[ids.ID]Index{},
		vtxIndices:           map[ids.ID]Index{},
		blockIndices:         map[ids.ID]Index{},
		routeAdder:           config.APIServer,
		shutdownF:            config.ShutdownF,
	}
	if err := indexer.codec.RegisterCodec(
		codecVersion,
		linearcodec.New(reflectcodec.DefaultTagName, math.MaxUint32),
	); err != nil {
		return nil, fmt.Errorf("couldn't register codec: %s", err)
	}
	hasRun, err := indexer.hasRun()
	if err != nil {
		return nil, err
	}
	indexer.hasRunBefore = hasRun
	return indexer, indexer.markHasRun()
}

// indexer implements Indexer
type indexer struct {
	// Name of endpoint
	name   string
	codec  codec.Manager
	clock  timer.Clock
	lock   sync.RWMutex
	log    logging.Logger
	db     database.Database
	closed bool

	// Called in a goroutine on shutdown
	shutdownF func()

	// true if this is not the first run using this database
	hasRunBefore bool

	// Used to add API endpoint for new indices
	routeAdder api.RouteAdder

	// If true, allow running in such a way that could allow the creation
	// of an index which could be missing accepted containers.
	allowIncompleteIndex bool

	// If false, don't create index for a chain when RegisterChain is called
	indexingEnabled bool

	// Chain ID --> index of blocks of that chain (if applicable)
	blockIndices map[ids.ID]Index
	// Chain ID --> index of vertices of that chain (if applicable)
	vtxIndices map[ids.ID]Index
	// Chain ID --> index of txs of that chain (if applicable)
	txIndices map[ids.ID]Index

	// Notifies of newly accepted blocks and vertices
	consensusDispatcher *triggers.EventDispatcher
	// Notifies of newly accepted transactions
	decisionDispatcher *triggers.EventDispatcher
}

// Assumes [engineIntf]'s context lock is not held
func (i *indexer) RegisterChain(name string, ctx *snow.Context, engineIntf interface{}) {
	i.lock.Lock()
	defer i.lock.Unlock()

	if i.closed {
		i.log.Debug("not registering chain %s because indexer is closed", name)
		return
	} else if ctx.SubnetID != constants.PrimaryNetworkID {
		i.log.Debug("not registering chain %s because it's not in primary network", name)
		return
	}

	chainID := ctx.ChainID
	if i.blockIndices[chainID] != nil || i.txIndices[chainID] != nil || i.vtxIndices[chainID] != nil {
		i.log.Warn("chain %s is already being indexed", chainID)
		return
	}

	// If the index is incomplete, make sure that's OK. Otherwise, cause node to die.
	isIncomplete, err := i.isIncomplete(chainID)
	if err != nil {
		i.log.Error("couldn't get whether chain %s is incomplete: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	// See if this chain was indexed in a previous run
	previouslyIndexed, err := i.previouslyIndexed(chainID)
	if err != nil {
		i.log.Error("couldn't get whether chain %s was previously indexed: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	if !i.allowIncompleteIndex && isIncomplete && (previouslyIndexed || i.hasRunBefore) {
		i.log.Fatal("index %s is incomplete but incomplete indices are disabled. Shutting down", name)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	if !i.indexingEnabled { // Indexing is disabled
		if previouslyIndexed && !i.allowIncompleteIndex {
			// We indexed this chain in a previous run but not in this run.
			// This would create an incomplete index, which is not allowed, so exit.
			i.log.Fatal("running would cause index %s would become incomplete but incomplete indices are disabled", name)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}

		// Creating an incomplete index is allowed. Mark index as incomplete.
		err := i.markIncomplete(chainID)
		if err == nil {
			return
		}
		i.log.Fatal("couldn't mark chain %s as incomplete: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	// Mark that in this run, this chain was indexed
	if err := i.markPreviouslyIndexed(chainID); err != nil {
		i.log.Error("couldn't mark chain %s as indexed: %s", name, err)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

	switch engine := engineIntf.(type) {
	case snowman.Engine:
		isAcceptedFunc := func(blkID ids.ID) bool {
			ctx.Lock.Lock()
			defer ctx.Lock.Unlock()

			blk, err := engine.GetVM().GetBlock(blkID)
			return err == nil && blk.Status() == choices.Accepted
		}

		index, err := i.registerChainHelper(chainID, blockPrefix, name, "block", i.consensusDispatcher, isAcceptedFunc)
		if err != nil {
			i.log.Fatal("couldn't create block index for %s: %s", name, err)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}
		i.blockIndices[chainID] = index
	case avalanche.Engine:
		isVtxAcceptedFunc := func(vtxID ids.ID) bool {
			ctx.Lock.Lock()
			defer ctx.Lock.Unlock()

			vtx, err := engine.GetVtx(vtxID)
			return err == nil && vtx.Status() == choices.Accepted
		}

		vtxIndex, err := i.registerChainHelper(chainID, vtxPrefix, name, "vtx", i.consensusDispatcher, isVtxAcceptedFunc)
		if err != nil {
			i.log.Fatal("couldn't create vertex index for %s: %s", name, err)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}
		i.vtxIndices[chainID] = vtxIndex

		isTxAcceptedFunc := func(txID ids.ID) bool {
			ctx.Lock.Lock()
			defer ctx.Lock.Unlock()

			tx, err := engine.GetVM().GetTx(txID)
			return err == nil && tx.Status() == choices.Accepted
		}
		txIndex, err := i.registerChainHelper(chainID, txPrefix, name, "tx", i.decisionDispatcher, isTxAcceptedFunc)
		if err != nil {
			i.log.Fatal("couldn't create tx index for %s: %s", name, err)
			if err := i.close(); err != nil {
				i.log.Error("error while closing indexer: %s", err)
			}
			return
		}
		i.txIndices[chainID] = txIndex
	default:
		i.log.Error("got unexpected engine type %T", engineIntf)
		if err := i.close(); err != nil {
			i.log.Error("error while closing indexer: %s", err)
		}
		return
	}

}

func (i *indexer) registerChainHelper(
	chainID ids.ID,
	prefixEnd byte,
	name, endpoint string,
	dispatcher *triggers.EventDispatcher,
	isAcceptedF func(containerID ids.ID) bool,
) (Index, error) {
	prefix := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(prefix, chainID[:])
	prefix[hashing.HashLen] = prefixEnd
	indexDB := prefixdb.New(prefix, i.db)
	index, err := newIndex(indexDB, i.log, i.codec, i.clock, isAcceptedF)
	if err != nil {
		_ = indexDB.Close()
		return nil, err
	}

	// Register index to learn about new accepted vertices
	if err := dispatcher.RegisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID), index); err != nil {
		_ = index.Close()
		return nil, err
	}

	// Create an API endpoint for this index
	apiServer := rpc.NewServer()
	codec := json.NewCodec()
	apiServer.RegisterCodec(codec, "application/json")
	apiServer.RegisterCodec(codec, "application/json;charset=UTF-8")
	if err := apiServer.RegisterService(&service{Index: index}, "index"); err != nil {
		_ = index.Close()
		return nil, err
	}
	handler := &common.HTTPHandler{LockOptions: common.NoLock, Handler: apiServer}
	if err := i.routeAdder.AddRoute(handler, &sync.RWMutex{}, "index/"+name, "/"+endpoint, i.log); err != nil {
		_ = index.Close()
		return nil, err
	}
	return index, nil
}

// Close this indexer. Stops indexing all chains.
// Closes [i.db]. Assumes Close is only called after
// the node is done making decisions.
// Calling Close after it has been called does nothing.
func (i *indexer) Close() error {
	i.lock.Lock()
	defer i.lock.Unlock()

	return i.close()
}

func (i *indexer) close() error {
	if i.closed {
		return nil
	}
	i.closed = true

	errs := &wrappers.Errs{}
	for chainID, txIndex := range i.txIndices {
		errs.Add(txIndex.Close())
		errs.Add(i.decisionDispatcher.DeregisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID)))
	}
	for chainID, vtxIndex := range i.vtxIndices {
		errs.Add(vtxIndex.Close())
		errs.Add(i.consensusDispatcher.DeregisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID)))
	}
	for chainID, blockIndex := range i.blockIndices {
		errs.Add(blockIndex.Close())
		errs.Add(i.consensusDispatcher.DeregisterChain(chainID, fmt.Sprintf("%s%s", indexNamePrefix, chainID)))
	}
	errs.Add(i.db.Close())

	go i.shutdownF()
	return errs.Err
}

func (i *indexer) markIncomplete(chainID ids.ID) error {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = isIncompletePrefix
	return i.db.Put(key, nil)
}

// Returns true if this chain is incomplete
func (i *indexer) isIncomplete(chainID ids.ID) (bool, error) {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = isIncompletePrefix
	return i.db.Has(key)
}

func (i *indexer) markPreviouslyIndexed(chainID ids.ID) error {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = previouslyIndexedPrefix
	return i.db.Put(key, nil)
}

// Returns true if this chain is incomplete
func (i *indexer) previouslyIndexed(chainID ids.ID) (bool, error) {
	key := make([]byte, hashing.HashLen+wrappers.ByteLen)
	copy(key, chainID[:])
	key[hashing.HashLen] = previouslyIndexedPrefix
	return i.db.Has(key)
}

// Mark that the node has run at least once
func (i *indexer) markHasRun() error {
	return i.db.Put(hasRunKey, nil)
}

// Returns true if the node has run before
func (i *indexer) hasRun() (bool, error) {
	return i.db.Has(hasRunKey)
}
<|endoftext|>