<commit_before>package dockergen

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/fsouza/go-dockerclient"
)

type generator struct {
	Client                     *docker.Client
	Configs                    ConfigFile
	Endpoint                   string
	TLSVerify                  bool
	TLSCert, TLSCaCert, TLSKey string

	wg sync.WaitGroup
}

type GeneratorConfig struct {
	Endpoint string

	TLSCert   string
	TLSKey    string
	TLSCACert string
	TLSVerify bool

	ConfigFile ConfigFile
}

func NewGenerator(gc GeneratorConfig) (*generator, error) {
	endpoint, err := GetEndpoint(gc.Endpoint)
	if err != nil {
		return nil, fmt.Errorf("Bad endpoint: %s", err)
	}

	client, err := NewDockerClient(endpoint, gc.TLSVerify, gc.TLSCert, gc.TLSCACert, gc.TLSKey)
	if err != nil {
		return nil, fmt.Errorf("Unable to create docker client: %s", err)
	}

	apiVersion, err := client.Version()
	if err != nil {
		log.Printf("error retrieving docker server version info: %s\n", err)
	}

	// Grab the docker daemon info once and hold onto it
	SetDockerEnv(apiVersion)

	return &generator{
		Client:    client,
		Endpoint:  gc.Endpoint,
		TLSVerify: gc.TLSVerify,
		TLSCert:   gc.TLSCert,
		TLSCaCert: gc.TLSCACert,
		TLSKey:    gc.TLSKey,
		Configs:   gc.ConfigFile,
	}, nil
}

func (g *generator) Generate() error {
	g.generateFromContainers(g.Client)
	g.generateAtInterval(g.Client, g.Configs)
	g.generateFromEvents(g.Client, g.Configs)
	g.generateFromSignals()
	g.wg.Wait()

	return nil
}

func (g *generator) generateFromSignals() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL)

	g.wg.Add(1)
	go func() {
		defer g.wg.Done()

		for {
			sig := <-sigs
			log.Printf("Received signal: %s\n", sig)
			switch sig {
			case syscall.SIGHUP:
				g.generateFromContainers(g.Client)
			case syscall.SIGQUIT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGINT:
				// exit when context is done
				return
			}
		}
	}()
}

func (g *generator) generateFromContainers(client *docker.Client) {
	containers, err := g.getContainers(client)
	if err != nil {
		log.Printf("error listing containers: %s\n", err)
		return
	}
	for _, config := range g.Configs.Config {
		changed := GenerateFile(config, containers)
		if !changed {
			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
			continue
		}
		g.runNotifyCmd(config)
		g.sendSignalToContainer(client, config)
	}
}

func (g *generator) generateAtInterval(client *docker.Client, configs ConfigFile) {
	for _, config := range configs.Config {

		if config.Interval == 0 {
			continue
		}

		log.Printf("Generating every %d seconds", config.Interval)
		g.wg.Add(1)
		ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
		quit := make(chan struct{})
		configCopy := config
		go func() {
			defer g.wg.Done()
			for {
				select {
				case <-ticker.C:
					containers, err := g.getContainers(client)
					if err != nil {
						log.Printf("Error listing containers: %s\n", err)
						continue
					}
					// ignore changed return value. always run notify command
					GenerateFile(configCopy, containers)
					g.runNotifyCmd(configCopy)
					g.sendSignalToContainer(client, configCopy)
				case <-quit:
					ticker.Stop()
					return
				}
			}
		}()
	}
}

func (g *generator) generateFromEvents(client *docker.Client, configs ConfigFile) {
	configs = configs.FilterWatches()
	if len(configs.Config) == 0 {
		return
	}

	g.wg.Add(1)
	defer g.wg.Done()

	for {
		if client == nil {
			var err error
			endpoint, err := GetEndpoint(g.Endpoint)
			if err != nil {
				log.Printf("Bad endpoint: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}

			client, err = NewDockerClient(endpoint, g.TLSVerify, g.TLSCert, g.TLSCaCert, g.TLSKey)
			if err != nil {
				log.Printf("Unable to connect to docker daemon: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}
			g.generateFromContainers(client)
		}

		eventChan := make(chan *docker.APIEvents, 100)
		defer close(eventChan)

		watching := false
		for {

			if client == nil {
				break
			}
			err := client.Ping()
			if err != nil {
				log.Printf("Unable to ping docker daemon: %s", err)
				if watching {
					client.RemoveEventListener(eventChan)
					watching = false
					client = nil
				}
				time.Sleep(10 * time.Second)
				break

			}

			if !watching {
				err = client.AddEventListener(eventChan)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
				log.Println("Watching docker events")
			}

			select {

			case event := <-eventChan:
				if event == nil {
					if watching {
						client.RemoveEventListener(eventChan)
						watching = false
						client = nil
					}
					break
				}

				if event.Status == "start" || event.Status == "stop" || event.Status == "die" {
					log.Printf("Received event %s for container %s", event.Status, event.ID[:12])
					g.generateFromContainers(client)
				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}
}

func (g *generator) runNotifyCmd(config Config) {
	if config.NotifyCmd == "" {
		return
	}

	log.Printf("Running '%s'", config.NotifyCmd)
	cmd := exec.Command("/bin/sh", "-c", config.NotifyCmd)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running notify command: %s, %s\n", config.NotifyCmd, err)
	}
	if config.NotifyOutput {
		for _, line := range strings.Split(string(out), "\n") {
			if line != "" {
				log.Printf("[%s]: %s", config.NotifyCmd, line)
			}
		}
	}
}

func (g *generator) sendSignalToContainer(client *docker.Client, config Config) {
	if len(config.NotifyContainers) < 1 {
		return
	}

	for container, signal := range config.NotifyContainers {
		log.Printf("Sending container '%s' signal '%v'", container, signal)
		killOpts := docker.KillContainerOptions{
			ID:     container,
			Signal: signal,
		}
		if err := client.KillContainer(killOpts); err != nil {
			log.Printf("Error sending signal to container: %s", err)
		}
	}
}

func (g *generator) getContainers(client *docker.Client) ([]*RuntimeContainer, error) {
	apiInfo, err := client.Info()
	if err != nil {
		log.Printf("error retrieving docker server info: %s\n", err)
	}

	SetServerInfo(apiInfo)

	apiContainers, err := client.ListContainers(docker.ListContainersOptions{
		All:  false,
		Size: false,
	})
	if err != nil {
		return nil, err
	}

	containers := []*RuntimeContainer{}
	for _, apiContainer := range apiContainers {
		container, err := client.InspectContainer(apiContainer.ID)
		if err != nil {
			log.Printf("error inspecting container: %s: %s\n", apiContainer.ID, err)
			continue
		}

		registry, repository, tag := splitDockerImage(container.Config.Image)
		runtimeContainer := &RuntimeContainer{
			ID: container.ID,
			Image: DockerImage{
				Registry:   registry,
				Repository: repository,
				Tag:        tag,
			},
			Name:         strings.TrimLeft(container.Name, "/"),
			Hostname:     container.Config.Hostname,
			Gateway:      container.NetworkSettings.Gateway,
			Addresses:    []Address{},
			Networks:     []Network{},
			Env:          make(map[string]string),
			Volumes:      make(map[string]Volume),
			Node:         SwarmNode{},
			Labels:       make(map[string]string),
			IP:           container.NetworkSettings.IPAddress,
			IP6LinkLocal: container.NetworkSettings.LinkLocalIPv6Address,
			IP6Global:    container.NetworkSettings.GlobalIPv6Address,
		}
		for k, v := range container.NetworkSettings.Ports {
			address := Address{
				IP:           container.NetworkSettings.IPAddress,
				IP6LinkLocal: container.NetworkSettings.LinkLocalIPv6Address,
				IP6Global:    container.NetworkSettings.GlobalIPv6Address,
				Port:         k.Port(),
				Proto:        k.Proto(),
			}
			if len(v) > 0 {
				address.HostPort = v[0].HostPort
				address.HostIP = v[0].HostIP
			}
			runtimeContainer.Addresses = append(runtimeContainer.Addresses,
				address)

		}
		for k, v := range container.NetworkSettings.Networks {
			network := Network{
				IP:                  v.IPAddress,
				Name:                k,
				Gateway:             v.Gateway,
				EndpointID:          v.EndpointID,
				IPv6Gateway:         v.IPv6Gateway,
				GlobalIPv6Address:   v.GlobalIPv6Address,
				MacAddress:          v.MacAddress,
				GlobalIPv6PrefixLen: v.GlobalIPv6PrefixLen,
				IPPrefixLen:         v.IPPrefixLen,
			}

			runtimeContainer.Networks = append(runtimeContainer.Networks,
				network)
		}
		for k, v := range container.Volumes {
			runtimeContainer.Volumes[k] = Volume{
				Path:      k,
				HostPath:  v,
				ReadWrite: container.VolumesRW[k],
			}
		}
		if container.Node != nil {
			runtimeContainer.Node.ID = container.Node.ID
			runtimeContainer.Node.Name = container.Node.Name
			runtimeContainer.Node.Address = Address{
				IP: container.Node.IP,
			}
		}

		for _, v := range container.Mounts {
			runtimeContainer.Mounts = append(runtimeContainer.Mounts, Mount{
				Name:        v.Name,
				Source:      v.Source,
				Destination: v.Destination,
				Driver:      v.Driver,
				Mode:        v.Mode,
				RW:          v.RW,
			})
		}

		runtimeContainer.Env = splitKeyValueSlice(container.Config.Env)
		runtimeContainer.Labels = container.Config.Labels
		containers = append(containers, runtimeContainer)
	}
	return containers, nil

}
<commit_msg>Cleanup generator func signatures<commit_after>package dockergen

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/fsouza/go-dockerclient"
)

type generator struct {
	Client                     *docker.Client
	Configs                    ConfigFile
	Endpoint                   string
	TLSVerify                  bool
	TLSCert, TLSCaCert, TLSKey string

	wg sync.WaitGroup
}

type GeneratorConfig struct {
	Endpoint string

	TLSCert   string
	TLSKey    string
	TLSCACert string
	TLSVerify bool

	ConfigFile ConfigFile
}

func NewGenerator(gc GeneratorConfig) (*generator, error) {
	endpoint, err := GetEndpoint(gc.Endpoint)
	if err != nil {
		return nil, fmt.Errorf("Bad endpoint: %s", err)
	}

	client, err := NewDockerClient(endpoint, gc.TLSVerify, gc.TLSCert, gc.TLSCACert, gc.TLSKey)
	if err != nil {
		return nil, fmt.Errorf("Unable to create docker client: %s", err)
	}

	apiVersion, err := client.Version()
	if err != nil {
		log.Printf("error retrieving docker server version info: %s\n", err)
	}

	// Grab the docker daemon info once and hold onto it
	SetDockerEnv(apiVersion)

	return &generator{
		Client:    client,
		Endpoint:  gc.Endpoint,
		TLSVerify: gc.TLSVerify,
		TLSCert:   gc.TLSCert,
		TLSCaCert: gc.TLSCACert,
		TLSKey:    gc.TLSKey,
		Configs:   gc.ConfigFile,
	}, nil
}

func (g *generator) Generate() error {
	g.generateFromContainers()
	g.generateAtInterval()
	g.generateFromEvents()
	g.generateFromSignals()
	g.wg.Wait()

	return nil
}

func (g *generator) generateFromSignals() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGKILL)

	g.wg.Add(1)
	go func() {
		defer g.wg.Done()

		for {
			sig := <-sigs
			log.Printf("Received signal: %s\n", sig)
			switch sig {
			case syscall.SIGHUP:
				g.generateFromContainers()
			case syscall.SIGQUIT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGINT:
				// exit when context is done
				return
			}
		}
	}()
}

func (g *generator) generateFromContainers() {
	containers, err := g.getContainers()
	if err != nil {
		log.Printf("error listing containers: %s\n", err)
		return
	}
	for _, config := range g.Configs.Config {
		changed := GenerateFile(config, containers)
		if !changed {
			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
			continue
		}
		g.runNotifyCmd(config)
		g.sendSignalToContainer(config)
	}
}

func (g *generator) generateAtInterval() {
	for _, config := range g.Configs.Config {

		if config.Interval == 0 {
			continue
		}

		log.Printf("Generating every %d seconds", config.Interval)
		g.wg.Add(1)
		ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
		quit := make(chan struct{})
		go func(config Config) {
			defer g.wg.Done()
			for {
				select {
				case <-ticker.C:
					containers, err := g.getContainers()
					if err != nil {
						log.Printf("Error listing containers: %s\n", err)
						continue
					}
					// ignore changed return value. always run notify command
					GenerateFile(config, containers)
					g.runNotifyCmd(config)
					g.sendSignalToContainer(config)
				case <-quit:
					ticker.Stop()
					return
				}
			}
		}(config)
	}
}

func (g *generator) generateFromEvents() {
	configs := g.Configs.FilterWatches()
	if len(configs.Config) == 0 {
		return
	}

	g.wg.Add(1)
	defer g.wg.Done()

	client := g.Client
	for {
		if client == nil {
			var err error
			endpoint, err := GetEndpoint(g.Endpoint)
			if err != nil {
				log.Printf("Bad endpoint: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}

			client, err = NewDockerClient(endpoint, g.TLSVerify, g.TLSCert, g.TLSCaCert, g.TLSKey)
			if err != nil {
				log.Printf("Unable to connect to docker daemon: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}
			g.generateFromContainers()
		}

		eventChan := make(chan *docker.APIEvents, 100)
		defer close(eventChan)

		watching := false
		for {

			if client == nil {
				break
			}
			err := client.Ping()
			if err != nil {
				log.Printf("Unable to ping docker daemon: %s", err)
				if watching {
					client.RemoveEventListener(eventChan)
					watching = false
					client = nil
				}
				time.Sleep(10 * time.Second)
				break

			}

			if !watching {
				err = client.AddEventListener(eventChan)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
				log.Println("Watching docker events")
			}

			select {

			case event := <-eventChan:
				if event == nil {
					if watching {
						client.RemoveEventListener(eventChan)
						watching = false
						client = nil
					}
					break
				}

				if event.Status == "start" || event.Status == "stop" || event.Status == "die" {
					log.Printf("Received event %s for container %s", event.Status, event.ID[:12])
					g.generateFromContainers()
				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}
}

func (g *generator) runNotifyCmd(config Config) {
	if config.NotifyCmd == "" {
		return
	}

	log.Printf("Running '%s'", config.NotifyCmd)
	cmd := exec.Command("/bin/sh", "-c", config.NotifyCmd)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running notify command: %s, %s\n", config.NotifyCmd, err)
	}
	if config.NotifyOutput {
		for _, line := range strings.Split(string(out), "\n") {
			if line != "" {
				log.Printf("[%s]: %s", config.NotifyCmd, line)
			}
		}
	}
}

func (g *generator) sendSignalToContainer(config Config) {
	if len(config.NotifyContainers) < 1 {
		return
	}

	for container, signal := range config.NotifyContainers {
		log.Printf("Sending container '%s' signal '%v'", container, signal)
		killOpts := docker.KillContainerOptions{
			ID:     container,
			Signal: signal,
		}
		if err := g.Client.KillContainer(killOpts); err != nil {
			log.Printf("Error sending signal to container: %s", err)
		}
	}
}

func (g *generator) getContainers() ([]*RuntimeContainer, error) {
	apiInfo, err := g.Client.Info()
	if err != nil {
		log.Printf("error retrieving docker server info: %s\n", err)
	}

	SetServerInfo(apiInfo)

	apiContainers, err := g.Client.ListContainers(docker.ListContainersOptions{
		All:  false,
		Size: false,
	})
	if err != nil {
		return nil, err
	}

	containers := []*RuntimeContainer{}
	for _, apiContainer := range apiContainers {
		container, err := g.Client.InspectContainer(apiContainer.ID)
		if err != nil {
			log.Printf("error inspecting container: %s: %s\n", apiContainer.ID, err)
			continue
		}

		registry, repository, tag := splitDockerImage(container.Config.Image)
		runtimeContainer := &RuntimeContainer{
			ID: container.ID,
			Image: DockerImage{
				Registry:   registry,
				Repository: repository,
				Tag:        tag,
			},
			Name:         strings.TrimLeft(container.Name, "/"),
			Hostname:     container.Config.Hostname,
			Gateway:      container.NetworkSettings.Gateway,
			Addresses:    []Address{},
			Networks:     []Network{},
			Env:          make(map[string]string),
			Volumes:      make(map[string]Volume),
			Node:         SwarmNode{},
			Labels:       make(map[string]string),
			IP:           container.NetworkSettings.IPAddress,
			IP6LinkLocal: container.NetworkSettings.LinkLocalIPv6Address,
			IP6Global:    container.NetworkSettings.GlobalIPv6Address,
		}
		for k, v := range container.NetworkSettings.Ports {
			address := Address{
				IP:           container.NetworkSettings.IPAddress,
				IP6LinkLocal: container.NetworkSettings.LinkLocalIPv6Address,
				IP6Global:    container.NetworkSettings.GlobalIPv6Address,
				Port:         k.Port(),
				Proto:        k.Proto(),
			}
			if len(v) > 0 {
				address.HostPort = v[0].HostPort
				address.HostIP = v[0].HostIP
			}
			runtimeContainer.Addresses = append(runtimeContainer.Addresses,
				address)

		}
		for k, v := range container.NetworkSettings.Networks {
			network := Network{
				IP:                  v.IPAddress,
				Name:                k,
				Gateway:             v.Gateway,
				EndpointID:          v.EndpointID,
				IPv6Gateway:         v.IPv6Gateway,
				GlobalIPv6Address:   v.GlobalIPv6Address,
				MacAddress:          v.MacAddress,
				GlobalIPv6PrefixLen: v.GlobalIPv6PrefixLen,
				IPPrefixLen:         v.IPPrefixLen,
			}

			runtimeContainer.Networks = append(runtimeContainer.Networks,
				network)
		}
		for k, v := range container.Volumes {
			runtimeContainer.Volumes[k] = Volume{
				Path:      k,
				HostPath:  v,
				ReadWrite: container.VolumesRW[k],
			}
		}
		if container.Node != nil {
			runtimeContainer.Node.ID = container.Node.ID
			runtimeContainer.Node.Name = container.Node.Name
			runtimeContainer.Node.Address = Address{
				IP: container.Node.IP,
			}
		}

		for _, v := range container.Mounts {
			runtimeContainer.Mounts = append(runtimeContainer.Mounts, Mount{
				Name:        v.Name,
				Source:      v.Source,
				Destination: v.Destination,
				Driver:      v.Driver,
				Mode:        v.Mode,
				RW:          v.RW,
			})
		}

		runtimeContainer.Env = splitKeyValueSlice(container.Config.Env)
		runtimeContainer.Labels = container.Config.Labels
		containers = append(containers, runtimeContainer)
	}
	return containers, nil

}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
)

// ParseStack is a standard stack implementation, but also holds
// methods for transforming the top k elements into a new element.
type ParseStack struct {
	top  *stackElement
	size int
}

// stackElement is a stack-internal data structure that is used
// as a wrapper for the actual data.
type stackElement struct {
	value *ParsedComponent
	next  *stackElement
}

// ParsedComponent is an element of the parse stack that represents
// a section of the input string that was successfully parsed.
type ParsedComponent struct {
	// begin is the index of the first character that belongs to
	// the parsed statement
	begin int
	// end is the index of the last character that belongs to the
	// parsed statement + 1
	end int
	// comp stores the struct that the string was parsed into
	comp interface{}
}

// Len return the stack's size.
func (ps *ParseStack) Len() int {
	return ps.size
}

// Push pushes a new element onto the stack.
func (ps *ParseStack) Push(value *ParsedComponent) {
	ps.top = &stackElement{value, ps.top}
	ps.size++
}

// Pop removes the top element from the stack and returns its value.
// If the stack is empty, returns nil.
func (ps *ParseStack) Pop() (value *ParsedComponent) {
	if ps.size > 0 {
		value, ps.top = ps.top.value, ps.top.next
		ps.size--
		return
	}
	return nil
}

// Peek returns the top element from the stack but doesn't remove it.
// If the stack is empty, returns nil.
func (ps *ParseStack) Peek() (value *ParsedComponent) {
	if ps.size > 0 {
		return ps.top.value
	}
	return nil
}

// AssembleSelect takes the topmost elements from the stack, assuming
// they are components of a SELECT statement, and replaces them by
// a single SelectStmt element.
func (ps *ParseStack) AssembleSelect() {
	// pop the components from the stack in reverse order
	_having, _grouping, _filter, _from, _projections := ps.pop5()

	// extract and convert the contained structure
	// (if this fails, this is a fundamental parser bug => panic ok)
	having := _having.comp.(Having)
	grouping := _grouping.comp.(Grouping)
	filter := _filter.comp.(Filter)
	from := _from.comp.(From)
	projections := _projections.comp.(Projections)

	// assemble the SelectStmt and push it back
	s := SelectStmt{projections, from, filter, grouping, having}
	se := ParsedComponent{_projections.begin, _having.end, s}
	ps.Push(&se)
}

/* Projections/Columns */

// AssembleProjections takes the elements from the stack that
// correspond to the input[begin:end] string and wraps a
// Projections struct around them.
func (ps *ParseStack) AssembleProjections(begin int, end int) {
	elems := ps.collectElements(begin, end)
	// push the grouped list back
	ps.PushComponent(begin, end, Projections{elems})
}

/* FROM clause */

// AssembleFrom takes the elements from the stack that
// correspond to the input[begin:end] string, makes sure
// they are all Relation elements and wraps a From struct
// around them. If there are no such elements, adds an
// empty From struct to the stack.
func (ps *ParseStack) AssembleFrom(begin int, end int) {
	if begin == end {
		// push an empty from clause
		ps.PushComponent(begin, end, From{})
	} else {
		elems := ps.collectElements(begin, end)
		rels := make([]Relation, len(elems), len(elems))
		for i, elem := range elems {
			// (if this conversion fails, this is a fundamental parser bug)
			e := elem.(Relation)
			rels[i] = e
		}
		// push the grouped list back
		ps.PushComponent(begin, end, From{rels})
	}
}

/* WHERE clause */

// AssembleFilter takes the expression on top of the stack
// (if there is a WHERE clause) and wraps a Filter struct
// around it. If there is no WHERE clause, an empty Filter
// struct is used.
func (ps *ParseStack) AssembleFilter(begin int, end int) {
	if begin == end {
		// push an empty from clause
		ps.PushComponent(begin, end, Filter{})
	} else {
		// if the stack is empty at this point, this is
		// a serious parser bug
		f := ps.Pop()
		if begin > f.begin || end < f.end {
			panic("the item on top of the stack is not within given range")
		}
		ps.PushComponent(begin, end, Filter{f.comp})
	}
}

/* GROUP BY clause */

// AssembleGrouping takes the elements from the stack that
// correspond to the input[begin:end] string and wraps a
// Grouping struct around them. If there are no such elements,
// adds an empty Grouping struct to the stack.
func (ps *ParseStack) AssembleGrouping(begin int, end int) {
	elems := ps.collectElements(begin, end)
	// push the grouped list back
	ps.PushComponent(begin, end, Grouping{elems})
}

/* HAVING clause */

// AssembleHaving takes the expression on top of the stack
// (if there is a HAVING clause) and wraps a Having struct
// around it. If there is no HAVING clause, an empty Having
// struct is used.
func (ps *ParseStack) AssembleHaving(begin int, end int) {
	if begin == end {
		// push an empty from clause
		ps.PushComponent(begin, end, Having{})
	} else {
		// if the stack is empty at this point, this is
		// a serious parser bug
		h := ps.Pop()
		if begin > h.begin || end < h.end {
			panic("the item on top of the stack is not within given range")
		}
		ps.PushComponent(begin, end, Having{h.comp})
	}
}

/* Expressions */

// AssembleBinaryOperation takes the two elements from the stack that
// correspond to the input[begin:end] string and adds the given
// binary operator in between. If there is just one element, push
// it back unmodified.
func (ps *ParseStack) AssembleBinaryOperation(begin int, end int, op string) {
	elems := ps.collectElements(begin, end)
	if len(elems) == 1 {
		// there is no "binary" operation, push back the single element
		ps.PushComponent(begin, end, elems[0])
	} else if len(elems) == 2 {
		// connect left and right with the given operator
		ps.PushComponent(begin, end, BinaryOp{op, elems[0], elems[1]})
	} else {
		panic(fmt.Sprintf("cannot turn %+v into a binary operation", elems))
	}
}

// PushComponent pushes the given component to the top of the stack
// wrapped in a ParsedComponent struct. It's the caller's responsibility
// to make sure that the parameter is one of the AST classes, or there
// will almost surely be a panic at a later point in the parsing process.
func (ps *ParseStack) PushComponent(begin int, end int, comp interface{}) {
	if begin > end {
		panic("begin must be less or equal to end")
	}
	if top := ps.Peek(); top != nil && top.end > begin {
		panic("begin must be larger or equal to the previous item's end")
	}
	se := ParsedComponent{begin, end, comp}
	ps.Push(&se)
}

/* helper functions to reduce code duplication */

// collectElements pops all elements with begin/end contained in
// the parameter range from the stack, reverses their order and
// returns them.
func (ps *ParseStack) collectElements(begin int, end int) []interface{} {
	elems := []interface{}{}
	// look at elements on the stack as long as there are some and
	// they are contained in our interval
	for ps.Peek() != nil {
		if ps.Peek().end <= begin {
			break
		}
		top := ps.Pop().comp
		elems = append(elems, top)
	}
	// reverse the list to restore original order
	size := len(elems)
	for i := 0; i < size/2; i++ {
		elems[i], elems[size-i-1] = elems[size-i-1], elems[i]
	}
	return elems
}

func (ps *ParseStack) pop2() (*ParsedComponent, *ParsedComponent) {
	if ps.Len() < 2 {
		panic("not enough elements on stack to pop 2 of them")
	}
	return ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop3() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent) {
	if ps.Len() < 3 {
		panic("not enough elements on stack to pop 3 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop4() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent) {
	if ps.Len() < 4 {
		panic("not enough elements on stack to pop 4 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop5() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent) {
	if ps.Len() < 5 {
		panic("not enough elements on stack to pop 5 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop6() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent, *ParsedComponent) {
	if ps.Len() < 6 {
		panic("not enough elements on stack to pop 6 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop7() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent, *ParsedComponent,
	*ParsedComponent) {
	if ps.Len() < 7 {
		panic("not enough elements on stack to pop 7 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop8() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent) {
	if ps.Len() < 8 {
		panic("not enough elements on stack to pop 8 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}
<commit_msg>add stack diagrams to the existing AssembleXyz functions<commit_after>package parser

import (
	"fmt"
)

// ParseStack is a standard stack implementation, but also holds
// methods for transforming the top k elements into a new element.
type ParseStack struct {
	top  *stackElement
	size int
}

// stackElement is a stack-internal data structure that is used
// as a wrapper for the actual data.
type stackElement struct {
	value *ParsedComponent
	next  *stackElement
}

// ParsedComponent is an element of the parse stack that represents
// a section of the input string that was successfully parsed.
type ParsedComponent struct {
	// begin is the index of the first character that belongs to
	// the parsed statement
	begin int
	// end is the index of the last character that belongs to the
	// parsed statement + 1
	end int
	// comp stores the struct that the string was parsed into
	comp interface{}
}

// Len return the stack's size.
func (ps *ParseStack) Len() int {
	return ps.size
}

// Push pushes a new element onto the stack.
func (ps *ParseStack) Push(value *ParsedComponent) {
	ps.top = &stackElement{value, ps.top}
	ps.size++
}

// Pop removes the top element from the stack and returns its value.
// If the stack is empty, returns nil.
func (ps *ParseStack) Pop() (value *ParsedComponent) {
	if ps.size > 0 {
		value, ps.top = ps.top.value, ps.top.next
		ps.size--
		return
	}
	return nil
}

// Peek returns the top element from the stack but doesn't remove it.
// If the stack is empty, returns nil.
func (ps *ParseStack) Peek() (value *ParsedComponent) {
	if ps.size > 0 {
		return ps.top.value
	}
	return nil
}

// AssembleSelect takes the topmost elements from the stack, assuming
// they are components of a SELECT statement, and replaces them by
// a single SelectStmt element.
//
//  Having
//  Grouping
//  Filter
//  From
//  Projections
//   =>
//  SelectStmt{Projections, From, Filter, Grouping, Having}
func (ps *ParseStack) AssembleSelect() {
	// pop the components from the stack in reverse order
	_having, _grouping, _filter, _from, _projections := ps.pop5()

	// extract and convert the contained structure
	// (if this fails, this is a fundamental parser bug => panic ok)
	having := _having.comp.(Having)
	grouping := _grouping.comp.(Grouping)
	filter := _filter.comp.(Filter)
	from := _from.comp.(From)
	projections := _projections.comp.(Projections)

	// assemble the SelectStmt and push it back
	s := SelectStmt{projections, from, filter, grouping, having}
	se := ParsedComponent{_projections.begin, _having.end, s}
	ps.Push(&se)
}

/* Projections/Columns */

// AssembleProjections takes the elements from the stack that
// correspond to the input[begin:end] string and wraps a
// Projections struct around them.
//
//  Any
//  Any
//  Any
//   =>
//  Projections{[Any, Any, Any]}
func (ps *ParseStack) AssembleProjections(begin int, end int) {
	elems := ps.collectElements(begin, end)
	// push the grouped list back
	ps.PushComponent(begin, end, Projections{elems})
}

/* FROM clause */

// AssembleFrom takes the elements from the stack that
// correspond to the input[begin:end] string, makes sure
// they are all Relation elements and wraps a From struct
// around them. If there are no such elements, adds an
// empty From struct to the stack.
//
//  Relation
//  Relation
//  Relation
//   =>
//  From{[Relation, Relation, Relation]}
func (ps *ParseStack) AssembleFrom(begin int, end int) {
	if begin == end {
		// push an empty from clause
		ps.PushComponent(begin, end, From{})
	} else {
		elems := ps.collectElements(begin, end)
		rels := make([]Relation, len(elems), len(elems))
		for i, elem := range elems {
			// (if this conversion fails, this is a fundamental parser bug)
			e := elem.(Relation)
			rels[i] = e
		}
		// push the grouped list back
		ps.PushComponent(begin, end, From{rels})
	}
}

/* WHERE clause */

// AssembleFilter takes the expression on top of the stack
// (if there is a WHERE clause) and wraps a Filter struct
// around it. If there is no WHERE clause, an empty Filter
// struct is used.
//
//  Any
//   =>
//  Filter{Any}
func (ps *ParseStack) AssembleFilter(begin int, end int) {
	if begin == end {
		// push an empty from clause
		ps.PushComponent(begin, end, Filter{})
	} else {
		// if the stack is empty at this point, this is
		// a serious parser bug
		f := ps.Pop()
		if begin > f.begin || end < f.end {
			panic("the item on top of the stack is not within given range")
		}
		ps.PushComponent(begin, end, Filter{f.comp})
	}
}

/* GROUP BY clause */

// AssembleGrouping takes the elements from the stack that
// correspond to the input[begin:end] string and wraps a
// Grouping struct around them. If there are no such elements,
// adds an empty Grouping struct to the stack.
//
//  Any
//  Any
//  Any
//   =>
//  Grouping{[Any, Any, Any]}
func (ps *ParseStack) AssembleGrouping(begin int, end int) {
	elems := ps.collectElements(begin, end)
	// push the grouped list back
	ps.PushComponent(begin, end, Grouping{elems})
}

/* HAVING clause */

// AssembleHaving takes the expression on top of the stack
// (if there is a HAVING clause) and wraps a Having struct
// around it. If there is no HAVING clause, an empty Having
// struct is used.
//
//  Any
//   =>
//  Having{Any}
func (ps *ParseStack) AssembleHaving(begin int, end int) {
	if begin == end {
		// push an empty from clause
		ps.PushComponent(begin, end, Having{})
	} else {
		// if the stack is empty at this point, this is
		// a serious parser bug
		h := ps.Pop()
		if begin > h.begin || end < h.end {
			panic("the item on top of the stack is not within given range")
		}
		ps.PushComponent(begin, end, Having{h.comp})
	}
}

/* Expressions */

// AssembleBinaryOperation takes the two elements from the stack that
// correspond to the input[begin:end] string and adds the given
// binary operator in between. If there is just one element, push
// it back unmodified.
//
//  Any
//  Any
//   =>
//  BinaryOp{op, Any, Any}
func (ps *ParseStack) AssembleBinaryOperation(begin int, end int, op string) {
	elems := ps.collectElements(begin, end)
	if len(elems) == 1 {
		// there is no "binary" operation, push back the single element
		ps.PushComponent(begin, end, elems[0])
	} else if len(elems) == 2 {
		// connect left and right with the given operator
		ps.PushComponent(begin, end, BinaryOp{op, elems[0], elems[1]})
	} else {
		panic(fmt.Sprintf("cannot turn %+v into a binary operation", elems))
	}
}

// PushComponent pushes the given component to the top of the stack
// wrapped in a ParsedComponent struct. It's the caller's responsibility
// to make sure that the parameter is one of the AST classes, or there
// will almost surely be a panic at a later point in the parsing process.
func (ps *ParseStack) PushComponent(begin int, end int, comp interface{}) {
	if begin > end {
		panic("begin must be less or equal to end")
	}
	if top := ps.Peek(); top != nil && top.end > begin {
		panic("begin must be larger or equal to the previous item's end")
	}
	se := ParsedComponent{begin, end, comp}
	ps.Push(&se)
}

/* helper functions to reduce code duplication */

// collectElements pops all elements with begin/end contained in
// the parameter range from the stack, reverses their order and
// returns them.
func (ps *ParseStack) collectElements(begin int, end int) []interface{} {
	elems := []interface{}{}
	// look at elements on the stack as long as there are some and
	// they are contained in our interval
	for ps.Peek() != nil {
		if ps.Peek().end <= begin {
			break
		}
		top := ps.Pop().comp
		elems = append(elems, top)
	}
	// reverse the list to restore original order
	size := len(elems)
	for i := 0; i < size/2; i++ {
		elems[i], elems[size-i-1] = elems[size-i-1], elems[i]
	}
	return elems
}

func (ps *ParseStack) pop2() (*ParsedComponent, *ParsedComponent) {
	if ps.Len() < 2 {
		panic("not enough elements on stack to pop 2 of them")
	}
	return ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop3() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent) {
	if ps.Len() < 3 {
		panic("not enough elements on stack to pop 3 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop4() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent) {
	if ps.Len() < 4 {
		panic("not enough elements on stack to pop 4 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop5() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent) {
	if ps.Len() < 5 {
		panic("not enough elements on stack to pop 5 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop6() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent, *ParsedComponent) {
	if ps.Len() < 6 {
		panic("not enough elements on stack to pop 6 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop7() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent, *ParsedComponent,
	*ParsedComponent) {
	if ps.Len() < 7 {
		panic("not enough elements on stack to pop 7 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}

func (ps *ParseStack) pop8() (*ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent, *ParsedComponent, *ParsedComponent,
	*ParsedComponent, *ParsedComponent) {
	if ps.Len() < 8 {
		panic("not enough elements on stack to pop 8 of them")
	}
	return ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop(), ps.Pop()
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcutil

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	_ "crypto/sha512" // Needed for RegisterHash in init
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"net"
	"os"
	"time"
)

// NewTLSCertPair returns a new PEM-encoded x.509 certificate pair
// based on a 521-bit ECDSA private key.  The machine's local interface
// addresses and all variants of IPv4 and IPv6 localhost are included as
// valid IP addresses.
func NewTLSCertPair(organization string, validUntil time.Time, extraHosts []string) (cert, key []byte, err error) {
	now := time.Now()
	if validUntil.Before(now) {
		return nil, nil, errors.New("validUntil would create an already-expired certificate")
	}

	priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
	if err != nil {
		return nil, nil, err
	}

	// end of ASN.1 time
	endOfTime := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
	if validUntil.After(endOfTime) {
		validUntil = endOfTime
	}

	template := x509.Certificate{
		SerialNumber: new(big.Int).SetInt64(0),
		Subject: pkix.Name{
			Organization: []string{organization},
		},
		NotBefore: now,
		NotAfter:  validUntil,

		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		IsCA:        true, // so can sign self.
		BasicConstraintsValid: true,
	}

	host, err := os.Hostname()
	if err != nil {
		return nil, nil, err
	}

	// Use maps to prevent adding duplicates.
	ipAddresses := map[string]net.IP{
		"127.0.0.1": net.ParseIP("127.0.0.1"),
		"::1":       net.ParseIP("::1"),
	}
	dnsNames := map[string]bool{
		host:        true,
		"localhost": true,
	}

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, nil, err
	}
	for _, a := range addrs {
		ip, _, err := net.ParseCIDR(a.String())
		if err == nil {
			ipAddresses[ip.String()] = ip
		}
	}

	for _, hostStr := range extraHosts {
		host, _, err := net.SplitHostPort(hostStr)
		if err != nil {
			host = hostStr
		}
		if ip := net.ParseIP(host); ip != nil {
			ipAddresses[ip.String()] = ip
		} else {
			dnsNames[host] = true
		}
	}

	template.DNSNames = make([]string, 0, len(dnsNames))
	for dnsName := range dnsNames {
		template.DNSNames = append(template.DNSNames, dnsName)
	}
	template.IPAddresses = make([]net.IP, 0, len(ipAddresses))
	for _, ip := range ipAddresses {
		template.IPAddresses = append(template.IPAddresses, ip)
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template,
		&template, &priv.PublicKey, priv)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create certificate: %v\n", err)
	}

	certBuf := &bytes.Buffer{}
	pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})

	keybytes, err := x509.MarshalECPrivateKey(priv)
	if err != nil {
		return nil, nil, err
	}
	keyBuf := &bytes.Buffer{}
	pem.Encode(keyBuf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keybytes})

	return certBuf.Bytes(), keyBuf.Bytes(), nil
}
<commit_msg>Correct and improve cert generation.<commit_after>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcutil

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	_ "crypto/sha512" // Needed for RegisterHash in init
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"net"
	"os"
	"time"
)

// NewTLSCertPair returns a new PEM-encoded x.509 certificate pair
// based on a 521-bit ECDSA private key.  The machine's local interface
// addresses and all variants of IPv4 and IPv6 localhost are included as
// valid IP addresses.
func NewTLSCertPair(organization string, validUntil time.Time, extraHosts []string) (cert, key []byte, err error) {
	now := time.Now()
	if validUntil.Before(now) {
		return nil, nil, errors.New("validUntil would create an already-expired certificate")
	}

	priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
	if err != nil {
		return nil, nil, err
	}

	// end of ASN.1 time
	endOfTime := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
	if validUntil.After(endOfTime) {
		validUntil = endOfTime
	}

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate serial number: %s", err)
	}

	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{organization},
		},
		NotBefore: now.Add(-time.Hour * 24),
		NotAfter:  validUntil,

		KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature |
			x509.KeyUsageCertSign,
		IsCA: true, // so can sign self.
		BasicConstraintsValid: true,
	}

	host, err := os.Hostname()
	if err != nil {
		return nil, nil, err
	}

	// Use maps to prevent adding duplicates.
	ipAddresses := map[string]net.IP{
		"127.0.0.1": net.ParseIP("127.0.0.1"),
		"::1":       net.ParseIP("::1"),
	}
	dnsNames := map[string]bool{
		host:        true,
		"localhost": true,
	}

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, nil, err
	}
	for _, a := range addrs {
		ip, _, err := net.ParseCIDR(a.String())
		if err == nil {
			ipAddresses[ip.String()] = ip
		}
	}

	for _, hostStr := range extraHosts {
		host, _, err := net.SplitHostPort(hostStr)
		if err != nil {
			host = hostStr
		}
		if ip := net.ParseIP(host); ip != nil {
			ipAddresses[ip.String()] = ip
		} else {
			dnsNames[host] = true
		}
	}

	template.DNSNames = make([]string, 0, len(dnsNames))
	for dnsName := range dnsNames {
		template.DNSNames = append(template.DNSNames, dnsName)
	}
	template.IPAddresses = make([]net.IP, 0, len(ipAddresses))
	for _, ip := range ipAddresses {
		template.IPAddresses = append(template.IPAddresses, ip)
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template,
		&template, &priv.PublicKey, priv)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create certificate: %v\n", err)
	}

	certBuf := &bytes.Buffer{}
	pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})

	keybytes, err := x509.MarshalECPrivateKey(priv)
	if err != nil {
		return nil, nil, err
	}
	keyBuf := &bytes.Buffer{}
	pem.Encode(keyBuf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keybytes})

	return certBuf.Bytes(), keyBuf.Bytes(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/DECK36/go-gelf/gelf"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor                    string `json:"__CURSOR"`
	Realtime_timestamp        int64  `json:"__REALTIME_TIMESTAMP,string"`
	Monotonic_timestamp       string `json:"__MONOTONIC_TIMESTAMP"`
	Boot_id                   string `json:"_BOOT_ID"`
	Transport                 string `json:"_TRANSPORT"`
	Priority                  int32  `json:"PRIORITY,string"`
	Syslog_facility           string `json:"SYSLOG_FACILITY"`
	Syslog_identifier         string `json:"SYSLOG_IDENTIFIER"`
	Message                   string `json:"MESSAGE"`
	Pid                       string `json:"_PID"`
	Uid                       string `json:"_UID"`
	Gid                       string `json:"_GID"`
	Comm                      string `json:"_COMM"`
	Exe                       string `json:"_EXE"`
	Cmdline                   string `json:"_CMDLINE"`
	Systemd_cgroup            string `json:"_SYSTEMD_CGROUP"`
	Systemd_session           string `json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid         string `json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit              string `json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp string `json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id                string `json:"_MACHINE_ID"`
	Hostname                  string `json:"_HOSTNAME"`
	FullMessage               string
}

// Strip date from message-content. Use named subpatterns to override other fields
var messageReplace = map[*regexp.Regexp]string{
	regexp.MustCompile("^20[0-9][0-9]/[01][0-9]/[0123][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] \\[(?P<Priority>[a-z]+)\\] "):            "", //nginx
	regexp.MustCompile("^20[0-9][0-9]-[01][0-9]-[0123][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9],[0-9]{3} (?P<Priority>[A-Z]+) : "):       "", //graylog2-server
	regexp.MustCompile("^[0-9]{6} [0-1]?[0-9]:[0-5][0-9]:[0-5][0-9] \\[(?P<Priority>[A-Z]+)\\] "):                                     "", //mysqld
	regexp.MustCompile("^\\[([A-Z][a-z][a-] ){2} [0-9]+ [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\\.[0-9]{3} 20[0-9][0-9]\\] \\[ [0-9]+ \\] "): "", //sphinx
	regexp.MustCompile("^[A-Z][a-z]{2} [01][0-9], 20[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] [AP]M "):                              "", //jenkins
	regexp.MustCompile("^pool [a-z]+: "):                                                                                "", //php-fpm
}

var priorities = map[string]int32{
	"emergency": 0,
	"emerg":     0,
	"alert":     1,
	"critical":  2,
	"crit":      2,
	"error":     3,
	"err":       3,
	"warning":   4,
	"warn":      4,
	"notice":    5,
	"info":      6,
	"debug":     7,
}

func (this *SystemdJournalEntry) toGelf() *gelf.Message {
	var extra = map[string]interface{}{
		"Boot_id": this.Boot_id,
		"Pid":     this.Pid,
		"Uid":     this.Uid,
	}

	if this.isJsonMessage() {
		if err := json.Unmarshal([]byte(this.Message), &extra); err == nil {
			this.Message = extra["Message"].(string)
		}
	} else if -1 != strings.Index(this.Message, "\n") {
		this.FullMessage = this.Message
		this.Message = strings.Split(this.Message, "\n")[0]
	}

	// php-fpm refuses to fill identifier
	facility := this.Syslog_identifier
	if "" == facility {
		facility = this.Comm
	}

	if this.isJsonMessage() {
		if err := json.Unmarshal([]byte(this.Message), &extra); err == nil {
			if m, ok := extra["Message"]; ok {
				this.Message = m.(string)
				delete(extra, "Message")
			}

			if f, ok := extra["FullMessage"]; ok {
				this.FullMessage = f.(string)
				delete(extra, "FullMessage")
			}
		}
	}

	return &gelf.Message{
		Version:  "1.1",
		Host:     this.Hostname,
		Short:    this.Message,
		Full:     this.FullMessage,
		TimeUnix: float64(this.Realtime_timestamp) / 1000 / 1000,
		Level:    this.Priority,
		Facility: facility,
		Extra:    extra,
	}
}

func (this *SystemdJournalEntry) process() {
	for re, replace := range messageReplace {
		m := re.FindStringSubmatch(this.Message)
		if nil == m {
			continue
		}

		for idx, key := range re.SubexpNames() {
			// no need for reflect, just define desired keys here
			if "Priority" == key {
				this.Priority = priorities[strings.ToLower(m[idx])]
			}
		}

		this.Message = re.ReplaceAllString(this.Message, replace)
	}
}

func (this *SystemdJournalEntry) sameSource(message *SystemdJournalEntry) bool {
	if this.Syslog_identifier != message.Syslog_identifier {
		return false
	}

	if this.Priority != message.Priority {
		return false
	}

	if this.Realtime_timestamp-message.Realtime_timestamp > SAMESOURCE_TIME_DIFFERENCE {
		return false
	}

	return true
}

func (this *SystemdJournalEntry) send() {
	message := this.toGelf()

	if err := gelfWriter.WriteMessage(message); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func (this *SystemdJournalEntry) isJsonMessage() bool {
	return len(this.Message) > 64 && this.Message[0] == '{' && this.Message[1] == '"'
}

func (this *SystemdJournalEntry) extendWith(message *SystemdJournalEntry) {
	if this.FullMessage == "" {
		this.FullMessage = this.Message
	}

	this.FullMessage += "\n" + message.Message
}

var (
	pendingEntry *SystemdJournalEntry
	gelfWriter   *gelf.Writer
)

const (
	WRITE_INTERVAL             = 50 * time.Millisecond
	SAMESOURCE_TIME_DIFFERENCE = 100 * 1000
	// Systemd's inline coredumps are typically ~ 14Kb
	JOURNAL_READER_BUFFER = 16384
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintln(os.Stderr, "Pass server:12201 as first argument and append journalctl parameters to use")
		os.Exit(1)
	}

	serverAddr := os.Args[1]
	journalArgs := []string{"--all", "--output=json"}
	journalArgs = append(journalArgs, os.Args[2:]...)
	cmd := exec.Command("journalctl", journalArgs...)

	var err error
	gelfWriter, err = gelf.NewWriter(serverAddr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "While connecting to Graylog server: %s\n", err)
		os.Exit(1)
	}

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)

	go writePendingEntry()

	r := bufio.NewReaderSize(stdout, JOURNAL_READER_BUFFER)
	cmd.Start()

	for line, _, err := r.ReadLine(); err != io.EOF; line, _, err = r.ReadLine() {
		if err != nil {
			break
		}

		var entry = &SystemdJournalEntry{}
		if err = json.Unmarshal(line, &entry); err != nil {
			//fmt.Fprintf(os.Stderr, "Could not parse line, skipping: %s\n", line)
			continue
		}

		entry.process()

		if pendingEntry == nil {
			pendingEntry = entry
		} else if !pendingEntry.sameSource(entry) || pendingEntry.isJsonMessage() {
			go pendingEntry.send()
			pendingEntry = entry
		} else {
			pendingEntry.extendWith(entry)

			// Keeps writePendingEntry waiting longer for us to append even more
			pendingEntry.Realtime_timestamp = entry.Realtime_timestamp
		}

		// Prevent saturation and throttling
		time.Sleep(1 * time.Millisecond)
	}

	cmd.Wait()
}

/*
 * Sleep for WritePending_interval, then check if
 */
func writePendingEntry() {
	for {
		time.Sleep(WRITE_INTERVAL)

		if pendingEntry != nil && (time.Now().UnixNano()/1000-pendingEntry.Realtime_timestamp) > SAMESOURCE_TIME_DIFFERENCE {
			go pendingEntry.send()
			pendingEntry = nil
		}
	}
}<commit_msg>Removed double isJsonMessage check; moved from r.Readline to s.Scan<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/DECK36/go-gelf/gelf"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor                    string `json:"__CURSOR"`
	Realtime_timestamp        int64  `json:"__REALTIME_TIMESTAMP,string"`
	Monotonic_timestamp       string `json:"__MONOTONIC_TIMESTAMP"`
	Boot_id                   string `json:"_BOOT_ID"`
	Transport                 string `json:"_TRANSPORT"`
	Priority                  int32  `json:"PRIORITY,string"`
	Syslog_facility           string `json:"SYSLOG_FACILITY"`
	Syslog_identifier         string `json:"SYSLOG_IDENTIFIER"`
	Message                   string `json:"MESSAGE"`
	Pid                       string `json:"_PID"`
	Uid                       string `json:"_UID"`
	Gid                       string `json:"_GID"`
	Comm                      string `json:"_COMM"`
	Exe                       string `json:"_EXE"`
	Cmdline                   string `json:"_CMDLINE"`
	Systemd_cgroup            string `json:"_SYSTEMD_CGROUP"`
	Systemd_session           string `json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid         string `json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit              string `json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp string `json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id                string `json:"_MACHINE_ID"`
	Hostname                  string `json:"_HOSTNAME"`
	FullMessage               string
}

// Strip date from message-content. Use named subpatterns to override other fields
var messageReplace = map[*regexp.Regexp]string{
	regexp.MustCompile("^20[0-9][0-9]/[01][0-9]/[0123][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] \\[(?P<Priority>[a-z]+)\\] "):            "", //nginx
	regexp.MustCompile("^20[0-9][0-9]-[01][0-9]-[0123][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9],[0-9]{3} (?P<Priority>[A-Z]+) : "):       "", //graylog2-server
	regexp.MustCompile("^[0-9]{6} [0-1]?[0-9]:[0-5][0-9]:[0-5][0-9] \\[(?P<Priority>[A-Z]+)\\] "):                                     "", //mysqld
	regexp.MustCompile("^\\[([A-Z][a-z][a-] ){2} [0-9]+ [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\\.[0-9]{3} 20[0-9][0-9]\\] \\[ [0-9]+ \\] "): "", //sphinx
	regexp.MustCompile("^[A-Z][a-z]{2} [01][0-9], 20[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] [AP]M "):                              "", //jenkins
	regexp.MustCompile("^pool [a-z]+: "):                                                                                "", //php-fpm
}

var priorities = map[string]int32{
	"emergency": 0,
	"emerg":     0,
	"alert":     1,
	"critical":  2,
	"crit":      2,
	"error":     3,
	"err":       3,
	"warning":   4,
	"warn":      4,
	"notice":    5,
	"info":      6,
	"debug":     7,
}

func (this *SystemdJournalEntry) toGelf() *gelf.Message {
	var extra = map[string]interface{}{
		"Boot_id": this.Boot_id,
		"Pid":     this.Pid,
		"Uid":     this.Uid,
	}

	// php-fpm refuses to fill identifier
	facility := this.Syslog_identifier
	if "" == facility {
		facility = this.Comm
	}

	if this.isJsonMessage() {
		if err := json.Unmarshal([]byte(this.Message), &extra); err == nil {
			if m, ok := extra["Message"]; ok {
				this.Message = m.(string)
				delete(extra, "Message")
			}

			if f, ok := extra["FullMessage"]; ok {
				this.FullMessage = f.(string)
				delete(extra, "FullMessage")
			}
		}
	} else if -1 != strings.Index(this.Message, "\n") {
		this.FullMessage = this.Message
		this.Message = strings.Split(this.Message, "\n")[0]
	}

	return &gelf.Message{
		Version:  "1.1",
		Host:     this.Hostname,
		Short:    this.Message,
		Full:     this.FullMessage,
		TimeUnix: float64(this.Realtime_timestamp) / 1000 / 1000,
		Level:    this.Priority,
		Facility: facility,
		Extra:    extra,
	}
}

func (this *SystemdJournalEntry) process() {
	for re, replace := range messageReplace {
		m := re.FindStringSubmatch(this.Message)
		if m == nil {
			continue
		}

		// Store subpatterns in fields
		for idx, key := range re.SubexpNames() {
			if "Priority" == key {
				this.Priority = priorities[strings.ToLower(m[idx])]
			}
		}

		this.Message = re.ReplaceAllString(this.Message, replace)

		// We won't match multiple replaces
		break
	}
}

func (this *SystemdJournalEntry) sameSource(message *SystemdJournalEntry) bool {
	if this.Syslog_identifier != message.Syslog_identifier {
		return false
	}

	if this.Priority != message.Priority {
		return false
	}

	if this.Realtime_timestamp-message.Realtime_timestamp > SAMESOURCE_TIME_DIFFERENCE {
		return false
	}

	return true
}

func (this *SystemdJournalEntry) send() {
	message := this.toGelf()

	if err := writer.WriteMessage(message); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func (this *SystemdJournalEntry) isJsonMessage() bool {
	return len(this.Message) > 64 && this.Message[0] == '{' && this.Message[1] == '"'
}

func (this *SystemdJournalEntry) extendWith(message *SystemdJournalEntry) {
	if this.FullMessage == "" {
		this.FullMessage = this.Message
	}

	this.FullMessage += "\n" + message.Message
}

var (
	pendingEntry *SystemdJournalEntry
	writer       *gelf.Writer
)

const (
	WRITE_INTERVAL             = 50 * time.Millisecond
	SAMESOURCE_TIME_DIFFERENCE = 100 * 1000
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintln(os.Stderr, "Pass server:12201 as first argument and append journalctl parameters to use")
		os.Exit(1)
	}

	if w, err := gelf.NewWriter(os.Args[1]); err != nil {
		fmt.Fprintf(os.Stderr, "While connecting to Graylog server: %s\n", err)
		os.Exit(1)
	} else {
		writer = w
	}

	journalArgs := []string{"--all", "--output=json"}
	journalArgs = append(journalArgs, os.Args[2:]...)
	cmd := exec.Command("journalctl", journalArgs...)

	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)
	stdout, _ := cmd.StdoutPipe()
	s := bufio.NewScanner(stdout)

	go writePendingEntry()

	cmd.Start()

	for s.Scan() {
		line := s.Text()

		var entry = &SystemdJournalEntry{}
		if err := json.Unmarshal([]byte(line), &entry); err != nil {
			//fmt.Fprintf(os.Stderr, "Could not parse line, skipping: %s\n", line)
			continue
		}

		entry.process()

		if pendingEntry == nil {
			pendingEntry = entry
		} else if !pendingEntry.sameSource(entry) || pendingEntry.isJsonMessage() {
			pendingEntry.send()
			pendingEntry = entry
		} else {
			pendingEntry.extendWith(entry)

			// Keeps writePendingEntry waiting longer for us to append even more
			pendingEntry.Realtime_timestamp = entry.Realtime_timestamp
		}

		// Prevent saturation and throttling
		time.Sleep(1 * time.Millisecond)
	}

	if err := s.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "Error from journalctl: %s\n", err)
	}

	cmd.Wait()
	pendingEntry.send()
}

func writePendingEntry() {
	var entry *SystemdJournalEntry

	for {
		time.Sleep(WRITE_INTERVAL)

		if pendingEntry != nil && (time.Now().UnixNano()/1000-pendingEntry.Realtime_timestamp) > SAMESOURCE_TIME_DIFFERENCE {
			entry = pendingEntry
			pendingEntry = nil

			entry.send()
		}
	}
}<|endoftext|>


<commit_before>package completion

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"strings"
	"unicode"

	"../conio"
	"../lua"
	"../readline"
	/* dbg "github.com/zetamatta/goutputdebugstring" */)

type CompletionList struct {
	AllLine string
	List    []string
	RawWord string // have quotation
	Word    string
	Pos     int
}

var Hook lua.Pushable = lua.TNil{}

func listUpComplete(this *readline.Buffer) (*CompletionList, rune, error) {
	var err error
	rv := CompletionList{}

	// environment completion.
	rv.AllLine = this.String()
	rv.List, rv.Pos, err = listUpEnv(rv.AllLine)
	default_delimiter := rune(readline.Delimiters[0])
	if len(rv.List) > 0 && rv.Pos >= 0 && err == nil {
		rv.RawWord = rv.AllLine[rv.Pos:]
		rv.Word = rv.RawWord
		return &rv, default_delimiter, nil
	}

	// filename or commandname completion
	rv.RawWord, rv.Pos = this.CurrentWord()
	found_delimter := false
	rv.Word = strings.Map(func(c rune) rune {
		if strings.ContainsRune(readline.Delimiters, c) {
			if !found_delimter {
				default_delimiter = c
			}
			found_delimter = true
			return -1
		} else {
			return c
		}
	}, rv.RawWord)
	if rv.Pos > 0 {
		rv.List, err = listUpFiles(rv.Word)
	} else {
		rv.List, err = listUpCommands(rv.Word)
	}

	L, L_ok := this.Context.Value("lua").(lua.Lua)
	if !L_ok {
		return &rv, default_delimiter, errors.New("listUpComplete: could not get lua instance")
	}

	L.Push(Hook)
	if L.IsFunction(-1) {
		L.NewTable()
		L.PushString(rv.RawWord)
		L.SetField(-2, "rawword")
		L.Push(rv.Pos + 1)
		L.SetField(-2, "pos")
		L.PushString(rv.AllLine)
		L.SetField(-2, "text")
		L.PushString(rv.Word)
		L.SetField(-2, "word")
		L.NewTable()
		for key, val := range rv.List {
			L.Push(1 + key)
			L.PushString(val)
			L.SetTable(-3)
		}
		L.SetField(-2, "list")
		if err := L.Call(1, 1); err != nil {
			fmt.Println(err)
		}
		if L.IsTable(-1) {
			list := make([]string, 0, len(rv.List)+32)
			wordUpr := strings.ToUpper(rv.Word)
			for i := 1; true; i++ {
				L.Push(i)
				L.GetTable(-2)
				str, strErr := L.ToString(-1)
				L.Pop(1)
				if strErr != nil || str == "" {
					break
				}
				strUpr := strings.ToUpper(str)
				if strings.HasPrefix(strUpr, wordUpr) {
					list = append(list, str)
				}
			}
			if len(list) > 0 {
				rv.List = list
			}
		}
	}
	L.Pop(1) // remove something not function or result-table
	return &rv, default_delimiter, err
}

func KeyFuncCompletionList(ctx context.Context, this *readline.Buffer) readline.Result {
	comp, _, err := listUpComplete(this)
	if comp == nil {
		return readline.CONTINUE
	}
	fmt.Print("\n")
	os.Stdout.Sync()
	if err != nil {
		fmt.Printf("(warning) %s\n", err.Error())
		os.Stderr.Sync()
	}
	conio.BoxPrint(ctx, comp.List, os.Stdout)
	this.RepaintAll()
	return readline.CONTINUE
}

func CommonPrefix(list []string) string {
	if len(list) < 1 {
		return ""
	}
	common := list[0]
	for _, f := range list[1:] {
		cr := strings.NewReader(common)
		fr := strings.NewReader(f)
		i := 0
		var buffer bytes.Buffer
		for {
			ch, _, cerr := cr.ReadRune()
			fh, _, ferr := fr.ReadRune()
			if cerr != nil || ferr != nil || unicode.ToUpper(ch) != unicode.ToUpper(fh) {
				break
			}
			buffer.WriteRune(ch)
			i++
		}
		common = buffer.String()
	}
	return common
}

func endWithRoot(path string) bool {
	return strings.HasSuffix(path, "\\") || strings.HasSuffix(path, "/")
}

func KeyFuncCompletion(this *readline.Buffer) readline.Result {
	comp, default_delimiter, err := listUpComplete(this)
	if comp.List == nil || len(comp.List) <= 0 {
		return readline.CONTINUE
	}

	slashToBackSlash := true
	firstFoundSlashPos := strings.IndexRune(comp.Word, '/')
	firstFoundBackSlashPos := strings.IndexRune(comp.Word, '\\')
	if firstFoundSlashPos >= 0 && (firstFoundBackSlashPos == -1 || firstFoundSlashPos < firstFoundBackSlashPos) {
		slashToBackSlash = false
	}

	commonStr := CommonPrefix(comp.List)
	quotechar := byte(0)
	if i := strings.IndexAny(comp.Word, readline.Delimiters); i >= 0 {
		quotechar = comp.Word[i]
	} else {
		for _, node := range comp.List {
			if strings.ContainsAny(node, " &") {
				quotechar = byte(default_delimiter)
				break
			}
		}
	}
	if quotechar != 0 {
		buffer := make([]byte, 0, 100)
		buffer = append(buffer, quotechar)
		buffer = append(buffer, commonStr...)
		if len(comp.List) == 1 && !endWithRoot(comp.List[0]) {
			buffer = append(buffer, quotechar)
		}
		commonStr = string(buffer)
	}
	if len(comp.List) == 1 && !endWithRoot(commonStr) {
		commonStr += " "
	}
	if slashToBackSlash {
		commonStr = strings.Replace(commonStr, "/", "\\", -1)
	}
	if comp.RawWord == commonStr {
		fmt.Print("\n")
		if err != nil {
			fmt.Printf("(warning) %s\n", err.Error())
		}
		conio.BoxPrint(nil, comp.List, os.Stdout)
		this.RepaintAll()
		return readline.CONTINUE
	}
	this.ReplaceAndRepaint(comp.Pos, commonStr)
	return readline.CONTINUE
}
<commit_msg>At completion, ignore the string before `;`,`=` (close #180)<commit_after>package completion

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"strings"
	"unicode"

	"../conio"
	"../lua"
	"../readline"
	/* dbg "github.com/zetamatta/goutputdebugstring" */)

type CompletionList struct {
	AllLine string
	List    []string
	RawWord string // have quotation
	Word    string
	Pos     int
}

var Hook lua.Pushable = lua.TNil{}

func listUpComplete(this *readline.Buffer) (*CompletionList, rune, error) {
	var err error
	rv := CompletionList{}

	// environment completion.
	rv.AllLine = this.String()
	rv.List, rv.Pos, err = listUpEnv(rv.AllLine)
	default_delimiter := rune(readline.Delimiters[0])
	if len(rv.List) > 0 && rv.Pos >= 0 && err == nil {
		rv.RawWord = rv.AllLine[rv.Pos:]
		rv.Word = rv.RawWord
		return &rv, default_delimiter, nil
	}

	// filename or commandname completion
	rv.RawWord, rv.Pos = this.CurrentWord()
	found_delimter := false
	rv.Word = strings.Map(func(c rune) rune {
		if strings.ContainsRune(readline.Delimiters, c) {
			if !found_delimter {
				default_delimiter = c
			}
			found_delimter = true
			return -1
		} else {
			return c
		}
	}, rv.RawWord)

	start := strings.LastIndexAny(rv.Word, ";=") + 1

	if rv.Pos > 0 {
		rv.List, err = listUpFiles(rv.Word[start:])
	} else {
		rv.List, err = listUpCommands(rv.Word[start:])
	}

	for i := 0; i < len(rv.List); i++ {
		rv.List[i] = rv.Word[:start] + rv.List[i]
	}

	L, L_ok := this.Context.Value("lua").(lua.Lua)
	if !L_ok {
		return &rv, default_delimiter, errors.New("listUpComplete: could not get lua instance")
	}

	L.Push(Hook)
	if L.IsFunction(-1) {
		L.NewTable()
		L.PushString(rv.RawWord)
		L.SetField(-2, "rawword")
		L.Push(rv.Pos + 1)
		L.SetField(-2, "pos")
		L.PushString(rv.AllLine)
		L.SetField(-2, "text")
		L.PushString(rv.Word)
		L.SetField(-2, "word")
		L.NewTable()
		for key, val := range rv.List {
			L.Push(1 + key)
			L.PushString(val)
			L.SetTable(-3)
		}
		L.SetField(-2, "list")
		if err := L.Call(1, 1); err != nil {
			fmt.Println(err)
		}
		if L.IsTable(-1) {
			list := make([]string, 0, len(rv.List)+32)
			wordUpr := strings.ToUpper(rv.Word)
			for i := 1; true; i++ {
				L.Push(i)
				L.GetTable(-2)
				str, strErr := L.ToString(-1)
				L.Pop(1)
				if strErr != nil || str == "" {
					break
				}
				strUpr := strings.ToUpper(str)
				if strings.HasPrefix(strUpr, wordUpr) {
					list = append(list, str)
				}
			}
			if len(list) > 0 {
				rv.List = list
			}
		}
	}
	L.Pop(1) // remove something not function or result-table
	return &rv, default_delimiter, err
}

func KeyFuncCompletionList(ctx context.Context, this *readline.Buffer) readline.Result {
	comp, _, err := listUpComplete(this)
	if comp == nil {
		return readline.CONTINUE
	}
	fmt.Print("\n")
	os.Stdout.Sync()
	if err != nil {
		fmt.Printf("(warning) %s\n", err.Error())
		os.Stderr.Sync()
	}
	conio.BoxPrint(ctx, comp.List, os.Stdout)
	this.RepaintAll()
	return readline.CONTINUE
}

func CommonPrefix(list []string) string {
	if len(list) < 1 {
		return ""
	}
	common := list[0]
	for _, f := range list[1:] {
		cr := strings.NewReader(common)
		fr := strings.NewReader(f)
		i := 0
		var buffer bytes.Buffer
		for {
			ch, _, cerr := cr.ReadRune()
			fh, _, ferr := fr.ReadRune()
			if cerr != nil || ferr != nil || unicode.ToUpper(ch) != unicode.ToUpper(fh) {
				break
			}
			buffer.WriteRune(ch)
			i++
		}
		common = buffer.String()
	}
	return common
}

func endWithRoot(path string) bool {
	return strings.HasSuffix(path, "\\") || strings.HasSuffix(path, "/")
}

func KeyFuncCompletion(this *readline.Buffer) readline.Result {
	comp, default_delimiter, err := listUpComplete(this)
	if comp.List == nil || len(comp.List) <= 0 {
		return readline.CONTINUE
	}

	slashToBackSlash := true
	firstFoundSlashPos := strings.IndexRune(comp.Word, '/')
	firstFoundBackSlashPos := strings.IndexRune(comp.Word, '\\')
	if firstFoundSlashPos >= 0 && (firstFoundBackSlashPos == -1 || firstFoundSlashPos < firstFoundBackSlashPos) {
		slashToBackSlash = false
	}

	commonStr := CommonPrefix(comp.List)
	quotechar := byte(0)
	if i := strings.IndexAny(comp.Word, readline.Delimiters); i >= 0 {
		quotechar = comp.Word[i]
	} else {
		for _, node := range comp.List {
			if strings.ContainsAny(node, " &") {
				quotechar = byte(default_delimiter)
				break
			}
		}
	}
	if quotechar != 0 {
		buffer := make([]byte, 0, 100)
		buffer = append(buffer, quotechar)
		buffer = append(buffer, commonStr...)
		if len(comp.List) == 1 && !endWithRoot(comp.List[0]) {
			buffer = append(buffer, quotechar)
		}
		commonStr = string(buffer)
	}
	if len(comp.List) == 1 && !endWithRoot(commonStr) {
		commonStr += " "
	}
	if slashToBackSlash {
		commonStr = strings.Replace(commonStr, "/", "\\", -1)
	}
	if comp.RawWord == commonStr {
		fmt.Print("\n")
		if err != nil {
			fmt.Printf("(warning) %s\n", err.Error())
		}
		conio.BoxPrint(nil, comp.List, os.Stdout)
		this.RepaintAll()
		return readline.CONTINUE
	}
	this.ReplaceAndRepaint(comp.Pos, commonStr)
	return readline.CONTINUE
}
<|endoftext|>


<commit_before>package twitch

import (
	"time"
)

type Channel struct {
	Mature                       bool      `json:"mature"`
	Status                       string    `json:"status"`
	BroadcasterLanguage          string    `json:"broadcaster_language"`
	DisplayName                  string    `json:"display_name"`
	Game                         string    `json:"game"`
	Delay                        int64     `json:"delay"`
	Language                     string    `json:"language"`
	Id                           int64     `json:"_id"`
	Name                         string    `json:"name"`
	CreatedAt                    time.Time `json:"created_at"`
	UpdatedAt                    time.Time `json:"updated_at"`
	Logo                         string    `json:"logo"`
	Banner                       string    `json:"banner"`
	VideoBanner                  string    `json:"video_banner"`
	ProfileBanner                string    `json:"profile_banner"`
	ProfileBannerBackgroundColor string    `json:"profile_banner_background_color"`
	Partner                      bool      `json:"partner"`
	Views                        int64     `json:"views"`
	Followers                    int64     `json:"followers"`
}
<commit_msg>Add function to get channel information<commit_after>package twitch

import (
	"fmt"
	"time"
)

type Channel struct {
	Mature                       bool      `json:"mature"`
	Status                       string    `json:"status"`
	BroadcasterLanguage          string    `json:"broadcaster_language"`
	DisplayName                  string    `json:"display_name"`
	Game                         string    `json:"game"`
	Delay                        int64     `json:"delay"`
	Language                     string    `json:"language"`
	Id                           int64     `json:"_id"`
	Name                         string    `json:"name"`
	CreatedAt                    time.Time `json:"created_at"`
	UpdatedAt                    time.Time `json:"updated_at"`
	Logo                         string    `json:"logo"`
	Banner                       string    `json:"banner"`
	VideoBanner                  string    `json:"video_banner"`
	ProfileBanner                string    `json:"profile_banner"`
	ProfileBannerBackgroundColor string    `json:"profile_banner_background_color"`
	Partner                      bool      `json:"partner"`
	Views                        int64     `json:"views"`
	Followers                    int64     `json:"followers"`
}

// GetChannel requests channel information from Twitch.
// It returns a Channel struct if successful and any error encountered.
func (client *TwitchClient) GetChannel(channel string) (*Channel, error) {
	res := Channel{}

	err := client.getRequest(fmt.Sprintf("/channels/%s", channel), nil, &res)
	if err != nil {
		return nil, err
	}

	return &res, nil
}
<|endoftext|>


<commit_before>package btree

import (
	"bytes"
	"encoding/binary"
	"math/rand"
	"testing"

	"github.com/avisagie/indexes"
)

func TestBtreeCreate(t *testing.T) {
	var index indexes.Index
	index = NewInMemoryBtree()
	t.Log(index)
	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func TestKeys(t *testing.T) {
	key1 := []byte{5, 5, 5}
	key2 := []byte{6, 5, 5}
	var empty []byte
	if keyLess(key2, key1) {
		t.Error("1")
	}
	if !keyLess(key1, key2) {
		t.Error("2")
	}
	if keyLess(key1, key1) {
		t.Error("3")
	}
	if keyLess(key1, empty) || keyLess(key2, empty) {
		t.Error("4")
	}
	if !keyLess(empty, key1) || !keyLess(empty, key2) {
		t.Error("5")
	}
}

func TestBtreeSearchEmpty(t *testing.T) {
	index := NewInMemoryBtree()
	value, ok := index.Get([]byte{1, 2, 3})
	if ok {
		t.Error("Did not expect to find anything")
	}
	if value != nil {
		t.Error("Should not have a value")
	}
	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func TestBtreeInsert1(t *testing.T) {
	index := NewInMemoryBtree()
	value, ok := index.Get([]byte{1, 2, 3})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}

	k, v := []byte{1, 2, 3}, []byte{4, 5, 6}
	replaced := index.Put(k, v)
	if replaced {
		t.Fatal("Empty btree, could not have inserted anything")
	}

	value, ok = index.Get([]byte{1, 2, 3})
	if !ok {
		t.Fatal("Expected to find it", index)
	}
	if bytes.Compare(v, value) != 0 {
		t.Fatal("Got wrong value out")
	}

	value, ok = index.Get([]byte{5})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}

	replaced = index.Put(k, v)
	if !replaced {
		t.Fatal("Expected it to have been replaced")
	}

	replaced = index.Put([]byte{3, 2, 1}, v)
	if replaced {
		t.Fatal("Expected it to NOT have been replaced")
	}

	value, ok = index.Get([]byte{1, 2, 3})
	if !ok {
		t.Fatal("Expected to find it")
	}
	if bytes.Compare(v, value) != 0 {
		t.Fatal("Got wrong value out")
	}

	value, ok = index.Get([]byte{5})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}

	t.Log(index)
}

func TestBtreeAppend(t *testing.T) {
	index := NewInMemoryBtree()
	index.Put([]byte{1, 2, 3}, []byte{4, 5, 6})
	index.Put([]byte{1, 2, 1}, []byte{6, 5, 4})
	index.Append([]byte{1, 2, 3}, []byte{7, 8, 9})

	value, ok := index.Get([]byte{1, 2, 3})
	expected := []byte{4, 5, 6, 7, 8, 9}
	if !ok || bytes.Compare(value, expected) != 0 {
		t.Fatal("Expected", expected, ", got", ok, value)
	}

	t.Log(index)

	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func TestBtreeOverride(t *testing.T) {
	index := NewInMemoryBtree()
	value, ok := index.Get([]byte{1, 2, 3})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}
	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func pfxm(t *testing.T, k, pfx []byte, must bool) {
	if prefixMatches(k, pfx) != must {
		t.Fatal("k =", k, "pfx =", pfx, "expected", must, "got", !must)
	}
}

func TestPrefixMatches(t *testing.T) {
	pfxm(t, []byte{}, []byte{}, true)
	pfxm(t, []byte{1}, []byte{1}, true)
	pfxm(t, []byte{1, 2}, []byte{1}, true)
	pfxm(t, []byte{1, 2}, []byte{1, 2}, true)
	pfxm(t, []byte{1, 2, 3, 4, 5, 6}, []byte{1, 2}, true)
	pfxm(t, []byte{1, 2}, []byte{1, 2, 3}, false)
	pfxm(t, []byte{1, 2}, []byte{2, 1}, false)
	pfxm(t, []byte{1, 2}, []byte{2}, false)
}

func TestShortIter(t *testing.T) {
	index := NewInMemoryBtree()
	index.Put([]byte{1, 2, 3}, []byte{1, 2, 3})
	index.Put([]byte{1, 2, 4}, []byte{1, 2, 3})
	index.Put([]byte{1, 2, 5}, []byte{1, 2, 3})

	t.Log("[]byte{1, 2}:")
	count := 0
	it := index.Start([]byte{1, 2})
	for {
		k, v, ok := it.Next()
		if !ok {
			break
		}
		t.Log("  ", k, "=>", v)
		count += 1
	}
	if count != 3 {
		t.Fatal("Expected 3, got", count)
	}

	t.Log("[]byte{2}:")
	count = 0
	it = index.Start([]byte{2})
	for {
		k, v, ok := it.Next()
		if !ok {
			break
		}
		t.Log("  ", k, "=>", v)
		count += 1
	}
	if count != 0 {
		t.Fatal("Expected 0, got", count)
	}

	t.Log("[]byte{1, 2, 5}:")
	count = 0
	it = index.Start([]byte{1, 2, 5})
	for {
		k, v, ok := it.Next()
		if !ok {
			break
		}
		t.Log("  ", k, "=>", v)
		count += 1
	}
	if count != 1 {
		t.Fatal("Expected 1, got", count)
	}

	for ii := 0; ii < 100; ii++ {
		// After iteration it must remain in the done state and not do anything else
		k, v, ok := it.Next()
		if ok || k != nil || v != nil {
			t.Fatal(ok, k, v)
		}
	}
}

func fill(t *testing.T, index indexes.Index) (keys [][]byte) {
	buffer := &bytes.Buffer{}
	count := int32(0)

	keys = make([][]byte, 0)

	for ; count < int32(30*pageSize/(4+4+4)+5); count++ {
		binary.Write(buffer, binary.LittleEndian, count)
		b := copyBytes(buffer.Bytes())
		keys = append(keys, b)
		buffer.Reset()
	}

	// randomize ish
	for i := range keys {
		j := rand.Int31n(int32(len(keys)))
		keys[i], keys[j] = keys[j], keys[i]
	}

	for _, b := range keys {
		index.Put(b, b)
	}

	//	pager := index.(*Btree).pager.(*ramPager)
	//	for i, p := range pager.pages {
	//		t.Log(i, len(p.keys), p.keys)
	//	}

	for _, k := range keys {
		v, ok := index.Get(k)
		if !ok || bytes.Compare(k, v) != 0 {
			t.Fatal("Expected", k, "got", v)
		}
	}

	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}

	return
}

func TestLarger(t *testing.T) {
	index := NewInMemoryBtree()
	keys := fill(t, index)

	// iteration
	x := 0
	iter := index.Start([]byte{})
	prev := []byte{}
	for {
		k, v, ok := iter.Next()
		if !ok {
			break
		}
		if !keyLess(prev, k) {
			t.Fatal("Expected ", prev, " < ", k)
		}
		if bytes.Compare(k, v) != 0 {
			t.Fatal("Inserted values equal to keys. Got ", k, "=>", v)
		}
		x++
	}

	if x != len(keys) {
		t.Fatal("Expected", len(keys), "got", x)
	}
}

func TestIteration2(t *testing.T) {
	index := NewInMemoryBtree()
	fill(t, index)

	iter := index.Start([]byte{4})
	for {
		k, v, ok := iter.Next()
		if !ok {
			break
		}
		if k[0] != 4 {
			t.Fatal("Expected something with prefix 4")
		}
		t.Log(k, "=>", v)
	}
}

func TestBulk(t *testing.T) {
	index1 := NewInMemoryBtree()
	fill(t, index1)

	index2 := NewInMemoryBtree()
	bt := index2.(*Btree)

	iter := index1.Start([]byte{})
	for {
		k, v, ok := iter.Next()
		if !ok {
			break
		}
		bt.PutNext(k, v)
	}

	if err := bt.CheckConsistency(); err != nil {
		t.Fatal(err)
	}

	iter1 := index1.Start([]byte{})
	iter2 := index2.Start([]byte{})
	for count := 1; ; count++ {
		k1, v1, ok1 := iter1.Next()
		k2, v2, ok2 := iter2.Next()

		if ok1 != ok2 || bytes.Compare(k1, k2) != 0 || bytes.Compare(v1, v2) != 0 {
			t.Fatal("Not the same:", ok1, ok2, k1, k2, v1, v2)
		}

		if !ok1 {
			break
		}
	}
	t.Log("Bulk filled used pages:", len(bt.pager.(*inplacePager).pages))
	t.Log("Random filled used pages:", len(index1.(*Btree).pager.(*inplacePager).pages))
}
<commit_msg>add bulk load benchmark<commit_after>package btree

import (
	"bytes"
	"encoding/binary"
	"math/rand"
	"testing"

	"github.com/avisagie/indexes"
)

func TestBtreeCreate(t *testing.T) {
	var index indexes.Index
	index = NewInMemoryBtree()
	t.Log(index)
	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func TestKeys(t *testing.T) {
	key1 := []byte{5, 5, 5}
	key2 := []byte{6, 5, 5}
	var empty []byte
	if keyLess(key2, key1) {
		t.Error("1")
	}
	if !keyLess(key1, key2) {
		t.Error("2")
	}
	if keyLess(key1, key1) {
		t.Error("3")
	}
	if keyLess(key1, empty) || keyLess(key2, empty) {
		t.Error("4")
	}
	if !keyLess(empty, key1) || !keyLess(empty, key2) {
		t.Error("5")
	}
}

func TestBtreeSearchEmpty(t *testing.T) {
	index := NewInMemoryBtree()
	value, ok := index.Get([]byte{1, 2, 3})
	if ok {
		t.Error("Did not expect to find anything")
	}
	if value != nil {
		t.Error("Should not have a value")
	}
	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func TestBtreeInsert1(t *testing.T) {
	index := NewInMemoryBtree()
	value, ok := index.Get([]byte{1, 2, 3})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}

	k, v := []byte{1, 2, 3}, []byte{4, 5, 6}
	replaced := index.Put(k, v)
	if replaced {
		t.Fatal("Empty btree, could not have inserted anything")
	}

	value, ok = index.Get([]byte{1, 2, 3})
	if !ok {
		t.Fatal("Expected to find it", index)
	}
	if bytes.Compare(v, value) != 0 {
		t.Fatal("Got wrong value out")
	}

	value, ok = index.Get([]byte{5})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}

	replaced = index.Put(k, v)
	if !replaced {
		t.Fatal("Expected it to have been replaced")
	}

	replaced = index.Put([]byte{3, 2, 1}, v)
	if replaced {
		t.Fatal("Expected it to NOT have been replaced")
	}

	value, ok = index.Get([]byte{1, 2, 3})
	if !ok {
		t.Fatal("Expected to find it")
	}
	if bytes.Compare(v, value) != 0 {
		t.Fatal("Got wrong value out")
	}

	value, ok = index.Get([]byte{5})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}

	t.Log(index)
}

func TestBtreeAppend(t *testing.T) {
	index := NewInMemoryBtree()
	index.Put([]byte{1, 2, 3}, []byte{4, 5, 6})
	index.Put([]byte{1, 2, 1}, []byte{6, 5, 4})
	index.Append([]byte{1, 2, 3}, []byte{7, 8, 9})

	value, ok := index.Get([]byte{1, 2, 3})
	expected := []byte{4, 5, 6, 7, 8, 9}
	if !ok || bytes.Compare(value, expected) != 0 {
		t.Fatal("Expected", expected, ", got", ok, value)
	}

	t.Log(index)

	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func TestBtreeOverride(t *testing.T) {
	index := NewInMemoryBtree()
	value, ok := index.Get([]byte{1, 2, 3})
	if ok {
		t.Fatal("Did not expect to find anything")
	}
	if value != nil {
		t.Fatal("Should not have a value")
	}
	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}
}

func pfxm(t *testing.T, k, pfx []byte, must bool) {
	if prefixMatches(k, pfx) != must {
		t.Fatal("k =", k, "pfx =", pfx, "expected", must, "got", !must)
	}
}

func TestPrefixMatches(t *testing.T) {
	pfxm(t, []byte{}, []byte{}, true)
	pfxm(t, []byte{1}, []byte{1}, true)
	pfxm(t, []byte{1, 2}, []byte{1}, true)
	pfxm(t, []byte{1, 2}, []byte{1, 2}, true)
	pfxm(t, []byte{1, 2, 3, 4, 5, 6}, []byte{1, 2}, true)
	pfxm(t, []byte{1, 2}, []byte{1, 2, 3}, false)
	pfxm(t, []byte{1, 2}, []byte{2, 1}, false)
	pfxm(t, []byte{1, 2}, []byte{2}, false)
}

func TestShortIter(t *testing.T) {
	index := NewInMemoryBtree()
	index.Put([]byte{1, 2, 3}, []byte{1, 2, 3})
	index.Put([]byte{1, 2, 4}, []byte{1, 2, 3})
	index.Put([]byte{1, 2, 5}, []byte{1, 2, 3})

	t.Log("[]byte{1, 2}:")
	count := 0
	it := index.Start([]byte{1, 2})
	for {
		k, v, ok := it.Next()
		if !ok {
			break
		}
		t.Log("  ", k, "=>", v)
		count += 1
	}
	if count != 3 {
		t.Fatal("Expected 3, got", count)
	}

	t.Log("[]byte{2}:")
	count = 0
	it = index.Start([]byte{2})
	for {
		k, v, ok := it.Next()
		if !ok {
			break
		}
		t.Log("  ", k, "=>", v)
		count += 1
	}
	if count != 0 {
		t.Fatal("Expected 0, got", count)
	}

	t.Log("[]byte{1, 2, 5}:")
	count = 0
	it = index.Start([]byte{1, 2, 5})
	for {
		k, v, ok := it.Next()
		if !ok {
			break
		}
		t.Log("  ", k, "=>", v)
		count += 1
	}
	if count != 1 {
		t.Fatal("Expected 1, got", count)
	}

	for ii := 0; ii < 100; ii++ {
		// After iteration it must remain in the done state and not do anything else
		k, v, ok := it.Next()
		if ok || k != nil || v != nil {
			t.Fatal(ok, k, v)
		}
	}
}

func fill(t *testing.T, index indexes.Index) (keys [][]byte) {
	buffer := &bytes.Buffer{}
	count := int32(0)

	keys = make([][]byte, 0)

	for ; count < int32(30*pageSize/(4+4+4)+5); count++ {
		binary.Write(buffer, binary.LittleEndian, count)
		b := copyBytes(buffer.Bytes())
		keys = append(keys, b)
		buffer.Reset()
	}

	// randomize ish
	for i := range keys {
		j := rand.Int31n(int32(len(keys)))
		keys[i], keys[j] = keys[j], keys[i]
	}

	for _, b := range keys {
		index.Put(b, b)
	}

	//	pager := index.(*Btree).pager.(*ramPager)
	//	for i, p := range pager.pages {
	//		t.Log(i, len(p.keys), p.keys)
	//	}

	for _, k := range keys {
		v, ok := index.Get(k)
		if !ok || bytes.Compare(k, v) != 0 {
			t.Fatal("Expected", k, "got", v)
		}
	}

	if err := index.(*Btree).CheckConsistency(); err != nil {
		t.Fatal(err)
	}

	return
}

func BenchmarkBulkLoad(b *testing.B) {
	buffer := &bytes.Buffer{}
	count := int32(0)

	index := NewInMemoryBtree()
	keys := make([][]byte, 0)

	for ; count < int32(30*pageSize/(4+4+4)+5); count++ {
		binary.Write(buffer, binary.LittleEndian, count)
		b := copyBytes(buffer.Bytes())
		keys = append(keys, b)
		buffer.Reset()
	}

	// randomize ish
	for i, pos := range rand.Perm(len(keys)) {
		keys[i], keys[pos] = keys[pos], keys[i]
	}

	b.ResetTimer()

	// if b.N < 1e9 {
	// 	b.N = 1e9
	// }
	for i := 0; i < b.N; i++ {
		k := keys[i%len(keys)]
		v := k
		index.Put(k, v)
	}
}

func TestLarger(t *testing.T) {
	index := NewInMemoryBtree()
	keys := fill(t, index)

	// iteration
	x := 0
	iter := index.Start([]byte{})
	prev := []byte{}
	for {
		k, v, ok := iter.Next()
		if !ok {
			break
		}
		if !keyLess(prev, k) {
			t.Fatal("Expected ", prev, " < ", k)
		}
		if bytes.Compare(k, v) != 0 {
			t.Fatal("Inserted values equal to keys. Got ", k, "=>", v)
		}
		x++
	}

	if x != len(keys) {
		t.Fatal("Expected", len(keys), "got", x)
	}
}

func TestIteration2(t *testing.T) {
	index := NewInMemoryBtree()
	fill(t, index)

	iter := index.Start([]byte{4})
	for {
		k, v, ok := iter.Next()
		if !ok {
			break
		}
		if k[0] != 4 {
			t.Fatal("Expected something with prefix 4")
		}
		t.Log(k, "=>", v)
	}
}

func TestBulk(t *testing.T) {
	index1 := NewInMemoryBtree()
	fill(t, index1)

	index2 := NewInMemoryBtree()
	bt := index2.(*Btree)

	iter := index1.Start([]byte{})
	for {
		k, v, ok := iter.Next()
		if !ok {
			break
		}
		bt.PutNext(k, v)
	}

	if err := bt.CheckConsistency(); err != nil {
		t.Fatal(err)
	}

	iter1 := index1.Start([]byte{})
	iter2 := index2.Start([]byte{})
	for count := 1; ; count++ {
		k1, v1, ok1 := iter1.Next()
		k2, v2, ok2 := iter2.Next()

		if ok1 != ok2 || bytes.Compare(k1, k2) != 0 || bytes.Compare(v1, v2) != 0 {
			t.Fatal("Not the same:", ok1, ok2, k1, k2, v1, v2)
		}

		if !ok1 {
			break
		}
	}
	t.Log("Bulk filled used pages:", len(bt.pager.(*inplacePager).pages))
	t.Log("Random filled used pages:", len(index1.(*Btree).pager.(*inplacePager).pages))
}
<|endoftext|>