<commit_before>package graph

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/log"
	"github.com/docker/docker/pkg/truncindex"
	"github.com/docker/docker/runconfig"
	"github.com/docker/docker/utils"
)

// A Graph is a store for versioned filesystem images and the relationship between them.
type Graph struct {
	Root    string
	idIndex *truncindex.TruncIndex
	driver  graphdriver.Driver
}

// NewGraph instantiates a new graph at the given root path in the filesystem.
// `root` will be created if it doesn't exist.
func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	// Create the root directory if it doesn't exists
	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
		return nil, err
	}

	graph := &Graph{
		Root:    abspath,
		idIndex: truncindex.NewTruncIndex([]string{}),
		driver:  driver,
	}
	if err := graph.restore(); err != nil {
		return nil, err
	}
	return graph, nil
}

func (graph *Graph) restore() error {
	dir, err := ioutil.ReadDir(graph.Root)
	if err != nil {
		return err
	}
	var ids = []string{}
	for _, v := range dir {
		id := v.Name()
		if graph.driver.Exists(id) {
			ids = append(ids, id)
		}
	}
	graph.idIndex = truncindex.NewTruncIndex(ids)
	log.Debugf("Restored %d elements", len(dir))
	return nil
}

// FIXME: Implement error subclass instead of looking at the error text
// Note: This is the way golang implements os.IsNotExists on Plan9
func (graph *Graph) IsNotExist(err error) bool {
	return err != nil && (strings.Contains(err.Error(), "does not exist") || strings.Contains(err.Error(), "No such"))
}

// Exists returns true if an image is registered at the given id.
// If the image doesn't exist or if an error is encountered, false is returned.
func (graph *Graph) Exists(id string) bool {
	if _, err := graph.Get(id); err != nil {
		return false
	}
	return true
}

// Get returns the image with the given id, or an error if the image doesn't exist.
func (graph *Graph) Get(name string) (*image.Image, error) {
	id, err := graph.idIndex.Get(name)
	if err != nil {
		return nil, err
	}
	img, err := image.LoadImage(graph.ImageRoot(id))
	if err != nil {
		return nil, err
	}
	if img.ID != id {
		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
	}
	img.SetGraph(graph)

	if img.Size < 0 {
		size, err := graph.driver.DiffSize(img.ID, img.Parent)
		if err != nil {
			return nil, fmt.Errorf("unable to calculate size of image id %q: %s", img.ID, err)
		}

		img.Size = size
		if err := img.SaveSize(graph.ImageRoot(id)); err != nil {
			return nil, err
		}
	}
	return img, nil
}

// Create creates a new image and registers it in the graph.
func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) {
	img := &image.Image{
		ID:            utils.GenerateRandomID(),
		Comment:       comment,
		Created:       time.Now().UTC(),
		DockerVersion: dockerversion.VERSION,
		Author:        author,
		Config:        config,
		Architecture:  runtime.GOARCH,
		OS:            runtime.GOOS,
	}

	if containerID != "" {
		img.Parent = containerImage
		img.Container = containerID
		img.ContainerConfig = *containerConfig
	}

	if err := graph.Register(img, nil, layerData); err != nil {
		return nil, err
	}
	return img, nil
}

// Register imports a pre-existing image into the graph.
func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archive.ArchiveReader) (err error) {
	defer func() {
		// If any error occurs, remove the new dir from the driver.
		// Don't check for errors since the dir might not have been created.
		// FIXME: this leaves a possible race condition.
		if err != nil {
			graph.driver.Remove(img.ID)
		}
	}()
	if err := utils.ValidateID(img.ID); err != nil {
		return err
	}
	// (This is a convenience to save time. Race conditions are taken care of by os.Rename)
	if graph.Exists(img.ID) {
		return fmt.Errorf("Image %s already exists", img.ID)
	}

	// Ensure that the image root does not exist on the filesystem
	// when it is not registered in the graph.
	// This is common when you switch from one graph driver to another
	if err := os.RemoveAll(graph.ImageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
		return err
	}

	// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
	// (the graph is the source of truth).
	// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
	// (FIXME: make that mandatory for drivers).
	graph.driver.Remove(img.ID)

	tmp, err := graph.Mktemp("")
	defer os.RemoveAll(tmp)
	if err != nil {
		return fmt.Errorf("Mktemp failed: %s", err)
	}

	// Create root filesystem in the driver
	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
	}
	// Apply the diff/layer
	img.SetGraph(graph)
	if err := image.StoreImage(img, jsonData, layerData, tmp); err != nil {
		return err
	}
	// Commit
	if err := os.Rename(tmp, graph.ImageRoot(img.ID)); err != nil {
		return err
	}
	graph.idIndex.Add(img.ID)
	return nil
}

// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
//   The archive is stored on disk and will be automatically deleted as soon as has been read.
//   If output is not nil, a human-readable progress bar will be written to it.
//   FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, sf *utils.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
	image, err := graph.Get(id)
	if err != nil {
		return nil, err
	}
	tmp, err := graph.Mktemp("")
	if err != nil {
		return nil, err
	}
	a, err := image.TarLayer()
	if err != nil {
		return nil, err
	}
	progress := utils.ProgressReader(a, 0, output, sf, false, utils.TruncateID(id), "Buffering to disk")
	defer progress.Close()
	return archive.NewTempArchive(progress, tmp)
}

// Mktemp creates a temporary sub-directory inside the graph's filesystem.
func (graph *Graph) Mktemp(id string) (string, error) {
	dir := path.Join(graph.Root, "_tmp", utils.GenerateRandomID())
	if err := os.MkdirAll(dir, 0700); err != nil {
		return "", err
	}
	return dir, nil
}

// setupInitLayer populates a directory with mountpoints suitable
// for bind-mounting dockerinit into the container. The mountpoint is simply an
// empty file at /.dockerinit
//
// This extra layer is used by all containers as the top-most ro layer. It protects
// the container from unwanted side-effects on the rw layer.
func SetupInitLayer(initLayer string) error {
	for pth, typ := range map[string]string{
		"/dev/pts":         "dir",
		"/dev/shm":         "dir",
		"/proc":            "dir",
		"/sys":             "dir",
		"/.dockerinit":     "file",
		"/.dockerenv":      "file",
		"/etc/resolv.conf": "file",
		"/etc/hosts":       "file",
		"/etc/hostname":    "file",
		"/dev/console":     "file",
		"/etc/mtab":        "/proc/mounts",
	} {
		parts := strings.Split(pth, "/")
		prev := "/"
		for _, p := range parts[1:] {
			prev = path.Join(prev, p)
			syscall.Unlink(path.Join(initLayer, prev))
		}

		if _, err := os.Stat(path.Join(initLayer, pth)); err != nil {
			if os.IsNotExist(err) {
				if err := os.MkdirAll(path.Join(initLayer, path.Dir(pth)), 0755); err != nil {
					return err
				}
				switch typ {
				case "dir":
					if err := os.MkdirAll(path.Join(initLayer, pth), 0755); err != nil {
						return err
					}
				case "file":
					f, err := os.OpenFile(path.Join(initLayer, pth), os.O_CREATE, 0755)
					if err != nil {
						return err
					}
					f.Close()
				default:
					if err := os.Symlink(typ, path.Join(initLayer, pth)); err != nil {
						return err
					}
				}
			} else {
				return err
			}
		}
	}

	// Layer is ready to use, if it wasn't before.
	return nil
}

// Check if given error is "not empty".
// Note: this is the way golang does it internally with os.IsNotExists.
func isNotEmpty(err error) bool {
	switch pe := err.(type) {
	case nil:
		return false
	case *os.PathError:
		err = pe.Err
	case *os.LinkError:
		err = pe.Err
	}
	return strings.Contains(err.Error(), " not empty")
}

// Delete atomically removes an image from the graph.
func (graph *Graph) Delete(name string) error {
	id, err := graph.idIndex.Get(name)
	if err != nil {
		return err
	}
	tmp, err := graph.Mktemp("")
	graph.idIndex.Delete(id)
	if err == nil {
		err = os.Rename(graph.ImageRoot(id), tmp)
		// On err make tmp point to old dir and cleanup unused tmp dir
		if err != nil {
			os.RemoveAll(tmp)
			tmp = graph.ImageRoot(id)
		}
	} else {
		// On err make tmp point to old dir for cleanup
		tmp = graph.ImageRoot(id)
	}
	// Remove rootfs data from the driver
	graph.driver.Remove(id)
	// Remove the trashed image directory
	return os.RemoveAll(tmp)
}

// Map returns a list of all images in the graph, addressable by ID.
func (graph *Graph) Map() (map[string]*image.Image, error) {
	images := make(map[string]*image.Image)
	err := graph.walkAll(func(image *image.Image) {
		images[image.ID] = image
	})
	if err != nil {
		return nil, err
	}
	return images, nil
}

// walkAll iterates over each image in the graph, and passes it to a handler.
// The walking order is undetermined.
func (graph *Graph) walkAll(handler func(*image.Image)) error {
	files, err := ioutil.ReadDir(graph.Root)
	if err != nil {
		return err
	}
	for _, st := range files {
		if img, err := graph.Get(st.Name()); err != nil {
			// Skip image
			continue
		} else if handler != nil {
			handler(img)
		}
	}
	return nil
}

// ByParent returns a lookup table of images by their parent.
// If an image of id ID has 3 children images, then the value for key ID
// will be a list of 3 images.
// If an image has no children, it will not have an entry in the table.
func (graph *Graph) ByParent() (map[string][]*image.Image, error) {
	byParent := make(map[string][]*image.Image)
	err := graph.walkAll(func(img *image.Image) {
		parent, err := graph.Get(img.Parent)
		if err != nil {
			return
		}
		if children, exists := byParent[parent.ID]; exists {
			byParent[parent.ID] = append(children, img)
		} else {
			byParent[parent.ID] = []*image.Image{img}
		}
	})
	return byParent, err
}

// Heads returns all heads in the graph, keyed by id.
// A head is an image which is not the parent of another image in the graph.
func (graph *Graph) Heads() (map[string]*image.Image, error) {
	heads := make(map[string]*image.Image)
	byParent, err := graph.ByParent()
	if err != nil {
		return nil, err
	}
	err = graph.walkAll(func(image *image.Image) {
		// If it's not in the byParent lookup table, then
		// it's not a parent -> so it's a head!
		if _, exists := byParent[image.ID]; !exists {
			heads[image.ID] = image
		}
	})
	return heads, err
}

func (graph *Graph) ImageRoot(id string) string {
	return path.Join(graph.Root, id)
}

func (graph *Graph) Driver() graphdriver.Driver {
	return graph.driver
}
<commit_msg>Fixes #8690<commit_after>package graph

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/log"
	"github.com/docker/docker/pkg/truncindex"
	"github.com/docker/docker/runconfig"
	"github.com/docker/docker/utils"
)

// A Graph is a store for versioned filesystem images and the relationship between them.
type Graph struct {
	Root    string
	idIndex *truncindex.TruncIndex
	driver  graphdriver.Driver
}

// NewGraph instantiates a new graph at the given root path in the filesystem.
// `root` will be created if it doesn't exist.
func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	// Create the root directory if it doesn't exists
	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
		return nil, err
	}

	graph := &Graph{
		Root:    abspath,
		idIndex: truncindex.NewTruncIndex([]string{}),
		driver:  driver,
	}
	if err := graph.restore(); err != nil {
		return nil, err
	}
	return graph, nil
}

func (graph *Graph) restore() error {
	dir, err := ioutil.ReadDir(graph.Root)
	if err != nil {
		return err
	}
	var ids = []string{}
	for _, v := range dir {
		id := v.Name()
		if graph.driver.Exists(id) {
			ids = append(ids, id)
		}
	}
	graph.idIndex = truncindex.NewTruncIndex(ids)
	log.Debugf("Restored %d elements", len(dir))
	return nil
}

// FIXME: Implement error subclass instead of looking at the error text
// Note: This is the way golang implements os.IsNotExists on Plan9
func (graph *Graph) IsNotExist(err error) bool {
	return err != nil && (strings.Contains(strings.ToLower(err.Error()), "does not exist") || strings.Contains(strings.ToLower(err.Error()), "no such"))
}

// Exists returns true if an image is registered at the given id.
// If the image doesn't exist or if an error is encountered, false is returned.
func (graph *Graph) Exists(id string) bool {
	if _, err := graph.Get(id); err != nil {
		return false
	}
	return true
}

// Get returns the image with the given id, or an error if the image doesn't exist.
func (graph *Graph) Get(name string) (*image.Image, error) {
	id, err := graph.idIndex.Get(name)
	if err != nil {
		return nil, err
	}
	img, err := image.LoadImage(graph.ImageRoot(id))
	if err != nil {
		return nil, err
	}
	if img.ID != id {
		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
	}
	img.SetGraph(graph)

	if img.Size < 0 {
		size, err := graph.driver.DiffSize(img.ID, img.Parent)
		if err != nil {
			return nil, fmt.Errorf("unable to calculate size of image id %q: %s", img.ID, err)
		}

		img.Size = size
		if err := img.SaveSize(graph.ImageRoot(id)); err != nil {
			return nil, err
		}
	}
	return img, nil
}

// Create creates a new image and registers it in the graph.
func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, containerImage, comment, author string, containerConfig, config *runconfig.Config) (*image.Image, error) {
	img := &image.Image{
		ID:            utils.GenerateRandomID(),
		Comment:       comment,
		Created:       time.Now().UTC(),
		DockerVersion: dockerversion.VERSION,
		Author:        author,
		Config:        config,
		Architecture:  runtime.GOARCH,
		OS:            runtime.GOOS,
	}

	if containerID != "" {
		img.Parent = containerImage
		img.Container = containerID
		img.ContainerConfig = *containerConfig
	}

	if err := graph.Register(img, nil, layerData); err != nil {
		return nil, err
	}
	return img, nil
}

// Register imports a pre-existing image into the graph.
func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archive.ArchiveReader) (err error) {
	defer func() {
		// If any error occurs, remove the new dir from the driver.
		// Don't check for errors since the dir might not have been created.
		// FIXME: this leaves a possible race condition.
		if err != nil {
			graph.driver.Remove(img.ID)
		}
	}()
	if err := utils.ValidateID(img.ID); err != nil {
		return err
	}
	// (This is a convenience to save time. Race conditions are taken care of by os.Rename)
	if graph.Exists(img.ID) {
		return fmt.Errorf("Image %s already exists", img.ID)
	}

	// Ensure that the image root does not exist on the filesystem
	// when it is not registered in the graph.
	// This is common when you switch from one graph driver to another
	if err := os.RemoveAll(graph.ImageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
		return err
	}

	// If the driver has this ID but the graph doesn't, remove it from the driver to start fresh.
	// (the graph is the source of truth).
	// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
	// (FIXME: make that mandatory for drivers).
	graph.driver.Remove(img.ID)

	tmp, err := graph.Mktemp("")
	defer os.RemoveAll(tmp)
	if err != nil {
		return fmt.Errorf("Mktemp failed: %s", err)
	}

	// Create root filesystem in the driver
	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
	}
	// Apply the diff/layer
	img.SetGraph(graph)
	if err := image.StoreImage(img, jsonData, layerData, tmp); err != nil {
		return err
	}
	// Commit
	if err := os.Rename(tmp, graph.ImageRoot(img.ID)); err != nil {
		return err
	}
	graph.idIndex.Add(img.ID)
	return nil
}

// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
//   The archive is stored on disk and will be automatically deleted as soon as has been read.
//   If output is not nil, a human-readable progress bar will be written to it.
//   FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, sf *utils.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
	image, err := graph.Get(id)
	if err != nil {
		return nil, err
	}
	tmp, err := graph.Mktemp("")
	if err != nil {
		return nil, err
	}
	a, err := image.TarLayer()
	if err != nil {
		return nil, err
	}
	progress := utils.ProgressReader(a, 0, output, sf, false, utils.TruncateID(id), "Buffering to disk")
	defer progress.Close()
	return archive.NewTempArchive(progress, tmp)
}

// Mktemp creates a temporary sub-directory inside the graph's filesystem.
func (graph *Graph) Mktemp(id string) (string, error) {
	dir := path.Join(graph.Root, "_tmp", utils.GenerateRandomID())
	if err := os.MkdirAll(dir, 0700); err != nil {
		return "", err
	}
	return dir, nil
}

// setupInitLayer populates a directory with mountpoints suitable
// for bind-mounting dockerinit into the container. The mountpoint is simply an
// empty file at /.dockerinit
//
// This extra layer is used by all containers as the top-most ro layer. It protects
// the container from unwanted side-effects on the rw layer.
func SetupInitLayer(initLayer string) error {
	for pth, typ := range map[string]string{
		"/dev/pts":         "dir",
		"/dev/shm":         "dir",
		"/proc":            "dir",
		"/sys":             "dir",
		"/.dockerinit":     "file",
		"/.dockerenv":      "file",
		"/etc/resolv.conf": "file",
		"/etc/hosts":       "file",
		"/etc/hostname":    "file",
		"/dev/console":     "file",
		"/etc/mtab":        "/proc/mounts",
	} {
		parts := strings.Split(pth, "/")
		prev := "/"
		for _, p := range parts[1:] {
			prev = path.Join(prev, p)
			syscall.Unlink(path.Join(initLayer, prev))
		}

		if _, err := os.Stat(path.Join(initLayer, pth)); err != nil {
			if os.IsNotExist(err) {
				if err := os.MkdirAll(path.Join(initLayer, path.Dir(pth)), 0755); err != nil {
					return err
				}
				switch typ {
				case "dir":
					if err := os.MkdirAll(path.Join(initLayer, pth), 0755); err != nil {
						return err
					}
				case "file":
					f, err := os.OpenFile(path.Join(initLayer, pth), os.O_CREATE, 0755)
					if err != nil {
						return err
					}
					f.Close()
				default:
					if err := os.Symlink(typ, path.Join(initLayer, pth)); err != nil {
						return err
					}
				}
			} else {
				return err
			}
		}
	}

	// Layer is ready to use, if it wasn't before.
	return nil
}

// Check if given error is "not empty".
// Note: this is the way golang does it internally with os.IsNotExists.
func isNotEmpty(err error) bool {
	switch pe := err.(type) {
	case nil:
		return false
	case *os.PathError:
		err = pe.Err
	case *os.LinkError:
		err = pe.Err
	}
	return strings.Contains(err.Error(), " not empty")
}

// Delete atomically removes an image from the graph.
func (graph *Graph) Delete(name string) error {
	id, err := graph.idIndex.Get(name)
	if err != nil {
		return err
	}
	tmp, err := graph.Mktemp("")
	graph.idIndex.Delete(id)
	if err == nil {
		err = os.Rename(graph.ImageRoot(id), tmp)
		// On err make tmp point to old dir and cleanup unused tmp dir
		if err != nil {
			os.RemoveAll(tmp)
			tmp = graph.ImageRoot(id)
		}
	} else {
		// On err make tmp point to old dir for cleanup
		tmp = graph.ImageRoot(id)
	}
	// Remove rootfs data from the driver
	graph.driver.Remove(id)
	// Remove the trashed image directory
	return os.RemoveAll(tmp)
}

// Map returns a list of all images in the graph, addressable by ID.
func (graph *Graph) Map() (map[string]*image.Image, error) {
	images := make(map[string]*image.Image)
	err := graph.walkAll(func(image *image.Image) {
		images[image.ID] = image
	})
	if err != nil {
		return nil, err
	}
	return images, nil
}

// walkAll iterates over each image in the graph, and passes it to a handler.
// The walking order is undetermined.
func (graph *Graph) walkAll(handler func(*image.Image)) error {
	files, err := ioutil.ReadDir(graph.Root)
	if err != nil {
		return err
	}
	for _, st := range files {
		if img, err := graph.Get(st.Name()); err != nil {
			// Skip image
			continue
		} else if handler != nil {
			handler(img)
		}
	}
	return nil
}

// ByParent returns a lookup table of images by their parent.
// If an image of id ID has 3 children images, then the value for key ID
// will be a list of 3 images.
// If an image has no children, it will not have an entry in the table.
func (graph *Graph) ByParent() (map[string][]*image.Image, error) {
	byParent := make(map[string][]*image.Image)
	err := graph.walkAll(func(img *image.Image) {
		parent, err := graph.Get(img.Parent)
		if err != nil {
			return
		}
		if children, exists := byParent[parent.ID]; exists {
			byParent[parent.ID] = append(children, img)
		} else {
			byParent[parent.ID] = []*image.Image{img}
		}
	})
	return byParent, err
}

// Heads returns all heads in the graph, keyed by id.
// A head is an image which is not the parent of another image in the graph.
func (graph *Graph) Heads() (map[string]*image.Image, error) {
	heads := make(map[string]*image.Image)
	byParent, err := graph.ByParent()
	if err != nil {
		return nil, err
	}
	err = graph.walkAll(func(image *image.Image) {
		// If it's not in the byParent lookup table, then
		// it's not a parent -> so it's a head!
		if _, exists := byParent[image.ID]; !exists {
			heads[image.ID] = image
		}
	})
	return heads, err
}

func (graph *Graph) ImageRoot(id string) string {
	return path.Join(graph.Root, id)
}

func (graph *Graph) Driver() graphdriver.Driver {
	return graph.driver
}
<|endoftext|>


<commit_before>//
// Copyright (C) 2014-2015 Sebastian 'tokkee' Harl <sh@tokkee.org>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package graph handles time-series data provided by SysDB. It supports
// querying and post-processing of the data.
package graph

import (
	"fmt"
	"time"

	"github.com/gonum/plot"
	"github.com/gonum/plot/plotter"
	"github.com/gonum/plot/plotutil"
	"github.com/sysdb/go/client"
	"github.com/sysdb/go/sysdb"
)

// A Metric represents a single data-source of a graph.
type Metric struct {
	// The unique identifier of the metric.
	Hostname, Identifier string

	// Attributes describing details of the metric.
	Attributes map[string]string
}

// A Graph represents a single graph. It may reference multiple data-sources.
type Graph struct {
	// Time range of the graph.
	Start, End time.Time

	// Content of the graph.
	Metrics []Metric
}

type pl struct {
	*plot.Plot

	ts int // Index of the current time-series.
}

func (p *pl) addTimeseries(c *client.Client, metric Metric, start, end time.Time, verbose bool) error {
	q, err := client.QueryString("TIMESERIES %s.%s START %s END %s",
		metric.Hostname, metric.Identifier, start, end)
	if err != nil {
		return fmt.Errorf("Failed to retrieve graph data: %v", err)
	}
	res, err := c.Query(q)
	if err != nil {
		return fmt.Errorf("Failed to retrieve graph data: %v", err)
	}

	ts, ok := res.(*sysdb.Timeseries)
	if !ok {
		return fmt.Errorf("TIMESERIES did not return a time-series but %T", res)
	}

	for name, data := range ts.Data {
		pts := make(plotter.XYs, len(data))
		for i, p := range data {
			pts[i].X = float64(time.Time(p.Timestamp).UnixNano())
			pts[i].Y = p.Value
		}

		l, err := plotter.NewLine(pts)
		if err != nil {
			return fmt.Errorf("Failed to create line plotter: %v", err)
		}
		l.LineStyle.Color = plotutil.DarkColors[p.ts%len(plotutil.DarkColors)]

		p.Add(l)
		if verbose {
			p.Legend.Add(fmt.Sprintf("%s %s %s", metric.Hostname, metric.Identifier, name), l)
		} else {
			p.Legend.Add(name, l)
		}
		p.ts++
	}
	return nil
}

// Plot fetches a graph's time-series data using the specified client and
// plots it.
func (g *Graph) Plot(c *client.Client) (*plot.Plot, error) {
	var err error

	p := &pl{}
	p.Plot, err = plot.New()
	if err != nil {
		return nil, fmt.Errorf("Failed to create plot: %v", err)
	}
	p.Add(plotter.NewGrid())
	p.X.Tick.Marker = dateTicks{}

	for _, m := range g.Metrics {
		if err := p.addTimeseries(c, m, g.Start, g.End, len(g.Metrics) > 1); err != nil {
			return nil, err
		}
	}
	return p.Plot, nil
}

type dateTicks struct{}

func (dateTicks) Ticks(min, max float64) []plot.Tick {
	// TODO: this is surely not the best we can do
	// but it'll distribute ticks evenly.
	ticks := plot.DefaultTicks{}.Ticks(min, max)
	for i, t := range ticks {
		if t.Label == "" {
			// Skip minor ticks.
			continue
		}
		ticks[i].Label = time.Unix(0, int64(t.Value)).Format(time.RFC822)
	}
	return ticks
}

// vim: set tw=78 sw=4 sw=4 noexpandtab :
<commit_msg>graph: Add support for grouping of metrics.<commit_after>//
// Copyright (C) 2014-2015 Sebastian 'tokkee' Harl <sh@tokkee.org>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package graph handles time-series data provided by SysDB. It supports
// querying and post-processing of the data.
package graph

import (
	"fmt"
	"strings"
	"time"

	"github.com/gonum/plot"
	"github.com/gonum/plot/plotter"
	"github.com/gonum/plot/plotutil"
	"github.com/sysdb/go/client"
	"github.com/sysdb/go/sysdb"
)

// A Metric represents a single data-source of a graph.
type Metric struct {
	// The unique identifier of the metric.
	Hostname, Identifier string

	// Attributes describing details of the metric.
	Attributes map[string]string

	ts *sysdb.Timeseries
}

// A Graph represents a single graph. It may reference multiple data-sources.
type Graph struct {
	// Time range of the graph.
	Start, End time.Time

	// Content of the graph.
	Metrics []Metric

	// List of attributes to group by.
	GroupBy []string
}

type pl struct {
	*plot.Plot

	ts int // Index of the current time-series.
}

func queryTimeseries(c *client.Client, metric Metric, start, end time.Time) (*sysdb.Timeseries, error) {
	q, err := client.QueryString("TIMESERIES %s.%s START %s END %s",
		metric.Hostname, metric.Identifier, start, end)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve graph data: %v", err)
	}
	res, err := c.Query(q)
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve graph data: %v", err)
	}

	ts, ok := res.(*sysdb.Timeseries)
	if !ok {
		return nil, fmt.Errorf("TIMESERIES did not return a time-series but %T", res)
	}
	return ts, nil
}

func (p *pl) addTimeseries(c *client.Client, metric Metric, verbose bool) error {
	for name, data := range metric.ts.Data {
		pts := make(plotter.XYs, len(data))
		for i, p := range data {
			pts[i].X = float64(time.Time(p.Timestamp).UnixNano())
			pts[i].Y = p.Value
		}

		l, err := plotter.NewLine(pts)
		if err != nil {
			return fmt.Errorf("Failed to create line plotter: %v", err)
		}
		l.LineStyle.Color = plotutil.DarkColors[p.ts%len(plotutil.DarkColors)]

		p.Add(l)
		if verbose {
			p.Legend.Add(fmt.Sprintf("%s %s %s", metric.Hostname, metric.Identifier, name), l)
		} else {
			p.Legend.Add(name, l)
		}
		p.ts++
	}
	return nil
}

// sum is an aggregation function that adds ts2 to ts1.
func sum(ts1, ts2 *sysdb.Timeseries) error {
	if !ts1.Start.Equal(ts1.Start) || !ts1.End.Equal(ts2.End) {
		return fmt.Errorf("Timeseries cover different ranges: [%s, %s] != [%s, %s]",
			ts1.Start, ts1.End, ts2.Start, ts2.End)
	}
	if len(ts1.Data) != len(ts2.Data) {
		return fmt.Errorf("Incompatible time-series: %v != %v", ts1.Data, ts2.Data)
	}

	for name := range ts1.Data {
		if len(ts1.Data[name]) != len(ts2.Data[name]) {
			return fmt.Errorf("Time-series %q is not aligned", name)
		}
		for i := range ts1.Data[name] {
			if !ts1.Data[name][i].Timestamp.Equal(ts2.Data[name][i].Timestamp) {
				return fmt.Errorf("Time-series %q is not aligned", name)
			}
			ts1.Data[name][i].Value += ts2.Data[name][i].Value
		}
	}
	return nil
}

func (g *Graph) group(c *client.Client, start, end time.Time) ([]Metric, error) {
	if len(g.GroupBy) == 0 {
		for i, m := range g.Metrics {
			var err error
			if g.Metrics[i].ts, err = queryTimeseries(c, m, g.Start, g.End); err != nil {
				return nil, err
			}
		}
		return g.Metrics, nil
	}

	groups := make(map[string][]Metric)
	for _, m := range g.Metrics {
		var key string
		for _, g := range g.GroupBy {
			key += "\x00" + m.Attributes[g]
		}
		groups[key] = append(groups[key], m)
	}

	var metrics []Metric
	for name, group := range groups {
		ts, err := queryTimeseries(c, group[0], g.Start, g.End)
		if err != nil {
			return nil, err
		}
		host := group[0].Hostname
		for _, m := range group[1:] {
			ts2, err := queryTimeseries(c, m, g.Start, g.End)
			if err != nil {
				return nil, err
			}
			if err := sum(ts, ts2); err != nil {
				return nil, err
			}
			if host != "" && host != m.Hostname {
				host = ""
			}
		}

		metrics = append(metrics, Metric{
			Hostname:   host,
			Identifier: strings.Replace(name[1:], "\x00", "-", -1),
			ts:         ts,
		})
	}
	return metrics, nil
}

// Plot fetches a graph's time-series data using the specified client and
// plots it.
func (g *Graph) Plot(c *client.Client) (*plot.Plot, error) {
	var err error

	p := &pl{}
	p.Plot, err = plot.New()
	if err != nil {
		return nil, fmt.Errorf("Failed to create plot: %v", err)
	}
	p.Add(plotter.NewGrid())
	p.X.Tick.Marker = dateTicks{}

	metrics, err := g.group(c, g.Start, g.End)
	if err != nil {
		return nil, err
	}
	for _, m := range metrics {
		if err := p.addTimeseries(c, m, len(g.Metrics) > 1); err != nil {
			return nil, err
		}
	}
	return p.Plot, nil
}

type dateTicks struct{}

func (dateTicks) Ticks(min, max float64) []plot.Tick {
	// TODO: this is surely not the best we can do
	// but it'll distribute ticks evenly.
	ticks := plot.DefaultTicks{}.Ticks(min, max)
	for i, t := range ticks {
		if t.Label == "" {
			// Skip minor ticks.
			continue
		}
		ticks[i].Label = time.Unix(0, int64(t.Value)).Format(time.RFC822)
	}
	return ticks
}

// vim: set tw=78 sw=4 sw=4 noexpandtab :
<|endoftext|>


<commit_before>package controller

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/tidwall/btree"
	"github.com/tidwall/resp"
	"github.com/tidwall/tile38/controller/server"
	"github.com/tidwall/tile38/core"
)

func (c *Controller) cmdStats(msg *server.Message) (res resp.Value, err error) {
	start := time.Now()
	vs := msg.Values[1:]
	var ms = []map[string]interface{}{}

	if len(vs) == 0 {
		return server.NOMessage, errInvalidNumberOfArguments
	}
	var vals []resp.Value
	var key string
	var ok bool
	for {
		vs, key, ok = tokenval(vs)
		if !ok {
			break
		}
		col := c.getCol(key)
		if col != nil {
			m := make(map[string]interface{})
			m["num_points"] = col.PointCount()
			m["in_memory_size"] = col.TotalWeight()
			m["num_objects"] = col.Count()
			m["num_strings"] = col.StringCount()
			switch msg.OutputType {
			case server.JSON:
				ms = append(ms, m)
			case server.RESP:
				vals = append(vals, resp.ArrayValue(respValuesSimpleMap(m)))
			}
		} else {
			switch msg.OutputType {
			case server.JSON:
				ms = append(ms, nil)
			case server.RESP:
				vals = append(vals, resp.NullValue())
			}
		}
	}
	switch msg.OutputType {
	case server.JSON:

		data, err := json.Marshal(ms)
		if err != nil {
			return server.NOMessage, err
		}
		res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
	case server.RESP:
		res = resp.ArrayValue(vals)
	}
	return res, nil
}
func (c *Controller) cmdServer(msg *server.Message) (res resp.Value, err error) {
	start := time.Now()

	if len(msg.Values) != 1 {
		return server.NOMessage, errInvalidNumberOfArguments
	}
	m := make(map[string]interface{})
	m["id"] = c.config.serverID()
	if c.config.followHost() != "" {
		m["following"] = fmt.Sprintf("%s:%d", c.config.followHost(), c.config.followPort())
		m["caught_up"] = c.fcup
		m["caught_up_once"] = c.fcuponce
	}
	m["http_transport"] = c.http
	m["pid"] = os.Getpid()
	m["aof_size"] = c.aofsz
	m["num_collections"] = c.cols.Len()
	m["num_hooks"] = len(c.hooks)
	sz := 0
	c.cols.Ascend(func(item btree.Item) bool {
		col := item.(*collectionT).Collection
		sz += col.TotalWeight()
		return true
	})
	m["in_memory_size"] = sz
	points := 0
	objects := 0
	strings := 0
	c.cols.Ascend(func(item btree.Item) bool {
		col := item.(*collectionT).Collection
		points += col.PointCount()
		objects += col.Count()
		strings += col.StringCount()
		return true
	})
	m["num_points"] = points
	m["num_objects"] = objects
	m["num_strings"] = strings
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	avgsz := 0
	if points != 0 {
		avgsz = int(mem.HeapAlloc) / points
	}
	m["mem_alloc"] = mem.Alloc
	m["heap_size"] = mem.HeapAlloc
	m["heap_released"] = mem.HeapReleased
	m["max_heap_size"] = c.config.maxMemory()
	m["avg_item_size"] = avgsz
	m["pointer_size"] = (32 << uintptr(uint64(^uintptr(0))>>63)) / 8
	m["read_only"] = c.config.readOnly()

	switch msg.OutputType {
	case server.JSON:
		data, err := json.Marshal(m)
		if err != nil {
			return server.NOMessage, err
		}
		res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
	case server.RESP:
		vals := respValuesSimpleMap(m)
		res = resp.ArrayValue(vals)
	}
	return res, nil
}

func (c *Controller) writeInfoServer(w *bytes.Buffer) {
	fmt.Fprintf(w, "tile38_version:%s\r\n", core.Version)
	fmt.Fprintf(w, "redis_version:%s\r\n", core.Version)                              //Version of the Redis server
	fmt.Fprintf(w, "uptime_in_seconds:%d\r\n", time.Now().Sub(c.started)/time.Second) //Number of seconds since Redis server start
}
func (c *Controller) writeInfoClients(w *bytes.Buffer) {
	c.connsmu.RLock()
	fmt.Fprintf(w, "connected_clients:%d\r\n", len(c.conns)) // Number of client connections (excluding connections from slaves)
	c.connsmu.RUnlock()
}
func (c *Controller) writeInfoMemory(w *bytes.Buffer) {
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	fmt.Fprintf(w, "used_memory:%d\r\n", mem.Alloc) // total number of bytes allocated by Redis using its allocator (either standard libc, jemalloc, or an alternative allocator such as tcmalloc
}
func boolInt(t bool) int {
	if t {
		return 1
	}
	return 0
}
func (c *Controller) writeInfoPersistence(w *bytes.Buffer) {
	fmt.Fprintf(w, "aof_enabled:1\r\n")
	fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(c.shrinking))                          // Flag indicating a AOF rewrite operation is on-going
	fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", c.lastShrinkDuration.get()/int(time.Second)) // Duration of the last AOF rewrite operation in seconds
	currentShrinkStart := c.currentShrinkStart.get()
	if currentShrinkStart.IsZero() {
		fmt.Fprintf(w, "aof_current_rewrite_time_sec:0\r\n") // Duration of the on-going AOF rewrite operation if any
	} else {
		fmt.Fprintf(w, "aof_current_rewrite_time_sec:%d\r\n", time.Now().Sub(currentShrinkStart)/time.Second) // Duration of the on-going AOF rewrite operation if any
	}
}

func (c *Controller) writeInfoStats(w *bytes.Buffer) {
	fmt.Fprintf(w, "total_connections_received:%d\r\n", c.statsTotalConns.get())  // Total number of connections accepted by the server
	fmt.Fprintf(w, "total_commands_processed:%d\r\n", c.statsTotalCommands.get()) // Total number of commands processed by the server
	fmt.Fprintf(w, "expired_keys:%d\r\n", c.statsExpired.get())                   // Total number of key expiration events
}
func (c *Controller) writeInfoReplication(w *bytes.Buffer) {
	fmt.Fprintf(w, "connected_slaves:%d\r\n", len(c.aofconnM)) // Number of connected slaves
}
func (c *Controller) writeInfoCluster(w *bytes.Buffer) {
	fmt.Fprintf(w, "cluster_enabled:0\r\n")
}

func (c *Controller) cmdInfo(msg *server.Message) (res resp.Value, err error) {
	start := time.Now()

	sections := []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "cluster", "keyspace"}
	switch len(msg.Values) {
	default:
		return server.NOMessage, errInvalidNumberOfArguments
	case 1:
	case 2:
		section := strings.ToLower(msg.Values[1].String())
		switch section {
		default:
			sections = []string{section}
		case "all":
			sections = []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "commandstats", "cluster", "keyspace"}
		case "default":
		}
	}

	w := &bytes.Buffer{}
	for i, section := range sections {
		if i > 0 {
			w.WriteString("\r\n")
		}
		switch strings.ToLower(section) {
		default:
			continue
		case "server":
			w.WriteString("# Server\r\n")
			c.writeInfoServer(w)
		case "clients":
			w.WriteString("# Clients\r\n")
			c.writeInfoClients(w)
		case "memory":
			w.WriteString("# Memory\r\n")
			c.writeInfoMemory(w)
		case "persistence":
			w.WriteString("# Persistence\r\n")
			c.writeInfoPersistence(w)
		case "stats":
			w.WriteString("# Stats\r\n")
			c.writeInfoStats(w)
		case "replication":
			w.WriteString("# Replication\r\n")
			c.writeInfoReplication(w)
		case "cpu":
			w.WriteString("# CPU\r\n")
			c.writeInfoCPU(w)
		case "cluster":
			w.WriteString("# Cluster\r\n")
			c.writeInfoCluster(w)
		}
	}

	switch msg.OutputType {
	case server.JSON:
		data, err := json.Marshal(w.String())
		if err != nil {
			return server.NOMessage, err
		}
		res = resp.StringValue(`{"ok":true,"info":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
	case server.RESP:
		res = resp.BytesValue(w.Bytes())
	}

	return res, nil
}
func respValuesSimpleMap(m map[string]interface{}) []resp.Value {
	var keys []string
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var vals []resp.Value
	for _, key := range keys {
		val := m[key]
		vals = append(vals, resp.StringValue(key))
		vals = append(vals, resp.StringValue(fmt.Sprintf("%v", val)))
	}
	return vals
}

func (c *Controller) statsCollections(line string) (string, error) {
	start := time.Now()
	var key string
	var ms = []map[string]interface{}{}
	for len(line) > 0 {
		line, key = token(line)
		col := c.getCol(key)
		if col != nil {
			m := make(map[string]interface{})
			points := col.PointCount()
			m["num_points"] = points
			m["in_memory_size"] = col.TotalWeight()
			m["num_objects"] = col.Count()
			ms = append(ms, m)
		} else {
			ms = append(ms, nil)
		}
	}
	data, err := json.Marshal(ms)
	if err != nil {
		return "", err
	}
	return `{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}", nil
}
<commit_msg>added cpu and threads to stats<commit_after>package controller

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/tidwall/btree"
	"github.com/tidwall/resp"
	"github.com/tidwall/tile38/controller/server"
	"github.com/tidwall/tile38/core"
)

func (c *Controller) cmdStats(msg *server.Message) (res resp.Value, err error) {
	start := time.Now()
	vs := msg.Values[1:]
	var ms = []map[string]interface{}{}

	if len(vs) == 0 {
		return server.NOMessage, errInvalidNumberOfArguments
	}
	var vals []resp.Value
	var key string
	var ok bool
	for {
		vs, key, ok = tokenval(vs)
		if !ok {
			break
		}
		col := c.getCol(key)
		if col != nil {
			m := make(map[string]interface{})
			m["num_points"] = col.PointCount()
			m["in_memory_size"] = col.TotalWeight()
			m["num_objects"] = col.Count()
			m["num_strings"] = col.StringCount()
			switch msg.OutputType {
			case server.JSON:
				ms = append(ms, m)
			case server.RESP:
				vals = append(vals, resp.ArrayValue(respValuesSimpleMap(m)))
			}
		} else {
			switch msg.OutputType {
			case server.JSON:
				ms = append(ms, nil)
			case server.RESP:
				vals = append(vals, resp.NullValue())
			}
		}
	}
	switch msg.OutputType {
	case server.JSON:

		data, err := json.Marshal(ms)
		if err != nil {
			return server.NOMessage, err
		}
		res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
	case server.RESP:
		res = resp.ArrayValue(vals)
	}
	return res, nil
}
func (c *Controller) cmdServer(msg *server.Message) (res resp.Value, err error) {
	start := time.Now()

	if len(msg.Values) != 1 {
		return server.NOMessage, errInvalidNumberOfArguments
	}
	m := make(map[string]interface{})
	m["id"] = c.config.serverID()
	if c.config.followHost() != "" {
		m["following"] = fmt.Sprintf("%s:%d", c.config.followHost(), c.config.followPort())
		m["caught_up"] = c.fcup
		m["caught_up_once"] = c.fcuponce
	}
	m["http_transport"] = c.http
	m["pid"] = os.Getpid()
	m["aof_size"] = c.aofsz
	m["num_collections"] = c.cols.Len()
	m["num_hooks"] = len(c.hooks)
	sz := 0
	c.cols.Ascend(func(item btree.Item) bool {
		col := item.(*collectionT).Collection
		sz += col.TotalWeight()
		return true
	})
	m["in_memory_size"] = sz
	points := 0
	objects := 0
	strings := 0
	c.cols.Ascend(func(item btree.Item) bool {
		col := item.(*collectionT).Collection
		points += col.PointCount()
		objects += col.Count()
		strings += col.StringCount()
		return true
	})
	m["num_points"] = points
	m["num_objects"] = objects
	m["num_strings"] = strings
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	avgsz := 0
	if points != 0 {
		avgsz = int(mem.HeapAlloc) / points
	}
	m["mem_alloc"] = mem.Alloc
	m["heap_size"] = mem.HeapAlloc
	m["heap_released"] = mem.HeapReleased
	m["max_heap_size"] = c.config.maxMemory()
	m["avg_item_size"] = avgsz
	m["pointer_size"] = (32 << uintptr(uint64(^uintptr(0))>>63)) / 8
	m["read_only"] = c.config.readOnly()
	m["cpus"] = runtime.NumCPU()
	m["threads"] = runtime.GOMAXPROCS(0)

	switch msg.OutputType {
	case server.JSON:
		data, err := json.Marshal(m)
		if err != nil {
			return server.NOMessage, err
		}
		res = resp.StringValue(`{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
	case server.RESP:
		vals := respValuesSimpleMap(m)
		res = resp.ArrayValue(vals)
	}
	return res, nil
}

func (c *Controller) writeInfoServer(w *bytes.Buffer) {
	fmt.Fprintf(w, "tile38_version:%s\r\n", core.Version)
	fmt.Fprintf(w, "redis_version:%s\r\n", core.Version)                              //Version of the Redis server
	fmt.Fprintf(w, "uptime_in_seconds:%d\r\n", time.Now().Sub(c.started)/time.Second) //Number of seconds since Redis server start
}
func (c *Controller) writeInfoClients(w *bytes.Buffer) {
	c.connsmu.RLock()
	fmt.Fprintf(w, "connected_clients:%d\r\n", len(c.conns)) // Number of client connections (excluding connections from slaves)
	c.connsmu.RUnlock()
}
func (c *Controller) writeInfoMemory(w *bytes.Buffer) {
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	fmt.Fprintf(w, "used_memory:%d\r\n", mem.Alloc) // total number of bytes allocated by Redis using its allocator (either standard libc, jemalloc, or an alternative allocator such as tcmalloc
}
func boolInt(t bool) int {
	if t {
		return 1
	}
	return 0
}
func (c *Controller) writeInfoPersistence(w *bytes.Buffer) {
	fmt.Fprintf(w, "aof_enabled:1\r\n")
	fmt.Fprintf(w, "aof_rewrite_in_progress:%d\r\n", boolInt(c.shrinking))                          // Flag indicating a AOF rewrite operation is on-going
	fmt.Fprintf(w, "aof_last_rewrite_time_sec:%d\r\n", c.lastShrinkDuration.get()/int(time.Second)) // Duration of the last AOF rewrite operation in seconds
	currentShrinkStart := c.currentShrinkStart.get()
	if currentShrinkStart.IsZero() {
		fmt.Fprintf(w, "aof_current_rewrite_time_sec:0\r\n") // Duration of the on-going AOF rewrite operation if any
	} else {
		fmt.Fprintf(w, "aof_current_rewrite_time_sec:%d\r\n", time.Now().Sub(currentShrinkStart)/time.Second) // Duration of the on-going AOF rewrite operation if any
	}
}

func (c *Controller) writeInfoStats(w *bytes.Buffer) {
	fmt.Fprintf(w, "total_connections_received:%d\r\n", c.statsTotalConns.get())  // Total number of connections accepted by the server
	fmt.Fprintf(w, "total_commands_processed:%d\r\n", c.statsTotalCommands.get()) // Total number of commands processed by the server
	fmt.Fprintf(w, "expired_keys:%d\r\n", c.statsExpired.get())                   // Total number of key expiration events
}
func (c *Controller) writeInfoReplication(w *bytes.Buffer) {
	fmt.Fprintf(w, "connected_slaves:%d\r\n", len(c.aofconnM)) // Number of connected slaves
}
func (c *Controller) writeInfoCluster(w *bytes.Buffer) {
	fmt.Fprintf(w, "cluster_enabled:0\r\n")
}

func (c *Controller) cmdInfo(msg *server.Message) (res resp.Value, err error) {
	start := time.Now()

	sections := []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "cluster", "keyspace"}
	switch len(msg.Values) {
	default:
		return server.NOMessage, errInvalidNumberOfArguments
	case 1:
	case 2:
		section := strings.ToLower(msg.Values[1].String())
		switch section {
		default:
			sections = []string{section}
		case "all":
			sections = []string{"server", "clients", "memory", "persistence", "stats", "replication", "cpu", "commandstats", "cluster", "keyspace"}
		case "default":
		}
	}

	w := &bytes.Buffer{}
	for i, section := range sections {
		if i > 0 {
			w.WriteString("\r\n")
		}
		switch strings.ToLower(section) {
		default:
			continue
		case "server":
			w.WriteString("# Server\r\n")
			c.writeInfoServer(w)
		case "clients":
			w.WriteString("# Clients\r\n")
			c.writeInfoClients(w)
		case "memory":
			w.WriteString("# Memory\r\n")
			c.writeInfoMemory(w)
		case "persistence":
			w.WriteString("# Persistence\r\n")
			c.writeInfoPersistence(w)
		case "stats":
			w.WriteString("# Stats\r\n")
			c.writeInfoStats(w)
		case "replication":
			w.WriteString("# Replication\r\n")
			c.writeInfoReplication(w)
		case "cpu":
			w.WriteString("# CPU\r\n")
			c.writeInfoCPU(w)
		case "cluster":
			w.WriteString("# Cluster\r\n")
			c.writeInfoCluster(w)
		}
	}

	switch msg.OutputType {
	case server.JSON:
		data, err := json.Marshal(w.String())
		if err != nil {
			return server.NOMessage, err
		}
		res = resp.StringValue(`{"ok":true,"info":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}")
	case server.RESP:
		res = resp.BytesValue(w.Bytes())
	}

	return res, nil
}
func respValuesSimpleMap(m map[string]interface{}) []resp.Value {
	var keys []string
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	var vals []resp.Value
	for _, key := range keys {
		val := m[key]
		vals = append(vals, resp.StringValue(key))
		vals = append(vals, resp.StringValue(fmt.Sprintf("%v", val)))
	}
	return vals
}

func (c *Controller) statsCollections(line string) (string, error) {
	start := time.Now()
	var key string
	var ms = []map[string]interface{}{}
	for len(line) > 0 {
		line, key = token(line)
		col := c.getCol(key)
		if col != nil {
			m := make(map[string]interface{})
			points := col.PointCount()
			m["num_points"] = points
			m["in_memory_size"] = col.TotalWeight()
			m["num_objects"] = col.Count()
			ms = append(ms, m)
		} else {
			ms = append(ms, nil)
		}
	}
	data, err := json.Marshal(ms)
	if err != nil {
		return "", err
	}
	return `{"ok":true,"stats":` + string(data) + `,"elapsed":"` + time.Now().Sub(start).String() + "\"}", nil
}
<|endoftext|>


<commit_before>package controllers

import (
	"fmt"

	"github.com/astaxie/beego"
)

type AuthController struct {
	beego.Controller
}

func (this *AuthController) Prepare() {
	beego.Debug(fmt.Sprintf("[%s] %s | %s", this.Ctx.Input.Host(), this.Ctx.Input.Request.Method, this.Ctx.Input.Request.RequestURI))

	beego.Debug("[Header] ")
	beego.Debug(this.Ctx.Request.Header)
}

func (this *AuthController) Get() {
	this.TplNames = "auth.html"

	this.Data["description"] = ""
	this.Data["author"] = ""

	this.Render()
}
<commit_msg>注册用户存储到ledis<commit_after>package controllers

import (
	"fmt"
	"github.com/astaxie/beego"
	"github.com/dockercn/docker-bucket/models"
	"log"
	"net/http"
)

type Result struct {
	Success bool
	Message string
	Url     string
}

type AuthController struct {
	beego.Controller
}

func (this *AuthController) Prepare() {
	beego.Debug(fmt.Sprintf("[%s] %s | %s", this.Ctx.Input.Host(), this.Ctx.Input.Request.Method, this.Ctx.Input.Request.RequestURI))

	beego.Debug("[Header] ")
	beego.Debug(this.Ctx.Request.Header)
}

func (this *AuthController) Get() {
	this.TplNames = "auth.html"

	this.Data["description"] = ""
	this.Data["author"] = ""

	this.Render()
}

func (this *AuthController) Signup() {
	this.Ctx.Output.Context.Output.SetStatus(http.StatusOK)
	this.Ctx.Output.Context.ResponseWriter.Header().Set("Content-Type", "application/json;charset=UTF-8")
	var result Result

	//form的attr name:username,email,password,password_confirm
	user := new(models.User)
	if err := user.Put(this.GetString("username"), this.GetString("password"), this.GetString("email")); err != nil {
		result = Result{Success: false, Message: fmt.Sprint(err), Url: "/auth"}
		this.Data["json"] = &result
		this.ServeJson()
		return
	}
	log.Println("数据存储成功")
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE.txt file.

package graphs

// isFeasible returns true if c is inside the maze and is white.
func isFeasible(maze [][]bool, c Coordinate) bool {
	return c.x >= 0 && c.x < len(maze) && c.y >= 0 && c.y < len(maze[c.x]) && maze[c.x][c.y] == white
}

// SearchMaze returns true if a path exists from start to end in the maze.
// The time complexity is O(v+e) where v is the number of vertices and e is the
// number of edges in maze. Vertex is represented by boolean value true and edge
// connects two vertices. O(v) additional space is needed.
func SearchMaze(maze [][]bool, start, end Coordinate) (path []Coordinate) {
	if !isFeasible(maze, start) {
		return path
	}
	path = append(path, start)
	maze[start.x][start.y] = black // Mark as visited.

	// pathExists returns true if path from curr to end exists in maze.
	// Function is recursive implementation of DFS.
	var pathExists func(maze [][]bool, curr, end Coordinate) bool
	pathExists = func(maze [][]bool, curr, end Coordinate) bool {
		if curr == end { // Base case.
			return true
		}

		for _, shift := range []Coordinate{right, left, down, up} {
			next := Coordinate{curr.x + shift.x, curr.y + shift.y}
			if isFeasible(maze, next) { // Check if we can move this way.
				path = append(path, next)
				maze[next.x][next.y] = black // Mark as visited.
				if pathExists(maze, next, end) {
					return true
				}
				path = path[:len(path)-1] // End wasn't found, take step back and try another one.
			}
		}
		return false
	}

	if !pathExists(maze, start, end) {
		return nil
	}
	return path
}
<commit_msg>Simplify boolean logic<commit_after>// Copyright (c) 2015, Peter Mrekaj. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE.txt file.

package graphs

// isFeasible returns true if c is inside the maze and is white.
func isFeasible(maze [][]bool, c Coordinate) bool {
	return c.x >= 0 && c.x < len(maze) && c.y >= 0 && c.y < len(maze[c.x]) && maze[c.x][c.y] == white
}

// SearchMaze returns true if a path exists from start to end in the maze.
// The time complexity is O(v+e) where v is the number of vertices and e is the
// number of edges in maze. Vertex is represented by boolean value true and edge
// connects two vertices. O(v) additional space is needed.
func SearchMaze(maze [][]bool, start, end Coordinate) (path []Coordinate) {
	if !isFeasible(maze, start) {
		return path
	}
	path = append(path, start)
	maze[start.x][start.y] = black // Mark as visited.

	// pathExists returns true if path from curr to end exists in maze.
	// Function is recursive implementation of DFS.
	var pathExists func(maze [][]bool, curr, end Coordinate) bool
	pathExists = func(maze [][]bool, curr, end Coordinate) bool {
		if curr == end { // Base case.
			return true
		}

		for _, shift := range []Coordinate{right, left, down, up} {
			next := Coordinate{curr.x + shift.x, curr.y + shift.y}
			if isFeasible(maze, next) { // Check if we can move this way.
				path = append(path, next)
				maze[next.x][next.y] = black // Mark as visited.
				if pathExists(maze, next, end) {
					return true
				}
				path = path[:len(path)-1] // End wasn't found, take step back and try another one.
			}
		}
		return false
	}

	if pathExists(maze, start, end) {
		return path
	}
	return nil
}
<|endoftext|>


<commit_before>package gsort

import (
	"bufio"
	"compress/gzip"
	"container/heap"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"sort"
	"sync"

	"github.com/brentp/xopen"
)

type LineDeco struct {
	line []byte
	Cols []int
	i    int // used internally to indicate which file it came from.
}

type Lines []LineDeco

func (l Lines) Len() int {
	return len(l)
}

func (l Lines) Less(i, j int) bool {

	for k := 0; k < len(l[i].Cols); k++ {
		if l[j].Cols[k] == l[i].Cols[k] {
			continue
		}
		return l[i].Cols[k] < l[j].Cols[k]
	}
	return false
}
func (l Lines) Swap(i, j int) {
	l[j], l[i] = l[i], l[j]
}

// for Heap

func (l Lines) Push(i interface{}) {
	l = append(l, i.(LineDeco))
}

func (l Lines) Pop() interface{} {
	n := len(l)
	if n == 0 {
		return nil
	}

	v := l[n-1]
	l = l[:n-1]
	return v
}

type Processor func(line []byte) LineDeco

func readLines(rdr *bufio.Reader, memMb int) ([][]byte, error) {

	var processed [][]byte
	j := 0

	for {

		line, err := rdr.ReadBytes('\n')
		if err != nil && err != io.EOF {
			return nil, err
		}
		if len(line) == 0 {
			if err == io.EOF {
				return processed[:j], io.EOF
			}
		}
		if j == 0 {
			n := 1000000 * memMb / (len(line) - 1)
			processed = make([][]byte, n)
		}
		processed[j] = line

		j += 1
		if err == io.EOF {
			return processed[:j], io.EOF
		}
		if j == len(processed) {
			break
		}
	}
	return processed, nil
}

func writeHeader(wtr *xopen.Writer, rdr *xopen.Reader) error {
	for {
		b, err := rdr.Peek(1)
		if err != nil {
			return err
		}
		if b[0] != '#' {
			break
		}
		line, err := rdr.ReadBytes('\n')
		if err != nil {
			return err
		}
		wtr.Write(line)
	}
	return nil
}

// TODO: should take io.Reader, and io.Writer for in and out.
func Sort(inpath string, outpath string, preprocess Processor, memMB int) error {

	rdr, err := xopen.Ropen(inpath)
	if err != nil {
		return err
	}
	defer rdr.Close()
	wtr, err := xopen.Wopen(outpath)
	if err != nil {
		return err
	}
	defer wtr.Close()
	err = writeHeader(wtr, rdr)
	if err != nil {
		return err
	}
	// ch make sure we don't have too many processes running
	ch := make(chan bool, runtime.GOMAXPROCS(-1))
	// wg makes sure we wait until all is done
	wg := &sync.WaitGroup{}
	var rerr error
	fileNames := make([]string, 0)
	for rerr == nil {
		var chunk [][]byte
		chunk, rerr = readLines(rdr.Reader, memMB)
		if len(chunk) != 0 {
			f, err := ioutil.TempFile("", fmt.Sprintf("gsort.%d", len(fileNames)))
			if err != nil {
				log.Fatal(err)
			}
			fileNames = append(fileNames, f.Name())
			defer os.Remove(f.Name())
			ch <- true
			wg.Add(1)
			// decorating and sorting is done in parallel.
			go sortAndWrite(f, wg, chunk, preprocess)
		}
		if rerr == io.EOF {
			break
		}
	}
	wg.Wait()
	// currently merging is serial. Should parallelize.
	return merge(fileNames, wtr.Writer, preprocess)
}

func merge(fileNames []string, wtr *bufio.Writer, process Processor) error {
	fhs := make([]*bufio.Reader, len(fileNames))

	cache := make(Lines, len(fileNames))

	for i, fn := range fileNames {
		defer os.Remove(fn)
		fh, err := os.Open(fn)
		if err != nil {
			return err
		}
		defer fh.Close()
		gz, err := gzip.NewReader(fh)
		if err != nil {
			return err
		}
		defer gz.Close()
		fhs[i] = bufio.NewReader(gz)

		line, err := fhs[i].ReadBytes('\n')
		if len(line) > 0 {
			cache[i] = process(line)
			cache[i].line = line
			cache[i].i = i
		} else if err == io.EOF {
			continue
		} else if err != nil {
			return err
		}
	}

	heap.Init(cache)

	for {
		o := heap.Pop(cache)
		if o == nil {
			break
		}
		c := o.(LineDeco)
		// refill from same file
		line, err := fhs[c.i].ReadBytes('\n')
		if err != io.EOF && err != nil {
			return err
		}
		next := process(line)
		next.line = line
		next.i = c.i
		heap.Push(cache, next)
		wtr.Write(c.line)

	}
	return nil
}

func sortAndWrite(f *os.File, wg *sync.WaitGroup, chunk [][]byte, process Processor) {
	defer wg.Done()
	defer f.Close()
	gz := gzip.NewWriter(f)
	defer gz.Close()
	dchunk := make(Lines, len(chunk))
	for i, l := range chunk {
		dchunk[i] = process(l)
		dchunk[i].line = l
	}

	sort.Sort(dchunk)
	wtr := bufio.NewWriter(gz)
	for _, dl := range dchunk {
		wtr.Write(dl.line)
	}
}
<commit_msg>remove extraneous break. handled by for<commit_after>package gsort

import (
	"bufio"
	"compress/gzip"
	"container/heap"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"sort"
	"sync"

	"github.com/brentp/xopen"
)

type LineDeco struct {
	line []byte
	Cols []int
	i    int // used internally to indicate which file it came from.
}

type Lines []LineDeco

func (l Lines) Len() int {
	return len(l)
}

func (l Lines) Less(i, j int) bool {

	for k := 0; k < len(l[i].Cols); k++ {
		if l[j].Cols[k] == l[i].Cols[k] {
			continue
		}
		return l[i].Cols[k] < l[j].Cols[k]
	}
	return false
}
func (l Lines) Swap(i, j int) {
	l[j], l[i] = l[i], l[j]
}

// for Heap

func (l Lines) Push(i interface{}) {
	l = append(l, i.(LineDeco))
}

func (l Lines) Pop() interface{} {
	n := len(l)
	if n == 0 {
		return nil
	}

	v := l[n-1]
	l = l[:n-1]
	return v
}

type Processor func(line []byte) LineDeco

func readLines(rdr *bufio.Reader, memMb int) ([][]byte, error) {

	var processed [][]byte
	j := 0

	for {

		line, err := rdr.ReadBytes('\n')
		if err != nil && err != io.EOF {
			return nil, err
		}
		if len(line) == 0 {
			if err == io.EOF {
				return processed[:j], io.EOF
			}
		}
		if j == 0 {
			n := 1000000 * memMb / (len(line) - 1)
			processed = make([][]byte, n)
		}
		processed[j] = line

		j += 1
		if err == io.EOF {
			return processed[:j], io.EOF
		}
		if j == len(processed) {
			break
		}
	}
	return processed, nil
}

func writeHeader(wtr *xopen.Writer, rdr *xopen.Reader) error {
	for {
		b, err := rdr.Peek(1)
		if err != nil {
			return err
		}
		if b[0] != '#' {
			break
		}
		line, err := rdr.ReadBytes('\n')
		if err != nil {
			return err
		}
		wtr.Write(line)
	}
	return nil
}

// TODO: should take io.Reader, and io.Writer for in and out.
func Sort(inpath string, outpath string, preprocess Processor, memMB int) error {

	rdr, err := xopen.Ropen(inpath)
	if err != nil {
		return err
	}
	defer rdr.Close()
	wtr, err := xopen.Wopen(outpath)
	if err != nil {
		return err
	}
	defer wtr.Close()
	err = writeHeader(wtr, rdr)
	if err != nil {
		return err
	}
	// ch make sure we don't have too many processes running
	ch := make(chan bool, runtime.GOMAXPROCS(-1))
	// wg makes sure we wait until all is done
	wg := &sync.WaitGroup{}
	var rerr error
	fileNames := make([]string, 0)
	for rerr == nil {
		var chunk [][]byte
		chunk, rerr = readLines(rdr.Reader, memMB)
		if len(chunk) != 0 {
			f, err := ioutil.TempFile("", fmt.Sprintf("gsort.%d", len(fileNames)))
			if err != nil {
				log.Fatal(err)
			}
			fileNames = append(fileNames, f.Name())
			defer os.Remove(f.Name())
			ch <- true
			wg.Add(1)
			// decorating and sorting is done in parallel.
			go sortAndWrite(f, wg, chunk, preprocess)
		}
	}
	wg.Wait()
	// currently merging is serial. Should parallelize.
	return merge(fileNames, wtr.Writer, preprocess)
}

func merge(fileNames []string, wtr *bufio.Writer, process Processor) error {
	fhs := make([]*bufio.Reader, len(fileNames))

	cache := make(Lines, len(fileNames))

	for i, fn := range fileNames {
		defer os.Remove(fn)
		fh, err := os.Open(fn)
		if err != nil {
			return err
		}
		defer fh.Close()
		gz, err := gzip.NewReader(fh)
		if err != nil {
			return err
		}
		defer gz.Close()
		fhs[i] = bufio.NewReader(gz)

		line, err := fhs[i].ReadBytes('\n')
		if len(line) > 0 {
			cache[i] = process(line)
			cache[i].line = line
			cache[i].i = i
		} else if err == io.EOF {
			continue
		} else if err != nil {
			return err
		}
	}

	heap.Init(cache)

	for {
		o := heap.Pop(cache)
		if o == nil {
			break
		}
		c := o.(LineDeco)
		// refill from same file
		line, err := fhs[c.i].ReadBytes('\n')
		if err != io.EOF && err != nil {
			return err
		}
		next := process(line)
		next.line = line
		next.i = c.i
		heap.Push(cache, next)
		wtr.Write(c.line)

	}
	return nil
}

func sortAndWrite(f *os.File, wg *sync.WaitGroup, chunk [][]byte, process Processor) {
	defer wg.Done()
	defer f.Close()
	gz := gzip.NewWriter(f)
	defer gz.Close()
	dchunk := make(Lines, len(chunk))
	for i, l := range chunk {
		dchunk[i] = process(l)
		dchunk[i].line = l
	}

	sort.Sort(dchunk)
	wtr := bufio.NewWriter(gz)
	for _, dl := range dchunk {
		wtr.Write(dl.line)
	}
}
<|endoftext|>


<commit_before>package conversation

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strconv"
	"time"

	messagebird "github.com/messagebird/go-rest-api"
)

const (
	// apiRoot is the absolute URL of the Converstations API. All paths are
	// relative to apiRoot (e.g.
	// https://conversations.messagebird.com/v1/webhooks).
	apiRoot = "https://conversations.messagebird.com/v1"

	// path is the path for the Conversation resource, relative to apiRoot.
	path = "conversations"

	// messagesPath is the path for the Message resource, relative to apiRoot
	// and path.
	messagesPath = "messages"

	// webhooksPath is the path for the Webhook resource, relative to apiRoot.
	webhooksPath = "webhooks"
)

type ConversationList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Items      []*Conversation
}

type Conversation struct {
	ID                   string
	ContactID            string
	Contact              *Contact
	LastUsedChannelID    string
	Channels             []*Channel
	Messages             *MessagesCount
	Status               ConversationStatus
	CreatedDatetime      *time.Time
	UpdatedDatetime      *time.Time
	LastReceivedDatetime *time.Time
}

type Contact struct {
	ID            string
	Href          string
	MSISDN        string
	FirstName     string
	LastName      string
	CustomDetails map[string]interface{}
	CreatedAt     *time.Time
	UpdatedAt     *time.Time
}

type Channel struct {
	ID              string
	Name            string
	PlatformID      string
	Status          string
	CreatedDatetime *time.Time
	UpdatedDatetime *time.Time
}

type MessagesCount struct {
	HRef       string
	TotalCount int
}

// ConversationStatus indicates what state a Conversation is in.
type ConversationStatus string

const (
	// ConversationStatusActive is returned when the Conversation is active.
	// Only one active conversation can ever exist for a given contact.
	ConversationStatusActive ConversationStatus = "active"

	// ConversationStatusArchived is returned when the Conversation is
	// archived. When this is the case, a new Conversation is created when a
	// message is received from a contact.
	ConversationStatusArchived ConversationStatus = "archived"
)

type MessageList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Items      []*Message
}

type Message struct {
	ID              string
	ConversationID  string
	ChannelID       string
	Direction       MessageDirection
	Status          MessageStatus
	Type            MessageType
	Content         MessageContent
	CreatedDatetime *time.Time
	UpdatedDatetime *time.Time
}

type MessageDirection string

const (
	// MessageDirectionReceived indicates an inbound message received from the customer.
	MessageDirectionReceived MessageDirection = "received"

	// MessageDirectionSent indicates an outbound message sent from the API.
	MessageDirectionSent MessageDirection = "sent"
)

// MessageStatus is a field set by the API. It indicates what the state of the
// message is, e.g. whether it has been successfully delivered or read.
type MessageStatus string

const (
	MessageStatusDeleted     MessageStatus = "deleted"
	MessageStatusDelivered   MessageStatus = "delivered"
	MessageStatusFailed      MessageStatus = "failed"
	MessageStatusPending     MessageStatus = "pending"
	MessageStatusRead        MessageStatus = "read"
	MessageStatusReceived    MessageStatus = "received"
	MessageStatusSent        MessageStatus = "sent"
	MessageStatusUnsupported MessageStatus = "unsupported"
)

// MessageType indicates what kind of content a Message has, e.g. audio or
// text.
type MessageType string

const (
	MessageTypeAudio    MessageType = "audio"
	MessageTypeFile     MessageType = "file"
	MessageTypeHSM      MessageType = "hsm"
	MessageTypeImage    MessageType = "image"
	MessageTypeLocation MessageType = "location"
	MessageTypeText     MessageType = "text"
	MessageTypeVideo    MessageType = "video"
)

// MessageContent holds a message's actual content. Only one field can be set
// per request.
type MessageContent struct {
	Audio    *Audio    `json:"audio,omitempty"`
	File     *File     `json:"file,omitempty"`
	Image    *Image    `json:"image,omitempty"`
	Location *Location `json:"location,omitempty"`
	Video    *Video    `json:"video,omitempty"`
	Text     string    `json:"text,omitempty"`

	// HSM is a highly structured message for WhatsApp. Its definition lives in
	// hsm.go.
	HSM *HSM `json:"hsm,omitempty"`
}

type Media struct {
	URL string `json:"url"`
}

type Audio Media
type File Media
type Image Media
type Video Media

type Location struct {
	Latitude  float32 `json:"latitude"`
	Longitude float32 `json:"longitude"`
}

type WebhookList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Items      []*Webhook
}

type Webhook struct {
	ID              string
	ChannelID       string
	Events          []WebhookEvent
	URL             string
	CreatedDatetime *time.Time
	UpdatedDatetime *time.Time
}

type WebhookEvent string

const (
	WebhookEventConversationCreated WebhookEvent = "conversation.created"
	WebhookEventConversationUpdated WebhookEvent = "conversation.updated"
	WebhookEventMessageCreated      WebhookEvent = "message.created"
	WebhookEventMessageUpdated      WebhookEvent = "message.updated"
)

// request does the exact same thing as Client.Request. It does, however,
// prefix the path with the Conversation API's root. This ensures the client
// doesn't "handle" this for us: by default, it uses the REST API.
func request(c *messagebird.Client, v interface{}, method, path string, data interface{}) error {
	return c.Request(v, method, fmt.Sprintf("%s/%s", apiRoot, path), data)
}

// paginationQuery builds the query string for paginated endpoints.
func paginationQuery(options *ListOptions) string {
	query := url.Values{}
	query.Set("limit", strconv.Itoa(options.Limit))
	query.Set("offset", strconv.Itoa(options.Offset))

	return query.Encode()
}

// UnmarshalJSON is used to unmarshal the MSISDN to a string rather than an
// int64. The API returns integers, but this client always uses strings.
// Exposing a json.Number doesn't seem nice.
func (c *Contact) UnmarshalJSON(data []byte) error {
	target := struct {
		ID            string
		Href          string
		MSISDN        json.Number
		FirstName     string
		LastName      string
		CustomDetails map[string]interface{}
		CreatedAt     *time.Time
		UpdatedAt     *time.Time
	}{}

	if err := json.Unmarshal(data, &target); err != nil {
		return err
	}

	// In many cases, the CustomDetails will contain the user ID. As
	// CustomDetails has interface{} values, these are unmarshalled as floats.
	// Convert them to int64.
	// Map key is not a typo: API returns userId and not userID.
	if val, ok := target.CustomDetails["userId"]; ok {
		var userID float64
		if userID, ok = val.(float64); ok {
			target.CustomDetails["userId"] = int64(userID)
		}
	}

	*c = Contact{
		target.ID,
		target.Href,
		target.MSISDN.String(),
		target.FirstName,
		target.LastName,
		target.CustomDetails,
		target.CreatedAt,
		target.UpdatedAt,
	}

	return nil
}
<commit_msg>add rejected to MessageStatus and make conversation created at not a pointer<commit_after>package conversation

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strconv"
	"time"

	messagebird "github.com/messagebird/go-rest-api"
)

const (
	// apiRoot is the absolute URL of the Converstations API. All paths are
	// relative to apiRoot (e.g.
	// https://conversations.messagebird.com/v1/webhooks).
	apiRoot = "https://conversations.messagebird.com/v1"

	// path is the path for the Conversation resource, relative to apiRoot.
	path = "conversations"

	// messagesPath is the path for the Message resource, relative to apiRoot
	// and path.
	messagesPath = "messages"

	// webhooksPath is the path for the Webhook resource, relative to apiRoot.
	webhooksPath = "webhooks"
)

type ConversationList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Items      []*Conversation
}

type Conversation struct {
	ID                   string
	ContactID            string
	Contact              *Contact
	LastUsedChannelID    string
	Channels             []*Channel
	Messages             *MessagesCount
	Status               ConversationStatus
	CreatedDatetime      time.Time
	UpdatedDatetime      *time.Time
	LastReceivedDatetime *time.Time
}

type Contact struct {
	ID            string
	Href          string
	MSISDN        string
	FirstName     string
	LastName      string
	CustomDetails map[string]interface{}
	CreatedAt     *time.Time
	UpdatedAt     *time.Time
}

type Channel struct {
	ID              string
	Name            string
	PlatformID      string
	Status          string
	CreatedDatetime *time.Time
	UpdatedDatetime *time.Time
}

type MessagesCount struct {
	HRef       string
	TotalCount int
}

// ConversationStatus indicates what state a Conversation is in.
type ConversationStatus string

const (
	// ConversationStatusActive is returned when the Conversation is active.
	// Only one active conversation can ever exist for a given contact.
	ConversationStatusActive ConversationStatus = "active"

	// ConversationStatusArchived is returned when the Conversation is
	// archived. When this is the case, a new Conversation is created when a
	// message is received from a contact.
	ConversationStatusArchived ConversationStatus = "archived"
)

type MessageList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Items      []*Message
}

type Message struct {
	ID              string
	ConversationID  string
	ChannelID       string
	Direction       MessageDirection
	Status          MessageStatus
	Type            MessageType
	Content         MessageContent
	CreatedDatetime *time.Time
	UpdatedDatetime *time.Time
}

type MessageDirection string

const (
	// MessageDirectionReceived indicates an inbound message received from the customer.
	MessageDirectionReceived MessageDirection = "received"

	// MessageDirectionSent indicates an outbound message sent from the API.
	MessageDirectionSent MessageDirection = "sent"
)

// MessageStatus is a field set by the API. It indicates what the state of the
// message is, e.g. whether it has been successfully delivered or read.
type MessageStatus string

const (
	MessageStatusDeleted     MessageStatus = "deleted"
	MessageStatusDelivered   MessageStatus = "delivered"
	MessageStatusFailed      MessageStatus = "failed"
	MessageStatusPending     MessageStatus = "pending"
	MessageStatusRead        MessageStatus = "read"
	MessageStatusReceived    MessageStatus = "received"
	MessageStatusSent        MessageStatus = "sent"
	MessageStatusUnsupported MessageStatus = "unsupported"
	MessageStatusRejected    MessageStatus = "rejected"
)

// MessageType indicates what kind of content a Message has, e.g. audio or
// text.
type MessageType string

const (
	MessageTypeAudio    MessageType = "audio"
	MessageTypeFile     MessageType = "file"
	MessageTypeHSM      MessageType = "hsm"
	MessageTypeImage    MessageType = "image"
	MessageTypeLocation MessageType = "location"
	MessageTypeText     MessageType = "text"
	MessageTypeVideo    MessageType = "video"
)

// MessageContent holds a message's actual content. Only one field can be set
// per request.
type MessageContent struct {
	Audio    *Audio    `json:"audio,omitempty"`
	File     *File     `json:"file,omitempty"`
	Image    *Image    `json:"image,omitempty"`
	Location *Location `json:"location,omitempty"`
	Video    *Video    `json:"video,omitempty"`
	Text     string    `json:"text,omitempty"`

	// HSM is a highly structured message for WhatsApp. Its definition lives in
	// hsm.go.
	HSM *HSM `json:"hsm,omitempty"`
}

type Media struct {
	URL string `json:"url"`
}

type Audio Media
type File Media
type Image Media
type Video Media

type Location struct {
	Latitude  float32 `json:"latitude"`
	Longitude float32 `json:"longitude"`
}

type WebhookList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Items      []*Webhook
}

type Webhook struct {
	ID              string
	ChannelID       string
	Events          []WebhookEvent
	URL             string
	CreatedDatetime *time.Time
	UpdatedDatetime *time.Time
}

type WebhookEvent string

const (
	WebhookEventConversationCreated WebhookEvent = "conversation.created"
	WebhookEventConversationUpdated WebhookEvent = "conversation.updated"
	WebhookEventMessageCreated      WebhookEvent = "message.created"
	WebhookEventMessageUpdated      WebhookEvent = "message.updated"
)

// request does the exact same thing as Client.Request. It does, however,
// prefix the path with the Conversation API's root. This ensures the client
// doesn't "handle" this for us: by default, it uses the REST API.
func request(c *messagebird.Client, v interface{}, method, path string, data interface{}) error {
	return c.Request(v, method, fmt.Sprintf("%s/%s", apiRoot, path), data)
}

// paginationQuery builds the query string for paginated endpoints.
func paginationQuery(options *ListOptions) string {
	query := url.Values{}
	query.Set("limit", strconv.Itoa(options.Limit))
	query.Set("offset", strconv.Itoa(options.Offset))

	return query.Encode()
}

// UnmarshalJSON is used to unmarshal the MSISDN to a string rather than an
// int64. The API returns integers, but this client always uses strings.
// Exposing a json.Number doesn't seem nice.
func (c *Contact) UnmarshalJSON(data []byte) error {
	target := struct {
		ID            string
		Href          string
		MSISDN        json.Number
		FirstName     string
		LastName      string
		CustomDetails map[string]interface{}
		CreatedAt     *time.Time
		UpdatedAt     *time.Time
	}{}

	if err := json.Unmarshal(data, &target); err != nil {
		return err
	}

	// In many cases, the CustomDetails will contain the user ID. As
	// CustomDetails has interface{} values, these are unmarshalled as floats.
	// Convert them to int64.
	// Map key is not a typo: API returns userId and not userID.
	if val, ok := target.CustomDetails["userId"]; ok {
		var userID float64
		if userID, ok = val.(float64); ok {
			target.CustomDetails["userId"] = int64(userID)
		}
	}

	*c = Contact{
		target.ID,
		target.Href,
		target.MSISDN.String(),
		target.FirstName,
		target.LastName,
		target.CustomDetails,
		target.CreatedAt,
		target.UpdatedAt,
	}

	return nil
}
<|endoftext|>


<commit_before>package query

import (
	"context"
	"encoding/json"
	"fmt"

	"chain/cos/bc"
	"chain/database/pg"
	"chain/errors"
	"chain/log"
)

// Annotator describes a function capable of adding annotations
// to transactions, inputs and outputs.
type Annotator func(ctx context.Context, txs []map[string]interface{}) error

// RegisterAnnotator adds an additional annotator capable of mutating
// the annotated transaction object.
func (ind *Indexer) RegisterAnnotator(annotator Annotator) {
	ind.annotators = append(ind.annotators, annotator)
}

// indexBlockCallback is registered as a block callback on the cos.FC. It
// saves all annotated transactions to the database and indexes them according
// to the Core's configured indexes.
func (ind *Indexer) indexBlockCallback(ctx context.Context, b *bc.Block) {
	err := ind.insertBlock(ctx, b)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	txs, err := ind.insertAnnotatedTxs(ctx, b)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	err = ind.insertAnnotatedOutputs(ctx, b, txs)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	// TODO(jackson): Build indexes
}

func (ind *Indexer) insertBlock(ctx context.Context, b *bc.Block) error {
	const q = `
		INSERT INTO query_blocks (height, timestamp) VALUES($1, $2)
		ON CONFLICT (height) DO NOTHING
	`
	_, err := ind.db.Exec(ctx, q, b.Height, b.TimestampMS)
	return errors.Wrap(err, "inserting block timestamp")
}

func (ind *Indexer) insertAnnotatedTxs(ctx context.Context, b *bc.Block) ([]map[string]interface{}, error) {
	var (
		hashes              = pg.Strings(make([]string, 0, len(b.Transactions)))
		positions           = pg.Uint32s(make([]uint32, 0, len(b.Transactions)))
		annotatedTxs        = pg.Strings(make([]string, 0, len(b.Transactions)))
		annotatedTxsDecoded = make([]map[string]interface{}, 0, len(b.Transactions))
	)
	for pos, tx := range b.Transactions {
		hashes = append(hashes, tx.Hash.String())
		positions = append(positions, uint32(pos))
		annotatedTxsDecoded = append(annotatedTxsDecoded, transactionObject(tx, b, uint32(pos)))
	}

	dbctx := pg.NewContext(ctx, ind.db)

	for _, annotator := range ind.annotators {
		err := annotator(dbctx, annotatedTxsDecoded)
		if err != nil {
			return nil, errors.Wrap(err, "adding external annotations")
		}
	}

	for _, decoded := range annotatedTxsDecoded {
		b, err := json.Marshal(decoded)
		if err != nil {
			return nil, err
		}
		annotatedTxs = append(annotatedTxs, string(b))
	}

	// Save the annotated txs to the database.
	const insertQ = `
		INSERT INTO annotated_txs(block_height, tx_pos, tx_hash, data)
		SELECT $1, unnest($2::integer[]), unnest($3::text[]), unnest($4::jsonb[])
		ON CONFLICT (block_height, tx_pos) DO NOTHING;
	`
	_, err := ind.db.Exec(ctx, insertQ, b.Height, positions, hashes, annotatedTxs)
	if err != nil {
		return nil, errors.Wrap(err, "inserting annotated_txs to db")
	}
	return annotatedTxsDecoded, nil
}

func (ind *Indexer) insertAnnotatedOutputs(ctx context.Context, b *bc.Block, annotatedTxs []map[string]interface{}) error {
	var (
		outputTxPositions pg.Uint32s
		outputIndexes     pg.Uint32s
		outputTxHashes    pg.Strings
		outputData        pg.Strings
		prevoutHashes     pg.Strings
		prevoutIndexes    pg.Uint32s
	)

	for pos, tx := range b.Transactions {
		for _, in := range tx.Inputs {
			if !in.IsIssuance() {
				prevoutHashes = append(prevoutHashes, in.Outpoint().Hash.String())
				prevoutIndexes = append(prevoutIndexes, in.Outpoint().Index)
			}
		}

		outs, ok := annotatedTxs[pos]["outputs"].([]interface{})
		if !ok {
			return errors.Wrap(fmt.Errorf("bad outputs type %T", annotatedTxs[pos]["outputs"]))
		}
		for outIndex, out := range outs {
			txOut, ok := out.(map[string]interface{})
			if !ok {
				return errors.Wrap(fmt.Errorf("bad output type %T", out))
			}

			serializedData, err := json.Marshal(txOut)
			if err != nil {
				return errors.Wrap(err, "serializing annotated output")
			}

			outputTxPositions = append(outputTxPositions, uint32(pos))
			outputIndexes = append(outputIndexes, uint32(outIndex))
			outputTxHashes = append(outputTxHashes, tx.Hash.String())
			outputData = append(outputData, string(serializedData))
		}
	}

	// Insert all of the block's outputs at once.
	const insertQ = `
		INSERT INTO annotated_outputs (block_height, tx_pos, output_index, tx_hash, data, timespan)
		SELECT $1, unnest($2::integer[]), unnest($3::integer[]), unnest($4::text[]),
		           unnest($5::jsonb[]),   int8range($6, NULL)
		ON CONFLICT (block_height, tx_pos, output_index) DO NOTHING;
	`
	_, err := ind.db.Exec(ctx, insertQ, b.Height, outputTxPositions,
		outputIndexes, outputTxHashes, outputData, b.TimestampMS)
	if err != nil {
		return errors.Wrap(err, "batch inserting annotated outputs")
	}

	const updateQ = `
		UPDATE annotated_outputs SET timespan = INT8RANGE(LOWER(timespan), $1)
		WHERE (tx_hash, output_index) IN (SELECT unnest($2::text[]), unnest($3::integer[]))
	`
	_, err = ind.db.Exec(ctx, updateQ, b.TimestampMS, prevoutHashes, prevoutIndexes)
	return errors.Wrap(err, "updating spent annotated outputs")
}
<commit_msg>core/query: query outputs by txid<commit_after>package query

import (
	"context"
	"encoding/json"
	"fmt"

	"chain/cos/bc"
	"chain/database/pg"
	"chain/errors"
	"chain/log"
)

// Annotator describes a function capable of adding annotations
// to transactions, inputs and outputs.
type Annotator func(ctx context.Context, txs []map[string]interface{}) error

// RegisterAnnotator adds an additional annotator capable of mutating
// the annotated transaction object.
func (ind *Indexer) RegisterAnnotator(annotator Annotator) {
	ind.annotators = append(ind.annotators, annotator)
}

// indexBlockCallback is registered as a block callback on the cos.FC. It
// saves all annotated transactions to the database and indexes them according
// to the Core's configured indexes.
func (ind *Indexer) indexBlockCallback(ctx context.Context, b *bc.Block) {
	err := ind.insertBlock(ctx, b)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	txs, err := ind.insertAnnotatedTxs(ctx, b)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	err = ind.insertAnnotatedOutputs(ctx, b, txs)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	// TODO(jackson): Build indexes
}

func (ind *Indexer) insertBlock(ctx context.Context, b *bc.Block) error {
	const q = `
		INSERT INTO query_blocks (height, timestamp) VALUES($1, $2)
		ON CONFLICT (height) DO NOTHING
	`
	_, err := ind.db.Exec(ctx, q, b.Height, b.TimestampMS)
	return errors.Wrap(err, "inserting block timestamp")
}

func (ind *Indexer) insertAnnotatedTxs(ctx context.Context, b *bc.Block) ([]map[string]interface{}, error) {
	var (
		hashes              = pg.Strings(make([]string, 0, len(b.Transactions)))
		positions           = pg.Uint32s(make([]uint32, 0, len(b.Transactions)))
		annotatedTxs        = pg.Strings(make([]string, 0, len(b.Transactions)))
		annotatedTxsDecoded = make([]map[string]interface{}, 0, len(b.Transactions))
	)
	for pos, tx := range b.Transactions {
		hashes = append(hashes, tx.Hash.String())
		positions = append(positions, uint32(pos))
		annotatedTxsDecoded = append(annotatedTxsDecoded, transactionObject(tx, b, uint32(pos)))
	}

	dbctx := pg.NewContext(ctx, ind.db)

	for _, annotator := range ind.annotators {
		err := annotator(dbctx, annotatedTxsDecoded)
		if err != nil {
			return nil, errors.Wrap(err, "adding external annotations")
		}
	}

	for _, decoded := range annotatedTxsDecoded {
		b, err := json.Marshal(decoded)
		if err != nil {
			return nil, err
		}
		annotatedTxs = append(annotatedTxs, string(b))
	}

	// Save the annotated txs to the database.
	const insertQ = `
		INSERT INTO annotated_txs(block_height, tx_pos, tx_hash, data)
		SELECT $1, unnest($2::integer[]), unnest($3::text[]), unnest($4::jsonb[])
		ON CONFLICT (block_height, tx_pos) DO NOTHING;
	`
	_, err := ind.db.Exec(ctx, insertQ, b.Height, positions, hashes, annotatedTxs)
	if err != nil {
		return nil, errors.Wrap(err, "inserting annotated_txs to db")
	}
	return annotatedTxsDecoded, nil
}

func (ind *Indexer) insertAnnotatedOutputs(ctx context.Context, b *bc.Block, annotatedTxs []map[string]interface{}) error {
	var (
		outputTxPositions pg.Uint32s
		outputIndexes     pg.Uint32s
		outputTxHashes    pg.Strings
		outputData        pg.Strings
		prevoutHashes     pg.Strings
		prevoutIndexes    pg.Uint32s
	)

	for pos, tx := range b.Transactions {
		for _, in := range tx.Inputs {
			if !in.IsIssuance() {
				prevoutHashes = append(prevoutHashes, in.Outpoint().Hash.String())
				prevoutIndexes = append(prevoutIndexes, in.Outpoint().Index)
			}
		}

		outs, ok := annotatedTxs[pos]["outputs"].([]interface{})
		if !ok {
			return errors.Wrap(fmt.Errorf("bad outputs type %T", annotatedTxs[pos]["outputs"]))
		}
		for outIndex, out := range outs {
			txOut, ok := out.(map[string]interface{})
			if !ok {
				return errors.Wrap(fmt.Errorf("bad output type %T", out))
			}

			txOutCopy := make(map[string]interface{}, len(txOut))
			for k, v := range txOut {
				txOutCopy[k] = v // be extra paranoid; don't modify txOut
			}

			txOutCopy["transaction_id"] = tx.Hash
			serializedData, err := json.Marshal(txOutCopy)
			if err != nil {
				return errors.Wrap(err, "serializing annotated output")
			}

			outputTxPositions = append(outputTxPositions, uint32(pos))
			outputIndexes = append(outputIndexes, uint32(outIndex))
			outputTxHashes = append(outputTxHashes, tx.Hash.String())
			outputData = append(outputData, string(serializedData))
		}
	}

	// Insert all of the block's outputs at once.
	const insertQ = `
		INSERT INTO annotated_outputs (block_height, tx_pos, output_index, tx_hash, data, timespan)
		SELECT $1, unnest($2::integer[]), unnest($3::integer[]), unnest($4::text[]),
		           unnest($5::jsonb[]),   int8range($6, NULL)
		ON CONFLICT (block_height, tx_pos, output_index) DO NOTHING;
	`
	_, err := ind.db.Exec(ctx, insertQ, b.Height, outputTxPositions,
		outputIndexes, outputTxHashes, outputData, b.TimestampMS)
	if err != nil {
		return errors.Wrap(err, "batch inserting annotated outputs")
	}

	const updateQ = `
		UPDATE annotated_outputs SET timespan = INT8RANGE(LOWER(timespan), $1)
		WHERE (tx_hash, output_index) IN (SELECT unnest($2::text[]), unnest($3::integer[]))
	`
	_, err = ind.db.Exec(ctx, updateQ, b.TimestampMS, prevoutHashes, prevoutIndexes)
	return errors.Wrap(err, "updating spent annotated outputs")
}
<|endoftext|>


<commit_before>// Package ast declares the types used to represent syntax trees for HCL
// (HashiCorp Configuration Language)
package ast

import (
	"strings"

	"github.com/hashicorp/hcl/hcl/token"
)

// Node is an element in the abstract syntax tree.
type Node interface {
	node()
	Pos() token.Pos
}

func (File) node()         {}
func (ObjectList) node()   {}
func (ObjectKey) node()    {}
func (ObjectItem) node()   {}
func (Comment) node()      {}
func (CommentGroup) node() {}
func (ObjectType) node()   {}
func (LiteralType) node()  {}
func (ListType) node()     {}

// File represents a single HCL file
type File struct {
	Node     Node            // usually a *ObjectList
	Comments []*CommentGroup // list of all comments in the source
}

func (f *File) Pos() token.Pos {
	return f.Node.Pos()
}

// ObjectList represents a list of ObjectItems. An HCL file itself is an
// ObjectList.
type ObjectList struct {
	Items []*ObjectItem
}

func (o *ObjectList) Add(item *ObjectItem) {
	o.Items = append(o.Items, item)
}

// Filter filters out the objects with the given key list as a prefix.
//
// The returned list of objects contain ObjectItems where the keys have
// this prefix already stripped off. This might result in objects with
// zero-length key lists if they have no children.
//
// If no matches are found, an empty ObjectList (non-nil) is returned.
func (o *ObjectList) Filter(keys ...string) *ObjectList {
	var result ObjectList
	for _, item := range o.Items {
		// If there aren't enough keys, then ignore this
		if len(item.Keys) < len(keys) {
			continue
		}

		match := true
		for i, key := range item.Keys[:len(keys)] {
			key := key.Token.Value().(string)
			if key != keys[i] && !strings.EqualFold(key, keys[i]) {
				match = false
				break
			}
		}
		if !match {
			continue
		}

		// Strip off the prefix from the children
		newItem := *item
		newItem.Keys = newItem.Keys[len(keys):]
		result.Add(&newItem)
	}

	return &result
}

// Children returns further nested objects (key length > 0) within this
// ObjectList. This should be used with Filter to get at child items.
func (o *ObjectList) Children() *ObjectList {
	var result ObjectList
	for _, item := range o.Items {
		if len(item.Keys) > 0 {
			result.Add(item)
		}
	}

	return &result
}

// Elem returns items in the list that are direct element assignments
// (key length == 0). This should be used with Filter to get at elements.
func (o *ObjectList) Elem() *ObjectList {
	var result ObjectList
	for _, item := range o.Items {
		if len(item.Keys) == 0 {
			result.Add(item)
		}
	}

	return &result
}

func (o *ObjectList) Pos() token.Pos {
	// always returns the uninitiliazed position
	return o.Items[0].Pos()
}

// ObjectItem represents a HCL Object Item. An item is represented with a key
// (or keys). It can be an assignment or an object (both normal and nested)
type ObjectItem struct {
	// keys is only one length long if it's of type assignment. If it's a
	// nested object it can be larger than one. In that case "assign" is
	// invalid as there is no assignments for a nested object.
	Keys []*ObjectKey

	// assign contains the position of "=", if any
	Assign token.Pos

	// val is the item itself. It can be an object,list, number, bool or a
	// string. If key length is larger than one, val can be only of type
	// Object.
	Val Node

	LeadComment *CommentGroup // associated lead comment
	LineComment *CommentGroup // associated line comment
}

func (o *ObjectItem) Pos() token.Pos {
	return o.Keys[0].Pos()
}

// ObjectKeys are either an identifier or of type string.
type ObjectKey struct {
	Token token.Token
}

func (o *ObjectKey) Pos() token.Pos {
	return o.Token.Pos
}

// LiteralType represents a literal of basic type. Valid types are:
// token.NUMBER, token.FLOAT, token.BOOL and token.STRING
type LiteralType struct {
	Token token.Token

	// associated line comment, only when used in a list
	LineComment *CommentGroup
}

func (l *LiteralType) Pos() token.Pos {
	return l.Token.Pos
}

// ListStatement represents a HCL List type
type ListType struct {
	Lbrack token.Pos // position of "["
	Rbrack token.Pos // position of "]"
	List   []Node    // the elements in lexical order
}

func (l *ListType) Pos() token.Pos {
	return l.Lbrack
}

func (l *ListType) Add(node Node) {
	l.List = append(l.List, node)
}

// ObjectType represents a HCL Object Type
type ObjectType struct {
	Lbrace token.Pos   // position of "{"
	Rbrace token.Pos   // position of "}"
	List   *ObjectList // the nodes in lexical order
}

func (o *ObjectType) Pos() token.Pos {
	return o.Lbrace
}

// Comment node represents a single //, # style or /*- style commment
type Comment struct {
	Start token.Pos // position of / or #
	Text  string
}

func (c *Comment) Pos() token.Pos {
	return c.Start
}

// CommentGroup node represents a sequence of comments with no other tokens and
// no empty lines between.
type CommentGroup struct {
	List []*Comment // len(List) > 0
}

func (c *CommentGroup) Pos() token.Pos {
	return c.List[0].Pos()
}
<commit_msg>hcl/ast: just adding a GoSTringer impl<commit_after>// Package ast declares the types used to represent syntax trees for HCL
// (HashiCorp Configuration Language)
package ast

import (
	"fmt"
	"strings"

	"github.com/hashicorp/hcl/hcl/token"
)

// Node is an element in the abstract syntax tree.
type Node interface {
	node()
	Pos() token.Pos
}

func (File) node()         {}
func (ObjectList) node()   {}
func (ObjectKey) node()    {}
func (ObjectItem) node()   {}
func (Comment) node()      {}
func (CommentGroup) node() {}
func (ObjectType) node()   {}
func (LiteralType) node()  {}
func (ListType) node()     {}

// File represents a single HCL file
type File struct {
	Node     Node            // usually a *ObjectList
	Comments []*CommentGroup // list of all comments in the source
}

func (f *File) Pos() token.Pos {
	return f.Node.Pos()
}

// ObjectList represents a list of ObjectItems. An HCL file itself is an
// ObjectList.
type ObjectList struct {
	Items []*ObjectItem
}

func (o *ObjectList) Add(item *ObjectItem) {
	o.Items = append(o.Items, item)
}

// Filter filters out the objects with the given key list as a prefix.
//
// The returned list of objects contain ObjectItems where the keys have
// this prefix already stripped off. This might result in objects with
// zero-length key lists if they have no children.
//
// If no matches are found, an empty ObjectList (non-nil) is returned.
func (o *ObjectList) Filter(keys ...string) *ObjectList {
	var result ObjectList
	for _, item := range o.Items {
		// If there aren't enough keys, then ignore this
		if len(item.Keys) < len(keys) {
			continue
		}

		match := true
		for i, key := range item.Keys[:len(keys)] {
			key := key.Token.Value().(string)
			if key != keys[i] && !strings.EqualFold(key, keys[i]) {
				match = false
				break
			}
		}
		if !match {
			continue
		}

		// Strip off the prefix from the children
		newItem := *item
		newItem.Keys = newItem.Keys[len(keys):]
		result.Add(&newItem)
	}

	return &result
}

// Children returns further nested objects (key length > 0) within this
// ObjectList. This should be used with Filter to get at child items.
func (o *ObjectList) Children() *ObjectList {
	var result ObjectList
	for _, item := range o.Items {
		if len(item.Keys) > 0 {
			result.Add(item)
		}
	}

	return &result
}

// Elem returns items in the list that are direct element assignments
// (key length == 0). This should be used with Filter to get at elements.
func (o *ObjectList) Elem() *ObjectList {
	var result ObjectList
	for _, item := range o.Items {
		if len(item.Keys) == 0 {
			result.Add(item)
		}
	}

	return &result
}

func (o *ObjectList) Pos() token.Pos {
	// always returns the uninitiliazed position
	return o.Items[0].Pos()
}

// ObjectItem represents a HCL Object Item. An item is represented with a key
// (or keys). It can be an assignment or an object (both normal and nested)
type ObjectItem struct {
	// keys is only one length long if it's of type assignment. If it's a
	// nested object it can be larger than one. In that case "assign" is
	// invalid as there is no assignments for a nested object.
	Keys []*ObjectKey

	// assign contains the position of "=", if any
	Assign token.Pos

	// val is the item itself. It can be an object,list, number, bool or a
	// string. If key length is larger than one, val can be only of type
	// Object.
	Val Node

	LeadComment *CommentGroup // associated lead comment
	LineComment *CommentGroup // associated line comment
}

func (o *ObjectItem) Pos() token.Pos {
	return o.Keys[0].Pos()
}

// ObjectKeys are either an identifier or of type string.
type ObjectKey struct {
	Token token.Token
}

func (o *ObjectKey) Pos() token.Pos {
	return o.Token.Pos
}

// LiteralType represents a literal of basic type. Valid types are:
// token.NUMBER, token.FLOAT, token.BOOL and token.STRING
type LiteralType struct {
	Token token.Token

	// associated line comment, only when used in a list
	LineComment *CommentGroup
}

func (l *LiteralType) Pos() token.Pos {
	return l.Token.Pos
}

// ListStatement represents a HCL List type
type ListType struct {
	Lbrack token.Pos // position of "["
	Rbrack token.Pos // position of "]"
	List   []Node    // the elements in lexical order
}

func (l *ListType) Pos() token.Pos {
	return l.Lbrack
}

func (l *ListType) Add(node Node) {
	l.List = append(l.List, node)
}

// ObjectType represents a HCL Object Type
type ObjectType struct {
	Lbrace token.Pos   // position of "{"
	Rbrace token.Pos   // position of "}"
	List   *ObjectList // the nodes in lexical order
}

func (o *ObjectType) Pos() token.Pos {
	return o.Lbrace
}

// Comment node represents a single //, # style or /*- style commment
type Comment struct {
	Start token.Pos // position of / or #
	Text  string
}

func (c *Comment) Pos() token.Pos {
	return c.Start
}

// CommentGroup node represents a sequence of comments with no other tokens and
// no empty lines between.
type CommentGroup struct {
	List []*Comment // len(List) > 0
}

func (c *CommentGroup) Pos() token.Pos {
	return c.List[0].Pos()
}

//-------------------------------------------------------------------
// GoStringer
//-------------------------------------------------------------------

func (o *ObjectKey) GoString() string { return fmt.Sprintf("*%#v", *o) }
<|endoftext|>