<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type dataHandler struct {
	cond condition

	// dependent objects, which will be collected in the second pass
	depNodes    []int64
	depNodesMtx sync.Mutex
	depWays     []int64
	depWaysMtx  sync.Mutex

	nodes    []gosmparse.Node
	nodesMtx sync.Mutex
	ways     []gosmparse.Way
	waysMtx  sync.Mutex
	rels     []gosmparse.Relation
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if v, ok := n.Tags[d.cond.key]; ok {
		if len(d.cond.value) != 0 && d.cond.value != v {
			return
		}
		d.nodesMtx.Lock()
		d.nodes = append(d.nodes, n)
		d.nodesMtx.Unlock()
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if v, ok := w.Tags[d.cond.key]; ok {
		if len(d.cond.value) != 0 && d.cond.value != v {
			return
		}

		d.depNodesMtx.Lock()
		d.depNodes = append(d.depNodes, w.NodeIDs...)
		d.depNodesMtx.Unlock()

		d.waysMtx.Lock()
		d.ways = append(d.ways, w)
		d.waysMtx.Unlock()
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if v, ok := r.Tags[d.cond.key]; ok {
		if len(d.cond.value) != 0 && d.cond.value != v {
			return
		}

		d.relsMtx.Lock()
		d.rels = append(d.rels, r)
		d.relsMtx.Unlock()

		for _, memb := range r.Members {
			switch memb.Type {
			case gosmparse.NodeType:
				d.depNodesMtx.Lock()
				d.depNodes = append(d.depNodes, memb.ID)
				d.depNodesMtx.Unlock()
			case gosmparse.WayType:
				d.depWaysMtx.Lock()
				d.depWays = append(d.depWays, memb.ID)
				d.depWaysMtx.Unlock()
			} // TODO: check if relations of relations are necessary
		}
	}
}

type wayNodeCollector struct {
	wys map[int64]struct{}

	depNodes    []int64
	depNodesMtx sync.Mutex
}

func (wnc *wayNodeCollector) ReadNode(n gosmparse.Node) {}
func (wnc *wayNodeCollector) ReadWay(w gosmparse.Way) {
	if _, ok := wnc.wys[w.ID]; ok {
		wnc.depNodesMtx.Lock()
		wnc.depNodes = append(wnc.depNodes, w.NodeIDs...)
		wnc.depNodesMtx.Unlock()
	}
}
func (wnc *wayNodeCollector) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	nds    map[int64]spatial.Point
	ndsMtx sync.Mutex
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ndsMtx.Lock()
	defer d.ndsMtx.Unlock()
	if _, ok := d.nds[n.ID]; ok {
		d.nds[n.ID] = spatial.Point{float64(n.Lon), float64(n.Lat)}
	}
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

// const (
// 	typAny      = 0
// 	typNode     = 1
// 	typWay      = 2
// 	typRelation = 3
// )

type condition struct {
	key   string
	value string
}

func main() {
	cond := condition{"building", ""}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cudgf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	dh := dataHandler{
		cond: cond,
	}
	log.Println("Collecting data...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	clNds := dh.depNodes

	// Second pass
	log.Println("Collecting way nodes...")
	wayMap := map[int64]struct{}{}
	for _, wayID := range dh.depWays {
		wayMap[wayID] = struct{}{}
	}
	wnc := wayNodeCollector{
		wys: wayMap,
	}
	err = dec.Parse(&wnc)
	if err != nil {
		log.Fatal(err)
	}
	clNds = append(clNds, wnc.depNodes...)

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects")
	ndmap := map[int64]spatial.Point{}
	for _, ndID := range clNds {
		ndmap[ndID] = spatial.Point{}
	}
	rc := nodeCollector{
		nds: ndmap,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Assembling ways")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		props := map[string]interface{}{}
		for k, v := range wy.Tags {
			props[k] = v
		}

		var ls spatial.Line
		for _, nID := range wy.NodeIDs {
			ls = append(ls, rc.nds[nID])
		}

		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(ls),
		})
	}

	log.Println("Assembling relations")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" {
				if len(poly) != 0 {
					// TODO: allow polygons with multiple outer rings and split them
					break
				}
				poly = append(poly, spatial.Line{})
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}

// func wayToLine(w gosmparse.Way) spatial.Line {

// }
<commit_msg>cmd/spatialize: separated element cache<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type dataHandler struct {
	cond condition

	ec *elemCache

	nodes    []gosmparse.Node
	nodesMtx sync.Mutex
	ways     []gosmparse.Way
	waysMtx  sync.Mutex
	rels     []gosmparse.Relation
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if v, ok := n.Tags[d.cond.key]; ok {
		if len(d.cond.value) != 0 && d.cond.value != v {
			return
		}
		d.nodesMtx.Lock()
		d.nodes = append(d.nodes, n)
		d.nodesMtx.Unlock()
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if v, ok := w.Tags[d.cond.key]; ok {
		if len(d.cond.value) != 0 && d.cond.value != v {
			return
		}
		d.ec.AddNodes(w.NodeIDs...)
		d.ec.setMembers(w.ID, w.NodeIDs)

		d.waysMtx.Lock()
		d.ways = append(d.ways, w)
		d.waysMtx.Unlock()
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	// TODO: make it possible to specify condition type (node/way/rel)
	if v, ok := r.Tags[d.cond.key]; ok {
		if len(d.cond.value) != 0 && d.cond.value != v {
			return
		}

		d.relsMtx.Lock()
		d.rels = append(d.rels, r)
		d.relsMtx.Unlock()

		for _, memb := range r.Members {
			switch memb.Type {
			case gosmparse.WayType:
				d.ec.AddWay(memb.ID)
			} // TODO: check if relations of nodes/relations are necessary
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

// const (
// 	typAny      = 0
// 	typNode     = 1
// 	typWay      = 2
// 	typRelation = 3
// )

type condition struct {
	key   string
	value string
}

func main() {
	cond := condition{"building", ""}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		cond: cond,
		ec:   ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")
	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		props := map[string]interface{}{}
		for k, v := range wy.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(ec.Line(wy.ID)),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" {
				if len(poly) != 0 {
					// TODO: allow polygons with multiple outer rings and split them
					break
				}
				poly = append(poly, spatial.Line{})
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}

func wayToLine(w gosmparse.Way) spatial.Line {
	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2021 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package cli

import (
	"github.com/urfave/cli"
)

var pendingCommand = cli.Command{
	Name:     "pending",
	HideHelp: true,
	Usage:    "Pending subcommand group",
	Subcommands: []cli.Command{
		{
			Name:   "devices",
			Usage:  "Show pending devices",
			Action: expects(0, indexDumpOutput("cluster/pending/devices")),
		},
		{
			Name:   "folders",
			Usage:  "Show pending folders",
			Action: expects(0, indexDumpOutput("cluster/pending/folders")),
		},
	},
}
<commit_msg>cmd/syncthing/cli: Add showing pending folders for given device (fixes #8130) (#8131)<commit_after>// Copyright (C) 2021 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package cli

import (
	"net/url"

	"github.com/urfave/cli"
)

var pendingCommand = cli.Command{
	Name:     "pending",
	HideHelp: true,
	Usage:    "Pending subcommand group",
	Subcommands: []cli.Command{
		{
			Name:   "devices",
			Usage:  "Show pending devices",
			Action: expects(0, indexDumpOutput("cluster/pending/devices")),
		},
		{
			Name:  "folders",
			Usage: "Show pending folders",
			Flags: []cli.Flag{
				cli.StringFlag{Name: "device", Usage: "Show pending folders offered by given device"},
			},
			Action: expects(0, folders()),
		},
	},
}

func folders() cli.ActionFunc {
	return func(c *cli.Context) error {
		if c.String("device") != "" {
			query := make(url.Values)
			query.Set("device", c.String("device"))
			return indexDumpOutput("cluster/pending/folders?" + query.Encode())(c)
		}
		return indexDumpOutput("cluster/pending/folders")(c)
	}
}
<|endoftext|>


<commit_before>// Package ircclient provides the main interface for library users
// It manages a single connection to the server and the associated
// configuration and plugins.
package ircclient

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

type IRCClient struct {
	conn       *ircConn
	plugins    map[string]Plugin
	handlers   map[string]handler
	disconnect chan bool
}

type handler struct {
	Handler   Plugin
	Command   string
	Minparams int
	Minaccess int
}

// Returns a new IRCClient connection with the given configuration options.
// It will not connect to the given server until Connect() has been called,
// so you can register plugins before connecting
func NewIRCClient(configfile string) *IRCClient {
	c := &IRCClient{nil, make(map[string]Plugin), make(map[string]handler), make(chan bool)}
	c.RegisterPlugin(&basicProtocol{})
	c.RegisterPlugin(NewConfigPlugin(configfile))
	c.RegisterPlugin(new(authPlugin))
	return c
}

// Registers a new plugin. Plugins can be registered at any time, even before
// the actual connection attempt. The plugin's Unregister() function will already
// be called when the connection is lost.
func (ic *IRCClient) RegisterPlugin(p Plugin) error {
	if _, ok := ic.plugins[p.String()]; ok == true {
		return errors.New("Plugin already exists")
	}
	p.Register(ic)
	ic.plugins[p.String()] = p
	return nil
}

// Registers a command handler. Plugin callbacks will only be called if
// the command matches. Note that only a single plugin per command may
// be registered. This function is not synchronized, e.g., it shall only
// be called during registration (as Plugin.Register()-calls are currently
// sequential).
func (ic *IRCClient) RegisterCommandHandler(command string, minparams int, minaccess int, plugin Plugin) error {
	if plug, err := ic.handlers[command]; err {
		return errors.New("Handler is already registered by plugin: " + plug.Handler.String())
	}
	ic.handlers[command] = handler{plugin, command, minparams, minaccess}
	return nil
}

// Gets one of the configuration options stored in the config object. Valid config
// options for section "Server" usually include:
//  - nick
//  - hostport (colon-seperated host and port to connect to)
//  - realname (the real name)
//  - ident
//  - trigger
// All other sections are managed by the library user. Returns an
// empty string if the option is empty, this means: you currently can't
// use empty config values - they will be deemed non-existent!
func (ic *IRCClient) GetStringOption(section, option string) string {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	retval, _ := cf.Conf.String(section, option)
	cf.Unlock()
	return retval
}

// Sets a single config option. Existing parameters are overriden,
// if necessary, a new config section is automatically added.
func (ic *IRCClient) SetStringOption(section, option, value string) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	if !cf.Conf.HasSection(section) {
		cf.Conf.AddSection(section)
	}
	if cf.Conf.HasOption(section, option) {
		cf.Conf.RemoveOption(section, option)
	}
	cf.Conf.AddOption(section, option, value)
	cf.Unlock()
}

// Removes a single config option. Note: This does not delete the section,
// even if it's empty.
func (ic *IRCClient) RemoveOption(section, option string) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()

	if !cf.Conf.HasSection(section) {
		// nothing to do
		return
	}
	cf.Conf.RemoveOption(section, option)
}

// Gets a list of all config keys for a given section. The return value is
// an empty slice if there are no options present _or_ if there is no
// section present. There is currently no way to check whether a section
// exists, it is automatically added when calling one of the SetOption()
// methods.
func (ic *IRCClient) GetOptions(section string) []string {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()
	opts, err := cf.Conf.Options(section)
	if err != nil {
		return []string{}
	}
	return opts
}

// Does the same as GetStringOption(), but with integers. Returns an os.Error,
// if the given config option does not exist.
func (ic *IRCClient) GetIntOption(section, option string) (int, error) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()
	v, err := cf.Conf.Int(section, option)
	if err != nil {
		return -1, err
	}
	return v, nil
}

// See SetStringOption()
func (ic *IRCClient) SetIntOption(section, option string, value int) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()
	stropt := fmt.Sprintf("%d", value)
	if !cf.Conf.HasSection(section) {
		cf.Conf.AddSection(section)
	}
	cf.Conf.AddOption(section, option, stropt)
}

// Gets the highest matching access level for a given hostmask by comparing
// the mask against all authorization entries. Default return value is 0
// (no access).
func (ic *IRCClient) GetAccessLevel(host string) int {
	a := ic.plugins["auth"]
	auth, _ := a.(*authPlugin)
	return auth.GetAccessLevel(host)
}

// Sets the access level for the given hostmask to level. Note that host may
// be a regular expression, if exactly the same expression is already present
// in the database, it is overridden.
func (ic *IRCClient) SetAccessLevel(host string, level int) {
	a := ic.plugins["auth"]
	auth, _ := a.(*authPlugin)
	auth.SetAccessLevel(host, level)
}

// Delete the given regular expression from auth database. The "host" parameter
// has to be exactly the string stored in the database, otherwise, the command
// will have no effect.
func (ic *IRCClient) DelAccessLevel(host string) {
	a := ic.plugins["auth"]
	auth, _ := a.(*authPlugin)
	auth.DelAccessLevel(host)
}

// Connects to the server specified on object creation. If the chosen nickname is
// already in use, it will automatically be suffixed with an single underscore until
// an unused nickname is found. This function blocks until the connection attempt
// has been finished.
func (ic *IRCClient) Connect() error {
	ic.conn = NewircConn()
	e := ic.conn.Connect(ic.GetStringOption("Server", "host"))
	if e != nil {
		return e
	}

	// Doing bot online restart. Don't reregister.
	if len(os.Args) > 1 {
		return nil
	}

	ic.conn.Output <- "NICK " + ic.GetStringOption("Server", "nick")
	ic.conn.Output <- "USER " + ic.GetStringOption("Server", "ident") + " * Q :" + ic.GetStringOption("Server", "realname")
	nick := ic.GetStringOption("Server", "nick")

	for {
		line, ok := <-ic.conn.Input
		if !ok {
			return <-ic.conn.Err
		}

		// Invoke plugin line handlers.
		// At this point, it makes no sense to
		// process "commands". If a plugin needs
		// interaction in this state, it should be
		// low-level.
		s := ParseServerLine(line)
		if s == nil {
			continue
		}
		for _, p := range ic.plugins {
			go p.ProcessLine(s)
		}

		switch s.Command {
		case "433":
			// Nickname already in use
			nick = nick + "_"
			ic.SetStringOption("Server", "nick", nick)
			ic.conn.Output <- "NICK " + nick
		case "001":
			// Successfully registered
			return nil
		}
	}
	return nil
}

func (ic *IRCClient) dispatchHandlers(in string) {
	var c *IRCCommand = nil

	s := ParseServerLine(in)
	if s == nil {
		return
	}
	if (s.Command == "PRIVMSG" || s.Command == "NOTICE") && (s.Target == ic.GetStringOption("Server", "nick") || strings.Index(s.Args[0], ic.GetStringOption("Server", "trigger")) == 0) {
		c = ParseCommand(s)
		// Strip trigger, if necessary
		if c != nil && s.Target != ic.GetStringOption("Server", "nick") && len(c.Command) != 0 {
			c.Command = c.Command[len(ic.GetStringOption("Server", "trigger")):len(c.Command)]
		}
	}

	// Call line handlers
	for _, p := range ic.plugins {
		go p.ProcessLine(s)
	}

	// Call command handler
	if c == nil {
		return
	}
	if handler, err := ic.handlers[c.Command]; err == true {
		// Don't do regexp matching, if we don't need access anyway
		if handler.Minaccess > 0 && ic.GetAccessLevel(c.Source) < handler.Minaccess {
			ic.Reply(c, "You are not authorized to do that.")
			return
		}
		if len(c.Args) < handler.Minparams {
			ic.Reply(c, "This command requires at least "+fmt.Sprintf("%d", handler.Minparams)+" parameters")
			return
		}
		go handler.Handler.ProcessCommand(c)
	}
}

// Starts the actual command processing. This function will block until the connection
// has either been lost or Disconnect() has been called (by a plugin or by the library
// user).
func (ic *IRCClient) InputLoop() error {
	for {
		in, ok := <-ic.conn.Input
		if !ok {
			return <-ic.conn.Err
		}
		ic.dispatchHandlers(in)
	}
	panic("This never happens")
}

// Disconnects from the server with the given quit message. All plugins wil be unregistered
// and pending messages in queue (e.g. because of floodprotection) will be flushed. This will
// also make InputLoop() return.
func (ic *IRCClient) Disconnect(quitmsg string) {
	ic.Shutdown()
	ic.conn.Output <- "QUIT :" + quitmsg
	ic.conn.Quit()
}

// Dumps a raw line to the server socket. This is usually called by plugins, but may also
// be used by the library user.
func (ic *IRCClient) SendLine(line string) {
	ic.conn.Output <- line
}

func (ic *IRCClient) Shutdown() {
	for _, p := range ic.plugins {
		p.Unregister()
	}
}

// Returns a channel on which all command handlers will be sent.
func (ic *IRCClient) IterHandlers() <-chan handler {
	ch := make(chan handler, len(ic.handlers))
	go func() {
		for _, e := range ic.handlers {
			ch <- e
		}
		close(ch)
	}()
	return ch
}

// Get the pointer to a specific plugin that has been registered using RegisterPlugin()
// Name is the name the plugin identifies itself with when String() is called on it.
func (ic *IRCClient) GetPlugin(name string) Plugin {
	return ic.plugins[name]
}

// Get the Usage string from the Plugin that has registered itself as handler for
// the Command cmd. we need to wrap this to ircclient because the handlers are not
// public, and GetPlugin doesn't help us either, because the plugin<->command mapping
// is not known
func (ic *IRCClient) GetUsage(cmd string) string {
	plugin, exists := ic.handlers[cmd]
	if !exists {
		return "no such command"
	}
	return plugin.Handler.Usage(cmd)
}

// Sends a reply to a parsed message from a user. This is mostly intended for plugins
// and will automatically distinguish between channel and query messages. Note: Notice
// replies will currently be sent to the client using PRIVMSG, this may change in the
// future.
func (ic *IRCClient) Reply(cmd *IRCCommand, message string) {
	var target string
	if cmd.Target != ic.GetStringOption("Server", "nick") {
		target = cmd.Target
	} else {
		target = strings.SplitN(cmd.Source, "!", 2)[0]
	}
	ic.SendLine("NOTICE " + target + " :" + message)
}
func (ic *IRCClient) ReplyMsg(msg *IRCMessage, message string) {
	var target string
	if msg.Target != ic.GetStringOption("Server", "nick") {
		target = msg.Target
	} else {
		target = strings.SplitN(msg.Source, "!", 2)[0]
	}
	ic.SendLine("NOTICE " + target + " :" + message)
}

// Returns socket fd. Needed for kexec
func (ic *IRCClient) GetSocket() int {
	return ic.conn.GetSocket()
}

func (ic *IRCClient) GetPlugins() map[string]Plugin {
	return ic.plugins
}
<commit_msg>reworked dispatchHandlers()<commit_after>// Package ircclient provides the main interface for library users
// It manages a single connection to the server and the associated
// configuration and plugins.
package ircclient

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

type IRCClient struct {
	conn       *ircConn
	plugins    map[string]Plugin
	handlers   map[string]handler
	disconnect chan bool
}

type handler struct {
	Handler   Plugin
	Command   string
	Minparams int
	Minaccess int
}

// Returns a new IRCClient connection with the given configuration options.
// It will not connect to the given server until Connect() has been called,
// so you can register plugins before connecting
func NewIRCClient(configfile string) *IRCClient {
	c := &IRCClient{nil, make(map[string]Plugin), make(map[string]handler), make(chan bool)}
	c.RegisterPlugin(&basicProtocol{})
	c.RegisterPlugin(NewConfigPlugin(configfile))
	c.RegisterPlugin(new(authPlugin))
	return c
}

// Registers a new plugin. Plugins can be registered at any time, even before
// the actual connection attempt. The plugin's Unregister() function will already
// be called when the connection is lost.
func (ic *IRCClient) RegisterPlugin(p Plugin) error {
	if _, ok := ic.plugins[p.String()]; ok == true {
		return errors.New("Plugin already exists")
	}
	p.Register(ic)
	ic.plugins[p.String()] = p
	return nil
}

// Registers a command handler. Plugin callbacks will only be called if
// the command matches. Note that only a single plugin per command may
// be registered. This function is not synchronized, e.g., it shall only
// be called during registration (as Plugin.Register()-calls are currently
// sequential).
func (ic *IRCClient) RegisterCommandHandler(command string, minparams int, minaccess int, plugin Plugin) error {
	if plug, err := ic.handlers[command]; err {
		return errors.New("Handler is already registered by plugin: " + plug.Handler.String())
	}
	ic.handlers[command] = handler{plugin, command, minparams, minaccess}
	return nil
}

// Gets one of the configuration options stored in the config object. Valid config
// options for section "Server" usually include:
//  - nick
//  - hostport (colon-seperated host and port to connect to)
//  - realname (the real name)
//  - ident
//  - trigger
// All other sections are managed by the library user. Returns an
// empty string if the option is empty, this means: you currently can't
// use empty config values - they will be deemed non-existent!
func (ic *IRCClient) GetStringOption(section, option string) string {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	retval, _ := cf.Conf.String(section, option)
	cf.Unlock()
	return retval
}

// Sets a single config option. Existing parameters are overriden,
// if necessary, a new config section is automatically added.
func (ic *IRCClient) SetStringOption(section, option, value string) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	if !cf.Conf.HasSection(section) {
		cf.Conf.AddSection(section)
	}
	if cf.Conf.HasOption(section, option) {
		cf.Conf.RemoveOption(section, option)
	}
	cf.Conf.AddOption(section, option, value)
	cf.Unlock()
}

// Removes a single config option. Note: This does not delete the section,
// even if it's empty.
func (ic *IRCClient) RemoveOption(section, option string) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()

	if !cf.Conf.HasSection(section) {
		// nothing to do
		return
	}
	cf.Conf.RemoveOption(section, option)
}

// Gets a list of all config keys for a given section. The return value is
// an empty slice if there are no options present _or_ if there is no
// section present. There is currently no way to check whether a section
// exists, it is automatically added when calling one of the SetOption()
// methods.
func (ic *IRCClient) GetOptions(section string) []string {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()
	opts, err := cf.Conf.Options(section)
	if err != nil {
		return []string{}
	}
	return opts
}

// Does the same as GetStringOption(), but with integers. Returns an os.Error,
// if the given config option does not exist.
func (ic *IRCClient) GetIntOption(section, option string) (int, error) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()
	v, err := cf.Conf.Int(section, option)
	if err != nil {
		return -1, err
	}
	return v, nil
}

// See SetStringOption()
func (ic *IRCClient) SetIntOption(section, option string, value int) {
	c := ic.plugins["conf"]
	cf, _ := c.(*ConfigPlugin)
	cf.Lock()
	defer cf.Unlock()
	stropt := fmt.Sprintf("%d", value)
	if !cf.Conf.HasSection(section) {
		cf.Conf.AddSection(section)
	}
	cf.Conf.AddOption(section, option, stropt)
}

// Gets the highest matching access level for a given hostmask by comparing
// the mask against all authorization entries. Default return value is 0
// (no access).
func (ic *IRCClient) GetAccessLevel(host string) int {
	a := ic.plugins["auth"]
	auth, _ := a.(*authPlugin)
	return auth.GetAccessLevel(host)
}

// Sets the access level for the given hostmask to level. Note that host may
// be a regular expression, if exactly the same expression is already present
// in the database, it is overridden.
func (ic *IRCClient) SetAccessLevel(host string, level int) {
	a := ic.plugins["auth"]
	auth, _ := a.(*authPlugin)
	auth.SetAccessLevel(host, level)
}

// Delete the given regular expression from auth database. The "host" parameter
// has to be exactly the string stored in the database, otherwise, the command
// will have no effect.
func (ic *IRCClient) DelAccessLevel(host string) {
	a := ic.plugins["auth"]
	auth, _ := a.(*authPlugin)
	auth.DelAccessLevel(host)
}

// Connects to the server specified on object creation. If the chosen nickname is
// already in use, it will automatically be suffixed with an single underscore until
// an unused nickname is found. This function blocks until the connection attempt
// has been finished.
func (ic *IRCClient) Connect() error {
	ic.conn = NewircConn()
	e := ic.conn.Connect(ic.GetStringOption("Server", "host"))
	if e != nil {
		return e
	}

	// Doing bot online restart. Don't reregister.
	if len(os.Args) > 1 {
		return nil
	}

	ic.conn.Output <- "NICK " + ic.GetStringOption("Server", "nick")
	ic.conn.Output <- "USER " + ic.GetStringOption("Server", "ident") + " * Q :" + ic.GetStringOption("Server", "realname")
	nick := ic.GetStringOption("Server", "nick")

	for {
		line, ok := <-ic.conn.Input
		if !ok {
			return <-ic.conn.Err
		}

		// Invoke plugin line handlers.
		// At this point, it makes no sense to
		// process "commands". If a plugin needs
		// interaction in this state, it should be
		// low-level.
		s := ParseServerLine(line)
		if s == nil {
			continue
		}
		for _, p := range ic.plugins {
			go p.ProcessLine(s)
		}

		switch s.Command {
		case "433":
			// Nickname already in use
			nick = nick + "_"
			ic.SetStringOption("Server", "nick", nick)
			ic.conn.Output <- "NICK " + nick
		case "001":
			// Successfully registered
			return nil
		}
	}
	return nil
}

func (ic *IRCClient) dispatchHandlers(in string) {
	var c *IRCCommand = nil

	s := ParseServerLine(in)
	if s == nil {
		return
	}

	// Call line handlers
	for _, p := range ic.plugins {
		go p.ProcessLine(s)
	}

	if (s.Command != "PRIVMSG" && s.Command != "NOTICE") || strings.Index(s.Args[0], ic.GetStringOption("Server", "trigger")) != 0 {
		return
	}

	c = ParseCommand(s)
	if c == nil || len(c.Command) == 0 {
		return
	}

	// Strip trigger
	c.Command = c.Command[len(ic.GetStringOption("Server", "trigger")):]

	// Call command handler
	handler, ok := ic.handlers[c.Command]
	if !ok {
		return
	}

	// Don't do regexp matching, if we don't need access anyway
	if handler.Minaccess > 0 && ic.GetAccessLevel(c.Source) < handler.Minaccess {
		ic.Reply(c, "You are not authorized to do that.")
		return
	}
	if len(c.Args) < handler.Minparams {
		//ic.Reply(c, "This command requires at least "+fmt.Sprintf("%d", handler.Minparams)+" parameters")
		ic.Reply(c, ic.GetUsage(c.Command))
		return
	}
	go handler.Handler.ProcessCommand(c)
}

// Starts the actual command processing. This function will block until the connection
// has either been lost or Disconnect() has been called (by a plugin or by the library
// user).
func (ic *IRCClient) InputLoop() error {
	for {
		in, ok := <-ic.conn.Input
		if !ok {
			return <-ic.conn.Err
		}
		ic.dispatchHandlers(in)
	}
	panic("This never happens")
}

// Disconnects from the server with the given quit message. All plugins wil be unregistered
// and pending messages in queue (e.g. because of floodprotection) will be flushed. This will
// also make InputLoop() return.
func (ic *IRCClient) Disconnect(quitmsg string) {
	ic.Shutdown()
	ic.conn.Output <- "QUIT :" + quitmsg
	ic.conn.Quit()
}

// Dumps a raw line to the server socket. This is usually called by plugins, but may also
// be used by the library user.
func (ic *IRCClient) SendLine(line string) {
	ic.conn.Output <- line
}

func (ic *IRCClient) Shutdown() {
	for _, p := range ic.plugins {
		p.Unregister()
	}
}

// Returns a channel on which all command handlers will be sent.
func (ic *IRCClient) IterHandlers() <-chan handler {
	ch := make(chan handler, len(ic.handlers))
	go func() {
		for _, e := range ic.handlers {
			ch <- e
		}
		close(ch)
	}()
	return ch
}

// Get the pointer to a specific plugin that has been registered using RegisterPlugin()
// Name is the name the plugin identifies itself with when String() is called on it.
func (ic *IRCClient) GetPlugin(name string) Plugin {
	return ic.plugins[name]
}

// Get the Usage string from the Plugin that has registered itself as handler for
// the Command cmd. we need to wrap this to ircclient because the handlers are not
// public, and GetPlugin doesn't help us either, because the plugin<->command mapping
// is not known
func (ic *IRCClient) GetUsage(cmd string) string {
	plugin, exists := ic.handlers[cmd]
	if !exists {
		return "no such command"
	}
	return plugin.Handler.Usage(cmd)
}

// Sends a reply to a parsed message from a user. This is mostly intended for plugins
// and will automatically distinguish between channel and query messages. Note: Notice
// replies will currently be sent to the client using PRIVMSG, this may change in the
// future.
func (ic *IRCClient) Reply(cmd *IRCCommand, message string) {
	var target string
	if cmd.Target != ic.GetStringOption("Server", "nick") {
		target = cmd.Target
	} else {
		target = strings.SplitN(cmd.Source, "!", 2)[0]
	}
	ic.SendLine("NOTICE " + target + " :" + message)
}
func (ic *IRCClient) ReplyMsg(msg *IRCMessage, message string) {
	var target string
	if msg.Target != ic.GetStringOption("Server", "nick") {
		target = msg.Target
	} else {
		target = strings.SplitN(msg.Source, "!", 2)[0]
	}
	ic.SendLine("NOTICE " + target + " :" + message)
}

// Returns socket fd. Needed for kexec
func (ic *IRCClient) GetSocket() int {
	return ic.conn.GetSocket()
}

func (ic *IRCClient) GetPlugins() map[string]Plugin {
	return ic.plugins
}
<|endoftext|>


<commit_before>package k8s

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	model "github.com/servicebroker/servicebroker/k8s/service_controller/model"
)

var _ model.ServiceStorage = (*K8sServiceStorage)(nil)

type K8sServiceStorage struct {
	// Host is the location where we'll talk to k8s
	host            string
	defaultResource string
}

const serviceDomain string = "cncf.org"
const apiVersion string = "v1alpha1"
const brokerResource string = "servicebrokers"
const defaultUri string = "http://%v/apis/" + serviceDomain + "/" + apiVersion + "/namespaces/default/" + brokerResource

// The k8s implementation should leverage Third Party Resources
// https://github.com/kubernetes/kubernetes/blob/master/docs/design/extending-api.md

var _ model.ServiceStorage = (*K8sServiceStorage)(nil)

type Meta struct {
	Name string `json:"name"`
}

type KubeData struct {
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type k8sServiceBroker struct {
	*model.ServiceBroker
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type k8sService struct {
	*model.Service
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type k8sPlan struct {
	*model.ServicePlan
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type VName struct {
	Name string `json:"name"`
}

type TPR struct {
	Meta       `json:"metadata"`
	ApiVersion string  `json:"apiVersion"`
	kind       string  `json:"kind"`
	Versions   []VName `json:"versions"`
}

const TPRapiVersion string = "extensions/v1beta1"
const thirdPartyResourceString string = "ThirdPartyResource"

var versionMap []VName = []VName{{"v1alpha1"}}

// Kubernetes ThirdPartyResources definitions
var serviceBrokerDefinition TPR = TPR{Meta{"service-broker.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}

// sbservice so it does not conflict with the built in Service
var serviceDefinition TPR = TPR{Meta{"sbservice.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}
var servicePlanDefinition TPR = TPR{Meta{"service-plan.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}
var serviceInstanceDefinition TPR = TPR{Meta{"service-instance.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}
var serviceBindingDefinition TPR = TPR{Meta{"service-binding.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}

func CreateServiceStorage(host string) model.ServiceStorage {
	k := &K8sServiceStorage{host: host,
		defaultResource: fmt.Sprintf(defaultUri, host)}
	fmt.Println(" root host is:", k.defaultUri())
	// define the resources once at startup
	// results in ServiceBrokers

	k.createTPR(serviceBrokerDefinition)
	k.createTPR(serviceDefinition)
	k.createTPR(servicePlanDefinition)
	k.createTPR(serviceBindingDefinition)
	k.createTPR(serviceInstanceDefinition)
	// cleanup afterwards by `kubectl delete thirdpartyresource service-broker.cncf.org`

	return k
}

// listSB is only used for unmarshalling the list of service brokers
// for returning to the client
type listSB struct {
	Items []*k8sServiceBroker `json:"items"`
}

func (kss *K8sServiceStorage) defaultUri() string {
	return kss.defaultResource
}

func (kss *K8sServiceStorage) createTPR(tpr TPR) {
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(&tpr)
	fmt.Printf("encoded bytes: %v\n", b.String())
	r, e := http.Post("http://"+kss.host+"/apis/extensions/v1beta1/thirdpartyresources", "application/json", b)
	fmt.Printf("result: %v\n", r)
	if nil != e || 201 != r.StatusCode {
		fmt.Printf("Error creating k8s TPR [%s]...\n%v\n", e, r)
	}
}

/* BROKER */
/**********/

func (kss *K8sServiceStorage) ListBrokers() ([]string, error) {
	fmt.Println("listing all brokers")
	// get the ServiceBroker

	r, e := http.Get(kss.defaultUri())
	if nil != e {
		return nil, fmt.Errorf("couldn't get the service brokers. %v, [%v]", e, r)
	}

	var lsb listSB
	e = json.NewDecoder(r.Body).Decode(&lsb)
	if nil != e { // wrong json format error
		fmt.Println("json not unmarshalled:", e, r)
		return nil, e
	}
	fmt.Println("Got", len(lsb.Items), "brokers.")
	ret := make([]string, 0, len(lsb.Items))
	for _, v := range lsb.Items {
		ret = append(ret, v.ServiceBroker.ID)
	}
	return ret, nil
}

func (kss *K8sServiceStorage) AddBroker(broker *model.ServiceBroker) error {
	fmt.Println("adding broker to k8s", broker)
	// create TPR
	// tpr is
	//    kind.fqdn
	// or
	//    kind.domain.tld
	//
	// use service-broker.cncf.org
	// end up with k8s resource of ServiceBroker
	// version v1alpha1 for now
	//
	// store name/host/port/user/pass as metadata
	//
	// example yaml
	// metadata:
	//   name: service-broker.cncf.org
	//   (service)name/host/port/user/pass
	// apiVersion: extensions/v1beta1
	// kind: ThirdPartyResource
	// versions:
	// - name: v1alpha1
	ksb := NewK8sSB()
	ksb.Metadata = Meta{Name: broker.ID}
	ksb.ServiceBroker = broker

	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(&ksb)
	fmt.Printf("sending: %v", b)
	r, e := http.Post(kss.defaultUri(), "application/json", b)
	fmt.Sprintf("result: %v", r)
	if nil != e || 201 != r.StatusCode {
		fmt.Printf("Error creating k8s service broker TPR [%s]...\n%v\n", e, r)
		return e
	}

	fmt.Println("installing the", len(broker.Services), "services for this broker")
	for i, serviceID := range broker.Services {
		fmt.Println(i, serviceID)

		ks := NewK8sService()
		ks.Metadata = Meta{Name: serviceID}

		b := new(bytes.Buffer)
		if err := json.NewEncoder(b).Encode(&ks); nil != err {
			fmt.Println("failed to encode")
			return err
		}
		defaultUri := "http://%v/apis/" + serviceDomain + "/" + apiVersion + "/namespaces/default/" + "sbservices"
		fmt.Printf("sending: %v\n to %v", b, defaultUri)
		r, e := http.Post(fmt.Sprintf(defaultUri, kss.host), "application/json", b)
		fmt.Sprintf("result: %v", r)
		if nil != e || 201 != r.StatusCode {
			fmt.Printf("Error creating k8s service TPR [%s]...\n%v\n", e, r)
			return e
		}
	}

	return nil
}

func (kss *K8sServiceStorage) GetBroker(name string) (*model.ServiceBroker, error) {
	uri := kss.defaultUri() + "/" + name
	fmt.Println("uri is:", uri)
	r, e := http.Get(uri)
	if nil != e {
		return nil, fmt.Errorf("couldn't get the service broker. %v, [%v]", e, r)
	}
	defer r.Body.Close()
	var sb k8sServiceBroker
	e = json.NewDecoder(r.Body).Decode(&sb)
	if nil != e { // wrong json format error
		return nil, e
	}
	fmt.Printf("returned json: %+v\n", sb)
	return sb.ServiceBroker, nil
}

func (kss *K8sServiceStorage) SetBroker(si *model.ServiceBroker) error {
	return fmt.Errorf("SetBroker: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteBroker(name string) error {
	uri := kss.defaultUri() + "/" + name
	fmt.Println("uri is:", uri)

	// utter failure of an http API
	req, _ := http.NewRequest("DELETE", uri, nil)
	_, e := http.DefaultClient.Do(req)
	if nil != e {
		return fmt.Errorf("couldn't nuke %v, [%v]", name, e)
	}
	return nil
}

func NewK8sSB() *k8sServiceBroker {
	return &k8sServiceBroker{ApiVersion: serviceDomain + "/" + apiVersion,
		Kind: "ServiceBroker"}
}

/* Service */
/***********/

func NewK8sService() *k8sService {
	return &k8sService{ApiVersion: serviceDomain + "/" + apiVersion,
		Kind: "Sbservice"}
}

func (kss *K8sServiceStorage) ListServices() ([]string, error) {
	return nil, fmt.Errorf("ListServices: Not implemented yet")
}

func (s *K8sServiceStorage) GetServices() ([]*model.Service, error) {
	return nil, fmt.Errorf("GetServices: Not implemented yet")
}

func (kss *K8sServiceStorage) GetService(id string) (*model.Service, error) {
	return nil, fmt.Errorf("GetService: Not implemented yet")
}

func (kss *K8sServiceStorage) AddService(si *model.Service) error {
	return fmt.Errorf("AddService: Not implemented yet")
}

func (kss *K8sServiceStorage) SetService(si *model.Service) error {
	return fmt.Errorf("SetService: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteService(id string) error {
	return fmt.Errorf("DeleteService: Not implemented yet")
}

/* Plan */
/********/

func (kss *K8sServiceStorage) ListPlans() ([]string, error) {
	return nil, fmt.Errorf("ListPlans: Not implemented yet")
}

func (s *K8sServiceStorage) GetPlans() ([]*model.ServicePlan, error) {
	return nil, fmt.Errorf("GetPlans: Not implemented yet")
}

func (kss *K8sServiceStorage) GetPlan(id string) (*model.ServicePlan, error) {
	return nil, fmt.Errorf("GetPlan: Not implemented yet")
}

func (kss *K8sServiceStorage) AddPlan(plan *model.ServicePlan) error {
	return fmt.Errorf("AddPlan: Not implemented yet")
}

func (kss *K8sServiceStorage) SetPlan(si *model.ServicePlan) error {
	return fmt.Errorf("SetPlan: Not implemented yet")
}

func (kss *K8sServiceStorage) DeletePlan(id string) error {
	return fmt.Errorf("DeletePlan: Not implemented yet")
}

/* Instance */
/************/

func (kss *K8sServiceStorage) ListInstances() ([]string, error) {
	return nil, fmt.Errorf("ListInstances: Not implemented yet")
}

func (s *K8sServiceStorage) GetInstances() ([]*model.Service, error) {
	return nil, fmt.Errorf("GetInstances: Not implemented yet")
}

func (kss *K8sServiceStorage) GetInstance(id string) (*model.ServiceInstance, error) {
	return nil, fmt.Errorf("GetInstance: Not implemented yet")
}

func (kss *K8sServiceStorage) AddInstance(si *model.ServiceInstance) error {
	return fmt.Errorf("AddInstance: Not implemented yet")
}

func (kss *K8sServiceStorage) SetInstance(si *model.ServiceInstance) error {
	return fmt.Errorf("SetInstance: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteInstance(id string) error {
	return fmt.Errorf("DeleteInstance: Not implemented yet")
}

/* Binding */
/***********/
func (kss *K8sServiceStorage) ListBindings() ([]string, error) {
	return nil, fmt.Errorf("ListBindings: Not implemented yet")
}

func (kss *K8sServiceStorage) GetBinding(id string) (*model.ServiceBinding, error) {
	return nil, fmt.Errorf("GetBinding: Not implemented yet")
}

func (kss *K8sServiceStorage) AddBinding(binding *model.ServiceBinding) error {
	return fmt.Errorf("AddBinding: Not implemented yet")
}

func (kss *K8sServiceStorage) SetBinding(binding *model.ServiceBinding) error {
	return fmt.Errorf("SetBinding: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteBinding(id string) error {
	return fmt.Errorf("DeleteBinding: Not implemented yet")
}
<commit_msg>k8s AddService and AddPlan implementation<commit_after>package k8s

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	model "github.com/servicebroker/servicebroker/k8s/service_controller/model"
)

var _ model.ServiceStorage = (*K8sServiceStorage)(nil)

type K8sServiceStorage struct {
	// Host is the location where we'll talk to k8s
	host            string
	defaultResource string
}

const serviceDomain string = "cncf.org"
const apiVersion string = "v1alpha1"
const brokerResource string = "servicebrokers"
const defaultUri string = "http://%v/apis/" + serviceDomain + "/" + apiVersion + "/namespaces/default/" + brokerResource

// The k8s implementation should leverage Third Party Resources
// https://github.com/kubernetes/kubernetes/blob/master/docs/design/extending-api.md

var _ model.ServiceStorage = (*K8sServiceStorage)(nil)

type Meta struct {
	Name string `json:"name"`
}

type KubeData struct {
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type k8sServiceBroker struct {
	*model.ServiceBroker
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type k8sService struct {
	*model.Service
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type k8sPlan struct {
	*model.ServicePlan
	ApiVersion string `json:"apiVersion"`
	Kind       string `json:"kind"`
	Metadata   Meta   `json:"metadata"`
}

type VName struct {
	Name string `json:"name"`
}

type TPR struct {
	Meta       `json:"metadata"`
	ApiVersion string  `json:"apiVersion"`
	kind       string  `json:"kind"`
	Versions   []VName `json:"versions"`
}

const TPRapiVersion string = "extensions/v1beta1"
const thirdPartyResourceString string = "ThirdPartyResource"

var versionMap []VName = []VName{{"v1alpha1"}}

// Kubernetes ThirdPartyResources definitions
var serviceBrokerDefinition TPR = TPR{Meta{"service-broker.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}

// sbservice so it does not conflict with the built in Service
var serviceDefinition TPR = TPR{Meta{"sbservice.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}
var servicePlanDefinition TPR = TPR{Meta{"service-plan.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}
var serviceInstanceDefinition TPR = TPR{Meta{"service-instance.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}
var serviceBindingDefinition TPR = TPR{Meta{"service-binding.cncf.org"},
	TPRapiVersion, thirdPartyResourceString, versionMap}

func CreateServiceStorage(host string) model.ServiceStorage {
	k := &K8sServiceStorage{host: host,
		defaultResource: fmt.Sprintf(defaultUri, host)}
	fmt.Println(" root host is:", k.defaultUri())
	// define the resources once at startup
	// results in ServiceBrokers

	k.createTPR(serviceBrokerDefinition)
	k.createTPR(serviceDefinition)
	k.createTPR(servicePlanDefinition)
	k.createTPR(serviceBindingDefinition)
	k.createTPR(serviceInstanceDefinition)
	// cleanup afterwards by `kubectl delete thirdpartyresource service-broker.cncf.org`

	return k
}

// listSB is only used for unmarshalling the list of service brokers
// for returning to the client
type listSB struct {
	Items []*k8sServiceBroker `json:"items"`
}

func (kss *K8sServiceStorage) defaultUri() string {
	return kss.defaultResource
}

func (kss *K8sServiceStorage) createTPR(tpr TPR) {
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(&tpr)
	fmt.Printf("encoded bytes: %v\n", b.String())
	r, e := http.Post("http://"+kss.host+"/apis/extensions/v1beta1/thirdpartyresources", "application/json", b)
	fmt.Printf("result: %v\n", r)
	if nil != e || 201 != r.StatusCode {
		fmt.Printf("Error creating k8s TPR [%s]...\n%v\n", e, r)
	}
}

/* BROKER */
/**********/

func (kss *K8sServiceStorage) ListBrokers() ([]string, error) {
	fmt.Println("listing all brokers")
	// get the ServiceBroker

	r, e := http.Get(kss.defaultUri())
	if nil != e {
		return nil, fmt.Errorf("couldn't get the service brokers. %v, [%v]", e, r)
	}

	var lsb listSB
	e = json.NewDecoder(r.Body).Decode(&lsb)
	if nil != e { // wrong json format error
		fmt.Println("json not unmarshalled:", e, r)
		return nil, e
	}
	fmt.Println("Got", len(lsb.Items), "brokers.")
	ret := make([]string, 0, len(lsb.Items))
	for _, v := range lsb.Items {
		ret = append(ret, v.ServiceBroker.ID)
	}
	return ret, nil
}

func (kss *K8sServiceStorage) AddBroker(broker *model.ServiceBroker) error {
	fmt.Println("adding broker to k8s", broker)
	// create TPR
	// tpr is
	//    kind.fqdn
	// or
	//    kind.domain.tld
	//
	// use service-broker.cncf.org
	// end up with k8s resource of ServiceBroker
	// version v1alpha1 for now
	//
	// store name/host/port/user/pass as metadata
	//
	// example yaml
	// metadata:
	//   name: service-broker.cncf.org
	//   (service)name/host/port/user/pass
	// apiVersion: extensions/v1beta1
	// kind: ThirdPartyResource
	// versions:
	// - name: v1alpha1
	ksb := NewK8sSB()
	ksb.Metadata = Meta{Name: broker.ID}
	ksb.ServiceBroker = broker

	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(&ksb)
	fmt.Printf("sending: %v", b)
	r, e := http.Post(kss.defaultUri(), "application/json", b)
	fmt.Sprintf("result: %v", r)
	if nil != e || 201 != r.StatusCode {
		fmt.Printf("Error creating k8s service broker TPR [%s]...\n%v\n", e, r)
		return e
	}

	return nil
}

func (kss *K8sServiceStorage) GetBroker(name string) (*model.ServiceBroker, error) {
	uri := kss.defaultUri() + "/" + name
	fmt.Println("uri is:", uri)
	r, e := http.Get(uri)
	if nil != e {
		return nil, fmt.Errorf("couldn't get the service broker. %v, [%v]", e, r)
	}
	defer r.Body.Close()
	var sb k8sServiceBroker
	e = json.NewDecoder(r.Body).Decode(&sb)
	if nil != e { // wrong json format error
		return nil, e
	}
	fmt.Printf("returned json: %+v\n", sb)
	return sb.ServiceBroker, nil
}

func (kss *K8sServiceStorage) SetBroker(si *model.ServiceBroker) error {
	return fmt.Errorf("SetBroker: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteBroker(name string) error {
	uri := kss.defaultUri() + "/" + name
	fmt.Println("uri is:", uri)

	// utter failure of an http API
	req, _ := http.NewRequest("DELETE", uri, nil)
	_, e := http.DefaultClient.Do(req)
	if nil != e {
		return fmt.Errorf("couldn't nuke %v, [%v]", name, e)
	}
	return nil
}

func NewK8sSB() *k8sServiceBroker {
	return &k8sServiceBroker{ApiVersion: serviceDomain + "/" + apiVersion,
		Kind: "ServiceBroker"}
}

/* Service */
/***********/

func NewK8sService() *k8sService {
	return &k8sService{ApiVersion: serviceDomain + "/" + apiVersion,
		Kind: "Sbservice"}
}

func (kss *K8sServiceStorage) ListServices() ([]string, error) {
	return nil, fmt.Errorf("ListServices: Not implemented yet")
}

func (s *K8sServiceStorage) GetServices() ([]*model.Service, error) {
	return nil, fmt.Errorf("GetServices: Not implemented yet")
}

func (kss *K8sServiceStorage) GetService(id string) (*model.Service, error) {
	return nil, fmt.Errorf("GetService: Not implemented yet")
}

func (kss *K8sServiceStorage) AddService(si *model.Service) error {
	fmt.Println(si, si.ID)

	ks := NewK8sService()
	ks.Metadata = Meta{Name: si.ID}

	b := new(bytes.Buffer)
	if err := json.NewEncoder(b).Encode(&ks); nil != err {
		fmt.Println("failed to encode", si, "as", ks)
		return err
	}
	defaultUri := "http://%v/apis/" + serviceDomain + "/" + apiVersion + "/namespaces/default/" + "sbservices"
	fmt.Printf("sending: %v\n to %v", b, defaultUri)
	r, e := http.Post(fmt.Sprintf(defaultUri, kss.host), "application/json", b)
	fmt.Sprintf("result: %v", r)
	if nil != e || 201 != r.StatusCode {
		fmt.Printf("Error creating k8s service TPR [%s]...\n%v\n", e, r)
		return e
	}
	return nil
}

func (kss *K8sServiceStorage) SetService(si *model.Service) error {
	return fmt.Errorf("SetService: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteService(id string) error {
	return fmt.Errorf("DeleteService: Not implemented yet")
}

/* Plan */
/********/

func NewK8sPlan() *k8sPlan {
	return &k8sPlan{ApiVersion: serviceDomain + "/" + apiVersion,
		Kind: "ServicePlan"}
}

func (kss *K8sServiceStorage) ListPlans() ([]string, error) {
	return nil, fmt.Errorf("ListPlans: Not implemented yet")
}

func (s *K8sServiceStorage) GetPlans() ([]*model.ServicePlan, error) {
	return nil, fmt.Errorf("GetPlans: Not implemented yet")
}

func (kss *K8sServiceStorage) GetPlan(id string) (*model.ServicePlan, error) {
	return nil, fmt.Errorf("GetPlan: Not implemented yet")
}

func (kss *K8sServiceStorage) AddPlan(plan *model.ServicePlan) error {
	fmt.Println(plan, plan.ID)

	ks := NewK8sPlan()
	ks.Metadata = Meta{Name: plan.ID}

	b := new(bytes.Buffer)
	if err := json.NewEncoder(b).Encode(&ks); nil != err {
		fmt.Println("failed to encode", plan, "as", ks)
		return err
	}
	defaultUri := "http://%v/apis/" + serviceDomain + "/" + apiVersion + "/namespaces/default/" + "serviceplans"
	fmt.Printf("sending: %v\n to %v\n", b, defaultUri)
	r, e := http.Post(fmt.Sprintf(defaultUri, kss.host), "application/json", b)
	fmt.Sprintf("result: %v", r)
	if nil != e || 201 != r.StatusCode {
		fmt.Printf("Error creating k8s service TPR [%s]...\n%v\n", e, r)
		return e
	}
	return nil
}

func (kss *K8sServiceStorage) SetPlan(si *model.ServicePlan) error {
	return fmt.Errorf("SetPlan: Not implemented yet")
}

func (kss *K8sServiceStorage) DeletePlan(id string) error {
	return fmt.Errorf("DeletePlan: Not implemented yet")
}

/* Instance */
/************/

func (kss *K8sServiceStorage) ListInstances() ([]string, error) {
	return nil, fmt.Errorf("ListInstances: Not implemented yet")
}

func (s *K8sServiceStorage) GetInstances() ([]*model.Service, error) {
	return nil, fmt.Errorf("GetInstances: Not implemented yet")
}

func (kss *K8sServiceStorage) GetInstance(id string) (*model.ServiceInstance, error) {
	return nil, fmt.Errorf("GetInstance: Not implemented yet")
}

func (kss *K8sServiceStorage) AddInstance(si *model.ServiceInstance) error {
	return fmt.Errorf("AddInstance: Not implemented yet")
}

func (kss *K8sServiceStorage) SetInstance(si *model.ServiceInstance) error {
	return fmt.Errorf("SetInstance: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteInstance(id string) error {
	return fmt.Errorf("DeleteInstance: Not implemented yet")
}

/* Binding */
/***********/
func (kss *K8sServiceStorage) ListBindings() ([]string, error) {
	return nil, fmt.Errorf("ListBindings: Not implemented yet")
}

func (kss *K8sServiceStorage) GetBinding(id string) (*model.ServiceBinding, error) {
	return nil, fmt.Errorf("GetBinding: Not implemented yet")
}

func (kss *K8sServiceStorage) AddBinding(binding *model.ServiceBinding) error {
	return fmt.Errorf("AddBinding: Not implemented yet")
}

func (kss *K8sServiceStorage) SetBinding(binding *model.ServiceBinding) error {
	return fmt.Errorf("SetBinding: Not implemented yet")
}

func (kss *K8sServiceStorage) DeleteBinding(id string) error {
	return fmt.Errorf("DeleteBinding: Not implemented yet")
}
<|endoftext|>


<commit_before>package main

import (
	zmq "github.com/pebbe/zmq4"
	"github.com/anchor/chevalier"
	"github.com/anchor/picolog"
	"syscall"
)

var IndexerLogger *picolog.Logger

func originUpdate(w *chevalier.ElasticsearchWriter, endpoint string, origin string) {
	indexed := 0
	IndexerLogger.Infof("Requesting sources for origin %v.", origin)
	// We want to retry if we get interrupted.
	var err error
	var burst *chevalier.DataSourceBurst
	err = syscall.EAGAIN
	for err == syscall.EAGAIN || err == syscall.EINTR {
		burst, err = chevalier.GetContents(endpoint, origin)
	}
	if err != nil {
		IndexerLogger.Errorf("Could not read contents for origin %v: %v", origin, err)
		return
	}
	for _, s := range burst.Sources {
		err = w.Write(origin, s)
		if err != nil {
			IndexerLogger.Errorf("Could not index source: %v", err)
		} else {
			indexed += 1
		}
	}
	IndexerLogger.Infof("Indexed %v sources for origin %v.", indexed, origin)
}

func fullUpdate(w *chevalier.ElasticsearchWriter, endpoint string, origins []string) {
	for _, o := range origins {
		go originUpdate(w, endpoint, o)
	}
}

func subscribeUpdate(endpoint string) error {
	sock, err := zmq.NewSocket(zmq.SUB)
	if err != nil {
		return err
	}
	sock.SetSubscribe("")
	sock.Connect(endpoint)
	return nil
}

func getElasticsearchWriter(cfg Config) *chevalier.ElasticsearchWriter {
	writer := chevalier.NewElasticsearchWriter(cfg.Elasticsearch.Host, cfg.Elasticsearch.MaxConns, cfg.Elasticsearch.RetrySeconds, cfg.Elasticsearch.Index, cfg.Elasticsearch.DataType)
	return writer
}

func RunIndexerOnce(cfg Config) {
	IndexerLogger = Logger.NewSubLogger("indexer")
	IndexerLogger.Infof("Starting single indexer run.")
	writer := getElasticsearchWriter(cfg)
	fullUpdate(writer, cfg.Vaultaire.ReadEndpoint, cfg.Vaultaire.Origins)
}

func RunIndexer(cfg Config) {
	Logger.Infof("Starting chevalierd %v in indexer mode.", Version)
	IndexerLogger = Logger.NewSubLogger("indexer")
	writer := getElasticsearchWriter(cfg)
	for {
		IndexerLogger.Infof("Starting run.")
		fullUpdate(writer, cfg.Vaultaire.ReadEndpoint, cfg.Vaultaire.Origins)
	}
}
<commit_msg>Add more debug logging to the indexer<commit_after>package main

import (
	zmq "github.com/pebbe/zmq4"
	"github.com/anchor/chevalier"
	"github.com/anchor/picolog"
	"syscall"
)

var IndexerLogger *picolog.Logger

func originUpdate(w *chevalier.ElasticsearchWriter, endpoint string, origin string) {
	indexed := 0
	IndexerLogger.Infof("Requesting sources for origin %v.", origin)
	// We want to retry if we get interrupted.
	var err error
	var burst *chevalier.DataSourceBurst
	err = syscall.EAGAIN
	for err == syscall.EAGAIN || err == syscall.EINTR {
		burst, err = chevalier.GetContents(endpoint, origin)
	}
	if err != nil {
		IndexerLogger.Errorf("Could not read contents for origin %v: %v", origin, err)
		return
	}
	for _, s := range burst.Sources {
		err = w.Write(origin, s)
		IndexerLogger.Debugf("Writing source %v for origin %s.", s, origin)
		if err != nil {
			IndexerLogger.Errorf("Could not index source: %v", err)
		} else {
			indexed += 1
		}
	}
	IndexerLogger.Infof("Indexed %v sources for origin %v.", indexed, origin)
}

func fullUpdate(w *chevalier.ElasticsearchWriter, endpoint string, origins []string) {
	for _, o := range origins {
		go originUpdate(w, endpoint, o)
	}
}

func subscribeUpdate(endpoint string) error {
	sock, err := zmq.NewSocket(zmq.SUB)
	if err != nil {
		return err
	}
	sock.SetSubscribe("")
	sock.Connect(endpoint)
	return nil
}

func getElasticsearchWriter(cfg Config) *chevalier.ElasticsearchWriter {
	writer := chevalier.NewElasticsearchWriter(cfg.Elasticsearch.Host, cfg.Elasticsearch.MaxConns, cfg.Elasticsearch.RetrySeconds, cfg.Elasticsearch.Index, cfg.Elasticsearch.DataType)
	return writer
}

func RunIndexerOnce(cfg Config) {
	IndexerLogger = Logger.NewSubLogger("indexer")
	IndexerLogger.Infof("Starting single indexer run.")
	writer := getElasticsearchWriter(cfg)
	fullUpdate(writer, cfg.Vaultaire.ReadEndpoint, cfg.Vaultaire.Origins)
}

func RunIndexer(cfg Config) {
	Logger.Infof("Starting chevalierd %v in indexer mode.", Version)
	IndexerLogger = Logger.NewSubLogger("indexer")
	writer := getElasticsearchWriter(cfg)
	for {
		IndexerLogger.Infof("Starting run.")
		fullUpdate(writer, cfg.Vaultaire.ReadEndpoint, cfg.Vaultaire.Origins)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/json"
)

func GetReceipt(hash string) (*Receipt, error) {
	type receiptResponse struct {
		Receipt *Receipt `json:"receipt"`
	}

	params := hashRequest{Hash: hash}
	req := NewJSON2Request("receipt", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	rec := new(receiptResponse)
	if err := json.Unmarshal(resp.JSONResult(), rec); err != nil {
		return nil, err
	}

	return rec.Receipt, nil
}

type Receipt struct {
	Entry struct {
		Raw  string `json:"raw,omitempty"`
		Key  string `json:"key,omitempty"`
		Json string `json:"json,omitempty"`
	} `json:"entry,omitempty"`
	MerkleBranch []struct {
		Left  string `json:"left,omitempty"`
		Right string `json:"right,omitempty"`
		Top   string `json:"top,omitempty"`
	} `json:"merklebranch,omitempty"`
	EntryBlockKeyMR        string `json:"entryblockkeymr,omitempty"`
	DirectoryBlockKeyMR    string `json:"directoryblockkeymr,omitempty"`
	BitcoinTransactionHash string `json:"bitcointransactionhash,omitempty"`
	BitcoinBlockHash       string `json:"bitcoinblockhash,omitempty"`
}
<commit_msg>changed receipt field "key" to "entryhash"<commit_after>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/json"
)

func GetReceipt(hash string) (*Receipt, error) {
	type receiptResponse struct {
		Receipt *Receipt `json:"receipt"`
	}

	params := hashRequest{Hash: hash}
	req := NewJSON2Request("receipt", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	rec := new(receiptResponse)
	if err := json.Unmarshal(resp.JSONResult(), rec); err != nil {
		return nil, err
	}

	return rec.Receipt, nil
}

type Receipt struct {
	Entry struct {
		Raw       string `json:"raw,omitempty"`
		EntryHash string `json:"entryhash,omitempty"`
		Json      string `json:"json,omitempty"`
	} `json:"entry,omitempty"`
	MerkleBranch []struct {
		Left  string `json:"left,omitempty"`
		Right string `json:"right,omitempty"`
		Top   string `json:"top,omitempty"`
	} `json:"merklebranch,omitempty"`
	EntryBlockKeyMR        string `json:"entryblockkeymr,omitempty"`
	DirectoryBlockKeyMR    string `json:"directoryblockkeymr,omitempty"`
	BitcoinTransactionHash string `json:"bitcointransactionhash,omitempty"`
	BitcoinBlockHash       string `json:"bitcoinblockhash,omitempty"`
}
<|endoftext|>


<commit_before>// defines timestamp types to interface with go-coubhbase and also provides
// functions for set-operations on time-stamps.

package common

import "github.com/couchbase/indexing/secondary/logging"
import "bytes"
import "fmt"

// TsVb is logical clock for a subset of vbuckets.
type TsVb struct {
	Bucket string
	Vbnos  []uint16
	Seqnos []uint64
}

// TsVbFull is logical clock for full set of vbuckets.
type TsVbFull struct {
	Bucket string
	Seqnos []uint64
}

// TsVbuuid is logical clock for full set of vbuckets along with branch value
// and last seen snapshot.
type TsVbuuid struct {
	Bucket    string
	Seqnos    []uint64
	Vbuuids   []uint64
	Snapshots [][2]uint64
	Persisted bool
}

// NewTsVbuuid returns reference to new instance of TsVbuuid.
// `numVbuckets` is same as `maxVbuckets`.
func NewTsVbuuid(bucket string, numVbuckets int) *TsVbuuid {
	return &TsVbuuid{
		Bucket:    bucket,
		Seqnos:    make([]uint64, numVbuckets),
		Vbuuids:   make([]uint64, numVbuckets),
		Snapshots: make([][2]uint64, numVbuckets),
	}
}

// GetVbnos will return the list of all vbnos.
func (ts *TsVbuuid) GetVbnos() []uint16 {
	var vbnos []uint16
	for i := 0; i < len(ts.Vbuuids); i++ {
		if ts.Vbuuids[i] != 0 { //if vbuuid is valid
			vbnos = append(vbnos, uint16(i))
		}
	}
	return vbnos
}

// CompareVbuuids will compare two timestamps for its bucket and vbuuids
func (ts *TsVbuuid) CompareVbuuids(other *TsVbuuid) bool {
	if ts == nil || other == nil {
		return false
	}
	if ts.Bucket != other.Bucket || ts.Len() != other.Len() {
		return false
	}
	for i, vbuuid := range ts.Vbuuids {
		if (vbuuid != other.Vbuuids[i]) ||
			(ts.Snapshots[i][0] != other.Snapshots[i][0]) ||
			(ts.Snapshots[i][1] != other.Snapshots[i][1]) {
			return false
		}
	}
	return true
}

// AsRecent will check whether timestamp `ts` is atleast as recent as
// timestamp `other`.
func (ts *TsVbuuid) AsRecent(other *TsVbuuid) bool {
	if ts == nil || other == nil {
		return false
	}
	if ts.Bucket != other.Bucket {
		return false
	}
	for i, vbuuid := range ts.Vbuuids {
		//skip comparing the vbucket if "other" ts has vbuuid 0
		if other.Vbuuids[i] == 0 {
			continue
		}

		if vbuuid != other.Vbuuids[i] || ts.Seqnos[i] < other.Seqnos[i] {
			return false
		}
	}
	return true
}

// Len return number of entries in the timestamp.
func (ts *TsVbuuid) Len() int {
	length := 0
	for i := 0; i < len(ts.Vbuuids); i++ {
		if ts.Vbuuids[i] != 0 { //if vbuuid is valid
			length++
		}
	}
	return length
}

//Persisted returns the value of persisted flag
func (ts *TsVbuuid) IsPersisted() bool {
	return ts.Persisted
}

//Persisted sets the persisted flag
func (ts *TsVbuuid) SetPersisted(persist bool) {
	ts.Persisted = persist
}

// Copy will return a clone of this timestamp.
func (ts *TsVbuuid) Copy() *TsVbuuid {
	newTs := NewTsVbuuid(ts.Bucket, len(ts.Seqnos))
	copy(newTs.Seqnos, ts.Seqnos)
	copy(newTs.Vbuuids, ts.Vbuuids)
	copy(newTs.Snapshots, ts.Snapshots)
	return newTs
}

// Equal returns whether `ts` and `other` compare equal.
func (ts *TsVbuuid) Equal(other *TsVbuuid) bool {
	if ts != nil && other == nil ||
		ts == nil && other != nil {
		return false
	}

	if ts == nil && other == nil {
		return true
	}

	if len(ts.Seqnos) != len(other.Seqnos) {
		return false
	}

	for i, seqno := range ts.Seqnos {
		if other.Seqnos[i] != seqno {
			return false
		}
	}

	for i, vbuuid := range ts.Vbuuids {
		if other.Vbuuids[i] != vbuuid {
			return false
		}
	}

	for i, sn := range ts.Snapshots {
		if other.Snapshots[i][0] != sn[0] {
			return false
		}

		if other.Snapshots[i][1] != sn[1] {
			return false
		}
	}

	return true
}

// Clone of TsVbuuid
func (ts *TsVbuuid) Clone() *TsVbuuid {

	other := NewTsVbuuid(ts.Bucket, len(ts.Seqnos))
	for i, seqno := range ts.Seqnos {
		other.Seqnos[i] = seqno
	}

	for i, vbuuid := range ts.Vbuuids {
		other.Vbuuids[i] = vbuuid
	}

	for i, sn := range ts.Snapshots {
		other.Snapshots[i][0] = sn[0]
		other.Snapshots[i][1] = sn[1]
	}

	return other
}

// Convert into a human readable format
func (ts *TsVbuuid) String() string {
	var buf bytes.Buffer
	vbnos := ts.GetVbnos()
	buf.WriteString(fmt.Sprintf("bucket: %v, vbuckets: %v -\n",
		ts.Bucket, len(vbnos)))
	buf.WriteString(fmt.Sprintf("    vbno, vbuuid, seqno, snapshot-start, snapshot-end\n"))
	for _, v := range vbnos {
		start, end := ts.Snapshots[v][0], ts.Snapshots[v][1]
		buf.WriteString(fmt.Sprintf("    {%5d %16x %10d %10d %10d}\n",
			v, ts.Vbuuids[v], ts.Seqnos[v], start, end))
	}
	return buf.String()
}

// Convert the difference between two timestamps to human readable format
func (ts *TsVbuuid) Diff(other *TsVbuuid) string {

	var buf bytes.Buffer
	if ts.Equal(other) {
		buf.WriteString("Timestamps are equal\n")
		return buf.String()
	}

	if other == nil {
		buf.WriteString("This timestamp:\n")
		buf.WriteString(ts.String())
		buf.WriteString("Other timestamp is nil\n")
		return buf.String()
	}

	if len(other.Seqnos) != len(ts.Seqnos) {
		logging.Debugf("Two timestamps contain different number of vbuckets\n")
		buf.WriteString("This timestamp:\n")
		buf.WriteString(ts.String())
		buf.WriteString("Other timestamp:\n")
		buf.WriteString(other.String())
		return buf.String()
	}

	for i := range ts.Seqnos {
		if ts.Seqnos[i] != other.Seqnos[i] || ts.Vbuuids[i] != other.Vbuuids[i] ||
			ts.Snapshots[i][0] != other.Snapshots[i][0] || ts.Snapshots[i][1] != other.Snapshots[i][1] {
			buf.WriteString(fmt.Sprintf("This timestamp: bucket %s, vb = %d, vbuuid = %d, seqno = %d, snapshot[0] = %d, snapshot[1] = %d\n",
				ts.Bucket, i, ts.Vbuuids[i], ts.Seqnos[i], ts.Snapshots[0], ts.Snapshots[1]))
			buf.WriteString(fmt.Sprintf("Other timestamp: bucket %s, vb = %d, vbuuid = %d, seqno = %d, snapshot[0] = %d, snapshot[1] = %d\n",
				other.Bucket, i, other.Vbuuids[i], other.Seqnos[i], other.Snapshots[0], other.Snapshots[1]))
		}
	}

	return buf.String()
}
<commit_msg>bugfix: copy persisted flag in Copy()<commit_after>// defines timestamp types to interface with go-coubhbase and also provides
// functions for set-operations on time-stamps.

package common

import "github.com/couchbase/indexing/secondary/logging"
import "bytes"
import "fmt"

// TsVb is logical clock for a subset of vbuckets.
type TsVb struct {
	Bucket string
	Vbnos  []uint16
	Seqnos []uint64
}

// TsVbFull is logical clock for full set of vbuckets.
type TsVbFull struct {
	Bucket string
	Seqnos []uint64
}

// TsVbuuid is logical clock for full set of vbuckets along with branch value
// and last seen snapshot.
type TsVbuuid struct {
	Bucket    string
	Seqnos    []uint64
	Vbuuids   []uint64
	Snapshots [][2]uint64
	Persisted bool
}

// NewTsVbuuid returns reference to new instance of TsVbuuid.
// `numVbuckets` is same as `maxVbuckets`.
func NewTsVbuuid(bucket string, numVbuckets int) *TsVbuuid {
	return &TsVbuuid{
		Bucket:    bucket,
		Seqnos:    make([]uint64, numVbuckets),
		Vbuuids:   make([]uint64, numVbuckets),
		Snapshots: make([][2]uint64, numVbuckets),
	}
}

// GetVbnos will return the list of all vbnos.
func (ts *TsVbuuid) GetVbnos() []uint16 {
	var vbnos []uint16
	for i := 0; i < len(ts.Vbuuids); i++ {
		if ts.Vbuuids[i] != 0 { //if vbuuid is valid
			vbnos = append(vbnos, uint16(i))
		}
	}
	return vbnos
}

// CompareVbuuids will compare two timestamps for its bucket and vbuuids
func (ts *TsVbuuid) CompareVbuuids(other *TsVbuuid) bool {
	if ts == nil || other == nil {
		return false
	}
	if ts.Bucket != other.Bucket || ts.Len() != other.Len() {
		return false
	}
	for i, vbuuid := range ts.Vbuuids {
		if (vbuuid != other.Vbuuids[i]) ||
			(ts.Snapshots[i][0] != other.Snapshots[i][0]) ||
			(ts.Snapshots[i][1] != other.Snapshots[i][1]) {
			return false
		}
	}
	return true
}

// AsRecent will check whether timestamp `ts` is atleast as recent as
// timestamp `other`.
func (ts *TsVbuuid) AsRecent(other *TsVbuuid) bool {
	if ts == nil || other == nil {
		return false
	}
	if ts.Bucket != other.Bucket {
		return false
	}
	for i, vbuuid := range ts.Vbuuids {
		//skip comparing the vbucket if "other" ts has vbuuid 0
		if other.Vbuuids[i] == 0 {
			continue
		}

		if vbuuid != other.Vbuuids[i] || ts.Seqnos[i] < other.Seqnos[i] {
			return false
		}
	}
	return true
}

// Len return number of entries in the timestamp.
func (ts *TsVbuuid) Len() int {
	length := 0
	for i := 0; i < len(ts.Vbuuids); i++ {
		if ts.Vbuuids[i] != 0 { //if vbuuid is valid
			length++
		}
	}
	return length
}

//Persisted returns the value of persisted flag
func (ts *TsVbuuid) IsPersisted() bool {
	return ts.Persisted
}

//Persisted sets the persisted flag
func (ts *TsVbuuid) SetPersisted(persist bool) {
	ts.Persisted = persist
}

// Copy will return a clone of this timestamp.
func (ts *TsVbuuid) Copy() *TsVbuuid {
	newTs := NewTsVbuuid(ts.Bucket, len(ts.Seqnos))
	copy(newTs.Seqnos, ts.Seqnos)
	copy(newTs.Vbuuids, ts.Vbuuids)
	copy(newTs.Snapshots, ts.Snapshots)
	newTs.Persisted = ts.Persisted
	return newTs
}

// Equal returns whether `ts` and `other` compare equal.
func (ts *TsVbuuid) Equal(other *TsVbuuid) bool {
	if ts != nil && other == nil ||
		ts == nil && other != nil {
		return false
	}

	if ts == nil && other == nil {
		return true
	}

	if len(ts.Seqnos) != len(other.Seqnos) {
		return false
	}

	for i, seqno := range ts.Seqnos {
		if other.Seqnos[i] != seqno {
			return false
		}
	}

	for i, vbuuid := range ts.Vbuuids {
		if other.Vbuuids[i] != vbuuid {
			return false
		}
	}

	for i, sn := range ts.Snapshots {
		if other.Snapshots[i][0] != sn[0] {
			return false
		}

		if other.Snapshots[i][1] != sn[1] {
			return false
		}
	}

	return true
}

// Clone of TsVbuuid
func (ts *TsVbuuid) Clone() *TsVbuuid {

	other := NewTsVbuuid(ts.Bucket, len(ts.Seqnos))
	for i, seqno := range ts.Seqnos {
		other.Seqnos[i] = seqno
	}

	for i, vbuuid := range ts.Vbuuids {
		other.Vbuuids[i] = vbuuid
	}

	for i, sn := range ts.Snapshots {
		other.Snapshots[i][0] = sn[0]
		other.Snapshots[i][1] = sn[1]
	}

	return other
}

// Convert into a human readable format
func (ts *TsVbuuid) String() string {
	var buf bytes.Buffer
	vbnos := ts.GetVbnos()
	buf.WriteString(fmt.Sprintf("bucket: %v, vbuckets: %v -\n",
		ts.Bucket, len(vbnos)))
	buf.WriteString(fmt.Sprintf("    vbno, vbuuid, seqno, snapshot-start, snapshot-end\n"))
	for _, v := range vbnos {
		start, end := ts.Snapshots[v][0], ts.Snapshots[v][1]
		buf.WriteString(fmt.Sprintf("    {%5d %16x %10d %10d %10d}\n",
			v, ts.Vbuuids[v], ts.Seqnos[v], start, end))
	}
	return buf.String()
}

// Convert the difference between two timestamps to human readable format
func (ts *TsVbuuid) Diff(other *TsVbuuid) string {

	var buf bytes.Buffer
	if ts.Equal(other) {
		buf.WriteString("Timestamps are equal\n")
		return buf.String()
	}

	if other == nil {
		buf.WriteString("This timestamp:\n")
		buf.WriteString(ts.String())
		buf.WriteString("Other timestamp is nil\n")
		return buf.String()
	}

	if len(other.Seqnos) != len(ts.Seqnos) {
		logging.Debugf("Two timestamps contain different number of vbuckets\n")
		buf.WriteString("This timestamp:\n")
		buf.WriteString(ts.String())
		buf.WriteString("Other timestamp:\n")
		buf.WriteString(other.String())
		return buf.String()
	}

	for i := range ts.Seqnos {
		if ts.Seqnos[i] != other.Seqnos[i] || ts.Vbuuids[i] != other.Vbuuids[i] ||
			ts.Snapshots[i][0] != other.Snapshots[i][0] || ts.Snapshots[i][1] != other.Snapshots[i][1] {
			buf.WriteString(fmt.Sprintf("This timestamp: bucket %s, vb = %d, vbuuid = %d, seqno = %d, snapshot[0] = %d, snapshot[1] = %d\n",
				ts.Bucket, i, ts.Vbuuids[i], ts.Seqnos[i], ts.Snapshots[0], ts.Snapshots[1]))
			buf.WriteString(fmt.Sprintf("Other timestamp: bucket %s, vb = %d, vbuuid = %d, seqno = %d, snapshot[0] = %d, snapshot[1] = %d\n",
				other.Bucket, i, other.Vbuuids[i], other.Seqnos[i], other.Snapshots[0], other.Snapshots[1]))
		}
	}

	return buf.String()
}
<|endoftext|>


<commit_before>package wikifier

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	httpdate "github.com/Songmu/go-httpdate"
	"github.com/cooper/quiki/markdown"
	strip "github.com/grokify/html-strip-tags-go"
)

// Page represents a single page or article, generally associated with a .page file.
// It provides the most basic public interface to parsing with the wikifier engine.
type Page struct {
	Source     string   // source content
	FilePath   string   // Path to the .page file
	VarsOnly   bool     // True if Parse() should only extract variables
	Opt        *PageOpt // page options
	styles     []styleEntry
	parser     *parser            // wikifier parser instance
	main       block              // main block
	Images     map[string][][]int // references to images
	Models     map[string]bool    // references to models
	PageLinks  map[string][]int   // references to other pages
	sectionN   int
	name       string
	headingIDs map[string]int
	Wiki       interface{} // only available during Parse() and HTML()
	markdown   bool        // true if FilePath points to a markdown source
	*variableScope
}

// PageInfo represents metadata associated with a page.
type PageInfo struct {
	Created   *time.Time `json:"created,omitempty"`   // creation time
	Modified  *time.Time `json:"modified,omitempty"`  // modify time
	Draft     bool       `json:"draft,omitempty"`     // true if page is marked as draft
	Generated bool       `json:"generated,omitempty"` // true if page was generated from another source
	Redirect  string     `json:"redirect,omitempty"`  // path page is to redirect to
	FmtTitle  HTML       `json:"fmt_title,omitempty"` // title with formatting tags
	Title     string     `json:"title,omitempty"`     // title without tags
	Author    string     `json:"author,omitempty"`    // author's name
}

// NewPage creates a page given its filepath.
func NewPage(filePath string) *Page {
	myOpt := defaultPageOpt // copy
	return &Page{
		FilePath:      filePath,
		Opt:           &myOpt,
		variableScope: newVariableScope(),
		Images:        make(map[string][][]int),
		Models:        make(map[string]bool),
		PageLinks:     make(map[string][]int),
		headingIDs:    make(map[string]int),
		markdown:      strings.HasSuffix(filePath, ".md"),
	}
}

// NewPageSource creates a page given some source code.
func NewPageSource(source string) *Page {
	p := NewPage("")
	p.Source = source
	return p
}

// NewPageNamed creates a page given its filepath and relative name.
func NewPageNamed(filePath, name string) *Page {
	p := NewPage(filePath)
	p.name = name
	return p
}

// Parse opens the page file and attempts to parse it, returning any errors encountered.
func (p *Page) Parse() error {
	p.parser = newParser()
	p.main = p.parser.block
	defer p.resetParseState()

	// create reader from file path or source code provided
	var reader io.Reader
	if p.Source != "" {
		reader = strings.NewReader(p.Source)
	} else if p.markdown && p.FilePath != "" {
		md, err := ioutil.ReadFile(p.FilePath)
		if err != nil {
			return err
		}
		d := markdown.Run(md)
		fmt.Println(string(d))
		reader = bytes.NewReader(d)
	} else if p.FilePath != "" {
		file, err := os.Open(p.FilePath)
		if err != nil {
			return err
		}
		defer file.Close()
		reader = file
	} else {
		return errors.New("neither Source nor FilePath provided")
	}

	// parse line-by-line
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		if err := p.parser.parseLine(scanner.Bytes(), p); err != nil {
			return err
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	// TODO: check if p.parser.catch != main block
	if p.parser.catch != p.main {
		if p.parser.catch == p.parser.block {
			return fmt.Errorf("%s{} not closed; started at %d", p.parser.block.blockType(), p.parser.block.openPosition())
		}
		return errors.New(string(p.parser.catch.catchType()) + " not closed")
	}

	// parse the blocks, unless we only want vars
	if !p.VarsOnly {
		p.main.parse(p)
	}

	// inject variables set in the page to page opts
	if err := InjectPageOpt(p, p.Opt); err != nil {
		// TODO: position
		return err
	}

	return nil
}

// HTML generates and returns the HTML code for the page.
// The page must be parsed with Parse before attempting this method.
func (p *Page) HTML() HTML {
	// TODO: cache and then recursively destroy elements
	return generateBlock(p.main, p)
}

// Exists is true if the page exists.
func (p *Page) Exists() bool {
	if p.Source != "" {
		return true
	}
	_, err := os.Stat(p.FilePath)
	return err == nil
}

// CacheExists is true if the page cache file exists.
func (p *Page) CacheExists() bool {
	_, err := os.Stat(p.CachePath())
	return err == nil
}

// Name returns the resolved page name, with or without extension.
//
// This DOES take symbolic links into account
// and DOES include the page prefix if applicable.
//
func (p *Page) Name() string {
	dir := pageAbs(p.Opt.Dir.Page)        // /path/to/quiki/wikis/mywiki/pages
	path := filepath.ToSlash(p.Path())    // /path/to/quiki/doc/language.md
	name := strings.TrimPrefix(path, dir) // /path/to/quiki/doc/language.md
	name = strings.TrimPrefix(name, "/")  // path/to/quiki/doc/language.md
	if strings.Index(path, dir) != 0 {    // if path does not start with /path/to/quiki/wikis/mywiki/pages
		return p.RelName() // return language.md
	}
	return name
}

// NameNE returns the resolved page name with No Extension.
func (p *Page) NameNE() string {
	return PageNameNE(p.Name())
}

// Prefix returns the page prefix.
//
// For example, for a page named a/b.page, this is a.
// For a page named a.page, this is an empty string.
//
func (p *Page) Prefix() string {
	dir := strings.TrimSuffix(filepath.ToSlash(filepath.Dir(p.Name())), "/")
	if dir == "." {
		return ""
	}
	return dir
}

// Path returns the absolute path to the page as resolved.
// If the path does not resolve, returns an empty string.
func (p *Page) Path() string {
	return pageAbs(p.RelPath())
}

// RelName returns the unresolved page filename, with or without extension.
// This does NOT take symbolic links into account.
// It is not guaranteed to exist.
func (p *Page) RelName() string {
	if p.name != "" {
		return p.name
	}
	dir := pageAbs(p.Opt.Dir.Page) // /path/to/quiki/wikis/mywiki/pages
	path := p.RelPath()            // doc/parsing.md
	name := strings.TrimPrefix(path, dir)
	name = strings.TrimPrefix(filepath.ToSlash(name), "/")
	if strings.Index(path, dir) == 0 {
		return filepath.Base(p.RelPath())
	}
	return name
}

// RelNameNE returns the unresolved page name with No Extension, relative to
// the page directory option.
// This does NOT take symbolic links into account.
// It is not guaranteed to exist.
func (p *Page) RelNameNE() string {
	return PageNameNE(p.RelName())
}

// RelPath returns the unresolved file path to the page.
// It may be a relative or absolute path.
// It is not guaranteed to exist.
func (p *Page) RelPath() string {
	if p.FilePath != "" {
		return p.FilePath
	}
	return filepath.Join(p.Opt.Dir.Page, p.name)
}

// Redirect returns the location to which the page redirects, if any.
// This may be a relative or absolute URL, suitable for use in a Location header.
func (p *Page) Redirect() string {

	// symbolic link redirect
	if p.IsSymlink() {
		return pageAbs(filepath.Join(p.Opt.Root.Page, p.NameNE()))
	}

	// @page.redirect
	if link, err := p.GetStr("page.redirect"); err != nil {
		// FIXME: is there anyway to produce a warning for wrong variable type?
	} else if ok, target, _, _, _ := p.parseLink(link); ok {
		return target
	}

	return ""
}

// IsSymlink returns true if the page is a symbolic link to another file within
// the page directory. If it is symlinked to somewhere outside the page directory,
// it is treated as a normal page rather than a redirect.
func (p *Page) IsSymlink() bool {
	dirPage := pageAbs(p.Opt.Dir.Page)
	if !strings.HasPrefix(p.Path(), dirPage) {
		return false
	}
	fi, _ := os.Lstat(p.RelPath())
	return fi.Mode()&os.ModeSymlink != 0
}

// Created returns the page creation time.
func (p *Page) Created() time.Time {
	var t time.Time
	// FIXME: maybe produce a warning if this is not in the right format
	created, _ := p.GetStr("page.created")
	if created == "" {
		return t
	}
	if unix, err := strconv.ParseInt(created, 10, 0); err == nil {
		return time.Unix(unix, 0)
	}
	t, _ = httpdate.Str2Time(created, time.UTC)
	return t
}

// Modified returns the page modification time.
func (p *Page) Modified() time.Time {
	fi, _ := os.Lstat(p.Path())
	return fi.ModTime()
}

// CachePath returns the absolute path to the page cache file.
func (p *Page) CachePath() string {
	MakeDir(p.Opt.Dir.Cache+"/page", p.Name())
	return pageAbs(p.Opt.Dir.Cache + "/page/" + p.Name() + ".cache")
}

// CacheModified returns the page cache file time.
func (p *Page) CacheModified() time.Time {
	fi, _ := os.Lstat(p.CachePath())
	return fi.ModTime()
}

// SearchPath returns the absolute path to the page search text file.
func (p *Page) SearchPath() string {
	MakeDir(p.Opt.Dir.Cache+"/page", p.Name())
	return pageAbs(p.Opt.Dir.Cache + "/page/" + p.Name() + ".txt")
}

// Draft returns true if the page is marked as a draft.
func (p *Page) Draft() bool {
	b, _ := p.GetBool("page.draft")
	return b
}

// Generated returns true if the page was auto-generated
// from some other source content.
func (p *Page) Generated() bool {
	b, _ := p.GetBool("page.generated")
	return b
}

// Author returns the page author's name, if any.
func (p *Page) Author() string {
	s, _ := p.GetStr("page.author")
	return s
}

// FmtTitle returns the page title, preserving any possible text formatting.
func (p *Page) FmtTitle() HTML {
	s, _ := p.GetStr("page.title")
	return HTML(s)
}

// Title returns the page title with HTML text formatting tags stripped.
func (p *Page) Title() string {
	return strip.StripTags(string(p.FmtTitle()))
}

// TitleOrName returns the result of Title if available, otherwise that of Name.
func (p *Page) TitleOrName() string {
	if title := p.Title(); title != "" {
		return title
	}
	return p.Name()
}

// Categories returns a list of categories the page belongs to.
func (p *Page) Categories() []string {
	obj, err := p.GetObj("category")
	if err != nil {
		return nil
	}
	catMap, ok := obj.(*Map)
	if !ok {
		return nil
	}
	return catMap.Keys()
}

// Info returns the PageInfo for the page.
func (p *Page) Info() PageInfo {
	info := PageInfo{
		Draft:     p.Draft(),
		Generated: p.Generated(),
		Redirect:  p.Redirect(),
		FmtTitle:  p.FmtTitle(),
		Title:     p.Title(),
		Author:    p.Author(),
	}
	mod, create := p.Modified(), p.Created()
	if !mod.IsZero() {
		info.Modified = &mod
	}
	if !create.IsZero() {
		info.Created = &create
	}
	return info
}

func (p *Page) mainBlock() block {
	return p.main
}

// resets the parser
func (p *Page) resetParseState() {
	// TODO: recursively destroy blocks
	p.parser = nil
}

func pageAbs(path string) string {
	if abs, _ := filepath.Abs(path); abs != "" {
		path = abs
	}
	if followed, _ := filepath.EvalSymlinks(path); followed != "" {
		return followed
	}
	return path
}
<commit_msg>fix cache and text paths<commit_after>package wikifier

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	httpdate "github.com/Songmu/go-httpdate"
	"github.com/cooper/quiki/markdown"
	strip "github.com/grokify/html-strip-tags-go"
)

// Page represents a single page or article, generally associated with a .page file.
// It provides the most basic public interface to parsing with the wikifier engine.
type Page struct {
	Source     string   // source content
	FilePath   string   // Path to the .page file
	VarsOnly   bool     // True if Parse() should only extract variables
	Opt        *PageOpt // page options
	styles     []styleEntry
	parser     *parser            // wikifier parser instance
	main       block              // main block
	Images     map[string][][]int // references to images
	Models     map[string]bool    // references to models
	PageLinks  map[string][]int   // references to other pages
	sectionN   int
	name       string
	headingIDs map[string]int
	Wiki       interface{} // only available during Parse() and HTML()
	markdown   bool        // true if FilePath points to a markdown source
	*variableScope
}

// PageInfo represents metadata associated with a page.
type PageInfo struct {
	Created   *time.Time `json:"created,omitempty"`   // creation time
	Modified  *time.Time `json:"modified,omitempty"`  // modify time
	Draft     bool       `json:"draft,omitempty"`     // true if page is marked as draft
	Generated bool       `json:"generated,omitempty"` // true if page was generated from another source
	Redirect  string     `json:"redirect,omitempty"`  // path page is to redirect to
	FmtTitle  HTML       `json:"fmt_title,omitempty"` // title with formatting tags
	Title     string     `json:"title,omitempty"`     // title without tags
	Author    string     `json:"author,omitempty"`    // author's name
}

// NewPage creates a page given its filepath.
func NewPage(filePath string) *Page {
	myOpt := defaultPageOpt // copy
	return &Page{
		FilePath:      filePath,
		Opt:           &myOpt,
		variableScope: newVariableScope(),
		Images:        make(map[string][][]int),
		Models:        make(map[string]bool),
		PageLinks:     make(map[string][]int),
		headingIDs:    make(map[string]int),
		markdown:      strings.HasSuffix(filePath, ".md"),
	}
}

// NewPageSource creates a page given some source code.
func NewPageSource(source string) *Page {
	p := NewPage("")
	p.Source = source
	return p
}

// NewPageNamed creates a page given its filepath and relative name.
func NewPageNamed(filePath, name string) *Page {
	p := NewPage(filePath)
	p.name = name
	return p
}

// Parse opens the page file and attempts to parse it, returning any errors encountered.
func (p *Page) Parse() error {
	p.parser = newParser()
	p.main = p.parser.block
	defer p.resetParseState()

	// create reader from file path or source code provided
	var reader io.Reader
	if p.Source != "" {
		reader = strings.NewReader(p.Source)
	} else if p.markdown && p.FilePath != "" {
		md, err := ioutil.ReadFile(p.FilePath)
		if err != nil {
			return err
		}
		d := markdown.Run(md)
		fmt.Println(string(d))
		reader = bytes.NewReader(d)
	} else if p.FilePath != "" {
		file, err := os.Open(p.FilePath)
		if err != nil {
			return err
		}
		defer file.Close()
		reader = file
	} else {
		return errors.New("neither Source nor FilePath provided")
	}

	// parse line-by-line
	scanner := bufio.NewScanner(reader)
	for scanner.Scan() {
		if err := p.parser.parseLine(scanner.Bytes(), p); err != nil {
			return err
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	// TODO: check if p.parser.catch != main block
	if p.parser.catch != p.main {
		if p.parser.catch == p.parser.block {
			return fmt.Errorf("%s{} not closed; started at %d", p.parser.block.blockType(), p.parser.block.openPosition())
		}
		return errors.New(string(p.parser.catch.catchType()) + " not closed")
	}

	// parse the blocks, unless we only want vars
	if !p.VarsOnly {
		p.main.parse(p)
	}

	// inject variables set in the page to page opts
	if err := InjectPageOpt(p, p.Opt); err != nil {
		// TODO: position
		return err
	}

	return nil
}

// HTML generates and returns the HTML code for the page.
// The page must be parsed with Parse before attempting this method.
func (p *Page) HTML() HTML {
	// TODO: cache and then recursively destroy elements
	return generateBlock(p.main, p)
}

// Exists is true if the page exists.
func (p *Page) Exists() bool {
	if p.Source != "" {
		return true
	}
	_, err := os.Stat(p.FilePath)
	return err == nil
}

// CacheExists is true if the page cache file exists.
func (p *Page) CacheExists() bool {
	_, err := os.Stat(p.CachePath())
	return err == nil
}

// Name returns the resolved page name, with or without extension.
//
// This DOES take symbolic links into account
// and DOES include the page prefix if applicable.
//
func (p *Page) Name() string {
	dir := pageAbs(p.Opt.Dir.Page)        // /path/to/quiki/wikis/mywiki/pages
	path := filepath.ToSlash(p.Path())    // /path/to/quiki/doc/language.md
	name := strings.TrimPrefix(path, dir) // /path/to/quiki/doc/language.md
	name = strings.TrimPrefix(name, "/")  // path/to/quiki/doc/language.md
	if strings.Index(path, dir) != 0 {    // if path does not start with /path/to/quiki/wikis/mywiki/pages
		return p.RelName() // return language.md
	}
	return name
}

// NameNE returns the resolved page name with No Extension.
func (p *Page) NameNE() string {
	return PageNameNE(p.Name())
}

// Prefix returns the page prefix.
//
// For example, for a page named a/b.page, this is a.
// For a page named a.page, this is an empty string.
//
func (p *Page) Prefix() string {
	dir := strings.TrimSuffix(filepath.ToSlash(filepath.Dir(p.Name())), "/")
	if dir == "." {
		return ""
	}
	return dir
}

// Path returns the absolute path to the page as resolved.
// If the path does not resolve, returns an empty string.
func (p *Page) Path() string {
	return pageAbs(p.RelPath())
}

// RelName returns the unresolved page filename, with or without extension.
// This does NOT take symbolic links into account.
// It is not guaranteed to exist.
func (p *Page) RelName() string {
	if p.name != "" {
		return p.name
	}
	dir := pageAbs(p.Opt.Dir.Page) // /path/to/quiki/wikis/mywiki/pages
	path := p.RelPath()            // doc/parsing.md
	name := strings.TrimPrefix(path, dir)
	name = strings.TrimPrefix(filepath.ToSlash(name), "/")
	if strings.Index(path, dir) == 0 {
		return filepath.Base(p.RelPath())
	}
	return name
}

// RelNameNE returns the unresolved page name with No Extension, relative to
// the page directory option.
// This does NOT take symbolic links into account.
// It is not guaranteed to exist.
func (p *Page) RelNameNE() string {
	return PageNameNE(p.RelName())
}

// RelPath returns the unresolved file path to the page.
// It may be a relative or absolute path.
// It is not guaranteed to exist.
func (p *Page) RelPath() string {
	if p.FilePath != "" {
		return p.FilePath
	}
	return filepath.Join(p.Opt.Dir.Page, p.name)
}

// Redirect returns the location to which the page redirects, if any.
// This may be a relative or absolute URL, suitable for use in a Location header.
func (p *Page) Redirect() string {

	// symbolic link redirect
	if p.IsSymlink() {
		return pageAbs(filepath.Join(p.Opt.Root.Page, p.NameNE()))
	}

	// @page.redirect
	if link, err := p.GetStr("page.redirect"); err != nil {
		// FIXME: is there anyway to produce a warning for wrong variable type?
	} else if ok, target, _, _, _ := p.parseLink(link); ok {
		return target
	}

	return ""
}

// IsSymlink returns true if the page is a symbolic link to another file within
// the page directory. If it is symlinked to somewhere outside the page directory,
// it is treated as a normal page rather than a redirect.
func (p *Page) IsSymlink() bool {
	dirPage := pageAbs(p.Opt.Dir.Page)
	if !strings.HasPrefix(p.Path(), dirPage) {
		return false
	}
	fi, _ := os.Lstat(p.RelPath())
	return fi.Mode()&os.ModeSymlink != 0
}

// Created returns the page creation time.
func (p *Page) Created() time.Time {
	var t time.Time
	// FIXME: maybe produce a warning if this is not in the right format
	created, _ := p.GetStr("page.created")
	if created == "" {
		return t
	}
	if unix, err := strconv.ParseInt(created, 10, 0); err == nil {
		return time.Unix(unix, 0)
	}
	t, _ = httpdate.Str2Time(created, time.UTC)
	return t
}

// Modified returns the page modification time.
func (p *Page) Modified() time.Time {
	fi, _ := os.Lstat(p.Path())
	return fi.ModTime()
}

// CachePath returns the absolute path to the page cache file.
func (p *Page) CachePath() string {
	osName := filepath.FromSlash(p.Name()) + ".cache" // os-specific cache name
	MakeDir(filepath.Join(p.Opt.Dir.Cache, "page"), osName)
	return pageAbs(filepath.Join(p.Opt.Dir.Cache, "page", osName))
}

// CacheModified returns the page cache file time.
func (p *Page) CacheModified() time.Time {
	fi, _ := os.Lstat(p.CachePath())
	return fi.ModTime()
}

// SearchPath returns the absolute path to the page search text file.
func (p *Page) SearchPath() string {
	osName := filepath.FromSlash(p.Name()) + ".txt" // os-specific text file name
	MakeDir(filepath.Join(p.Opt.Dir.Cache, "page"), osName)
	return pageAbs(filepath.Join(p.Opt.Dir.Cache, "page", osName))
}

// Draft returns true if the page is marked as a draft.
func (p *Page) Draft() bool {
	b, _ := p.GetBool("page.draft")
	return b
}

// Generated returns true if the page was auto-generated
// from some other source content.
func (p *Page) Generated() bool {
	b, _ := p.GetBool("page.generated")
	return b
}

// Author returns the page author's name, if any.
func (p *Page) Author() string {
	s, _ := p.GetStr("page.author")
	return s
}

// FmtTitle returns the page title, preserving any possible text formatting.
func (p *Page) FmtTitle() HTML {
	s, _ := p.GetStr("page.title")
	return HTML(s)
}

// Title returns the page title with HTML text formatting tags stripped.
func (p *Page) Title() string {
	return strip.StripTags(string(p.FmtTitle()))
}

// TitleOrName returns the result of Title if available, otherwise that of Name.
func (p *Page) TitleOrName() string {
	if title := p.Title(); title != "" {
		return title
	}
	return p.Name()
}

// Categories returns a list of categories the page belongs to.
func (p *Page) Categories() []string {
	obj, err := p.GetObj("category")
	if err != nil {
		return nil
	}
	catMap, ok := obj.(*Map)
	if !ok {
		return nil
	}
	return catMap.Keys()
}

// Info returns the PageInfo for the page.
func (p *Page) Info() PageInfo {
	info := PageInfo{
		Draft:     p.Draft(),
		Generated: p.Generated(),
		Redirect:  p.Redirect(),
		FmtTitle:  p.FmtTitle(),
		Title:     p.Title(),
		Author:    p.Author(),
	}
	mod, create := p.Modified(), p.Created()
	if !mod.IsZero() {
		info.Modified = &mod
	}
	if !create.IsZero() {
		info.Created = &create
	}
	return info
}

func (p *Page) mainBlock() block {
	return p.main
}

// resets the parser
func (p *Page) resetParseState() {
	// TODO: recursively destroy blocks
	p.parser = nil
}

func pageAbs(path string) string {
	if abs, _ := filepath.Abs(path); abs != "" {
		path = abs
	}
	if followed, _ := filepath.EvalSymlinks(path); followed != "" {
		return followed
	}
	return path
}
<|endoftext|>