<commit_before>// A facebook graph api client in go.
// https://github.com/huandu/facebook/
//
// Copyright 2012 - 2015, Huan Du
// Licensed under the MIT license
// https://github.com/huandu/facebook/blob/master/LICENSE

package facebook

import (
	"encoding/json"
	"reflect"
	"regexp"
	"time"
)

// Facebook graph api methods.
const (
	GET    Method = "GET"
	POST   Method = "POST"
	DELETE Method = "DELETE"
	PUT    Method = "PUT"
)

const (
	ERROR_CODE_UNKNOWN = -1 // unknown facebook graph api error code.

	_MIME_FORM_URLENCODED = "application/x-www-form-urlencoded"
	_MIME_FORM_DATA       = "multipart/form-data"
)

// Graph API debug mode values.
const (
	DEBUG_OFF DebugMode = "" // turn off debug.

	DEBUG_ALL     DebugMode = "all"
	DEBUG_INFO    DebugMode = "info"
	DEBUG_WARNING DebugMode = "warning"
)

const (
	debugInfoKey   = "__debug__"
	debugProtoKey  = "__proto__"
	debugHeaderKey = "__header__"

	facebookApiVersionHeader = "facebook-api-version"
	facebookDebugHeader      = "x-fb-debug"
	facebookRevHeader        = "x-fb-rev"
)

var (
	// Maps aliases to Facebook domains.
	// Copied from Facebook PHP SDK.
	domainMap = map[string]string{
		"api":         "https://api.facebook.com/",
		"api_video":   "https://api-video.facebook.com/",
		"api_read":    "https://api-read.facebook.com/",
		"graph":       "https://graph.facebook.com/",
		"graph_video": "https://graph-video.facebook.com/",
		"www":         "https://www.facebook.com/",
	}

	// checks whether it's a video post.
	regexpIsVideoPost = regexp.MustCompile("/videos")

	// default facebook session.
	defaultSession = &Session{}

	typeOfPointerToBinaryData = reflect.TypeOf(&binaryData{})
	typeOfPointerToBinaryFile = reflect.TypeOf(&binaryFile{})
	typeOfJSONNumber          = reflect.TypeOf(json.Number(""))
	typeOfTime                = reflect.TypeOf(time.Time{})
	typeOfUnmarshaler         = reflect.TypeOf((*json.Unmarshaler)(nil)).Elem()

	facebookSuccessJsonBytes = []byte("true")
)
<commit_msg>add leading \ and trailing $<commit_after>// A facebook graph api client in go.
// https://github.com/huandu/facebook/
//
// Copyright 2012 - 2015, Huan Du
// Licensed under the MIT license
// https://github.com/huandu/facebook/blob/master/LICENSE

package facebook

import (
	"encoding/json"
	"reflect"
	"regexp"
	"time"
)

// Facebook graph api methods.
const (
	GET    Method = "GET"
	POST   Method = "POST"
	DELETE Method = "DELETE"
	PUT    Method = "PUT"
)

const (
	ERROR_CODE_UNKNOWN = -1 // unknown facebook graph api error code.

	_MIME_FORM_URLENCODED = "application/x-www-form-urlencoded"
	_MIME_FORM_DATA       = "multipart/form-data"
)

// Graph API debug mode values.
const (
	DEBUG_OFF DebugMode = "" // turn off debug.

	DEBUG_ALL     DebugMode = "all"
	DEBUG_INFO    DebugMode = "info"
	DEBUG_WARNING DebugMode = "warning"
)

const (
	debugInfoKey   = "__debug__"
	debugProtoKey  = "__proto__"
	debugHeaderKey = "__header__"

	facebookApiVersionHeader = "facebook-api-version"
	facebookDebugHeader      = "x-fb-debug"
	facebookRevHeader        = "x-fb-rev"
)

var (
	// Maps aliases to Facebook domains.
	// Copied from Facebook PHP SDK.
	domainMap = map[string]string{
		"api":         "https://api.facebook.com/",
		"api_video":   "https://api-video.facebook.com/",
		"api_read":    "https://api-read.facebook.com/",
		"graph":       "https://graph.facebook.com/",
		"graph_video": "https://graph-video.facebook.com/",
		"www":         "https://www.facebook.com/",
	}

	// checks whether it's a video post.
	regexpIsVideoPost = regexp.MustCompile("\\/videos$")

	// default facebook session.
	defaultSession = &Session{}

	typeOfPointerToBinaryData = reflect.TypeOf(&binaryData{})
	typeOfPointerToBinaryFile = reflect.TypeOf(&binaryFile{})
	typeOfJSONNumber          = reflect.TypeOf(json.Number(""))
	typeOfTime                = reflect.TypeOf(time.Time{})
	typeOfUnmarshaler         = reflect.TypeOf((*json.Unmarshaler)(nil)).Elem()

	facebookSuccessJsonBytes = []byte("true")
)
<|endoftext|>


<commit_before>package models

import "errors"

var (
	ErrMessageAlreadyInTheChannel = errors.New("message is already in the channel")
	ErrIdIsNotSet                 = errors.New("Id is not set")
	ErrAccountIdIsNotSet          = errors.New("account id is not set")
	ErrOldIdIsNotSet              = errors.New("old id is not set")
	ErrNickIsNotSet               = errors.New("nick is not set")
	ErrGuestsAreNotAllowed        = errors.New("guests are not allowed")

	ErrMessageIdIsNotSet       = errors.New("message id is not set")
	ErrMessageIsNotSet         = errors.New("message is not set")
	ErrParentMessageIsNotSet   = errors.New("parent message is not set")
	ErrParentMessageIdIsNotSet = errors.New("parent message id is not set")
	ErrCreatorIdIsNotSet       = errors.New("creator id is not set")

	ErrChannelIsNotSet                = errors.New("channel is not set")
	ErrChannelIdIsNotSet              = errors.New("channel id is not set")
	ErrChannelContainerIsNotSet       = errors.New("channel container is not set")
	ErCouldntFindAccountIdFromContent = errors.New("couldnt find account id from content")
	ErrAccountIsAlreadyInTheChannel   = errors.New("account is already in the channel")

	ErrChannelParticipantIsNotSet             = errors.New("channel participant is not set")
	ErrCannotAddNewParticipantToPinnedChannel = errors.New("you can not add any participants to pinned activity channel")

	ErrChannelMessageIdIsNotSet        = errors.New("channel message id is not set")
	ErrChannelMessageUpdatedNotAllowed = errors.New("join/leave message update is not allowed")

	ErrNameIsNotSet       = errors.New("name is not set")
	ErrGroupNameIsNotSet  = errors.New("group name is not set")
	ErrLastSeenAtIsNotSet = errors.New("lastSeenAt is not set")
	ErrAddedAtIsNotSet    = errors.New("addedAt is not set")

	ErrRecipientsNotDefined = errors.New("recipients are not defined")
	ErrCannotOpenChannel    = errors.New("you can not open the channel")
	ErrSlugIsNotSet         = errors.New("slug is not set")

	ErrChannelOrMessageIdIsNotSet = errors.New("channelId/messageId is not set")

	ErrNotLoggedIn = errors.New("not logged in")

	ErrAccessDenied = errors.New("access denied")

	ErrRoleNotSet          = errors.New("role not set")
	ErrAccountNotFound     = errors.New("account not found")
	ErrChannelNotFound     = errors.New("channel not found")
	ErrParticipantNotFound = errors.New("participant not found")
)
<commit_msg>Socialapi: created errors<commit_after>package models

import "errors"

var (
	ErrMessageAlreadyInTheChannel = errors.New("message is already in the channel")
	ErrIdIsNotSet                 = errors.New("Id is not set")
	ErrAccountIdIsNotSet          = errors.New("account id is not set")
	ErrOldIdIsNotSet              = errors.New("old id is not set")
	ErrNickIsNotSet               = errors.New("nick is not set")
	ErrGuestsAreNotAllowed        = errors.New("guests are not allowed")

	ErrMessageIdIsNotSet       = errors.New("message id is not set")
	ErrMessageIsNotSet         = errors.New("message is not set")
	ErrParentMessageIsNotSet   = errors.New("parent message is not set")
	ErrParentMessageIdIsNotSet = errors.New("parent message id is not set")
	ErrCreatorIdIsNotSet       = errors.New("creator id is not set")

	ErrChannelIsNotSet                = errors.New("channel is not set")
	ErrChannelIdIsNotSet              = errors.New("channel id is not set")
	ErrChannelContainerIsNotSet       = errors.New("channel container is not set")
	ErCouldntFindAccountIdFromContent = errors.New("couldnt find account id from content")
	ErrAccountIsAlreadyInTheChannel   = errors.New("account is already in the channel")

	ErrChannelParticipantIsNotSet             = errors.New("channel participant is not set")
	ErrCannotAddNewParticipantToPinnedChannel = errors.New("you can not add any participants to pinned activity channel")

	ErrChannelMessageIdIsNotSet        = errors.New("channel message id is not set")
	ErrChannelMessageUpdatedNotAllowed = errors.New("join/leave message update is not allowed")

	ErrNameIsNotSet       = errors.New("name is not set")
	ErrGroupNameIsNotSet  = errors.New("group name is not set")
	ErrLastSeenAtIsNotSet = errors.New("lastSeenAt is not set")
	ErrAddedAtIsNotSet    = errors.New("addedAt is not set")

	ErrRecipientsNotDefined = errors.New("recipients are not defined")
	ErrCannotOpenChannel    = errors.New("you can not open the channel")
	ErrSlugIsNotSet         = errors.New("slug is not set")

	ErrChannelOrMessageIdIsNotSet = errors.New("channelId/messageId is not set")

	ErrNotLoggedIn = errors.New("not logged in")

	ErrAccessDenied = errors.New("access denied")

	ErrRoleNotSet          = errors.New("role not set")
	ErrAccountNotFound     = errors.New("account not found")
	ErrChannelNotFound     = errors.New("channel not found")
	ErrParticipantNotFound = errors.New("participant not found")

	// moderation
	ErrLeafIsNotSet     = errors.New("leaf channel is not set")
	ErrRootIsNotSet     = errors.New("root channel is not set")
	ErrChannelHasLeaves = errors.New("channel has leaves")
)
<|endoftext|>


<commit_before>package models

import "errors"

var (
	ErrMessageAlreadyInTheChannel = errors.New("message is already in the channel")
	ErrIdIsNotSet                 = errors.New("Id is not set")
	ErrAccountIdIsNotSet          = errors.New("account id is not set")
	ErrOldIdIsNotSet              = errors.New("old id is not set")
	ErrNickIsNotSet               = errors.New("nick is not set")
	ErrGuestsAreNotAllowed        = errors.New("guests are not allowed")

	ErrMessageIdIsNotSet       = errors.New("message id is not set")
	ErrMessageIsNotSet         = errors.New("message is not set")
	ErrParentMessageIsNotSet   = errors.New("parent message is not set")
	ErrParentMessageIdIsNotSet = errors.New("parent message id is not set")
	ErrCreatorIdIsNotSet       = errors.New("creator id is not set")

	ErrChannelIsNotSet                = errors.New("channel is not set")
	ErrChannelIdIsNotSet              = errors.New("channel id is not set")
	ErrChannelContainerIsNotSet       = errors.New("channel container is not set")
	ErCouldntFindAccountIdFromContent = errors.New("couldnt find account id from content")
	ErrAccountIsAlreadyInTheChannel   = errors.New("account is already in the channel")

	ErrChannelParticipantIsNotSet             = errors.New("channel participant is not set")
	ErrCannotAddNewParticipantToPinnedChannel = errors.New("you can not add any participants to pinned activity channel")

	ErrChannelMessageIdIsNotSet        = errors.New("channel message id is not set")
	ErrChannelMessageUpdatedNotAllowed = errors.New("join/leave message update is not allowed")

	ErrNameIsNotSet       = errors.New("name is not set")
	ErrGroupNameIsNotSet  = errors.New("group name is not set")
	ErrLastSeenAtIsNotSet = errors.New("lastSeenAt is not set")
	ErrAddedAtIsNotSet    = errors.New("addedAt is not set")

	ErrRecipientsNotDefined = errors.New("recipients are not defined")
	ErrCannotOpenChannel    = errors.New("you can not open the channel")
	ErrSlugIsNotSet         = errors.New("slug is not set")

	ErrChannelOrMessageIdIsNotSet = errors.New("channelId/messageId is not set")

	ErrNotLoggedIn = errors.New("not logged in")

	ErrAccessDenied = errors.New("access denied")

	ErrRoleNotSet          = errors.New("role not set")
	ErrAccountNotFound     = errors.New("account not found")
	ErrChannelNotFound     = errors.New("channel not found")
	ErrParticipantNotFound = errors.New("participant not found")

	// moderation
	ErrLeafIsNotSet     = errors.New("leaf channel is not set")
	ErrRootIsNotSet     = errors.New("root channel is not set")
	ErrChannelHasLeaves = errors.New("channel has leaves")
)
<commit_msg>Socialapi: define a new error variable for channel link constraint<commit_after>package models

import "errors"

var (
	ErrMessageAlreadyInTheChannel = errors.New("message is already in the channel")
	ErrIdIsNotSet                 = errors.New("Id is not set")
	ErrAccountIdIsNotSet          = errors.New("account id is not set")
	ErrOldIdIsNotSet              = errors.New("old id is not set")
	ErrNickIsNotSet               = errors.New("nick is not set")
	ErrGuestsAreNotAllowed        = errors.New("guests are not allowed")

	ErrMessageIdIsNotSet       = errors.New("message id is not set")
	ErrMessageIsNotSet         = errors.New("message is not set")
	ErrParentMessageIsNotSet   = errors.New("parent message is not set")
	ErrParentMessageIdIsNotSet = errors.New("parent message id is not set")
	ErrCreatorIdIsNotSet       = errors.New("creator id is not set")

	ErrChannelIsNotSet                = errors.New("channel is not set")
	ErrChannelIdIsNotSet              = errors.New("channel id is not set")
	ErrChannelContainerIsNotSet       = errors.New("channel container is not set")
	ErCouldntFindAccountIdFromContent = errors.New("couldnt find account id from content")
	ErrAccountIsAlreadyInTheChannel   = errors.New("account is already in the channel")

	ErrChannelParticipantIsNotSet             = errors.New("channel participant is not set")
	ErrCannotAddNewParticipantToPinnedChannel = errors.New("you can not add any participants to pinned activity channel")

	ErrChannelMessageIdIsNotSet        = errors.New("channel message id is not set")
	ErrChannelMessageUpdatedNotAllowed = errors.New("join/leave message update is not allowed")

	ErrNameIsNotSet       = errors.New("name is not set")
	ErrGroupNameIsNotSet  = errors.New("group name is not set")
	ErrLastSeenAtIsNotSet = errors.New("lastSeenAt is not set")
	ErrAddedAtIsNotSet    = errors.New("addedAt is not set")

	ErrRecipientsNotDefined = errors.New("recipients are not defined")
	ErrCannotOpenChannel    = errors.New("you can not open the channel")
	ErrSlugIsNotSet         = errors.New("slug is not set")

	ErrChannelOrMessageIdIsNotSet = errors.New("channelId/messageId is not set")

	ErrNotLoggedIn = errors.New("not logged in")

	ErrAccessDenied = errors.New("access denied")

	ErrRoleNotSet          = errors.New("role not set")
	ErrAccountNotFound     = errors.New("account not found")
	ErrChannelNotFound     = errors.New("channel not found")
	ErrParticipantNotFound = errors.New("participant not found")

	// moderation
	ErrLeafIsNotSet     = errors.New("leaf channel is not set")
	ErrRootIsNotSet     = errors.New("root channel is not set")
	ErrChannelHasLeaves = errors.New("channel has leaves")
	ErrGroupsAreNotSame = errors.New("groups are not same")
)
<|endoftext|>


<commit_before>package tree

// Define a function Build(records []Record) (*Node, error)
// where Record is a struct containing int fields ID and Parent
// and Node is a struct containing int field ID and []*Node field Children.

type Record struct {
	ID     int
	Parent int
}

type Node struct {
	ID       int
	Children []*Node
}

func Build(records []Record) (*Node, error) {
	if len(records) == 0 {
		return nil, nil
	}
	nodes := map[int]*Node{}
	nodes[0] = &Node{ID: 0}

	for _, record := range records {
		// Initialize node if it doesn't exist
		if _, ok := nodes[record.ID]; !ok {
			nodes[record.ID] = &Node{ID: record.ID}
		}
		// Initialize parent if it doesn't exist
		if _, ok := nodes[record.Parent]; !ok {
			nodes[record.Parent] = &Node{ID: record.Parent}
		}
		// Populate children for parent
		children := nodes[record.Parent].Children
		node := nodes[record.ID]
		if !contains(children, node) {
			children = append(children, node)
			nodes[record.Parent].Children = children
		}
	}
	return nodes[0], nil
}

func contains(slice []*Node, node *Node) bool {
	for _, v := range slice {
		if v == node {
			return true
		}
	}
	return false
}
<commit_msg>Still recursion bug<commit_after>package tree

import "reflect"

// Define a function Build(records []Record) (*Node, error)
// where Record is a struct containing int fields ID and Parent
// and Node is a struct containing int field ID and []*Node field Children.

type Record struct {
	ID     int
	Parent int
}

type Node struct {
	ID       int
	Children []*Node
}

func Build(records []Record) (*Node, error) {
	if len(records) == 0 {
		return nil, nil
	}
	nodes := map[int]*Node{}
	nodes[0] = &Node{ID: 0}

	for _, record := range records {
		// Initialize node if it doesn't exist
		if _, ok := nodes[record.ID]; !ok {
			nodes[record.ID] = &Node{ID: record.ID}
		}
		// Initialize parent if it doesn't exist
		if _, ok := nodes[record.Parent]; !ok {
			nodes[record.Parent] = &Node{ID: record.Parent}
		}
		// Populate children for parent
		children := nodes[record.Parent].Children
		node := nodes[record.ID]
		if !contains(children, node) {
			children = append(children, node)
			nodes[record.Parent].Children = children
		}
	}
	return nodes[0], nil
}

func contains(slice []*Node, node *Node) bool {
	for _, v := range slice {
		if reflect.DeepEqual(v, node) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/andrew-d/go-termutil"
	"github.com/codegangsta/cli"
	"github.com/garyburd/redigo/redis"
)

//Pop all values from the given redis list and write the values to stdout
func readAll(list string, client redis.Conn) {
	for {
		read(list, client, 64)
	}
}

func read(list string, conn redis.Conn, count int) {
	for i := 0; i < count; i++ {
		conn.Send("LPOP", list)
	}
	conn.Flush()
	for i := 0; i < count; i++ {
		value, err := redis.String(conn.Receive())

		if err != nil {
			os.Exit(0)
		}
		if value != "" {
			fmt.Println(value)
		}
	}
}

func writeBatch(list string, conn redis.Conn, values []string) {
	conn.Send("MULTI")
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value != "" {
			conn.Send("LPUSH", list, value)
		}
	}
	_, err := redis.Values(conn.Do("EXEC"))
	if err != nil {
		log.Fatalf("Could not LPUSH: %s \"%s\"\n", list, err.Error())
	}
}

//Push all values from stdin to a given redis list
func write(list string, conn redis.Conn) {
	var values []string
	count := 0
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		value := scanner.Text()
		values = append(values, value)
		count++

		if count >= 64 {
			writeBatch(list, conn, values)
			values = make([]string, 0)
			count = 0
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "Reading standard input:", err)
	}
	//writeBatch(list, conn, values)
}

func main() {
	app := cli.NewApp()
	app.Name = "redis-pipe"
	app.Usage = "connect stdin with LPUSH and LPOP with stout."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "Redis host",
			EnvVar: "REDIS_HOST",
		},
		cli.IntFlag{
			Name:   "port",
			Value:  6379,
			Usage:  "Redis port",
			EnvVar: "REDIS_PORT",
		},
		cli.IntFlag{
			Name:  "count",
			Value: -1,
			Usage: "Redis Stop reading from list after count",
		},
	}

	app.Action = func(c *cli.Context) {
		list := c.Args().First()
		if list == "" {
			fmt.Println("Please provide name of the list")
			os.Exit(1)
		}

		conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", c.String("host"), c.Int("port")))
		if err != nil {
			log.Fatalf("Connecting to Redis failed: %s\n", err.Error())
		}

		if termutil.Isatty(os.Stdin.Fd()) {
			count := c.Int("count")
			if count > 0 {
				read(list, conn, count)
			} else {
				readAll(list, conn)
			}
		} else {
			write(list, conn)
		}
		conn.Close()
	}

	app.Run(os.Args)
}
<commit_msg>Forgot final writeBatch call<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/andrew-d/go-termutil"
	"github.com/codegangsta/cli"
	"github.com/garyburd/redigo/redis"
)

//Pop all values from the given redis list and write the values to stdout
func readAll(list string, client redis.Conn) {
	for {
		read(list, client, 64)
	}
}

func read(list string, conn redis.Conn, count int) {
	for i := 0; i < count; i++ {
		conn.Send("LPOP", list)
	}
	conn.Flush()
	for i := 0; i < count; i++ {
		value, err := redis.String(conn.Receive())

		if err != nil {
			os.Exit(0)
		}
		if value != "" {
			fmt.Println(value)
		}
	}
}

func writeBatch(list string, conn redis.Conn, values []string) {
	conn.Send("MULTI")
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value != "" {
			conn.Send("LPUSH", list, value)
		}
	}
	_, err := redis.Values(conn.Do("EXEC"))
	if err != nil {
		log.Fatalf("Could not LPUSH: %s \"%s\"\n", list, err.Error())
	}
}

//Push all values from stdin to a given redis list
func write(list string, conn redis.Conn) {
	var values []string
	count := 0
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		value := scanner.Text()
		values = append(values, value)
		count++

		if count >= 64 {
			writeBatch(list, conn, values)
			values = make([]string, 0)
			count = 0
		}
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "Reading standard input:", err)
	}
	writeBatch(list, conn, values)
}

func main() {
	app := cli.NewApp()
	app.Name = "redis-pipe"
	app.Usage = "connect stdin with LPUSH and LPOP with stout."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "Redis host",
			EnvVar: "REDIS_HOST",
		},
		cli.IntFlag{
			Name:   "port",
			Value:  6379,
			Usage:  "Redis port",
			EnvVar: "REDIS_PORT",
		},
		cli.IntFlag{
			Name:  "count",
			Value: -1,
			Usage: "Redis Stop reading from list after count",
		},
	}

	app.Action = func(c *cli.Context) {
		list := c.Args().First()
		if list == "" {
			fmt.Println("Please provide name of the list")
			os.Exit(1)
		}

		conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", c.String("host"), c.Int("port")))
		if err != nil {
			log.Fatalf("Connecting to Redis failed: %s\n", err.Error())
		}

		if termutil.Isatty(os.Stdin.Fd()) {
			count := c.Int("count")
			if count > 0 {
				read(list, conn, count)
			} else {
				readAll(list, conn)
			}
		} else {
			write(list, conn)
		}
		conn.Close()
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package networker_test

import (
	"fmt"
	"io/ioutil"

	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	"github.com/juju/utils/set"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/juju/testing"
	"github.com/juju/juju/state"
	"github.com/juju/juju/state/api"
	apinetworker "github.com/juju/juju/state/api/networker"
	"github.com/juju/juju/state/api/params"
	"github.com/juju/juju/worker"
	"github.com/juju/juju/worker/networker"
)

type networkerSuite struct {
	testing.JujuConnSuite
	cfg *config.Config

	networks      []state.NetworkInfo
	machine       *state.Machine
	machineIfaces []state.NetworkInterfaceInfo

	st             *api.State
	networkerState *apinetworker.State

	networkDir          string
	interfacesFile      string
	executorCommands    []string
	executorHasFinished chan struct{}
}

var _ = gc.Suite(&networkerSuite{})

// Create several networks.
func (s *networkerSuite) setUpNetworks(c *gc.C) {
	s.networks = []state.NetworkInfo{{
		Name:       "net1",
		ProviderId: "net1",
		CIDR:       "0.1.2.0/24",
		VLANTag:    0,
	}, {
		Name:       "vlan42",
		ProviderId: "vlan42",
		CIDR:       "0.2.2.0/24",
		VLANTag:    42,
	}, {
		Name:       "vlan69",
		ProviderId: "vlan69",
		CIDR:       "0.3.2.0/24",
		VLANTag:    69,
	}, {
		Name:       "vlan123",
		ProviderId: "vlan123",
		CIDR:       "0.4.2.0/24",
		VLANTag:    123,
	}, {
		Name:       "net2",
		ProviderId: "net2",
		CIDR:       "0.5.2.0/24",
		VLANTag:    0,
	}}
}

// Create a machine and login to it.
func (s *networkerSuite) setUpMachine(c *gc.C) {
	var err error
	s.machine, err = s.State.AddMachine("quantal", state.JobHostUnits)
	c.Assert(err, gc.IsNil)
	password, err := utils.RandomPassword()
	c.Assert(err, gc.IsNil)
	err = s.machine.SetPassword(password)
	c.Assert(err, gc.IsNil)
	hwChars := instance.MustParseHardware("cpu-cores=123", "mem=4G")
	s.machineIfaces = []state.NetworkInterfaceInfo{{
		MACAddress:    "aa:bb:cc:dd:ee:f0",
		InterfaceName: "eth0",
		NetworkName:   "net1",
		IsVirtual:     false,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f1",
		InterfaceName: "eth1",
		NetworkName:   "net1",
		IsVirtual:     false,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f1",
		InterfaceName: "eth1.42",
		NetworkName:   "vlan42",
		IsVirtual:     true,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f0",
		InterfaceName: "eth0.69",
		NetworkName:   "vlan69",
		IsVirtual:     true,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f2",
		InterfaceName: "eth2",
		NetworkName:   "net2",
		IsVirtual:     false,
	}}
	err = s.machine.SetInstanceInfo("i-am", "fake_nonce", &hwChars, s.networks, s.machineIfaces)
	c.Assert(err, gc.IsNil)
	s.st = s.OpenAPIAsMachine(c, s.machine.Tag().String(), password, "fake_nonce")
	c.Assert(s.st, gc.NotNil)
}

func (s *networkerSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)

	// Create test juju state.
	s.setUpNetworks(c)
	s.setUpMachine(c)

	// Create the networker API facade.
	s.networkerState = s.st.Networker()
	c.Assert(s.networkerState, gc.NotNil)
}

func (s *networkerSuite) TestMachineNetworkInfoPermissionDenied(c *gc.C) {
	tags := []string{"foo-42", "unit-mysql-0", "service-mysql", "user-foo", "machine-1"}
	for _, tag := range tags {
		info, err := s.networkerState.MachineNetworkInfo(tag)
		c.Assert(err, gc.ErrorMatches, "permission denied")
		c.Assert(err, jc.Satisfies, params.IsCodeUnauthorized)
		c.Assert(info, gc.IsNil)
	}
}

type mockConfig struct {
	agent.Config
	tag string
}

func (mock *mockConfig) Tag() string {
	return mock.tag
}

func agentConfig(tag string) agent.Config {
	return &mockConfig{tag: tag}
}

const sampleInterfacesFile = `# This file describes the network interfaces available on your system
# and how to activate them. For more information see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

auto eth0
source %s/eth0.config

auto eth1
iface eth1 inet dhcp

auto eth1.2
iface eth1.2 inet dhcp

auto eth2
iface eth2 inet dhcp
`
var readyInterfaces = set.NewStrings("eth0", "eth1")
var interfacesWithAddress = set.NewStrings("eth0", "eth2")

const expectedInterfacesFile = `# This file describes the network interfaces available on your system
# and how to activate them. For more information see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

auto eth0
source %s/eth0.config

auto eth1
iface eth1 inet dhcp

auto eth1.2
iface eth1.2 inet dhcp

auto eth2
iface eth2 inet dhcp

auto eth1.42
iface eth1.42 inet dhcp

auto eth0.69
iface eth0.69 inet dhcp
`

func (s *networkerSuite) TestNetworker(c *gc.C) {
	// Create temporary directory to store interfaces file.
	s.networkDir = c.MkDir()
	s.PatchValue(&networker.NetworkDir, s.networkDir)
	s.interfacesFile = s.networkDir + "/interfaces"

	// Create a sample interfaces file
	interfacesFileContents := fmt.Sprintf(sampleInterfacesFile, s.networkDir)
	err := ioutil.WriteFile(s.interfacesFile, []byte(interfacesFileContents), 0644)
	c.Assert(err, gc.IsNil)

	// Patch the network interface functions
	s.PatchValue(&networker.InterfaceIsUp,
		func(name string) bool {
			return readyInterfaces.Contains(name)
		})
	s.PatchValue(&networker.InterfaceHasAddress,
		func(name string) bool {
			return interfacesWithAddress.Contains(name)
		})

	// Path the command executor
	s.PatchValue(&networker.ExecuteCommands,
		func(commands []string) error {
			s.executorCommands = commands
			close(s.executorHasFinished)
			return nil
		})
	s.executorHasFinished = make(chan struct{})

	// Create and setup networker.
	nw := networker.NewNetworker(s.networkerState, agentConfig(s.machine.Tag().String()))
	defer func() { c.Assert(worker.Stop(nw), gc.IsNil) }()

	// Wait until networker setup has finished execute commands.
	<-s.executorHasFinished

	// Verify the contents of generated interfaces file.
	contents, err := ioutil.ReadFile(s.interfacesFile)
	c.Assert(err, gc.IsNil)
	expected := fmt.Sprintf(expectedInterfacesFile, s.networkDir)
	c.Assert(string(contents), gc.Equals, expected)

	// Verify the executed commands
	expectedCommands := []string{
		"dpkg-query -s vlan || apt-get --option Dpkg::Options::=--force-confold --assume-yes install vlan",
		"lsmod | grep -q 8021q || modprobe 8021q",
		"grep -q 8021q /etc/modules || echo 8021q >> /etc/modules",
		"vconfig set_name_type DEV_PLUS_VID_NO_PAD",
		"ifup eth1.42",
		"ifup eth0.69",
		"ifup eth2",
	}
	c.Assert(s.executorCommands, gc.DeepEquals, expectedCommands)
}

func (s *networkerSuite) TestExecuteCommands(c *gc.C) {
	commands := []string{
		"echo start",
		"sh -c 'echo STDOUT; echo STDERR >&2; exit 123'",
		"echo end",
		"exit 111",
	}
	err := networker.ExecuteCommands(commands)
	expected := "command \"sh -c 'echo STDOUT; echo STDERR >&2; exit 123'\" failed " +
		"(code: 123, stdout: STDOUT\n, stderr: STDERR\n)"
	c.Assert(err, gc.NotNil)
	c.Assert(err.Error(), gc.Equals, expected)
}
<commit_msg>dropped TestMachineNetworkInfoPermissionDenied<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package networker_test

import (
	"fmt"
	"io/ioutil"

	"github.com/juju/utils"
	"github.com/juju/utils/set"
	gc "launchpad.net/gocheck"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/juju/testing"
	"github.com/juju/juju/state"
	"github.com/juju/juju/state/api"
	apinetworker "github.com/juju/juju/state/api/networker"
	"github.com/juju/juju/worker"
	"github.com/juju/juju/worker/networker"
)

type networkerSuite struct {
	testing.JujuConnSuite
	cfg *config.Config

	networks      []state.NetworkInfo
	machine       *state.Machine
	machineIfaces []state.NetworkInterfaceInfo

	st             *api.State
	networkerState *apinetworker.State

	networkDir          string
	interfacesFile      string
	executorCommands    []string
	executorHasFinished chan struct{}
}

var _ = gc.Suite(&networkerSuite{})

// Create several networks.
func (s *networkerSuite) setUpNetworks(c *gc.C) {
	s.networks = []state.NetworkInfo{{
		Name:       "net1",
		ProviderId: "net1",
		CIDR:       "0.1.2.0/24",
		VLANTag:    0,
	}, {
		Name:       "vlan42",
		ProviderId: "vlan42",
		CIDR:       "0.2.2.0/24",
		VLANTag:    42,
	}, {
		Name:       "vlan69",
		ProviderId: "vlan69",
		CIDR:       "0.3.2.0/24",
		VLANTag:    69,
	}, {
		Name:       "vlan123",
		ProviderId: "vlan123",
		CIDR:       "0.4.2.0/24",
		VLANTag:    123,
	}, {
		Name:       "net2",
		ProviderId: "net2",
		CIDR:       "0.5.2.0/24",
		VLANTag:    0,
	}}
}

// Create a machine and login to it.
func (s *networkerSuite) setUpMachine(c *gc.C) {
	var err error
	s.machine, err = s.State.AddMachine("quantal", state.JobHostUnits)
	c.Assert(err, gc.IsNil)
	password, err := utils.RandomPassword()
	c.Assert(err, gc.IsNil)
	err = s.machine.SetPassword(password)
	c.Assert(err, gc.IsNil)
	hwChars := instance.MustParseHardware("cpu-cores=123", "mem=4G")
	s.machineIfaces = []state.NetworkInterfaceInfo{{
		MACAddress:    "aa:bb:cc:dd:ee:f0",
		InterfaceName: "eth0",
		NetworkName:   "net1",
		IsVirtual:     false,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f1",
		InterfaceName: "eth1",
		NetworkName:   "net1",
		IsVirtual:     false,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f1",
		InterfaceName: "eth1.42",
		NetworkName:   "vlan42",
		IsVirtual:     true,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f0",
		InterfaceName: "eth0.69",
		NetworkName:   "vlan69",
		IsVirtual:     true,
	}, {
		MACAddress:    "aa:bb:cc:dd:ee:f2",
		InterfaceName: "eth2",
		NetworkName:   "net2",
		IsVirtual:     false,
	}}
	err = s.machine.SetInstanceInfo("i-am", "fake_nonce", &hwChars, s.networks, s.machineIfaces)
	c.Assert(err, gc.IsNil)
	s.st = s.OpenAPIAsMachine(c, s.machine.Tag().String(), password, "fake_nonce")
	c.Assert(s.st, gc.NotNil)
}

func (s *networkerSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)

	// Create test juju state.
	s.setUpNetworks(c)
	s.setUpMachine(c)

	// Create the networker API facade.
	s.networkerState = s.st.Networker()
	c.Assert(s.networkerState, gc.NotNil)
}

type mockConfig struct {
	agent.Config
	tag string
}

func (mock *mockConfig) Tag() string {
	return mock.tag
}

func agentConfig(tag string) agent.Config {
	return &mockConfig{tag: tag}
}

const sampleInterfacesFile = `# This file describes the network interfaces available on your system
# and how to activate them. For more information see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

auto eth0
source %s/eth0.config

auto eth1
iface eth1 inet dhcp

auto eth1.2
iface eth1.2 inet dhcp

auto eth2
iface eth2 inet dhcp
`
var readyInterfaces = set.NewStrings("eth0", "eth1")
var interfacesWithAddress = set.NewStrings("eth0", "eth2")

const expectedInterfacesFile = `# This file describes the network interfaces available on your system
# and how to activate them. For more information see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

auto eth0
source %s/eth0.config

auto eth1
iface eth1 inet dhcp

auto eth1.2
iface eth1.2 inet dhcp

auto eth2
iface eth2 inet dhcp

auto eth1.42
iface eth1.42 inet dhcp

auto eth0.69
iface eth0.69 inet dhcp
`

func (s *networkerSuite) TestNetworker(c *gc.C) {
	// Create temporary directory to store interfaces file.
	s.networkDir = c.MkDir()
	s.PatchValue(&networker.NetworkDir, s.networkDir)
	s.interfacesFile = s.networkDir + "/interfaces"

	// Create a sample interfaces file
	interfacesFileContents := fmt.Sprintf(sampleInterfacesFile, s.networkDir)
	err := ioutil.WriteFile(s.interfacesFile, []byte(interfacesFileContents), 0644)
	c.Assert(err, gc.IsNil)

	// Patch the network interface functions
	s.PatchValue(&networker.InterfaceIsUp,
		func(name string) bool {
			return readyInterfaces.Contains(name)
		})
	s.PatchValue(&networker.InterfaceHasAddress,
		func(name string) bool {
			return interfacesWithAddress.Contains(name)
		})

	// Path the command executor
	s.PatchValue(&networker.ExecuteCommands,
		func(commands []string) error {
			s.executorCommands = commands
			close(s.executorHasFinished)
			return nil
		})
	s.executorHasFinished = make(chan struct{})

	// Create and setup networker.
	nw := networker.NewNetworker(s.networkerState, agentConfig(s.machine.Tag().String()))
	defer func() { c.Assert(worker.Stop(nw), gc.IsNil) }()

	// Wait until networker setup has finished execute commands.
	<-s.executorHasFinished

	// Verify the contents of generated interfaces file.
	contents, err := ioutil.ReadFile(s.interfacesFile)
	c.Assert(err, gc.IsNil)
	expected := fmt.Sprintf(expectedInterfacesFile, s.networkDir)
	c.Assert(string(contents), gc.Equals, expected)

	// Verify the executed commands
	expectedCommands := []string{
		"dpkg-query -s vlan || apt-get --option Dpkg::Options::=--force-confold --assume-yes install vlan",
		"lsmod | grep -q 8021q || modprobe 8021q",
		"grep -q 8021q /etc/modules || echo 8021q >> /etc/modules",
		"vconfig set_name_type DEV_PLUS_VID_NO_PAD",
		"ifup eth1.42",
		"ifup eth0.69",
		"ifup eth2",
	}
	c.Assert(s.executorCommands, gc.DeepEquals, expectedCommands)
}

func (s *networkerSuite) TestExecuteCommands(c *gc.C) {
	commands := []string{
		"echo start",
		"sh -c 'echo STDOUT; echo STDERR >&2; exit 123'",
		"echo end",
		"exit 111",
	}
	err := networker.ExecuteCommands(commands)
	expected := "command \"sh -c 'echo STDOUT; echo STDERR >&2; exit 123'\" failed " +
		"(code: 123, stdout: STDOUT\n, stderr: STDERR\n)"
	c.Assert(err, gc.NotNil)
	c.Assert(err.Error(), gc.Equals, expected)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Godoc comment extraction and comment -> HTML formatting.

package doc

import (
	"go/ast"
	"http" // for URLEscape
	"io"
	"regexp"
	"strings"
	"template" // for htmlEscape
)

func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' }

func stripTrailingWhitespace(s string) string {
	i := len(s)
	for i > 0 && isWhitespace(s[i-1]) {
		i--
	}
	return s[0:i]
}

// CommentText returns the text of comment,
// with the comment markers - //, /*, and */ - removed.
func CommentText(comment *ast.CommentGroup) string {
	if comment == nil {
		return ""
	}
	comments := make([]string, len(comment.List))
	for i, c := range comment.List {
		comments[i] = string(c.Text)
	}

	lines := make([]string, 0, 10) // most comments are less than 10 lines
	for _, c := range comments {
		// Remove comment markers.
		// The parser has given us exactly the comment text.
		switch c[1] {
		case '/':
			//-style comment
			c = c[2:]
			// Remove leading space after //, if there is one.
			// TODO(gri) This appears to be necessary in isolated
			//           cases (bignum.RatFromString) - why?
			if len(c) > 0 && c[0] == ' ' {
				c = c[1:]
			}
		case '*':
			/*-style comment */
			c = c[2 : len(c)-2]
		}

		// Split on newlines.
		cl := strings.Split(c, "\n", -1)

		// Walk lines, stripping trailing white space and adding to list.
		for _, l := range cl {
			l = stripTrailingWhitespace(l)
			// Add to list.
			n := len(lines)
			if n+1 >= cap(lines) {
				newlines := make([]string, n, 2*cap(lines))
				copy(newlines, lines)
				lines = newlines
			}
			lines = lines[0 : n+1]
			lines[n] = l
		}
	}

	// Remove leading blank lines; convert runs of
	// interior blank lines to a single blank line.
	n := 0
	for _, line := range lines {
		if line != "" || n > 0 && lines[n-1] != "" {
			lines[n] = line
			n++
		}
	}
	lines = lines[0:n]

	// Add final "" entry to get trailing newline from Join.
	// The original loop always leaves room for one more.
	if n > 0 && lines[n-1] != "" {
		lines = lines[0 : n+1]
		lines[n] = ""
	}

	return strings.Join(lines, "\n")
}

// Split bytes into lines.
func split(text []byte) [][]byte {
	// count lines
	n := 0
	last := 0
	for i, c := range text {
		if c == '\n' {
			last = i + 1
			n++
		}
	}
	if last < len(text) {
		n++
	}

	// split
	out := make([][]byte, n)
	last = 0
	n = 0
	for i, c := range text {
		if c == '\n' {
			out[n] = text[last : i+1]
			last = i + 1
			n++
		}
	}
	if last < len(text) {
		out[n] = text[last:]
	}

	return out
}

var (
	ldquo = []byte("“")
	rdquo = []byte("”")
)

// Escape comment text for HTML. If nice is set,
// also turn `` into “ and '' into ”.
func commentEscape(w io.Writer, s []byte, nice bool) {
	last := 0
	if nice {
		for i := 0; i < len(s)-1; i++ {
			ch := s[i]
			if ch == s[i+1] && (ch == '`' || ch == '\'') {
				template.HTMLEscape(w, s[last:i])
				last = i + 2
				switch ch {
				case '`':
					w.Write(ldquo)
				case '\'':
					w.Write(rdquo)
				}
				i++ // loop will add one more
			}
		}
	}
	template.HTMLEscape(w, s[last:])
}

const (
	// Regexp for Go identifiers
	identRx = `[a-zA-Z_][a-zA-Z_0-9]*` // TODO(gri) ASCII only for now - fix this

	// Regexp for URLs
	protocol = `(https?|ftp|file|gopher|mailto|news|nntp|telnet|wais|prospero):`
	hostPart = `[a-zA-Z0-9_@\-]+`
	filePart = `[a-zA-Z0-9_?%#~&/\-+=]+`
	urlRx    = protocol + `//` + // http://
		hostPart + `([.:]` + hostPart + `)*/?` + // //www.google.com:8080/
		filePart + `([:.,]` + filePart + `)*`
)

var matchRx = regexp.MustCompile(`(` + identRx + `)|(` + urlRx + `)`)

var (
	html_a      = []byte(`<a href="`)
	html_aq     = []byte(`">`)
	html_enda   = []byte("")
	html_i      = []byte("")
	html_endi   = []byte("")
	html_p      = []byte("
\n")
	html_endp   = []byte("
\n")
	html_pre    = []byte("
")
	html_endpre = []byte("
\n")
)

// Emphasize and escape a line of text for HTML. URLs are converted into links;
// if the URL also appears in the words map, the link is taken from the map (if
// the corresponding map value is the empty string, the URL is not converted
// into a link). Go identifiers that appear in the words map are italicized; if
// the corresponding map value is not the empty string, it is considered a URL
// and the word is converted into a link. If nice is set, the remaining text's
// appearance is improved where is makes sense (e.g., `` is turned into “
// and '' into ”).
func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) {
	for {
		m := matchRx.FindSubmatchIndex(line)
		if m == nil {
			break
		}
		// m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is identRx)

		// write text before match
		commentEscape(w, line[0:m[0]], nice)

		// analyze match
		match := line[m[0]:m[1]]
		url := ""
		italics := false
		if words != nil {
			url, italics = words[string(match)]
		}
		if m[2] < 0 {
			// didn't match against first parenthesized sub-regexp; must be match against urlRx
			if !italics {
				// no alternative URL in words list, use match instead
				url = string(match)
			}
			italics = false // don't italicize URLs
		}

		// write match
		if len(url) > 0 {
			w.Write(html_a)
			w.Write([]byte(http.URLEscape(url)))
			w.Write(html_aq)
		}
		if italics {
			w.Write(html_i)
		}
		commentEscape(w, match, nice)
		if italics {
			w.Write(html_endi)
		}
		if len(url) > 0 {
			w.Write(html_enda)
		}

		// advance
		line = line[m[1]:]
	}
	commentEscape(w, line, nice)
}

func indentLen(s []byte) int {
	i := 0
	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
		i++
	}
	return i
}

func isBlank(s []byte) bool { return len(s) == 0 || (len(s) == 1 && s[0] == '\n') }

func commonPrefix(a, b []byte) []byte {
	i := 0
	for i < len(a) && i < len(b) && a[i] == b[i] {
		i++
	}
	return a[0:i]
}

func unindent(block [][]byte) {
	if len(block) == 0 {
		return
	}

	// compute maximum common white prefix
	prefix := block[0][0:indentLen(block[0])]
	for _, line := range block {
		if !isBlank(line) {
			prefix = commonPrefix(prefix, line[0:indentLen(line)])
		}
	}
	n := len(prefix)

	// remove
	for i, line := range block {
		if !isBlank(line) {
			block[i] = line[n:]
		}
	}
}

// Convert comment text to formatted HTML.
// The comment was prepared by DocReader,
// so it is known not to have leading, trailing blank lines
// nor to have trailing spaces at the end of lines.
// The comment markers have already been removed.
//
// Turn each run of multiple \n into

// Turn each run of indented lines into a
 block without indent.
//
// URLs in the comment text are converted into links; if the URL also appears
// in the words map, the link is taken from the map (if the corresponding map
// value is the empty string, the URL is not converted into a link).
//
// Go identifiers that appear in the words map are italicized; if the corresponding
// map value is not the empty string, it is considered a URL and the word is converted
// into a link.
func ToHTML(w io.Writer, s []byte, words map[string]string) {
	inpara := false

	close := func() {
		if inpara {
			w.Write(html_endp)
			inpara = false
		}
	}
	open := func() {
		if !inpara {
			w.Write(html_p)
			inpara = true
		}
	}

	lines := split(s)
	unindent(lines)
	for i := 0; i < len(lines); {
		line := lines[i]
		if isBlank(line) {
			// close paragraph
			close()
			i++
			continue
		}
		if indentLen(line) > 0 {
			// close paragraph
			close()

			// count indented or blank lines
			j := i + 1
			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
				j++
			}
			// but not trailing blank lines
			for j > i && isBlank(lines[j-1]) {
				j--
			}
			block := lines[i:j]
			i = j

			unindent(block)

			// put those lines in a pre block
			w.Write(html_pre)
			for _, line := range block {
				emphasize(w, line, nil, false) // no nice text formatting
			}
			w.Write(html_endpre)
			continue
		}
		// open paragraph
		open()
		emphasize(w, lines[i], words, true) // nice text formatting
		i++
	}
	close()
}
<commit_msg>go/doc: use correct escaper for URL<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Godoc comment extraction and comment -> HTML formatting.

package doc

import (
	"go/ast"
	"io"
	"regexp"
	"strings"
	"template" // for htmlEscape
)

func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' }

func stripTrailingWhitespace(s string) string {
	i := len(s)
	for i > 0 && isWhitespace(s[i-1]) {
		i--
	}
	return s[0:i]
}

// CommentText returns the text of comment,
// with the comment markers - //, /*, and */ - removed.
func CommentText(comment *ast.CommentGroup) string {
	if comment == nil {
		return ""
	}
	comments := make([]string, len(comment.List))
	for i, c := range comment.List {
		comments[i] = string(c.Text)
	}

	lines := make([]string, 0, 10) // most comments are less than 10 lines
	for _, c := range comments {
		// Remove comment markers.
		// The parser has given us exactly the comment text.
		switch c[1] {
		case '/':
			//-style comment
			c = c[2:]
			// Remove leading space after //, if there is one.
			// TODO(gri) This appears to be necessary in isolated
			//           cases (bignum.RatFromString) - why?
			if len(c) > 0 && c[0] == ' ' {
				c = c[1:]
			}
		case '*':
			/*-style comment */
			c = c[2 : len(c)-2]
		}

		// Split on newlines.
		cl := strings.Split(c, "\n", -1)

		// Walk lines, stripping trailing white space and adding to list.
		for _, l := range cl {
			l = stripTrailingWhitespace(l)
			// Add to list.
			n := len(lines)
			if n+1 >= cap(lines) {
				newlines := make([]string, n, 2*cap(lines))
				copy(newlines, lines)
				lines = newlines
			}
			lines = lines[0 : n+1]
			lines[n] = l
		}
	}

	// Remove leading blank lines; convert runs of
	// interior blank lines to a single blank line.
	n := 0
	for _, line := range lines {
		if line != "" || n > 0 && lines[n-1] != "" {
			lines[n] = line
			n++
		}
	}
	lines = lines[0:n]

	// Add final "" entry to get trailing newline from Join.
	// The original loop always leaves room for one more.
	if n > 0 && lines[n-1] != "" {
		lines = lines[0 : n+1]
		lines[n] = ""
	}

	return strings.Join(lines, "\n")
}

// Split bytes into lines.
func split(text []byte) [][]byte {
	// count lines
	n := 0
	last := 0
	for i, c := range text {
		if c == '\n' {
			last = i + 1
			n++
		}
	}
	if last < len(text) {
		n++
	}

	// split
	out := make([][]byte, n)
	last = 0
	n = 0
	for i, c := range text {
		if c == '\n' {
			out[n] = text[last : i+1]
			last = i + 1
			n++
		}
	}
	if last < len(text) {
		out[n] = text[last:]
	}

	return out
}

var (
	ldquo = []byte("“")
	rdquo = []byte("”")
)

// Escape comment text for HTML. If nice is set,
// also turn `` into “ and '' into ”.
func commentEscape(w io.Writer, s []byte, nice bool) {
	last := 0
	if nice {
		for i := 0; i < len(s)-1; i++ {
			ch := s[i]
			if ch == s[i+1] && (ch == '`' || ch == '\'') {
				template.HTMLEscape(w, s[last:i])
				last = i + 2
				switch ch {
				case '`':
					w.Write(ldquo)
				case '\'':
					w.Write(rdquo)
				}
				i++ // loop will add one more
			}
		}
	}
	template.HTMLEscape(w, s[last:])
}

const (
	// Regexp for Go identifiers
	identRx = `[a-zA-Z_][a-zA-Z_0-9]*` // TODO(gri) ASCII only for now - fix this

	// Regexp for URLs
	protocol = `(https?|ftp|file|gopher|mailto|news|nntp|telnet|wais|prospero):`
	hostPart = `[a-zA-Z0-9_@\-]+`
	filePart = `[a-zA-Z0-9_?%#~&/\-+=]+`
	urlRx    = protocol + `//` + // http://
		hostPart + `([.:]` + hostPart + `)*/?` + // //www.google.com:8080/
		filePart + `([:.,]` + filePart + `)*`
)

var matchRx = regexp.MustCompile(`(` + identRx + `)|(` + urlRx + `)`)

var (
	html_a      = []byte(`<a href="`)
	html_aq     = []byte(`">`)
	html_enda   = []byte("")
	html_i      = []byte("")
	html_endi   = []byte("")
	html_p      = []byte("
\n")
	html_endp   = []byte("
\n")
	html_pre    = []byte("
")
	html_endpre = []byte("
\n")
)

// Emphasize and escape a line of text for HTML. URLs are converted into links;
// if the URL also appears in the words map, the link is taken from the map (if
// the corresponding map value is the empty string, the URL is not converted
// into a link). Go identifiers that appear in the words map are italicized; if
// the corresponding map value is not the empty string, it is considered a URL
// and the word is converted into a link. If nice is set, the remaining text's
// appearance is improved where is makes sense (e.g., `` is turned into “
// and '' into ”).
func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) {
	for {
		m := matchRx.FindSubmatchIndex(line)
		if m == nil {
			break
		}
		// m >= 6 (two parenthesized sub-regexps in matchRx, 1st one is identRx)

		// write text before match
		commentEscape(w, line[0:m[0]], nice)

		// analyze match
		match := line[m[0]:m[1]]
		url := ""
		italics := false
		if words != nil {
			url, italics = words[string(match)]
		}
		if m[2] < 0 {
			// didn't match against first parenthesized sub-regexp; must be match against urlRx
			if !italics {
				// no alternative URL in words list, use match instead
				url = string(match)
			}
			italics = false // don't italicize URLs
		}

		// write match
		if len(url) > 0 {
			w.Write(html_a)
			template.HTMLEscape(w, []byte(url))
			w.Write(html_aq)
		}
		if italics {
			w.Write(html_i)
		}
		commentEscape(w, match, nice)
		if italics {
			w.Write(html_endi)
		}
		if len(url) > 0 {
			w.Write(html_enda)
		}

		// advance
		line = line[m[1]:]
	}
	commentEscape(w, line, nice)
}

func indentLen(s []byte) int {
	i := 0
	for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
		i++
	}
	return i
}

func isBlank(s []byte) bool { return len(s) == 0 || (len(s) == 1 && s[0] == '\n') }

func commonPrefix(a, b []byte) []byte {
	i := 0
	for i < len(a) && i < len(b) && a[i] == b[i] {
		i++
	}
	return a[0:i]
}

func unindent(block [][]byte) {
	if len(block) == 0 {
		return
	}

	// compute maximum common white prefix
	prefix := block[0][0:indentLen(block[0])]
	for _, line := range block {
		if !isBlank(line) {
			prefix = commonPrefix(prefix, line[0:indentLen(line)])
		}
	}
	n := len(prefix)

	// remove
	for i, line := range block {
		if !isBlank(line) {
			block[i] = line[n:]
		}
	}
}

// Convert comment text to formatted HTML.
// The comment was prepared by DocReader,
// so it is known not to have leading, trailing blank lines
// nor to have trailing spaces at the end of lines.
// The comment markers have already been removed.
//
// Turn each run of multiple \n into

// Turn each run of indented lines into a
 block without indent.
//
// URLs in the comment text are converted into links; if the URL also appears
// in the words map, the link is taken from the map (if the corresponding map
// value is the empty string, the URL is not converted into a link).
//
// Go identifiers that appear in the words map are italicized; if the corresponding
// map value is not the empty string, it is considered a URL and the word is converted
// into a link.
func ToHTML(w io.Writer, s []byte, words map[string]string) {
	inpara := false

	close := func() {
		if inpara {
			w.Write(html_endp)
			inpara = false
		}
	}
	open := func() {
		if !inpara {
			w.Write(html_p)
			inpara = true
		}
	}

	lines := split(s)
	unindent(lines)
	for i := 0; i < len(lines); {
		line := lines[i]
		if isBlank(line) {
			// close paragraph
			close()
			i++
			continue
		}
		if indentLen(line) > 0 {
			// close paragraph
			close()

			// count indented or blank lines
			j := i + 1
			for j < len(lines) && (isBlank(lines[j]) || indentLen(lines[j]) > 0) {
				j++
			}
			// but not trailing blank lines
			for j > i && isBlank(lines[j-1]) {
				j--
			}
			block := lines[i:j]
			i = j

			unindent(block)

			// put those lines in a pre block
			w.Write(html_pre)
			for _, line := range block {
				emphasize(w, line, nil, false) // no nice text formatting
			}
			w.Write(html_endpre)
			continue
		}
		// open paragraph
		open()
		emphasize(w, lines[i], words, true) // nice text formatting
		i++
	}
	close()
}
<|endoftext|>


<commit_before>package plugins

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	base "github.com/jriddick/geoffrey/bot"
	"github.com/jriddick/geoffrey/irc"
	"github.com/jriddick/geoffrey/msg"
	log "github.com/sirupsen/logrus"

	"github.com/tidwall/gjson"
)

func init() {
	base.RegisterHandler(CurrencyHandler)
}

// CurrencyHandler extracts title from posted links and sends
// them to the channel.
var CurrencyHandler = base.Handler{
	Name:        "Currency",
	Description: "Currency converter.",
	Event:       irc.Message,
	Run: func(bot *base.Bot, msg *msg.Message) (bool, error) {
		// Get configuration
		config := bot.Config()

		client := http.Client{}

		returnData := "Currency error, usage: !c <amount> <from> <to>"

		// Check if channel message
		if msg.Params[0] == config.Identification.Nick || msg.Prefix.Name == "nibbler" || msg.Prefix.Name == "geoffrey-bot" {
			return false, nil
		}

		go func(bot *base.Bot, channel string) {
			if strings.HasPrefix(msg.Trailing, "!c") {
				if len(msg.Trailing) > 2 {
					cData := strings.Split(msg.Trailing, " ")

					if len(cData) == 4 {
						url := fmt.Sprintf("https://api.exchangerate.host/latest?base=%s&amount=%s", cData[2], cData[1])

						request, reqErr := http.NewRequest(http.MethodGet, url, nil)
						if reqErr != nil {
							log.Errorf("[currency] Failed NewRequest: %s", reqErr)
						}

						resp, respErr := client.Do(request)
						if respErr != nil {
							log.Errorf("[currency] Failed client.Do: %s", respErr)
						}

						cConvertTo := fmt.Sprintf("rates.%s", strings.ToUpper(cData[3]))

						body, bodyErr := ioutil.ReadAll(resp.Body)
						if bodyErr != nil {
							log.Errorf("[currency] Failed ReadAll: %s", bodyErr)
						}

						cValue := gjson.GetBytes(body, cConvertTo)

						returnData = fmt.Sprintf("%s %s in %s: %v", cData[1], cData[2], cData[3], cValue)
					}
				}
			}

			bot.Send(channel, returnData)
		}(bot, msg.Params[0])
		return true, nil
	},
}
<commit_msg>Moved usage error due to catching all messages<commit_after>package plugins

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	base "github.com/jriddick/geoffrey/bot"
	"github.com/jriddick/geoffrey/irc"
	"github.com/jriddick/geoffrey/msg"
	log "github.com/sirupsen/logrus"

	"github.com/tidwall/gjson"
)

func init() {
	base.RegisterHandler(CurrencyHandler)
}

// CurrencyHandler extracts title from posted links and sends
// them to the channel.
var CurrencyHandler = base.Handler{
	Name:        "Currency",
	Description: "Currency converter.",
	Event:       irc.Message,
	Run: func(bot *base.Bot, msg *msg.Message) (bool, error) {
		// Get configuration
		config := bot.Config()

		client := http.Client{}

		returnData := "Currency error, usage: !c <amount> <from> <to>"

		// Check if channel message
		if msg.Params[0] == config.Identification.Nick || msg.Prefix.Name == "nibbler" || msg.Prefix.Name == "geoffrey-bot" {
			return false, nil
		}

		go func(bot *base.Bot, channel string) {
			if strings.HasPrefix(msg.Trailing, "!c") {
				if len(msg.Trailing) > 2 {
					cData := strings.Split(msg.Trailing, " ")

					if len(cData) == 4 {
						url := fmt.Sprintf("https://api.exchangerate.host/latest?base=%s&amount=%s", cData[2], cData[1])

						request, reqErr := http.NewRequest(http.MethodGet, url, nil)
						if reqErr != nil {
							log.Errorf("[currency] Failed NewRequest: %s", reqErr)
						}

						resp, respErr := client.Do(request)
						if respErr != nil {
							log.Errorf("[currency] Failed client.Do: %s", respErr)
						}

						cConvertTo := fmt.Sprintf("rates.%s", strings.ToUpper(cData[3]))

						body, bodyErr := ioutil.ReadAll(resp.Body)
						if bodyErr != nil {
							log.Errorf("[currency] Failed ReadAll: %s", bodyErr)
						}

						cValue := gjson.GetBytes(body, cConvertTo)

						returnData = fmt.Sprintf("%s %s in %s: %v", cData[1], cData[2], cData[3], cValue)
					}
				}
				bot.Send(channel, returnData)
			}
		}(bot, msg.Params[0])
		return true, nil
	},
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package defines constants representing the lexical
// tokens of the Go programming language and basic operations
// on tokens (printing, predicates).
//
package token

import (
	"fmt";
	"strconv";
)

// Token is the set of lexical tokens of the Go programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	ILLEGAL Token = iota;
	EOF;
	COMMENT;

	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	literal_beg;
	IDENT;
	INT;
	FLOAT;
	CHAR;
	STRING;
	literal_end;

	// Operators and delimiters
	operator_beg;
	ADD;
	SUB;
	MUL;
	QUO;
	REM;

	AND;
	OR;
	XOR;
	SHL;
	SHR;
	AND_NOT;

	ADD_ASSIGN;
	SUB_ASSIGN;
	MUL_ASSIGN;
	QUO_ASSIGN;
	REM_ASSIGN;

	AND_ASSIGN;
	OR_ASSIGN;
	XOR_ASSIGN;
	SHL_ASSIGN;
	SHR_ASSIGN;
	AND_NOT_ASSIGN;

	LAND;
	LOR;
	ARROW;
	INC;
	DEC;

	EQL;
	LSS;
	GTR;
	ASSIGN;
	NOT;

	NEQ;
	LEQ;
	GEQ;
	DEFINE;
	ELLIPSIS;

	LPAREN;
	LBRACK;
	LBRACE;
	COMMA;
	PERIOD;

	RPAREN;
	RBRACK;
	RBRACE;
	SEMICOLON;
	COLON;
	operator_end;

	// Keywords
	keyword_beg;
	BREAK;
	CASE;
	CHAN;
	CONST;
	CONTINUE;

	DEFAULT;
	DEFER;
	ELSE;
	FALLTHROUGH;
	FOR;

	FUNC;
	GO;
	GOTO;
	IF;
	IMPORT;

	INTERFACE;
	MAP;
	PACKAGE;
	RANGE;
	RETURN;

	SELECT;
	STRUCT;
	SWITCH;
	TYPE;
	VAR;
	keyword_end;
)

// At the moment we have no array literal syntax that lets us describe
// the index for each element - use a map for now to make sure they are
// in sync.
var tokens = map [Token] string {
	ILLEGAL : "ILLEGAL",

	EOF : "EOF",
	COMMENT : "COMMENT",

	IDENT : "IDENT",
	INT : "INT",
	FLOAT : "FLOAT",
	CHAR : "CHAR",
	STRING : "STRING",

	ADD : "+",
	SUB : "-",
	MUL : "*",
	QUO : "/",
	REM : "%",

	AND : "&",
	OR : "|",
	XOR : "^",
	SHL : "<<",
	SHR : ">>",
	AND_NOT : "&^",

	ADD_ASSIGN : "+=",
	SUB_ASSIGN : "-=",
	MUL_ASSIGN : "*=",
	QUO_ASSIGN : "/=",
	REM_ASSIGN : "%=",

	AND_ASSIGN : "&=",
	OR_ASSIGN : "|=",
	XOR_ASSIGN : "^=",
	SHL_ASSIGN : "<<=",
	SHR_ASSIGN : ">>=",
	AND_NOT_ASSIGN : "&^=",

	LAND : "&&",
	LOR : "||",
	ARROW : "<-",
	INC : "++",
	DEC : "--",

	EQL : "==",
	LSS : "<",
	GTR : ">",
	ASSIGN : "=",
	NOT : "!",

	NEQ : "!=",
	LEQ : "<=",
	GEQ : ">=",
	DEFINE : ":=",
	ELLIPSIS : "...",

	LPAREN : "(",
	LBRACK : "[",
	LBRACE : "{",
	COMMA : ",",
	PERIOD : ".",

	RPAREN : ")",
	RBRACK : "]",
	RBRACE : "}",
	SEMICOLON : ";",
	COLON : ":",

	BREAK : "break",
	CASE : "case",
	CHAN : "chan",
	CONST : "const",
	CONTINUE : "continue",

	DEFAULT : "default",
	DEFER : "defer",
	ELSE : "else",
	FALLTHROUGH : "fallthrough",
	FOR : "for",

	FUNC : "func",
	GO : "go",
	GOTO : "goto",
	IF : "if",
	IMPORT : "import",

	INTERFACE : "interface",
	MAP : "map",
	PACKAGE : "package",
	RANGE : "range",
	RETURN : "return",

	SELECT : "select",
	STRUCT : "struct",
	SWITCH : "switch",
	TYPE : "type",
	VAR : "var",
}

// String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT").
//
func (tok Token) String() string {
	if str, exists := tokens[tok]; exists {
		return str;
	}
	return "token(" + strconv.Itoa(int(tok)) + ")";
}

// A set of constants for precedence-based expression parsing.
// Non-operators have lowest precedence, followed by operators
// starting with precedence 1 up to unary operators. The highest
// precedence corresponds serves as "catch-all" precedence for
// selector, indexing, and other operator and delimiter tokens.
//
const (
	LowestPrec = 0;  // non-operators
	UnaryPrec = 7;
	HighestPrec = 8;
)

// Precedence returns the operator precedence of the binary
// operator op. If op is not a binary operator, the result
// is LowestPrecedence.
//
func (op Token) Precedence() int {
	switch op {
	case LOR:
		return 1;
	case LAND:
		return 2;
	case ARROW:
		return 3;
	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
		return 4;
	case ADD, SUB, OR, XOR:
		return 5;
	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
		return 6;
	}
	return LowestPrec;
}

var keywords map [string] Token;

func init() {
	keywords = make(map [string] Token);
	for i := keyword_beg + 1; i < keyword_end; i++ {
		keywords[tokens[i]] = i;
	}
}

// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
//
func Lookup(ident []byte) Token {
	// TODO Maps with []byte key are illegal because []byte does not
	//      support == . Should find a more efficient solution eventually.
	if tok, is_keyword := keywords[string(ident)]; is_keyword {
		return tok;
	}
	return IDENT;
}

// Predicates

// IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; returns false otherwise.
//
func (tok Token) IsLiteral() bool {
	return literal_beg < tok && tok < literal_end;
}

// IsOperator returns true for tokens corresponding to operators and
// delimiters; returns false otherwise.
//
func (tok Token) IsOperator() bool {
	return operator_beg < tok && tok < operator_end;
}

// IsKeyword returns true for tokens corresponding to keywords;
// returns false otherwise.
//
func (tok Token) IsKeyword() bool {
	return keyword_beg < tok && tok < keyword_end;
}

// Token source positions are represented by a Position value.
// A Position is valid if the line number is > 0.
//
type Position struct {
	Filename string;  // filename, if any
	Offset int;  // byte offset, starting at 0
	Line int;  // line number, starting at 1
	Column int;  // column number, starting at 1 (character count)
}

// Pos is an accessor method for anonymous Position fields.
// It returns its receiver.
//
func (pos *Position) Pos() Position {
	return *pos;
}

// IsValid returns true if the position is valid.
func (pos *Position) IsValid() bool {
	return pos.Line > 0
}

func (pos *Position) String() string {
	if pos != nil {
		s := pos.Filename;
		if pos.IsValid() {
			if s != "" {
				s += ":";
			}
			s += fmt.Sprintf("%d:%d", pos.Line, pos.Column);
		}
		if s != "" {
			return s;
		}
	}
	return "<unknown position>";
}
<commit_msg>documentation edits: add comments showing Go syntax for each token. move doc comments below hidden begin constants<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package defines constants representing the lexical
// tokens of the Go programming language and basic operations
// on tokens (printing, predicates).
//
package token

import (
	"fmt";
	"strconv";
)

// Token is the set of lexical tokens of the Go programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	ILLEGAL Token = iota;
	EOF;
	COMMENT;

	literal_beg;
	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	IDENT;		// main
	INT;		// 12345
	FLOAT;		// 123.45
	CHAR;		// 'a'
	STRING;		// "abc"
	literal_end;

	operator_beg;
	// Operators and delimiters
	ADD;		// +
	SUB;		// -
	MUL;		// *
	QUO;		// /
	REM;		// %

	AND;		// &
	OR;		// |
	XOR;		// ^
	SHL;		// <<
	SHR;		// >>
	AND_NOT;	// &^

	ADD_ASSIGN;	// +=
	SUB_ASSIGN;	// -=
	MUL_ASSIGN;	// *=
	QUO_ASSIGN;	// /=
	REM_ASSIGN;	// %=

	AND_ASSIGN;	// &=
	OR_ASSIGN;	// |=
	XOR_ASSIGN;	// ^=
	SHL_ASSIGN;	// <<=
	SHR_ASSIGN;	// >>=
	AND_NOT_ASSIGN;	// &^=

	LAND;		// &&
	LOR;		// ||
	ARROW;		// <-
	INC;		// ++
	DEC;		// --

	EQL;		// ==
	LSS;		// <
	GTR;		// >
	ASSIGN;		// =
	NOT;		// !

	NEQ;		// !=
	LEQ;		// <=
	GEQ;		// >=
	DEFINE;		// :=
	ELLIPSIS;	// ...

	LPAREN;		// (
	LBRACK;		// [
	LBRACE;		// {
	COMMA;		// ,
	PERIOD;		// .

	RPAREN;		// )
	RBRACK;		// ]
	RBRACE;		// }
	SEMICOLON;	// ;
	COLON;		// :
	operator_end;

	keyword_beg;
	// Keywords
	BREAK;
	CASE;
	CHAN;
	CONST;
	CONTINUE;

	DEFAULT;
	DEFER;
	ELSE;
	FALLTHROUGH;
	FOR;

	FUNC;
	GO;
	GOTO;
	IF;
	IMPORT;

	INTERFACE;
	MAP;
	PACKAGE;
	RANGE;
	RETURN;

	SELECT;
	STRUCT;
	SWITCH;
	TYPE;
	VAR;
	keyword_end;
)

// At the moment we have no array literal syntax that lets us describe
// the index for each element - use a map for now to make sure they are
// in sync.
var tokens = map [Token] string {
	ILLEGAL : "ILLEGAL",

	EOF : "EOF",
	COMMENT : "COMMENT",

	IDENT : "IDENT",
	INT : "INT",
	FLOAT : "FLOAT",
	CHAR : "CHAR",
	STRING : "STRING",

	ADD : "+",
	SUB : "-",
	MUL : "*",
	QUO : "/",
	REM : "%",

	AND : "&",
	OR : "|",
	XOR : "^",
	SHL : "<<",
	SHR : ">>",
	AND_NOT : "&^",

	ADD_ASSIGN : "+=",
	SUB_ASSIGN : "-=",
	MUL_ASSIGN : "*=",
	QUO_ASSIGN : "/=",
	REM_ASSIGN : "%=",

	AND_ASSIGN : "&=",
	OR_ASSIGN : "|=",
	XOR_ASSIGN : "^=",
	SHL_ASSIGN : "<<=",
	SHR_ASSIGN : ">>=",
	AND_NOT_ASSIGN : "&^=",

	LAND : "&&",
	LOR : "||",
	ARROW : "<-",
	INC : "++",
	DEC : "--",

	EQL : "==",
	LSS : "<",
	GTR : ">",
	ASSIGN : "=",
	NOT : "!",

	NEQ : "!=",
	LEQ : "<=",
	GEQ : ">=",
	DEFINE : ":=",
	ELLIPSIS : "...",

	LPAREN : "(",
	LBRACK : "[",
	LBRACE : "{",
	COMMA : ",",
	PERIOD : ".",

	RPAREN : ")",
	RBRACK : "]",
	RBRACE : "}",
	SEMICOLON : ";",
	COLON : ":",

	BREAK : "break",
	CASE : "case",
	CHAN : "chan",
	CONST : "const",
	CONTINUE : "continue",

	DEFAULT : "default",
	DEFER : "defer",
	ELSE : "else",
	FALLTHROUGH : "fallthrough",
	FOR : "for",

	FUNC : "func",
	GO : "go",
	GOTO : "goto",
	IF : "if",
	IMPORT : "import",

	INTERFACE : "interface",
	MAP : "map",
	PACKAGE : "package",
	RANGE : "range",
	RETURN : "return",

	SELECT : "select",
	STRUCT : "struct",
	SWITCH : "switch",
	TYPE : "type",
	VAR : "var",
}

// String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT").
//
func (tok Token) String() string {
	if str, exists := tokens[tok]; exists {
		return str;
	}
	return "token(" + strconv.Itoa(int(tok)) + ")";
}

// A set of constants for precedence-based expression parsing.
// Non-operators have lowest precedence, followed by operators
// starting with precedence 1 up to unary operators. The highest
// precedence corresponds serves as "catch-all" precedence for
// selector, indexing, and other operator and delimiter tokens.
//
const (
	LowestPrec = 0;  // non-operators
	UnaryPrec = 7;
	HighestPrec = 8;
)

// Precedence returns the operator precedence of the binary
// operator op. If op is not a binary operator, the result
// is LowestPrecedence.
//
func (op Token) Precedence() int {
	switch op {
	case LOR:
		return 1;
	case LAND:
		return 2;
	case ARROW:
		return 3;
	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
		return 4;
	case ADD, SUB, OR, XOR:
		return 5;
	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
		return 6;
	}
	return LowestPrec;
}

var keywords map [string] Token;

func init() {
	keywords = make(map [string] Token);
	for i := keyword_beg + 1; i < keyword_end; i++ {
		keywords[tokens[i]] = i;
	}
}

// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
//
func Lookup(ident []byte) Token {
	// TODO Maps with []byte key are illegal because []byte does not
	//      support == . Should find a more efficient solution eventually.
	if tok, is_keyword := keywords[string(ident)]; is_keyword {
		return tok;
	}
	return IDENT;
}

// Predicates

// IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; returns false otherwise.
//
func (tok Token) IsLiteral() bool {
	return literal_beg < tok && tok < literal_end;
}

// IsOperator returns true for tokens corresponding to operators and
// delimiters; returns false otherwise.
//
func (tok Token) IsOperator() bool {
	return operator_beg < tok && tok < operator_end;
}

// IsKeyword returns true for tokens corresponding to keywords;
// returns false otherwise.
//
func (tok Token) IsKeyword() bool {
	return keyword_beg < tok && tok < keyword_end;
}

// Token source positions are represented by a Position value.
// A Position is valid if the line number is > 0.
//
type Position struct {
	Filename string;  // filename, if any
	Offset int;  // byte offset, starting at 0
	Line int;  // line number, starting at 1
	Column int;  // column number, starting at 1 (character count)
}

// Pos is an accessor method for anonymous Position fields.
// It returns its receiver.
//
func (pos *Position) Pos() Position {
	return *pos;
}

// IsValid returns true if the position is valid.
func (pos *Position) IsValid() bool {
	return pos.Line > 0
}

func (pos *Position) String() string {
	if pos != nil {
		s := pos.Filename;
		if pos.IsValid() {
			if s != "" {
				s += ":";
			}
			s += fmt.Sprintf("%d:%d", pos.Line, pos.Column);
		}
		if s != "" {
			return s;
		}
	}
	return "<unknown position>";
}
<|endoftext|>


<commit_before>package paxos

import (
	"doozer/store"
	"rand"
	"time"
)

const initialBound = 1e6 // ns == 1ms

type instance chan Packet

type clusterer interface {
	cluster(seqn int64) *cluster
}

func (it instance) process(seqn int64, cf clusterer, res chan<- store.Op) {
	var sched bool
	var waitBound int64 = initialBound
	cx := cf.cluster(seqn)

	co := coordinator{cx: cx, crnd: int64(cx.SelfIndex()), outs: cx}
	ac := acceptor{outs: cx}
	ln := *newLearner(int64(cx.Quorum()))
	var sk sink

	for p := range it {
		if p.Cmd() == tick {
			sched = false
		}

		p.SetFrom(cx.indexByAddr(p.Addr))
		co.Put(p.Msg)
		ac.Put(p.Msg)
		ln.Put(p.Msg)
		sk.Put(p.Msg)

		if co.seen > co.crnd && !sched {
			sched = true
			waitBound *= 2
			go func() {
				time.Sleep(rand.Int63n(waitBound))
				it.PutFrom("", msgTick)
			}()
		}

		if sk.done {
			cx.Put(newLearn(sk.v))
			res <- store.Op{seqn, sk.v}
		}
		if ln.done {
			cx.Put(newLearn(ln.v))
			res <- store.Op{seqn, ln.v}
		}
	}
}

func (it instance) PutFrom(addr string, m Msg) {
	// TODO try eliminating the goroutine
	go func() {
		it <- Packet{m, addr}
	}()
}

func (ins instance) Propose(v string) {
	// The from address doesn't matter.
	ins.PutFrom("", newPropose(v))
}
<commit_msg>don't repeat a learn message<commit_after>package paxos

import (
	"doozer/store"
	"rand"
	"time"
)

const initialBound = 1e6 // ns == 1ms

type instance chan Packet

type clusterer interface {
	cluster(seqn int64) *cluster
}

func (it instance) process(seqn int64, cf clusterer, res chan<- store.Op) {
	var sched bool
	var waitBound int64 = initialBound
	cx := cf.cluster(seqn)

	co := coordinator{cx: cx, crnd: int64(cx.SelfIndex()), outs: cx}
	ac := acceptor{outs: cx}
	ln := *newLearner(int64(cx.Quorum()))
	var sk sink

	for p := range it {
		if p.Cmd() == tick {
			sched = false
		}

		p.SetFrom(cx.indexByAddr(p.Addr))
		co.Put(p.Msg)
		ac.Put(p.Msg)
		ln.Put(p.Msg)
		sk.Put(p.Msg)

		if co.seen > co.crnd && !sched {
			sched = true
			waitBound *= 2
			go func() {
				time.Sleep(rand.Int63n(waitBound))
				it.PutFrom("", msgTick)
			}()
		}

		if sk.done {
			res <- store.Op{seqn, sk.v}
		}
		if ln.done {
			cx.Put(newLearn(ln.v))
			res <- store.Op{seqn, ln.v}
		}
	}
}

func (it instance) PutFrom(addr string, m Msg) {
	// TODO try eliminating the goroutine
	go func() {
		it <- Packet{m, addr}
	}()
}

func (ins instance) Propose(v string) {
	// The from address doesn't matter.
	ins.PutFrom("", newPropose(v))
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"io"
	"os"
	"strings"
	"text/template"

	"github.com/Masterminds/squirrel"
	"github.com/codegangsta/cli"

	_ "github.com/lib/pq"
)

const version = "DEV"

const Usage = `Read a schema and generate Structable structs.

This utility generates Structable structs be reading your database table and
generating the appropriate code.
`

const fileHeader = `package %s

// This file is automatically generated by schema2struct.

import (
	"time"

	"github.com/Masterminds/squirrel"
	"github.com/Masterminds/structable"
	_ "github.com/lib/pq"
)

`

const structTemplate = `// {{.StructName}} maps to database table {{.TableName}}
type {{.StructName}} struct {
	tableName string {{ann "tablename" .TableName}}
	structable.Recorder
	builder squirrel.StatementBuilderType
	{{range .Fields}}{{.}}
	{{end}}
}

// New{{.StructName}} creates a new {{.StructName}} wired to structable.
func New{{.StructName}}(db squirrel.DBProxyBeginner, flavor string) *{{.StructName}} {
	o := new({{.StructName}})
	o.Recorder = structable.New(db, flavor).Bind("{{.TableName}}", o)
	return o
}

// List lists the {{.StructName}} items found in the table.
//
// limit is the max number of items to return, offset is the offset (for paging).
func (v {{.StructName}}) List(limit, offset uint64) ([]*{{.StructName}}, error) {
	o, err := structable.List(v, limit, offset)
	if err != nil {
		return []*{{.StructName}}{}, err
	}

	res := make([]*{{.StructName}}, len(o))
	for i, obj := range o {
		res[i] = obj.(*{{.StructName}})
	}
	return res, nil
}

// List{{.StructName}} returns a list of {{.StructName}} objects.
//
// Limit is the max number of items. Offset is the offset the results will
// begin with.
func List{{.StructName}}(db squirrel.DBProxyBeginner, flavor string, limit, offset uint64) ([]*{{.StructName}}, error) {
	var tn string = "{{.TableName}}"

	// We need a prototype structable to learn about the table structure.
	ps := New{{.StructName}}(db, flavor)
	cols := ps.Columns(true)

	q := ps.Builder().Select(cols...).From(tn).Limit(limit).Offset(offset)
	rows, err := q.Query()
	if err != nil || rows == nil {
		return []*{{.StructName}}{}, err
	}
	defer rows.Close()

	buf := []*{{.StructName}}{}
	for rows.Next() {
		o := New{{.StructName}}(db, flavor)
		dest := o.FieldReferences(false)
		if err := rows.Scan(dest...); err != nil {
			return buf, err
		}
		buf = append(buf, o)
	}

	return buf, rows.Err()
}

`

type structDesc struct {
	StructName string
	TableName  string
	Fields     []string
}

func main() {
	app := cli.NewApp()
	app.Name = "schema2struct"
	app.Version = "version"
	app.Usage = Usage
	app.Action = importTables
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "driver,d",
			Value: "postgres",
			Usage: "The name of the SQL driver to use.",
		},
		cli.StringFlag{
			Name:  "connection,c",
			Value: "user=$USER dbname=$USER sslmode=disable",
			Usage: "The database connection string. Environment variables are expanded.",
		},
		cli.StringFlag{
			Name:  "tables,t",
			Value: "",
			Usage: "The list of tables to generate, comma separated. If none specified, the entire schema is used.",
		},
		cli.StringFlag{
			Name:  "file,f",
			Value: "",
			Usage: "The file to send the output.",
		},
		cli.StringFlag{
			Name:   "package,p",
			Value:  "main",
			Usage:  "The name of the destination package.",
			EnvVar: "GOPACKAGE",
		},
	}

	app.Run(os.Args)
}

func driver(c *cli.Context) string {
	return c.String("driver")
}
func conn(c *cli.Context) string {
	return os.ExpandEnv(c.String("connection"))
}

// dest gets the destination output writer.
func dest(c *cli.Context) io.Writer {
	if out := c.String("file"); out != "" {
		f, err := os.Create(out)
		if err != nil {
			panic(f)
		}
		return f
	}
	return os.Stdout
}

func tableList(c *cli.Context) []string {
	z := c.String("tables")
	if z != "" {
		return strings.Split(z, ",")
	}
	return []string{}
}

func cxdie(c *cli.Context, err error) {
	fmt.Fprintf(os.Stderr, "Failed to connect to %s (type %s): %s", conn(c), driver(c), err)
	os.Exit(1)
}

var funcMap = map[string]interface{}{
	"ann": func(tag, val string) string {
		return fmt.Sprintf("`%s:\"%s\"`", tag, val)
	},
}

func importTables(c *cli.Context) {
	ttt := template.Must(template.New("st").Funcs(funcMap).Parse(structTemplate))
	cxn, err := sql.Open(driver(c), conn(c))
	if err != nil {
		cxdie(c, err)
	}
	// Many drivers defer connections until the first statement. We test
	// that here.
	if err := cxn.Ping(); err != nil {
		cxdie(c, err)
	}
	defer cxn.Close()

	// Set up Squirrel
	stmts := squirrel.NewStmtCacher(cxn)
	bldr := squirrel.StatementBuilder.RunWith(stmts)
	if driver(c) == "postgres" {
		bldr = bldr.PlaceholderFormat(squirrel.Dollar)
	}

	// Set up destination
	out := dest(c)
	fmt.Fprintf(out, fileHeader, c.String("package"))

	tables := tableList(c)

	if len(tables) == 0 {
		tables, err = publicTables(bldr)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot fetch list of tables: %s\n", err)
			os.Exit(2)
		}
	}

	for _, t := range tables {
		f, err := importTable(t, bldr)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to import table %s: %s", t, err)
		}

		//fmt.Fprintf(out, "%s %s %s\n", f.StructName, f.TableName, f.Fields)
		ttt.Execute(out, f)
	}
}

type column struct {
	Name, DataType string
	Max            int64
}

func publicTables(b squirrel.StatementBuilderType) ([]string, error) {
	rows, err := b.Select("table_name").From("INFORMATION_SCHEMA.TABLES").
		Where("table_schema = 'public'").Query()

	res := []string{}
	if err != nil {
		return res, err
	}

	for rows.Next() {
		var s string
		rows.Scan(&s)
		res = append(res, s)
	}

	return res, nil
}

// importTable reads a table definition and writes a corresponding struct.
// SELECT table_name, column_name, data_type, character_maximum_length
//   FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = 'goose_db_version'
func importTable(tbl string, b squirrel.StatementBuilderType) (*structDesc, error) {

	pks, err := primaryKeyField(tbl, b)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting primary keys: %s", err)
	}

	q := b.Select("column_name, data_type, character_maximum_length").
		From("INFORMATION_SCHEMA.COLUMNS").
		Where("table_name = ?", tbl)

	rows, err := q.Query()
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	ff := []string{}
	for rows.Next() {
		c := &column{}
		var length sql.NullInt64
		if err := rows.Scan(&c.Name, &c.DataType, &length); err != nil {
			return nil, err
		}
		c.Max = length.Int64
		ff = append(ff, structField(c, pks, tbl, b))
	}
	sd := &structDesc{
		StructName: goName(tbl),
		TableName:  tbl,
		Fields:     ff,
	}

	return sd, nil
}

func primaryKeyField(tbl string, b squirrel.StatementBuilderType) ([]string, error) {
	q := b.Select("column_name").
		From("INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS c").
		LeftJoin("INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS t USING(constraint_name)").
		Where("t.table_name = ? AND t.constraint_type = 'PRIMARY KEY'", tbl).
		OrderBy("ordinal_position")

	rows, err := q.Query()
	if err != nil {
		return []string{}, err
	}

	res := []string{}
	for rows.Next() {
		var s string
		rows.Scan(&s)
		res = append(res, s)
	}
	return res, nil
}

func sequentialKey(tbl, pk string, b squirrel.StatementBuilderType) bool {

	tlen := 58

	stbl := tbl
	if len(tbl) > 29 {
		stbl = tbl[0:29]
	}

	left := tlen - len(stbl)
	spk := pk
	if len(pk) > left {
		spk = pk[0:left]
	}
	seq := fmt.Sprintf("%s_%s_seq", stbl, spk)

	q := b.Select("COUNT(*)").
		From("INFORMATION_SCHEMA.SEQUENCES").
		Where("sequence_name = ?", seq)

	var num int
	if err := q.Scan(&num); err != nil {
		panic(err)
	}
	return num > 0
}

func structField(c *column, pks []string, tbl string, b squirrel.StatementBuilderType) string {
	tpl := "%s %s `stbl:\"%s\"`"
	gn := destutter(goName(c.Name), goName(tbl))
	tt := goType(c.DataType)

	tag := c.Name
	for _, p := range pks {
		if c.Name == p {
			tag += ",PRIMARY_KEY"
			if sequentialKey(tbl, c.Name, b) {
				tag += ",SERIAL"
			}
		}
	}

	return fmt.Sprintf(tpl, gn, tt, tag)
}

// goType takes a SQL type and returns a string containin the name of a Go type.
//
// The goal is not to provide an exact match for every type, but to provide a
// safe Go representation of a SQL type.
//
// For some floating point SQL types, for example, we store them as strings
// so as not to lose precision while also not adding new types.
//
// The default type is string.
func goType(sqlType string) string {
	switch sqlType {
	case "smallint", "smallserial":
		return "int16"
	case "integer", "serial":
		return "int32"
	case "bigint", "bigserial":
		return "int"
	case "real":
		return "float32"
	case "double precision":
		return "float64"
	// Because we need to preserve base-10 precision.
	case "money":
		return "string"
	case "text", "varchar", "char", "character", "character varying", "uuid":
		return "string"
	case "bytea":
		return "[]byte"
	case "boolean":
		return "bool"
	case "timezone", "timezonetz", "date", "time":
		return "time.Time"
	case "interval":
		return "time.Duration"
	}
	return "string"
}

// Convert a SQL name to a Go name.
func goName(sqlName string) string {
	// This can definitely be done better.
	goName := strings.Replace(sqlName, "_", " ", -1)
	goName = strings.Replace(goName, ".", " ", -1)
	goName = strings.Title(goName)
	goName = strings.Replace(goName, " ", "", -1)

	return goName
}

// destutter removes a stutter prefix.
func destutter(str, prefix string) string {
	return strings.TrimPrefix(str, prefix)
}
<commit_msg>Add Len and better List functions to schema2struct.<commit_after>package main

import (
	"database/sql"
	"fmt"
	"io"
	"os"
	"strings"
	"text/template"

	"github.com/Masterminds/squirrel"
	"github.com/codegangsta/cli"

	_ "github.com/lib/pq"
)

const version = "DEV"

const Usage = `Read a schema and generate Structable structs.

This utility generates Structable structs be reading your database table and
generating the appropriate code.
`

const fileHeader = `package %s

// This file is automatically generated by schema2struct.

import (
	"time"

	"github.com/Masterminds/squirrel"
	"github.com/Masterminds/structable"
	_ "github.com/lib/pq"
)

// QueryFunc modifies a SelectBuilder prior to execution.
//
// The SelectBuilder is modified in place. An error is returned under any
// conditions where the query should not be executed.
type QueryFunc func(q squirrel.SelectBuilder) (squirrel.SelectBuilder, error)

`

const structTemplate = `// {{.StructName}} maps to database table {{.TableName}}
type {{.StructName}} struct {
	tableName string {{ann "tablename" .TableName}}
	structable.Recorder
	builder squirrel.StatementBuilderType
	{{range .Fields}}{{.}}
	{{end}}
}

// New{{.StructName}} creates a new {{.StructName}} wired to structable.
func New{{.StructName}}(db squirrel.DBProxyBeginner, flavor string) *{{.StructName}} {
	o := new({{.StructName}})
	o.Recorder = structable.New(db, flavor).Bind("{{.TableName}}", o)
	return o
}

// List{{.StructName}} returns a list of {{.StructName}} objects.
//
// Limit is the max number of items. Offset is the offset the results will
// begin with.
func List{{.StructName}}(db squirrel.DBProxyBeginner, flavor string, limit, offset uint64) ([]*{{.StructName}}, error) {
	fn := func(q squirrel.SelectBuilder) (squirrel.SelectBuilder, error) {
		return q.Limit(limit).Offset(offset), nil
	}
	return Query{{.StructName}}(db, flavor, fn)
}

// Query{{.StructName}} builds a base query, but allows the query to be modified before execution.
//
// This creates a new Select, settings the columns and table name, and then calling QueryFunc with the
// query. The QueryFunc can then add a Where clause, etc. Provided QueryFunc does not exit with an
// error, Query{{.StructName}} will then execute the query, extract the results into a slice of
// {{.StructName}} structs, and then return.
//
// The QueryFunc should not modify the list of fields returned or the table name,
// as the intent is to construct a complete {{.StructName}} from each result.
// More sophisticated queries should be written directly.
func Query{{.StructName}}(db squirrel.DBProxyBeginner, flavor string, fn QueryFunc) ([]*{{.StructName}}, error){
	var tn string = "{{.TableName}}"

	// We need a prototype structable to learn about the table structure.
	ps := New{{.StructName}}(db, flavor)
	cols := ps.Columns(true)

	q := ps.Builder().Select(cols...).From(tn)
	var err error
	if q, err = fn(q); err != nil {
		return []*{{.StructName}}{}, err
	}
	rows, err := q.Query()
	if err != nil || rows == nil {
		return []*{{.StructName}}{}, err
	}
	defer rows.Close()

	buf := []*{{.StructName}}{}
	for rows.Next() {
		o := New{{.StructName}}(db, flavor)
		dest := o.FieldReferences(true)
		if err := rows.Scan(dest...); err != nil {
			return buf, err
		}
		buf = append(buf, o)
	}
	return buf, rows.Err()
}

// Len{{.StructName}} returns the number of {{.StructName}} objects in the database.
func Len{{.StructName}}(db squirrel.DBProxyBeginner, flavor string) (int, error) {
	fn := func(q squirrel.SelectBuilder) (squirrel.SelectBuilder, error) {return q, nil}
	return QueryLen{{.StructName}}(db, flavor, fn)
}

// QueryLen{{.StructName}} returns the length of a table.
//
// The QueryFunc can be used to modify the query. For a simple length call, you
// may prefer to use Len{{.StructName}}.
func QueryLen{{.StructName}}(db squirrel.DBProxyBeginner, flavor string, fn QueryFunc) (int, error) {
	tn := "{{.TableName}}"
	ps := New{{.StructName}}(db, flavor)
	q := ps.Builder().Select("COUNT(*)").From(tn)
	var err error
	if q, err = fn(q); err != nil {
		return 0, err
	}
	var count int
	err = q.Scan(&count)
	return count, err
}

`

type structDesc struct {
	StructName string
	TableName  string
	Fields     []string
}

func main() {
	app := cli.NewApp()
	app.Name = "schema2struct"
	app.Version = "version"
	app.Usage = Usage
	app.Action = importTables
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "driver,d",
			Value: "postgres",
			Usage: "The name of the SQL driver to use.",
		},
		cli.StringFlag{
			Name:  "connection,c",
			Value: "user=$USER dbname=$USER sslmode=disable",
			Usage: "The database connection string. Environment variables are expanded.",
		},
		cli.StringFlag{
			Name:  "tables,t",
			Value: "",
			Usage: "The list of tables to generate, comma separated. If none specified, the entire schema is used.",
		},
		cli.StringFlag{
			Name:  "file,f",
			Value: "",
			Usage: "The file to send the output.",
		},
		cli.StringFlag{
			Name:   "package,p",
			Value:  "main",
			Usage:  "The name of the destination package.",
			EnvVar: "GOPACKAGE",
		},
	}

	app.Run(os.Args)
}

func driver(c *cli.Context) string {
	return c.String("driver")
}
func conn(c *cli.Context) string {
	return os.ExpandEnv(c.String("connection"))
}

// dest gets the destination output writer.
func dest(c *cli.Context) io.Writer {
	if out := c.String("file"); out != "" {
		f, err := os.Create(out)
		if err != nil {
			panic(f)
		}
		return f
	}
	return os.Stdout
}

func tableList(c *cli.Context) []string {
	z := c.String("tables")
	if z != "" {
		return strings.Split(z, ",")
	}
	return []string{}
}

func cxdie(c *cli.Context, err error) {
	fmt.Fprintf(os.Stderr, "Failed to connect to %s (type %s): %s", conn(c), driver(c), err)
	os.Exit(1)
}

var funcMap = map[string]interface{}{
	"ann": func(tag, val string) string {
		return fmt.Sprintf("`%s:\"%s\"`", tag, val)
	},
}

func importTables(c *cli.Context) {
	ttt := template.Must(template.New("st").Funcs(funcMap).Parse(structTemplate))
	cxn, err := sql.Open(driver(c), conn(c))
	if err != nil {
		cxdie(c, err)
	}
	// Many drivers defer connections until the first statement. We test
	// that here.
	if err := cxn.Ping(); err != nil {
		cxdie(c, err)
	}
	defer cxn.Close()

	// Set up Squirrel
	stmts := squirrel.NewStmtCacher(cxn)
	bldr := squirrel.StatementBuilder.RunWith(stmts)
	if driver(c) == "postgres" {
		bldr = bldr.PlaceholderFormat(squirrel.Dollar)
	}

	// Set up destination
	out := dest(c)
	fmt.Fprintf(out, fileHeader, c.String("package"))

	tables := tableList(c)

	if len(tables) == 0 {
		tables, err = publicTables(bldr)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot fetch list of tables: %s\n", err)
			os.Exit(2)
		}
	}

	for _, t := range tables {
		f, err := importTable(t, bldr)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to import table %s: %s", t, err)
		}

		//fmt.Fprintf(out, "%s %s %s\n", f.StructName, f.TableName, f.Fields)
		ttt.Execute(out, f)
	}
}

type column struct {
	Name, DataType string
	Max            int64
}

func publicTables(b squirrel.StatementBuilderType) ([]string, error) {
	rows, err := b.Select("table_name").From("INFORMATION_SCHEMA.TABLES").
		Where("table_schema = 'public'").Query()

	res := []string{}
	if err != nil {
		return res, err
	}

	for rows.Next() {
		var s string
		rows.Scan(&s)
		res = append(res, s)
	}

	return res, nil
}

// importTable reads a table definition and writes a corresponding struct.
// SELECT table_name, column_name, data_type, character_maximum_length
//   FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = 'goose_db_version'
func importTable(tbl string, b squirrel.StatementBuilderType) (*structDesc, error) {

	pks, err := primaryKeyField(tbl, b)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting primary keys: %s", err)
	}

	q := b.Select("column_name, data_type, character_maximum_length").
		From("INFORMATION_SCHEMA.COLUMNS").
		Where("table_name = ?", tbl)

	rows, err := q.Query()
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	ff := []string{}
	for rows.Next() {
		c := &column{}
		var length sql.NullInt64
		if err := rows.Scan(&c.Name, &c.DataType, &length); err != nil {
			return nil, err
		}
		c.Max = length.Int64
		ff = append(ff, structField(c, pks, tbl, b))
	}
	sd := &structDesc{
		StructName: goName(tbl),
		TableName:  tbl,
		Fields:     ff,
	}

	return sd, nil
}

func primaryKeyField(tbl string, b squirrel.StatementBuilderType) ([]string, error) {
	q := b.Select("column_name").
		From("INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS c").
		LeftJoin("INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS t USING(constraint_name)").
		Where("t.table_name = ? AND t.constraint_type = 'PRIMARY KEY'", tbl).
		OrderBy("ordinal_position")

	rows, err := q.Query()
	if err != nil {
		return []string{}, err
	}

	res := []string{}
	for rows.Next() {
		var s string
		rows.Scan(&s)
		res = append(res, s)
	}
	return res, nil
}

func sequentialKey(tbl, pk string, b squirrel.StatementBuilderType) bool {

	tlen := 58

	stbl := tbl
	if len(tbl) > 29 {
		stbl = tbl[0:29]
	}

	left := tlen - len(stbl)
	spk := pk
	if len(pk) > left {
		spk = pk[0:left]
	}
	seq := fmt.Sprintf("%s_%s_seq", stbl, spk)

	q := b.Select("COUNT(*)").
		From("INFORMATION_SCHEMA.SEQUENCES").
		Where("sequence_name = ?", seq)

	var num int
	if err := q.Scan(&num); err != nil {
		panic(err)
	}
	return num > 0
}

func structField(c *column, pks []string, tbl string, b squirrel.StatementBuilderType) string {
	tpl := "%s %s `stbl:\"%s\"`"
	gn := destutter(goName(c.Name), goName(tbl))
	tt := goType(c.DataType)

	tag := c.Name
	for _, p := range pks {
		if c.Name == p {
			tag += ",PRIMARY_KEY"
			if sequentialKey(tbl, c.Name, b) {
				tag += ",SERIAL"
			}
		}
	}

	return fmt.Sprintf(tpl, gn, tt, tag)
}

// goType takes a SQL type and returns a string containin the name of a Go type.
//
// The goal is not to provide an exact match for every type, but to provide a
// safe Go representation of a SQL type.
//
// For some floating point SQL types, for example, we store them as strings
// so as not to lose precision while also not adding new types.
//
// The default type is string.
func goType(sqlType string) string {
	switch sqlType {
	case "smallint", "smallserial":
		return "int16"
	case "integer", "serial":
		return "int32"
	case "bigint", "bigserial":
		return "int"
	case "real":
		return "float32"
	case "double precision":
		return "float64"
	// Because we need to preserve base-10 precision.
	case "money":
		return "string"
	case "text", "varchar", "char", "character", "character varying", "uuid":
		return "string"
	case "bytea":
		return "[]byte"
	case "boolean":
		return "bool"
	case "timezone", "timezonetz", "date", "time":
		return "time.Time"
	case "interval":
		return "time.Duration"
	}
	return "string"
}

// Convert a SQL name to a Go name.
func goName(sqlName string) string {
	// This can definitely be done better.
	goName := strings.Replace(sqlName, "_", " ", -1)
	goName = strings.Replace(goName, ".", " ", -1)
	goName = strings.Title(goName)
	goName = strings.Replace(goName, " ", "", -1)

	return goName
}

// destutter removes a stutter prefix.
func destutter(str, prefix string) string {
	return strings.TrimPrefix(str, prefix)
}
<|endoftext|>


<commit_before>// Alexandria
//
// Copyright (C) 2015  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package xelatex

import (
	"io"
	"log"
	"os/exec"
	"strconv"
	"strings"

	. "github.com/yzhs/alexandria-go"
	parser "github.com/yzhs/alexandria-go/parser"
)

type errorHandler struct {
	err error
}

func latexToPdf(id Id) error {
	// TODO return error message if the execution fails
	msg, err := exec.Command("rubber", "--module", "xelatex", "--force", "--into",
		Config.TempDirectory, Config.TempDirectory+string(id)+".tex").CombinedOutput()
	if err != nil {
		log.Fatal(string(msg))
		return err
	}
	return nil
}

func pdfToPng(i Id) error {
	id := string(i)
	return exec.Command("convert", "-quality", strconv.Itoa(Config.Quality),
		"-density", strconv.Itoa(Config.Dpi), Config.TempDirectory+id+".pdf",
		Config.CacheDirectory+id+".png").Run()
}

func searchSwish(query []string) ([]Id, error) {
	tmp := append([]string{"-c", Config.SwishConfig}, query...)
	cmd := exec.Command("search++", tmp...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	cmd.Start()

	buffer := make([]byte, 65536)
	bytesRead, _ := io.ReadFull(stdout, buffer)
	cmd.Wait()
	stdout.Close()

	output := strings.Split(string(buffer[:bytesRead]), "\n")
	//num, _ := strconv.Atoi(strings.TrimPrefix(output[0], "# results: "))

	result := make([]Id, 100)
	i := 0
	for _, line := range output {
		if line == "" || strings.HasPrefix(line, "# ") {
			continue
		}
		fields := strings.Fields(line)
		result[i] = Id(strings.TrimSuffix(fields[len(fields)-1], ".tex"))
		i++
	}
	result = result[:i]

	return result, nil
}

func render(id Id) (string, error) {
	doc, err := readTemplate("header")
	if err != nil {
		log.Fatal(err)
		return "", err
	}

	scroll, err := readScroll(id)
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	tags := parser.ParseTags(scroll)

	temp, err := readTemplate(parser.DocumentType(tags) + "_header")
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	doc += temp + scroll

	temp, err = readTemplate(parser.DocumentType(tags) + "_footer")
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	doc += temp

	temp, err = readTemplate("footer")
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	doc += temp

	return doc, nil
}

func processScroll(id Id) error {
	if isUpToDate(id) {
		return nil
	}

	temp, err := render(id)
	if err != nil {
		return err
	}

	err = writeTemp(id, temp)
	if err != nil {
		return err
	}

	err = latexToPdf(id)
	if err != nil {
		return err
	}

	err = pdfToPng(id)
	if err != nil {
		return err
	}

	return nil
}

func processAllScrolls(ids []Id) {
	for _, id := range ids {
		err := processScroll(id)
		if err != nil {
			log.Panic("An error ocurred when processing scroll ", id, ": ", err)
		}
	}
}

func FindScrolls(query []string) ([]Id, error) {
	ids, err := searchSwish(query)
	if err != nil {
		return nil, err
	}
	processAllScrolls(ids)

	return ids, nil
}
<commit_msg>Tell swish++ to show at most Config.MaxResults results<commit_after>// Alexandria
//
// Copyright (C) 2015  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package xelatex

import (
	"io"
	"log"
	"os/exec"
	"strconv"
	"strings"

	. "github.com/yzhs/alexandria-go"
	parser "github.com/yzhs/alexandria-go/parser"
)

type errorHandler struct {
	err error
}

func latexToPdf(id Id) error {
	// TODO return error message if the execution fails
	msg, err := exec.Command("rubber", "--module", "xelatex", "--force", "--into",
		Config.TempDirectory, Config.TempDirectory+string(id)+".tex").CombinedOutput()
	if err != nil {
		log.Fatal(string(msg))
		return err
	}
	return nil
}

func pdfToPng(i Id) error {
	id := string(i)
	return exec.Command("convert", "-quality", strconv.Itoa(Config.Quality),
		"-density", strconv.Itoa(Config.Dpi), Config.TempDirectory+id+".pdf",
		Config.CacheDirectory+id+".png").Run()
}

func searchSwish(query []string) ([]Id, error) {
	tmp := append([]string{"-c", Config.SwishConfig, "--max-results=" + strconv.Itoa(Config.MaxResults)}, query...)
	cmd := exec.Command("search++", tmp...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	cmd.Start()

	buffer := make([]byte, 65536)
	bytesRead, _ := io.ReadFull(stdout, buffer)
	cmd.Wait()
	stdout.Close()

	output := strings.Split(string(buffer[:bytesRead]), "\n")
	//num, _ := strconv.Atoi(strings.TrimPrefix(output[0], "# results: "))

	result := make([]Id, Config.MaxResults)
	i := 0
	for _, line := range output {
		if line == "" || strings.HasPrefix(line, "# ") {
			continue
		}
		fields := strings.Fields(line)
		result[i] = Id(strings.TrimSuffix(fields[len(fields)-1], ".tex"))
		i++
	}
	result = result[:i]

	return result, nil
}

func render(id Id) (string, error) {
	doc, err := readTemplate("header")
	if err != nil {
		log.Fatal(err)
		return "", err
	}

	scroll, err := readScroll(id)
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	tags := parser.ParseTags(scroll)

	temp, err := readTemplate(parser.DocumentType(tags) + "_header")
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	doc += temp + scroll

	temp, err = readTemplate(parser.DocumentType(tags) + "_footer")
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	doc += temp

	temp, err = readTemplate("footer")
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	doc += temp

	return doc, nil
}

func processScroll(id Id) error {
	if isUpToDate(id) {
		return nil
	}

	temp, err := render(id)
	if err != nil {
		return err
	}

	err = writeTemp(id, temp)
	if err != nil {
		return err
	}

	err = latexToPdf(id)
	if err != nil {
		return err
	}

	err = pdfToPng(id)
	if err != nil {
		return err
	}

	return nil
}

func processAllScrolls(ids []Id) {
	for _, id := range ids {
		err := processScroll(id)
		if err != nil {
			log.Panic("An error ocurred when processing scroll ", id, ": ", err)
		}
	}
}

func FindScrolls(query []string) ([]Id, error) {
	ids, err := searchSwish(query)
	if err != nil {
		return nil, err
	}
	processAllScrolls(ids)

	return ids, nil
}
<|endoftext|>