<commit_before>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nftables

import (
	"math"

	"github.com/google/nftables/binaryutil"
	"github.com/mdlayher/netlink"
	"golang.org/x/sys/unix"
)

// ChainHook specifies at which step in packet processing the Chain should be
// executed. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Base_chain_hooks
type ChainHook uint32

// Possible ChainHook values.
const (
	ChainHookPrerouting  ChainHook = unix.NF_INET_PRE_ROUTING
	ChainHookInput       ChainHook = unix.NF_INET_LOCAL_IN
	ChainHookForward     ChainHook = unix.NF_INET_FORWARD
	ChainHookOutput      ChainHook = unix.NF_INET_LOCAL_OUT
	ChainHookPostrouting ChainHook = unix.NF_INET_POST_ROUTING
	ChainHookIngress     ChainHook = unix.NF_NETDEV_INGRESS
)

// ChainPriority orders the chain relative to Netfilter internal operations. See
// also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Base_chain_priority
type ChainPriority int32

// Possible ChainPriority values.
const ( // from /usr/include/linux/netfilter_ipv4.h
	ChainPriorityFirst            ChainPriority = math.MinInt32
	ChainPriorityConntrackDefrag  ChainPriority = -400
	ChainPriorityRaw              ChainPriority = -300
	ChainPrioritySELinuxFirst     ChainPriority = -225
	ChainPriorityConntrack        ChainPriority = -200
	ChainPriorityMangle           ChainPriority = -150
	ChainPriorityNATDest          ChainPriority = -100
	ChainPriorityFilter           ChainPriority = 0
	ChainPrioritySecurity         ChainPriority = 50
	ChainPriorityNATSource        ChainPriority = 100
	ChainPrioritySELinuxLast      ChainPriority = 225
	ChainPriorityConntrackHelper  ChainPriority = 300
	ChainPriorityConntrackConfirm ChainPriority = math.MaxInt32
	ChainPriorityLast             ChainPriority = math.MaxInt32
)

// ChainType defines what this chain will be used for. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Base_chain_types
type ChainType string

// Possible ChainType values.
const (
	ChainTypeFilter ChainType = "filter"
	ChainTypeRoute  ChainType = "route"
	ChainTypeNAT    ChainType = "nat"
)

// A Chain contains Rules. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains
type Chain struct {
	Name     string
	Table    *Table
	Hooknum  ChainHook
	Priority ChainPriority
	Type     ChainType
}

// AddChain adds the specified Chain. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Adding_base_chains
func (cc *Conn) AddChain(c *Chain) *Chain {
	chainHook := cc.marshalAttr([]netlink.Attribute{
		{Type: unix.NFTA_HOOK_HOOKNUM, Data: binaryutil.BigEndian.PutUint32(uint32(c.Hooknum))},
		{Type: unix.NFTA_HOOK_PRIORITY, Data: binaryutil.BigEndian.PutUint32(uint32(c.Priority))},
	})

	data := cc.marshalAttr([]netlink.Attribute{
		{Type: unix.NFTA_CHAIN_TABLE, Data: []byte(c.Table.Name + "\x00")},
		{Type: unix.NFTA_CHAIN_NAME, Data: []byte(c.Name + "\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_CHAIN_HOOK, Data: chainHook},
		{Type: unix.NFTA_CHAIN_TYPE, Data: []byte(c.Type + "\x00")},
	})

	cc.messages = append(cc.messages, netlink.Message{
		Header: netlink.Header{
			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWCHAIN),
			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
		},
		Data: append(extraHeader(uint8(c.Table.Family), 0), data...),
	})

	return c
}
<commit_msg>List chains (#25)<commit_after>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nftables

import (
	"encoding/binary"
	"fmt"
	"math"

	"github.com/google/nftables/binaryutil"
	"github.com/mdlayher/netlink"
	"golang.org/x/sys/unix"
)

// ChainHook specifies at which step in packet processing the Chain should be
// executed. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Base_chain_hooks
type ChainHook uint32

// Possible ChainHook values.
const (
	ChainHookPrerouting  ChainHook = unix.NF_INET_PRE_ROUTING
	ChainHookInput       ChainHook = unix.NF_INET_LOCAL_IN
	ChainHookForward     ChainHook = unix.NF_INET_FORWARD
	ChainHookOutput      ChainHook = unix.NF_INET_LOCAL_OUT
	ChainHookPostrouting ChainHook = unix.NF_INET_POST_ROUTING
	ChainHookIngress     ChainHook = unix.NF_NETDEV_INGRESS
)

// ChainPriority orders the chain relative to Netfilter internal operations. See
// also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Base_chain_priority
type ChainPriority int32

// Possible ChainPriority values.
const ( // from /usr/include/linux/netfilter_ipv4.h
	ChainPriorityFirst            ChainPriority = math.MinInt32
	ChainPriorityConntrackDefrag  ChainPriority = -400
	ChainPriorityRaw              ChainPriority = -300
	ChainPrioritySELinuxFirst     ChainPriority = -225
	ChainPriorityConntrack        ChainPriority = -200
	ChainPriorityMangle           ChainPriority = -150
	ChainPriorityNATDest          ChainPriority = -100
	ChainPriorityFilter           ChainPriority = 0
	ChainPrioritySecurity         ChainPriority = 50
	ChainPriorityNATSource        ChainPriority = 100
	ChainPrioritySELinuxLast      ChainPriority = 225
	ChainPriorityConntrackHelper  ChainPriority = 300
	ChainPriorityConntrackConfirm ChainPriority = math.MaxInt32
	ChainPriorityLast             ChainPriority = math.MaxInt32
)

// ChainType defines what this chain will be used for. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Base_chain_types
type ChainType string

// Possible ChainType values.
const (
	ChainTypeFilter ChainType = "filter"
	ChainTypeRoute  ChainType = "route"
	ChainTypeNAT    ChainType = "nat"
)

// A Chain contains Rules. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains
type Chain struct {
	Name     string
	Table    *Table
	Hooknum  ChainHook
	Priority ChainPriority
	Type     ChainType
}

// AddChain adds the specified Chain. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Configuring_chains#Adding_base_chains
func (cc *Conn) AddChain(c *Chain) *Chain {
	chainHook := cc.marshalAttr([]netlink.Attribute{
		{Type: unix.NFTA_HOOK_HOOKNUM, Data: binaryutil.BigEndian.PutUint32(uint32(c.Hooknum))},
		{Type: unix.NFTA_HOOK_PRIORITY, Data: binaryutil.BigEndian.PutUint32(uint32(c.Priority))},
	})

	data := cc.marshalAttr([]netlink.Attribute{
		{Type: unix.NFTA_CHAIN_TABLE, Data: []byte(c.Table.Name + "\x00")},
		{Type: unix.NFTA_CHAIN_NAME, Data: []byte(c.Name + "\x00")},
		{Type: unix.NLA_F_NESTED | unix.NFTA_CHAIN_HOOK, Data: chainHook},
		{Type: unix.NFTA_CHAIN_TYPE, Data: []byte(c.Type + "\x00")},
	})

	cc.messages = append(cc.messages, netlink.Message{
		Header: netlink.Header{
			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWCHAIN),
			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
		},
		Data: append(extraHeader(uint8(c.Table.Family), 0), data...),
	})

	return c
}

// ListChains returns currently configured chains in the kernel
func (cc *Conn) ListChains() ([]*Chain, error) {
	conn, err := cc.dialNetlink()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	msg := netlink.Message{
		Header: netlink.Header{
			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_GETCHAIN),
			Flags: netlink.Request | netlink.Dump,
		},
		Data: extraHeader(uint8(unix.AF_UNSPEC), 0),
	}

	response, err := conn.Execute(msg)
	if err != nil {
		return nil, err
	}

	var chains []*Chain
	for _, m := range response {
		c, err := chainFromMsg(m)
		if err != nil {
			return nil, err
		}

		chains = append(chains, c)
	}

	return chains, nil
}

func chainFromMsg(msg netlink.Message) (*Chain, error) {
	chainHeaderType := netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWCHAIN)
	if got, want := msg.Header.Type, chainHeaderType; got != want {
		return nil, fmt.Errorf("unexpected header type: got %v, want %v", got, want)
	}

	var c Chain

	ad, err := netlink.NewAttributeDecoder(msg.Data[4:])
	if err != nil {
		return nil, err
	}

	for ad.Next() {
		switch ad.Type() {
		case unix.NFTA_CHAIN_NAME:
			c.Name = ad.String()
		case unix.NFTA_TABLE_NAME:
			c.Table = &Table{Name: ad.String()}
		case unix.NFTA_CHAIN_TYPE:
			c.Type = ChainType(ad.String())
		case unix.NFTA_CHAIN_HOOK:
			ad.Do(func(b []byte) error {
				c.Hooknum, c.Priority, err = hookFromMsg(b)
				return err
			})
		}

	}

	return &c, nil
}

func hookFromMsg(b []byte) (ChainHook, ChainPriority, error) {
	ad, err := netlink.NewAttributeDecoder(b)
	if err != nil {
		return 0, 0, err
	}

	ad.ByteOrder = binary.BigEndian

	var hooknum ChainHook
	var prio ChainPriority

	for ad.Next() {
		switch ad.Type() {
		case unix.NFTA_HOOK_HOOKNUM:
			hooknum = ChainHook(ad.Uint32())
		case unix.NFTA_HOOK_PRIORITY:
			prio = ChainPriority(ad.Uint32())
		}
	}

	return hooknum, prio, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/samuel/go-gettext/gettext"
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
)

// page model
type Page struct {
	IsTor    bool
	UpToDate bool
	NotSmall bool
	OnOff    string
	Lang     string
	IP       string
	Extra    string
	Locales  map[string]string
}

type Port struct {
	min int
	max int
}

type Policy struct {
	accept bool
	ports  []Port
}

func (p Policy) CanExit(exitPort int) bool {
	for _, port := range p.ports {
		if port.min <= exitPort && exitPort <= port.max {
			return p.accept
		}
	}
	return !p.accept
}

var (

	// map the exit list
	// TODO: investigate other data structures
	ExitMap  map[string]Policy
	ExitLock = new(sync.RWMutex)

	// layout template
	Layout = template.New("")

	// public file server
	Phttp = http.NewServeMux()

	// locales map
	Locales = map[string]string{
		"ar":    "عربية (Arabiya)",
		"bms":   "Burmese",
		"cs":    "česky",
		"da":    "Dansk",
		"de":    "Deutsch",
		"el":    "Ελληνικά (Ellinika)",
		"en_US": "English",
		"es":    "Español",
		"et":    "Estonian",
		"fa_IR": "فارسی (Fārsī)",
		"fr":    "Français",
		"it_IT": "Italiano",
		"ja":    "日本語 (Nihongo)",
		"nb":    "Norsk (Bokmål)",
		"nl":    "Nederlands",
		"pl":    "Polski",
		"pt":    "Português",
		"pt_BR": "Português do Brasil",
		"ro":    "Română",
		"fi":    "Suomi",
		"ru":    "Русский (Russkij)",
		"th":    "Thai",
		"tr":    "Türkçe",
		"uk":    "українська (Ukrajins\"ka)",
		"vi":    "Vietnamese",
		"zh_CN": "中文(简)",
	}
)

func GetExits() map[string]Policy {
	ExitLock.RLock()
	defer ExitLock.RUnlock()
	return ExitMap
}

// load exit list
func LoadList() {

	file, err := os.Open("data/exit-policies")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	exits := make(map[string]Policy)
	scan := bufio.NewScanner(file)
	for scan.Scan() {
		strs := strings.Fields(scan.Text())
		if len(strs) > 0 {
			policy := Policy{}
			if strs[1] == "accept" {
				policy.accept = true
			}
			ports := strings.Split(strs[2], ",")
			for _, p := range ports {
				s := strings.Split(p, "-")
				min, err := strconv.Atoi(s[0])
				if err != nil {
					log.Fatal(err)
				}
				port := Port{
					min: min,
					max: min,
				}
				if len(s) > 1 {
					port.max, err = strconv.Atoi(s[1])
					if err != nil {
						log.Fatal(err)
					}
				}
				policy.ports = append(policy.ports, port)
			}
			exits[strs[0]] = policy
		}
	}

	if err = scan.Err(); err != nil {
		log.Fatal(err)
	}

	// swap in exits
	ExitLock.Lock()
	ExitMap = exits
	ExitLock.Unlock()

}

func IsTor(remoteAddr string) bool {
	if net.ParseIP(remoteAddr).To4() == nil {
		return false
	}
	return GetExits()[remoteAddr].CanExit(443)
}

func UpToDate(r *http.Request) bool {
	if r.URL.Query().Get("uptodate") == "0" {
		return false
	}
	return true
}

func Small(r *http.Request) bool {
	if len(r.URL.Query().Get("small")) > 0 {
		return true
	}
	return false
}

// determine which language to use. default to english
func Lang(r *http.Request) string {
	lang := r.URL.Query().Get("lang")
	if len(lang) == 0 {
		lang = "en_US"
	}
	return lang
}

func RootHandler(w http.ResponseWriter, r *http.Request) {

	// serve public files
	if len(r.URL.Path) > 1 {
		Phttp.ServeHTTP(w, r)
		return
	}

	// get remote ip
	host, _, _ := net.SplitHostPort(r.RemoteAddr)

	// determine if we're in Tor
	isTor := IsTor(host)

	// short circuit for torbutton
	if len(r.URL.Query().Get("TorButton")) > 0 {
		Layout.ExecuteTemplate(w, "torbutton.html", isTor)
		return
	}

	// string used for classes and such
	// in the template
	var onOff string
	if isTor {
		onOff = "on"
	} else {
		onOff = "off"
	}

	small := Small(r)
	upToDate := UpToDate(r)

	// querystring params
	extra := ""
	if small {
		extra += "&small=1"
	}
	if !upToDate {
		extra += "&uptodate=0"
	}

	// instance of your page model
	p := Page{
		isTor,
		isTor && !upToDate,
		!small,
		onOff,
		Lang(r),
		host,
		extra,
		Locales,
	}

	// render the template
	Layout.ExecuteTemplate(w, "index.html", p)

}

func main() {

	// determine which port to run on
	port := os.Getenv("PORT")
	if len(port) == 0 {
		port = "8000"
	}

	// load i18n
	domain, err := gettext.NewDomain("check", "locale")
	if err != nil {
		log.Fatal(err)
	}

	// add template funcs
	Layout = Layout.Funcs(template.FuncMap{
		"UnEscaped": func(x string) interface{} {
			return template.HTML(x)
		},
		"UnEscapedURL": func(x string) interface{} {
			return template.URL(x)
		},
		"GetText": func(lang string, text string) string {
			return domain.GetText(lang, text)
		},
	})

	// load layout
	Layout, err = Layout.ParseFiles(
		"public/index.html",
		"public/torbutton.html",
	)
	if err != nil {
		log.Fatal(err)
	}

	// load exits
	LoadList()

	// listen for signal to reload exits
	s := make(chan os.Signal, 1)
	signal.Notify(s, syscall.SIGUSR2)
	go func() {
		for {
			<-s
			LoadList()
			log.Println("Exit list reloaded.")
		}
	}()

	// routes
	http.HandleFunc("/", RootHandler)

	// files
	files := http.FileServer(http.Dir("./public"))
	Phttp.Handle("/torcheck/", http.StripPrefix("/torcheck/", files))
	Phttp.Handle("/", files)

	// start the server
	log.Printf("Listening on port: %s\n", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))

}
<commit_msg>default to false<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/samuel/go-gettext/gettext"
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
)

// page model
type Page struct {
	IsTor    bool
	UpToDate bool
	NotSmall bool
	OnOff    string
	Lang     string
	IP       string
	Extra    string
	Locales  map[string]string
}

type Port struct {
	min int
	max int
}

type Policy struct {
	accept bool
	ports  []Port
}

func (p Policy) CanExit(exitPort int) bool {
	if len(p.ports) == 0 {
		return false
	}
	for _, port := range p.ports {
		if port.min <= exitPort && exitPort <= port.max {
			return p.accept
		}
	}
	return !p.accept
}

var (

	// map the exit list
	// TODO: investigate other data structures
	ExitMap  map[string]Policy
	ExitLock = new(sync.RWMutex)

	// layout template
	Layout = template.New("")

	// public file server
	Phttp = http.NewServeMux()

	// locales map
	Locales = map[string]string{
		"ar":    "عربية (Arabiya)",
		"bms":   "Burmese",
		"cs":    "česky",
		"da":    "Dansk",
		"de":    "Deutsch",
		"el":    "Ελληνικά (Ellinika)",
		"en_US": "English",
		"es":    "Español",
		"et":    "Estonian",
		"fa_IR": "فارسی (Fārsī)",
		"fr":    "Français",
		"it_IT": "Italiano",
		"ja":    "日本語 (Nihongo)",
		"nb":    "Norsk (Bokmål)",
		"nl":    "Nederlands",
		"pl":    "Polski",
		"pt":    "Português",
		"pt_BR": "Português do Brasil",
		"ro":    "Română",
		"fi":    "Suomi",
		"ru":    "Русский (Russkij)",
		"th":    "Thai",
		"tr":    "Türkçe",
		"uk":    "українська (Ukrajins\"ka)",
		"vi":    "Vietnamese",
		"zh_CN": "中文(简)",
	}
)

func GetExits() map[string]Policy {
	ExitLock.RLock()
	defer ExitLock.RUnlock()
	return ExitMap
}

// load exit list
func LoadList() {

	file, err := os.Open("data/exit-policies")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	exits := make(map[string]Policy)
	scan := bufio.NewScanner(file)
	for scan.Scan() {
		strs := strings.Fields(scan.Text())
		if len(strs) > 0 {
			policy := Policy{}
			if strs[1] == "accept" {
				policy.accept = true
			}
			ports := strings.Split(strs[2], ",")
			for _, p := range ports {
				s := strings.Split(p, "-")
				min, err := strconv.Atoi(s[0])
				if err != nil {
					log.Fatal(err)
				}
				port := Port{
					min: min,
					max: min,
				}
				if len(s) > 1 {
					port.max, err = strconv.Atoi(s[1])
					if err != nil {
						log.Fatal(err)
					}
				}
				policy.ports = append(policy.ports, port)
			}
			exits[strs[0]] = policy
		}
	}

	if err = scan.Err(); err != nil {
		log.Fatal(err)
	}

	// swap in exits
	ExitLock.Lock()
	ExitMap = exits
	ExitLock.Unlock()

}

func IsTor(remoteAddr string) bool {
	if net.ParseIP(remoteAddr).To4() == nil {
		return false
	}
	return GetExits()[remoteAddr].CanExit(443)
}

func UpToDate(r *http.Request) bool {
	if r.URL.Query().Get("uptodate") == "0" {
		return false
	}
	return true
}

func Small(r *http.Request) bool {
	if len(r.URL.Query().Get("small")) > 0 {
		return true
	}
	return false
}

// determine which language to use. default to english
func Lang(r *http.Request) string {
	lang := r.URL.Query().Get("lang")
	if len(lang) == 0 {
		lang = "en_US"
	}
	return lang
}

func RootHandler(w http.ResponseWriter, r *http.Request) {

	// serve public files
	if len(r.URL.Path) > 1 {
		Phttp.ServeHTTP(w, r)
		return
	}

	// get remote ip
	host, _, _ := net.SplitHostPort(r.RemoteAddr)

	// determine if we're in Tor
	isTor := IsTor(host)

	// short circuit for torbutton
	if len(r.URL.Query().Get("TorButton")) > 0 {
		Layout.ExecuteTemplate(w, "torbutton.html", isTor)
		return
	}

	// string used for classes and such
	// in the template
	var onOff string
	if isTor {
		onOff = "on"
	} else {
		onOff = "off"
	}

	small := Small(r)
	upToDate := UpToDate(r)

	// querystring params
	extra := ""
	if small {
		extra += "&small=1"
	}
	if !upToDate {
		extra += "&uptodate=0"
	}

	// instance of your page model
	p := Page{
		isTor,
		isTor && !upToDate,
		!small,
		onOff,
		Lang(r),
		host,
		extra,
		Locales,
	}

	// render the template
	Layout.ExecuteTemplate(w, "index.html", p)

}

func main() {

	// determine which port to run on
	port := os.Getenv("PORT")
	if len(port) == 0 {
		port = "8000"
	}

	// load i18n
	domain, err := gettext.NewDomain("check", "locale")
	if err != nil {
		log.Fatal(err)
	}

	// add template funcs
	Layout = Layout.Funcs(template.FuncMap{
		"UnEscaped": func(x string) interface{} {
			return template.HTML(x)
		},
		"UnEscapedURL": func(x string) interface{} {
			return template.URL(x)
		},
		"GetText": func(lang string, text string) string {
			return domain.GetText(lang, text)
		},
	})

	// load layout
	Layout, err = Layout.ParseFiles(
		"public/index.html",
		"public/torbutton.html",
	)
	if err != nil {
		log.Fatal(err)
	}

	// load exits
	LoadList()

	// listen for signal to reload exits
	s := make(chan os.Signal, 1)
	signal.Notify(s, syscall.SIGUSR2)
	go func() {
		for {
			<-s
			LoadList()
			log.Println("Exit list reloaded.")
		}
	}()

	// routes
	http.HandleFunc("/", RootHandler)

	// files
	files := http.FileServer(http.Dir("./public"))
	Phttp.Handle("/torcheck/", http.StripPrefix("/torcheck/", files))
	Phttp.Handle("/", files)

	// start the server
	log.Printf("Listening on port: %s\n", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/itchio/butler/comm"
)

type CleanPlan struct {
	BasePath string   `json:"basePath"`
	Entries  []string `json:"entries"`
}

func clean(planPath string) {
	startTime := time.Now()

	contents, err := ioutil.ReadFile(planPath)
	must(err)

	plan := CleanPlan{}

	err = json.Unmarshal(contents, &plan)
	must(err)

	comm.Logf("Cleaning %d entries from %s", len(plan.Entries), plan.BasePath)

	for _, entry := range plan.Entries {
		fullPath := filepath.Join(plan.BasePath, entry)
		must(os.Remove(fullPath))
	}

	duration := time.Since(startTime)
	entriesPerSec := float64(len(plan.Entries)) / duration.Seconds()
	comm.Statf("Done in %s (%.2f entries/s)", duration, entriesPerSec)
}
<commit_msg>Don't complain if file we want to remove is already deleted<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/itchio/butler/comm"
)

type CleanPlan struct {
	BasePath string   `json:"basePath"`
	Entries  []string `json:"entries"`
}

func clean(planPath string) {
	startTime := time.Now()

	contents, err := ioutil.ReadFile(planPath)
	must(err)

	plan := CleanPlan{}

	err = json.Unmarshal(contents, &plan)
	must(err)

	comm.Logf("Cleaning %d entries from %s", len(plan.Entries), plan.BasePath)

	for _, entry := range plan.Entries {
		fullPath := filepath.Join(plan.BasePath, entry)
		err := os.Remove(fullPath)
		if err != nil {
			if os.IsNotExist(err) {
				// ignore
			} else {
				must(err)
			}
		}
	}

	duration := time.Since(startTime)
	entriesPerSec := float64(len(plan.Entries)) / duration.Seconds()
	comm.Statf("Done in %s (%.2f entries/s)", duration, entriesPerSec)
}
<|endoftext|>


<commit_before>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 5, 16, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>Go 1.8.2<commit_after>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 5, 23, 20, 0, 0, 0, time.UTC)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>


<commit_before>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 11, 20, 0, 0, 0, 0, time.UTC)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>Local time<commit_after>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 11, 20, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/LK4D4/vndr/godl"
)

type depEntry struct {
	importPath string
	rev        string
	repoPath   string
}

func (d depEntry) String() string {
	return fmt.Sprintf("%s %s\n", d.importPath, d.rev)
}

func parseDeps(r io.Reader) ([]depEntry, error) {
	var deps []depEntry
	s := bufio.NewScanner(r)
	for s.Scan() {
		ln := strings.TrimSpace(s.Text())
		if strings.HasPrefix(ln, "#") || ln == "" {
			continue
		}
		cidx := strings.Index(ln, "#")
		if cidx > 0 {
			ln = ln[:cidx]
		}
		ln = strings.TrimSpace(ln)
		parts := strings.Fields(ln)
		if len(parts) != 2 && len(parts) != 3 {
			return nil, fmt.Errorf("invalid config format: %s", ln)
		}
		d := depEntry{
			importPath: parts[0],
			rev:        parts[1],
		}
		if len(parts) == 3 {
			d.repoPath = parts[2]
		}
		deps = append(deps, d)
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	return deps, nil
}

func cloneAll(vd string, ds []depEntry) error {
	var wg sync.WaitGroup
	errCh := make(chan error, len(ds))
	limit := make(chan struct{}, 16)
	for _, d := range ds {
		wg.Add(1)
		go func(d depEntry) {
			var err error
			limit <- struct{}{}
			for i := 0; i < 20; i++ {
				if err = cloneDep(vd, d); err == nil {
					errCh <- nil
					wg.Done()
					<-limit
					return
				}
				time.Sleep(1 * time.Second)
			}
			errCh <- err
			wg.Done()
			<-limit
		}(d)
	}
	wg.Wait()
	close(errCh)
	var errs []string
	for err := range errCh {
		if err != nil {
			errs = append(errs, err.Error())
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("Errors on clone:\n%s", strings.Join(errs, "\n"))
}

func cloneDep(vd string, d depEntry) error {
	if d.repoPath != "" {
		log.Printf("\tClone %s to %s, revision %s", d.repoPath, d.importPath, d.rev)
	} else {
		log.Printf("\tClone %s, revision %s", d.importPath, d.rev)
	}
	defer log.Printf("\tFinished clone %s", d.importPath)
	vcs, err := godl.Download(d.importPath, d.repoPath, vd, d.rev)
	if err != nil {
		return fmt.Errorf("%s: %v", d.importPath, err)
	}
	return cleanVCS(vcs)
}
<commit_msg>Add more clear messages around clone failures<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/LK4D4/vndr/godl"
)

type depEntry struct {
	importPath string
	rev        string
	repoPath   string
}

func (d depEntry) String() string {
	return fmt.Sprintf("%s %s\n", d.importPath, d.rev)
}

func parseDeps(r io.Reader) ([]depEntry, error) {
	var deps []depEntry
	s := bufio.NewScanner(r)
	for s.Scan() {
		ln := strings.TrimSpace(s.Text())
		if strings.HasPrefix(ln, "#") || ln == "" {
			continue
		}
		cidx := strings.Index(ln, "#")
		if cidx > 0 {
			ln = ln[:cidx]
		}
		ln = strings.TrimSpace(ln)
		parts := strings.Fields(ln)
		if len(parts) != 2 && len(parts) != 3 {
			return nil, fmt.Errorf("invalid config format: %s", ln)
		}
		d := depEntry{
			importPath: parts[0],
			rev:        parts[1],
		}
		if len(parts) == 3 {
			d.repoPath = parts[2]
		}
		deps = append(deps, d)
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	return deps, nil
}

func cloneAll(vd string, ds []depEntry) error {
	var wg sync.WaitGroup
	errCh := make(chan error, len(ds))
	limit := make(chan struct{}, 16)
	for _, d := range ds {
		wg.Add(1)
		go func(d depEntry) {
			var err error
			limit <- struct{}{}
			for i := 0; i < 20; i++ {
				if d.repoPath != "" {
					log.Printf("\tClone %s to %s, revision %s, attempt %d/20", d.repoPath, d.importPath, d.rev, i+1)
				} else {
					log.Printf("\tClone %s, revision %s, attempt %d/20", d.importPath, d.rev, i+1)
				}
				if err = cloneDep(vd, d); err == nil {
					errCh <- nil
					wg.Done()
					<-limit
					log.Printf("\tFinished clone %s", d.importPath)
					return
				}
				log.Printf("\tClone %s, attempt %d/20 finished with error %v", d.importPath, i+1, err)
				time.Sleep(1 * time.Second)
			}
			errCh <- err
			wg.Done()
			<-limit
		}(d)
	}
	wg.Wait()
	close(errCh)
	var errs []string
	for err := range errCh {
		if err != nil {
			errs = append(errs, err.Error())
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("Errors on clone:\n%s", strings.Join(errs, "\n"))
}

func cloneDep(vd string, d depEntry) error {
	vcs, err := godl.Download(d.importPath, d.repoPath, vd, d.rev)
	if err != nil {
		return fmt.Errorf("%s: %v", d.importPath, err)
	}
	return cleanVCS(vcs)
}
<|endoftext|>


<commit_before>package shared

import (
	"errors"
	"os"
)

/*
Errors of Tinzenite.
*/
var (
	ErrIllegalParameters = errors.New("illegal parameters given")
	ErrUnsupported       = errors.New("feature currently unsupported")
	ErrIsTinzenite       = errors.New("already a Tinzenite directory")
	ErrNotTinzenite      = errors.New("path is not valid Tinzenite directory")
	ErrNoTinIgnore       = errors.New("no .tinignore file found")
	ErrUntracked         = errors.New("object is not tracked in the model")
	ErrNilInternalState  = errors.New("internal state has illegal NIL values")
	ErrConflict          = errors.New("conflict, can not apply")
	ErrIllegalFileState  = errors.New("illegal file state detected")
)

/*
Internal errors of Tinzenite.
*/
var (
	errWrongObject = errors.New("wrong ObjectInfo")
)

// constant value here
const (
	/*RANDOMSEEDLENGTH is the amount of bytes used as cryptographic hash seed.*/
	RANDOMSEEDLENGTH = 32
	/*IDMAXLENGTH is the length in chars of new random identification hashes.*/
	IDMAXLENGTH = 16
	/*KEYLENGTH is the length of the encryption key used for challenges and file encryption.*/
	KEYLENGTH = 256
	/*FILEPERMISSIONMODE used for all file operations.*/
	FILEPERMISSIONMODE = 0777
	/*FILEFLAGCREATEAPPEND is the flag required to create a file or append to it if it already exists.*/
	FILEFLAGCREATEAPPEND = os.O_CREATE | os.O_RDWR | os.O_APPEND
	/*CHUNKSIZE for hashing and encryption.*/
	CHUNKSIZE = 8 * 1024
)

// Path constants here
const (
	TINZENITEDIR   = ".tinzenite"
	TINIGNORE      = ".tinignore" // correct valid name of .tinignore files
	DIRECTORYLIST  = "directory.list"
	LOCALDIR       = "local" // info to the local peer is stored here
	TEMPDIR        = "temp"
	RECEIVINGDIR   = "receiving" // dir for receiving transfers
	SENDINGDIR     = "sending"   // send dir for now only used in encrypted
	REMOVEDIR      = "removed"   // remove
	REMOVECHECKDIR = "check"     // remove
	REMOVEDONEDIR  = "done"      // remove
	REMOVESTOREDIR = "rmstore"   // local remove check dir
	ORGDIR         = "org"       // only directory that IS synchronized as normal
	PEERSDIR       = "peers"
	ENDING         = ".json"
	AUTHJSON       = "auth" + ENDING
	MODELJSON      = "model" + ENDING
	SELFPEERJSON   = "self" + ENDING
	BOOTJSON       = "boot" + ENDING
)

/*
Special path sets here. Mostly used to make writing to sub directories easier.

TODO aren't these all for core? Then move them there.
*/
const (
	STOREPEERDIR    = TINZENITEDIR + "/" + ORGDIR + "/" + PEERSDIR
	STORETOXDUMPDIR = TINZENITEDIR + "/" + LOCALDIR
	STOREAUTHDIR    = TINZENITEDIR + "/" + ORGDIR
	STOREMODELDIR   = TINZENITEDIR + "/" + LOCALDIR
)

// .tinignore content for .tinzenite directory
const TINDIRIGNORE = "# DO NOT MODIFY!\n/" + LOCALDIR + "\n/" +
	TEMPDIR + "\n/" + RECEIVINGDIR + "\n" + SENDINGDIR

// IDMODEL is the model identification used to differentiate models from files.
const IDMODEL = "MODEL"
<commit_msg>fixed .tinignore<commit_after>package shared

import (
	"errors"
	"os"
)

/*
Errors of Tinzenite.
*/
var (
	ErrIllegalParameters = errors.New("illegal parameters given")
	ErrUnsupported       = errors.New("feature currently unsupported")
	ErrIsTinzenite       = errors.New("already a Tinzenite directory")
	ErrNotTinzenite      = errors.New("path is not valid Tinzenite directory")
	ErrNoTinIgnore       = errors.New("no .tinignore file found")
	ErrUntracked         = errors.New("object is not tracked in the model")
	ErrNilInternalState  = errors.New("internal state has illegal NIL values")
	ErrConflict          = errors.New("conflict, can not apply")
	ErrIllegalFileState  = errors.New("illegal file state detected")
)

/*
Internal errors of Tinzenite.
*/
var (
	errWrongObject = errors.New("wrong ObjectInfo")
)

// constant value here
const (
	/*RANDOMSEEDLENGTH is the amount of bytes used as cryptographic hash seed.*/
	RANDOMSEEDLENGTH = 32
	/*IDMAXLENGTH is the length in chars of new random identification hashes.*/
	IDMAXLENGTH = 16
	/*KEYLENGTH is the length of the encryption key used for challenges and file encryption.*/
	KEYLENGTH = 256
	/*FILEPERMISSIONMODE used for all file operations.*/
	FILEPERMISSIONMODE = 0777
	/*FILEFLAGCREATEAPPEND is the flag required to create a file or append to it if it already exists.*/
	FILEFLAGCREATEAPPEND = os.O_CREATE | os.O_RDWR | os.O_APPEND
	/*CHUNKSIZE for hashing and encryption.*/
	CHUNKSIZE = 8 * 1024
)

// Path constants here
const (
	TINZENITEDIR   = ".tinzenite"
	TINIGNORE      = ".tinignore" // correct valid name of .tinignore files
	DIRECTORYLIST  = "directory.list"
	LOCALDIR       = "local" // info to the local peer is stored here
	TEMPDIR        = "temp"
	RECEIVINGDIR   = "receiving" // dir for receiving transfers
	SENDINGDIR     = "sending"   // send dir for now only used in encrypted
	REMOVEDIR      = "removed"   // remove
	REMOVECHECKDIR = "check"     // remove
	REMOVEDONEDIR  = "done"      // remove
	REMOVESTOREDIR = "rmstore"   // local remove check dir
	ORGDIR         = "org"       // only directory that IS synchronized as normal
	PEERSDIR       = "peers"
	ENDING         = ".json"
	AUTHJSON       = "auth" + ENDING
	MODELJSON      = "model" + ENDING
	SELFPEERJSON   = "self" + ENDING
	BOOTJSON       = "boot" + ENDING
)

/*
Special path sets here. Mostly used to make writing to sub directories easier.

TODO aren't these all for core? Then move them there.
*/
const (
	STOREPEERDIR    = TINZENITEDIR + "/" + ORGDIR + "/" + PEERSDIR
	STORETOXDUMPDIR = TINZENITEDIR + "/" + LOCALDIR
	STOREAUTHDIR    = TINZENITEDIR + "/" + ORGDIR
	STOREMODELDIR   = TINZENITEDIR + "/" + LOCALDIR
)

// .tinignore content for .tinzenite directory
const TINDIRIGNORE = "# DO NOT MODIFY!\n/" + LOCALDIR + "\n/" +
	TEMPDIR + "\n/" + RECEIVINGDIR + "\n/" + SENDINGDIR

// IDMODEL is the model identification used to differentiate models from files.
const IDMODEL = "MODEL"
<|endoftext|>


<commit_before>// Copyright 2013 Apcera Inc. All rights reserved.

package graft

import (
	"time"
)

const (
	VERSION = "0.1"

	// Election timeout MIN and MAX per RAFT spec suggestion.
	MIN_ELECTION_TIMEOUT = 500 * time.Millisecond
	MAX_ELECTION_TIMEOUT = 2 * MIN_ELECTION_TIMEOUT

	// Heartbeat tick for LEADERS.
	// Should be << MIN_ELECTION_TIMEOUT per RAFT spec.
	HEARTBEAT_INTERVAL = 100 * time.Millisecond

	NO_LEADER = ""
	NO_VOTE   = ""
)
<commit_msg>Bumped version<commit_after>// Copyright 2013-2015 Apcera Inc. All rights reserved.

package graft

import (
	"time"
)

const (
	VERSION = "0.2"

	// Election timeout MIN and MAX per RAFT spec suggestion.
	MIN_ELECTION_TIMEOUT = 500 * time.Millisecond
	MAX_ELECTION_TIMEOUT = 2 * MIN_ELECTION_TIMEOUT

	// Heartbeat tick for LEADERS.
	// Should be << MIN_ELECTION_TIMEOUT per RAFT spec.
	HEARTBEAT_INTERVAL = 100 * time.Millisecond

	NO_LEADER = ""
	NO_VOTE   = ""
)
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package memorytopo contains an implementation of the topo.Factory /
// topo.Conn interfaces based on an in-memory tree of data.
// It is constructed with an immutable set of cells.
package memorytopo

import (
	"math/rand"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/topo"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)

const (
	// Path components
	electionsPath = "elections"
)

var (
	nextWatchIndex = 0
)

// Factory is a memory-based implementation of topo.Factory.  It
// takes a file-system like approach, with directories at each level
// being an actual directory node. This is meant to be closer to
// file-system like servers, like ZooKeeper or Chubby. etcd or Consul
// implementations would be closer to a node-based implementation.
//
// It contains a single tree of nodes. Each cell topo.Conn will use
// a sub-directory in that tree.
type Factory struct {
	// mu protects the following fields.
	mu sync.Mutex
	// cells is the toplevel map that has one entry per cell.
	cells map[string]*node
	// generation is used to generate unique incrementing version
	// numbers.  We want a global counter so when creating a file,
	// then deleting it, then re-creating it, we don't restart the
	// version at 1. It is initialized with a random number,
	// so if we have two implementations, the numbers won't match.
	generation uint64
	// err is used for testing purposes to force queries / watches
	// to return the given error
	err error
}

// HasGlobalReadOnlyCell is part of the topo.Factory interface.
func (f *Factory) HasGlobalReadOnlyCell(serverAddr, root string) bool {
	return false
}

// Create is part of the topo.Factory interface.
func (f *Factory) Create(cell, serverAddr, root string) (topo.Conn, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	if _, ok := f.cells[cell]; !ok {
		return nil, topo.NewError(topo.NoNode, cell)
	}
	return &Conn{
		factory: f,
		cell:    cell,
	}, nil
}

// SetError forces the given error to be returned from all calls and propagates
// the error to all active watches.
func (f *Factory) SetError(err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.err = err
	if err != nil {
		for _, node := range f.cells {
			node.PropagateWatchError(err)
		}
	}
}

// Lock blocks all requests to the topo and is exposed to allow tests to
// simulate an unresponsive topo server
func (f *Factory) Lock() {
	f.mu.Lock()
}

// Unlock unblocks all requests to the topo and is exposed to allow tests to
// simulate an unresponsive topo server
func (f *Factory) Unlock() {
	f.mu.Unlock()
}

// Conn implements the topo.Conn interface. It remembers the cell, and
// points at the Factory that has all the data.
type Conn struct {
	factory *Factory
	cell    string
}

// Close is part of the topo.Conn interface.
// It nils out factory, so any subsequent call will panic.
func (c *Conn) Close() {
	c.factory = nil
}

// node contains a directory or a file entry.
// Exactly one of contents or children is not nil.
type node struct {
	name     string
	version  uint64
	contents []byte
	children map[string]*node

	// parent is a pointer to the parent node.
	// It is set to nil in toplevel and cell node.
	parent *node

	// watches is a map of all watches for this node.
	watches map[int]chan *topo.WatchData

	// lock is nil when the node is not locked.
	// otherwise it has a channel that is closed by unlock.
	lock chan struct{}

	// lockContents is the contents of the locks.
	// For regular locks, it has the contents that was passed in.
	// For master election, it has the id of the election leader.
	lockContents string
}

func (n *node) isDirectory() bool {
	return n.children != nil
}

// PropagateWatchError propagates the given error to all watches on this node
// and recursively applies to all children
func (n *node) PropagateWatchError(err error) {
	for _, ch := range n.watches {
		ch <- &topo.WatchData{
			Err: err,
		}
	}

	for _, c := range n.children {
		c.PropagateWatchError(err)
	}
}

// NewServerAndFactory returns a new MemoryTopo and the backing factory for all
// the cells. It will create one cell for each parameter passed in.  It will log.Exit out
// in case of a problem.
func NewServerAndFactory(cells ...string) (*topo.Server, *Factory) {
	f := &Factory{
		cells:      make(map[string]*node),
		generation: uint64(rand.Int63n(2 ^ 60)),
	}
	f.cells[topo.GlobalCell] = f.newDirectory(topo.GlobalCell, nil)

	ctx := context.Background()
	ts, err := topo.NewWithFactory(f, "" /*serverAddress*/, "" /*root*/)
	if err != nil {
		log.Exitf("topo.NewWithFactory() failed: %v", err)
	}
	for _, cell := range cells {
		f.cells[cell] = f.newDirectory(cell, nil)
		if err := ts.CreateCellInfo(ctx, cell, &topodatapb.CellInfo{}); err != nil {
			log.Exitf("ts.CreateCellInfo(%v) failed: %v", cell, err)
		}
	}
	return ts, f
}

// NewServer returns the new server
func NewServer(cells ...string) *topo.Server {
	server, _ := NewServerAndFactory(cells...)
	return server
}

func (f *Factory) getNextVersion() uint64 {
	f.generation++
	return f.generation
}

func (f *Factory) newFile(name string, contents []byte, parent *node) *node {
	return &node{
		name:     name,
		version:  f.getNextVersion(),
		contents: contents,
		parent:   parent,
		watches:  make(map[int]chan *topo.WatchData),
	}
}

func (f *Factory) newDirectory(name string, parent *node) *node {
	return &node{
		name:     name,
		version:  f.getNextVersion(),
		children: make(map[string]*node),
		parent:   parent,
	}
}

func (f *Factory) nodeByPath(cell, filePath string) *node {
	n, ok := f.cells[cell]
	if !ok {
		return nil
	}

	parts := strings.Split(filePath, "/")
	for _, part := range parts {
		if part == "" {
			// Skip empty parts, usually happens at the end.
			continue
		}
		if n.children == nil {
			// This is a file.
			return nil
		}
		child, ok := n.children[part]
		if !ok {
			// Path doesn't exist.
			return nil
		}
		n = child
	}
	return n
}

func (f *Factory) getOrCreatePath(cell, filePath string) *node {
	n, ok := f.cells[cell]
	if !ok {
		return nil
	}

	parts := strings.Split(filePath, "/")
	for _, part := range parts {
		if part == "" {
			// Skip empty parts, usually happens at the end.
			continue
		}
		if n.children == nil {
			// This is a file.
			return nil
		}
		child, ok := n.children[part]
		if !ok {
			// Path doesn't exist, create it.
			child = f.newDirectory(part, n)
			n.children[part] = child
		}
		n = child
	}
	return n
}

// recursiveDelete deletes a node and its parent directory if empty.
func (f *Factory) recursiveDelete(n *node) {
	parent := n.parent
	if parent == nil {
		return
	}
	delete(parent.children, n.name)
	if len(parent.children) == 0 {
		f.recursiveDelete(parent)
	}
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<commit_msg>Increased memorytopo generation pool size to 1 << 60<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreedto in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package memorytopo contains an implementation of the topo.Factory /
// topo.Conn interfaces based on an in-memory tree of data.
// It is constructed with an immutable set of cells.
package memorytopo

import (
	"math/rand"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"

	"vitess.io/vitess/go/vt/log"
	"vitess.io/vitess/go/vt/topo"

	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)

const (
	// Path components
	electionsPath = "elections"
)

var (
	nextWatchIndex = 0
)

// Factory is a memory-based implementation of topo.Factory.  It
// takes a file-system like approach, with directories at each level
// being an actual directory node. This is meant to be closer to
// file-system like servers, like ZooKeeper or Chubby. etcd or Consul
// implementations would be closer to a node-based implementation.
//
// It contains a single tree of nodes. Each cell topo.Conn will use
// a sub-directory in that tree.
type Factory struct {
	// mu protects the following fields.
	mu sync.Mutex
	// cells is the toplevel map that has one entry per cell.
	cells map[string]*node
	// generation is used to generate unique incrementing version
	// numbers.  We want a global counter so when creating a file,
	// then deleting it, then re-creating it, we don't restart the
	// version at 1. It is initialized with a random number,
	// so if we have two implementations, the numbers won't match.
	generation uint64
	// err is used for testing purposes to force queries / watches
	// to return the given error
	err error
}

// HasGlobalReadOnlyCell is part of the topo.Factory interface.
func (f *Factory) HasGlobalReadOnlyCell(serverAddr, root string) bool {
	return false
}

// Create is part of the topo.Factory interface.
func (f *Factory) Create(cell, serverAddr, root string) (topo.Conn, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	if _, ok := f.cells[cell]; !ok {
		return nil, topo.NewError(topo.NoNode, cell)
	}
	return &Conn{
		factory: f,
		cell:    cell,
	}, nil
}

// SetError forces the given error to be returned from all calls and propagates
// the error to all active watches.
func (f *Factory) SetError(err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.err = err
	if err != nil {
		for _, node := range f.cells {
			node.PropagateWatchError(err)
		}
	}
}

// Lock blocks all requests to the topo and is exposed to allow tests to
// simulate an unresponsive topo server
func (f *Factory) Lock() {
	f.mu.Lock()
}

// Unlock unblocks all requests to the topo and is exposed to allow tests to
// simulate an unresponsive topo server
func (f *Factory) Unlock() {
	f.mu.Unlock()
}

// Conn implements the topo.Conn interface. It remembers the cell, and
// points at the Factory that has all the data.
type Conn struct {
	factory *Factory
	cell    string
}

// Close is part of the topo.Conn interface.
// It nils out factory, so any subsequent call will panic.
func (c *Conn) Close() {
	c.factory = nil
}

// node contains a directory or a file entry.
// Exactly one of contents or children is not nil.
type node struct {
	name     string
	version  uint64
	contents []byte
	children map[string]*node

	// parent is a pointer to the parent node.
	// It is set to nil in toplevel and cell node.
	parent *node

	// watches is a map of all watches for this node.
	watches map[int]chan *topo.WatchData

	// lock is nil when the node is not locked.
	// otherwise it has a channel that is closed by unlock.
	lock chan struct{}

	// lockContents is the contents of the locks.
	// For regular locks, it has the contents that was passed in.
	// For master election, it has the id of the election leader.
	lockContents string
}

func (n *node) isDirectory() bool {
	return n.children != nil
}

// PropagateWatchError propagates the given error to all watches on this node
// and recursively applies to all children
func (n *node) PropagateWatchError(err error) {
	for _, ch := range n.watches {
		ch <- &topo.WatchData{
			Err: err,
		}
	}

	for _, c := range n.children {
		c.PropagateWatchError(err)
	}
}

// NewServerAndFactory returns a new MemoryTopo and the backing factory for all
// the cells. It will create one cell for each parameter passed in.  It will log.Exit out
// in case of a problem.
func NewServerAndFactory(cells ...string) (*topo.Server, *Factory) {
	f := &Factory{
		cells:      make(map[string]*node),
		generation: uint64(rand.Int63n(1 << 60)),
	}
	f.cells[topo.GlobalCell] = f.newDirectory(topo.GlobalCell, nil)

	ctx := context.Background()
	ts, err := topo.NewWithFactory(f, "" /*serverAddress*/, "" /*root*/)
	if err != nil {
		log.Exitf("topo.NewWithFactory() failed: %v", err)
	}
	for _, cell := range cells {
		f.cells[cell] = f.newDirectory(cell, nil)
		if err := ts.CreateCellInfo(ctx, cell, &topodatapb.CellInfo{}); err != nil {
			log.Exitf("ts.CreateCellInfo(%v) failed: %v", cell, err)
		}
	}
	return ts, f
}

// NewServer returns the new server
func NewServer(cells ...string) *topo.Server {
	server, _ := NewServerAndFactory(cells...)
	return server
}

func (f *Factory) getNextVersion() uint64 {
	f.generation++
	return f.generation
}

func (f *Factory) newFile(name string, contents []byte, parent *node) *node {
	return &node{
		name:     name,
		version:  f.getNextVersion(),
		contents: contents,
		parent:   parent,
		watches:  make(map[int]chan *topo.WatchData),
	}
}

func (f *Factory) newDirectory(name string, parent *node) *node {
	return &node{
		name:     name,
		version:  f.getNextVersion(),
		children: make(map[string]*node),
		parent:   parent,
	}
}

func (f *Factory) nodeByPath(cell, filePath string) *node {
	n, ok := f.cells[cell]
	if !ok {
		return nil
	}

	parts := strings.Split(filePath, "/")
	for _, part := range parts {
		if part == "" {
			// Skip empty parts, usually happens at the end.
			continue
		}
		if n.children == nil {
			// This is a file.
			return nil
		}
		child, ok := n.children[part]
		if !ok {
			// Path doesn't exist.
			return nil
		}
		n = child
	}
	return n
}

func (f *Factory) getOrCreatePath(cell, filePath string) *node {
	n, ok := f.cells[cell]
	if !ok {
		return nil
	}

	parts := strings.Split(filePath, "/")
	for _, part := range parts {
		if part == "" {
			// Skip empty parts, usually happens at the end.
			continue
		}
		if n.children == nil {
			// This is a file.
			return nil
		}
		child, ok := n.children[part]
		if !ok {
			// Path doesn't exist, create it.
			child = f.newDirectory(part, n)
			n.children[part] = child
		}
		n = child
	}
	return n
}

// recursiveDelete deletes a node and its parent directory if empty.
func (f *Factory) recursiveDelete(n *node) {
	parent := n.parent
	if parent == nil {
		return
	}
	delete(parent.children, n.name)
	if len(parent.children) == 0 {
		f.recursiveDelete(parent)
	}
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"errors"
	"net/http"
	"sort"
	"strings"
)

// CurlyRouter expects Routes with paths that contain zero or more parameters in curly brackets.
type CurlyRouter struct{}

// SelectRoute is part of the Router interface and returns the best match
// for the WebService and its Route for the given Request.
func (c CurlyRouter) SelectRoute(
	webServices []*WebService,
	httpRequest *http.Request) (selectedService *WebService, selected *Route, err error) {

	requestTokens := tokenizePath(httpRequest.URL.Path)

	detectedService := c.detectWebService(requestTokens, webServices)
	if detectedService == nil {
		return nil, nil, errors.New("[restful] no detected service")
	}
	candidateRoutes := c.selectRoutes(detectedService, requestTokens)
	if len(candidateRoutes) == 0 {
		return detectedService, nil, errors.New("[restful] no candidate routes")
	}
	selectedRoute, err := c.detectRoute(candidateRoutes, httpRequest)
	if selectedRoute == nil {
		return detectedService, nil, err
	}
	return detectedService, selectedRoute, nil
}

// selectRoutes return a collection of Route from a WebService that matches the path tokens from the request.
func (c CurlyRouter) selectRoutes(ws *WebService, requestTokens []string) []Route {
	candidates := &sortableCurlyRoutes{[]*curlyRoute{}}
	for _, each := range ws.routes {
		matches, paramCount, staticCount := c.matchesRouteByPathTokens(each.pathParts, requestTokens)
		if matches {
			candidates.add(&curlyRoute{each, paramCount, staticCount}) // TODO make sure Routes() return pointers?
		}
	}
	sort.Sort(sort.Reverse(candidates))
	return candidates.routes()
}

// matchesRouteByPathTokens computes whether it matches, howmany parameters do match and what the number of static path elements are.
func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []string) (matches bool, paramCount int, staticCount int) {
	if len(routeTokens) != len(requestTokens) {
		return false, 0, 0
	}
	for i, routeToken := range routeTokens {
		requestToken := requestTokens[i]
		if !strings.HasPrefix(routeToken, "{") {
			if requestToken != routeToken {
				return false, 0, 0
			}
			staticCount++
		} else {
			paramCount++
		}
	}
	return true, paramCount, staticCount
}

// detectRoute selectes from a list of Route the first match by inspecting both the Accept and Content-Type
// headers of the Request. See also RouterJSR311 in jsr311.go
func (c CurlyRouter) detectRoute(candidateRoutes []Route, httpRequest *http.Request) (*Route, error) {
	return RouterJSR311{}.detectRoute(candidateRoutes, httpRequest)
}

// detectWebService returns the best matching webService given the list of path tokens.
// see also computeWebserviceScore
func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService {
	var best *WebService
	score := -1
	for _, each := range webServices {
		matches, eachScore := c.computeWebserviceScore(requestTokens, each.pathExpr.tokens)
		if matches && (eachScore > score) {
			best = each
			score = eachScore
		}
	}
	return best
}

// computeWebserviceScore returns whether tokens match and
// the weighted score of the longest matching consecutive tokens from the beginning.
func (c CurlyRouter) computeWebserviceScore(requestTokens []string, tokens []string) (bool, int) {
	if len(tokens) > len(requestTokens) {
		return false, 0
	}
	score := 0
	for i := 0; i < len(tokens); i++ {
		each := requestTokens[i]
		other := tokens[i]
		if len(each) == 0 && len(other) == 0 {
			score++
			continue
		}
		if len(other) > 0 && strings.HasPrefix(other, "{") {
			// no empty match
			if len(each) == 0 {
				return false, score
			}
			score += 1
		} else {
			// not a parameter
			if each != other {
				return false, score
			}
			score += (len(tokens) - i) * 10 //fuzzy
		}
	}
	return true, score
}
<commit_msg>start with wildcard detect<commit_after>package restful

// Copyright 2013 Ernest Micklei. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

import (
	"errors"
	"net/http"
	"sort"
	"strings"
)

// CurlyRouter expects Routes with paths that contain zero or more parameters in curly brackets.
type CurlyRouter struct{}

// SelectRoute is part of the Router interface and returns the best match
// for the WebService and its Route for the given Request.
func (c CurlyRouter) SelectRoute(
	webServices []*WebService,
	httpRequest *http.Request) (selectedService *WebService, selected *Route, err error) {

	requestTokens := tokenizePath(httpRequest.URL.Path)

	detectedService := c.detectWebService(requestTokens, webServices)
	if detectedService == nil {
		return nil, nil, errors.New("[restful] no detected service")
	}
	candidateRoutes := c.selectRoutes(detectedService, requestTokens)
	if len(candidateRoutes) == 0 {
		return detectedService, nil, errors.New("[restful] no candidate routes")
	}
	selectedRoute, err := c.detectRoute(candidateRoutes, httpRequest)
	if selectedRoute == nil {
		return detectedService, nil, err
	}
	return detectedService, selectedRoute, nil
}

// selectRoutes return a collection of Route from a WebService that matches the path tokens from the request.
func (c CurlyRouter) selectRoutes(ws *WebService, requestTokens []string) []Route {
	candidates := &sortableCurlyRoutes{[]*curlyRoute{}}
	for _, each := range ws.routes {
		matches, paramCount, staticCount := c.matchesRouteByPathTokens(each.pathParts, requestTokens)
		if matches {
			candidates.add(&curlyRoute{each, paramCount, staticCount}) // TODO make sure Routes() return pointers?
		}
	}
	sort.Sort(sort.Reverse(candidates))
	return candidates.routes()
}

// matchesRouteByPathTokens computes whether it matches, howmany parameters do match and what the number of static path elements are.
func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []string) (matches bool, paramCount int, staticCount int) {
	if len(routeTokens) != len(requestTokens) {
		return false, 0, 0
	}
	for i, routeToken := range routeTokens {
		requestToken := requestTokens[i]
		if !strings.HasPrefix(routeToken, "{") {
			if strings.ContainsRune(routeToken, ':') {
				// match by regex
				matchesToken, matchesRemainder := c.regularMatchesPathToken(routeToken, requestToken)
				if !matchesToken {
					return false, 0, 0
				}
				if matcheshesRemainder {
					return true, paramCount, len(routeTokens) - i + 1
				}
			} else if requestToken != routeToken {
				return false, 0, 0
			}
			staticCount++
		} else {
			paramCount++
		}
	}
	return true, paramCount, staticCount
}

// regularMatchesPathToken tests whether the regular expression part of routeToken matches the requestToken or all remaining tokens
func (c CurlyRouter) regularMatchesPathToken(routeToken, requestToken) (matchesToken bool, matchesRemainder bool) {
	return false, false
}

// detectRoute selectes from a list of Route the first match by inspecting both the Accept and Content-Type
// headers of the Request. See also RouterJSR311 in jsr311.go
func (c CurlyRouter) detectRoute(candidateRoutes []Route, httpRequest *http.Request) (*Route, error) {
	return RouterJSR311{}.detectRoute(candidateRoutes, httpRequest)
}

// detectWebService returns the best matching webService given the list of path tokens.
// see also computeWebserviceScore
func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService {
	var best *WebService
	score := -1
	for _, each := range webServices {
		matches, eachScore := c.computeWebserviceScore(requestTokens, each.pathExpr.tokens)
		if matches && (eachScore > score) {
			best = each
			score = eachScore
		}
	}
	return best
}

// computeWebserviceScore returns whether tokens match and
// the weighted score of the longest matching consecutive tokens from the beginning.
func (c CurlyRouter) computeWebserviceScore(requestTokens []string, tokens []string) (bool, int) {
	if len(tokens) > len(requestTokens) {
		return false, 0
	}
	score := 0
	for i := 0; i < len(tokens); i++ {
		each := requestTokens[i]
		other := tokens[i]
		if len(each) == 0 && len(other) == 0 {
			score++
			continue
		}
		if len(other) > 0 && strings.HasPrefix(other, "{") {
			// no empty match
			if len(each) == 0 {
				return false, score
			}
			score += 1
		} else {
			// not a parameter
			if each != other {
				return false, score
			}
			score += (len(tokens) - i) * 10 //fuzzy
		}
	}
	return true, score
}
<|endoftext|>


<commit_before>package googleimage

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/kyokomi/slackbot/plugins"
)

const endpointURL = "https://www.googleapis.com/customsearch/v1"

type plugin struct {
	rd     *rand.Rand
	client *http.Client
	cx     string
	apiKey string
}

func NewPlugin(cx string, apiKey string) plugins.BotMessagePlugin {
	return &plugin{
		rd:     rand.New(rand.NewSource(time.Now().UnixNano())),
		client: http.DefaultClient,
		cx:     cx,
		apiKey: apiKey,
	}
}

func (p *plugin) CheckMessage(_ plugins.BotEvent, message string) (bool, string) {
	plugins.CheckMessageKeyword(message, "image me")
	return true, message
}

func (p *plugin) DoAction(event plugins.BotEvent, message string) bool {
	query := strings.Replace(strings.TrimLeft(message, "image me"), "image me", "", 1)

	params := url.Values{}
	params.Set("searchType", "image")
	params.Set("alt", "json")
	params.Set("cx", p.cx)
	params.Set("key", p.apiKey)
	params.Set("q", query)

	resp, err := p.client.Get(fmt.Sprintf("%s?%s", endpointURL, params.Encode()))
	if err != nil {
		log.Println(err)
		return false
	}

	j, err := simplejson.NewFromReader(resp.Body)
	if err != nil {
		log.Println(err)
		return false
	}

	items, err := j.Get("items").Array()
	if err != nil {
		log.Println(err)
		return false
	}

	var links []string
	for _, item := range items {
		link := item.(map[string]interface{})["link"].(string)
		links = append(links, link)
	}

	idx := int(p.rd.Int() % len(links))
	event.Reply(links[idx])

	return false // next ng
}

var _ plugins.BotMessagePlugin = (*plugin)(nil)
<commit_msg>bugfix: return CheckMessage<commit_after>package googleimage

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/kyokomi/slackbot/plugins"
)

const endpointURL = "https://www.googleapis.com/customsearch/v1"

type plugin struct {
	rd     *rand.Rand
	client *http.Client
	cx     string
	apiKey string
}

func NewPlugin(cx string, apiKey string) plugins.BotMessagePlugin {
	return &plugin{
		rd:     rand.New(rand.NewSource(time.Now().UnixNano())),
		client: http.DefaultClient,
		cx:     cx,
		apiKey: apiKey,
	}
}

func (p *plugin) CheckMessage(_ plugins.BotEvent, message string) (bool, string) {
	return plugins.CheckMessageKeyword(message, "image me")
}

func (p *plugin) DoAction(event plugins.BotEvent, message string) bool {
	query := strings.Replace(strings.TrimLeft(message, "image me"), "image me", "", 1)

	params := url.Values{}
	params.Set("searchType", "image")
	params.Set("alt", "json")
	params.Set("cx", p.cx)
	params.Set("key", p.apiKey)
	params.Set("q", query)

	resp, err := p.client.Get(fmt.Sprintf("%s?%s", endpointURL, params.Encode()))
	if err != nil {
		log.Println(err)
		return false
	}

	j, err := simplejson.NewFromReader(resp.Body)
	if err != nil {
		log.Println(err)
		return false
	}

	items, err := j.Get("items").Array()
	if err != nil {
		log.Println(err)
		return false
	}

	var links []string
	for _, item := range items {
		link := item.(map[string]interface{})["link"].(string)
		links = append(links, link)
	}

	idx := int(p.rd.Int() % len(links))
	event.Reply(links[idx])

	return false // next ng
}

var _ plugins.BotMessagePlugin = (*plugin)(nil)
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"reflect"
	"testing"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/stop"
	"github.com/cockroachdb/cockroach/util/uuid"
)

var (
	batchR           = roachpb.BatchResponse{}
	testTxnID        *uuid.UUID
	testTxnID2       *uuid.UUID
	testTxnKey       = []byte("a")
	testTxnTimestamp = roachpb.ZeroTimestamp.Add(123, 456)
	testTxnPriority  = int32(123)
)

func init() {
	incR := roachpb.IncrementResponse{
		NewValue: 1,
	}
	batchR.Add(&incR)

	var err error
	testTxnID, err = uuid.FromString("0ce61c17-5eb4-4587-8c36-dcf4062ada4c")
	if err != nil {
		panic(err)
	}
	testTxnID2, err = uuid.FromString("9ab49d02-eb45-beef-9212-c23a92bc8211")
	if err != nil {
		panic(err)
	}
}

// createTestAbortCache creates an in-memory engine and
// returns a abort cache using the supplied Range ID.
func createTestAbortCache(t *testing.T, rangeID roachpb.RangeID, stopper *stop.Stopper) (*AbortCache, engine.Engine) {
	return NewAbortCache(rangeID), engine.NewInMem(roachpb.Attributes{}, 1<<20, stopper)
}

// TestAbortCachePutGetClearData tests basic get & put functionality as well as
// clearing the cache.
func TestAbortCachePutGetClearData(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	sc, e := createTestAbortCache(t, 1, stopper)
	// Start with a get for an uncached id.
	entry := roachpb.AbortCacheEntry{}
	if aborted, readErr := sc.Get(context.Background(), e, testTxnID, &entry); aborted {
		t.Errorf("expected not aborted for id %s", testTxnID)
	} else if readErr != nil {
		t.Fatalf("unxpected read error: %s", readErr)
	}

	entry = roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	if err := sc.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting response: %s", err)
	}

	tryHit := func(expAbort bool, expEntry roachpb.AbortCacheEntry) {
		var actual roachpb.AbortCacheEntry
		if aborted, readErr := sc.Get(context.Background(), e, testTxnID, &actual); readErr != nil {
			t.Errorf("unexpected failure getting response: %s", readErr)
		} else if expAbort != aborted {
			t.Errorf("got aborted: %t; expected %t", aborted, expAbort)
		} else if !reflect.DeepEqual(expEntry, actual) {
			t.Fatalf("wanted %v, got %v", expEntry, actual)
		}
	}

	tryHit(true, entry)
	if err := sc.ClearData(e); err != nil {
		t.Error(err)
	}
	tryHit(false, roachpb.AbortCacheEntry{})
}

// TestAbortCacheEmptyParams tests operation with empty parameters.
func TestAbortCacheEmptyParams(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	sc, e := createTestAbortCache(t, 1, stopper)

	entry := roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	// Put value for test response.
	if err := sc.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting response: %s", err)
	}
	if err := sc.Put(context.Background(), e, nil, nil, &entry); err != errEmptyTxnID {
		t.Errorf("expected errEmptyTxnID error putting response; got %s", err)
	}
	if _, err := sc.Get(context.Background(), e, nil, nil); err != errEmptyTxnID {
		t.Fatalf("expected errEmptyTxnID error; got %s", err)
	}
}

// TestAbortCacheCopyInto tests that entries in one cache get
// transferred correctly to another cache using CopyInto().
func TestAbortCacheCopyInto(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	rc1, e := createTestAbortCache(t, 1, stopper)
	rc2, _ := createTestAbortCache(t, 2, stopper)
	const seq = 123

	entry := roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	if err := rc1.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting entry: %s", err)
	}
	// Copy the first cache into the second.
	if count, err := rc1.CopyInto(e, nil, rc2.rangeID); err != nil {
		t.Fatal(err)
	} else if expCount := 1; count != expCount {
		t.Errorf("unexpected number of copied entries: %d", count)
	}
	for _, cache := range []*AbortCache{rc1, rc2} {
		var actual roachpb.AbortCacheEntry
		// Get should return 1 for both caches.
		if aborted, readErr := cache.Get(context.Background(), e, testTxnID, &actual); !aborted || readErr != nil {
			t.Errorf("unexpected failure getting response from source: %t, %s", aborted, readErr)
		} else if !reflect.DeepEqual(entry, actual) {
			t.Fatalf("wanted %v, got %v", entry, actual)
		}
	}
}

// TestAbortCacheCopyFrom tests that entries in one cache get
// transferred correctly to another cache using CopyFrom().
func TestAbortCacheCopyFrom(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	rc1, e := createTestAbortCache(t, 1, stopper)
	rc2, _ := createTestAbortCache(t, 2, stopper)

	entry := roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	if err := rc1.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting response: %s", err)
	}

	// Copy the first cache into the second.
	if count, err := rc2.CopyFrom(context.Background(), e, nil, rc1.rangeID); err != nil {
		t.Fatal(err)
	} else if expCount := 1; count != expCount {
		t.Errorf("unexpected number of copied entries: %d", count)
	}

	// Get should hit both caches.
	for i, cache := range []*AbortCache{rc1, rc2} {
		var actual roachpb.AbortCacheEntry
		if aborted, readErr := cache.Get(context.Background(), e, testTxnID, &actual); !aborted || readErr != nil {
			t.Fatalf("%d: unxpected read error: %t, %s", i, aborted, readErr)
		} else if !reflect.DeepEqual(entry, actual) {
			t.Fatalf("expected %v, got %v", entry, actual)
		}
	}
}
<commit_msg>storage: remove unused test const<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"reflect"
	"testing"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/cockroachdb/cockroach/util/stop"
	"github.com/cockroachdb/cockroach/util/uuid"
)

var (
	batchR           = roachpb.BatchResponse{}
	testTxnID        *uuid.UUID
	testTxnID2       *uuid.UUID
	testTxnKey       = []byte("a")
	testTxnTimestamp = roachpb.ZeroTimestamp.Add(123, 456)
	testTxnPriority  = int32(123)
)

func init() {
	incR := roachpb.IncrementResponse{
		NewValue: 1,
	}
	batchR.Add(&incR)

	var err error
	testTxnID, err = uuid.FromString("0ce61c17-5eb4-4587-8c36-dcf4062ada4c")
	if err != nil {
		panic(err)
	}
	testTxnID2, err = uuid.FromString("9ab49d02-eb45-beef-9212-c23a92bc8211")
	if err != nil {
		panic(err)
	}
}

// createTestAbortCache creates an in-memory engine and
// returns a abort cache using the supplied Range ID.
func createTestAbortCache(t *testing.T, rangeID roachpb.RangeID, stopper *stop.Stopper) (*AbortCache, engine.Engine) {
	return NewAbortCache(rangeID), engine.NewInMem(roachpb.Attributes{}, 1<<20, stopper)
}

// TestAbortCachePutGetClearData tests basic get & put functionality as well as
// clearing the cache.
func TestAbortCachePutGetClearData(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	sc, e := createTestAbortCache(t, 1, stopper)
	// Start with a get for an uncached id.
	entry := roachpb.AbortCacheEntry{}
	if aborted, readErr := sc.Get(context.Background(), e, testTxnID, &entry); aborted {
		t.Errorf("expected not aborted for id %s", testTxnID)
	} else if readErr != nil {
		t.Fatalf("unxpected read error: %s", readErr)
	}

	entry = roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	if err := sc.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting response: %s", err)
	}

	tryHit := func(expAbort bool, expEntry roachpb.AbortCacheEntry) {
		var actual roachpb.AbortCacheEntry
		if aborted, readErr := sc.Get(context.Background(), e, testTxnID, &actual); readErr != nil {
			t.Errorf("unexpected failure getting response: %s", readErr)
		} else if expAbort != aborted {
			t.Errorf("got aborted: %t; expected %t", aborted, expAbort)
		} else if !reflect.DeepEqual(expEntry, actual) {
			t.Fatalf("wanted %v, got %v", expEntry, actual)
		}
	}

	tryHit(true, entry)
	if err := sc.ClearData(e); err != nil {
		t.Error(err)
	}
	tryHit(false, roachpb.AbortCacheEntry{})
}

// TestAbortCacheEmptyParams tests operation with empty parameters.
func TestAbortCacheEmptyParams(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	sc, e := createTestAbortCache(t, 1, stopper)

	entry := roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	// Put value for test response.
	if err := sc.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting response: %s", err)
	}
	if err := sc.Put(context.Background(), e, nil, nil, &entry); err != errEmptyTxnID {
		t.Errorf("expected errEmptyTxnID error putting response; got %s", err)
	}
	if _, err := sc.Get(context.Background(), e, nil, nil); err != errEmptyTxnID {
		t.Fatalf("expected errEmptyTxnID error; got %s", err)
	}
}

// TestAbortCacheCopyInto tests that entries in one cache get
// transferred correctly to another cache using CopyInto().
func TestAbortCacheCopyInto(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	rc1, e := createTestAbortCache(t, 1, stopper)
	rc2, _ := createTestAbortCache(t, 2, stopper)

	entry := roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	if err := rc1.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting entry: %s", err)
	}
	// Copy the first cache into the second.
	if count, err := rc1.CopyInto(e, nil, rc2.rangeID); err != nil {
		t.Fatal(err)
	} else if expCount := 1; count != expCount {
		t.Errorf("unexpected number of copied entries: %d", count)
	}
	for _, cache := range []*AbortCache{rc1, rc2} {
		var actual roachpb.AbortCacheEntry
		// Get should return 1 for both caches.
		if aborted, readErr := cache.Get(context.Background(), e, testTxnID, &actual); !aborted || readErr != nil {
			t.Errorf("unexpected failure getting response from source: %t, %s", aborted, readErr)
		} else if !reflect.DeepEqual(entry, actual) {
			t.Fatalf("wanted %v, got %v", entry, actual)
		}
	}
}

// TestAbortCacheCopyFrom tests that entries in one cache get
// transferred correctly to another cache using CopyFrom().
func TestAbortCacheCopyFrom(t *testing.T) {
	defer leaktest.AfterTest(t)()
	stopper := stop.NewStopper()
	defer stopper.Stop()
	rc1, e := createTestAbortCache(t, 1, stopper)
	rc2, _ := createTestAbortCache(t, 2, stopper)

	entry := roachpb.AbortCacheEntry{
		Key:       testTxnKey,
		Timestamp: testTxnTimestamp,
		Priority:  testTxnPriority,
	}
	if err := rc1.Put(context.Background(), e, nil, testTxnID, &entry); err != nil {
		t.Errorf("unexpected error putting response: %s", err)
	}

	// Copy the first cache into the second.
	if count, err := rc2.CopyFrom(context.Background(), e, nil, rc1.rangeID); err != nil {
		t.Fatal(err)
	} else if expCount := 1; count != expCount {
		t.Errorf("unexpected number of copied entries: %d", count)
	}

	// Get should hit both caches.
	for i, cache := range []*AbortCache{rc1, rc2} {
		var actual roachpb.AbortCacheEntry
		if aborted, readErr := cache.Get(context.Background(), e, testTxnID, &actual); !aborted || readErr != nil {
			t.Fatalf("%d: unxpected read error: %t, %s", i, aborted, readErr)
		} else if !reflect.DeepEqual(entry, actual) {
			t.Fatalf("expected %v, got %v", entry, actual)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"code.google.com/p/goprotobuf/proto"
	"fmt"
	"github.com/prometheus/prometheus/coding"
	"github.com/prometheus/prometheus/model"
	dto "github.com/prometheus/prometheus/model/generated"
	"github.com/prometheus/prometheus/storage/raw"
	"github.com/prometheus/prometheus/storage/raw/leveldb"
	"time"
)

// processor models a post-processing agent that performs work given a sample
// corpus.
type processor interface {
	// Name emits the name of this processor's signature encoder.  It must be
	// fully-qualified in the sense that it could be used via a Protocol Buffer
	// registry to extract the descriptor to reassemble this message.
	Name() string
	// Signature emits a byte signature for this process for the purpose of
	// remarking how far along it has been applied to the database.
	Signature() (signature []byte, err error)
	// Apply runs this processor against the sample set.  sampleIterator expects
	// to be pre-seeked to the initial starting position.  The processor will
	// run until up until stopAt has been reached.  It is imperative that the
	// provided stopAt is within the interval of the series frontier.
	//
	// Upon completion or error, the last time at which the processor finished
	// shall be emitted in addition to any errors.
	Apply(sampleIterator leveldb.Iterator, samples raw.Persistence, stopAt time.Time, fingerprint model.Fingerprint) (lastCurated time.Time, err error)
}

// compactionProcessor combines sparse values in the database together such
// that at least MinimumGroupSize is fulfilled from the
type compactionProcessor struct {
	// MaximumMutationPoolBatch represents approximately the largest pending
	// batch of mutation operations for the database before pausing to
	// commit before resumption.
	//
	// A reasonable value would be (MinimumGroupSize * 2) + 1.
	MaximumMutationPoolBatch int
	// MinimumGroupSize represents the smallest allowed sample chunk size in the
	// database.
	MinimumGroupSize int
	// signature is the byte representation of the compactionProcessor's settings,
	// used for purely memoization purposes across an instance.
	signature []byte
}

func (p compactionProcessor) Name() string {
	return "io.prometheus.CompactionProcessorDefinition"
}

func (p *compactionProcessor) Signature() (out []byte, err error) {
	if len(p.signature) == 0 {
		out, err = proto.Marshal(&dto.CompactionProcessorDefinition{
			MinimumGroupSize: proto.Uint32(uint32(p.MinimumGroupSize)),
		})

		p.signature = out
	}

	out = p.signature

	return
}

func (p compactionProcessor) String() string {
	return fmt.Sprintf("compactionProcess for minimum group size %d", p.MinimumGroupSize)
}

func (p compactionProcessor) Apply(sampleIterator leveldb.Iterator, samples raw.Persistence, stopAt time.Time, fingerprint model.Fingerprint) (lastCurated time.Time, err error) {
	var pendingBatch raw.Batch = nil

	defer func() {
		if pendingBatch != nil {
			pendingBatch.Close()
		}
	}()

	var pendingMutations = 0
	var pendingSamples model.Values
	var sampleKey model.SampleKey
	var sampleValues model.Values
	var lastTouchedTime time.Time
	var keyDropped bool

	sampleKey, err = extractSampleKey(sampleIterator)
	if err != nil {
		return
	}
	sampleValues, err = extractSampleValues(sampleIterator)
	if err != nil {
		return
	}

	for lastCurated.Before(stopAt) && lastTouchedTime.Before(stopAt) {
		switch {
		// Furnish a new pending batch operation if none is available.
		case pendingBatch == nil:
			pendingBatch = leveldb.NewBatch()

		// If there are no sample values to extract from the datastore, let's
		// continue extracting more values to use.  We know that the time.Before()
		// block would prevent us from going into unsafe territory.
		case len(sampleValues) == 0:
			if !sampleIterator.Next() {
				return lastCurated, fmt.Errorf("Illegal Condition: Invalid Iterator on Continuation")
			}

			keyDropped = false

			sampleKey, err = extractSampleKey(sampleIterator)
			if err != nil {
				return
			}
			sampleValues, err = extractSampleValues(sampleIterator)
			if err != nil {
				return
			}

		// If the number of pending mutations exceeds the allowed batch amount,
		// commit to disk and delete the batch.  A new one will be recreated if
		// necessary.
		case pendingMutations >= p.MaximumMutationPoolBatch:
			err = samples.Commit(pendingBatch)
			if err != nil {
				return
			}

			pendingMutations = 0

			pendingBatch.Close()
			pendingBatch = nil

		case len(pendingSamples) == 0 && len(sampleValues) >= p.MinimumGroupSize:
			lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
			sampleValues = model.Values{}

		case len(pendingSamples)+len(sampleValues) < p.MinimumGroupSize:
			if !keyDropped {
				key := coding.NewProtocolBuffer(sampleKey.ToDTO())
				pendingBatch.Drop(key)
				keyDropped = true
			}
			pendingSamples = append(pendingSamples, sampleValues...)
			lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
			sampleValues = model.Values{}
			pendingMutations++

		// If the number of pending writes equals the target group size
		case len(pendingSamples) == p.MinimumGroupSize:
			newSampleKey := pendingSamples.ToSampleKey(fingerprint)
			key := coding.NewProtocolBuffer(newSampleKey.ToDTO())
			value := coding.NewProtocolBuffer(pendingSamples.ToDTO())
			pendingBatch.Put(key, value)
			pendingMutations++
			lastCurated = newSampleKey.FirstTimestamp.In(time.UTC)
			if len(sampleValues) > 0 {
				if !keyDropped {
					key := coding.NewProtocolBuffer(sampleKey.ToDTO())
					pendingBatch.Drop(key)
					keyDropped = true
				}

				if len(sampleValues) > p.MinimumGroupSize {
					pendingSamples = sampleValues[:p.MinimumGroupSize]
					sampleValues = sampleValues[p.MinimumGroupSize:]
					lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
				} else {
					pendingSamples = sampleValues
					lastTouchedTime = pendingSamples[len(pendingSamples)-1].Timestamp
					sampleValues = model.Values{}
				}
			}

		case len(pendingSamples)+len(sampleValues) >= p.MinimumGroupSize:
			if !keyDropped {
				key := coding.NewProtocolBuffer(sampleKey.ToDTO())
				pendingBatch.Drop(key)
				keyDropped = true
			}
			remainder := p.MinimumGroupSize - len(pendingSamples)
			pendingSamples = append(pendingSamples, sampleValues[:remainder]...)
			sampleValues = sampleValues[remainder:]
			if len(sampleValues) == 0 {
				lastTouchedTime = pendingSamples[len(pendingSamples)-1].Timestamp
			} else {
				lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
			}
			pendingMutations++

		default:
			err = fmt.Errorf("Unhandled processing case.")
		}
	}

	if len(sampleValues) > 0 || len(pendingSamples) > 0 {
		pendingSamples = append(sampleValues, pendingSamples...)
		newSampleKey := pendingSamples.ToSampleKey(fingerprint)
		key := coding.NewProtocolBuffer(newSampleKey.ToDTO())
		value := coding.NewProtocolBuffer(pendingSamples.ToDTO())
		pendingBatch.Put(key, value)
		pendingSamples = model.Values{}
		pendingMutations++
		lastCurated = newSampleKey.FirstTimestamp.In(time.UTC)
	}

	// This is not deferred due to the off-chance that a pre-existing commit
	// failed.
	if pendingBatch != nil && pendingMutations > 0 {
		err = samples.Commit(pendingBatch)
		if err != nil {
			return
		}
	}

	return
}
<commit_msg>Complete sentence.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"code.google.com/p/goprotobuf/proto"
	"fmt"
	"github.com/prometheus/prometheus/coding"
	"github.com/prometheus/prometheus/model"
	dto "github.com/prometheus/prometheus/model/generated"
	"github.com/prometheus/prometheus/storage/raw"
	"github.com/prometheus/prometheus/storage/raw/leveldb"
	"time"
)

// processor models a post-processing agent that performs work given a sample
// corpus.
type processor interface {
	// Name emits the name of this processor's signature encoder.  It must be
	// fully-qualified in the sense that it could be used via a Protocol Buffer
	// registry to extract the descriptor to reassemble this message.
	Name() string
	// Signature emits a byte signature for this process for the purpose of
	// remarking how far along it has been applied to the database.
	Signature() (signature []byte, err error)
	// Apply runs this processor against the sample set.  sampleIterator expects
	// to be pre-seeked to the initial starting position.  The processor will
	// run until up until stopAt has been reached.  It is imperative that the
	// provided stopAt is within the interval of the series frontier.
	//
	// Upon completion or error, the last time at which the processor finished
	// shall be emitted in addition to any errors.
	Apply(sampleIterator leveldb.Iterator, samples raw.Persistence, stopAt time.Time, fingerprint model.Fingerprint) (lastCurated time.Time, err error)
}

// compactionProcessor combines sparse values in the database together such
// that at least MinimumGroupSize-sized chunks are grouped together.
type compactionProcessor struct {
	// MaximumMutationPoolBatch represents approximately the largest pending
	// batch of mutation operations for the database before pausing to
	// commit before resumption.
	//
	// A reasonable value would be (MinimumGroupSize * 2) + 1.
	MaximumMutationPoolBatch int
	// MinimumGroupSize represents the smallest allowed sample chunk size in the
	// database.
	MinimumGroupSize int
	// signature is the byte representation of the compactionProcessor's settings,
	// used for purely memoization purposes across an instance.
	signature []byte
}

func (p compactionProcessor) Name() string {
	return "io.prometheus.CompactionProcessorDefinition"
}

func (p *compactionProcessor) Signature() (out []byte, err error) {
	if len(p.signature) == 0 {
		out, err = proto.Marshal(&dto.CompactionProcessorDefinition{
			MinimumGroupSize: proto.Uint32(uint32(p.MinimumGroupSize)),
		})

		p.signature = out
	}

	out = p.signature

	return
}

func (p compactionProcessor) String() string {
	return fmt.Sprintf("compactionProcess for minimum group size %d", p.MinimumGroupSize)
}

func (p compactionProcessor) Apply(sampleIterator leveldb.Iterator, samples raw.Persistence, stopAt time.Time, fingerprint model.Fingerprint) (lastCurated time.Time, err error) {
	var pendingBatch raw.Batch = nil

	defer func() {
		if pendingBatch != nil {
			pendingBatch.Close()
		}
	}()

	var pendingMutations = 0
	var pendingSamples model.Values
	var sampleKey model.SampleKey
	var sampleValues model.Values
	var lastTouchedTime time.Time
	var keyDropped bool

	sampleKey, err = extractSampleKey(sampleIterator)
	if err != nil {
		return
	}
	sampleValues, err = extractSampleValues(sampleIterator)
	if err != nil {
		return
	}

	for lastCurated.Before(stopAt) && lastTouchedTime.Before(stopAt) {
		switch {
		// Furnish a new pending batch operation if none is available.
		case pendingBatch == nil:
			pendingBatch = leveldb.NewBatch()

		// If there are no sample values to extract from the datastore, let's
		// continue extracting more values to use.  We know that the time.Before()
		// block would prevent us from going into unsafe territory.
		case len(sampleValues) == 0:
			if !sampleIterator.Next() {
				return lastCurated, fmt.Errorf("Illegal Condition: Invalid Iterator on Continuation")
			}

			keyDropped = false

			sampleKey, err = extractSampleKey(sampleIterator)
			if err != nil {
				return
			}
			sampleValues, err = extractSampleValues(sampleIterator)
			if err != nil {
				return
			}

		// If the number of pending mutations exceeds the allowed batch amount,
		// commit to disk and delete the batch.  A new one will be recreated if
		// necessary.
		case pendingMutations >= p.MaximumMutationPoolBatch:
			err = samples.Commit(pendingBatch)
			if err != nil {
				return
			}

			pendingMutations = 0

			pendingBatch.Close()
			pendingBatch = nil

		case len(pendingSamples) == 0 && len(sampleValues) >= p.MinimumGroupSize:
			lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
			sampleValues = model.Values{}

		case len(pendingSamples)+len(sampleValues) < p.MinimumGroupSize:
			if !keyDropped {
				key := coding.NewProtocolBuffer(sampleKey.ToDTO())
				pendingBatch.Drop(key)
				keyDropped = true
			}
			pendingSamples = append(pendingSamples, sampleValues...)
			lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
			sampleValues = model.Values{}
			pendingMutations++

		// If the number of pending writes equals the target group size
		case len(pendingSamples) == p.MinimumGroupSize:
			newSampleKey := pendingSamples.ToSampleKey(fingerprint)
			key := coding.NewProtocolBuffer(newSampleKey.ToDTO())
			value := coding.NewProtocolBuffer(pendingSamples.ToDTO())
			pendingBatch.Put(key, value)
			pendingMutations++
			lastCurated = newSampleKey.FirstTimestamp.In(time.UTC)
			if len(sampleValues) > 0 {
				if !keyDropped {
					key := coding.NewProtocolBuffer(sampleKey.ToDTO())
					pendingBatch.Drop(key)
					keyDropped = true
				}

				if len(sampleValues) > p.MinimumGroupSize {
					pendingSamples = sampleValues[:p.MinimumGroupSize]
					sampleValues = sampleValues[p.MinimumGroupSize:]
					lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
				} else {
					pendingSamples = sampleValues
					lastTouchedTime = pendingSamples[len(pendingSamples)-1].Timestamp
					sampleValues = model.Values{}
				}
			}

		case len(pendingSamples)+len(sampleValues) >= p.MinimumGroupSize:
			if !keyDropped {
				key := coding.NewProtocolBuffer(sampleKey.ToDTO())
				pendingBatch.Drop(key)
				keyDropped = true
			}
			remainder := p.MinimumGroupSize - len(pendingSamples)
			pendingSamples = append(pendingSamples, sampleValues[:remainder]...)
			sampleValues = sampleValues[remainder:]
			if len(sampleValues) == 0 {
				lastTouchedTime = pendingSamples[len(pendingSamples)-1].Timestamp
			} else {
				lastTouchedTime = sampleValues[len(sampleValues)-1].Timestamp
			}
			pendingMutations++

		default:
			err = fmt.Errorf("Unhandled processing case.")
		}
	}

	if len(sampleValues) > 0 || len(pendingSamples) > 0 {
		pendingSamples = append(sampleValues, pendingSamples...)
		newSampleKey := pendingSamples.ToSampleKey(fingerprint)
		key := coding.NewProtocolBuffer(newSampleKey.ToDTO())
		value := coding.NewProtocolBuffer(pendingSamples.ToDTO())
		pendingBatch.Put(key, value)
		pendingSamples = model.Values{}
		pendingMutations++
		lastCurated = newSampleKey.FirstTimestamp.In(time.UTC)
	}

	// This is not deferred due to the off-chance that a pre-existing commit
	// failed.
	if pendingBatch != nil && pendingMutations > 0 {
		err = samples.Commit(pendingBatch)
		if err != nil {
			return
		}
	}

	return
}
<|endoftext|>