<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modfetch

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"cmd/go/internal/base"
	"cmd/go/internal/dirhash"
	"cmd/go/internal/module"
	"cmd/go/internal/par"
)

var downloadCache par.Cache

// Download downloads the specific module version to the
// local download cache and returns the name of the directory
// corresponding to the root of the module's file tree.
func Download(mod module.Version) (dir string, err error) {
	if SrcMod == "" {
		// Do not download to current directory.
		return "", fmt.Errorf("missing modfetch.SrcMod")
	}

	// The par.Cache here avoids duplicate work but also
	// avoids conflicts from simultaneous calls by multiple goroutines
	// for the same version.
	type cached struct {
		dir string
		err error
	}
	c := downloadCache.Do(mod, func() interface{} {
		dir, err := DownloadDir(mod)
		if err != nil {
			return cached{"", err}
		}
		if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
			zipfile, err := CachePath(mod, "zip")
			if err != nil {
				return cached{"", err}
			}
			if _, err := os.Stat(zipfile); err == nil {
				// Use it.
				// This should only happen if the mod/cache directory is preinitialized
				// or if src/mod/path was removed but not src/mod/cache/download.
				fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
			} else {
				if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
					return cached{"", err}
				}
				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
				if err := downloadZip(mod, zipfile); err != nil {
					return cached{"", err}
				}
			}
			modpath := mod.Path + "@" + mod.Version
			if err := Unzip(dir, zipfile, modpath, 0); err != nil {
				fmt.Fprintf(os.Stderr, "-> %s\n", err)
				return cached{"", err}
			}
		}
		checkSum(mod)
		return cached{dir, nil}
	}).(cached)
	return c.dir, c.err
}

func downloadZip(mod module.Version, target string) error {
	repo, err := Lookup(mod.Path)
	if err != nil {
		return err
	}
	tmpfile, err := repo.Zip(mod.Version, os.TempDir())
	if err != nil {
		return err
	}
	defer os.Remove(tmpfile)

	// Double-check zip file looks OK.
	z, err := zip.OpenReader(tmpfile)
	if err != nil {
		z.Close()
		return err
	}
	prefix := mod.Path + "@" + mod.Version
	for _, f := range z.File {
		if !strings.HasPrefix(f.Name, prefix) {
			z.Close()
			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
		}
	}
	z.Close()

	hash, err := dirhash.HashZip(tmpfile, dirhash.DefaultHash)
	if err != nil {
		return err
	}
	checkOneSum(mod, hash) // check before installing the zip file
	r, err := os.Open(tmpfile)
	if err != nil {
		return err
	}
	defer r.Close()
	w, err := os.Create(target)
	if err != nil {
		return err
	}
	if _, err := io.Copy(w, r); err != nil {
		w.Close()
		return fmt.Errorf("copying: %v", err)
	}
	if err := w.Close(); err != nil {
		return err
	}
	return ioutil.WriteFile(target+"hash", []byte(hash), 0666)
}

var GoSumFile string // path to go.sum; set by package modload

var goSum struct {
	mu        sync.Mutex
	m         map[module.Version][]string // content of go.sum file (+ go.modverify if present)
	enabled   bool                        // whether to use go.sum at all
	modverify string                      // path to go.modverify, to be deleted
}

// initGoSum initializes the go.sum data.
// It reports whether use of go.sum is now enabled.
// The goSum lock must be held.
func initGoSum() bool {
	if GoSumFile == "" {
		return false
	}
	if goSum.m != nil {
		return true
	}

	goSum.m = make(map[module.Version][]string)
	data, err := ioutil.ReadFile(GoSumFile)
	if err != nil && !os.IsNotExist(err) {
		base.Fatalf("go: %v", err)
	}
	goSum.enabled = true
	readGoSum(GoSumFile, data)

	// Add old go.modverify file.
	// We'll delete go.modverify in WriteGoSum.
	alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify"
	if data, err := ioutil.ReadFile(alt); err == nil {
		readGoSum(alt, data)
		goSum.modverify = alt
	}
	return true
}

// emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
// A bug caused us to write these into go.sum files for non-modules.
// We detect and remove them.
const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="

// readGoSum parses data, which is the content of file,
// and adds it to goSum.m. The goSum lock must be held.
func readGoSum(file string, data []byte) {
	lineno := 0
	for len(data) > 0 {
		var line []byte
		lineno++
		i := bytes.IndexByte(data, '\n')
		if i < 0 {
			line, data = data, nil
		} else {
			line, data = data[:i], data[i+1:]
		}
		f := strings.Fields(string(line))
		if len(f) == 0 {
			// blank line; skip it
			continue
		}
		if len(f) != 3 {
			base.Fatalf("go: malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
		}
		if f[2] == emptyGoModHash {
			// Old bug; drop it.
			continue
		}
		mod := module.Version{Path: f[0], Version: f[1]}
		goSum.m[mod] = append(goSum.m[mod], f[2])
	}
}

// checkSum checks the given module's checksum.
func checkSum(mod module.Version) {
	if SrcMod == "" {
		// Do not use current directory.
		return
	}

	// Do the file I/O before acquiring the go.sum lock.
	ziphash, err := CachePath(mod, "ziphash")
	if err != nil {
		base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
	}
	data, err := ioutil.ReadFile(ziphash)
	if err != nil {
		if os.IsNotExist(err) {
			// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
			return
		}
		base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
	}
	h := strings.TrimSpace(string(data))
	if !strings.HasPrefix(h, "h1:") {
		base.Fatalf("go: verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
	}

	checkOneSum(mod, h)
}

// checkGoMod checks the given module's go.mod checksum;
// data is the go.mod content.
func checkGoMod(path, version string, data []byte) {
	h, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
		return ioutil.NopCloser(bytes.NewReader(data)), nil
	})
	if err != nil {
		base.Fatalf("go: verifying %s %s go.mod: %v", path, version, err)
	}

	checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
}

// checkOneSum checks that the recorded hash for mod is h.
func checkOneSum(mod module.Version, h string) {
	goSum.mu.Lock()
	defer goSum.mu.Unlock()
	if !initGoSum() {
		return
	}

	for _, vh := range goSum.m[mod] {
		if h == vh {
			return
		}
		if strings.HasPrefix(vh, "h1:") {
			base.Fatalf("go: verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum:     %v", mod.Path, mod.Version, h, vh)
		}
	}
	if len(goSum.m[mod]) > 0 {
		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
	}
	goSum.m[mod] = append(goSum.m[mod], h)
}

// Sum returns the checksum for the downloaded copy of the given module,
// if present in the download cache.
func Sum(mod module.Version) string {
	if SrcMod == "" {
		// Do not use current directory.
		return ""
	}

	ziphash, err := CachePath(mod, "ziphash")
	if err != nil {
		return ""
	}
	data, err := ioutil.ReadFile(ziphash)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(data))
}

// WriteGoSum writes the go.sum file if it needs to be updated.
func WriteGoSum() {
	goSum.mu.Lock()
	defer goSum.mu.Unlock()
	if !initGoSum() {
		return
	}

	var mods []module.Version
	for m := range goSum.m {
		mods = append(mods, m)
	}
	module.Sort(mods)
	var buf bytes.Buffer
	for _, m := range mods {
		list := goSum.m[m]
		sort.Strings(list)
		for _, h := range list {
			fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
		}
	}

	data, _ := ioutil.ReadFile(GoSumFile)
	if !bytes.Equal(data, buf.Bytes()) {
		if err := ioutil.WriteFile(GoSumFile, buf.Bytes(), 0666); err != nil {
			base.Fatalf("go: writing go.sum: %v", err)
		}
	}

	if goSum.modverify != "" {
		os.Remove(goSum.modverify)
	}
}

// TrimGoSum trims go.sum to contain only the modules for which keep[m] is true.
func TrimGoSum(keep map[module.Version]bool) {
	goSum.mu.Lock()
	defer goSum.mu.Unlock()
	if !initGoSum() {
		return
	}

	for m := range goSum.m {
		// If we're keeping x@v we also keep x@v/go.mod.
		// Map x@v/go.mod back to x@v for the keep lookup.
		noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")}
		if !keep[m] && !keep[noGoMod] {
			delete(goSum.m, m)
		}
	}
}
<commit_msg>cmd/go/internal/modfetch: do not panic when zip is incorrect<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modfetch

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"cmd/go/internal/base"
	"cmd/go/internal/dirhash"
	"cmd/go/internal/module"
	"cmd/go/internal/par"
)

var downloadCache par.Cache

// Download downloads the specific module version to the
// local download cache and returns the name of the directory
// corresponding to the root of the module's file tree.
func Download(mod module.Version) (dir string, err error) {
	if SrcMod == "" {
		// Do not download to current directory.
		return "", fmt.Errorf("missing modfetch.SrcMod")
	}

	// The par.Cache here avoids duplicate work but also
	// avoids conflicts from simultaneous calls by multiple goroutines
	// for the same version.
	type cached struct {
		dir string
		err error
	}
	c := downloadCache.Do(mod, func() interface{} {
		dir, err := DownloadDir(mod)
		if err != nil {
			return cached{"", err}
		}
		if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
			zipfile, err := CachePath(mod, "zip")
			if err != nil {
				return cached{"", err}
			}
			if _, err := os.Stat(zipfile); err == nil {
				// Use it.
				// This should only happen if the mod/cache directory is preinitialized
				// or if src/mod/path was removed but not src/mod/cache/download.
				fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
			} else {
				if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
					return cached{"", err}
				}
				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
				if err := downloadZip(mod, zipfile); err != nil {
					return cached{"", err}
				}
			}
			modpath := mod.Path + "@" + mod.Version
			if err := Unzip(dir, zipfile, modpath, 0); err != nil {
				fmt.Fprintf(os.Stderr, "-> %s\n", err)
				return cached{"", err}
			}
		}
		checkSum(mod)
		return cached{dir, nil}
	}).(cached)
	return c.dir, c.err
}

func downloadZip(mod module.Version, target string) error {
	repo, err := Lookup(mod.Path)
	if err != nil {
		return err
	}
	tmpfile, err := repo.Zip(mod.Version, os.TempDir())
	if err != nil {
		return err
	}
	defer os.Remove(tmpfile)

	// Double-check zip file looks OK.
	z, err := zip.OpenReader(tmpfile)
	if err != nil {
		return err
	}
	prefix := mod.Path + "@" + mod.Version
	for _, f := range z.File {
		if !strings.HasPrefix(f.Name, prefix) {
			z.Close()
			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
		}
	}
	z.Close()

	hash, err := dirhash.HashZip(tmpfile, dirhash.DefaultHash)
	if err != nil {
		return err
	}
	checkOneSum(mod, hash) // check before installing the zip file
	r, err := os.Open(tmpfile)
	if err != nil {
		return err
	}
	defer r.Close()
	w, err := os.Create(target)
	if err != nil {
		return err
	}
	if _, err := io.Copy(w, r); err != nil {
		w.Close()
		return fmt.Errorf("copying: %v", err)
	}
	if err := w.Close(); err != nil {
		return err
	}
	return ioutil.WriteFile(target+"hash", []byte(hash), 0666)
}

var GoSumFile string // path to go.sum; set by package modload

var goSum struct {
	mu        sync.Mutex
	m         map[module.Version][]string // content of go.sum file (+ go.modverify if present)
	enabled   bool                        // whether to use go.sum at all
	modverify string                      // path to go.modverify, to be deleted
}

// initGoSum initializes the go.sum data.
// It reports whether use of go.sum is now enabled.
// The goSum lock must be held.
func initGoSum() bool {
	if GoSumFile == "" {
		return false
	}
	if goSum.m != nil {
		return true
	}

	goSum.m = make(map[module.Version][]string)
	data, err := ioutil.ReadFile(GoSumFile)
	if err != nil && !os.IsNotExist(err) {
		base.Fatalf("go: %v", err)
	}
	goSum.enabled = true
	readGoSum(GoSumFile, data)

	// Add old go.modverify file.
	// We'll delete go.modverify in WriteGoSum.
	alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify"
	if data, err := ioutil.ReadFile(alt); err == nil {
		readGoSum(alt, data)
		goSum.modverify = alt
	}
	return true
}

// emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
// A bug caused us to write these into go.sum files for non-modules.
// We detect and remove them.
const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="

// readGoSum parses data, which is the content of file,
// and adds it to goSum.m. The goSum lock must be held.
func readGoSum(file string, data []byte) {
	lineno := 0
	for len(data) > 0 {
		var line []byte
		lineno++
		i := bytes.IndexByte(data, '\n')
		if i < 0 {
			line, data = data, nil
		} else {
			line, data = data[:i], data[i+1:]
		}
		f := strings.Fields(string(line))
		if len(f) == 0 {
			// blank line; skip it
			continue
		}
		if len(f) != 3 {
			base.Fatalf("go: malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
		}
		if f[2] == emptyGoModHash {
			// Old bug; drop it.
			continue
		}
		mod := module.Version{Path: f[0], Version: f[1]}
		goSum.m[mod] = append(goSum.m[mod], f[2])
	}
}

// checkSum checks the given module's checksum.
func checkSum(mod module.Version) {
	if SrcMod == "" {
		// Do not use current directory.
		return
	}

	// Do the file I/O before acquiring the go.sum lock.
	ziphash, err := CachePath(mod, "ziphash")
	if err != nil {
		base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
	}
	data, err := ioutil.ReadFile(ziphash)
	if err != nil {
		if os.IsNotExist(err) {
			// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
			return
		}
		base.Fatalf("go: verifying %s@%s: %v", mod.Path, mod.Version, err)
	}
	h := strings.TrimSpace(string(data))
	if !strings.HasPrefix(h, "h1:") {
		base.Fatalf("go: verifying %s@%s: unexpected ziphash: %q", mod.Path, mod.Version, h)
	}

	checkOneSum(mod, h)
}

// checkGoMod checks the given module's go.mod checksum;
// data is the go.mod content.
func checkGoMod(path, version string, data []byte) {
	h, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
		return ioutil.NopCloser(bytes.NewReader(data)), nil
	})
	if err != nil {
		base.Fatalf("go: verifying %s %s go.mod: %v", path, version, err)
	}

	checkOneSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
}

// checkOneSum checks that the recorded hash for mod is h.
func checkOneSum(mod module.Version, h string) {
	goSum.mu.Lock()
	defer goSum.mu.Unlock()
	if !initGoSum() {
		return
	}

	for _, vh := range goSum.m[mod] {
		if h == vh {
			return
		}
		if strings.HasPrefix(vh, "h1:") {
			base.Fatalf("go: verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum:     %v", mod.Path, mod.Version, h, vh)
		}
	}
	if len(goSum.m[mod]) > 0 {
		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
	}
	goSum.m[mod] = append(goSum.m[mod], h)
}

// Sum returns the checksum for the downloaded copy of the given module,
// if present in the download cache.
func Sum(mod module.Version) string {
	if SrcMod == "" {
		// Do not use current directory.
		return ""
	}

	ziphash, err := CachePath(mod, "ziphash")
	if err != nil {
		return ""
	}
	data, err := ioutil.ReadFile(ziphash)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(data))
}

// WriteGoSum writes the go.sum file if it needs to be updated.
func WriteGoSum() {
	goSum.mu.Lock()
	defer goSum.mu.Unlock()
	if !initGoSum() {
		return
	}

	var mods []module.Version
	for m := range goSum.m {
		mods = append(mods, m)
	}
	module.Sort(mods)
	var buf bytes.Buffer
	for _, m := range mods {
		list := goSum.m[m]
		sort.Strings(list)
		for _, h := range list {
			fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
		}
	}

	data, _ := ioutil.ReadFile(GoSumFile)
	if !bytes.Equal(data, buf.Bytes()) {
		if err := ioutil.WriteFile(GoSumFile, buf.Bytes(), 0666); err != nil {
			base.Fatalf("go: writing go.sum: %v", err)
		}
	}

	if goSum.modverify != "" {
		os.Remove(goSum.modverify)
	}
}

// TrimGoSum trims go.sum to contain only the modules for which keep[m] is true.
func TrimGoSum(keep map[module.Version]bool) {
	goSum.mu.Lock()
	defer goSum.mu.Unlock()
	if !initGoSum() {
		return
	}

	for m := range goSum.m {
		// If we're keeping x@v we also keep x@v/go.mod.
		// Map x@v/go.mod back to x@v for the keep lookup.
		noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")}
		if !keep[m] && !keep[noGoMod] {
			delete(goSum.m, m)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	log "github.com/Sirupsen/logrus"

	"github.com/projectcalico/calicoctl/calico_node/calicoclient"
	"github.com/projectcalico/libcalico-go/lib/api"
	"github.com/projectcalico/libcalico-go/lib/client"
	"github.com/projectcalico/libcalico-go/lib/net"
)

// This file contains the main processing for the allocate_ipip_addr binary
// used by calico/node to set the host's tunnel address to an IPIP-enabled
// address if there are any available, otherwise it removes any tunnel address
// that is configured.

func main() {
	// Load the client config from environment.
	_, c := calicoclient.CreateClient()

	// The allocate_ipip_addr binary is only ever invoked _after_ the
	// startup binary has been invoked and the modified environments have
	// been sourced.  Therefore, the NODENAME environment will always be
	// set at this point.
	nodename := os.Getenv("NODENAME")
	if nodename == "" {
		log.Panic("NODENAME environment is not set")
	}

	// Query the IPIP enabled pools and either configure the tunnel
	// address, or remove it.
	if cidrs := getIPIPEnabledPoolCIDRs(c); len(cidrs) > 0 {
		ensureHostTunnelAddress(c, nodename, cidrs)
	} else {
		removeHostTunnelAddr(c, nodename)
	}
}

// ensureHostTunnelAddress that ensures the host has a valid IP address for the
// IPIP tunnel device. This must be an IP address claimed from one of the IPIP
// pools.  This function handles re-allocating the address if it finds an
// existing address that is not from an IPIP pool.
func ensureHostTunnelAddress(c *client.Client, nodename string, ipipCidrs []net.IPNet) {
	log.WithField("Node", nodename).Debug("Ensure IPIP tunnel address is set")

	// Get the currently configured IPIP address.
	if ipAddr, err := c.Config().GetNodeIPIPTunnelAddress(nodename); err != nil {
		log.WithError(err).Fatal("Unable to retrieve IPIP tunnel address")
	} else if ipAddr == nil {
		// The IPIP tunnel has no IP address assigned, assign one.
		log.Debug("IPIP tunnel is not assigned - assign IP")
		assignHostTunnelAddr(c, nodename, ipipCidrs)
	} else if isIpInPool(ipAddr, ipipCidrs) {
		// The IPIP tunnel address is still valid, so leave as it.
		log.WithField("IP", ipAddr.String()).Info("IPIP tunnel address is still valid")
	} else {
		// The address that is currently assigned is no longer part
		// of an IPIP pool, so release the IP, and reassign.
		log.WithField("IP", ipAddr.String()).Info("Reassigning IPIP tunnel address")

		ipsToRelease := []net.IP{*ipAddr}
		_, err := c.IPAM().ReleaseIPs(ipsToRelease)
		if err != nil {
			log.WithField("IP", ipAddr.String()).WithError(err).Fatal("Error releasing non IPIP address")
		}

		// Assign a new tunnel address.
		assignHostTunnelAddr(c, nodename, ipipCidrs)
	}
}

// removeHostTunnelAddr removes any existing IP address for this host's IPIP
// tunnel device and releases the IP from IPAM.  If no IP is assigned this function
// is a no-op.
func removeHostTunnelAddr(c *client.Client, nodename string) {
	if ipAddr, err := c.Config().GetNodeIPIPTunnelAddress(nodename); err != nil {
		log.WithError(err).Fatal("Unable to retrieve IPIP tunnel address for cleanup")
	} else if ipAddr == nil {
		log.Debug("No IPIP tunnel address assigned, and not required")
	} else if _, err := c.IPAM().ReleaseIPs([]net.IP{*ipAddr}); err != nil {
		log.WithError(err).WithField("IP", ipAddr.String()).Fatal("Error releasing IPIP address from IPAM")
	} else if err = c.Config().SetNodeIPIPTunnelAddress(nodename, nil); err != nil {
		log.WithError(err).Fatal("Unable to remove IPIP tunnel address")
	}
}

// assignHostTunnelAddr claims an IPIP-enabled IP address from the first pool
// with some space. Stores the result in the host's config as its tunnel
// address.
func assignHostTunnelAddr(c *client.Client, nodename string, ipipCidrs []net.IPNet) {
	args := client.AutoAssignArgs{
		Num4:      1,
		Num6:      0,
		HandleID:  nil,
		Attrs:     nil,
		Hostname:  nodename,
		IPv4Pools: ipipCidrs,
	}

	if ipv4Addrs, _, err := c.IPAM().AutoAssign(args); err != nil {
		log.WithError(err).Fatal("Unable to autoassign an address for IPIP")
	} else if len(ipv4Addrs) == 0 {
		log.Fatal("Unable to autoassign an address for IPIP - pools are likely exhausted.")
	} else if err = c.Config().SetNodeIPIPTunnelAddress(nodename, &ipv4Addrs[0]); err != nil {
		log.WithError(err).WithField("IP", ipv4Addrs[0].String()).Fatal("Unable to set IPIP tunnel address")
	} else {
		log.WithField("IP", ipv4Addrs[0].String()).Info("Set IPIP tunnel address")
	}
}

// isIpInPool returns if the IP address is in one of the supplied pools.
func isIpInPool(ipAddr *net.IP, ipipCidrs []net.IPNet) bool {
	for _, cidr := range ipipCidrs {
		if cidr.Contains(ipAddr.IP) {
			return true
		}
	}
	return false
}

// getIPIPEnabledPools returns all IPIP enabled pools.
func getIPIPEnabledPoolCIDRs(c *client.Client) []net.IPNet {
	meta := api.IPPoolMetadata{}
	ipPoolList, err := c.IPPools().List(meta)
	if err != nil {
		log.WithError(err).Fatal("Unable to query IP pool configuration")
	}

	var cidrs []net.IPNet
	for _, ipPool := range ipPoolList.Items {
		if ipPool.Spec.IPIP != nil && ipPool.Spec.IPIP.Enabled {
			cidrs = append(cidrs, ipPool.Metadata.CIDR)
		}
	}
	return cidrs
}
<commit_msg>Check if an IP pool is enabled and is IPv4 before allocating IPIP tunl IP<commit_after>package main

import (
	"os"

	log "github.com/Sirupsen/logrus"

	"github.com/projectcalico/calicoctl/calico_node/calicoclient"
	"github.com/projectcalico/libcalico-go/lib/api"
	"github.com/projectcalico/libcalico-go/lib/client"
	"github.com/projectcalico/libcalico-go/lib/net"
)

// This file contains the main processing for the allocate_ipip_addr binary
// used by calico/node to set the host's tunnel address to an IPIP-enabled
// address if there are any available, otherwise it removes any tunnel address
// that is configured.

func main() {
	// Load the client config from environment.
	_, c := calicoclient.CreateClient()

	// The allocate_ipip_addr binary is only ever invoked _after_ the
	// startup binary has been invoked and the modified environments have
	// been sourced.  Therefore, the NODENAME environment will always be
	// set at this point.
	nodename := os.Getenv("NODENAME")
	if nodename == "" {
		log.Panic("NODENAME environment is not set")
	}

	// Query the IPIP enabled pools and either configure the tunnel
	// address, or remove it.
	if cidrs := getIPIPEnabledPoolCIDRs(c); len(cidrs) > 0 {
		ensureHostTunnelAddress(c, nodename, cidrs)
	} else {
		removeHostTunnelAddr(c, nodename)
	}
}

// ensureHostTunnelAddress that ensures the host has a valid IP address for the
// IPIP tunnel device. This must be an IP address claimed from one of the IPIP
// pools.  This function handles re-allocating the address if it finds an
// existing address that is not from an IPIP pool.
func ensureHostTunnelAddress(c *client.Client, nodename string, ipipCidrs []net.IPNet) {
	log.WithField("Node", nodename).Debug("Ensure IPIP tunnel address is set")

	// Get the currently configured IPIP address.
	if ipAddr, err := c.Config().GetNodeIPIPTunnelAddress(nodename); err != nil {
		log.WithError(err).Fatal("Unable to retrieve IPIP tunnel address")
	} else if ipAddr == nil {
		// The IPIP tunnel has no IP address assigned, assign one.
		log.Debug("IPIP tunnel is not assigned - assign IP")
		assignHostTunnelAddr(c, nodename, ipipCidrs)
	} else if isIpInPool(ipAddr, ipipCidrs) {
		// The IPIP tunnel address is still valid, so leave as it.
		log.WithField("IP", ipAddr.String()).Info("IPIP tunnel address is still valid")
	} else {
		// The address that is currently assigned is no longer part
		// of an IPIP pool, so release the IP, and reassign.
		log.WithField("IP", ipAddr.String()).Info("Reassigning IPIP tunnel address")

		ipsToRelease := []net.IP{*ipAddr}
		_, err := c.IPAM().ReleaseIPs(ipsToRelease)
		if err != nil {
			log.WithField("IP", ipAddr.String()).WithError(err).Fatal("Error releasing non IPIP address")
		}

		// Assign a new tunnel address.
		assignHostTunnelAddr(c, nodename, ipipCidrs)
	}
}

// removeHostTunnelAddr removes any existing IP address for this host's IPIP
// tunnel device and releases the IP from IPAM.  If no IP is assigned this function
// is a no-op.
func removeHostTunnelAddr(c *client.Client, nodename string) {
	if ipAddr, err := c.Config().GetNodeIPIPTunnelAddress(nodename); err != nil {
		log.WithError(err).Fatal("Unable to retrieve IPIP tunnel address for cleanup")
	} else if ipAddr == nil {
		log.Debug("No IPIP tunnel address assigned, and not required")
	} else if _, err := c.IPAM().ReleaseIPs([]net.IP{*ipAddr}); err != nil {
		log.WithError(err).WithField("IP", ipAddr.String()).Fatal("Error releasing IPIP address from IPAM")
	} else if err = c.Config().SetNodeIPIPTunnelAddress(nodename, nil); err != nil {
		log.WithError(err).Fatal("Unable to remove IPIP tunnel address")
	}
}

// assignHostTunnelAddr claims an IPIP-enabled IP address from the first pool
// with some space. Stores the result in the host's config as its tunnel
// address.
func assignHostTunnelAddr(c *client.Client, nodename string, ipipCidrs []net.IPNet) {
	args := client.AutoAssignArgs{
		Num4:      1,
		Num6:      0,
		HandleID:  nil,
		Attrs:     nil,
		Hostname:  nodename,
		IPv4Pools: ipipCidrs,
	}

	if ipv4Addrs, _, err := c.IPAM().AutoAssign(args); err != nil {
		log.WithError(err).Fatal("Unable to autoassign an address for IPIP")
	} else if len(ipv4Addrs) == 0 {
		log.Fatal("Unable to autoassign an address for IPIP - pools are likely exhausted.")
	} else if err = c.Config().SetNodeIPIPTunnelAddress(nodename, &ipv4Addrs[0]); err != nil {
		log.WithError(err).WithField("IP", ipv4Addrs[0].String()).Fatal("Unable to set IPIP tunnel address")
	} else {
		log.WithField("IP", ipv4Addrs[0].String()).Info("Set IPIP tunnel address")
	}
}

// isIpInPool returns if the IP address is in one of the supplied pools.
func isIpInPool(ipAddr *net.IP, ipipCidrs []net.IPNet) bool {
	for _, cidr := range ipipCidrs {
		if cidr.Contains(ipAddr.IP) {
			return true
		}
	}
	return false
}

// getIPIPEnabledPools returns all IPIP enabled pools.
func getIPIPEnabledPoolCIDRs(c *client.Client) []net.IPNet {
	meta := api.IPPoolMetadata{}
	ipPoolList, err := c.IPPools().List(meta)
	if err != nil {
		log.WithError(err).Fatal("Unable to query IP pool configuration")
	}

	var cidrs []net.IPNet
	for _, ipPool := range ipPoolList.Items {
		// Check if IPIP is enabled in the IP pool, the IP pool is not disabled, and it is IPv4 pool since we don't support IPIP with IPv6.
		if ipPool.Spec.IPIP != nil && ipPool.Spec.IPIP.Enabled && !ipPool.Spec.Disabled && ipPool.Metadata.CIDR.Version() == 4 {
			cidrs = append(cidrs, ipPool.Metadata.CIDR)
		}
	}
	return cidrs
}
<|endoftext|>


<commit_before>package design

import (
	"os"

	. "github.com/enow-dev/enow/design/constant"
	_ "github.com/enow-dev/enow/design/resource"
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var _ = API("enow", func() {
	Title("enow")
	Description("enow")
	Host(func() string {
		switch os.Getenv("Op") {
		case "develop":
			return "localhost:8080"
		case "staging":
			return "enow-staging.appspot.com"
		case "production":
			return "enow.appspot.com"
		}
		return "localhost:8080"
	}())
	Scheme("http", "https")
	BasePath("/")
	Trait(AdminUserTrait, func() {
		Security(AdminAuth)
		Response(Unauthorized, ErrorMedia)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
	Trait(GeneralUserTrait, func() {
		Security(GeneralAuth)
		Response(Unauthorized, ErrorMedia)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
	Trait(GuestUserTrait, func() {
		Security(GuestAuth)
		Response(Unauthorized, ErrorMedia)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
	Trait(PaginatorHeaderTrait, func() {
		Header("x-search-hits-count")
		Header("link")
		Required("x-search-hits-count", "link")
	})
	Trait(PaginatorHeaderTrait2, func() {
		Header("link")
		Required("link")
	})
	Trait(GAECronTrait, func() {
		Security(GAECronAuth)
		Response(OK)
		Response(Unauthorized, ErrorMedia)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
})

var _ = Resource("swaggerui", func() {
	Origin("*", func() {
		Methods("GET")
	})
	Files("/swaggerui/*filepath", "swaggerui/")
})

var _ = Resource("swagger", func() {
	Files("/swagger.json", "swagger/swagger.json")
})

var _ = Resource("front", func() {
	Files("/favicon.ico", "build/favicon.ico")
	Files("/static/js/*filepath", "build/static/js")
	Files("/static/media/*filepath", "build/static/media")
	Files("/service-worker.js", "build/service-worker.js")
	Files("/manifest.json", "build/manifest.json")
	Files("/asset-manifest.json", "build/asset-manifest.json")
	Files("/reset.css", "build/reset.css")
	Files("/index.css", "build/index.css")
	Files("*", "build")
	Files("/", "build/index.html")
})
<commit_msg>fix: #214<commit_after>package design

import (
	"os"

	. "github.com/enow-dev/enow/design/constant"
	_ "github.com/enow-dev/enow/design/resource"
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var _ = API("enow", func() {
	Title("enow")
	Description("enow")
	Host(func() string {
		switch os.Getenv("Op") {
		case "develop":
			return "localhost:8080"
		case "staging":
			return "enow-staging.appspot.com"
		case "production":
			return "enow.appspot.com"
		}
		return "localhost:8080"
	}())
	Scheme(func() string {
		switch os.Getenv("Op") {
		case "develop":
			return "http"
		case "staging":
			return "https"
		case "production":
			return "https"
		}
		return "http"
	}())
	BasePath("/")
	Trait(AdminUserTrait, func() {
		Security(AdminAuth)
		Response(Unauthorized, ErrorMedia)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
	Trait(GeneralUserTrait, func() {
		Security(GeneralAuth)
		Response(Unauthorized, ErrorMedia)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
	Trait(GuestUserTrait, func() {
		Security(GuestAuth)
		Response(Unauthorized, ErrorMedia)
		Response(NotFound)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
	Trait(PaginatorHeaderTrait, func() {
		Header("x-search-hits-count")
		Header("link")
		Required("x-search-hits-count", "link")
	})
	Trait(PaginatorHeaderTrait2, func() {
		Header("link")
		Required("link")
	})
	Trait(GAECronTrait, func() {
		Security(GAECronAuth)
		Response(OK)
		Response(Unauthorized, ErrorMedia)
		Response(BadRequest, ErrorMedia)
		Response(InternalServerError, ErrorMedia)
	})
})

var _ = Resource("swaggerui", func() {
	Origin("*", func() {
		Methods("GET")
	})
	Files("/swaggerui/*filepath", "swaggerui/")
})

var _ = Resource("swagger", func() {
	Files("/swagger.json", "swagger/swagger.json")
})

var _ = Resource("front", func() {
	Files("/favicon.ico", "build/favicon.ico")
	Files("/static/js/*filepath", "build/static/js")
	Files("/static/media/*filepath", "build/static/media")
	Files("/service-worker.js", "build/service-worker.js")
	Files("/manifest.json", "build/manifest.json")
	Files("/asset-manifest.json", "build/asset-manifest.json")
	Files("/reset.css", "build/reset.css")
	Files("/index.css", "build/index.css")
	Files("*", "build")
	Files("/", "build/index.html")
})
<|endoftext|>


<commit_before>package session_test

import (
	"context"
	"encoding/json"
	"flag"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/mafredri/cdp"
	"github.com/mafredri/cdp/internal/testutil"
	"github.com/mafredri/cdp/protocol/page"
	"github.com/mafredri/cdp/protocol/runtime"
	"github.com/mafredri/cdp/session"
)

func TestManager(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	// Give time for goroutines to settle at the end (increases coverage).
	defer func() {
		time.Sleep(10 * time.Millisecond)
	}()

	c := testutil.NewClient(ctx, t)

	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

	newPage := c.NewPage(ctx)
	// Close later.
	// defer newPage.Close()

	// Test session usage.
	pageConn, err := m.Dial(ctx, newPage.ID())
	if err != nil {
		t.Fatal(err)
	}
	defer pageConn.Close()

	pageC := cdp.NewClient(pageConn)

	err = pageC.Page.Enable(ctx)
	if err != nil {
		t.Error(err)
	}
	fired, err := pageC.Page.DOMContentEventFired(ctx)
	if err != nil {
		t.Fatal(err)
	}
	defer fired.Close()

	// TODO(maf): Use testdata / sample HTML for test.
	_, err = pageC.Page.Navigate(ctx,
		page.NewNavigateArgs("https://www.google.com"))
	if err != nil {
		t.Error(err)
	}

	_, err = fired.Recv()
	if err != nil {
		t.Error(err)
	}

	eval, err := pageC.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(`document.title`))
	if err != nil {
		t.Error(err)
	}

	var title string
	err = json.Unmarshal(eval.Result.Value, &title)
	if err != nil {
		t.Error(err)
	}

	if !strings.Contains(title, "Google") {
		t.Error("bad title:", title)
	}

	// Close the page, this should also close pageConn.
	newPage.Close()
	select {
	case <-pageConn.Context().Done():
	case <-ctx.Done():
		t.Error("timed out waiting for session to close")
	}
}

func TestManager_Close(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)

	// Test connection closure, should close session client.
	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

	newPage := c.NewPage(ctx)
	defer newPage.Close()

	_, err = m.Dial(ctx, newPage.ID()) // Closed by sc.Close().
	if err != nil {
		t.Error(err)
	}

	m.Close()
	_, err = m.Dial(ctx, newPage.ID())
	if err == nil {
		t.Error("Dial: expected error after Close, got nil")
	}
}

func TestManager_ClosesErrorChan(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)
	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		m.Close()
	}()

	for range m.Err() {
		t.Fatal("channel should have been closed")
	}
}

func TestManager_CloseUnderlyingConn(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)

	// Get a target ID for use in Dial (doesn't matter that it's closed).
	p := c.NewPage(ctx)
	p.Close()
	targetID := p.ID()

	// Test connection closure, should close session Manager.
	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

	c.Conn.Close()
	time.Sleep(10 * time.Millisecond) // Give time for context propagation.

	_, err = m.Dial(ctx, targetID)
	if err == nil {
		t.Error("Dial succeeded on a closed connection")
	}
}

func TestManager_NewOnClosedConn(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)
	c.Conn.Close()

	_, err := session.NewManager(c.Client)
	if err == nil {
		t.Error("NewManager: rpcc.Conn is closed, expected error, got nil ")
	}
}

var (
	browserFlag = flag.Bool("browser", false, "Test with browser")
)

func TestMain(m *testing.M) {
	flag.Parse()
	os.Exit(m.Run())
}
<commit_msg>session: Add browser check to tests<commit_after>package session_test

import (
	"context"
	"encoding/json"
	"flag"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/mafredri/cdp"
	"github.com/mafredri/cdp/internal/testutil"
	"github.com/mafredri/cdp/protocol/page"
	"github.com/mafredri/cdp/protocol/runtime"
	"github.com/mafredri/cdp/session"
)

func checkBrowser(t *testing.T) {
	t.Helper()
	if !*browserFlag {
		t.Skip("Test requires browser, skipping...")
	}
}

func TestManager(t *testing.T) {
	checkBrowser(t)

	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	// Give time for goroutines to settle at the end (increases coverage).
	defer func() {
		time.Sleep(10 * time.Millisecond)
	}()

	c := testutil.NewClient(ctx, t)

	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

	newPage := c.NewPage(ctx)

	// Test session usage.
	pageConn, err := m.Dial(ctx, newPage.ID())
	if err != nil {
		t.Fatal(err)
	}
	defer pageConn.Close()

	pageC := cdp.NewClient(pageConn)

	err = pageC.Page.Enable(ctx)
	if err != nil {
		t.Error(err)
	}
	fired, err := pageC.Page.DOMContentEventFired(ctx)
	if err != nil {
		t.Fatal(err)
	}
	defer fired.Close()

	// TODO(maf): Use testdata / sample HTML for test.
	_, err = pageC.Page.Navigate(ctx,
		page.NewNavigateArgs("https://www.google.com"))
	if err != nil {
		t.Error(err)
	}

	_, err = fired.Recv()
	if err != nil {
		t.Error(err)
	}

	eval, err := pageC.Runtime.Evaluate(ctx, runtime.NewEvaluateArgs(`document.title`))
	if err != nil {
		t.Error(err)
	}

	var title string
	err = json.Unmarshal(eval.Result.Value, &title)
	if err != nil {
		t.Error(err)
	}

	if !strings.Contains(title, "Google") {
		t.Error("bad title:", title)
	}

	// Close the page, this should also close pageConn.
	newPage.Close()
	select {
	case <-pageConn.Context().Done():
	case <-ctx.Done():
		t.Error("timed out waiting for session to close")
	}
}

func TestManager_Close(t *testing.T) {
	checkBrowser(t)

	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)

	// Test connection closure, should close session client.
	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

	newPage := c.NewPage(ctx)
	defer newPage.Close()

	_, err = m.Dial(ctx, newPage.ID()) // Closed by sc.Close().
	if err != nil {
		t.Error(err)
	}

	m.Close()
	_, err = m.Dial(ctx, newPage.ID())
	if err == nil {
		t.Error("Dial: expected error after Close, got nil")
	}
}

func TestManager_ClosesErrorChan(t *testing.T) {
	checkBrowser(t)

	ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)
	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		m.Close()
	}()

	for range m.Err() {
		t.Fatal("channel should have been closed")
	}
}

func TestManager_CloseUnderlyingConn(t *testing.T) {
	checkBrowser(t)

	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)

	// Get a target ID for use in Dial (doesn't matter that it's closed).
	p := c.NewPage(ctx)
	p.Close()
	targetID := p.ID()

	// Test connection closure, should close session Manager.
	m, err := session.NewManager(c.Client)
	if err != nil {
		t.Fatal(err)
	}
	defer m.Close()

	c.Conn.Close()
	time.Sleep(10 * time.Millisecond) // Give time for context propagation.

	_, err = m.Dial(ctx, targetID)
	if err == nil {
		t.Error("Dial succeeded on a closed connection")
	}
}

func TestManager_NewOnClosedConn(t *testing.T) {
	checkBrowser(t)

	ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
	defer cancel()

	c := testutil.NewClient(ctx, t)
	c.Conn.Close()

	_, err := session.NewManager(c.Client)
	if err == nil {
		t.Error("NewManager: rpcc.Conn is closed, expected error, got nil ")
	}
}

var (
	browserFlag = flag.Bool("browser", false, "Test with browser")
)

func TestMain(m *testing.M) {
	flag.Parse()
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Build constraints copied from go's src/os/dir_unix.go
// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package settings

import (
	"os/user"
	"path/filepath"
)

var defaultConfigPaths = []string{
	// This will be prepended by $HOME/.config/gobuster.conf
	"/etc/gobuster.conf",
}

func init() {
	if usr, err := user.Current(); err != nil {
		path := filepath.Join(usr.HomeDir, ".config", "gobuster.conf")
		defaultConfigPaths = append([]string{path}, defaultConfigPaths...)
	}
}
<commit_msg>Fix logic error in loading unix config.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Build constraints copied from go's src/os/dir_unix.go
// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package settings

import (
	"os/user"
	"path/filepath"
)

var defaultConfigPaths = []string{
	// This will be prepended by $HOME/.config/gobuster.conf
	"/etc/gobuster.conf",
}

func init() {
	if usr, err := user.Current(); err == nil {
		path := filepath.Join(usr.HomeDir, ".config", "gobuster.conf")
		defaultConfigPaths = append([]string{path}, defaultConfigPaths...)
	}
}
<|endoftext|>


<commit_before>// Package mysql implements reform.Dialect for MySQL.
package mysql

import (
	"github.com/xaionaro/reform"
)

type mysql struct{}

func (mysql) String() string {
	return "mysql"
}

func (mysql) Placeholder(index int) string {
	return "?"
}

func (mysql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mysql) QuoteIdentifier(identifier string) string {
	return "`" + identifier + "`"
}

func (mysql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.LastInsertId
}

func (mysql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.Limit
}

func (mysql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.EmptyLists
}

func (mysql) ColumnDefinitionForField(field reform.FieldInfo) string {
	panic("Is not implemented, yet")
	return ""
}

func (mysql) ColumnDefinitionPostQueryForField(structInfo reform.StructInfo, field reform.FieldInfo) string {
	panic("Is not implemented, yet")
	return ""
}

// Dialect implements reform.Dialect for MySQL.
var Dialect mysql

// check interface
var _ reform.Dialect = Dialect
<commit_msg>Improved dialects/mysql/mysql.go<commit_after>// Package mysql implements reform.Dialect for MySQL.
package mysql

import (
	"github.com/xaionaro/reform"
)

type mysql struct{}

func (mysql) String() string {
	return "mysql"
}

func (mysql) Placeholder(index int) string {
	return "?"
}

func (mysql) Placeholders(start, count int) []string {
	res := make([]string, count)
	for i := 0; i < count; i++ {
		res[i] = "?"
	}
	return res
}

func (mysql) QuoteIdentifier(identifier string) string {
	return "`" + identifier + "`"
}

func (mysql) LastInsertIdMethod() reform.LastInsertIdMethod {
	return reform.LastInsertId
}

func (mysql) SelectLimitMethod() reform.SelectLimitMethod {
	return reform.Limit
}

func (mysql) DefaultValuesMethod() reform.DefaultValuesMethod {
	return reform.EmptyLists
}

func (mysql) ColumnTypeForField(field reform.FieldInfo) string {
	switch field.Type {
	case "time.Time", "extime.Time":
		return "timestamp"
	case "int":
		return "integer"
	case "int64":
		return "bigint"
	case "string":
		return "text"
	default:
		return "text"
	}
}

func (mysql) ColumnDefinitionForField(field reform.FieldInfo) string {
	canBeNull := false
	fieldType := field.Type
	if fieldType[0:1] == "*" {
		canBeNull = true
		fieldType = fieldType[1:]
	}

	columnType := Dialect.ColumnTypeForField(field)

	definition := field.Column + " " + columnType	// TODO: escape everything

	if field.IsPK {
		definition += " PRIMARY KEY"
		if fieldType == "int" {
			definition += " AUTO_INCREMENT"
		}
	}

	if field.IsUnique {
		panic("Is not implemented, yet")
	}

	if !canBeNull {
		definition += " NOT NULL"
	}

	return definition
}

func (mysql) ColumnDefinitionPostQueryForField(structInfo reform.StructInfo, field reform.FieldInfo) string {
	if field.HasIndex {
		panic("Is not implemented, yet")
	}

	return ""
}

// Dialect implements reform.Dialect for MySQL.
var Dialect mysql

// check interface
var _ reform.Dialect = Dialect
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dircache

// This file defines structures that keep track of individual target directories.
// It particular it keeps a count of entries from the directory still in the LRU
// and handles refreshing of directory entries.

import (
	"sync"
	"time"

	"upspin.io/access"
	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/path"
	"upspin.io/upspin"
)

// proxiedDir contains information about a proxied user directoies.
type proxiedDir struct {
	l     *clog
	atime time.Time // time of last access
	user  upspin.UserName

	// sequence is the last sequence number seen in a watch. It is only
	// set outside the watcher before any watcher starts
	// while reading the log files.
	sequence int64

	// ep is only used outside the watcher and is the
	// endpoint of the server being watched.
	ep upspin.Endpoint

	die   chan bool // channel used to tell watcher to die
	dying chan bool // channel used to confirm watcher is dying

	// For retrying a watch.
	retryInterval time.Duration
	wake          chan bool
}

// proxiedDirs is used to translate between a user name and the relevant cached directory.
type proxiedDirs struct {
	sync.Mutex

	closing bool // when this is true do not allocate any new watchers
	l       *clog
	m       map[upspin.UserName]*proxiedDir
}

func newProxiedDirs(l *clog) *proxiedDirs {
	return &proxiedDirs{m: make(map[upspin.UserName]*proxiedDir), l: l}
}

// proxyFor saves the endpoint and makes sure it is being watched.
func (p *proxiedDirs) proxyFor(name upspin.PathName, ep *upspin.Endpoint) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]

	// If the endpoint changed, kill off the current watcher.
	if d != nil && d.ep != *ep {
		d.close()
		d.ep = *ep
	}

	if d == nil {
		d = &proxiedDir{l: p.l, ep: *ep, user: u}
		p.m[u] = d
	}

	// Remember when we last accessed this proxied directory.
	// TODO: Use this time to stop listening to directories we
	// haven't looked at in a long time. We will also have to
	// forget about cached information for them if we stop
	// watching.
	d.atime = time.Now()

	// Start a watcher if none is running.
	if d.die == nil {
		d.die = make(chan bool)
		d.dying = make(chan bool)
		d.wake = make(chan bool, 1)
		go d.watcher(*ep)
	}
}

// retryWatch wakes up watcher (if it exists) to try the Watch again.
func (p *proxiedDirs) retryWatch(parsed path.Parsed) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}
	d := p.m[parsed.User()]
	if d == nil || d.wake == nil {
		return
	}
	select {
	case d.wake <- true:
	default:
	}
}

// setSequence remembers a sequence read from the logfile.
func (p *proxiedDirs) setSequence(name upspin.PathName, sequence int64) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]
	if d == nil {
		d = &proxiedDir{l: p.l, user: u}
		p.m[u] = d
	}
	d.sequence = sequence
}

// close terminates the goroutines associated with a proxied dir.
func (d *proxiedDir) close() {
	if d.die != nil {
		close(d.die)
		<-d.dying
		d.die = nil
	}
}

const (
	initialRetryInterval = time.Second
	maxRetryInterval     = time.Minute
)

// watcher watches a directory and caches any changes to something already in the LRU.
func (d *proxiedDir) watcher(ep upspin.Endpoint) {
	log.Debug.Printf("dircache.Watcher %s %s", d.user, ep)
	defer close(d.dying)

	// If we don't know better, always read in the whole state. It
	// is shorter than the the history of all operations.
	if d.sequence == 0 {
		d.sequence = -1
	}

	d.retryInterval = initialRetryInterval
	for {
		err := d.watch(ep)
		if err == nil {
			log.Debug.Printf("dircache.Watcher %s %s exiting", d.user, ep)
			// watch() only returns if the watcher has been told to die
			// or if there is an error requiring a new Watch.
			return
		}
		if err == upspin.ErrNotSupported {
			// Can't survive this.
			log.Debug.Printf("dir/dircache.watcher: %s: %s", d.user, err)
			return
		}
		if errors.Is(errors.Invalid, err) {
			// A bad record in the log or a bad sequence number. Reread current state.
			log.Info.Printf("dir/dircache.watcher restarting from -1: %s: %s", d.user, err)
			d.sequence = -1
		} else {
			log.Info.Printf("dir/dircache.watcher: %s: %s", d.user, err)
		}

		select {
		case <-time.After(d.retryInterval):
			d.retryInterval *= 2
			if d.retryInterval > maxRetryInterval {
				d.retryInterval = maxRetryInterval
			}
		case <-d.wake:
		}
	}
}

// watch loops receiving watch events. It returns nil if told to die.
// Otherwise it returns whatever error was encountered.
func (d *proxiedDir) watch(ep upspin.Endpoint) error {
	dir, err := bind.DirServer(d.l.cfg, ep)
	if err != nil {
		return err
	}
	done := make(chan struct{})
	defer close(done)
	event, err := dir.Watch(upspin.PathName(string(d.user)+"/"), d.sequence, done)
	if err != nil {
		return err
	}

	// If Watch succeeds, go back to the initial interval.
	d.retryInterval = initialRetryInterval

	// Loop receiving events until we are told to stop or the event stream is closed.
	for {
		select {
		case <-d.die:
			return nil
		case e, ok := <-event:
			if !ok {
				return errors.Str("Watch event stream closed")
			}
			if err := d.handleEvent(&e); err != nil {
				return err
			}
		}
	}
}

func (d *proxiedDir) handleEvent(e *upspin.Event) error {
	// Something odd happened?
	if e.Error != nil {
		return e.Error
	}

	// If we are rereading the current state, wipe what we know.
	if d.sequence == -1 {
		d.l.wipeLog(d.user)
	}
	log.Debug.Printf("watch entry %s %v", e.Entry.Name, e)

	// Is this a file we are watching? We always watch Access files since ones we never
	// saw before can affect our cached state.
	if !access.IsAccessFile(e.Entry.Name) {
		_, ok := d.l.lru.Get(lruKey{name: e.Entry.Name, glob: false})
		if !ok {
			// Not a file we are watching, how about in a directory we are watching?
			dirName := path.DropPath(e.Entry.Name, 1)
			if dirName == e.Entry.Name {
				return nil
			}
			_, ok := d.l.lru.Get(lruKey{name: dirName, glob: true})
			if !ok {
				return nil
			}
		}
	}

	// This is an event we care about.
	d.sequence = e.Entry.Sequence
	op := lookupReq
	if e.Delete {
		op = deleteReq
	}
	d.l.globalLock.Lock()
	d.l.logRequestWithSequence(op, e.Entry.Name, nil, e.Entry, e.Entry.Sequence)
	d.l.globalLock.Unlock()

	d.l.flush()
	return nil
}
<commit_msg>dir/dircache: add context to watch event debug log message<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dircache

// This file defines structures that keep track of individual target directories.
// It particular it keeps a count of entries from the directory still in the LRU
// and handles refreshing of directory entries.

import (
	"sync"
	"time"

	"upspin.io/access"
	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/path"
	"upspin.io/upspin"
)

// proxiedDir contains information about a proxied user directoies.
type proxiedDir struct {
	l     *clog
	atime time.Time // time of last access
	user  upspin.UserName

	// sequence is the last sequence number seen in a watch. It is only
	// set outside the watcher before any watcher starts
	// while reading the log files.
	sequence int64

	// ep is only used outside the watcher and is the
	// endpoint of the server being watched.
	ep upspin.Endpoint

	die   chan bool // channel used to tell watcher to die
	dying chan bool // channel used to confirm watcher is dying

	// For retrying a watch.
	retryInterval time.Duration
	wake          chan bool
}

// proxiedDirs is used to translate between a user name and the relevant cached directory.
type proxiedDirs struct {
	sync.Mutex

	closing bool // when this is true do not allocate any new watchers
	l       *clog
	m       map[upspin.UserName]*proxiedDir
}

func newProxiedDirs(l *clog) *proxiedDirs {
	return &proxiedDirs{m: make(map[upspin.UserName]*proxiedDir), l: l}
}

// proxyFor saves the endpoint and makes sure it is being watched.
func (p *proxiedDirs) proxyFor(name upspin.PathName, ep *upspin.Endpoint) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]

	// If the endpoint changed, kill off the current watcher.
	if d != nil && d.ep != *ep {
		d.close()
		d.ep = *ep
	}

	if d == nil {
		d = &proxiedDir{l: p.l, ep: *ep, user: u}
		p.m[u] = d
	}

	// Remember when we last accessed this proxied directory.
	// TODO: Use this time to stop listening to directories we
	// haven't looked at in a long time. We will also have to
	// forget about cached information for them if we stop
	// watching.
	d.atime = time.Now()

	// Start a watcher if none is running.
	if d.die == nil {
		d.die = make(chan bool)
		d.dying = make(chan bool)
		d.wake = make(chan bool, 1)
		go d.watcher(*ep)
	}
}

// retryWatch wakes up watcher (if it exists) to try the Watch again.
func (p *proxiedDirs) retryWatch(parsed path.Parsed) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}
	d := p.m[parsed.User()]
	if d == nil || d.wake == nil {
		return
	}
	select {
	case d.wake <- true:
	default:
	}
}

// setSequence remembers a sequence read from the logfile.
func (p *proxiedDirs) setSequence(name upspin.PathName, sequence int64) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]
	if d == nil {
		d = &proxiedDir{l: p.l, user: u}
		p.m[u] = d
	}
	d.sequence = sequence
}

// close terminates the goroutines associated with a proxied dir.
func (d *proxiedDir) close() {
	if d.die != nil {
		close(d.die)
		<-d.dying
		d.die = nil
	}
}

const (
	initialRetryInterval = time.Second
	maxRetryInterval     = time.Minute
)

// watcher watches a directory and caches any changes to something already in the LRU.
func (d *proxiedDir) watcher(ep upspin.Endpoint) {
	log.Debug.Printf("dircache.Watcher %s %s", d.user, ep)
	defer close(d.dying)

	// If we don't know better, always read in the whole state. It
	// is shorter than the the history of all operations.
	if d.sequence == 0 {
		d.sequence = -1
	}

	d.retryInterval = initialRetryInterval
	for {
		err := d.watch(ep)
		if err == nil {
			log.Debug.Printf("dircache.Watcher %s %s exiting", d.user, ep)
			// watch() only returns if the watcher has been told to die
			// or if there is an error requiring a new Watch.
			return
		}
		if err == upspin.ErrNotSupported {
			// Can't survive this.
			log.Debug.Printf("dir/dircache.watcher: %s: %s", d.user, err)
			return
		}
		if errors.Is(errors.Invalid, err) {
			// A bad record in the log or a bad sequence number. Reread current state.
			log.Info.Printf("dir/dircache.watcher restarting from -1: %s: %s", d.user, err)
			d.sequence = -1
		} else {
			log.Info.Printf("dir/dircache.watcher: %s: %s", d.user, err)
		}

		select {
		case <-time.After(d.retryInterval):
			d.retryInterval *= 2
			if d.retryInterval > maxRetryInterval {
				d.retryInterval = maxRetryInterval
			}
		case <-d.wake:
		}
	}
}

// watch loops receiving watch events. It returns nil if told to die.
// Otherwise it returns whatever error was encountered.
func (d *proxiedDir) watch(ep upspin.Endpoint) error {
	dir, err := bind.DirServer(d.l.cfg, ep)
	if err != nil {
		return err
	}
	name := upspin.PathName(string(d.user) + "/")
	done := make(chan struct{})
	defer close(done)
	event, err := dir.Watch(name, d.sequence, done)
	if err != nil {
		return err
	}

	// If Watch succeeds, go back to the initial interval.
	d.retryInterval = initialRetryInterval

	// Loop receiving events until we are told to stop or the event stream is closed.
	for {
		select {
		case <-d.die:
			return nil
		case e, ok := <-event:
			if !ok {
				return errors.Str("Watch event stream closed")
			}
			if e.Error != nil {
				log.Debug.Printf("dir/dircache: Watch(%q) error: %s", name, e.Error)
			} else {
				log.Debug.Printf("dir/dircache: Watch(%q) entry: %s (delete=%t)", name, e.Entry.Name, e.Delete)
			}
			if err := d.handleEvent(&e); err != nil {
				return err
			}
		}
	}
}

func (d *proxiedDir) handleEvent(e *upspin.Event) error {
	// Something odd happened?
	if e.Error != nil {
		return e.Error
	}

	// If we are rereading the current state, wipe what we know.
	if d.sequence == -1 {
		d.l.wipeLog(d.user)
	}

	// Is this a file we are watching? We always watch Access files since ones we never
	// saw before can affect our cached state.
	if !access.IsAccessFile(e.Entry.Name) {
		_, ok := d.l.lru.Get(lruKey{name: e.Entry.Name, glob: false})
		if !ok {
			// Not a file we are watching, how about in a directory we are watching?
			dirName := path.DropPath(e.Entry.Name, 1)
			if dirName == e.Entry.Name {
				return nil
			}
			_, ok := d.l.lru.Get(lruKey{name: dirName, glob: true})
			if !ok {
				return nil
			}
		}
	}

	// This is an event we care about.
	d.sequence = e.Entry.Sequence
	op := lookupReq
	if e.Delete {
		op = deleteReq
	}
	d.l.globalLock.Lock()
	d.l.logRequestWithSequence(op, e.Entry.Name, nil, e.Entry, e.Entry.Sequence)
	d.l.globalLock.Unlock()

	d.l.flush()
	return nil
}
<|endoftext|>


<commit_before>package raft

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

type requestForVote struct {
	Term         int
	CandidateID  string
	LastLogIndex int
	LastLogTerm  int
}

type voteRequest struct {
	Req        requestForVote
	ReturnChan chan requestForVoteResponse
}

type requestForVoteResponse struct {
	Term           int
	HasGrantedVote bool
}

type voteResponse struct {
	ServerIndex int
	Resp        requestForVoteResponse
}

func (s *server) stepDown(reason string) {
	fmt.Println("SETTING FOLLOWER & RESETTING ELECTION TIMEOUT: ", reason)
	s.state = "follower"
	s.electionTimeout.reset()
}

func (s *server) becomeLeader() {
	//Initialize nextIndex values to the index after the last one in leader's log
	for follower, _ := range s.nextIndex {
		s.nextIndex[follower] = len(s.db.Log.Entries)
	}
	fmt.Println(s.nextIndex)
	fmt.Println("SETTING LEADER: got majority vote")
	s.state = "leader"
	fmt.Println("IM THE LEADER :D :D :D :D :D ")
}

func (s *server) handleRequestForVote(v voteRequest) {
	fmt.Println("Got vote request:", v.Req)
	req := v.Req
	returnChan := v.ReturnChan
	if req.Term < s.term {
		resp := &requestForVoteResponse{s.term, false}
		returnChan <- *resp
	} else {
		if req.Term > s.term {
			s.votedFor = ""
			s.stepDown("Got vote request with term > current term")
		}
		cond1 := s.votedFor == ""
		cond2 := s.votedFor == req.CandidateID
		cond3 := req.LastLogIndex >= len(s.db.Log.Entries)-1
		if (cond1 || cond2) && cond3 {
			s.votedFor = req.CandidateID
			s.term = req.Term
			resp := &requestForVoteResponse{s.term, true}
			returnChan <- *resp
		}
	}
}

func (s *server) sendRequestForVote(receiverIndex int, respChan chan voteResponse) {
	receiver := s.config[receiverIndex]
	lastLogIndex := len(s.db.Log.Entries) - 1
	lastLogTerm := 0
	if lastLogIndex > 0 {
		lastLogTerm = s.db.Log.Entries[lastLogIndex].Term
	}
	v := &requestForVote{s.term, s.id, lastLogIndex, lastLogTerm}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(v)
	resp, err := http.Post("http://"+receiver+"/votes", "application/json", b)
	if err != nil {
		r := &requestForVoteResponse{0, false}
		v := &voteResponse{receiverIndex, *r}
		fmt.Println("Couldn't send request for votes to " + receiver)
		fmt.Println(err)
		respChan <- *v
	} else {
		r := &requestForVoteResponse{}
		err := json.NewDecoder(resp.Body).Decode(r)
		if err != nil {
			r := &requestForVoteResponse{0, false}
			v := &voteResponse{receiverIndex, *r}
			fmt.Println("Couldn't decode request for vote response from ", s.config[receiverIndex])
			respChan <- *v
			return
		}
		resp.Body.Close()
		voteResp := &voteResponse{receiverIndex, *r}
		respChan <- *voteResp
	}
}

func (s *server) startElection() {
	fmt.Println("SETTING CANDIDATE: Going to start election")
	s.state = "candidate"
	s.term += 1
	s.votedFor = s.id
	voteCount := 1
	respChan := make(chan voteResponse)
	for receiverIndex, receiverId := range s.config {
		if receiverId != s.id {
			go s.sendRequestForVote(receiverIndex, respChan)
		}
	}
	responseCount := 0
	if len(s.config) > 1 {
		for {
			vote := <-respChan
			responseCount++
			fmt.Println("Got vote response", vote)
			if vote.Resp.Term > s.term {
				s.stepDown("Got vote response with term greater than current term")
				break
			}
			if vote.Resp.HasGrantedVote {
				voteCount++
			}
			if voteCount > (len(s.config)-1)/2 {
				s.becomeLeader()
				break
			}
			if responseCount == len(s.config)-2 {
				fmt.Println("Got all the responses")
				break
			}
		}
	}
}
<commit_msg>print time when stepping down and starting election<commit_after>package raft

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"time"
)

type requestForVote struct {
	Term         int
	CandidateID  string
	LastLogIndex int
	LastLogTerm  int
}

type voteRequest struct {
	Req        requestForVote
	ReturnChan chan requestForVoteResponse
}

type requestForVoteResponse struct {
	Term           int
	HasGrantedVote bool
}

type voteResponse struct {
	ServerIndex int
	Resp        requestForVoteResponse
}

func (s *server) stepDown(reason string) {
	t := time.Now()
	fmt.Println(t.Format("15:04:05.999999"), "SETTING FOLLOWER & RESETTING ELECTION TIMEOUT: ", reason)
	s.state = "follower"
	s.electionTimeout.reset()
}

func (s *server) becomeLeader() {
	//Initialize nextIndex values to the index after the last one in leader's log
	for follower, _ := range s.nextIndex {
		s.nextIndex[follower] = len(s.db.Log.Entries)
	}
	fmt.Println(s.nextIndex)
	fmt.Println("SETTING LEADER: got majority vote")
	s.state = "leader"
	fmt.Println("IM THE LEADER :D :D :D :D :D ")
}

func (s *server) handleRequestForVote(v voteRequest) {
	fmt.Println("Got vote request:", v.Req)
	req := v.Req
	returnChan := v.ReturnChan
	if req.Term < s.term {
		resp := &requestForVoteResponse{s.term, false}
		returnChan <- *resp
	} else {
		if req.Term > s.term {
			s.votedFor = ""
			s.stepDown("Got vote request with term > current term")
		}
		cond1 := s.votedFor == ""
		cond2 := s.votedFor == req.CandidateID
		cond3 := req.LastLogIndex >= len(s.db.Log.Entries)-1
		if (cond1 || cond2) && cond3 {
			s.votedFor = req.CandidateID
			s.term = req.Term
			resp := &requestForVoteResponse{s.term, true}
			returnChan <- *resp
		}
	}
}

func (s *server) sendRequestForVote(receiverIndex int, respChan chan voteResponse) {
	receiver := s.config[receiverIndex]
	lastLogIndex := len(s.db.Log.Entries) - 1
	lastLogTerm := 0
	if lastLogIndex > 0 {
		lastLogTerm = s.db.Log.Entries[lastLogIndex].Term
	}
	v := &requestForVote{s.term, s.id, lastLogIndex, lastLogTerm}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(v)
	resp, err := http.Post("http://"+receiver+"/votes", "application/json", b)
	if err != nil {
		r := &requestForVoteResponse{0, false}
		v := &voteResponse{receiverIndex, *r}
		fmt.Println("Couldn't send request for votes to " + receiver)
		fmt.Println(err)
		respChan <- *v
	} else {
		r := &requestForVoteResponse{}
		err := json.NewDecoder(resp.Body).Decode(r)
		if err != nil {
			r := &requestForVoteResponse{0, false}
			v := &voteResponse{receiverIndex, *r}
			fmt.Println("Couldn't decode request for vote response from ", s.config[receiverIndex])
			respChan <- *v
			return
		}
		resp.Body.Close()
		voteResp := &voteResponse{receiverIndex, *r}
		respChan <- *voteResp
	}
}

func (s *server) startElection() {
	t := time.Now()
	fmt.Println(t.Format("15:04:05.999999"), "SETTING CANDIDATE: Going to start election")
	s.state = "candidate"
	s.term += 1
	s.votedFor = s.id
	voteCount := 1
	respChan := make(chan voteResponse)
	for receiverIndex, receiverId := range s.config {
		if receiverId != s.id {
			go s.sendRequestForVote(receiverIndex, respChan)
		}
	}
	responseCount := 0
	if len(s.config) > 1 {
		for {
			vote := <-respChan
			responseCount++
			fmt.Println("Got vote response", vote)
			if vote.Resp.Term > s.term {
				s.stepDown("Got vote response with term greater than current term")
				break
			}
			if vote.Resp.HasGrantedVote {
				voteCount++
			}
			if voteCount > (len(s.config)-1)/2 {
				s.becomeLeader()
				break
			}
			if responseCount == len(s.config)-2 {
				fmt.Println("Got all the responses")
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>package rbg2p

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/dlclark/regexp2"
)

func isVar(s string) bool {
	return strings.HasPrefix(s, "VAR ")
}

func isTest(s string) bool {
	return strings.HasPrefix(s, "TEST ")
}

func isFilter(s string) bool {
	return strings.HasPrefix(s, "FILTER ")
}

//var g2pLineRe = regexp.MustCompile("^(CHARACTER_SET|TEST|DEFAULT_PHONEME|FILTER|VAR|) .*")
var g2pLineRe = regexp.MustCompile("^(CHARACTER_SET|TEST|DEFAULT_PHONEME|FILTER|VAR|DOWNCASE_INPUT) .*")

func isG2PLine(s string) bool {
	return g2pLineRe.MatchString(s) || ruleRe.MatchString(s)
}

// LoadFile loads a g2p rule set from the specified file
func LoadFile(fName string) (RuleSet, error) {
	ruleSet := RuleSet{Vars: map[string]string{}}
	ruleSet.DefaultPhoneme = "_"
	ruleSet.PhonemeDelimiter = " "
	syllDefLines := []string{}
	fh, err := os.Open(fName)
	defer fh.Close()
	if err != nil {
		return ruleSet, err
	}
	n := 0
	s := bufio.NewScanner(fh)
	var inputLines []string
	var ruleLines []string
	var filterLines []string
	var phonemeSetLine string
	for s.Scan() {
		if err := s.Err(); err != nil {
			return ruleSet, err
		}
		n++
		lOrig := strings.TrimSpace(s.Text())
		l := trimComment(lOrig)
		inputLines = append(inputLines, lOrig)
		if isBlankLine(l) || isComment(l) {
		} else if isPhonemeDelimiter(l) {
			delim, err := parsePhonemeDelimiter(l)
			if err != nil {
				return ruleSet, err
			}
			ruleSet.PhonemeDelimiter = delim
		} else if isPhonemeSet(l) {
			phonemeSetLine = l
		} else if isConst(l) {
			err := parseConst(l, &ruleSet)
			if err != nil {
				return ruleSet, err
			}
		} else if isVar(l) {
			name, value, err := newVar(l)
			if err != nil {
				return ruleSet, err
			}
			ruleSet.Vars[name] = value
		} else if isSyllDefLine(l) {
			syllDefLines = append(syllDefLines, l)
		} else if isFilter(l) {
			filterLines = append(filterLines, l)
		} else if isTest(l) {
			t, err := newTest(l)
			if err != nil {
				return ruleSet, err
			}
			ruleSet.Tests = append(ruleSet.Tests, t)
		} else { // is a rule
			ruleLines = append(ruleLines, l)
		}

	}
	for k, v := range ruleSet.Vars {
		v = expandVarsWithBrackets(v, ruleSet.Vars)
		ruleSet.Vars[k] = v
	}
	if len(phonemeSetLine) > 0 {
		phnSet, err := parsePhonemeSet(phonemeSetLine, ruleSet.PhonemeDelimiter)
		if err != nil {
			return ruleSet, err
		}
		ruleSet.PhonemeSet = phnSet
	}
	if len(syllDefLines) > 0 {
		syllDef, stressPlacement, err := loadSyllDef(syllDefLines, ruleSet.PhonemeDelimiter)
		if err != nil {
			return ruleSet, err
		}
		ruleSet.SyllableDelimiter = syllDef.SyllableDelimiter()
		ruleSet.Syllabifier = Syllabifier{SyllDef: syllDef, StressPlacement: stressPlacement, PhonemeSet: ruleSet.PhonemeSet}
	}

	for _, l := range filterLines {
		t, err := newFilter(l, ruleSet.Vars)
		if err != nil {
			return ruleSet, err
		}
		ruleSet.Filters = append(ruleSet.Filters, t)
	}
	for _, l := range ruleLines {
		r, err := newRule(l, ruleSet.Vars)
		if err != nil {
			return ruleSet, err
		}
		for _, r0 := range ruleSet.Rules {
			if r0.equalsExceptOutput(r) {
				return ruleSet, fmt.Errorf("Duplicate rules for input file %s: %s vs. %s", fName, r0, r)
			}
		}
		ruleSet.Rules = append(ruleSet.Rules, r)
	}
	if ruleSet.CharacterSet == nil || len(ruleSet.CharacterSet) == 0 {
		return ruleSet, fmt.Errorf("No character set defined for input file %s", fName)
	}
	ruleSet.Content = strings.Join(inputLines, "\n")
	return ruleSet, nil
}

var constRe = regexp.MustCompile("^(CHARACTER_SET|DEFAULT_PHONEME|DOWNCASE_INPUT) (?:\"(.+)\"|([^\"]+))$")
var isConstRe = regexp.MustCompile("^(CHARACTER_SET|DEFAULT_PHONEME|DOWNCASE_INPUT) .*")
var isTrueRe = regexp.MustCompile("^(true|TRUE|1)$")
var isFalseRe = regexp.MustCompile("^(false|FALSE|0)$")

func isConst(s string) bool {
	return isConstRe.MatchString(s)
}

func parseConst(s string, ruleSet *RuleSet) error {
	var matchRes []string
	matchRes = constRe.FindStringSubmatch(s)
	var downcaseInputIsSet = false
	if matchRes != nil {
		name := matchRes[1]
		value := matchRes[2]
		if value == "" {
			value = matchRes[3]
		}
		if name == "CHARACTER_SET" {
			ruleSet.CharacterSet = strings.Split(value, "")
		} else if name == "DEFAULT_PHONEME" {
			ruleSet.DefaultPhoneme = value
		} else if name == "DOWNCASE_INPUT" {
			if isTrueRe.MatchString(value) {
				ruleSet.DowncaseInput = true
				downcaseInputIsSet = true
			} else if isFalseRe.MatchString(value) {
				ruleSet.DowncaseInput = false
				downcaseInputIsSet = true
			} else {
				return fmt.Errorf("invalid boolean value for " + name + ": " + value)
			}
		} else {
			return fmt.Errorf("invalid const definition: " + s)
		}
	} else {
		return fmt.Errorf("invalid const definition: " + s)
	}
	if !downcaseInputIsSet {
		ruleSet.DowncaseInput = true
	}
	return nil
}

var varRe = regexp.MustCompile("^VAR +([^ \"]+) +([^ \"]+|[^ ].*[^ ])$")
var varReQuoteFix = regexp.MustCompile(`^"(.*)"$`)

func newVar(s string) (string, string, error) {
	// VAR NAME VALUE
	matchRes := varRe.FindStringSubmatch(s)
	if matchRes == nil {
		return "", "", fmt.Errorf("invalid VAR definition: " + s)
	}
	name := matchRes[1]
	value := matchRes[2]
	_, err := regexp2.Compile(value, regexp2.None)
	if err != nil {
		msg := fmt.Sprintf("invalid VAR input - regular expression failed: %s : %v", s, err)
		return "", "", fmt.Errorf(msg)
	}
	if strings.Contains(name, "_") {
		return "", "", fmt.Errorf("invalid VAR input - var names cannot contain underscore: %s", s)
	}
	if strings.HasPrefix(strings.TrimSpace(value), "=") {
		return "", "", fmt.Errorf("invalid VAR input - var names cannot start with equal sign: %s", s)
	}
	quoteFix := varReQuoteFix.FindStringSubmatch(value)
	if quoteFix != nil {
		value = quoteFix[1]
	}
	return name, value, nil
}

var testReSimple = regexp.MustCompile("^TEST +([^ ]+) +-> +([^,()]+)$")
var testReVariants = regexp.MustCompile("^TEST +([^ ]+) +-> +[(](.+,.+)[)]$")

func newTest(s string) (Test, error) {
	var outputS string
	var matchRes []string
	matchRes = testReSimple.FindStringSubmatch(s)
	if matchRes != nil {
		outputS = matchRes[2]
	} else {
		matchRes = testReVariants.FindStringSubmatch(s)
		if matchRes == nil {
			return Test{}, fmt.Errorf("invalid TEST definition: " + s)
		}
		outputS = matchRes[2]
	}
	if strings.Contains(outputS, "->") {
		return Test{}, fmt.Errorf("invalid TEST definition: " + s)
	}
	input := matchRes[1]
	output := commaSplit.Split(outputS, -1)
	return Test{Input: input, Output: output}, nil
}

var filterRe = regexp.MustCompile("^FILTER +\"(.+)\" +-> +\"(.+)\"$")

func newFilter(s string, vars map[string]string) (Filter, error) {
	matchRes := filterRe.FindStringSubmatch(s)
	if matchRes == nil {
		return Filter{}, fmt.Errorf("invalid FILTER definition: " + s)
	}
	input := matchRes[1]
	output := strings.Replace(matchRes[2], "\\\"", "\"", -1)
	if strings.Contains(output, "->") {
		return Filter{}, fmt.Errorf("invalid FILTER definition: " + s)
	}
	input = expandVarsWithBrackets(input, vars)
	re, err := regexp2.Compile(input, regexp2.None)
	if err != nil {
		return Filter{}, fmt.Errorf("invalid FILTER definition (invalid regexp /%s/): %s", s, err)
	}
	return Filter{Regexp: re, Output: output}, nil
}

func expandVarsWithBrackets(re0 string, vars map[string]string) string {
	re := re0
	for k, v := range vars {
		k = fmt.Sprintf("{%s}", k)
		re = strings.Replace(re, k, v, -1)
	}
	return re
}

func expandVars(s0 string, isLeft bool, vars map[string]string) (*regexp2.Regexp, error) {
	if isLeft {
		s0 = strings.Replace(s0, "#", "^", -1)
	} else {
		s0 = strings.Replace(s0, "#", "$", -1)
	}
	splitted := strings.Split(s0, " ")
	for i, s := range splitted {
		if val, ok := vars[strings.TrimSpace(s)]; ok {
			splitted[i] = val
		}
	}
	if isLeft {
		return regexp2.Compile(strings.Join(splitted, "")+"$", regexp2.None)
	}
	return regexp2.Compile("^"+strings.Join(splitted, ""), regexp2.None)
}

var contextRe = regexp.MustCompile("^ +/ +((?:[^_>]+)?) *_ *((?:[^_>]+)?)$")

func newContext(s string, vars map[string]string) (Context, Context, error) {
	if len(strings.TrimSpace(s)) == 0 {
		return Context{}, Context{}, nil
	}
	matchRes := contextRe.FindStringSubmatch(s)
	if matchRes == nil {
		return Context{}, Context{}, fmt.Errorf("invalid context definition: " + s)
	}
	left := Context{}
	right := Context{}
	leftS := strings.TrimSpace(matchRes[1])
	if len(leftS) > 0 {
		re, err := expandVars(leftS, true, vars)
		if err != nil {
			return Context{}, Context{}, fmt.Errorf("invalid context definition: %s", err)
		}
		left.Regexp = re
		left.Input = leftS
	}
	rightS := strings.TrimSpace(matchRes[2])
	if len(rightS) > 0 {
		re, err := expandVars(rightS, false, vars)
		if err != nil {
			return Context{}, Context{}, fmt.Errorf("invalid context definition: %s", err)
		}
		right.Regexp = re
		right.Input = rightS
	}
	return left, right, nil
}

var ruleRe = regexp.MustCompile("^([^ ]+) +-> +([^/]+)( +/.*$|$)")
var ruleOutputReSimple = regexp.MustCompile("^([^,()]+)$")
var ruleOutputReVariants = regexp.MustCompile("^[(](.+,.+)[)]$")
var emptyOutput = "∅"

func newRuleOutput(s string, l string) ([]string, error) {
	s = strings.TrimSpace(s)
	var outputS string
	var matchRes []string
	matchRes = ruleOutputReSimple.FindStringSubmatch(s)
	if matchRes != nil {
		outputS = matchRes[1]
	} else {
		matchRes = ruleOutputReVariants.FindStringSubmatch(s)
		if matchRes == nil {
			return []string{}, fmt.Errorf("invalid rule output definition: " + l)
		}
		outputS = matchRes[1]
	}
	if strings.Contains(outputS, "->") {
		return []string{}, fmt.Errorf("invalid rule output definition: " + l)
	}
	outputS = strings.Replace(outputS, emptyOutput, "", -1)
	return commaSplit.Split(outputS, -1), nil
}

func newRule(s string, vars map[string]string) (Rule, error) {
	// INPUT -> OUTPUT
	// INPUT -> OUTPUT / LEFTCONTEXT _ RIGHTCONTEXT
	matchRes := ruleRe.FindStringSubmatch(s)
	if matchRes == nil {
		return Rule{}, fmt.Errorf("invalid rule definition: " + s)
	}
	input := matchRes[1]
	output, err := newRuleOutput(matchRes[2], s)
	if err != nil {
		return Rule{}, err
	}
	left, right, err := newContext(matchRes[3], vars)
	if err != nil {
		return Rule{}, err
	}
	return Rule{Input: input, Output: output, LeftContext: left, RightContext: right}, nil
}
<commit_msg>filter can have empty output<commit_after>package rbg2p

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/dlclark/regexp2"
)

func isVar(s string) bool {
	return strings.HasPrefix(s, "VAR ")
}

func isTest(s string) bool {
	return strings.HasPrefix(s, "TEST ")
}

func isFilter(s string) bool {
	return strings.HasPrefix(s, "FILTER ")
}

//var g2pLineRe = regexp.MustCompile("^(CHARACTER_SET|TEST|DEFAULT_PHONEME|FILTER|VAR|) .*")
var g2pLineRe = regexp.MustCompile("^(CHARACTER_SET|TEST|DEFAULT_PHONEME|FILTER|VAR|DOWNCASE_INPUT) .*")

func isG2PLine(s string) bool {
	return g2pLineRe.MatchString(s) || ruleRe.MatchString(s)
}

// LoadFile loads a g2p rule set from the specified file
func LoadFile(fName string) (RuleSet, error) {
	ruleSet := RuleSet{Vars: map[string]string{}}
	ruleSet.DefaultPhoneme = "_"
	ruleSet.PhonemeDelimiter = " "
	syllDefLines := []string{}
	fh, err := os.Open(fName)
	defer fh.Close()
	if err != nil {
		return ruleSet, err
	}
	n := 0
	s := bufio.NewScanner(fh)
	var inputLines []string
	var ruleLines []string
	var filterLines []string
	var phonemeSetLine string
	for s.Scan() {
		if err := s.Err(); err != nil {
			return ruleSet, err
		}
		n++
		lOrig := strings.TrimSpace(s.Text())
		l := trimComment(lOrig)
		inputLines = append(inputLines, lOrig)
		if isBlankLine(l) || isComment(l) {
		} else if isPhonemeDelimiter(l) {
			delim, err := parsePhonemeDelimiter(l)
			if err != nil {
				return ruleSet, err
			}
			ruleSet.PhonemeDelimiter = delim
		} else if isPhonemeSet(l) {
			phonemeSetLine = l
		} else if isConst(l) {
			err := parseConst(l, &ruleSet)
			if err != nil {
				return ruleSet, err
			}
		} else if isVar(l) {
			name, value, err := newVar(l)
			if err != nil {
				return ruleSet, err
			}
			ruleSet.Vars[name] = value
		} else if isSyllDefLine(l) {
			syllDefLines = append(syllDefLines, l)
		} else if isFilter(l) {
			filterLines = append(filterLines, l)
		} else if isTest(l) {
			t, err := newTest(l)
			if err != nil {
				return ruleSet, err
			}
			ruleSet.Tests = append(ruleSet.Tests, t)
		} else { // is a rule
			ruleLines = append(ruleLines, l)
		}

	}
	for k, v := range ruleSet.Vars {
		v = expandVarsWithBrackets(v, ruleSet.Vars)
		ruleSet.Vars[k] = v
	}
	if len(phonemeSetLine) > 0 {
		phnSet, err := parsePhonemeSet(phonemeSetLine, ruleSet.PhonemeDelimiter)
		if err != nil {
			return ruleSet, err
		}
		ruleSet.PhonemeSet = phnSet
	}
	if len(syllDefLines) > 0 {
		syllDef, stressPlacement, err := loadSyllDef(syllDefLines, ruleSet.PhonemeDelimiter)
		if err != nil {
			return ruleSet, err
		}
		ruleSet.SyllableDelimiter = syllDef.SyllableDelimiter()
		ruleSet.Syllabifier = Syllabifier{SyllDef: syllDef, StressPlacement: stressPlacement, PhonemeSet: ruleSet.PhonemeSet}
	}

	for _, l := range filterLines {
		t, err := newFilter(l, ruleSet.Vars)
		if err != nil {
			return ruleSet, err
		}
		ruleSet.Filters = append(ruleSet.Filters, t)
	}
	for _, l := range ruleLines {
		r, err := newRule(l, ruleSet.Vars)
		if err != nil {
			return ruleSet, err
		}
		for _, r0 := range ruleSet.Rules {
			if r0.equalsExceptOutput(r) {
				return ruleSet, fmt.Errorf("Duplicate rules for input file %s: %s vs. %s", fName, r0, r)
			}
		}
		ruleSet.Rules = append(ruleSet.Rules, r)
	}
	if ruleSet.CharacterSet == nil || len(ruleSet.CharacterSet) == 0 {
		return ruleSet, fmt.Errorf("No character set defined for input file %s", fName)
	}
	ruleSet.Content = strings.Join(inputLines, "\n")
	return ruleSet, nil
}

var constRe = regexp.MustCompile("^(CHARACTER_SET|DEFAULT_PHONEME|DOWNCASE_INPUT) (?:\"(.+)\"|([^\"]+))$")
var isConstRe = regexp.MustCompile("^(CHARACTER_SET|DEFAULT_PHONEME|DOWNCASE_INPUT) .*")
var isTrueRe = regexp.MustCompile("^(true|TRUE|1)$")
var isFalseRe = regexp.MustCompile("^(false|FALSE|0)$")

func isConst(s string) bool {
	return isConstRe.MatchString(s)
}

func parseConst(s string, ruleSet *RuleSet) error {
	var matchRes []string
	matchRes = constRe.FindStringSubmatch(s)
	var downcaseInputIsSet = false
	if matchRes != nil {
		name := matchRes[1]
		value := matchRes[2]
		if value == "" {
			value = matchRes[3]
		}
		if name == "CHARACTER_SET" {
			ruleSet.CharacterSet = strings.Split(value, "")
		} else if name == "DEFAULT_PHONEME" {
			ruleSet.DefaultPhoneme = value
		} else if name == "DOWNCASE_INPUT" {
			if isTrueRe.MatchString(value) {
				ruleSet.DowncaseInput = true
				downcaseInputIsSet = true
			} else if isFalseRe.MatchString(value) {
				ruleSet.DowncaseInput = false
				downcaseInputIsSet = true
			} else {
				return fmt.Errorf("invalid boolean value for " + name + ": " + value)
			}
		} else {
			return fmt.Errorf("invalid const definition: " + s)
		}
	} else {
		return fmt.Errorf("invalid const definition: " + s)
	}
	if !downcaseInputIsSet {
		ruleSet.DowncaseInput = true
	}
	return nil
}

var varRe = regexp.MustCompile("^VAR +([^ \"]+) +([^ \"]+|[^ ].*[^ ])$")
var varReQuoteFix = regexp.MustCompile(`^"(.*)"$`)

func newVar(s string) (string, string, error) {
	// VAR NAME VALUE
	matchRes := varRe.FindStringSubmatch(s)
	if matchRes == nil {
		return "", "", fmt.Errorf("invalid VAR definition: " + s)
	}
	name := matchRes[1]
	value := matchRes[2]
	_, err := regexp2.Compile(value, regexp2.None)
	if err != nil {
		msg := fmt.Sprintf("invalid VAR input - regular expression failed: %s : %v", s, err)
		return "", "", fmt.Errorf(msg)
	}
	if strings.Contains(name, "_") {
		return "", "", fmt.Errorf("invalid VAR input - var names cannot contain underscore: %s", s)
	}
	if strings.HasPrefix(strings.TrimSpace(value), "=") {
		return "", "", fmt.Errorf("invalid VAR input - var names cannot start with equal sign: %s", s)
	}
	quoteFix := varReQuoteFix.FindStringSubmatch(value)
	if quoteFix != nil {
		value = quoteFix[1]
	}
	return name, value, nil
}

var testReSimple = regexp.MustCompile("^TEST +([^ ]+) +-> +([^,()]+)$")
var testReVariants = regexp.MustCompile("^TEST +([^ ]+) +-> +[(](.+,.+)[)]$")

func newTest(s string) (Test, error) {
	var outputS string
	var matchRes []string
	matchRes = testReSimple.FindStringSubmatch(s)
	if matchRes != nil {
		outputS = matchRes[2]
	} else {
		matchRes = testReVariants.FindStringSubmatch(s)
		if matchRes == nil {
			return Test{}, fmt.Errorf("invalid TEST definition: " + s)
		}
		outputS = matchRes[2]
	}
	if strings.Contains(outputS, "->") {
		return Test{}, fmt.Errorf("invalid TEST definition: " + s)
	}
	input := matchRes[1]
	output := commaSplit.Split(outputS, -1)
	return Test{Input: input, Output: output}, nil
}

var filterRe = regexp.MustCompile("^FILTER +\"(.+)\" +-> +\"(.*)\"$")

func newFilter(s string, vars map[string]string) (Filter, error) {
	matchRes := filterRe.FindStringSubmatch(s)
	if matchRes == nil {
		return Filter{}, fmt.Errorf("invalid FILTER definition: " + s)
	}
	input := matchRes[1]
	output := strings.Replace(matchRes[2], "\\\"", "\"", -1)
	if strings.Contains(output, "->") {
		return Filter{}, fmt.Errorf("invalid FILTER definition: " + s)
	}
	input = expandVarsWithBrackets(input, vars)
	re, err := regexp2.Compile(input, regexp2.None)
	if err != nil {
		return Filter{}, fmt.Errorf("invalid FILTER definition (invalid regexp /%s/): %s", s, err)
	}
	return Filter{Regexp: re, Output: output}, nil
}

func expandVarsWithBrackets(re0 string, vars map[string]string) string {
	re := re0
	for k, v := range vars {
		k = fmt.Sprintf("{%s}", k)
		re = strings.Replace(re, k, v, -1)
	}
	return re
}

func expandVars(s0 string, isLeft bool, vars map[string]string) (*regexp2.Regexp, error) {
	if isLeft {
		s0 = strings.Replace(s0, "#", "^", -1)
	} else {
		s0 = strings.Replace(s0, "#", "$", -1)
	}
	splitted := strings.Split(s0, " ")
	for i, s := range splitted {
		if val, ok := vars[strings.TrimSpace(s)]; ok {
			splitted[i] = val
		}
	}
	if isLeft {
		return regexp2.Compile(strings.Join(splitted, "")+"$", regexp2.None)
	}
	return regexp2.Compile("^"+strings.Join(splitted, ""), regexp2.None)
}

var contextRe = regexp.MustCompile("^ +/ +((?:[^_>]+)?) *_ *((?:[^_>]+)?)$")

func newContext(s string, vars map[string]string) (Context, Context, error) {
	if len(strings.TrimSpace(s)) == 0 {
		return Context{}, Context{}, nil
	}
	matchRes := contextRe.FindStringSubmatch(s)
	if matchRes == nil {
		return Context{}, Context{}, fmt.Errorf("invalid context definition: " + s)
	}
	left := Context{}
	right := Context{}
	leftS := strings.TrimSpace(matchRes[1])
	if len(leftS) > 0 {
		re, err := expandVars(leftS, true, vars)
		if err != nil {
			return Context{}, Context{}, fmt.Errorf("invalid context definition: %s", err)
		}
		left.Regexp = re
		left.Input = leftS
	}
	rightS := strings.TrimSpace(matchRes[2])
	if len(rightS) > 0 {
		re, err := expandVars(rightS, false, vars)
		if err != nil {
			return Context{}, Context{}, fmt.Errorf("invalid context definition: %s", err)
		}
		right.Regexp = re
		right.Input = rightS
	}
	return left, right, nil
}

var ruleRe = regexp.MustCompile("^([^ ]+) +-> +([^/]+)( +/.*$|$)")
var ruleOutputReSimple = regexp.MustCompile("^([^,()]+)$")
var ruleOutputReVariants = regexp.MustCompile("^[(](.+,.+)[)]$")
var emptyOutput = "∅"

func newRuleOutput(s string, l string) ([]string, error) {
	s = strings.TrimSpace(s)
	var outputS string
	var matchRes []string
	matchRes = ruleOutputReSimple.FindStringSubmatch(s)
	if matchRes != nil {
		outputS = matchRes[1]
	} else {
		matchRes = ruleOutputReVariants.FindStringSubmatch(s)
		if matchRes == nil {
			return []string{}, fmt.Errorf("invalid rule output definition: " + l)
		}
		outputS = matchRes[1]
	}
	if strings.Contains(outputS, "->") {
		return []string{}, fmt.Errorf("invalid rule output definition: " + l)
	}
	outputS = strings.Replace(outputS, emptyOutput, "", -1)
	return commaSplit.Split(outputS, -1), nil
}

func newRule(s string, vars map[string]string) (Rule, error) {
	// INPUT -> OUTPUT
	// INPUT -> OUTPUT / LEFTCONTEXT _ RIGHTCONTEXT
	matchRes := ruleRe.FindStringSubmatch(s)
	if matchRes == nil {
		return Rule{}, fmt.Errorf("invalid rule definition: " + s)
	}
	input := matchRes[1]
	output, err := newRuleOutput(matchRes[2], s)
	if err != nil {
		return Rule{}, err
	}
	left, right, err := newContext(matchRes[3], vars)
	if err != nil {
		return Rule{}, err
	}
	return Rule{Input: input, Output: output, LeftContext: left, RightContext: right}, nil
}
<|endoftext|>