<commit_before>package hostess

import (
	"errors"
	"fmt"
	"net"
	"sort"
)

var InvalidVersionArgumentError = errors.New("Version argument must be 4 or 6")

type Hostlist []*Hostname

// NewHostList initializes a new Hostlist
func NewHostlist() *Hostlist {
	return &Hostlist{}
}

// Len returns the number of hostnames in the list, part of sort.Interface
func (h Hostlist) Len() int {
	return len(h)
}

// Less determines the sort order of two hostnames, part of sort.Interface
func (h Hostlist) Less(i, j int) bool {
	// Sort 127.0.0.1, 127.0.1.1 and "localhost" at the top
	if h[i].Domain == "localhost" {
		return true
	}
	if h[j].Domain == "localhost" {
		return false
	}

	// Sort ipv4 before ipv6
	if h[i].Ipv6 && !h[j].Ipv6 {
		return false
	}
	if !h[i].Ipv6 && h[j].Ipv6 {
		return true
	}

	// Compare the the ip addresses (byte array)
	for c, _ := range h[i].Ip {
		if h[i].Ip[c] < h[j].Ip[c] {
			return true
		} else if h[i].Ip[c] > h[j].Ip[c] {
			return false
		}
	}

	// Prep for domain sorting
	ilen := len(h[i].Domain)
	jlen := len(h[j].Domain)
	max := ilen
	if jlen > max {
		max = jlen
	}

	// Sort domains alphabetically
	// Note: This works best if domains are lowercased. However, we do not
	// enforce lowercase because of UTF-8 domain names, which may be broken by
	// case folding. There is a way to do this correctly but it's completed so
	// I'm not going to do it right now.
	for c := 0; c < max; c++ {
		if c > ilen {
			return true
		}
		if c > jlen {
			return false
		}
		if h[i].Domain[c] < h[j].Domain[c] {
			return true
		}
		if h[i].Domain[c] > h[j].Domain[c] {
			return false
		}
	}

	// Seems like everything was the same, so it can't be Less
	return false
}

// Swap changes the position of two hostnames, part of sort.Interface
func (h Hostlist) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

// Sort this list of Hostnames, according to Hostlist sorting rules:
// 1. localhost comes first
// 2. ipv4 comes first
// 3. ips are sorted in numerical order
// 4. domains are sorted in alphabetical
func (h *Hostlist) Sort() {
	sort.Sort(*h)
}

// Contains returns true if this Hostlist has the specified Hostname
func (h *Hostlist) Contains(b *Hostname) bool {
	for _, a := range *h {
		if a.Equal(b) {
			return true
		}
	}
	return false
}

// ContainsDomain returns true if a Hostname in this Hostlist matches domain
func (h *Hostlist) ContainsDomain(domain string) bool {
	for _, hostname := range *h {
		if hostname.Domain == domain {
			return true
		}
	}
	return false
}

// ContainsDomain returns true if a Hostname in this Hostlist matches ip
func (h *Hostlist) ContainsIp(ip net.IP) bool {
	for _, hostname := range *h {
		if hostname.EqualIp(ip) {
			return true
		}
	}
	return false
}

// Add a new Hostname to this hostlist. If a Hostname with the same domain name
// and Ip version is found, it will be replaced and an error will be returned.
// If you try to add an identical Hostname, an error will be returned.
// Note that in normal operation, you will sometimes expect an error, and the
// error data is mainly to alert you that you mis-entered information, not that
// the application has a problem.
func (h *Hostlist) Add(host *Hostname) error {
	for _, found := range *h {
		if found.Equal(host) {
			return errors.New(fmt.Sprintf("Duplicate hostname entry for %s -> %s",
				host.Domain, host.Ip))
		} else if found.Domain == host.Domain && found.Ipv6 == host.Ipv6 {
			return errors.New(fmt.Sprintf("Conflicting hostname entries for %s -> %s and -> %s",
				host.Domain, host.Ip, found.Ip))
		}
	}
	*h = append(*h, host)
	return nil
}

// IndexOf will indicate the index of a Hostname in Hostlist, or -1 if it is
// not found.
func (h *Hostlist) IndexOf(host *Hostname) int {
	for index, found := range *h {
		if found.Equal(host) {
			return index
		}
	}
	return -1
}

// IndexOfDomainV will indicate the index of a Hostname in Hostlist that has
// the same domain and IP version, or -1 if it is not found.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) IndexOfDomainV(domain string, version int) int {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	for index, hostname := range *h {
		if hostname.Domain == domain && hostname.Ipv6 == (version == 6) {
			return index
		}
	}
	return -1
}

// Remove will delete the Hostname at the specified index. If index is out of
// bounds (i.e. -1), Remove silently no-ops.
func (h *Hostlist) Remove(index int) {
	if index > -1 && index < len(*h) {
		*h = append((*h)[:index], (*h)[index+1:]...)
	}
}

// RemoveDomain removes both Ipv4 and Ipv6 Hostname entries matching domain.
func (h *Hostlist) RemoveDomain(domain string) {
	h.Remove(h.IndexOfDomainV(domain, 4))
	h.Remove(h.IndexOfDomainV(domain, 6))
}

// RemoveDomainV removes a Hostname entry matching the domain and IP version.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) RemoveDomainV(domain string, version int) {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	h.Remove(h.IndexOfDomainV(domain, version))
}

// Enable will change any Hostnames matching domain to be enabled.
func (h *Hostlist) Enable(domain string) {
	for _, hostname := range *h {
		if hostname.Domain == domain {
			hostname.Enabled = true
		}
	}
}

// EnableV will change a Hostname matching domain and IP version to be enabled.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) EnableV(domain string, version int) {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	for _, hostname := range *h {
		if hostname.Domain == domain && hostname.Ipv6 == (version == 6) {
			hostname.Enabled = true
		}
	}
}

// Enable will change any Hostnames matching domain to be disabled.
func (h *Hostlist) Disable(domain string) {
	for _, hostname := range *h {
		if hostname.Domain == domain {
			hostname.Enabled = false
		}
	}
}

// Enable will change any Hostnames matching domain and IP version to be disabled.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) DisableV(domain string, version int) {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	for _, hostname := range *h {
		if hostname.Domain == domain && hostname.Ipv6 == (version == 6) {
			hostname.Enabled = false
		}
	}
}

// Format takes the current list of Hostnames in this Hostfile and turns it
// into a string suitable for use as an /etc/hosts file.
// Sorting uses the following logic:
// 1. List is sorted by IP address
// 2. Commented items are left in place
// 3. 127.* appears at the top of the list (so boot resolvers don't break)
// 4. When present, localhost will always appear first in the domain list
func (h *Hostlist) Format() string {
	h.Sort()
	out := ""
	for _, hostname := range *h {
		out += hostname.Format() + "\n"
	}
	fmt.Println(h)
	return out
}
<commit_msg>Removed debug output<commit_after>package hostess

import (
	"errors"
	"fmt"
	"net"
	"sort"
)

var InvalidVersionArgumentError = errors.New("Version argument must be 4 or 6")

type Hostlist []*Hostname

// NewHostList initializes a new Hostlist
func NewHostlist() *Hostlist {
	return &Hostlist{}
}

// Len returns the number of hostnames in the list, part of sort.Interface
func (h Hostlist) Len() int {
	return len(h)
}

// Less determines the sort order of two hostnames, part of sort.Interface
func (h Hostlist) Less(i, j int) bool {
	// Sort 127.0.0.1, 127.0.1.1 and "localhost" at the top
	if h[i].Domain == "localhost" {
		return true
	}
	if h[j].Domain == "localhost" {
		return false
	}

	// Sort ipv4 before ipv6
	if h[i].Ipv6 && !h[j].Ipv6 {
		return false
	}
	if !h[i].Ipv6 && h[j].Ipv6 {
		return true
	}

	// Compare the the ip addresses (byte array)
	for c, _ := range h[i].Ip {
		if h[i].Ip[c] < h[j].Ip[c] {
			return true
		} else if h[i].Ip[c] > h[j].Ip[c] {
			return false
		}
	}

	// Prep for domain sorting
	ilen := len(h[i].Domain)
	jlen := len(h[j].Domain)
	max := ilen
	if jlen > max {
		max = jlen
	}

	// Sort domains alphabetically
	// Note: This works best if domains are lowercased. However, we do not
	// enforce lowercase because of UTF-8 domain names, which may be broken by
	// case folding. There is a way to do this correctly but it's completed so
	// I'm not going to do it right now.
	for c := 0; c < max; c++ {
		if c > ilen {
			return true
		}
		if c > jlen {
			return false
		}
		if h[i].Domain[c] < h[j].Domain[c] {
			return true
		}
		if h[i].Domain[c] > h[j].Domain[c] {
			return false
		}
	}

	// Seems like everything was the same, so it can't be Less
	return false
}

// Swap changes the position of two hostnames, part of sort.Interface
func (h Hostlist) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

// Sort this list of Hostnames, according to Hostlist sorting rules:
// 1. localhost comes first
// 2. ipv4 comes first
// 3. ips are sorted in numerical order
// 4. domains are sorted in alphabetical
func (h *Hostlist) Sort() {
	sort.Sort(*h)
}

// Contains returns true if this Hostlist has the specified Hostname
func (h *Hostlist) Contains(b *Hostname) bool {
	for _, a := range *h {
		if a.Equal(b) {
			return true
		}
	}
	return false
}

// ContainsDomain returns true if a Hostname in this Hostlist matches domain
func (h *Hostlist) ContainsDomain(domain string) bool {
	for _, hostname := range *h {
		if hostname.Domain == domain {
			return true
		}
	}
	return false
}

// ContainsDomain returns true if a Hostname in this Hostlist matches ip
func (h *Hostlist) ContainsIp(ip net.IP) bool {
	for _, hostname := range *h {
		if hostname.EqualIp(ip) {
			return true
		}
	}
	return false
}

// Add a new Hostname to this hostlist. If a Hostname with the same domain name
// and Ip version is found, it will be replaced and an error will be returned.
// If you try to add an identical Hostname, an error will be returned.
// Note that in normal operation, you will sometimes expect an error, and the
// error data is mainly to alert you that you mis-entered information, not that
// the application has a problem.
func (h *Hostlist) Add(host *Hostname) error {
	for _, found := range *h {
		if found.Equal(host) {
			return errors.New(fmt.Sprintf("Duplicate hostname entry for %s -> %s",
				host.Domain, host.Ip))
		} else if found.Domain == host.Domain && found.Ipv6 == host.Ipv6 {
			return errors.New(fmt.Sprintf("Conflicting hostname entries for %s -> %s and -> %s",
				host.Domain, host.Ip, found.Ip))
		}
	}
	*h = append(*h, host)
	return nil
}

// IndexOf will indicate the index of a Hostname in Hostlist, or -1 if it is
// not found.
func (h *Hostlist) IndexOf(host *Hostname) int {
	for index, found := range *h {
		if found.Equal(host) {
			return index
		}
	}
	return -1
}

// IndexOfDomainV will indicate the index of a Hostname in Hostlist that has
// the same domain and IP version, or -1 if it is not found.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) IndexOfDomainV(domain string, version int) int {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	for index, hostname := range *h {
		if hostname.Domain == domain && hostname.Ipv6 == (version == 6) {
			return index
		}
	}
	return -1
}

// Remove will delete the Hostname at the specified index. If index is out of
// bounds (i.e. -1), Remove silently no-ops.
func (h *Hostlist) Remove(index int) {
	if index > -1 && index < len(*h) {
		*h = append((*h)[:index], (*h)[index+1:]...)
	}
}

// RemoveDomain removes both Ipv4 and Ipv6 Hostname entries matching domain.
func (h *Hostlist) RemoveDomain(domain string) {
	h.Remove(h.IndexOfDomainV(domain, 4))
	h.Remove(h.IndexOfDomainV(domain, 6))
}

// RemoveDomainV removes a Hostname entry matching the domain and IP version.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) RemoveDomainV(domain string, version int) {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	h.Remove(h.IndexOfDomainV(domain, version))
}

// Enable will change any Hostnames matching domain to be enabled.
func (h *Hostlist) Enable(domain string) {
	for _, hostname := range *h {
		if hostname.Domain == domain {
			hostname.Enabled = true
		}
	}
}

// EnableV will change a Hostname matching domain and IP version to be enabled.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) EnableV(domain string, version int) {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	for _, hostname := range *h {
		if hostname.Domain == domain && hostname.Ipv6 == (version == 6) {
			hostname.Enabled = true
		}
	}
}

// Enable will change any Hostnames matching domain to be disabled.
func (h *Hostlist) Disable(domain string) {
	for _, hostname := range *h {
		if hostname.Domain == domain {
			hostname.Enabled = false
		}
	}
}

// Enable will change any Hostnames matching domain and IP version to be disabled.
// This function will panic if IP version is not 4 or 6.
func (h *Hostlist) DisableV(domain string, version int) {
	if version != 4 && version != 6 {
		panic(InvalidVersionArgumentError)
	}
	for _, hostname := range *h {
		if hostname.Domain == domain && hostname.Ipv6 == (version == 6) {
			hostname.Enabled = false
		}
	}
}

// Format takes the current list of Hostnames in this Hostfile and turns it
// into a string suitable for use as an /etc/hosts file.
// Sorting uses the following logic:
// 1. List is sorted by IP address
// 2. Commented items are left in place
// 3. 127.* appears at the top of the list (so boot resolvers don't break)
// 4. When present, localhost will always appear first in the domain list
func (h *Hostlist) Format() string {
	h.Sort()
	out := ""
	for _, hostname := range *h {
		out += hostname.Format() + "\n"
	}
	return out
}
<|endoftext|>


<commit_before>package hostess

import (
	"fmt"
	"net"
	"regexp"
	"strings"
)

var ipv4_pattern = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
var ipv6_pattern = regexp.MustCompile(`^[a-z0-9:]+$`)

func LooksLikeIpv4(ip string) bool {
	return ipv4_pattern.MatchString(ip)
}

func LooksLikeIpv6(ip string) bool {
	if !strings.Contains(ip, ":") {
		return false
	}
	return ipv6_pattern.MatchString(ip)
}

type Hostname struct {
	Domain  string
	Ip      net.IP
	Enabled bool
	Ipv6    bool
}

func NewHostname(domain, ip string, enabled bool) (hostname *Hostname) {
	IP := net.ParseIP(ip)
	hostname = &Hostname{domain, IP, enabled, LooksLikeIpv6(ip)}
	return
}

func (h *Hostname) Format() string {
	r := fmt.Sprintf("%s %s", h.Ip.String(), h.Domain)
	if !h.Enabled {
		r = "# " + r
	}
	return r
}

func (a *Hostname) Equals(b Hostname) bool {
	if a.Domain == b.Domain && a.Ip.Equal(b.Ip) {
		return true
	}
	return false
}
<commit_msg>Added Hostname.Equal() .EqualIp() and .IsValid()<commit_after>package hostess

import (
	"fmt"
	"net"
	"regexp"
	"strings"
)

var ipv4_pattern = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
var ipv6_pattern = regexp.MustCompile(`^[a-z0-9:]+$`)

func LooksLikeIpv4(ip string) bool {
	return ipv4_pattern.MatchString(ip)
}

func LooksLikeIpv6(ip string) bool {
	if !strings.Contains(ip, ":") {
		return false
	}
	return ipv6_pattern.MatchString(ip)
}

type Hostname struct {
	Domain  string
	Ip      net.IP
	Enabled bool
	Ipv6    bool
}

func NewHostname(domain, ip string, enabled bool) (hostname *Hostname) {
	IP := net.ParseIP(ip)
	hostname = &Hostname{domain, IP, enabled, LooksLikeIpv6(ip)}
	return
}

func (h *Hostname) Equal(n *Hostname) bool {
	return h.Ip.Equal(n.Ip) && h.Domain == n.Domain
}

func (h *Hostname) EqualIp(ip net.IP) bool {
	return h.Ip.Equal(ip)
}

func (h *Hostname) IsValid() bool {
	return h.Domain != "" && h.Ip != nil
}

func (h *Hostname) Format() string {
	r := fmt.Sprintf("%s %s", h.Ip.String(), h.Domain)
	if !h.Enabled {
		r = "# " + r
	}
	return r
}

func (a *Hostname) Equals(b Hostname) bool {
	if a.Domain == b.Domain && a.Ip.Equal(b.Ip) {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"fmt"
	"github.com/dghubble/sling"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"errors"
	"strings"
	"os"
	"os/exec"
)

type ApiStateResponse struct {
	Mode        string `json:"mode"`
	Destination string `json:"destination"`
}

type Hoverfly struct {
	Host       string
	AdminPort  string
	ProxyPort  string
	httpClient *http.Client
}

func NewHoverfly(config Config) (Hoverfly) {
	return Hoverfly {
		Host: config.HoverflyHost,
		AdminPort: config.HoverflyAdminPort,
		ProxyPort: config.HoverflyProxyPort,
		httpClient: http.DefaultClient,
	}
}

func (h *Hoverfly) Wipe() (error) {
	url := h.buildUrl("/api/records")

	request, err := sling.New().Delete(url).Request()
	if err != nil {
		log.Debug(err.Error())
		return err
	}

	response, err := h.httpClient.Do(request)
	if err != nil {
		log.Debug(err.Error())
		return err
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return errors.New("Hoverfly did not wipe the database")
	}

	return nil
}

func (h *Hoverfly) GetMode() (string, error) {
	url := h.buildUrl("/api/state")
	request, err := sling.New().Get(url).Request()

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	response, err := h.httpClient.Do(request)

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	defer response.Body.Close()

	apiResponse := h.createApiStateResponse(response)

	return apiResponse.Mode, nil
}

func (h *Hoverfly) SetMode(mode string) (string, error) {
	if mode != "simulate" && mode != "capture" && mode != "modify" && mode != "synthesize" {
		return "", errors.New(mode + " is not a valid mode")
	}

	url := h.buildUrl("/api/state")
	request, err := sling.New().Post(url).Body(strings.NewReader(`{"mode":"` + mode + `"}`)).Request()

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	response, err := h.httpClient.Do(request)

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	apiResponse := h.createApiStateResponse(response)

	return apiResponse.Mode, nil
}

func (h *Hoverfly) ImportSimulation(payload string) (error) {
	url := h.buildUrl("/api/records")
	request, err := sling.New().Post(url).Body(strings.NewReader(payload)).Request()

	if err != nil {
		log.Debug(err.Error())
		return err
	}

	response, err := h.httpClient.Do(request)

	if err != nil {
		log.Debug(err.Error())
		return err
	}

	if response.StatusCode != 200 {
		return errors.New("Import to Hoverfly failed")
	}

	return nil
}

func (h *Hoverfly) ExportSimulation() ([]byte, error) {
	url := h.buildUrl("/api/records")

	request, err := sling.New().Get(url).Request()
	if err != nil {
		log.Debug(err.Error())
		return nil, err
	}

	response, err := h.httpClient.Do(request)
	if err != nil {
		log.Debug(err.Error())
		return nil, err
	}

	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Debug(err.Error())
		return nil, err
	}

	return body, nil
}

func (h *Hoverfly) createApiStateResponse(response *http.Response) (ApiStateResponse) {
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Debug(err.Error())
	}

	var apiResponse ApiStateResponse

	err = json.Unmarshal(body, &apiResponse)
	if err != nil {
		log.Debug(err.Error())
	}

	return apiResponse
}

func (h *Hoverfly) buildUrl(endpoint string) (string) {
	return fmt.Sprintf("%v%v", h.buildBaseUrl(), endpoint)
}

func (h *Hoverfly) buildBaseUrl() string {
	return fmt.Sprintf("http://%v:%v", h.Host, h.AdminPort)
}

/*
This isn't working as intended, its working, just not how I imagined it.
 */

func (h *Hoverfly) start(hoverflyDirectory HoverflyDirectory) (error) {
	pid, err := hoverflyDirectory.GetPid(h.AdminPort, h.ProxyPort)
	if err != nil {
		return err
	}

	if pid == 0 {
		cmd := exec.Command("hoverfly", "-db", "memory", "-ap", h.AdminPort, "-pp", h.ProxyPort)

		err := cmd.Start()
		if err != nil {
			return err
		}

		err = hoverflyDirectory.WritePid(h.AdminPort, h.ProxyPort, cmd.Process.Pid)
		if err != nil {
			return err
		}

	} else {
		return errors.New("Hoverfly is already running")
	}

	return nil
}

func (h *Hoverfly) stop(hoverflyDirectory HoverflyDirectory) (error) {
	pid, err := hoverflyDirectory.GetPid(h.AdminPort, h.ProxyPort)
	if err != nil {
		return err
	}

	if pid == 0 {
		return errors.New("Hoverfly is not running")

	} else {
		hoverflyProcess := os.Process{Pid: pid}
		err := hoverflyProcess.Kill()
		if err != nil {
			return err
		}

		err = hoverflyDirectory.DeletePid(h.AdminPort, h.ProxyPort)
		if err != nil {
			return err
		}
	}

	return nil
}<commit_msg>If the pid file is for an instance of Hoverfly that is no longer running, it will delete it and start a new Hoverfly<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"fmt"
	"github.com/dghubble/sling"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"errors"
	"strings"
	"os"
	"os/exec"
)

type ApiStateResponse struct {
	Mode        string `json:"mode"`
	Destination string `json:"destination"`
}

type Hoverfly struct {
	Host       string
	AdminPort  string
	ProxyPort  string
	httpClient *http.Client
}

func NewHoverfly(config Config) (Hoverfly) {
	return Hoverfly {
		Host: config.HoverflyHost,
		AdminPort: config.HoverflyAdminPort,
		ProxyPort: config.HoverflyProxyPort,
		httpClient: http.DefaultClient,
	}
}

func (h *Hoverfly) Wipe() (error) {
	url := h.buildUrl("/api/records")

	request, err := sling.New().Delete(url).Request()
	if err != nil {
		log.Debug(err.Error())
		return err
	}

	response, err := h.httpClient.Do(request)
	if err != nil {
		log.Debug(err.Error())
		return err
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return errors.New("Hoverfly did not wipe the database")
	}

	return nil
}

func (h *Hoverfly) GetMode() (string, error) {
	url := h.buildUrl("/api/state")
	request, err := sling.New().Get(url).Request()

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	response, err := h.httpClient.Do(request)

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	defer response.Body.Close()

	apiResponse := h.createApiStateResponse(response)

	return apiResponse.Mode, nil
}

func (h *Hoverfly) SetMode(mode string) (string, error) {
	if mode != "simulate" && mode != "capture" && mode != "modify" && mode != "synthesize" {
		return "", errors.New(mode + " is not a valid mode")
	}

	url := h.buildUrl("/api/state")
	request, err := sling.New().Post(url).Body(strings.NewReader(`{"mode":"` + mode + `"}`)).Request()

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	response, err := h.httpClient.Do(request)

	if err != nil {
		log.Debug(err.Error())
		return "", err
	}

	apiResponse := h.createApiStateResponse(response)

	return apiResponse.Mode, nil
}

func (h *Hoverfly) ImportSimulation(payload string) (error) {
	url := h.buildUrl("/api/records")
	request, err := sling.New().Post(url).Body(strings.NewReader(payload)).Request()

	if err != nil {
		log.Debug(err.Error())
		return err
	}

	response, err := h.httpClient.Do(request)

	if err != nil {
		log.Debug(err.Error())
		return err
	}

	if response.StatusCode != 200 {
		return errors.New("Import to Hoverfly failed")
	}

	return nil
}

func (h *Hoverfly) ExportSimulation() ([]byte, error) {
	url := h.buildUrl("/api/records")

	request, err := sling.New().Get(url).Request()
	if err != nil {
		log.Debug(err.Error())
		return nil, err
	}

	response, err := h.httpClient.Do(request)
	if err != nil {
		log.Debug(err.Error())
		return nil, err
	}

	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Debug(err.Error())
		return nil, err
	}

	return body, nil
}

func (h *Hoverfly) createApiStateResponse(response *http.Response) (ApiStateResponse) {
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Debug(err.Error())
	}

	var apiResponse ApiStateResponse

	err = json.Unmarshal(body, &apiResponse)
	if err != nil {
		log.Debug(err.Error())
	}

	return apiResponse
}

func (h *Hoverfly) buildUrl(endpoint string) (string) {
	return fmt.Sprintf("%v%v", h.buildBaseUrl(), endpoint)
}

func (h *Hoverfly) buildBaseUrl() string {
	return fmt.Sprintf("http://%v:%v", h.Host, h.AdminPort)
}

/*
This isn't working as intended, its working, just not how I imagined it.
 */

func (h *Hoverfly) start(hoverflyDirectory HoverflyDirectory) (error) {
	pid, err := hoverflyDirectory.GetPid(h.AdminPort, h.ProxyPort)
	if err != nil {
		return err
	}

	if pid != 0 {
		_, err := h.GetMode()
		if err == nil {
			return errors.New("Hoverfly is already running")
		}
		hoverflyDirectory.DeletePid(h.AdminPort, h.ProxyPort)
	}

	cmd := exec.Command("hoverfly", "-db", "memory", "-ap", h.AdminPort, "-pp", h.ProxyPort)

	err = cmd.Start()
	if err != nil {
		return err
	}

	err = hoverflyDirectory.WritePid(h.AdminPort, h.ProxyPort, cmd.Process.Pid)
	if err != nil {
		return err
	}

	return nil
}

func (h *Hoverfly) stop(hoverflyDirectory HoverflyDirectory) (error) {
	pid, err := hoverflyDirectory.GetPid(h.AdminPort, h.ProxyPort)
	if err != nil {
		return err
	}

	if pid == 0 {
		return errors.New("Hoverfly is not running")

	} else {
		hoverflyProcess := os.Process{Pid: pid}
		err := hoverflyProcess.Kill()
		if err != nil {
			return err
		}

		err = hoverflyDirectory.DeletePid(h.AdminPort, h.ProxyPort)
		if err != nil {
			return err
		}
	}

	return nil
}<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"

	"bufio"
	"flag"
	"net"
	"net/http"
	"os"
	"regexp"
)

// modes
const VirtualizeMode = "virtualize"
const SynthesizeMode = "synthesize"
const ModifyMode = "modify"
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

func main() {
	// Output to stderr instead of stdout, could also be a file.
	log.SetOutput(os.Stderr)
	log.SetFormatter(&log.TextFormatter{})

	// getting proxy configuration
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	// modes
	capture := flag.Bool("capture", false, "should proxy capture requests")
	synthesize := flag.Bool("synthesize", false, "should proxy capture requests")
	modify := flag.Bool("modify", false, "should proxy only modify requests")

	destination := flag.String("destination", ".", "destination URI to catch")
	middleware := flag.String("middleware", "", "should proxy use middleware")
	flag.Parse()

	if *verbose {
		// Only log the warning severity or above.
		log.SetLevel(log.DebugLevel)
	}

	// getting settings
	initSettings()

	// overriding default middleware setting
	AppConfig.middleware = *middleware

	// setting default mode
	mode := VirtualizeMode

	if *capture {
		mode = CaptureMode
		// checking whether user supplied other modes
		if *synthesize == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *synthesize {
		mode = SynthesizeMode

		if AppConfig.middleware == "" {
			log.Fatal("Synthesize mode chosen although middleware not supplied")
		}

		if *capture == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *modify {
		mode = ModifyMode

		if AppConfig.middleware == "" {
			log.Fatal("Modify mode chosen although middleware not supplied")
		}

		if *capture == true || *synthesize == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	}

	// overriding default settings
	AppConfig.mode = mode

	// overriding destination
	AppConfig.destination = *destination

	// getting default database
	port := os.Getenv("ProxyPort")
	if port == "" {
		port = DefaultPort
	} else {
		port = fmt.Sprintf(":%s", port)
	}

	// getting boltDB
	db := getDB(AppConfig.databaseName)
	defer db.Close()

	cache := Cache{
		db:             db,
		requestsBucket: []byte(requestsBucketName),
	}

	// getting connections
	d := DBClient{
		cache: cache,
		http:  &http.Client{},
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(*destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
		defer func() {
			if e := recover(); e != nil {
				ctx.Logf("error connecting to remote: %v", e)
				client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
			}
			client.Close()
		}()
		clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
		remote, err := net.Dial("tcp", req.URL.Host)
		orPanic(err)
		remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
		for {
			req, err := http.ReadRequest(clientBuf.Reader)
			orPanic(err)
			orPanic(req.Write(remoteBuf))
			orPanic(remoteBuf.Flush())
			resp, err := http.ReadResponse(remoteBuf.Reader, req)

			orPanic(err)
			orPanic(resp.Write(clientBuf.Writer))
			orPanic(clientBuf.Flush())
		}
	})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(*destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	go d.startAdminInterface()

	proxy.Verbose = *verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": *destination,
		"ProxyPort":   port,
		"Mode":        AppConfig.mode,
	}).Info("Proxy is starting...")

	log.Warn(http.ListenAndServe(port, proxy))
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	if AppConfig.mode == CaptureMode {
		log.Info("*** Capture ***")
		newResponse, err := d.captureRequest(req)
		if err != nil {
			// something bad happened, passing through
			return req, nil
		} else {
			// discarding original requests and returns supplied response
			return req, newResponse
		}

	} else if AppConfig.mode == SynthesizeMode {
		log.Info("*** Sinthesize ***")
		response := synthesizeResponse(req, AppConfig.middleware)
		return req, response

	} else if AppConfig.mode == ModifyMode {
		log.Info("*** Modify ***")
		response, err := d.modifyRequestResponse(req, AppConfig.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": AppConfig.middleware,
			}).Error("Got error when performing request modification")
			return req, nil
		}

		// returning modified response
		return req, response

	} else {
		log.Info("*** Virtualize ***")
		newResponse := d.getResponse(req)
		return req, newResponse

	}
}
<commit_msg>getting initial settings<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"

	"bufio"
	"flag"
	"net"
	"net/http"
	"os"
	"regexp"
)

// modes
const VirtualizeMode = "virtualize"
const SynthesizeMode = "synthesize"
const ModifyMode = "modify"
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

func main() {
	// Output to stderr instead of stdout, could also be a file.
	log.SetOutput(os.Stderr)
	log.SetFormatter(&log.TextFormatter{})

	// getting proxy configuration
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	// modes
	capture := flag.Bool("capture", false, "should proxy capture requests")
	synthesize := flag.Bool("synthesize", false, "should proxy capture requests")
	modify := flag.Bool("modify", false, "should proxy only modify requests")

	destination := flag.String("destination", ".", "destination URI to catch")
	middleware := flag.String("middleware", "", "should proxy use middleware")
	flag.Parse()

	// getting settings
	cfg := InitSettings()

	if *verbose {
		// Only log the warning severity or above.
		log.SetLevel(log.DebugLevel)
	}
	cfg.verbose = *verbose

	// overriding default middleware setting
	cfg.middleware = *middleware

	// setting default mode
	mode := VirtualizeMode

	if *capture {
		mode = CaptureMode
		// checking whether user supplied other modes
		if *synthesize == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *synthesize {
		mode = SynthesizeMode

		if AppConfig.middleware == "" {
			log.Fatal("Synthesize mode chosen although middleware not supplied")
		}

		if *capture == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *modify {
		mode = ModifyMode

		if AppConfig.middleware == "" {
			log.Fatal("Modify mode chosen although middleware not supplied")
		}

		if *capture == true || *synthesize == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	}

	// overriding default settings
	AppConfig.mode = mode

	// overriding destination
	AppConfig.destination = *destination

	// getting default database
	port := os.Getenv("ProxyPort")
	if port == "" {
		port = DefaultPort
	} else {
		port = fmt.Sprintf(":%s", port)
	}

	// getting boltDB
	db := getDB(AppConfig.databaseName)
	defer db.Close()

	cache := Cache{
		db:             db,
		requestsBucket: []byte(requestsBucketName),
	}

	// getting connections
	d := DBClient{
		cache: cache,
		http:  &http.Client{},
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(*destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
		defer func() {
			if e := recover(); e != nil {
				ctx.Logf("error connecting to remote: %v", e)
				client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
			}
			client.Close()
		}()
		clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
		remote, err := net.Dial("tcp", req.URL.Host)
		orPanic(err)
		remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
		for {
			req, err := http.ReadRequest(clientBuf.Reader)
			orPanic(err)
			orPanic(req.Write(remoteBuf))
			orPanic(remoteBuf.Flush())
			resp, err := http.ReadResponse(remoteBuf.Reader, req)

			orPanic(err)
			orPanic(resp.Write(clientBuf.Writer))
			orPanic(clientBuf.Flush())
		}
	})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(*destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	go d.startAdminInterface()

	proxy.Verbose = *verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": *destination,
		"ProxyPort":   port,
		"Mode":        AppConfig.mode,
	}).Info("Proxy is starting...")

	log.Warn(http.ListenAndServe(port, proxy))
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	if AppConfig.mode == CaptureMode {
		log.Info("*** Capture ***")
		newResponse, err := d.captureRequest(req)
		if err != nil {
			// something bad happened, passing through
			return req, nil
		} else {
			// discarding original requests and returns supplied response
			return req, newResponse
		}

	} else if AppConfig.mode == SynthesizeMode {
		log.Info("*** Sinthesize ***")
		response := synthesizeResponse(req, AppConfig.middleware)
		return req, response

	} else if AppConfig.mode == ModifyMode {
		log.Info("*** Modify ***")
		response, err := d.modifyRequestResponse(req, AppConfig.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": AppConfig.middleware,
			}).Error("Got error when performing request modification")
			return req, nil
		}

		// returning modified response
		return req, response

	} else {
		log.Info("*** Virtualize ***")
		newResponse := d.getResponse(req)
		return req, newResponse

	}
}
<|endoftext|>


<commit_before>package restserver

/*
Original version copied from: github.com/bitly/oauth2_proxy

MIT License

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
	"crypto/sha1"
	"crypto/sha256"
	"crypto/subtle"
	"encoding/base64"
	"encoding/csv"
	"log"
	"os"
	"os/signal"
	"regexp"
	"sync"
	"syscall"
	"time"

	"golang.org/x/crypto/bcrypt"
)

// CheckInterval represents how often we check for changes in htpasswd file.
const CheckInterval = 30 * time.Second

// PasswordCacheDuration represents how long authentication credentials are
// cached in memory after they were successfully verified. This allows avoiding
// repeatedly verifying the same authentication credentials.
const PasswordCacheDuration = time.Minute

// Lookup passwords in a htpasswd file.  The entries must have been created with -s for SHA encryption.

type cacheEntry struct {
	expiry   time.Time
	verifier []byte
}

// HtpasswdFile is a map for usernames to passwords.
type HtpasswdFile struct {
	mutex    sync.Mutex
	path     string
	stat     os.FileInfo
	throttle chan struct{}
	users    map[string]string
	cache    map[string]cacheEntry
}

// NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.  If an error is encountered,
// it is returned, together with a nil-Pointer for the HtpasswdFile.
func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGHUP)
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	h := &HtpasswdFile{
		mutex:    sync.Mutex{},
		path:     path,
		stat:     stat,
		throttle: make(chan struct{}),
		cache:    make(map[string]cacheEntry),
	}

	if err := h.Reload(); err != nil {
		return nil, err
	}

	// Start a goroutine that limits reload checks to once per CheckInterval
	go h.throttleTimer()
	go h.expiryTimer()

	go func() {
		for range c {
			err := h.Reload()
			if err == nil {
				log.Printf("Reloaded htpasswd file")
			} else {
				log.Printf("Could not reload htpasswd file: %v", err)
			}
		}
	}()

	return h, nil
}

// throttleTimer sends at most one message per CheckInterval to throttle file change checks.
func (h *HtpasswdFile) throttleTimer() {
	var check struct{}
	for {
		time.Sleep(CheckInterval)
		h.throttle <- check
	}
}

func (h *HtpasswdFile) expiryTimer() {
	for {
		time.Sleep(5 * time.Second)
		now := time.Now()
		h.mutex.Lock()
		var zeros [sha256.Size]byte
		// try to wipe expired cache entries
		for user, entry := range h.cache {
			if entry.expiry.After(now) {
				copy(entry.verifier, zeros[:])
				delete(h.cache, user)
			}
		}
		h.mutex.Unlock()
	}
}

var validUsernameRegexp = regexp.MustCompile(`^[\p{L}\d@._-]+$`)

// Reload reloads the htpasswd file. If the reload fails, the Users map is not changed and the error is returned.
func (h *HtpasswdFile) Reload() error {
	r, err := os.Open(h.path)
	if err != nil {
		return err
	}

	cr := csv.NewReader(r)
	cr.Comma = ':'
	cr.Comment = '#'
	cr.TrimLeadingSpace = true

	records, err := cr.ReadAll()
	if err != nil {
		_ = r.Close()
		return err
	}
	users := make(map[string]string)
	for _, record := range records {
		if !validUsernameRegexp.MatchString(record[0]) {
			log.Printf("Ignoring invalid username %q in htpasswd, consists of characters other than letters, numbers, '_', '-', '.' and '@'", record[0])
			continue
		}
		users[record[0]] = record[1]
	}

	// Replace the Users map
	h.mutex.Lock()
	var zeros [sha256.Size]byte
	// try to wipe the old cache entries
	for _, entry := range h.cache {
		copy(entry.verifier, zeros[:])
	}
	h.cache = make(map[string]cacheEntry)

	h.users = users
	h.mutex.Unlock()

	_ = r.Close()
	return nil
}

// ReloadCheck checks at most once per CheckInterval if the file changed and will reload the file if it did.
// It logs errors and successful reloads, and returns an error if any was encountered.
func (h *HtpasswdFile) ReloadCheck() error {
	select {
	case <-h.throttle:
		stat, err := os.Stat(h.path)
		if err != nil {
			log.Printf("Could not stat htpasswd file: %v", err)
			return err
		}

		reload := false

		h.mutex.Lock()
		if stat.ModTime() != h.stat.ModTime() || stat.Size() != h.stat.Size() {
			reload = true
			h.stat = stat
		}
		h.mutex.Unlock()

		if reload {
			err := h.Reload()
			if err == nil {
				log.Printf("Reloaded htpasswd file")
			} else {
				log.Printf("Could not reload htpasswd file: %v", err)
				return err
			}
		}
	default:
		// No need to check
	}
	return nil
}

// Validate returns true if password matches the stored password for user.  If no password for user is stored, or the
// password is wrong, false is returned.
func (h *HtpasswdFile) Validate(user string, password string) bool {
	_ = h.ReloadCheck()

	hash := sha256.New()
	// hash.Write can never fail
	_, _ = hash.Write([]byte(user))
	_, _ = hash.Write([]byte(":"))
	_, _ = hash.Write([]byte(password))

	h.mutex.Lock()
	// avoid race conditions with cache replacements
	cache := h.cache
	realPassword, exists := h.users[user]
	entry, cacheExists := h.cache[user]
	h.mutex.Unlock()

	if !exists {
		return false
	}

	if cacheExists && subtle.ConstantTimeCompare(entry.verifier, hash.Sum(nil)) == 1 {
		return true
	}

	var shaRe = regexp.MustCompile(`^{SHA}`)
	var bcrRe = regexp.MustCompile(`^\$2b\$|^\$2a\$|^\$2y\$`)

	isValid := false

	switch {
	case shaRe.MatchString(realPassword):
		d := sha1.New()
		_, _ = d.Write([]byte(password))
		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
			isValid = true
		}
	case bcrRe.MatchString(realPassword):
		err := bcrypt.CompareHashAndPassword([]byte(realPassword), []byte(password))
		if err == nil {
			isValid = true
		}
	}

	if !isValid {
		log.Printf("Invalid htpasswd entry for %s.", user)
		return false
	}

	h.mutex.Lock()
	// repurpose mutex to prevent concurrent cache updates
	cache[user] = cacheEntry{
		verifier: hash.Sum(nil),
		expiry:   time.Now().Add(PasswordCacheDuration),
	}
	h.mutex.Unlock()

	return true
}
<commit_msg>Extend htpasswd auth cache entry expiry on use<commit_after>package restserver

/*
Original version copied from: github.com/bitly/oauth2_proxy

MIT License

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

import (
	"crypto/sha1"
	"crypto/sha256"
	"crypto/subtle"
	"encoding/base64"
	"encoding/csv"
	"log"
	"os"
	"os/signal"
	"regexp"
	"sync"
	"syscall"
	"time"

	"golang.org/x/crypto/bcrypt"
)

// CheckInterval represents how often we check for changes in htpasswd file.
const CheckInterval = 30 * time.Second

// PasswordCacheDuration represents how long authentication credentials are
// cached in memory after they were successfully verified. This allows avoiding
// repeatedly verifying the same authentication credentials.
const PasswordCacheDuration = time.Minute

// Lookup passwords in a htpasswd file.  The entries must have been created with -s for SHA encryption.

type cacheEntry struct {
	expiry   time.Time
	verifier []byte
}

// HtpasswdFile is a map for usernames to passwords.
type HtpasswdFile struct {
	mutex    sync.Mutex
	path     string
	stat     os.FileInfo
	throttle chan struct{}
	users    map[string]string
	cache    map[string]cacheEntry
}

// NewHtpasswdFromFile reads the users and passwords from a htpasswd file and returns them.  If an error is encountered,
// it is returned, together with a nil-Pointer for the HtpasswdFile.
func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGHUP)
	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	h := &HtpasswdFile{
		mutex:    sync.Mutex{},
		path:     path,
		stat:     stat,
		throttle: make(chan struct{}),
		cache:    make(map[string]cacheEntry),
	}

	if err := h.Reload(); err != nil {
		return nil, err
	}

	// Start a goroutine that limits reload checks to once per CheckInterval
	go h.throttleTimer()
	go h.expiryTimer()

	go func() {
		for range c {
			err := h.Reload()
			if err == nil {
				log.Printf("Reloaded htpasswd file")
			} else {
				log.Printf("Could not reload htpasswd file: %v", err)
			}
		}
	}()

	return h, nil
}

// throttleTimer sends at most one message per CheckInterval to throttle file change checks.
func (h *HtpasswdFile) throttleTimer() {
	var check struct{}
	for {
		time.Sleep(CheckInterval)
		h.throttle <- check
	}
}

func (h *HtpasswdFile) expiryTimer() {
	for {
		time.Sleep(5 * time.Second)
		now := time.Now()
		h.mutex.Lock()
		var zeros [sha256.Size]byte
		// try to wipe expired cache entries
		for user, entry := range h.cache {
			if entry.expiry.After(now) {
				copy(entry.verifier, zeros[:])
				delete(h.cache, user)
			}
		}
		h.mutex.Unlock()
	}
}

var validUsernameRegexp = regexp.MustCompile(`^[\p{L}\d@._-]+$`)

// Reload reloads the htpasswd file. If the reload fails, the Users map is not changed and the error is returned.
func (h *HtpasswdFile) Reload() error {
	r, err := os.Open(h.path)
	if err != nil {
		return err
	}

	cr := csv.NewReader(r)
	cr.Comma = ':'
	cr.Comment = '#'
	cr.TrimLeadingSpace = true

	records, err := cr.ReadAll()
	if err != nil {
		_ = r.Close()
		return err
	}
	users := make(map[string]string)
	for _, record := range records {
		if !validUsernameRegexp.MatchString(record[0]) {
			log.Printf("Ignoring invalid username %q in htpasswd, consists of characters other than letters, numbers, '_', '-', '.' and '@'", record[0])
			continue
		}
		users[record[0]] = record[1]
	}

	// Replace the Users map
	h.mutex.Lock()
	var zeros [sha256.Size]byte
	// try to wipe the old cache entries
	for _, entry := range h.cache {
		copy(entry.verifier, zeros[:])
	}
	h.cache = make(map[string]cacheEntry)

	h.users = users
	h.mutex.Unlock()

	_ = r.Close()
	return nil
}

// ReloadCheck checks at most once per CheckInterval if the file changed and will reload the file if it did.
// It logs errors and successful reloads, and returns an error if any was encountered.
func (h *HtpasswdFile) ReloadCheck() error {
	select {
	case <-h.throttle:
		stat, err := os.Stat(h.path)
		if err != nil {
			log.Printf("Could not stat htpasswd file: %v", err)
			return err
		}

		reload := false

		h.mutex.Lock()
		if stat.ModTime() != h.stat.ModTime() || stat.Size() != h.stat.Size() {
			reload = true
			h.stat = stat
		}
		h.mutex.Unlock()

		if reload {
			err := h.Reload()
			if err == nil {
				log.Printf("Reloaded htpasswd file")
			} else {
				log.Printf("Could not reload htpasswd file: %v", err)
				return err
			}
		}
	default:
		// No need to check
	}
	return nil
}

// Validate returns true if password matches the stored password for user.  If no password for user is stored, or the
// password is wrong, false is returned.
func (h *HtpasswdFile) Validate(user string, password string) bool {
	_ = h.ReloadCheck()

	hash := sha256.New()
	// hash.Write can never fail
	_, _ = hash.Write([]byte(user))
	_, _ = hash.Write([]byte(":"))
	_, _ = hash.Write([]byte(password))

	h.mutex.Lock()
	// avoid race conditions with cache replacements
	cache := h.cache
	realPassword, exists := h.users[user]
	entry, cacheExists := h.cache[user]
	h.mutex.Unlock()

	if !exists {
		return false
	}

	if cacheExists && subtle.ConstantTimeCompare(entry.verifier, hash.Sum(nil)) == 1 {
		h.mutex.Lock()
		// repurpose mutex to prevent concurrent cache updates
		// extend cache entry
		cache[user] = cacheEntry{
			verifier: entry.verifier,
			expiry:   time.Now().Add(PasswordCacheDuration),
		}
		h.mutex.Unlock()
		return true
	}

	var shaRe = regexp.MustCompile(`^{SHA}`)
	var bcrRe = regexp.MustCompile(`^\$2b\$|^\$2a\$|^\$2y\$`)

	isValid := false

	switch {
	case shaRe.MatchString(realPassword):
		d := sha1.New()
		_, _ = d.Write([]byte(password))
		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
			isValid = true
		}
	case bcrRe.MatchString(realPassword):
		err := bcrypt.CompareHashAndPassword([]byte(realPassword), []byte(password))
		if err == nil {
			isValid = true
		}
	}

	if !isValid {
		log.Printf("Invalid htpasswd entry for %s.", user)
		return false
	}

	h.mutex.Lock()
	// repurpose mutex to prevent concurrent cache updates
	cache[user] = cacheEntry{
		verifier: hash.Sum(nil),
		expiry:   time.Now().Add(PasswordCacheDuration),
	}
	h.mutex.Unlock()

	return true
}
<|endoftext|>


<commit_before>package fileServer

import (
	"crypto/md5"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
)

type Server struct {
	root string
	dir  string

	options Options

	hashes map[string]string
	mu     sync.RWMutex
}

func New(root, dir string, options *Options) Server {
	if options == nil {
		options = &Options{}
	}
	return Server{
		root: root,
		dir:  dir,

		options: *options,

		hashes: map[string]string{},
		mu:     sync.RWMutex{},
	}
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	urlPath := r.URL.Path
	if !strings.HasPrefix(urlPath, "/") {
		urlPath = "/" + urlPath
		r.URL.Path = urlPath
	}
	p := path.Clean(urlPath)

	if s.root != "" {
		if p = strings.TrimPrefix(p, s.root); len(p) >= len(r.URL.Path) {
			s.HTTPError(w, r, errNotFound)
			return
		}
	}

	if s.options.IndexPage != "" && strings.HasSuffix(r.URL.Path, s.options.IndexPage) {
		redirect(w, r, "./")
		return
	}

	if (s.options.Hashed && !s.options.NoHashQueryStrings) ||
		(s.options.Hashed && s.options.NoHashQueryStrings && len(r.URL.RawQuery) == 0) {
		cPath := canonicalPath(p)
		h, err := s.getHash(cPath)
		if err != errNotRegularFile { // continue as usual if it is not a regular file
			if err != nil {
				s.HTTPError(w, r, err)
				return
			}
			if hPath := hashedPath(cPath, h); hPath != p {
				redirect(w, r, path.Join(s.root, hPath))
				return
			}
			if s.options.RedirectTrailingSlash && urlPath[len(urlPath)-1] == '/' {
				redirect(w, r, path.Join(s.root, p))
				return
			}
			p = cPath
			r.URL.Path = path.Join(s.root, cPath)
		}
	}

	f, err := open(s.dir, p)
	if err != nil {
		s.HTTPError(w, r, err)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		s.HTTPError(w, r, err)
		return
	}

	if s.options.RedirectTrailingSlash {
		url := r.URL.Path
		if d.IsDir() {
			if url[len(url)-1] != '/' {
				redirect(w, r, url+"/")
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				redirect(w, r, "../"+path.Base(url))
				return
			}
		}
	}

	if d.IsDir() {
		index := strings.TrimSuffix(p, "/") + s.options.IndexPage
		ff, err := open(s.dir, index)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				p = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDir() {
		s.HTTPError(w, r, errNotFound)
		return
	}

	http.ServeContent(w, r, d.Name(), d.ModTime(), f)
}

func (s *Server) HashedPath(p string) (string, error) {
	if !s.options.Hashed {
		return p, nil
	}
	h, err := s.getHash(p)
	if err != nil {
		return "", err
	}
	return path.Join(s.root, hashedPath(p, h)), nil
}

func (s Server) HTTPError(w http.ResponseWriter, r *http.Request, err error) {
	if os.IsNotExist(err) || err == errNotFound {
		if s.options.NotFoundHandler != nil {
			s.options.NotFoundHandler.ServeHTTP(w, r)
			return
		}
		DefaultNotFoundHandler.ServeHTTP(w, r)
		return
	}
	if os.IsPermission(err) {
		if s.options.ForbiddenHandler != nil {
			s.options.ForbiddenHandler.ServeHTTP(w, r)
			return
		}
		DefaultForbiddenHandler.ServeHTTP(w, r)
		return
	}
	if s.options.InternalServerErrorHandler != nil {
		s.options.InternalServerErrorHandler.ServeHTTP(w, r)
		return
	}
	DefaultInternalServerErrorhandler.ServeHTTP(w, r)
}

func (s *Server) getHash(p string) (h string, err error) {
	s.mu.RLock()
	h, ok := s.hashes[p]
	s.mu.RUnlock()
	if ok {
		return
	}

	f, err := open(s.dir, p)
	if err != nil {
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		return
	}
	if !d.Mode().IsRegular() {
		err = errNotRegularFile
		return
	}

	hash := md5.New()
	if _, err = io.Copy(hash, f); err != nil {
		return
	}
	h = makeHash(strings.TrimRight(base32Encoding.EncodeToString(hash.Sum(nil)), "="))
	s.mu.Lock()
	s.hashes[p] = h
	s.mu.Unlock()
	return
}
<commit_msg>file-server: New() return pointer to Server<commit_after>package fileServer

import (
	"crypto/md5"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
)

type Server struct {
	root string
	dir  string

	options Options

	hashes map[string]string
	mu     sync.RWMutex
}

func New(root, dir string, options *Options) *Server {
	if options == nil {
		options = &Options{}
	}
	return &Server{
		root: root,
		dir:  dir,

		options: *options,

		hashes: map[string]string{},
		mu:     sync.RWMutex{},
	}
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	urlPath := r.URL.Path
	if !strings.HasPrefix(urlPath, "/") {
		urlPath = "/" + urlPath
		r.URL.Path = urlPath
	}
	p := path.Clean(urlPath)

	if s.root != "" {
		if p = strings.TrimPrefix(p, s.root); len(p) >= len(r.URL.Path) {
			s.HTTPError(w, r, errNotFound)
			return
		}
	}

	if s.options.IndexPage != "" && strings.HasSuffix(r.URL.Path, s.options.IndexPage) {
		redirect(w, r, "./")
		return
	}

	if (s.options.Hashed && !s.options.NoHashQueryStrings) ||
		(s.options.Hashed && s.options.NoHashQueryStrings && len(r.URL.RawQuery) == 0) {
		cPath := canonicalPath(p)
		h, err := s.getHash(cPath)
		if err != errNotRegularFile { // continue as usual if it is not a regular file
			if err != nil {
				s.HTTPError(w, r, err)
				return
			}
			if hPath := hashedPath(cPath, h); hPath != p {
				redirect(w, r, path.Join(s.root, hPath))
				return
			}
			if s.options.RedirectTrailingSlash && urlPath[len(urlPath)-1] == '/' {
				redirect(w, r, path.Join(s.root, p))
				return
			}
			p = cPath
			r.URL.Path = path.Join(s.root, cPath)
		}
	}

	f, err := open(s.dir, p)
	if err != nil {
		s.HTTPError(w, r, err)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		s.HTTPError(w, r, err)
		return
	}

	if s.options.RedirectTrailingSlash {
		url := r.URL.Path
		if d.IsDir() {
			if url[len(url)-1] != '/' {
				redirect(w, r, url+"/")
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				redirect(w, r, "../"+path.Base(url))
				return
			}
		}
	}

	if d.IsDir() {
		index := strings.TrimSuffix(p, "/") + s.options.IndexPage
		ff, err := open(s.dir, index)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				p = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDir() {
		s.HTTPError(w, r, errNotFound)
		return
	}

	http.ServeContent(w, r, d.Name(), d.ModTime(), f)
}

func (s *Server) HashedPath(p string) (string, error) {
	if !s.options.Hashed {
		return p, nil
	}
	h, err := s.getHash(p)
	if err != nil {
		return "", err
	}
	return path.Join(s.root, hashedPath(p, h)), nil
}

func (s Server) HTTPError(w http.ResponseWriter, r *http.Request, err error) {
	if os.IsNotExist(err) || err == errNotFound {
		if s.options.NotFoundHandler != nil {
			s.options.NotFoundHandler.ServeHTTP(w, r)
			return
		}
		DefaultNotFoundHandler.ServeHTTP(w, r)
		return
	}
	if os.IsPermission(err) {
		if s.options.ForbiddenHandler != nil {
			s.options.ForbiddenHandler.ServeHTTP(w, r)
			return
		}
		DefaultForbiddenHandler.ServeHTTP(w, r)
		return
	}
	if s.options.InternalServerErrorHandler != nil {
		s.options.InternalServerErrorHandler.ServeHTTP(w, r)
		return
	}
	DefaultInternalServerErrorhandler.ServeHTTP(w, r)
}

func (s *Server) getHash(p string) (h string, err error) {
	s.mu.RLock()
	h, ok := s.hashes[p]
	s.mu.RUnlock()
	if ok {
		return
	}

	f, err := open(s.dir, p)
	if err != nil {
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		return
	}
	if !d.Mode().IsRegular() {
		err = errNotRegularFile
		return
	}

	hash := md5.New()
	if _, err = io.Copy(hash, f); err != nil {
		return
	}
	h = makeHash(strings.TrimRight(base32Encoding.EncodeToString(hash.Sum(nil)), "="))
	s.mu.Lock()
	s.hashes[p] = h
	s.mu.Unlock()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"html"
	"io"
	"log"
	"os"
	"regexp"
	"strings"
)

var (
	extensionRE = regexp.MustCompile("!
 (.+)
")
	linkRE      = regexp.MustCompile(`\[\[(.+\|)?(.+?)\]\]`)
)

var outputFile *string = flag.String("output", "output.txt", "output file")
var wikiFile *string = flag.String("wikiinput", "download/data", "input Wikipedia data")

func main() {
	flag.Parse()
	lines := make(chan []byte)
	formats := make(chan fileFormat)

	go readData(lines)
	go parse(lines, formats)
	output(formats)
}

type fileFormat struct {
	ext string
	use []string
}

func output(formats chan fileFormat) {
	file, err := os.OpenFile(*outputFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	for f := range formats {
		out := []string{
			f.ext, // title (required)
			"A",   // type (required)
			"",    // redirect
			"",    // otheruses (ignore)
			"",    // categories
			"",    // references (ignore)
			"",    // see also
			"",    // further reading (ignore)
			"",    // external links
			"",    // disambiguation (ignore)
			"",    // images
			"A file with this extension may be a " + strings.Join(f.use, ", ") + ".", // abstract
			"",                                                                       // source url
		}
		_, err = file.WriteString(strings.Join(out, "\t") + "\n")
		if err != nil {
			log.Fatal(err)
		}
	}
}

func cleanWikiHTML(b []byte) string {
	return html.UnescapeString(string(wikilink(b)))
}

func wikilink(b []byte) []byte {
	return linkRE.ReplaceAll(b, []byte("$2"))
}

const (
	parseExtension = iota
	parseDescription
	parseApplication
)

func parse(lines chan []byte, formats chan fileFormat) {
	state := parseExtension

	// The format we're currently parsing.
	format := fileFormat{}

	for line := range lines {
		switch state {
		case parseExtension:
			// Determine if this line specifies an extension.
			if m := extensionRE.FindSubmatch(line); m != nil {
				ext := m[1]

				// If a wiki link, use the link text.
				ext = wikilink(ext)

				// Some extensions are used by multiple things. If this is a new extension,
				// send the old one.
				if thisExt := html.UnescapeString(string(ext)); thisExt != format.ext {
					if format.ext != "" {
						formats <- format
					}
					format = fileFormat{ext: thisExt}
				}
				state = parseDescription
			}
		case parseDescription:
			// Collect the possible uses for this extension.
			// This is [1:] to omit a leading "|" on the line.
			use := cleanWikiHTML(line[1:])
			format.use = append(format.use, use)
			state = parseApplication
		case parseApplication:
			if app := line[1:]; len(app) > 0 && app[0] != '-' {
				format.use[len(format.use)-1] += " for " + cleanWikiHTML(app)
			}
			state = parseExtension
		}
	}
	close(formats)
}

func readData(lines chan []byte) {
	wikiData, err := os.Open(*wikiFile)
	if err != nil {
		log.Fatal(err)
	}
	defer wikiData.Close()

	b := bufio.NewReader(wikiData)

	for {
		line, isPrefix, err := b.ReadLine()
		if err == io.EOF {
			break
		}

		if err != nil {
			log.Fatal(err)
		}

		if !isPrefix {
			lines <- line
		} else {
			log.Panic("abnormally long input line -- check your data")
		}
	}
	close(lines)
}
<commit_msg>Cleanup pass<commit_after>package main

import (
	"bufio"
	"flag"
	"html"
	"io"
	"log"
	"os"
	"regexp"
	"strings"
)

var (
	extensionRE = regexp.MustCompile("!
 (.+)
")
	linkRE      = regexp.MustCompile(`\[\[(.+\|)?(.+?)\]\]`)
)

var outputFile *string = flag.String("output", "output.txt", "output file")
var wikiFile *string = flag.String("wikiinput", "download/data", "input Wikipedia data")

func main() {
	flag.Parse()
	lines := make(chan string)
	formats := make(chan fileFormat)

	go readData(lines)
	go parse(lines, formats)
	output(formats)
}

type fileFormat struct {
	ext string
	use []string
}

func output(formats chan fileFormat) {
	file, err := os.OpenFile(*outputFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	for f := range formats {
		out := []string{
			f.ext, // title (required)
			"A",   // type (required)
			"",    // redirect
			"",    // otheruses (ignore)
			"",    // categories
			"",    // references (ignore)
			"",    // see also
			"",    // further reading (ignore)
			"",    // external links
			"",    // disambiguation (ignore)
			"",    // images
			"A file with this extension may be a " + strings.Join(f.use, ", ") + ".", // abstract
			"",                                                                       // source url
		}
		_, err = file.WriteString(strings.Join(out, "\t") + "\n")
		if err != nil {
			log.Fatal(err)
		}
	}
}

func cleanWikiHTML(s string) string {
	return html.UnescapeString(wikilink(s))
}

func wikilink(b string) string {
	return linkRE.ReplaceAllString(b, "$2")
}

const (
	parseExtension = iota
	parseDescription
	parseApplication
)

func parse(lines chan string, formats chan fileFormat) {
	state := parseExtension

	// The format we're currently parsing.
	format := fileFormat{}

	for line := range lines {
		switch state {
		case parseExtension:
			// Determine if this line specifies an extension.
			if m := extensionRE.FindStringSubmatch(line); m != nil {
				ext := m[1]

				// Some extensions are used by multiple things. If this is a new extension,
				// send the old one.
				if thisExt := cleanWikiHTML(ext); thisExt != format.ext {
					if format.ext != "" {
						formats <- format
					}
					format = fileFormat{ext: thisExt}
				}
				state = parseDescription
			}
		case parseDescription:
			// Collect the possible uses for this extension.
			// This is [1:] to omit a leading "|" on the line.
			format.use = append(format.use, cleanWikiHTML(line[1:]))
			state = parseApplication
		case parseApplication:
			if app := line[1:]; len(app) > 0 && app != "-" {
				format.use[len(format.use)-1] += " for " + cleanWikiHTML(app)
			}
			state = parseExtension
		}
	}
	close(formats)
}

func readData(lines chan string) {
	wikiData, err := os.Open(*wikiFile)
	if err != nil {
		log.Fatal(err)
	}
	defer wikiData.Close()

	b := bufio.NewReader(wikiData)

	for {
		line, isPrefix, err := b.ReadLine()
		if err == io.EOF {
			break
		}

		if err != nil {
			log.Fatal(err)
		}

		if !isPrefix {
			lines <- string(line)
		} else {
			log.Panic("abnormally long input line -- check your data")
		}
	}
	close(lines)
}
<|endoftext|>


<commit_before>package main

import (
	"exp/inotify"
	"fmt"
	"io"
	"koding/tools/dnode"
	"koding/tools/kite"
	"koding/tools/log"
	"koding/virt"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"
)

func init() {
	go func() {
		for err := range virt.WatchErrors {
			log.Warn("Watcher error", err)
		}
	}()
}

func registerFileSystemMethods(k *kite.Kite) {
	registerVmMethod(k, "fs.readDirectory", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path                string
			OnChange            dnode.Callback
			WatchSubdirectories bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], onChange: [function], watchSubdirectories: [bool] }"}
		}

		response := make(map[string]interface{})

		if params.OnChange != nil {
			watch, err := vos.WatchDirectory(params.Path, params.WatchSubdirectories, func(ev *inotify.Event, info os.FileInfo) {
				defer log.RecoverAndLog()

				if (ev.Mask & (inotify.IN_CREATE | inotify.IN_MOVED_TO | inotify.IN_ATTRIB)) != 0 {
					if info == nil {
						return // skip this event, file was deleted and deletion event will follow
					}
					params.OnChange(map[string]interface{}{
						"event": "added",
						"file":  makeFileEntry(vos, ev.Name, info),
					})
					return
				}
				if (ev.Mask & (inotify.IN_DELETE | inotify.IN_MOVED_FROM)) != 0 {
					params.OnChange(map[string]interface{}{
						"event": "removed",
						"file":  FileEntry{Name: path.Base(ev.Name), FullPath: ev.Name},
					})
					return
				}
			})
			if err != nil {
				return nil, err
			}
			channel.OnDisconnect(func() { watch.Close() })
			response["stopWatching"] = func() { watch.Close() }
		}

		dir, err := vos.Open(params.Path)
		if err != nil {
			return nil, err
		}
		defer dir.Close()

		infos, err := dir.Readdir(0)
		if err != nil {
			return nil, err
		}

		files := make([]FileEntry, len(infos))
		for i, info := range infos {
			files[i] = makeFileEntry(vos, path.Join(params.Path, info.Name()), info)
		}
		response["files"] = files

		return response, nil
	})

	registerVmMethod(k, "fs.glob", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Pattern string
		}
		if args.Unmarshal(&params) != nil || params.Pattern == "" {
			return nil, &kite.ArgumentError{Expected: "{ pattern: [string] }"}
		}

		return vos.Glob(params.Pattern)
	})

	registerVmMethod(k, "fs.readFile", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path string
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string] }"}
		}

		file, err := vos.Open(params.Path)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		fi, err := file.Stat()
		if err != nil {
			return nil, err
		}

		if fi.Size() > 10*1024*1024 {
			return nil, fmt.Errorf("File larger than 10MiB.")
		}

		buf := make([]byte, fi.Size())
		if _, err := io.ReadFull(file, buf); err != nil {
			return nil, err
		}

		return map[string]interface{}{"content": buf}, nil
	})

	registerVmMethod(k, "fs.writeFile", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path           string
			Content        []byte
			DoNotOverwrite bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" || params.Content == nil {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], content: [base64], doNotOverwrite: [bool] }"}
		}

		flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
		if params.DoNotOverwrite {
			flags |= os.O_EXCL
		}
		file, err := vos.OpenFile(params.Path, flags, 0666)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		return file.Write(params.Content)
	})

	suffixRegexp, err := regexp.Compile(`.((_\d+)?)(\.\w*)?$`)
	if err != nil {
		panic(err)
	}

	registerVmMethod(k, "fs.ensureNonexistentPath", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path string
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string] }"}
		}

		name := params.Path
		index := 1
		for {
			_, err := vos.Stat(name)
			if err != nil {
				if os.IsNotExist(err) {
					break
				}
				return nil, err
			}

			loc := suffixRegexp.FindStringSubmatchIndex(name)
			name = name[:loc[2]] + "_" + strconv.Itoa(index) + name[loc[3]:]
			index++
		}

		return name, nil
	})

	registerVmMethod(k, "fs.getInfo", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path string
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string] }"}
		}

		fi, err := vos.Stat(params.Path)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, nil
			}
			return nil, err
		}

		return makeFileEntry(vos, params.Path, fi), nil
	})

	registerVmMethod(k, "fs.setPermissions", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path      string
			Mode      os.FileMode
			Recursive bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], mode: [integer], recursive: [bool] }"}
		}

		var doChange func(name string) error
		doChange = func(name string) error {
			if err := vos.Chmod(name, params.Mode); err != nil {
				return err
			}
			if !params.Recursive {
				return nil
			}
			fi, err := vos.Stat(name)
			if err != nil {
				return err
			}
			if !fi.IsDir() {
				return nil
			}
			dir, err := vos.Open(name)
			if err != nil {
				return err
			}
			defer dir.Close()
			entries, err := dir.Readdirnames(0)
			if err != nil {
				return err
			}
			var firstErr error
			for _, entry := range entries {
				err := doChange(name + "/" + entry)
				if err != nil && firstErr == nil {
					firstErr = err
				}
			}
			return firstErr
		}
		if err := doChange(params.Path); err != nil {
			return nil, err
		}

		return true, nil
	})

	registerVmMethod(k, "fs.remove", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path      string
			Recursive bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], recursive: [bool] }"}
		}

		if params.Recursive {
			if err := vos.RemoveAll(params.Path); err != nil {
				return nil, err
			}
			return true, nil
		}

		if err := vos.Remove(params.Path); err != nil {
			return nil, err
		}
		return true, nil
	})

	registerVmMethod(k, "fs.rename", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			OldPath string
			NewPath string
		}
		if args.Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
			return nil, &kite.ArgumentError{Expected: "{ oldPath: [string], newPath: [string] }"}
		}

		if err := vos.Rename(params.OldPath, params.NewPath); err != nil {
			return nil, err
		}

		return true, nil
	})

	registerVmMethod(k, "fs.createDirectory", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path      string
			Recursive bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], recursive: [bool] }"}
		}

		if params.Recursive {
			if err := vos.MkdirAll(params.Path, 0755); err != nil {
				return nil, err
			}
			return true, nil
		}

		if err := vos.Mkdir(params.Path, 0755); err != nil {
			return nil, err
		}
		return true, nil
	})
}

type FileEntry struct {
	Name     string      `json:"name"`
	FullPath string      `json:"fullPath"`
	IsDir    bool        `json:"isDir"`
	Size     int64       `json:"size"`
	Mode     os.FileMode `json:"mode"`
	Time     time.Time   `json:"time"`
	IsBroken bool        `json:"isBroken"`
	Readable bool        `json:"readable"`
	Writable bool        `json:"writable"`
}

func makeFileEntry(vos *virt.VOS, fullPath string, fi os.FileInfo) FileEntry {
	entry := FileEntry{
		Name:     fi.Name(),
		FullPath: fullPath,
		IsDir:    fi.IsDir(),
		Size:     fi.Size(),
		Mode:     fi.Mode(),
		Time:     fi.ModTime(),
		Readable: vos.IsReadable(fi),
		Writable: vos.IsWritable(fi),
	}

	if fi.Mode()&os.ModeSymlink != 0 {
		symlinkInfo, err := vos.Stat(path.Dir(fullPath) + "/" + fi.Name())
		if err != nil {
			entry.IsBroken = true
			return entry
		}
		entry.IsDir = symlinkInfo.IsDir()
		entry.Size = symlinkInfo.Size()
		entry.Mode = symlinkInfo.Mode()
		entry.Time = symlinkInfo.ModTime()
	}

	return entry
}
<commit_msg>os kite: fs.glob returns empty array now if nothing matched.<commit_after>package main

import (
	"exp/inotify"
	"fmt"
	"io"
	"koding/tools/dnode"
	"koding/tools/kite"
	"koding/tools/log"
	"koding/virt"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"
)

func init() {
	go func() {
		for err := range virt.WatchErrors {
			log.Warn("Watcher error", err)
		}
	}()
}

func registerFileSystemMethods(k *kite.Kite) {
	registerVmMethod(k, "fs.readDirectory", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path                string
			OnChange            dnode.Callback
			WatchSubdirectories bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], onChange: [function], watchSubdirectories: [bool] }"}
		}

		response := make(map[string]interface{})

		if params.OnChange != nil {
			watch, err := vos.WatchDirectory(params.Path, params.WatchSubdirectories, func(ev *inotify.Event, info os.FileInfo) {
				defer log.RecoverAndLog()

				if (ev.Mask & (inotify.IN_CREATE | inotify.IN_MOVED_TO | inotify.IN_ATTRIB)) != 0 {
					if info == nil {
						return // skip this event, file was deleted and deletion event will follow
					}
					params.OnChange(map[string]interface{}{
						"event": "added",
						"file":  makeFileEntry(vos, ev.Name, info),
					})
					return
				}
				if (ev.Mask & (inotify.IN_DELETE | inotify.IN_MOVED_FROM)) != 0 {
					params.OnChange(map[string]interface{}{
						"event": "removed",
						"file":  FileEntry{Name: path.Base(ev.Name), FullPath: ev.Name},
					})
					return
				}
			})
			if err != nil {
				return nil, err
			}
			channel.OnDisconnect(func() { watch.Close() })
			response["stopWatching"] = func() { watch.Close() }
		}

		dir, err := vos.Open(params.Path)
		if err != nil {
			return nil, err
		}
		defer dir.Close()

		infos, err := dir.Readdir(0)
		if err != nil {
			return nil, err
		}

		files := make([]FileEntry, len(infos))
		for i, info := range infos {
			files[i] = makeFileEntry(vos, path.Join(params.Path, info.Name()), info)
		}
		response["files"] = files

		return response, nil
	})

	registerVmMethod(k, "fs.glob", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Pattern string
		}
		if args.Unmarshal(&params) != nil || params.Pattern == "" {
			return nil, &kite.ArgumentError{Expected: "{ pattern: [string] }"}
		}

		matches, err := vos.Glob(params.Pattern)
		if err == nil && matches == nil {
			matches = []string{}
		}
		return matches, err
	})

	registerVmMethod(k, "fs.readFile", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path string
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string] }"}
		}

		file, err := vos.Open(params.Path)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		fi, err := file.Stat()
		if err != nil {
			return nil, err
		}

		if fi.Size() > 10*1024*1024 {
			return nil, fmt.Errorf("File larger than 10MiB.")
		}

		buf := make([]byte, fi.Size())
		if _, err := io.ReadFull(file, buf); err != nil {
			return nil, err
		}

		return map[string]interface{}{"content": buf}, nil
	})

	registerVmMethod(k, "fs.writeFile", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path           string
			Content        []byte
			DoNotOverwrite bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" || params.Content == nil {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], content: [base64], doNotOverwrite: [bool] }"}
		}

		flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
		if params.DoNotOverwrite {
			flags |= os.O_EXCL
		}
		file, err := vos.OpenFile(params.Path, flags, 0666)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		return file.Write(params.Content)
	})

	suffixRegexp, err := regexp.Compile(`.((_\d+)?)(\.\w*)?$`)
	if err != nil {
		panic(err)
	}

	registerVmMethod(k, "fs.ensureNonexistentPath", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path string
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string] }"}
		}

		name := params.Path
		index := 1
		for {
			_, err := vos.Stat(name)
			if err != nil {
				if os.IsNotExist(err) {
					break
				}
				return nil, err
			}

			loc := suffixRegexp.FindStringSubmatchIndex(name)
			name = name[:loc[2]] + "_" + strconv.Itoa(index) + name[loc[3]:]
			index++
		}

		return name, nil
	})

	registerVmMethod(k, "fs.getInfo", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path string
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string] }"}
		}

		fi, err := vos.Stat(params.Path)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, nil
			}
			return nil, err
		}

		return makeFileEntry(vos, params.Path, fi), nil
	})

	registerVmMethod(k, "fs.setPermissions", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path      string
			Mode      os.FileMode
			Recursive bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], mode: [integer], recursive: [bool] }"}
		}

		var doChange func(name string) error
		doChange = func(name string) error {
			if err := vos.Chmod(name, params.Mode); err != nil {
				return err
			}
			if !params.Recursive {
				return nil
			}
			fi, err := vos.Stat(name)
			if err != nil {
				return err
			}
			if !fi.IsDir() {
				return nil
			}
			dir, err := vos.Open(name)
			if err != nil {
				return err
			}
			defer dir.Close()
			entries, err := dir.Readdirnames(0)
			if err != nil {
				return err
			}
			var firstErr error
			for _, entry := range entries {
				err := doChange(name + "/" + entry)
				if err != nil && firstErr == nil {
					firstErr = err
				}
			}
			return firstErr
		}
		if err := doChange(params.Path); err != nil {
			return nil, err
		}

		return true, nil
	})

	registerVmMethod(k, "fs.remove", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path      string
			Recursive bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], recursive: [bool] }"}
		}

		if params.Recursive {
			if err := vos.RemoveAll(params.Path); err != nil {
				return nil, err
			}
			return true, nil
		}

		if err := vos.Remove(params.Path); err != nil {
			return nil, err
		}
		return true, nil
	})

	registerVmMethod(k, "fs.rename", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			OldPath string
			NewPath string
		}
		if args.Unmarshal(&params) != nil || params.OldPath == "" || params.NewPath == "" {
			return nil, &kite.ArgumentError{Expected: "{ oldPath: [string], newPath: [string] }"}
		}

		if err := vos.Rename(params.OldPath, params.NewPath); err != nil {
			return nil, err
		}

		return true, nil
	})

	registerVmMethod(k, "fs.createDirectory", false, func(args *dnode.Partial, channel *kite.Channel, vos *virt.VOS) (interface{}, error) {
		var params struct {
			Path      string
			Recursive bool
		}
		if args.Unmarshal(&params) != nil || params.Path == "" {
			return nil, &kite.ArgumentError{Expected: "{ path: [string], recursive: [bool] }"}
		}

		if params.Recursive {
			if err := vos.MkdirAll(params.Path, 0755); err != nil {
				return nil, err
			}
			return true, nil
		}

		if err := vos.Mkdir(params.Path, 0755); err != nil {
			return nil, err
		}
		return true, nil
	})
}

type FileEntry struct {
	Name     string      `json:"name"`
	FullPath string      `json:"fullPath"`
	IsDir    bool        `json:"isDir"`
	Size     int64       `json:"size"`
	Mode     os.FileMode `json:"mode"`
	Time     time.Time   `json:"time"`
	IsBroken bool        `json:"isBroken"`
	Readable bool        `json:"readable"`
	Writable bool        `json:"writable"`
}

func makeFileEntry(vos *virt.VOS, fullPath string, fi os.FileInfo) FileEntry {
	entry := FileEntry{
		Name:     fi.Name(),
		FullPath: fullPath,
		IsDir:    fi.IsDir(),
		Size:     fi.Size(),
		Mode:     fi.Mode(),
		Time:     fi.ModTime(),
		Readable: vos.IsReadable(fi),
		Writable: vos.IsWritable(fi),
	}

	if fi.Mode()&os.ModeSymlink != 0 {
		symlinkInfo, err := vos.Stat(path.Dir(fullPath) + "/" + fi.Name())
		if err != nil {
			entry.IsBroken = true
			return entry
		}
		entry.IsDir = symlinkInfo.IsDir()
		entry.Size = symlinkInfo.Size()
		entry.Mode = symlinkInfo.Mode()
		entry.Time = symlinkInfo.ModTime()
	}

	return entry
}
<|endoftext|>


<commit_before>package systests

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/keybase/client/go/client"
	"github.com/keybase/client/go/libkb"
	"github.com/stretchr/testify/require"
)

func TestTeamAPI(t *testing.T) {
	tt := newTeamTester(t)
	defer tt.cleanup()

	tt.addUser("onr")
	tt.addUser("wtr")
	tt.addUser("botua")
	tt.addUser("rbot")

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "list-self-memberships"}`,
		`{"result":{"teams":null,"annotatedActiveInvites":{}}}`)

	teamName, err := libkb.RandHexString("t", 6)
	require.NoError(t, err)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "create-team", "params": {"options": {"team": "`+teamName+`"}}}`,
		`{"result":{"chatSent":true,"creatorAdded":true}}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "add-members", "params": {"options": {"team": "`+teamName+`", "usernames": [{"username": "`+tt.users[1].username+`", "role": "reader"}]}}}`,
		`{"result":[{"invited":false,"user":{"uid":"`+tt.users[1].uid.String()+`","username":"`+tt.users[1].username+`"},"emailSent":false,"chatSending":false}]}`)
	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "add-members", "params": {"options": {"team": "`+teamName+`", "usernames": [{"username": "`+tt.users[2].username+`", "role": "bot"}]}}}`,
		`{"result":[{"invited":false,"user":{"uid":"`+tt.users[2].uid.String()+`","username":"`+tt.users[2].username+`"},"emailSent":false,"chatSending":false}]}`)
	// TODO HOTPOT-227 reenable
	//assertTeamAPIOutput(t, tt.users[0],
	//	`{"method": "add-members", "params": {"options": {"team": "`+teamName+`", "usernames": [{"username": "`+tt.users[3].username+`", "role": "restrictedbot"}]}}}`,
	//	`{"result":[{"invited":false,"user":{"uid":"`+tt.users[3].uid.String()+`","username":"`+tt.users[3].username+`"},"emailSent":false,"chatSending":false}]}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "create-team", "params": {"options": {"team": "`+teamName+`.sub"}}}`,
		`{"result":{"chatSent":true,"creatorAdded":false}}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "rename-subteam", "params": {"options": {"team": "`+teamName+`.sub", "new-team-name": "`+teamName+`.sub2"}}}`,
		`{}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "edit-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[1].username+`", "role": "writer"}}}`,
		`{}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "remove-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[1].username+`"}}}`,
		`{}`)
	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "remove-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[2].username+`"}}}`,
		`{}`)
	// TODO HOTPOT-227 reenable
	//assertTeamAPIOutput(t, tt.users[0],
	//	`{"method": "remove-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[3].username+`"}}}`,
	//	`{}`)
}

func assertTeamAPIOutput(t *testing.T, u *userPlusDevice, in, expectedOut string) {
	out, err := runTeamAPI(t, u, in)
	require.NoError(t, err)

	out = strings.TrimSpace(out)
	require.Equal(t, expectedOut, out)
}

func runTeamAPI(t *testing.T, u *userPlusDevice, json string) (string, error) {
	cmd := client.NewCmdTeamAPIRunner(u.tc.G)
	cmd.SetMessage(json)
	r, err := libkb.RandString("teamapi", 4)
	if err != nil {
		return "", err
	}
	filename := filepath.Join(os.TempDir(), r+".json")
	cmd.SetOutputFile(filename)
	defer os.Remove(filename)
	if err := cmd.Run(); err != nil {
		return "", err
	}
	out, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}
	return string(out), nil
}
<commit_msg>Add a regex to match teamID output (#19003)<commit_after>package systests

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/keybase/client/go/client"
	"github.com/keybase/client/go/libkb"
	"github.com/stretchr/testify/require"
)

func TestTeamAPI(t *testing.T) {
	tt := newTeamTester(t)
	defer tt.cleanup()

	tt.addUser("onr")
	tt.addUser("wtr")
	tt.addUser("botua")
	tt.addUser("rbot")

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "list-self-memberships"}`,
		`{"result":{"teams":null,"annotatedActiveInvites":{}}}`)

	teamName, err := libkb.RandHexString("t", 6)
	require.NoError(t, err)

	assertTeamAPIOutputRegexp(t, tt.users[0],
		`{"method": "create-team", "params": {"options": {"team": "`+teamName+`"}}}`,
		`{"result":{"teamID":"[a-z0-9]{32}","chatSent":true,"creatorAdded":true}}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "add-members", "params": {"options": {"team": "`+teamName+`", "usernames": [{"username": "`+tt.users[1].username+`", "role": "reader"}]}}}`,
		`{"result":[{"invited":false,"user":{"uid":"`+tt.users[1].uid.String()+`","username":"`+tt.users[1].username+`"},"emailSent":false,"chatSending":false}]}`)
	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "add-members", "params": {"options": {"team": "`+teamName+`", "usernames": [{"username": "`+tt.users[2].username+`", "role": "bot"}]}}}`,
		`{"result":[{"invited":false,"user":{"uid":"`+tt.users[2].uid.String()+`","username":"`+tt.users[2].username+`"},"emailSent":false,"chatSending":false}]}`)
	// TODO HOTPOT-227 reenable
	//assertTeamAPIOutput(t, tt.users[0],
	//	`{"method": "add-members", "params": {"options": {"team": "`+teamName+`", "usernames": [{"username": "`+tt.users[3].username+`", "role": "restrictedbot"}]}}}`,
	//	`{"result":[{"invited":false,"user":{"uid":"`+tt.users[3].uid.String()+`","username":"`+tt.users[3].username+`"},"emailSent":false,"chatSending":false}]}`)

	assertTeamAPIOutputRegexp(t, tt.users[0],
		`{"method": "create-team", "params": {"options": {"team": "`+teamName+`.sub"}}}`,
		`{"result":{"teamID":"[a-z0-9]{32}","chatSent":true,"creatorAdded":false}}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "rename-subteam", "params": {"options": {"team": "`+teamName+`.sub", "new-team-name": "`+teamName+`.sub2"}}}`,
		`{}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "edit-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[1].username+`", "role": "writer"}}}`,
		`{}`)

	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "remove-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[1].username+`"}}}`,
		`{}`)
	assertTeamAPIOutput(t, tt.users[0],
		`{"method": "remove-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[2].username+`"}}}`,
		`{}`)
	// TODO HOTPOT-227 reenable
	//assertTeamAPIOutput(t, tt.users[0],
	//	`{"method": "remove-member", "params": {"options": {"team": "`+teamName+`", "username": "`+tt.users[3].username+`"}}}`,
	//	`{}`)
}

// Compares `in` to `expectedOut`, treating `expectedOut` as a regex to match.
func assertTeamAPIOutputRegexp(t *testing.T, u *userPlusDevice, in, expectedOut string) {
	out, err := runTeamAPI(t, u, in)
	require.NoError(t, err)

	out = strings.TrimSpace(out)
	require.Regexp(t, expectedOut, out)
}

func assertTeamAPIOutput(t *testing.T, u *userPlusDevice, in, expectedOut string) {
	out, err := runTeamAPI(t, u, in)
	require.NoError(t, err)

	out = strings.TrimSpace(out)
	require.Equal(t, expectedOut, out)
}

func runTeamAPI(t *testing.T, u *userPlusDevice, json string) (string, error) {
	cmd := client.NewCmdTeamAPIRunner(u.tc.G)
	cmd.SetMessage(json)
	r, err := libkb.RandString("teamapi", 4)
	if err != nil {
		return "", err
	}
	filename := filepath.Join(os.TempDir(), r+".json")
	cmd.SetOutputFile(filename)
	defer os.Remove(filename)
	if err := cmd.Run(); err != nil {
		return "", err
	}
	out, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", err
	}
	return string(out), nil
}
<|endoftext|>


<commit_before>package topology

import (
	"code.google.com/p/weed-fs/go/storage"
	"errors"
	"code.google.com/p/weed-fs/go/glog"
	"math/rand"
	"sync"
)

type VolumeLayout struct {
	repType         storage.ReplicationType
	vid2location    map[storage.VolumeId]*VolumeLocationList
	writables       []storage.VolumeId // transient array of writable volume id
	pulse           int64
	volumeSizeLimit uint64
	accessLock      sync.Mutex
}

func NewVolumeLayout(repType storage.ReplicationType, volumeSizeLimit uint64, pulse int64) *VolumeLayout {
	return &VolumeLayout{
		repType:         repType,
		vid2location:    make(map[storage.VolumeId]*VolumeLocationList),
		writables:       *new([]storage.VolumeId),
		pulse:           pulse,
		volumeSizeLimit: volumeSizeLimit,
	}
}

func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	if _, ok := vl.vid2location[v.Id]; !ok {
		vl.vid2location[v.Id] = NewVolumeLocationList()
	}
	if vl.vid2location[v.Id].Add(dn) {
		if len(vl.vid2location[v.Id].list) == v.RepType.GetCopyCount() {
			if vl.isWritable(v) {
				vl.writables = append(vl.writables, v.Id)
			}
		}
	}
}

func (vl *VolumeLayout) isWritable(v *storage.VolumeInfo) bool {
	return uint64(v.Size) < vl.volumeSizeLimit &&
		v.Version == storage.CurrentVersion &&
		!v.ReadOnly
}

func (vl *VolumeLayout) Lookup(vid storage.VolumeId) []*DataNode {
	if location := vl.vid2location[vid]; location != nil {
		return location.list
	}
	return nil
}

func (vl *VolumeLayout) PickForWrite(count int, dataCenter string) (*storage.VolumeId, int, *VolumeLocationList, error) {
	len_writers := len(vl.writables)
	if len_writers <= 0 {
		glog.V(0).Infoln("No more writable volumes!")
		return nil, 0, nil, errors.New("No more writable volumes!")
	}
	if dataCenter == "" {
		vid := vl.writables[rand.Intn(len_writers)]
		locationList := vl.vid2location[vid]
		if locationList != nil {
			return &vid, count, locationList, nil
		}
		return nil, 0, nil, errors.New("Strangely vid " + vid.String() + " is on no machine!")
	} else {
		var vid storage.VolumeId
		var locationList *VolumeLocationList
		counter := 0
		for _, v := range vl.writables {
			volumeLocationList := vl.vid2location[v]
			for _, dn := range volumeLocationList.list {
				if dn.GetDataCenter().Id() == NodeId(dataCenter) {
					counter++
					if rand.Intn(counter) < 1 {
						vid, locationList = v, volumeLocationList
					}
				}
			}
		}
		return &vid, count, locationList, nil
	}
	return nil, 0, nil, errors.New("Strangely This Should Never Have Happened!")
}

func (vl *VolumeLayout) GetActiveVolumeCount(dataCenter string) int {
	if dataCenter == "" {
		return len(vl.writables)
	}
	counter := 0
	for _, v := range vl.writables {
		for _, dn := range vl.vid2location[v].list {
			if dn.GetDataCenter().Id() == NodeId(dataCenter) {
				counter++
			}
		}
	}
	return counter
}

func (vl *VolumeLayout) removeFromWritable(vid storage.VolumeId) bool {
	for i, v := range vl.writables {
		if v == vid {
			glog.V(0).Infoln("Volume", vid, "becomes unwritable")
			vl.writables = append(vl.writables[:i], vl.writables[i+1:]...)
			return true
		}
	}
	return false
}
func (vl *VolumeLayout) setVolumeWritable(vid storage.VolumeId) bool {
	for _, v := range vl.writables {
		if v == vid {
			return false
		}
	}
	glog.V(0).Infoln("Volume", vid, "becomes writable")
	vl.writables = append(vl.writables, vid)
	return true
}

func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid storage.VolumeId) bool {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	if vl.vid2location[vid].Remove(dn) {
		if vl.vid2location[vid].Length() < vl.repType.GetCopyCount() {
			glog.V(0).Infoln("Volume", vid, "has", vl.vid2location[vid].Length(), "replica, less than required", vl.repType.GetCopyCount())
			return vl.removeFromWritable(vid)
		}
	}
	return false
}
func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid storage.VolumeId) bool {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	if vl.vid2location[vid].Add(dn) {
		if vl.vid2location[vid].Length() >= vl.repType.GetCopyCount() {
			return vl.setVolumeWritable(vid)
		}
	}
	return false
}

func (vl *VolumeLayout) SetVolumeCapacityFull(vid storage.VolumeId) bool {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	// glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
	return vl.removeFromWritable(vid)
}

func (vl *VolumeLayout) ToMap() interface{} {
	m := make(map[string]interface{})
	m["replication"] = vl.repType.String()
	m["writables"] = vl.writables
	//m["locations"] = vl.vid2location
	return m
}
<commit_msg>ensure unwritable volumes are not in writables list<commit_after>package topology

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/storage"
	"errors"
	"math/rand"
	"sync"
)

type VolumeLayout struct {
	repType         storage.ReplicationType
	vid2location    map[storage.VolumeId]*VolumeLocationList
	writables       []storage.VolumeId // transient array of writable volume id
	pulse           int64
	volumeSizeLimit uint64
	accessLock      sync.Mutex
}

func NewVolumeLayout(repType storage.ReplicationType, volumeSizeLimit uint64, pulse int64) *VolumeLayout {
	return &VolumeLayout{
		repType:         repType,
		vid2location:    make(map[storage.VolumeId]*VolumeLocationList),
		writables:       *new([]storage.VolumeId),
		pulse:           pulse,
		volumeSizeLimit: volumeSizeLimit,
	}
}

func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	if _, ok := vl.vid2location[v.Id]; !ok {
		vl.vid2location[v.Id] = NewVolumeLocationList()
	}
	if vl.vid2location[v.Id].Add(dn) {
		if len(vl.vid2location[v.Id].list) == v.RepType.GetCopyCount() {
			if vl.isWritable(v) {
				vl.writables = append(vl.writables, v.Id)
			} else {
				vl.removeFromWritable(v.Id)
			}
		}
	}
}

func (vl *VolumeLayout) isWritable(v *storage.VolumeInfo) bool {
	return uint64(v.Size) < vl.volumeSizeLimit &&
		v.Version == storage.CurrentVersion &&
		!v.ReadOnly
}

func (vl *VolumeLayout) Lookup(vid storage.VolumeId) []*DataNode {
	if location := vl.vid2location[vid]; location != nil {
		return location.list
	}
	return nil
}

func (vl *VolumeLayout) PickForWrite(count int, dataCenter string) (*storage.VolumeId, int, *VolumeLocationList, error) {
	len_writers := len(vl.writables)
	if len_writers <= 0 {
		glog.V(0).Infoln("No more writable volumes!")
		return nil, 0, nil, errors.New("No more writable volumes!")
	}
	if dataCenter == "" {
		vid := vl.writables[rand.Intn(len_writers)]
		locationList := vl.vid2location[vid]
		if locationList != nil {
			return &vid, count, locationList, nil
		}
		return nil, 0, nil, errors.New("Strangely vid " + vid.String() + " is on no machine!")
	} else {
		var vid storage.VolumeId
		var locationList *VolumeLocationList
		counter := 0
		for _, v := range vl.writables {
			volumeLocationList := vl.vid2location[v]
			for _, dn := range volumeLocationList.list {
				if dn.GetDataCenter().Id() == NodeId(dataCenter) {
					counter++
					if rand.Intn(counter) < 1 {
						vid, locationList = v, volumeLocationList
					}
				}
			}
		}
		return &vid, count, locationList, nil
	}
	return nil, 0, nil, errors.New("Strangely This Should Never Have Happened!")
}

func (vl *VolumeLayout) GetActiveVolumeCount(dataCenter string) int {
	if dataCenter == "" {
		return len(vl.writables)
	}
	counter := 0
	for _, v := range vl.writables {
		for _, dn := range vl.vid2location[v].list {
			if dn.GetDataCenter().Id() == NodeId(dataCenter) {
				counter++
			}
		}
	}
	return counter
}

func (vl *VolumeLayout) removeFromWritable(vid storage.VolumeId) bool {
	toDeleteIndex := -1
	for k, id := range vl.writables {
		if id == vid {
			toDeleteIndex = k
			break
		}
	}
	if toDeleteIndex >= 0 {
		glog.V(0).Infoln("Volume", vid, "becomes unwritable")
		vl.writables = append(vl.writables[0:toDeleteIndex], vl.writables[toDeleteIndex+1:]...)
		return true
	}
	return false
}
func (vl *VolumeLayout) setVolumeWritable(vid storage.VolumeId) bool {
	for _, v := range vl.writables {
		if v == vid {
			return false
		}
	}
	glog.V(0).Infoln("Volume", vid, "becomes writable")
	vl.writables = append(vl.writables, vid)
	return true
}

func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid storage.VolumeId) bool {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	if vl.vid2location[vid].Remove(dn) {
		if vl.vid2location[vid].Length() < vl.repType.GetCopyCount() {
			glog.V(0).Infoln("Volume", vid, "has", vl.vid2location[vid].Length(), "replica, less than required", vl.repType.GetCopyCount())
			return vl.removeFromWritable(vid)
		}
	}
	return false
}
func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid storage.VolumeId) bool {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	if vl.vid2location[vid].Add(dn) {
		if vl.vid2location[vid].Length() >= vl.repType.GetCopyCount() {
			return vl.setVolumeWritable(vid)
		}
	}
	return false
}

func (vl *VolumeLayout) SetVolumeCapacityFull(vid storage.VolumeId) bool {
	vl.accessLock.Lock()
	defer vl.accessLock.Unlock()

	// glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
	return vl.removeFromWritable(vid)
}

func (vl *VolumeLayout) ToMap() interface{} {
	m := make(map[string]interface{})
	m["replication"] = vl.repType.String()
	m["writables"] = vl.writables
	//m["locations"] = vl.vid2location
	return m
}
<|endoftext|>