<commit_before>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/nodetemple/nodetemple/common"
	flag "github.com/ogier/pflag"
)

var (
	cmdHelp = &Command{
		Name:        "help",
		Summary:     "Show a list of commands or help for one command",
		Usage:       "[COMMAND]",
		Description: "Show a list of commands or detailed help for one command",
		Run:         runHelp,
	}

	globalUsageTemplate  *template.Template
	commandUsageTemplate *template.Template
	templFuncs           = template.FuncMap{
		"descToLines": func(s string) []string {
			return strings.Split(strings.Trim(s, "\n\t "), "\n")
		},
		"printOption": func(name, defvalue, usage string) string {
			prefix := "--"
			if len(name) == 1 {
				prefix = "-"
			}
			return fmt.Sprintf("\t%s%s=%s\t%s", prefix, name, defvalue, usage)
		},
	}
)

func init() {
	globalUsageTemplate = template.Must(template.New("global_usage").Funcs(templFuncs).Parse(`
NAME:
{{printf "\t%s - %s" .Executable .Description}}

USAGE:
{{printf "\t%s" .Executable}} [global options] <command> [command options] [arguments...]

VERSION:
{{printf "\t%s" .Version}}

COMMANDS:{{range .Commands}}
{{printf "\t%s\t%s" .Name .Summary}}{{end}}

GLOBAL OPTIONS:{{range .Flags}}
{{printOption .Name .DefValue .Usage}}{{end}}

Run "{{.Executable}} help <command>" for more details on a specific command.
`[1:]))
	commandUsageTemplate = template.Must(template.New("command_usage").Funcs(templFuncs).Parse(`
NAME:
{{printf "\t%s - %s" .Cmd.Name .Cmd.Summary}}

USAGE:
{{printf "\t%s %s %s" .Executable .Cmd.Name .Cmd.Usage}}

DESCRIPTION:
{{range $line := descToLines .Cmd.Description}}{{printf "\t%s" $line}}
{{end}}
{{if .Cmd.Subcommands}}COMMANDS:{{range .Cmd.Subcommands}}
{{printf "\t%s\t%s" .Name .Summary}}{{end}}
{{end}}
{{if .CmdFlags}}OPTIONS:{{range .CmdFlags}}
{{printOption .Name .DefValue .Usage}}{{end}}
{{end}}For help on global options run "{{.Executable}} help"
`[1:]))
}

func runHelp(args []string, out *tabwriter.Writer) int {
	if len(args) < 1 {
		printGlobalUsage()
		return OK
	}

	var cmd *Command

	for _, c := range commands {
		if c.Name == args[0] {
			cmd = c
			break
		}
	}

	if cmd == nil {
		fmt.Println("Unrecognized command:", args[0])
		return ERROR_NO_COMMAND
	}

	printCommandUsage(cmd)
	return OK
}

func printGlobalUsage() {
	globalUsageTemplate.Execute(out, struct {
		Executable  string
		Commands    []*Command
		Flags       []*flag.Flag
		Description string
		Version     string
	}{
		cliName,
		commands,
		getAllFlags(),
		cliDescription,
		common.Version,
	})
}

func printCommandUsage(cmd *Command) {
	commandUsageTemplate.Execute(out, struct {
		Executable string
		Cmd        *Command
		CmdFlags   []*flag.Flag
	}{
		cliName,
		cmd,
		getFlags(&cmd.Flags),
	})
}

func getAllFlags() (flags []*flag.Flag) {
	return getFlags(globalFlagSet)
}

func getFlags(flagset *flag.FlagSet) (flags []*flag.Flag) {
	flags = make([]*flag.Flag, 0)
	flagset.VisitAll(func(f *flag.Flag) {
		flags = append(flags, f)
	})
	return
}
<commit_msg>Add short-hand flags to help<commit_after>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/nodetemple/nodetemple/common"
	flag "github.com/ogier/pflag"
)

var (
	cmdHelp = &Command{
		Name:        "help",
		Summary:     "Show a list of commands or help for one command",
		Usage:       "[COMMAND]",
		Description: "Show a list of commands or detailed help for one command",
		Run:         runHelp,
	}

	globalUsageTemplate  *template.Template
	commandUsageTemplate *template.Template
	templFuncs           = template.FuncMap{
		"descToLines": func(s string) []string {
			return strings.Split(strings.Trim(s, "\n\t "), "\n")
		},
		"printOption": func(name, defvalue, usage string) string {
			prefix := "--"
			if len(name) == 1 {
				prefix = "-"
			}
			return fmt.Sprintf("\t%s%s=%s\t%s", prefix, name, defvalue, usage)
		},
	}
)

func init() {
	globalUsageTemplate = template.Must(template.New("global_usage").Funcs(templFuncs).Parse(`
NAME:
{{printf "\t%s - %s" .Executable .Description}}

USAGE:
{{printf "\t%s" .Executable}} [global options] <command> [command options] [arguments...]

VERSION:
{{printf "\t%s" .Version}}

COMMANDS:{{range .Commands}}
{{printf "\t%s\t%s" .Name .Summary}}{{end}}

GLOBAL OPTIONS:{{range .Flags}}
{{printOption .Name .Shorthand .DefValue .Usage}}{{end}}

Run "{{.Executable}} help <command>" for more details on a specific command.
`[1:]))
	commandUsageTemplate = template.Must(template.New("command_usage").Funcs(templFuncs).Parse(`
NAME:
{{printf "\t%s - %s" .Cmd.Name .Cmd.Summary}}

USAGE:
{{printf "\t%s %s %s" .Executable .Cmd.Name .Cmd.Usage}}

DESCRIPTION:
{{range $line := descToLines .Cmd.Description}}{{printf "\t%s" $line}}
{{end}}
{{if .Cmd.Subcommands}}COMMANDS:{{range .Cmd.Subcommands}}
{{printf "\t%s\t%s" .Name .Summary}}{{end}}
{{end}}
{{if .CmdFlags}}OPTIONS:{{range .CmdFlags}}
{{printOption .Name .Shorthand .DefValue .Usage}}{{end}}
{{end}}For help on global options run "{{.Executable}} help"
`[1:]))
}

func runHelp(args []string, out *tabwriter.Writer) int {
	if len(args) < 1 {
		printGlobalUsage()
		return OK
	}

	var cmd *Command

	for _, c := range commands {
		if c.Name == args[0] {
			cmd = c
			break
		}
	}

	if cmd == nil {
		fmt.Println("Unrecognized command:", args[0])
		return ERROR_NO_COMMAND
	}

	printCommandUsage(cmd)
	return OK
}

func printGlobalUsage() {
	globalUsageTemplate.Execute(out, struct {
		Executable  string
		Commands    []*Command
		Flags       []*flag.Flag
		Description string
		Version     string
	}{
		cliName,
		commands,
		getAllFlags(),
		cliDescription,
		common.Version,
	})
}

func printCommandUsage(cmd *Command) {
	commandUsageTemplate.Execute(out, struct {
		Executable string
		Cmd        *Command
		CmdFlags   *flag.FlagSet
	}{
		cliName,
		cmd,
		cmd.Flags(),
	})
}

func getAllFlags() (flags []*flag.Flag) {
	return getFlags(globalFlagSet)
}

func getFlags(flagset *flag.FlagSet) (flags []*flag.Flag) {
	flags = make([]*flag.Flag, 0)
	flagset.VisitAll(func(f *flag.Flag) {
		flags = append(flags, f)
	})
	return
}
<|endoftext|>


<commit_before>package ipset

import (
	"os/exec"

	"github.com/pkg/errors"
)

type Name string

type Type string

const (
	ListSet = Type("list:set")
	HashIP  = Type("hash:ip")
)

type Interface interface {
	Create(ipsetName Name, ipsetType Type) error
	AddEntry(ipsetName Name, entry string) error
	DelEntry(ipsetName Name, entry string) error
	Flush(ipsetName Name) error
	Destroy(ipsetName Name) error

	FlushAll() error
	DestroyAll() error
}

type ipset struct {
	refCount
}

func New() Interface {
	return &ipset{refCount: newRefCount()}
}

func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
	return doExec("create", string(ipsetName), string(ipsetType))
}

func (i *ipset) AddEntry(ipsetName Name, entry string) error {
	if i.inc(ipsetName, entry) > 1 { // already in the set
		return nil
	}
	return doExec("add", string(ipsetName), entry)
}

func (i *ipset) DelEntry(ipsetName Name, entry string) error {
	if i.dec(ipsetName, entry) > 0 { // still needed
		return nil
	}
	return doExec("del", string(ipsetName), entry)
}

func (i *ipset) Flush(ipsetName Name) error {
	return doExec("flush", string(ipsetName))
}

func (i *ipset) FlushAll() error {
	return doExec("flush")
}

func (i *ipset) Destroy(ipsetName Name) error {
	return doExec("destroy", string(ipsetName))
}

func (i *ipset) DestroyAll() error {
	return doExec("destroy")
}

func doExec(args ...string) error {
	if output, err := exec.Command("ipset", args...).CombinedOutput(); err != nil {
		return errors.Wrapf(err, "ipset %v failed: %s", args, output)
	}
	return nil
}

// Reference-counting
type key struct {
	ipsetName Name
	entry     string
}

// note no locking is required as all operations are serialised in the controller
type refCount struct {
	ref map[key]int
}

func newRefCount() refCount {
	return refCount{ref: make(map[key]int)}
}

func (rc *refCount) inc(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]++
	return rc.ref[k]
}

func (rc *refCount) dec(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]--
	return rc.ref[k]
}
<commit_msg>Remove refcounts on destroy and flush<commit_after>package ipset

import (
	"os/exec"

	"github.com/pkg/errors"
)

type Name string

type Type string

const (
	ListSet = Type("list:set")
	HashIP  = Type("hash:ip")
)

type Interface interface {
	Create(ipsetName Name, ipsetType Type) error
	AddEntry(ipsetName Name, entry string) error
	DelEntry(ipsetName Name, entry string) error
	Flush(ipsetName Name) error
	Destroy(ipsetName Name) error

	FlushAll() error
	DestroyAll() error
}

type ipset struct {
	refCount
}

func New() Interface {
	return &ipset{refCount: newRefCount()}
}

func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
	return doExec("create", string(ipsetName), string(ipsetType))
}

func (i *ipset) AddEntry(ipsetName Name, entry string) error {
	if i.inc(ipsetName, entry) > 1 { // already in the set
		return nil
	}
	return doExec("add", string(ipsetName), entry)
}

func (i *ipset) DelEntry(ipsetName Name, entry string) error {
	if i.dec(ipsetName, entry) > 0 { // still needed
		return nil
	}
	return doExec("del", string(ipsetName), entry)
}

func (i *ipset) Flush(ipsetName Name) error {
	i.removeSet(ipsetName)
	return doExec("flush", string(ipsetName))
}

func (i *ipset) FlushAll() error {
	i.refCount = newRefCount()
	return doExec("flush")
}

func (i *ipset) Destroy(ipsetName Name) error {
	i.removeSet(ipsetName)
	return doExec("destroy", string(ipsetName))
}

func (i *ipset) DestroyAll() error {
	i.refCount = newRefCount()
	return doExec("destroy")
}

func doExec(args ...string) error {
	if output, err := exec.Command("ipset", args...).CombinedOutput(); err != nil {
		return errors.Wrapf(err, "ipset %v failed: %s", args, output)
	}
	return nil
}

// Reference-counting
type key struct {
	ipsetName Name
	entry     string
}

// note no locking is required as all operations are serialised in the controller
type refCount struct {
	ref map[key]int
}

func newRefCount() refCount {
	return refCount{ref: make(map[key]int)}
}

func (rc *refCount) inc(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]++
	return rc.ref[k]
}

func (rc *refCount) dec(ipsetName Name, entry string) int {
	k := key{ipsetName, entry}
	rc.ref[k]--
	return rc.ref[k]
}

func (rc *refCount) removeSet(ipsetName Name) {
	for k := range rc.ref {
		if k.ipsetName == ipsetName {
			delete(rc.ref, k)
		}
	}
}
<|endoftext|>


<commit_before>package libgobuster

import (
	"fmt"
	"net"
	"strings"

	uuid "github.com/satori/go.uuid"
)

func SetupDns(s *State) bool {
	// Resolve a subdomain that probably shouldn't exist
	guid := uuid.Must(uuid.NewV4())
	wildcardIps, err := net.LookupHost(fmt.Sprintf("%s.%s", guid, s.URL))
	if err == nil {
		s.IsWildcard = true
		s.WildcardIps.AddRange(wildcardIps)
		fmt.Println("[-] Wildcard DNS found. IP address(es): ", s.WildcardIps.Stringify())
		if !s.WildcardForced {
			fmt.Println("[-] To force processing of Wildcard DNS, specify the '-fw' switch.")
		}
		return s.WildcardForced
	}

	if !s.Quiet {
		// Provide a warning if the base domain doesn't resolve (in case of typo)
		_, err = net.LookupHost(s.URL)
		if err != nil {
			// Not an error, just a warning. Eg. `yp.to` doesn't resolve, but `cr.py.to` does!
			fmt.Println("[-] Unable to validate base domain:", s.URL)
		}
	}

	return true
}

func ProcessDnsEntry(s *State, word string, resultChan chan<- Result) {
	subdomain := fmt.Sprintf("%s.%s", word, s.URL)
	ips, err := net.LookupHost(subdomain)

	if err == nil {
		if !s.IsWildcard || !s.WildcardIps.ContainsAny(ips) {
			result := Result{
				Entity: subdomain,
			}
			if s.ShowIPs {
				result.Extra = strings.Join(ips, ", ")
			} else if s.ShowCNAME {
				cname, err := net.LookupCNAME(subdomain)
				if err == nil {
					result.Extra = cname
				}
			}
			resultChan <- result
		}
	} else if s.Verbose {
		result := Result{
			Entity: subdomain,
			Status: 404,
		}
		resultChan <- result
	}
}

func PrintDnsResult(s *State, r *Result) {
	var output string
	if r.Status == 404 {
		output = fmt.Sprintf("Missing: %s\n", r.Entity)
	} else if s.ShowIPs {
		output = fmt.Sprintf("Found: %s [%s]\n", r.Entity, r.Extra)
	} else if s.ShowCNAME {
		output = fmt.Sprintf("Found: %s [%s]\n", r.Entity, r.Extra)
	} else {
		output = fmt.Sprintf("Found: %s\n", r.Entity)
	}
	fmt.Printf("%s", output)

	if s.OutputFile != nil {
		WriteToFile(output, s)
	}
}
<commit_msg>status output also in dns mode<commit_after>package libgobuster

import (
	"bytes"
	"fmt"
	"net"
	"strings"

	uuid "github.com/satori/go.uuid"
)

func SetupDns(s *State) bool {
	// Resolve a subdomain that probably shouldn't exist
	guid := uuid.Must(uuid.NewV4())
	wildcardIps, err := net.LookupHost(fmt.Sprintf("%s.%s", guid, s.URL))
	if err == nil {
		s.IsWildcard = true
		s.WildcardIps.AddRange(wildcardIps)
		fmt.Println("[-] Wildcard DNS found. IP address(es): ", s.WildcardIps.Stringify())
		if !s.WildcardForced {
			fmt.Println("[-] To force processing of Wildcard DNS, specify the '-fw' switch.")
		}
		return s.WildcardForced
	}

	if !s.Quiet {
		// Provide a warning if the base domain doesn't resolve (in case of typo)
		_, err = net.LookupHost(s.URL)
		if err != nil {
			// Not an error, just a warning. Eg. `yp.to` doesn't resolve, but `cr.py.to` does!
			fmt.Println("[-] Unable to validate base domain:", s.URL)
		}
	}

	return true
}

func ProcessDnsEntry(s *State, word string, resultChan chan<- Result) {
	subdomain := fmt.Sprintf("%s.%s", word, s.URL)
	ips, err := net.LookupHost(subdomain)

	if err == nil {
		if !s.IsWildcard || !s.WildcardIps.ContainsAny(ips) {
			result := Result{
				Entity: subdomain,
			}
			if s.ShowIPs {
				result.Extra = strings.Join(ips, ", ")
			} else if s.ShowCNAME {
				cname, err := net.LookupCNAME(subdomain)
				if err == nil {
					result.Extra = cname
				}
			}
			resultChan <- result
		}
	} else if s.Verbose {
		result := Result{
			Entity: subdomain,
			Status: 404,
		}
		resultChan <- result
	}
}

func PrintDnsResult(s *State, r *Result) {
	buf := &bytes.Buffer{}
	// remove status output
	fmt.Fprintf(buf, "\r")

	if r.Status == 404 {
		fmt.Fprintf(buf, "Missing: %s\n", r.Entity)
	} else if s.ShowIPs {
		fmt.Fprintf(buf, "Found: %s [%s]\n", r.Entity, r.Extra)
	} else if s.ShowCNAME {
		fmt.Fprintf(buf, "Found: %s [%s]\n", r.Entity, r.Extra)
	} else {
		fmt.Fprintf(buf, "Found: %s\n", r.Entity)
	}

	output := buf.String()

	fmt.Printf("%s", output)

	if s.OutputFile != nil {
		WriteToFile(output, s)
	}
}
<|endoftext|>


<commit_before>package pq

import (
	"database/sql"
	_ "github.com/lib/pq"
	"github.com/oursky/ourd/oddb"
	. "github.com/smartystreets/goconvey/convey"

	"testing"
)

func getTestConn(t *testing.T) *conn {
	c, err := Open("com.oursky.ourd", "dbname=ourd_test sslmode=disable")
	if err != nil {
		t.Fatal(err)
	}
	return c.(*conn)
}

func cleanupDB(t *testing.T, c *conn) {
	_, err := c.DBMap.Db.Exec("DROP SCHEMA app_com_oursky_ourd CASCADE")
	if err != nil {
		t.Fatal(err)
	}
}

func TestUserCRUD(t *testing.T) {
	var c *conn

	Convey("Conn", t, func() {
		c = getTestConn(t)

		userinfo := oddb.UserInfo{
			ID:    "userid",
			Email: "john.doe@example.com",
			Auth: oddb.AuthInfo{
				"authproto": map[string]interface{}{
					"string": "string",
					"bool":   true,
					"number": float64(1),
				},
			},
		}

		Convey("creates user", func() {
			err := c.CreateUser(&userinfo)
			So(err, ShouldBeNil)

			email := ""
			auth := authInfoValue{}
			err = c.DBMap.Db.QueryRow("SELECT email, auth FROM app_com_oursky_ourd._user WHERE id = 'userid'").
				Scan(&email, &auth)
			So(err, ShouldBeNil)

			So(email, ShouldEqual, "john.doe@example.com")
			So(auth, ShouldResemble, authInfoValue{
				"authproto": map[string]interface{}{
					"string": "string",
					"bool":   true,
					"number": float64(1),
				},
			})
		})

		Convey("return ErrUserDuplicated when user to create already exists", func() {
			err := c.CreateUser(&userinfo)
			So(err, ShouldBeNil)

			err = c.CreateUser(&userinfo)
			So(err, ShouldEqual, oddb.ErrUserDuplicated)
		})

		Reset(func() {
			_, err := c.DBMap.Db.Exec("TRUNCATE app_com_oursky_ourd._user")
			So(err, ShouldBeNil)
		})
	})

	cleanupDB(t, c)
}

func TestInsert(t *testing.T) {
	var c *conn
	Convey("Database", t, func() {
		c = getTestConn(t)
		db := c.PublicDB()

		record := oddb.Record{
			Key:  "someid",
			Type: "note",
			Data: map[string]interface{}{
				"content": "some content",
			},
		}

		Convey("creates record if it doesn't exist", func() {
			err := db.Save(&record)
			So(err, ShouldBeNil)

			var content string
			err = db.(*database).DBMap.Db.QueryRow("SELECT content FROM app_com_oursky_ourd.note WHERE _id = 'someid'").Scan(&content)
			So(err, ShouldBeNil)
			So(content, ShouldEqual, "some content")
		})

		Convey("updates record if it already exists", func() {
			err := db.Save(&record)
			So(err, ShouldBeNil)

			record.Set("content", "more content")
			err = db.Save(&record)
			So(err, ShouldBeNil)

			var content string
			err = db.(*database).DBMap.Db.QueryRow("SELECT content FROM app_com_oursky_ourd.note WHERE _id = 'someid'").Scan(&content)
			So(err, ShouldBeNil)
			So(content, ShouldEqual, "more content")
		})

		Reset(func() {
			_, err := db.(*database).DBMap.Exec("TRUNCATE app_com_oursky_ourd.note")
			So(err, ShouldBeNil)
		})
	})

	cleanupDB(t, c)
}

func TestDelete(t *testing.T) {
	var c *conn
	Convey("Database", t, func() {
		c = getTestConn(t)
		db := c.PublicDB()

		record := oddb.Record{
			Key:  "someid",
			Type: "note",
			Data: map[string]interface{}{
				"content": "some content",
			},
		}

		Convey("deletes existing record", func() {
			err := db.Save(&record)
			So(err, ShouldBeNil)

			err = db.Delete("someid")
			So(err, ShouldBeNil)

			err = db.(*database).DBMap.Db.QueryRow("SELECT * FROM app_com_oursky_ourd.note WHERE _id = 'someid'").Scan((*string)(nil))
			So(err, ShouldEqual, sql.ErrNoRows)
		})

		Convey("returns ErrRecordNotFound when record to delete doesn't exist", func() {
			err := db.Delete("notexistid")
			So(err, ShouldEqual, oddb.ErrRecordNotFound)
		})
	})

	cleanupDB(t, c)
}
<commit_msg>Check for password in CreateUser test case, #55<commit_after>package pq

import (
	"database/sql"
	_ "github.com/lib/pq"
	"github.com/oursky/ourd/oddb"
	. "github.com/smartystreets/goconvey/convey"

	"testing"
)

func getTestConn(t *testing.T) *conn {
	c, err := Open("com.oursky.ourd", "dbname=ourd_test sslmode=disable")
	if err != nil {
		t.Fatal(err)
	}
	return c.(*conn)
}

func cleanupDB(t *testing.T, c *conn) {
	_, err := c.DBMap.Db.Exec("DROP SCHEMA app_com_oursky_ourd CASCADE")
	if err != nil {
		t.Fatal(err)
	}
}

func TestUserCRUD(t *testing.T) {
	var c *conn

	Convey("Conn", t, func() {
		c = getTestConn(t)

		userinfo := oddb.UserInfo{
			ID:             "userid",
			Email:          "john.doe@example.com",
			HashedPassword: []byte("$2a$10$RbmNb3Rw.PONA2QTcpjBg.1E00zdSI6dWTUwZi.XC0wZm9OhOEvKO"),
			Auth: oddb.AuthInfo{
				"authproto": map[string]interface{}{
					"string": "string",
					"bool":   true,
					"number": float64(1),
				},
			},
		}

		Convey("creates user", func() {
			err := c.CreateUser(&userinfo)
			So(err, ShouldBeNil)

			email := ""
			password := []byte{}
			auth := authInfoValue{}
			err = c.DBMap.Db.QueryRow("SELECT email, password, auth FROM app_com_oursky_ourd._user WHERE id = 'userid'").
				Scan(&email, &password, &auth)
			So(err, ShouldBeNil)

			So(email, ShouldEqual, "john.doe@example.com")
			So(password, ShouldResemble, []byte("$2a$10$RbmNb3Rw.PONA2QTcpjBg.1E00zdSI6dWTUwZi.XC0wZm9OhOEvKO"))
			So(auth, ShouldResemble, authInfoValue{
				"authproto": map[string]interface{}{
					"string": "string",
					"bool":   true,
					"number": float64(1),
				},
			})
		})

		Convey("return ErrUserDuplicated when user to create already exists", func() {
			err := c.CreateUser(&userinfo)
			So(err, ShouldBeNil)

			err = c.CreateUser(&userinfo)
			So(err, ShouldEqual, oddb.ErrUserDuplicated)
		})

		Reset(func() {
			_, err := c.DBMap.Db.Exec("TRUNCATE app_com_oursky_ourd._user")
			So(err, ShouldBeNil)
		})
	})

	cleanupDB(t, c)
}

func TestInsert(t *testing.T) {
	var c *conn
	Convey("Database", t, func() {
		c = getTestConn(t)
		db := c.PublicDB()

		record := oddb.Record{
			Key:  "someid",
			Type: "note",
			Data: map[string]interface{}{
				"content": "some content",
			},
		}

		Convey("creates record if it doesn't exist", func() {
			err := db.Save(&record)
			So(err, ShouldBeNil)

			var content string
			err = db.(*database).DBMap.Db.QueryRow("SELECT content FROM app_com_oursky_ourd.note WHERE _id = 'someid'").Scan(&content)
			So(err, ShouldBeNil)
			So(content, ShouldEqual, "some content")
		})

		Convey("updates record if it already exists", func() {
			err := db.Save(&record)
			So(err, ShouldBeNil)

			record.Set("content", "more content")
			err = db.Save(&record)
			So(err, ShouldBeNil)

			var content string
			err = db.(*database).DBMap.Db.QueryRow("SELECT content FROM app_com_oursky_ourd.note WHERE _id = 'someid'").Scan(&content)
			So(err, ShouldBeNil)
			So(content, ShouldEqual, "more content")
		})

		Reset(func() {
			_, err := db.(*database).DBMap.Exec("TRUNCATE app_com_oursky_ourd.note")
			So(err, ShouldBeNil)
		})
	})

	cleanupDB(t, c)
}

func TestDelete(t *testing.T) {
	var c *conn
	Convey("Database", t, func() {
		c = getTestConn(t)
		db := c.PublicDB()

		record := oddb.Record{
			Key:  "someid",
			Type: "note",
			Data: map[string]interface{}{
				"content": "some content",
			},
		}

		Convey("deletes existing record", func() {
			err := db.Save(&record)
			So(err, ShouldBeNil)

			err = db.Delete("someid")
			So(err, ShouldBeNil)

			err = db.(*database).DBMap.Db.QueryRow("SELECT * FROM app_com_oursky_ourd.note WHERE _id = 'someid'").Scan((*string)(nil))
			So(err, ShouldEqual, sql.ErrNoRows)
		})

		Convey("returns ErrRecordNotFound when record to delete doesn't exist", func() {
			err := db.Delete("notexistid")
			So(err, ShouldEqual, oddb.ErrRecordNotFound)
		})
	})

	cleanupDB(t, c)
}
<|endoftext|>


<commit_before>// Package quantize offers an implementation of the draw.Quantize interface using an optimized Median Cut method,
// including advanced functionality for fine-grained control of color priority
package quantize

import (
	"image"
	"image/color"
	"sync"
)

type bucketPool struct {
	sync.Pool
	maxCap int
	m      sync.Mutex
}

func (p *bucketPool) getBucket(c int) colorBucket {
	p.m.Lock()
	if p.maxCap > c {
		p.maxCap = p.maxCap * 99 / 100
	}
	if p.maxCap < c {
		p.maxCap = c
	}
	p.m.Unlock()
	val := p.Pool.Get()
	if val == nil || cap(val.(colorBucket)) < c {
		return make(colorBucket, p.maxCap)[0:c]
	}
	slice := val.(colorBucket)
	slice = slice[0:c]
	for i := range slice {
		slice[i] = colorPriority{}
	}
	return slice
}

var bpool bucketPool

// AggregationType specifies the type of aggregation to be done
type AggregationType uint8

const (
	// Mode - pick the highest priority value
	Mode AggregationType = iota
	// Mean - weighted average all values
	Mean
)

// MedianCutQuantizer implements the go draw.Quantizer interface using the Median Cut method
type MedianCutQuantizer struct {
	// The type of aggregation to be used to find final colors
	Aggregation AggregationType
	// The weighting function to use on each pixel
	Weighting func(image.Image, int, int) uint32
	// Whether to create a transparent entry
	AddTransparent bool
}

//bucketize takes a bucket and performs median cut on it to obtain the target number of grouped buckets
func bucketize(colors colorBucket, num int) (buckets []colorBucket) {
	if len(colors) == 0 || num == 0 {
		return nil
	}
	bucket := colors
	buckets = make([]colorBucket, 1, num*2)
	buckets[0] = bucket

	for len(buckets) < num && len(buckets) < len(colors) { // Limit to palette capacity or number of colors
		bucket, buckets = buckets[0], buckets[1:]
		if len(bucket) < 2 {
			buckets = append(buckets, bucket)
			continue
		}

		left, right := bucket.partition()
		buckets = append(buckets, left, right)
	}
	return
}

// palettize finds a single color to represent a set of color buckets
func (q MedianCutQuantizer) palettize(p color.Palette, buckets []colorBucket) color.Palette {
	for _, bucket := range buckets {
		switch q.Aggregation {
		case Mean:
			mean := bucket.mean()
			p = append(p, mean)
		case Mode:
			var best *colorPriority
			for _, c := range bucket {
				if best == nil || c.p > best.p {
					best = &c
				}
			}
			p = append(p, best.RGBA)
		}
	}
	return p
}

// quantizeSlice expands the provided bucket and then palettizes the result
func (q MedianCutQuantizer) quantizeSlice(p color.Palette, colors []colorPriority) color.Palette {
	numColors := cap(p) - len(p)
	addTransparent := q.AddTransparent
	if addTransparent {
		for _, c := range p {
			if _, _, _, a := c.RGBA(); a == 0 {
				addTransparent = false
			}
		}
		if addTransparent {
			numColors--
		}
	}
	buckets := bucketize(colors, numColors)
	p = q.palettize(p, buckets)
	if addTransparent {
		p = append(p, color.RGBA{0, 0, 0, 0})
	}
	return p
}

func rgbaAt(m image.Image, x int, y int) color.RGBA {
	switch i := m.(type) {
	case *image.YCbCr:
		c := i.YCbCrAt(x, y)
		r, g, b := color.YCbCrToRGB(c.Y, c.Cb, c.Cr)
		return color.RGBA{r, g, b, 255}
	case *image.RGBA:
		return i.RGBAAt(x, y)
	default:
		return color.RGBAModel.Convert(i.At(x, y)).(color.RGBA)
	}
}

// buildBucket creates a prioritized color slice with all the colors in the image
func (q MedianCutQuantizer) buildBucket(m image.Image) (bucket colorBucket) {
	bounds := m.Bounds()
	size := (bounds.Max.X - bounds.Min.X) * (bounds.Max.Y - bounds.Min.Y) * 2
	sparseBucket := bpool.getBucket(size)
	created := 0

	for x := bounds.Min.X; x < bounds.Max.X; x++ {
		for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
			priority := uint32(1)
			if q.Weighting != nil {
				priority = q.Weighting(m, x, y)
			}
			if priority != 0 {
				c := rgbaAt(m, x, y)
				index := int(c.R)<<16 | int(c.G)<<8 | int(c.B)
				for i := 1; ; i++ {
					if sparseBucket[index%size].p == 0 {
						sparseBucket[index%size] = colorPriority{priority, c}
						created++
						break
					}
					if sparseBucket[index%size].RGBA == c {
						sparseBucket[index%size].p += priority
						break
					}
					index += 1 + i
				}
			}
		}
	}
	bucket = sparseBucket[:0]
	for _, p := range sparseBucket {
		if p.p != 0 {
			bucket = append(bucket, p)
		}
	}
	return
}

// Quantize quantizes an image to a palette and returns the palette
func (q MedianCutQuantizer) Quantize(p color.Palette, m image.Image) color.Palette {
	bucket := q.buildBucket(m)
	defer bpool.Put(bucket)
	return q.quantizeSlice(p, bucket)
}
<commit_msg>Switched image iteration order to increase cache line hits<commit_after>// Package quantize offers an implementation of the draw.Quantize interface using an optimized Median Cut method,
// including advanced functionality for fine-grained control of color priority
package quantize

import (
	"image"
	"image/color"
	"sync"
)

type bucketPool struct {
	sync.Pool
	maxCap int
	m      sync.Mutex
}

func (p *bucketPool) getBucket(c int) colorBucket {
	p.m.Lock()
	if p.maxCap > c {
		p.maxCap = p.maxCap * 99 / 100
	}
	if p.maxCap < c {
		p.maxCap = c
	}
	p.m.Unlock()
	val := p.Pool.Get()
	if val == nil || cap(val.(colorBucket)) < c {
		return make(colorBucket, p.maxCap)[0:c]
	}
	slice := val.(colorBucket)
	slice = slice[0:c]
	for i := range slice {
		slice[i] = colorPriority{}
	}
	return slice
}

var bpool bucketPool

// AggregationType specifies the type of aggregation to be done
type AggregationType uint8

const (
	// Mode - pick the highest priority value
	Mode AggregationType = iota
	// Mean - weighted average all values
	Mean
)

// MedianCutQuantizer implements the go draw.Quantizer interface using the Median Cut method
type MedianCutQuantizer struct {
	// The type of aggregation to be used to find final colors
	Aggregation AggregationType
	// The weighting function to use on each pixel
	Weighting func(image.Image, int, int) uint32
	// Whether to create a transparent entry
	AddTransparent bool
}

//bucketize takes a bucket and performs median cut on it to obtain the target number of grouped buckets
func bucketize(colors colorBucket, num int) (buckets []colorBucket) {
	if len(colors) == 0 || num == 0 {
		return nil
	}
	bucket := colors
	buckets = make([]colorBucket, 1, num*2)
	buckets[0] = bucket

	for len(buckets) < num && len(buckets) < len(colors) { // Limit to palette capacity or number of colors
		bucket, buckets = buckets[0], buckets[1:]
		if len(bucket) < 2 {
			buckets = append(buckets, bucket)
			continue
		}

		left, right := bucket.partition()
		buckets = append(buckets, left, right)
	}
	return
}

// palettize finds a single color to represent a set of color buckets
func (q MedianCutQuantizer) palettize(p color.Palette, buckets []colorBucket) color.Palette {
	for _, bucket := range buckets {
		switch q.Aggregation {
		case Mean:
			mean := bucket.mean()
			p = append(p, mean)
		case Mode:
			var best *colorPriority
			for _, c := range bucket {
				if best == nil || c.p > best.p {
					best = &c
				}
			}
			p = append(p, best.RGBA)
		}
	}
	return p
}

// quantizeSlice expands the provided bucket and then palettizes the result
func (q MedianCutQuantizer) quantizeSlice(p color.Palette, colors []colorPriority) color.Palette {
	numColors := cap(p) - len(p)
	addTransparent := q.AddTransparent
	if addTransparent {
		for _, c := range p {
			if _, _, _, a := c.RGBA(); a == 0 {
				addTransparent = false
			}
		}
		if addTransparent {
			numColors--
		}
	}
	buckets := bucketize(colors, numColors)
	p = q.palettize(p, buckets)
	if addTransparent {
		p = append(p, color.RGBA{0, 0, 0, 0})
	}
	return p
}

func rgbaAt(m image.Image, x int, y int) color.RGBA {
	switch i := m.(type) {
	case *image.YCbCr:
		c := i.YCbCrAt(x, y)
		r, g, b := color.YCbCrToRGB(c.Y, c.Cb, c.Cr)
		return color.RGBA{r, g, b, 255}
	case *image.RGBA:
		return i.RGBAAt(x, y)
	default:
		return color.RGBAModel.Convert(i.At(x, y)).(color.RGBA)
	}
}

// buildBucket creates a prioritized color slice with all the colors in the image
func (q MedianCutQuantizer) buildBucket(m image.Image) (bucket colorBucket) {
	bounds := m.Bounds()
	size := (bounds.Max.X - bounds.Min.X) * (bounds.Max.Y - bounds.Min.Y) * 2
	sparseBucket := bpool.getBucket(size)
	created := 0

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			priority := uint32(1)
			if q.Weighting != nil {
				priority = q.Weighting(m, x, y)
			}
			if priority != 0 {
				c := rgbaAt(m, x, y)
				index := int(c.R)<<16 | int(c.G)<<8 | int(c.B)
				for i := 1; ; i++ {
					if sparseBucket[index%size].p == 0 {
						sparseBucket[index%size] = colorPriority{priority, c}
						created++
						break
					}
					if sparseBucket[index%size].RGBA == c {
						sparseBucket[index%size].p += priority
						break
					}
					index += 1 + i
				}
			}
		}
	}
	bucket = sparseBucket[:0]
	for _, p := range sparseBucket {
		if p.p != 0 {
			bucket = append(bucket, p)
		}
	}
	return
}

// Quantize quantizes an image to a palette and returns the palette
func (q MedianCutQuantizer) Quantize(p color.Palette, m image.Image) color.Palette {
	bucket := q.buildBucket(m)
	defer bpool.Put(bucket)
	return q.quantizeSlice(p, bucket)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/streadway/amqp"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
)

type ConfigParameters struct {
	QueueName      string
	RabbitmqURL    string
	PrefetchCount  int
	ConnRetryDelay int
	HttpTimeout    int
}

var config = ConfigParameters{}

type HttpRequestMessage struct {
	// RabbitMQ message
	delivery amqp.Delivery

	// Parsed fields
	url        string
	headers    map[string]string
	body       string
	expiration int64
	retries    int

	// Drop / Retry Indicator
	// Message is dropped after: Successful http request, message expiration, http response code 4XX or any other permanent error
	drop bool
}

var gracefulShutdown bool
var gracefulRestart bool

func main() {
	usageMessage()

	runLoadConfig()

	for {
		consumeHttpRequests()

		if gracefulShutdown {
			log.Println("Graceful Shutdown")
			break
		}

		if gracefulRestart {
			gracefulRestart = false
			log.Println("Graceful Restart...")
			runLoadConfig()
		} else {
			log.Println("Lost connection to RabbitMQ")
			time.Sleep(time.Duration(config.ConnRetryDelay) * time.Second)
		}

	}
}

func usageMessage() {
	help := make(map[string]bool)
	help["-h"] = true
	help["help"] = true
	help["-help"] = true
	help["--help"] = true

	argCnt := len(os.Args)

	if argCnt != 2 || help[os.Args[1]] == true {
		fmt.Println("Usage: rabbitmqHttpWorker CONFIG_FILE\n")
		os.Exit(1)
	}
}

func runLoadConfig() {
	if err := loadConfig(); err != nil {
		log.Println("Could not load the configuration file:", err)
		os.Exit(1)
	}
}

func loadConfig() error {
	configBytes, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		return errors.New("Error encountered reading file " + os.Args[1])
	}

	if err = json.Unmarshal(configBytes, &config); err != nil {
		return err
	}

	if len(config.QueueName) == 0 {
		return errors.New("QueueName is empty or missing")
	}

	if len(config.RabbitmqURL) == 0 {
		return errors.New("RabbitmqURL is empty or missing")
	}

	if config.PrefetchCount < 1 || config.PrefetchCount > 100 {
		return errors.New("PrefetchCount must be between 1 and 100")
	}

	if config.ConnRetryDelay < 10 || config.ConnRetryDelay > 300 {
		return errors.New("ConnRetryDelay must be between 10 and 300")
	}

	if config.HttpTimeout < 10 || config.HttpTimeout > 300 {
		return errors.New("HttpTimeout must be between 10 and 300")
	}

	return nil
}

func consumeHttpRequests() {
	log.Println("Connecting to RabbitMQ...")
	conn, err := amqp.Dial(config.RabbitmqURL)
	if err != nil {
		log.Println("Could not connect to RabbitMQ:", err)
		return
	}
	defer conn.Close()
	log.Println("Connected successfully")

	log.Println("Opening a channel to RabbitMQ...")
	ch, err := conn.Channel()
	if err != nil {
		log.Println("Could not open a channel:", err)
		return
	}
	defer ch.Close()
	log.Println("Channel opened successfully")

	log.Println("Setting prefetch count on the channel...")
	if err = ch.Qos(config.PrefetchCount, 0, false); err != nil {
		log.Println("Could not set prefetch count:", err)
		return
	}
	log.Println("Set prefetch count successfully")

	log.Println("Registering a consumer...")
	deliveries, err := ch.Consume(
		config.QueueName, // queue
		"",               // consumer
		false,            // auto-ack
		false,            // exclusive
		false,            // no-local
		false,            // no-wait
		nil,              // args
	)
	if err != nil {
		log.Println("Could not register a consumer:", err)
		return
	}
	log.Println("Registered a consumer successfully")

	closedChannelListener := make(chan *amqp.Error)
	ch.NotifyClose(closedChannelListener)
	log.Println("Started 'closed channel' listener")

	var msg HttpRequestMessage

	// Go channel to coordinate acknowledgment of RabbitMQ messages
	ackCh := make(chan HttpRequestMessage)

	unacknowledgedMsgs := 0

	for {
		select {
		// Process next available message from RabbitMQ
		case delivery := <-deliveries:
			unacknowledgedMsgs++
			log.Println("Unacknowledged message count:", unacknowledgedMsgs)
			log.Println("Message received from RabbitMQ. Parsing...")
			msg, err = parse(delivery)
			if err != nil {
				log.Println("Could not parse message:", err)
				msg.drop = true
				ackCh <- msg
			} else {
				log.Println("Message parsed successfully")
				go msg.httpPost(ackCh)
			}

		// Acknowledge RabbitMQ messages and indicate whether they should be dropped or retried
		case msg = <-ackCh:
			unacknowledgedMsgs--
			log.Println("Unacknowledged message count:", unacknowledgedMsgs)
			log.Println("Acknowledging message...")
			if err = msg.acknowledge(); err != nil {
				log.Println("Could not send message acknowledgement to RabbitMQ:", err)
				return
			}
			log.Println("Message acknowledged successfully")

		// Abort if a problem is detected with the RabbitMQ connection. The main() loop will attempt to reconnect.
		case <-closedChannelListener:
			return
		}
	}
}

func parse(rmqDelivery amqp.Delivery) (msg HttpRequestMessage, err error) {
	type MessageFields struct {
		Url        string
		Headers    []map[string]string
		Body       string
		Expiration int64
	}

	var fields MessageFields

	msg = HttpRequestMessage{delivery: rmqDelivery}

	if err := json.Unmarshal(rmqDelivery.Body, &fields); err != nil {
		return msg, err
	}

	// url
	if len(fields.Url) == 0 {
		err = errors.New("Field 'url' is empty or missing")
		return msg, err
	}
	msg.url = fields.Url

	// headers
	msg.headers = make(map[string]string)
	for _, m := range fields.Headers {
		for key, val := range m {
			msg.headers[key] = val
		}
	}

	// body
	msg.body = fields.Body

	// message expiration
	if fields.Expiration <= 0 {
		err = errors.New("Field 'expiration' is missing or invalid")
	}
	msg.expiration = fields.Expiration

	log.Println("Parsed fields:", fields)

	return msg, nil
}

func (msg HttpRequestMessage) httpPost(ackCh chan HttpRequestMessage) {
	req, err := http.NewRequest("POST", msg.url, bytes.NewBufferString(msg.body))
	if err != nil {
		log.Println("Invalid http request:", err)
		msg.drop = true
		ackCh <- msg
		return
	}

	client := &http.Client{Timeout: time.Duration(config.HttpTimeout) * time.Second}

	for hkey, hval := range msg.headers {
		req.Header.Set(hkey, hval)
	}

	log.Println("Http POST Request url:", msg.url)
	resp, err := client.Do(req)
	if err == nil {
		resp.Body.Close()
	}

	if err != nil {
		log.Println("Error on http POST:", err)
		ackCh <- msg
		return
	} else {
		htmlData, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Println("Error encountered when reading POST response body:", err)
		} else {
			log.Println("POST response status code:", resp.StatusCode)
			log.Println("POST response body:", string(htmlData))
		}
	}

	if resp.StatusCode >= 400 && resp.StatusCode <= 499 {
		log.Println("4XX error on http POST (no retry):", resp.Status)
		msg.drop = true
		ackCh <- msg
		return
	}

	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		log.Println("Success on http POST:", resp.Status)
		msg.drop = true
		ackCh <- msg
		return
	}

	log.Println("Error on http POST:", resp.Status)
	ackCh <- msg
}

func (msg HttpRequestMessage) acknowledge() (err error) {
	if msg.drop {
		log.Println("Sending ACK (drop) for request to url:", msg.url)
		return msg.delivery.Ack(false)
	}

	// Should message be dropped because it expired?
	expired := time.Now().Unix() >= msg.expiration

	if expired {
		log.Println("Sending ACK (drop) for EXPIRED request to url:", msg.url)
		return msg.delivery.Ack(false)
	}

	log.Println("Sending NACK (retry) for request to url:", msg.url)
	return msg.delivery.Nack(false, false)
}
<commit_msg>Added graceful shutdown / restart<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/streadway/amqp"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"
)

type ConfigParameters struct {
	QueueName      string
	RabbitmqURL    string
	PrefetchCount  int
	ConnRetryDelay int
	HttpTimeout    int
}

var config = ConfigParameters{}

type HttpRequestMessage struct {
	// RabbitMQ message
	delivery amqp.Delivery

	// Parsed fields
	url        string
	headers    map[string]string
	body       string
	expiration int64
	retries    int

	// Drop / Retry Indicator
	// Message is dropped after: Successful http request, message expiration, http response code 4XX or any other permanent error
	drop bool
}

var gracefulShutdown bool
var gracefulRestart bool
var connectionBroken bool

// Channel to receive asynchronous signals for graceful shutdown / restart
var signals chan os.Signal

func main() {
	usageMessage()

	runLoadConfig()

	// Register channel to receive OS signals
	// quit = graceful shutdown
	// hangup = graceful restart
	signals = make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGHUP, syscall.SIGQUIT)

	for {
		consumeHttpRequests()

		if gracefulShutdown {
			if connectionBroken {
				log.Println("Broken connection to RabbitMQ was detected during shutdown")
			} else {
				log.Println("Graceful shutdown completed")
			}
			break
		}

		if connectionBroken {
			connectionBroken = false
			gracefulRestart = false
			log.Printf("Broken RabbitMQ connection was detected. Reconnect will be attempted in %d seconds...", config.ConnRetryDelay)
			time.Sleep(time.Duration(config.ConnRetryDelay) * time.Second)
		}

		if gracefulRestart {
			gracefulRestart = false
			log.Println("Restarting...")
		}

		runLoadConfig()
	}
}

func usageMessage() {
	help := make(map[string]bool)
	help["-h"] = true
	help["help"] = true
	help["-help"] = true
	help["--help"] = true

	argCnt := len(os.Args)

	if argCnt != 2 || help[os.Args[1]] == true {
		fmt.Println("Usage: rabbitmqHttpWorker CONFIG_FILE\n")
		os.Exit(1)
	}
}

func runLoadConfig() {
	log.Println("Loading the configuration file...")
	if err := loadConfig(); err != nil {
		log.Println("Could not load the configuration file:", err)
		os.Exit(1)
	}
	log.Println("Configuration file successfully loaded")
}

func loadConfig() error {
	configBytes, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		return errors.New("Error encountered reading file " + os.Args[1])
	}

	if err = json.Unmarshal(configBytes, &config); err != nil {
		return err
	}

	if len(config.QueueName) == 0 {
		return errors.New("QueueName is empty or missing")
	}

	if len(config.RabbitmqURL) == 0 {
		return errors.New("RabbitmqURL is empty or missing")
	}

	if config.PrefetchCount < 1 || config.PrefetchCount > 100 {
		return errors.New("PrefetchCount must be between 1 and 100")
	}

	if config.ConnRetryDelay < 10 || config.ConnRetryDelay > 300 {
		return errors.New("ConnRetryDelay must be between 10 and 300")
	}

	if config.HttpTimeout < 10 || config.HttpTimeout > 300 {
		return errors.New("HttpTimeout must be between 10 and 300")
	}

	return nil
}

func consumeHttpRequests() {
	log.Println("Connecting to RabbitMQ...")
	conn, err := amqp.Dial(config.RabbitmqURL)
	if err != nil {
		log.Println("Could not connect to RabbitMQ:", err)
		return
	}
	defer conn.Close()
	log.Println("Connected successfully")

	log.Println("Opening a channel to RabbitMQ...")
	ch, err := conn.Channel()
	if err != nil {
		log.Println("Could not open a channel:", err)
		return
	}
	defer ch.Close()
	log.Println("Channel opened successfully")

	log.Println("Setting prefetch count on the channel...")
	if err = ch.Qos(config.PrefetchCount, 0, false); err != nil {
		log.Println("Could not set prefetch count:", err)
		return
	}
	log.Println("Set prefetch count successfully")

	log.Println("Registering a consumer...")
	deliveries, err := ch.Consume(
		config.QueueName, // queue
		"",               // consumer
		false,            // auto-ack
		false,            // exclusive
		false,            // no-local
		false,            // no-wait
		nil,              // args
	)
	if err != nil {
		log.Println("Could not register a consumer:", err)
		return
	}
	log.Println("Registered a consumer successfully")

	closedChannelListener := make(chan *amqp.Error, 1)
	ch.NotifyClose(closedChannelListener)
	log.Println("Started 'closed channel' listener")

	var msg HttpRequestMessage

	// Go channel to coordinate acknowledgment of RabbitMQ messages
	ackCh := make(chan HttpRequestMessage, config.PrefetchCount)

	unacknowledgedMsgs := 0

	for {
		select {
		// Process next available message from RabbitMQ
		case delivery := <-deliveries:
			unacknowledgedMsgs++
			log.Println("Unacknowledged message count:", unacknowledgedMsgs)
			log.Println("Message received from RabbitMQ. Parsing...")
			msg, err = parse(delivery)
			if err != nil {
				log.Println("Could not parse message:", err)
				msg.drop = true
				ackCh <- msg
			} else {
				log.Println("Message parsed successfully")
				go msg.httpPost(ackCh)
			}

		// Acknowledge RabbitMQ messages and indicate whether they should be dropped or retried
		case msg = <-ackCh:
			log.Println("Acknowledging message...")
			if err = msg.acknowledge(); err != nil {
				log.Println("Could not send message acknowledgement to RabbitMQ:", err)
				return
			}
			log.Println("Message acknowledged successfully")

			unacknowledgedMsgs--
			log.Println("Unacknowledged message count:", unacknowledgedMsgs)
			if unacknowledgedMsgs == 0 && (gracefulShutdown || gracefulRestart) {
				return
			}

		// Abort if a problem is detected with the RabbitMQ connection. The main() loop will attempt to reconnect.
		case <-closedChannelListener:
			connectionBroken = true
			return

		// Process request to gracefully shutdown / restart
		case sig := <-signals:
			switch signalName := sig.String(); signalName {
			case "hangup":
				log.Println("Graceful restart requested")

				// Substitute a dummy delivery channel to halt consumption from RabbitMQ
				deliveries = make(chan amqp.Delivery, 1)

				gracefulRestart = true
				if unacknowledgedMsgs == 0 {
					return
				}
			case "quit":
				log.Println("Graceful shutdown requested")

				// Substitute a dummy delivery channel to halt consumption from RabbitMQ
				deliveries = make(chan amqp.Delivery, 1)

				gracefulShutdown = true
				if unacknowledgedMsgs == 0 {
					return
				}
			}
		}
	}
}

func parse(rmqDelivery amqp.Delivery) (msg HttpRequestMessage, err error) {
	type MessageFields struct {
		Url        string
		Headers    []map[string]string
		Body       string
		Expiration int64
	}

	var fields MessageFields

	msg = HttpRequestMessage{delivery: rmqDelivery}

	if err := json.Unmarshal(rmqDelivery.Body, &fields); err != nil {
		return msg, err
	}

	// url
	if len(fields.Url) == 0 {
		err = errors.New("Field 'url' is empty or missing")
		return msg, err
	}
	msg.url = fields.Url

	// headers
	msg.headers = make(map[string]string)
	for _, m := range fields.Headers {
		for key, val := range m {
			msg.headers[key] = val
		}
	}

	// body
	msg.body = fields.Body

	// message expiration
	if fields.Expiration <= 0 {
		err = errors.New("Field 'expiration' is missing or invalid")
	}
	msg.expiration = fields.Expiration

	log.Println("Parsed fields:", fields)

	return msg, nil
}

func (msg HttpRequestMessage) httpPost(ackCh chan HttpRequestMessage) {
	req, err := http.NewRequest("POST", msg.url, bytes.NewBufferString(msg.body))
	if err != nil {
		log.Println("Invalid http request:", err)
		msg.drop = true
		ackCh <- msg
		return
	}

	client := &http.Client{Timeout: time.Duration(config.HttpTimeout) * time.Second}

	for hkey, hval := range msg.headers {
		req.Header.Set(hkey, hval)
	}

	log.Println("Http POST Request url:", msg.url)
	resp, err := client.Do(req)

	if err != nil {
		log.Println("Error on http POST:", err)
		ackCh <- msg
		return
	} else {
		htmlData, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Println("Error encountered when reading POST response body:", err)
		} else {
			log.Println("POST response status code:", resp.StatusCode)
			log.Println("POST response body:", string(htmlData))
			resp.Body.Close()
		}
	}

	if resp.StatusCode >= 400 && resp.StatusCode <= 499 {
		log.Println("4XX error on http POST (no retry):", resp.Status)
		msg.drop = true
		ackCh <- msg
		return
	}

	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		log.Println("Success on http POST:", resp.Status)
		msg.drop = true
		ackCh <- msg
		return
	}

	log.Println("Error on http POST:", resp.Status)
	ackCh <- msg
}

func (msg HttpRequestMessage) acknowledge() (err error) {
	if msg.drop {
		log.Println("Sending ACK (drop) for request to url:", msg.url)
		return msg.delivery.Ack(false)
	}

	// Should message be dropped because it expired?
	expired := time.Now().Unix() >= msg.expiration

	if expired {
		log.Println("Sending ACK (drop) for EXPIRED request to url:", msg.url)
		return msg.delivery.Ack(false)
	}

	log.Println("Sending NACK (retry) for request to url:", msg.url)
	return msg.delivery.Nack(false, false)
}
<|endoftext|>


<commit_before>// Package raft_store implements a storage backend for raft on top of LevelDB.
//
// LevelDBStore implements the LogStore and StableStore interfaces of
// https://godoc.org/github.com/hashicorp/raft by using
// https://godoc.org/github.com/syndtr/goleveldb as a storage backend.
package raft_store

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"sync"

	"github.com/hashicorp/raft"
	"github.com/syndtr/goleveldb/leveldb"
	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
)

var metaKey = []byte("logstore-meta")

// LevelDBStore implements the raft.LogStore and raft.StableStore interfaces on
// top of leveldb.
type LevelDBStore struct {
	mu   sync.RWMutex
	meta logstoreMeta
	db   *leveldb.DB
}

type logstoreMeta struct {
	Lo uint64
	Hi uint64
}

// NewLevelDBStore opens a leveldb at the given directory to be used as a log-
// and stable storage for raft.
func NewLevelDBStore(dir string) (*LevelDBStore, error) {
	db, err := leveldb.OpenFile(dir, nil)
	if err != nil {
		if _, ok := err.(*leveldb_errors.ErrCorrupted); !ok {
			return nil, fmt.Errorf("could not open: %v", err)
		}
		db, err = leveldb.RecoverFile(dir, nil)
		if err != nil {
			return nil, fmt.Errorf("could not recover: %v", err)
		}
	}

	v, err := db.Get(metaKey, nil)
	if err != nil {
		if err != leveldb.ErrNotFound {
			db.Close()
			return nil, fmt.Errorf("error reading metadata: %v", err)
		}
		v = []byte(`{"Lo":0,"Hi":0}`)
	}
	var m logstoreMeta
	if err = json.Unmarshal(v, &m); err != nil {
		db.Close()
		return nil, err
	}

	return &LevelDBStore{db: db, meta: m}, nil
}

// Close closes the LevelDBStore. No other methods may be called after this.
func (s *LevelDBStore) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	err := s.db.Close()
	s.db = nil
	return err
}

// FirstIndex implements raft.LogStore.
func (s *LevelDBStore) FirstIndex() (uint64, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.meta.Lo, nil
}

// LastIndex implements raft.LogStore.
func (s *LevelDBStore) LastIndex() (uint64, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.meta.Hi, nil
}

// GetLog implements raft.LogStore.
func (s *LevelDBStore) GetLog(index uint64, rlog *raft.Log) error {
	s.mu.RLock()
	defer s.mu.RUnlock()

	key := make([]byte, binary.Size(index))
	binary.LittleEndian.PutUint64(key, index)
	value, err := s.db.Get(key, nil)
	if err != nil {
		return err
	}
	return json.Unmarshal(value, rlog)
}

// StoreLog implements raft.LogStore.
func (s *LevelDBStore) StoreLog(entry *raft.Log) error {
	return s.StoreLogs([]*raft.Log{entry})
}

// StoreLogs implements raft.LogStore.
func (s *LevelDBStore) StoreLogs(logs []*raft.Log) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var batch leveldb.Batch
	key := make([]byte, binary.Size(uint64(0)))
	meta := s.meta

	for _, entry := range logs {
		binary.LittleEndian.PutUint64(key, entry.Index)
		v, err := json.Marshal(entry)
		if err != nil {
			return err
		}
		batch.Put(key, v)

		if entry.Index < meta.Lo || meta.Lo == 0 {
			meta.Lo = entry.Index
		}
		if entry.Index > meta.Hi {
			meta.Hi = entry.Index
		}
	}
	data, err := json.Marshal(meta)
	if err != nil {
		return err
	}
	batch.Put(metaKey, data)

	if err := s.db.Write(&batch, nil); err != nil {
		return err
	}
	s.meta = meta
	return nil
}

// DeleteRange implements raft.LogStore.
func (s *LevelDBStore) DeleteRange(min, max uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var batch leveldb.Batch
	key := make([]byte, binary.Size(uint64(0)))
	meta := s.meta

	if min > meta.Lo && max < meta.Hi {
		panic("wrongly assumed that the range of stored keys is always contiguous")
	}

	for n := min; n <= max; n++ {
		binary.LittleEndian.PutUint64(key, n)
		batch.Delete(key)
	}
	if max == meta.Hi && min == meta.Lo {
		meta.Lo = 0
		meta.Hi = 0
	} else if max < meta.Hi {
		// We are deleting from the beginning. Update meta.Lo
		meta.Lo = max + 1
	} else if min > meta.Lo {
		// We are deleting from the end. Update meta.Hi
		meta.Hi = min - 1
	}

	data, err := json.Marshal(meta)
	if err != nil {
		return err
	}
	batch.Put(metaKey, data)

	if err := s.db.Write(&batch, nil); err != nil {
		return err
	}
	s.meta = meta
	return nil
}

// Set implements raft.StableStore.
func (s *LevelDBStore) Set(key []byte, val []byte) error {
	key = append([]byte("stablestore-"), key...)
	return s.db.Put(key, val, nil)
}

// Get implements raft.StableStore.
func (s *LevelDBStore) Get(key []byte) ([]byte, error) {
	key = append([]byte("stablestore-"), key...)
	value, err := s.db.Get(key, nil)
	if err == leveldb.ErrNotFound {
		return nil, nil
	}
	return value, err
}

// SetUint64 implements raft.StableStore.
func (s *LevelDBStore) SetUint64(key []byte, val uint64) error {
	key = append([]byte("stablestore-"), key...)

	v := make([]byte, binary.Size(val))
	binary.LittleEndian.PutUint64(v, val)

	return s.db.Put(key, v, nil)
}

// GetUint64 implements raft.StableStore.
func (s *LevelDBStore) GetUint64(key []byte) (uint64, error) {
	key = append([]byte("stablestore-"), key...)
	v, err := s.db.Get(key, nil)
	if err == leveldb.ErrNotFound {
		return 0, nil
	}
	return binary.LittleEndian.Uint64(v), err
}
<commit_msg>Bugfix: return raft error code, otherwise snapshot recovery is broken.<commit_after>// Package raft_store implements a storage backend for raft on top of LevelDB.
//
// LevelDBStore implements the LogStore and StableStore interfaces of
// https://godoc.org/github.com/hashicorp/raft by using
// https://godoc.org/github.com/syndtr/goleveldb as a storage backend.
package raft_store

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"sync"

	"github.com/hashicorp/raft"
	"github.com/syndtr/goleveldb/leveldb"
	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
)

var metaKey = []byte("logstore-meta")

// LevelDBStore implements the raft.LogStore and raft.StableStore interfaces on
// top of leveldb.
type LevelDBStore struct {
	mu   sync.RWMutex
	meta logstoreMeta
	db   *leveldb.DB
}

type logstoreMeta struct {
	Lo uint64
	Hi uint64
}

// NewLevelDBStore opens a leveldb at the given directory to be used as a log-
// and stable storage for raft.
func NewLevelDBStore(dir string) (*LevelDBStore, error) {
	db, err := leveldb.OpenFile(dir, nil)
	if err != nil {
		if _, ok := err.(*leveldb_errors.ErrCorrupted); !ok {
			return nil, fmt.Errorf("could not open: %v", err)
		}
		db, err = leveldb.RecoverFile(dir, nil)
		if err != nil {
			return nil, fmt.Errorf("could not recover: %v", err)
		}
	}

	v, err := db.Get(metaKey, nil)
	if err != nil {
		if err != leveldb.ErrNotFound {
			db.Close()
			return nil, fmt.Errorf("error reading metadata: %v", err)
		}
		v = []byte(`{"Lo":0,"Hi":0}`)
	}
	var m logstoreMeta
	if err = json.Unmarshal(v, &m); err != nil {
		db.Close()
		return nil, err
	}

	return &LevelDBStore{db: db, meta: m}, nil
}

// Close closes the LevelDBStore. No other methods may be called after this.
func (s *LevelDBStore) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	err := s.db.Close()
	s.db = nil
	return err
}

// FirstIndex implements raft.LogStore.
func (s *LevelDBStore) FirstIndex() (uint64, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.meta.Lo, nil
}

// LastIndex implements raft.LogStore.
func (s *LevelDBStore) LastIndex() (uint64, error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	return s.meta.Hi, nil
}

// GetLog implements raft.LogStore.
func (s *LevelDBStore) GetLog(index uint64, rlog *raft.Log) error {
	s.mu.RLock()
	defer s.mu.RUnlock()

	key := make([]byte, binary.Size(index))
	binary.LittleEndian.PutUint64(key, index)
	value, err := s.db.Get(key, nil)
	if err != nil {
		if err == leveldb.ErrNotFound {
			return raft.ErrLogNotFound
		}
		return err
	}
	return json.Unmarshal(value, rlog)
}

// StoreLog implements raft.LogStore.
func (s *LevelDBStore) StoreLog(entry *raft.Log) error {
	return s.StoreLogs([]*raft.Log{entry})
}

// StoreLogs implements raft.LogStore.
func (s *LevelDBStore) StoreLogs(logs []*raft.Log) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var batch leveldb.Batch
	key := make([]byte, binary.Size(uint64(0)))
	meta := s.meta

	for _, entry := range logs {
		binary.LittleEndian.PutUint64(key, entry.Index)
		v, err := json.Marshal(entry)
		if err != nil {
			return err
		}
		batch.Put(key, v)

		if entry.Index < meta.Lo || meta.Lo == 0 {
			meta.Lo = entry.Index
		}
		if entry.Index > meta.Hi {
			meta.Hi = entry.Index
		}
	}
	data, err := json.Marshal(meta)
	if err != nil {
		return err
	}
	batch.Put(metaKey, data)

	if err := s.db.Write(&batch, nil); err != nil {
		return err
	}
	s.meta = meta
	return nil
}

// DeleteRange implements raft.LogStore.
func (s *LevelDBStore) DeleteRange(min, max uint64) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	var batch leveldb.Batch
	key := make([]byte, binary.Size(uint64(0)))
	meta := s.meta

	if min > meta.Lo && max < meta.Hi {
		panic("wrongly assumed that the range of stored keys is always contiguous")
	}

	for n := min; n <= max; n++ {
		binary.LittleEndian.PutUint64(key, n)
		batch.Delete(key)
	}
	if max == meta.Hi && min == meta.Lo {
		meta.Lo = 0
		meta.Hi = 0
	} else if max < meta.Hi {
		// We are deleting from the beginning. Update meta.Lo
		meta.Lo = max + 1
	} else if min > meta.Lo {
		// We are deleting from the end. Update meta.Hi
		meta.Hi = min - 1
	}

	data, err := json.Marshal(meta)
	if err != nil {
		return err
	}
	batch.Put(metaKey, data)

	if err := s.db.Write(&batch, nil); err != nil {
		return err
	}
	s.meta = meta
	return nil
}

// Set implements raft.StableStore.
func (s *LevelDBStore) Set(key []byte, val []byte) error {
	key = append([]byte("stablestore-"), key...)
	return s.db.Put(key, val, nil)
}

// Get implements raft.StableStore.
func (s *LevelDBStore) Get(key []byte) ([]byte, error) {
	key = append([]byte("stablestore-"), key...)
	value, err := s.db.Get(key, nil)
	if err == leveldb.ErrNotFound {
		return nil, nil
	}
	return value, err
}

// SetUint64 implements raft.StableStore.
func (s *LevelDBStore) SetUint64(key []byte, val uint64) error {
	key = append([]byte("stablestore-"), key...)

	v := make([]byte, binary.Size(val))
	binary.LittleEndian.PutUint64(v, val)

	return s.db.Put(key, v, nil)
}

// GetUint64 implements raft.StableStore.
func (s *LevelDBStore) GetUint64(key []byte) (uint64, error) {
	key = append([]byte("stablestore-"), key...)
	v, err := s.db.Get(key, nil)
	if err == leveldb.ErrNotFound {
		return 0, nil
	}
	return binary.LittleEndian.Uint64(v), err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	lua "github.com/anaminus/gopher-lua"
	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/anaminus/snek"
	"github.com/robloxapi/types"
)

const testdata = "testdata"

// Replace scriptArguments[x] with test script.
const replaceIndex = 3

var scriptArguments = [...]string{
	"rbxmk_test",
	"run",
	"--debug",
	"-",
	"true",
	"false",
	"nil",
	"42",
	"3.141592653589793",
	"-1e-8",
	"hello, world!",
	"hello\000world!",
}

type dummyFile struct {
	r    io.Reader
	info *dummyInfo
}

func (d *dummyFile) Name() string               { return "test" }
func (d *dummyFile) Stat() (os.FileInfo, error) { return d.info, nil }
func (d *dummyFile) Read(b []byte) (int, error) { return d.r.Read(b) }
func (d *dummyFile) Write([]byte) (int, error)  { return 0, nil }

type dummyInfo struct {
	name  string
	size  int64
	mode  os.FileMode
	time  time.Time
	isdir bool
}

func (d *dummyInfo) Name() string       { return d.name }
func (d *dummyInfo) Size() int64        { return d.size }
func (d *dummyInfo) Mode() os.FileMode  { return d.mode }
func (d *dummyInfo) ModTime() time.Time { return d.time }
func (d *dummyInfo) IsDir() bool        { return d.isdir }
func (d *dummyInfo) Sys() interface{}   { return d }

func deepeq(t *testing.T, s rbxmk.State, msg string, a, b lua.LValue) bool {
	if b, ok := b.(*lua.LTable); ok {
		if a, ok := a.(*lua.LTable); ok {
			visited := map[lua.LValue]struct{}{}
			b.ForEach(func(k, v lua.LValue) error {
				switch k := k.(type) {
				case lua.LNumber:
					deepeq(t, s, msg+"["+k.String()+"]", a.RawGetInt(int(k)), v)
				case lua.LString:
					deepeq(t, s, msg+"."+k.String(), a.RawGetString(string(k)), v)
				default:
					deepeq(t, s, msg+"["+k.String()+"]", a.RawGet(k), v)
				}
				visited[k] = struct{}{}
				return nil
			})
			a.ForEach(func(k, v lua.LValue) error {
				if _, ok := visited[k]; ok {
					return nil
				}
				switch k := k.(type) {
				case lua.LNumber:
					deepeq(t, s, msg+"["+k.String()+"]", v, b.RawGetInt(int(k)))
				case lua.LString:
					deepeq(t, s, msg+"."+k.String(), v, b.RawGetString(string(k)))
				default:
					deepeq(t, s, msg+"["+k.String()+"]", v, b.RawGet(k))
				}
				return nil
			})
			return true
		}
	}
	if !s.L.Equal(a, b) {
		if a.Type() != b.Type() {
			t.Errorf("%s: expected type %s, got %s", msg, b.Type(), a.Type())
			return false
		}
		if a.Type() == lua.LTString {
			t.Errorf("%s: expected %q, got %q", msg, b.String(), a.String())
			return false
		}
		t.Errorf("%s: expected %s, got %s", msg, b.String(), a.String())
		return false
	}
	return true
}

func initMain(s rbxmk.State, t *testing.T) {
	T := s.L.CreateTable(0, 2)

	// Pass makes a positive assertion. If the first argument is a non-function,
	// then an error is emitted if the value is falsy. Otherwise, the function
	// is called. If the call errors or returns a falsy value, then an error is
	// emitted. Returning no value counts as truthy. The error is emitted to
	// testing.T, but does not cause a Lua error to be thrown. The second
	// optional argument is a string that describes the assertion, which is
	// included with an emitted error.
	T.RawSetString("Pass", s.WrapFunc(func(s rbxmk.State) int {
		v := s.CheckAny(1)
		msg := s.OptString(2, "expected pass")
		switch v := v.(type) {
		case *lua.LFunction:
			n := s.Count()
			s.L.Push(v)
			if err := s.L.PCall(0, lua.MultRet, nil); err != nil {
				t.Errorf("%s: %s", msg, err.Error())
				return 0
			}
			if s.Count() > n {
				if !s.L.ToBool(n + 1) {
					if m := s.L.ToString(n + 2); m != "" {
						t.Errorf("%s: %s", msg, m)
					} else {
						t.Errorf(msg)
					}
					return 0
				}
			}
		default:
			if !lua.LVAsBool(v) {
				t.Errorf(msg)
				return 0
			}
		}
		return 0
	}))

	// Fail makes a negative assertion. If the first argument is a non-function,
	// then an error is emitted if the value is truthy. Otherwise, the function
	// is called. If the call does not error or returns a truthy value, then an
	// error is emitted. Returning no value counts as falsy. The error is
	// emitted to testing.T, but does not cause a Lua error to be thrown. The
	// second optional argument is a string that describes the assertion, which
	// is included with an emitted error.
	T.RawSetString("Fail", s.WrapFunc(func(s rbxmk.State) int {
		v := s.CheckAny(1)
		msg := s.OptString(2, "expected fail")
		switch v := v.(type) {
		case *lua.LFunction:
			n := s.Count()
			s.L.Push(v)
			if err := s.L.PCall(0, lua.MultRet, nil); err == nil {
				if s.Count() > n {
					if s.L.ToBool(n + 1) {
						if m := s.L.ToString(n + 2); m != "" {
							t.Errorf("%s: %s", msg, m)
						} else {
							t.Errorf(msg)
						}
					}
					return 0
				}
				t.Errorf(msg)
				return 0
			} else if lua.LVAsBool(s.L.GetGlobal("SHOW_ERRORS")) {
				t.Logf("ERROR: %s\n", err)
			}
		default:
			if lua.LVAsBool(v) {
				t.Errorf(msg)
				return 0
			}
		}
		return 0
	}))

	// Equal asserts whether two values are deeply equal.
	T.RawSetString("Equal", s.L.NewFunction(func(l *lua.LState) int {
		msg := s.CheckString(1)
		a := s.CheckAny(2)
		b := s.CheckAny(3)
		deepeq(t, s, msg, a, b)
		return 0
	}))

	// GC runs the garbage collector.
	T.RawSetString("GC", s.L.NewFunction(func(l *lua.LState) int {
		runtime.GC()
		return 0
	}))

	// UserDataCacheLen returns the number of cached userdata values.
	T.RawSetString("UserDataCacheLen", s.WrapFunc(func(s rbxmk.State) int {
		return s.Push(types.Int(s.UserDataCacheLen()))
	}))

	// UserDataID returns a string that identifies a userdata value.
	T.RawSetString("UserDataID", s.WrapFunc(func(s rbxmk.State) int {
		return s.Push(types.String(fmt.Sprintf("%p", s.CheckUserData(1))))
	}))

	T.RawSetString("DummySymbol", s.UserDataOf(rtypes.Symbol{Name: "DummySymbol"}, "Symbol"))

	s.L.SetGlobal("T", T)
}

// TestScripts runs each .lua file in testdata as a Lua script. If the first
// line starts with a comment that contains "fail", then the script is expected
// to throw an error. All scripts receive the arguments from scriptArguments.
func TestScripts(t *testing.T) {
	program := snek.NewProgram("", scriptArguments[:])
	program.Register(snek.Def{
		Name: "run",
		New: func() snek.Command {
			return &RunCommand{Init: func(s rbxmk.State) { initMain(s, t) }}
		},
	})

	var files []string
	wd, _ := os.Getwd()
	for _, arg := range os.Args[2:] {
		if strings.HasPrefix(arg, "-test.") {
			continue
		}
		rel, err := filepath.Rel(wd, arg)
		if err != nil {
			rel = arg
		}
		files = append(files, rel)
	}
	if len(files) == 0 {
		err := filepath.Walk(testdata, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() &&
				filepath.Ext(info.Name()) == ".lua" &&
				!strings.HasPrefix(filepath.Base(info.Name()), "_") {
				files = append(files, path)
			}
			return nil
		})
		if err != nil {
			t.Fatalf("error walking testdata: %s", err)
		}
	}
	for _, file := range files {
		t.Run(filepath.ToSlash(file), func(t *testing.T) {
			args := make([]string, len(scriptArguments))
			copy(args, scriptArguments[:])
			args[replaceIndex] = file
			err := program.RunWithInput("run", snek.Input{
				Program:   args[0],
				Arguments: args[2:],
			})
			if err != nil {
				t.Errorf("script %s: %s", file, err)
			}
		})
	}
}
<commit_msg>Annotate test errors with line number.<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	lua "github.com/anaminus/gopher-lua"
	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/anaminus/snek"
	"github.com/robloxapi/types"
)

const testdata = "testdata"

// Replace scriptArguments[x] with test script.
const replaceIndex = 3

var scriptArguments = [...]string{
	"rbxmk_test",
	"run",
	"--debug",
	"-",
	"true",
	"false",
	"nil",
	"42",
	"3.141592653589793",
	"-1e-8",
	"hello, world!",
	"hello\000world!",
}

type dummyFile struct {
	r    io.Reader
	info *dummyInfo
}

func (d *dummyFile) Name() string               { return "test" }
func (d *dummyFile) Stat() (os.FileInfo, error) { return d.info, nil }
func (d *dummyFile) Read(b []byte) (int, error) { return d.r.Read(b) }
func (d *dummyFile) Write([]byte) (int, error)  { return 0, nil }

type dummyInfo struct {
	name  string
	size  int64
	mode  os.FileMode
	time  time.Time
	isdir bool
}

func (d *dummyInfo) Name() string       { return d.name }
func (d *dummyInfo) Size() int64        { return d.size }
func (d *dummyInfo) Mode() os.FileMode  { return d.mode }
func (d *dummyInfo) ModTime() time.Time { return d.time }
func (d *dummyInfo) IsDir() bool        { return d.isdir }
func (d *dummyInfo) Sys() interface{}   { return d }

func deepeq(s rbxmk.State, t *testing.T, msg string, a, b lua.LValue) bool {
	if b, ok := b.(*lua.LTable); ok {
		if a, ok := a.(*lua.LTable); ok {
			visited := map[lua.LValue]struct{}{}
			b.ForEach(func(k, v lua.LValue) error {
				switch k := k.(type) {
				case lua.LNumber:
					deepeq(s, t, msg+"["+k.String()+"]", a.RawGetInt(int(k)), v)
				case lua.LString:
					deepeq(s, t, msg+"."+k.String(), a.RawGetString(string(k)), v)
				default:
					deepeq(s, t, msg+"["+k.String()+"]", a.RawGet(k), v)
				}
				visited[k] = struct{}{}
				return nil
			})
			a.ForEach(func(k, v lua.LValue) error {
				if _, ok := visited[k]; ok {
					return nil
				}
				switch k := k.(type) {
				case lua.LNumber:
					deepeq(s, t, msg+"["+k.String()+"]", v, b.RawGetInt(int(k)))
				case lua.LString:
					deepeq(s, t, msg+"."+k.String(), v, b.RawGetString(string(k)))
				default:
					deepeq(s, t, msg+"["+k.String()+"]", v, b.RawGet(k))
				}
				return nil
			})
			return true
		}
	}
	if !s.L.Equal(a, b) {
		if a.Type() != b.Type() {
			lineError(s, t, "expected type %s, got %s", msg, b.Type(), a.Type())
			return false
		}
		if a.Type() == lua.LTString {
			lineError(s, t, "expected %q, got %q", msg, b.String(), a.String())
			return false
		}
		lineError(s, t, "expected %s, got %s", msg, b.String(), a.String())
		return false
	}
	return true
}

func lineError(s rbxmk.State, t *testing.T, f, msg string, v ...interface{}) {
	d, ok := s.L.GetStack(-1)
	if ok {
		s.L.GetInfo("l", d, lua.LNil)
		if d.CurrentLine > 0 {
			v = append([]interface{}{d.CurrentLine, msg}, v...)
			t.Errorf("line %d: %s: "+f, v...)
			return
		}
	}
	v = append([]interface{}{msg}, v...)
	t.Errorf("%s: "+f, v...)
}

func initMain(s rbxmk.State, t *testing.T) {
	T := s.L.CreateTable(0, 2)

	// Pass makes a positive assertion. If the first argument is a non-function,
	// then an error is emitted if the value is falsy. Otherwise, the function
	// is called. If the call errors or returns a falsy value, then an error is
	// emitted. Returning no value counts as truthy. The error is emitted to
	// testing.T, but does not cause a Lua error to be thrown. The second
	// optional argument is a string that describes the assertion, which is
	// included with an emitted error.
	T.RawSetString("Pass", s.WrapFunc(func(s rbxmk.State) int {
		v := s.CheckAny(1)
		msg := s.OptString(2, "expected pass")
		switch v := v.(type) {
		case *lua.LFunction:
			n := s.Count()
			s.L.Push(v)
			if err := s.L.PCall(0, lua.MultRet, nil); err != nil {
				lineError(s, t, "%s", msg, err.Error())
				return 0
			}
			if s.Count() > n {
				if !s.L.ToBool(n + 1) {
					if m := s.L.ToString(n + 2); m != "" {
						lineError(s, t, "%s", msg, m)
					} else {
						lineError(s, t, "", msg)
					}
					return 0
				}
			}
		default:
			if !lua.LVAsBool(v) {
				lineError(s, t, "", msg)
				return 0
			}
		}
		return 0
	}))

	// Fail makes a negative assertion. If the first argument is a non-function,
	// then an error is emitted if the value is truthy. Otherwise, the function
	// is called. If the call does not error or returns a truthy value, then an
	// error is emitted. Returning no value counts as falsy. The error is
	// emitted to testing.T, but does not cause a Lua error to be thrown. The
	// second optional argument is a string that describes the assertion, which
	// is included with an emitted error.
	T.RawSetString("Fail", s.WrapFunc(func(s rbxmk.State) int {
		v := s.CheckAny(1)
		msg := s.OptString(2, "expected fail")
		switch v := v.(type) {
		case *lua.LFunction:
			n := s.Count()
			s.L.Push(v)
			if err := s.L.PCall(0, lua.MultRet, nil); err == nil {
				if s.Count() > n {
					if s.L.ToBool(n + 1) {
						if m := s.L.ToString(n + 2); m != "" {
							lineError(s, t, "%s", msg, m)
						} else {
							lineError(s, t, "", msg)
						}
					}
					return 0
				}
				t.Errorf(msg)
				return 0
			} else if lua.LVAsBool(s.L.GetGlobal("SHOW_ERRORS")) {
				t.Logf("ERROR: %s\n", err)
			}
		default:
			if lua.LVAsBool(v) {
				lineError(s, t, "", msg)
				return 0
			}
		}
		return 0
	}))

	// Equal asserts whether two values are deeply equal.
	T.RawSetString("Equal", s.L.NewFunction(func(l *lua.LState) int {
		msg := s.CheckString(1)
		a := s.CheckAny(2)
		b := s.CheckAny(3)
		deepeq(s, t, msg, a, b)
		return 0
	}))

	// GC runs the garbage collector.
	T.RawSetString("GC", s.L.NewFunction(func(l *lua.LState) int {
		runtime.GC()
		return 0
	}))

	// UserDataCacheLen returns the number of cached userdata values.
	T.RawSetString("UserDataCacheLen", s.WrapFunc(func(s rbxmk.State) int {
		return s.Push(types.Int(s.UserDataCacheLen()))
	}))

	// UserDataID returns a string that identifies a userdata value.
	T.RawSetString("UserDataID", s.WrapFunc(func(s rbxmk.State) int {
		return s.Push(types.String(fmt.Sprintf("%p", s.CheckUserData(1))))
	}))

	T.RawSetString("DummySymbol", s.UserDataOf(rtypes.Symbol{Name: "DummySymbol"}, "Symbol"))

	s.L.SetGlobal("T", T)
}

// TestScripts runs each .lua file in testdata as a Lua script. If the first
// line starts with a comment that contains "fail", then the script is expected
// to throw an error. All scripts receive the arguments from scriptArguments.
func TestScripts(t *testing.T) {
	program := snek.NewProgram("", scriptArguments[:])
	program.Register(snek.Def{
		Name: "run",
		New: func() snek.Command {
			return &RunCommand{Init: func(s rbxmk.State) { initMain(s, t) }}
		},
	})

	var files []string
	wd, _ := os.Getwd()
	for _, arg := range os.Args[2:] {
		if strings.HasPrefix(arg, "-test.") {
			continue
		}
		rel, err := filepath.Rel(wd, arg)
		if err != nil {
			rel = arg
		}
		files = append(files, rel)
	}
	if len(files) == 0 {
		err := filepath.Walk(testdata, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() &&
				filepath.Ext(info.Name()) == ".lua" &&
				!strings.HasPrefix(filepath.Base(info.Name()), "_") {
				files = append(files, path)
			}
			return nil
		})
		if err != nil {
			t.Fatalf("error walking testdata: %s", err)
		}
	}
	for _, file := range files {
		t.Run(filepath.ToSlash(file), func(t *testing.T) {
			args := make([]string, len(scriptArguments))
			copy(args, scriptArguments[:])
			args[replaceIndex] = file
			err := program.RunWithInput("run", snek.Input{
				Program:   args[0],
				Arguments: args[2:],
			})
			if err != nil {
				t.Errorf("script %s: %s", file, err)
			}
		})
	}
}
<|endoftext|>


<commit_before>package rccsv

/*
	CSV reader for RingCentral Call Log CSV files

	SYNOPSIS

 	csv := rccsv.NewCallLogRecordsCsvReader()
	err := csv.ReadFile("/path/to/my.csv")
	if err != nil {
		fmt.Printf("ERR %v", err)
	}
	stats := csv.CallLogRecordsCsv.GetStatsForVoiceRecordings()
*/

import (
	"errors"
	"io"
	"reflect"
	"regexp"
	"strconv"
	"time"

	"github.com/grokify/gotilla/encoding/csvutil"
	"github.com/grokify/gotilla/time/timeutil"
	"github.com/ttacon/libphonenumber"
)

type CallLogStats struct {
	Days           float64
	NumCalls       int64
	TotalSeconds   int64
	TotalMinutes   float32
	TotalHours     float32
	AverageMinutes float32
	DailyCalls     float32
	DailyMinutes   float32
	DailyHours     float32
	Dt14Start      int64
	Dt14End        int64
}

func NewCallLogStats(days int64) CallLogStats {
	s := CallLogStats{}
	s.Days = -1
	s.NumCalls = 0
	s.TotalSeconds = 0
	s.TotalMinutes = 0.0
	s.TotalHours = 0.0
	s.AverageMinutes = 0.0
	s.DailyCalls = 0.0
	s.DailyMinutes = 0.0
	s.DailyHours = 0.0
	s.Dt14Start = -1
	s.Dt14End = -1
	return s
}

func (s *CallLogStats) Inflate() {
	if s.NumCalls < 1 {
		return
	}
	if s.Dt14Start > 0 && s.Dt14End > 0 && s.Dt14Start < s.Dt14End {
		dtStart, err1 := timeutil.TimeForDt14(s.Dt14Start)
		if err1 == nil {
			dtEnd, err2 := timeutil.TimeForDt14(s.Dt14End)
			if err2 == nil {
				dur := dtEnd.Sub(dtStart)
				durHrs := dur.Hours()
				days := durHrs / 24
				s.Days = days
			}
		}
	}
	s.TotalMinutes = float32(s.TotalSeconds / 60)
	s.TotalHours = float32(s.TotalSeconds / 60 / 60)
	s.AverageMinutes = s.TotalMinutes / float32(s.NumCalls)
	if s.Days > 0 {
		s.DailyCalls = float32(s.NumCalls) / float32(s.Days)
		s.DailyMinutes = s.DailyCalls * s.AverageMinutes
		s.DailyHours = s.DailyMinutes / 60
	}
}

type CallLogRecordsCsv struct {
	CallLogRecords []CallLogRecordCsv
}

func (rs *CallLogRecordsCsv) GetStatsForVoiceRecordings() CallLogStats {
	stats := NewCallLogStats(int64(30))
	stats.NumCalls = 0
	for _, rec := range rs.CallLogRecords {
		rec.Inflate()
		if rec.Action != "Phone Call" && rec.Action != "VoIP Call" && rec.Action != "FindMe" {
			//continue
		}
		if rec.ActionResult != "Accepted" && rec.ActionResult != "Call connected" {
			continue
		}
		stats.NumCalls += 1
		stats.TotalSeconds += rec.DurationSeconds

		if len(rec.TimeRfc3339) > 0 {
			dt, err := time.Parse(time.RFC3339, rec.TimeRfc3339)
			if err == nil {
				dt14 := timeutil.Dt14ForTime(dt)
				if stats.Dt14Start < 0 || dt14 < stats.Dt14Start {
					stats.Dt14Start = dt14
				}
				if stats.Dt14End < 0 || dt14 > stats.Dt14Start {
					stats.Dt14End = dt14
				}
			}
		}
	}
	stats.Inflate()
	return stats
}

type CallLogRecordCsv struct {
	Type                  string
	PhoneNumber           string
	PhoneNumberE164       string
	PhoneNumberIso31661a2 string
	Name                  string
	Date                  string
	Time                  string
	TimeRfc3339           string
	Action                string
	ActionResult          string
	ResultDescription     string
	Duration              string
	DurationSeconds       int64
}

func (r *CallLogRecordCsv) SetCountry() {}

func (r *CallLogRecordCsv) Inflate() {
	r.PhoneNumberIso31661a2 = ""
	if 1 == 0 && len(r.PhoneNumber) > 0 {
		country := "US"
		num, err := libphonenumber.Parse(r.PhoneNumber, country)
		if err == nil {
			r.PhoneNumberE164 = libphonenumber.Format(num, libphonenumber.E164)
			r.PhoneNumberIso31661a2 = country
		}
	}
	rx1 := regexp.MustCompile(`^([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$`)
	rs1 := rx1.FindStringSubmatch(r.Duration)
	if len(rs1) > 0 {
		hr, err := strconv.ParseInt(rs1[1], 10, 64)
		if err != nil {
			hr = 0
		}
		mn, err := strconv.ParseInt(rs1[2], 10, 64)
		if err != nil {
			mn = 0
		}
		sc, err := strconv.ParseInt(rs1[3], 10, 64)
		if err != nil {
			sc = 0
		}
		sc = sc + (mn * 60) + (hr * 60 * 60)
		r.DurationSeconds = sc
	}
	if len(r.Date) > 0 && len(r.Time) > 0 {
		dateTimeRaw := r.Date + " " + r.Time
		layout := "Mon 01/02/2006 3:04 PM"
		myTime, err := time.Parse(layout, dateTimeRaw)
		if err == nil {
			r.TimeRfc3339 = myTime.Format(time.RFC3339)
		}
	}
}

func (r *CallLogRecordCsv) LoadRow(cols []string, vals []string) error {
	lc := len(cols)
	lv := len(vals)
	if lc != lv {
		return errors.New("RC CSV COL ROW LENGTH MISMATCH")
	}
	rx1 := regexp.MustCompile(`\s+`)
	for i := 0; i < lc; i++ {
		col := cols[i]
		val := vals[i]
		key := rx1.ReplaceAllString(col, "")
		reflect.ValueOf(r).Elem().FieldByName(key).Set(reflect.ValueOf(val))
	}
	return nil
}

type CallLogRecordsCsvReader struct {
	CallLogRecordsCsv CallLogRecordsCsv
}

func NewCallLogRecordsCsvReader() CallLogRecordsCsvReader {
	rd := CallLogRecordsCsvReader{}
	return rd
}

func (rd *CallLogRecordsCsvReader) ReadFile(path string) error {
	reader, file, err := csvutil.NewReader(path, ',')
	if err != nil {
		return err
	}
	i := 0
	cols := []string{}
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		if i == 0 {
			cols = record
			i += 1
			continue
		}
		obj := CallLogRecordCsv{}
		_ = obj.LoadRow(cols, record)
		obj.Inflate()
		rd.CallLogRecordsCsv.CallLogRecords = append(rd.CallLogRecordsCsv.CallLogRecords, obj)
		i += 1
	}
	file.Close()
	return nil
}
<commit_msg>add BOM comment<commit_after>package rccsv

/*
	CSV reader for RingCentral Call Log CSV files

	SYNOPSIS

 	csv := rccsv.NewCallLogRecordsCsvReader()
	err := csv.ReadFile("/path/to/my.csv")
	if err != nil {
		fmt.Printf("ERR %v", err)
	}
	stats := csv.CallLogRecordsCsv.GetStatsForVoiceRecordings()

	// NOTE on stripping UTF-8 BOM
	http://cautery.blogspot.com/2013/04/stripping-utf-8-byte-order-mark-with-go.html
*/

import (
	"errors"
	"io"
	"reflect"
	"regexp"
	"strconv"
	"time"

	"github.com/grokify/gotilla/encoding/csvutil"
	"github.com/grokify/gotilla/time/timeutil"
	"github.com/ttacon/libphonenumber"
)

type CallLogStats struct {
	Days           float64
	NumCalls       int64
	TotalSeconds   int64
	TotalMinutes   float32
	TotalHours     float32
	AverageMinutes float32
	DailyCalls     float32
	DailyMinutes   float32
	DailyHours     float32
	Dt14Start      int64
	Dt14End        int64
}

func NewCallLogStats(days int64) CallLogStats {
	s := CallLogStats{}
	s.Days = -1
	s.NumCalls = 0
	s.TotalSeconds = 0
	s.TotalMinutes = 0.0
	s.TotalHours = 0.0
	s.AverageMinutes = 0.0
	s.DailyCalls = 0.0
	s.DailyMinutes = 0.0
	s.DailyHours = 0.0
	s.Dt14Start = -1
	s.Dt14End = -1
	return s
}

func (s *CallLogStats) Inflate() {
	if s.NumCalls < 1 {
		return
	}
	if s.Dt14Start > 0 && s.Dt14End > 0 && s.Dt14Start < s.Dt14End {
		dtStart, err1 := timeutil.TimeForDt14(s.Dt14Start)
		if err1 == nil {
			dtEnd, err2 := timeutil.TimeForDt14(s.Dt14End)
			if err2 == nil {
				dur := dtEnd.Sub(dtStart)
				durHrs := dur.Hours()
				days := durHrs / 24
				s.Days = days
			}
		}
	}
	s.TotalMinutes = float32(s.TotalSeconds / 60)
	s.TotalHours = float32(s.TotalSeconds / 60 / 60)
	s.AverageMinutes = s.TotalMinutes / float32(s.NumCalls)
	if s.Days > 0 {
		s.DailyCalls = float32(s.NumCalls) / float32(s.Days)
		s.DailyMinutes = s.DailyCalls * s.AverageMinutes
		s.DailyHours = s.DailyMinutes / 60
	}
}

type CallLogRecordsCsv struct {
	CallLogRecords []CallLogRecordCsv
}

func (rs *CallLogRecordsCsv) GetStatsForVoiceRecordings() CallLogStats {
	stats := NewCallLogStats(int64(30))
	stats.NumCalls = 0
	for _, rec := range rs.CallLogRecords {
		rec.Inflate()
		if rec.Action != "Phone Call" && rec.Action != "VoIP Call" && rec.Action != "FindMe" {
			//continue
		}
		if rec.ActionResult != "Accepted" && rec.ActionResult != "Call connected" {
			continue
		}
		stats.NumCalls += 1
		stats.TotalSeconds += rec.DurationSeconds

		if len(rec.TimeRfc3339) > 0 {
			dt, err := time.Parse(time.RFC3339, rec.TimeRfc3339)
			if err == nil {
				dt14 := timeutil.Dt14ForTime(dt)
				if stats.Dt14Start < 0 || dt14 < stats.Dt14Start {
					stats.Dt14Start = dt14
				}
				if stats.Dt14End < 0 || dt14 > stats.Dt14Start {
					stats.Dt14End = dt14
				}
			}
		}
	}
	stats.Inflate()
	return stats
}

type CallLogRecordCsv struct {
	Type                  string
	PhoneNumber           string
	PhoneNumberE164       string
	PhoneNumberIso31661a2 string
	Name                  string
	Date                  string
	Time                  string
	TimeRfc3339           string
	Action                string
	ActionResult          string
	ResultDescription     string
	Duration              string
	DurationSeconds       int64
}

func (r *CallLogRecordCsv) SetCountry() {}

func (r *CallLogRecordCsv) Inflate() {
	r.PhoneNumberIso31661a2 = ""
	if 1 == 0 && len(r.PhoneNumber) > 0 {
		country := "US"
		num, err := libphonenumber.Parse(r.PhoneNumber, country)
		if err == nil {
			r.PhoneNumberE164 = libphonenumber.Format(num, libphonenumber.E164)
			r.PhoneNumberIso31661a2 = country
		}
	}
	rx1 := regexp.MustCompile(`^([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$`)
	rs1 := rx1.FindStringSubmatch(r.Duration)
	if len(rs1) > 0 {
		hr, err := strconv.ParseInt(rs1[1], 10, 64)
		if err != nil {
			hr = 0
		}
		mn, err := strconv.ParseInt(rs1[2], 10, 64)
		if err != nil {
			mn = 0
		}
		sc, err := strconv.ParseInt(rs1[3], 10, 64)
		if err != nil {
			sc = 0
		}
		sc = sc + (mn * 60) + (hr * 60 * 60)
		r.DurationSeconds = sc
	}
	if len(r.Date) > 0 && len(r.Time) > 0 {
		dateTimeRaw := r.Date + " " + r.Time
		layout := "Mon 01/02/2006 3:04 PM"
		myTime, err := time.Parse(layout, dateTimeRaw)
		if err == nil {
			r.TimeRfc3339 = myTime.Format(time.RFC3339)
		}
	}
}

func (r *CallLogRecordCsv) LoadRow(cols []string, vals []string) error {
	lc := len(cols)
	lv := len(vals)
	if lc != lv {
		return errors.New("RC CSV COL ROW LENGTH MISMATCH")
	}
	rx1 := regexp.MustCompile(`\s+`)
	for i := 0; i < lc; i++ {
		col := cols[i]
		val := vals[i]
		key := rx1.ReplaceAllString(col, "")
		reflect.ValueOf(r).Elem().FieldByName(key).Set(reflect.ValueOf(val))
	}
	return nil
}

type CallLogRecordsCsvReader struct {
	CallLogRecordsCsv CallLogRecordsCsv
}

func NewCallLogRecordsCsvReader() CallLogRecordsCsvReader {
	rd := CallLogRecordsCsvReader{}
	return rd
}

func (rd *CallLogRecordsCsvReader) ReadFile(path string) error {
	reader, file, err := csvutil.NewReader(path, ',')
	if err != nil {
		return err
	}
	i := 0
	cols := []string{}
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		if i == 0 {
			cols = record
			i += 1
			continue
		}
		obj := CallLogRecordCsv{}
		_ = obj.LoadRow(cols, record)
		obj.Inflate()
		rd.CallLogRecordsCsv.CallLogRecords = append(rd.CallLogRecordsCsv.CallLogRecords, obj)
		i += 1
	}
	file.Close()
	return nil
}
<|endoftext|>


<commit_before>package reaper_test

import (
	"errors"
	"time"

	. "github.com/concourse/baggageclaim/reaper"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/baggageclaim/volume/fakes"
	"github.com/pivotal-golang/clock/fakeclock"
	"github.com/pivotal-golang/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Reaper", func() {
	var (
		repository *fakes.FakeRepository
		clock      *fakeclock.FakeClock

		reaper *Reaper
	)

	now := time.Unix(123, 456)

	BeforeEach(func() {
		repository = new(fakes.FakeRepository)
		clock = fakeclock.NewFakeClock(now)

		reaper = NewReaper(clock, repository)
	})

	Describe("Reap", func() {
		var reapErr error

		JustBeforeEach(func() {
			reapErr = reaper.Reap(lagertest.NewTestLogger("test"))
		})

		Context("when listing the volumes works", func() {
			nonExpiringVolume := volume.Volume{
				Handle: "non-expiring",
				TTL:    0,
			}

			expiringVolume10sec := volume.Volume{
				Handle:    "expiring-10sec",
				TTL:       10,
				ExpiresAt: now.Add(10 * time.Second),
			}

			expiringVolume20sec := volume.Volume{
				Handle:    "expiring-20sec",
				TTL:       20,
				ExpiresAt: now.Add(20 * time.Second),
			}

			BeforeEach(func() {
				repository.ListVolumesReturns([]volume.Volume{
					nonExpiringVolume,
					expiringVolume10sec,
					expiringVolume20sec,
				}, nil)
			})

			It("lists volumes with no filter", func() {
				Expect(repository.ListVolumesArgsForCall(0)).To(BeEmpty())
			})

			Context("when no volumes have expired", func() {
				It("does nothin'", func() {
					Expect(repository.DestroyVolumeCallCount()).To(BeZero())
				})
			})

			Context("when a volume has expired", func() {
				BeforeEach(func() {
					clock.Increment(10*time.Second + 1)
				})

				It("destroys it", func() {
					Expect(repository.DestroyVolumeCallCount()).To(Equal(1))

					handle := repository.DestroyVolumeArgsForCall(0)
					Expect(handle).To(Equal(expiringVolume10sec.Handle))
				})

				Context("when determining if a volume has a parent fails", func() {
					BeforeEach(func() {
						repository.VolumeParentReturns(volume.Volume{}, false, errors.New("nope"))
					})

					It("returns an error", func() {
						Expect(reapErr).To(MatchError("failed to determine volume parent: nope"))
					})

					It("does not destroy any volumes", func() {
						Expect(repository.DestroyVolumeCallCount()).To(BeZero())
					})
				})

				Context("when the expired volume has children", func() {
					BeforeEach(func() {
						repository.VolumeParentStub = func(handle string) (volume.Volume, bool, error) {
							switch handle {
							case nonExpiringVolume.Handle:
								return expiringVolume10sec, true, nil
							default:
								return volume.Volume{}, false, nil
							}
						}
					})

					It("is not destroyed", func() {
						Expect(repository.DestroyVolumeCallCount()).To(BeZero())
					})

					Context("regardless of the order in which the volumes are returned", func() {
						BeforeEach(func() {
							repository.ListVolumesReturns([]volume.Volume{
								expiringVolume20sec,
								expiringVolume10sec,
								nonExpiringVolume,
							}, nil)
						})

						It("is not destroyed", func() {
							Expect(repository.DestroyVolumeCallCount()).To(BeZero())
						})
					})
				})
			})

			Context("when multiple volumes have expired", func() {
				BeforeEach(func() {
					clock.Increment(20*time.Second + 1)
				})

				It("destroys the expired volumes", func() {
					Expect(repository.DestroyVolumeCallCount()).To(Equal(2))

					handle1 := repository.DestroyVolumeArgsForCall(0)
					Expect(handle1).To(Equal(expiringVolume10sec.Handle))

					handle2 := repository.DestroyVolumeArgsForCall(1)
					Expect(handle2).To(Equal(expiringVolume20sec.Handle))
				})

				Context("when destroying any volumes fails", func() {
					BeforeEach(func() {
						repository.DestroyVolumeStub = func(handle string) error {
							return errors.New("nope to " + handle)
						}
					})

					It("returns an aggregated error", func() {
						Expect(reapErr).To(HaveOccurred())
						Expect(reapErr.Error()).To(ContainSubstring("failed to destroy expiring-10sec: nope to expiring-10sec"))
						Expect(reapErr.Error()).To(ContainSubstring("failed to destroy expiring-20sec: nope to expiring-20sec"))
					})
				})
			})
		})

		Context("when listing the volumes blows up", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				repository.ListVolumesReturns(nil, disaster)
			})

			It("returns the error", func() {
				Expect(reapErr).To(MatchError("failed to list volumes: nope"))
			})

			It("does nothin'", func() {
				Expect(repository.DestroyVolumeCallCount()).To(BeZero())
			})
		})
	})
})
<commit_msg>woops, fix unit test<commit_after>package reaper_test

import (
	"errors"
	"time"

	. "github.com/concourse/baggageclaim/reaper"
	"github.com/concourse/baggageclaim/volume"
	"github.com/concourse/baggageclaim/volume/fakes"
	"github.com/pivotal-golang/clock/fakeclock"
	"github.com/pivotal-golang/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Reaper", func() {
	var (
		repository *fakes.FakeRepository
		clock      *fakeclock.FakeClock

		reaper *Reaper
	)

	now := time.Unix(123, 456)

	BeforeEach(func() {
		repository = new(fakes.FakeRepository)
		clock = fakeclock.NewFakeClock(now)

		reaper = NewReaper(clock, repository)
	})

	Describe("Reap", func() {
		var reapErr error

		JustBeforeEach(func() {
			reapErr = reaper.Reap(lagertest.NewTestLogger("test"))
		})

		Context("when listing the volumes works", func() {
			nonExpiringVolume := volume.Volume{
				Handle: "non-expiring",
				TTL:    0,
			}

			expiringVolume10sec := volume.Volume{
				Handle:    "expiring-10sec",
				TTL:       10,
				ExpiresAt: now.Add(10 * time.Second),
			}

			expiringVolume20sec := volume.Volume{
				Handle:    "expiring-20sec",
				TTL:       20,
				ExpiresAt: now.Add(20 * time.Second),
			}

			BeforeEach(func() {
				repository.ListVolumesReturns([]volume.Volume{
					nonExpiringVolume,
					expiringVolume10sec,
					expiringVolume20sec,
				}, nil)
			})

			It("lists volumes with no filter", func() {
				Expect(repository.ListVolumesArgsForCall(0)).To(BeEmpty())
			})

			Context("when no volumes have expired", func() {
				It("does nothin'", func() {
					Expect(repository.DestroyVolumeCallCount()).To(BeZero())
				})
			})

			Context("when a volume has expired", func() {
				BeforeEach(func() {
					clock.Increment(10*time.Second + 1)
				})

				It("destroys it", func() {
					Expect(repository.DestroyVolumeCallCount()).To(Equal(1))

					handle := repository.DestroyVolumeArgsForCall(0)
					Expect(handle).To(Equal(expiringVolume10sec.Handle))
				})

				Context("when determining if a volume has a parent fails", func() {
					BeforeEach(func() {
						repository.VolumeParentReturns(volume.Volume{}, false, errors.New("nope"))
					})

					It("returns an error", func() {
						Expect(reapErr).To(MatchError("failed to determine parent of volume 'non-expiring': nope"))
					})

					It("does not destroy any volumes", func() {
						Expect(repository.DestroyVolumeCallCount()).To(BeZero())
					})
				})

				Context("when the expired volume has children", func() {
					BeforeEach(func() {
						repository.VolumeParentStub = func(handle string) (volume.Volume, bool, error) {
							switch handle {
							case nonExpiringVolume.Handle:
								return expiringVolume10sec, true, nil
							default:
								return volume.Volume{}, false, nil
							}
						}
					})

					It("is not destroyed", func() {
						Expect(repository.DestroyVolumeCallCount()).To(BeZero())
					})

					Context("regardless of the order in which the volumes are returned", func() {
						BeforeEach(func() {
							repository.ListVolumesReturns([]volume.Volume{
								expiringVolume20sec,
								expiringVolume10sec,
								nonExpiringVolume,
							}, nil)
						})

						It("is not destroyed", func() {
							Expect(repository.DestroyVolumeCallCount()).To(BeZero())
						})
					})
				})
			})

			Context("when multiple volumes have expired", func() {
				BeforeEach(func() {
					clock.Increment(20*time.Second + 1)
				})

				It("destroys the expired volumes", func() {
					Expect(repository.DestroyVolumeCallCount()).To(Equal(2))

					handle1 := repository.DestroyVolumeArgsForCall(0)
					Expect(handle1).To(Equal(expiringVolume10sec.Handle))

					handle2 := repository.DestroyVolumeArgsForCall(1)
					Expect(handle2).To(Equal(expiringVolume20sec.Handle))
				})

				Context("when destroying any volumes fails", func() {
					BeforeEach(func() {
						repository.DestroyVolumeStub = func(handle string) error {
							return errors.New("nope to " + handle)
						}
					})

					It("returns an aggregated error", func() {
						Expect(reapErr).To(HaveOccurred())
						Expect(reapErr.Error()).To(ContainSubstring("failed to destroy expiring-10sec: nope to expiring-10sec"))
						Expect(reapErr.Error()).To(ContainSubstring("failed to destroy expiring-20sec: nope to expiring-20sec"))
					})
				})
			})
		})

		Context("when listing the volumes blows up", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				repository.ListVolumesReturns(nil, disaster)
			})

			It("returns the error", func() {
				Expect(reapErr).To(MatchError("failed to list volumes: nope"))
			})

			It("does nothin'", func() {
				Expect(repository.DestroyVolumeCallCount()).To(BeZero())
			})
		})
	})
})
<|endoftext|>