<commit_before>package ws

import (
	"bytes"
	"flag"
	"fmt"
	"math/rand"
	"net/textproto"
	"strings"
	"testing"
)

var compareWithStd = flag.Bool("std", false, "compare with standard library implementation (if exists)")

func TestHasToken(t *testing.T) {
	for i, test := range []struct {
		header string
		token  string
		exp    bool
	}{
		{"Keep-Alive, Close, Upgrade", "upgrade", true},
		{"Keep-Alive, Close, upgrade, hello", "upgrade", true},
		{"Keep-Alive, Close,  hello", "upgrade", false},
	} {
		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
			if has := strHasToken(test.header, test.token); has != test.exp {
				t.Errorf("hasToken(%q, %q) = %v; want %v", test.header, test.token, has, test.exp)
			}
		})
	}
}

func BenchmarkHasToken(b *testing.B) {
	for i, bench := range []struct {
		header string
		token  string
	}{
		{"Keep-Alive, Close, Upgrade", "upgrade"},
		{"Keep-Alive, Close, upgrade, hello", "upgrade"},
		{"Keep-Alive, Close,  hello", "upgrade"},
	} {
		b.Run(fmt.Sprintf("#%d", i), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				_ = strHasToken(bench.header, bench.token)
			}
		})
	}
}

type equalFoldCase struct {
	label string
	a, b  string
}

var equalFoldCases = []equalFoldCase{
	{"websocket", "WebSocket", "websocket"},
	{"upgrade", "Upgrade", "upgrade"},
	randomEqualLetters(512),
	inequalAt(randomEqualLetters(512), 256),
}

func TestStrEqualFold(t *testing.T) {
	for i, test := range equalFoldCases {
		t.Run(fmt.Sprintf("%s#%d", test.label, i), func(t *testing.T) {
			if len(test.a) < 100 && len(test.b) < 100 {
				t.Logf("\n\ta: %s\n\tb: %s\n", test.a, test.b)
			}
			exp := strings.EqualFold(test.a, test.b)
			if act := strEqualFold(test.a, test.b); act != exp {
				t.Errorf("strEqualFold(%q, %q) = %v; want %v", test.a, test.b, act, exp)
			}
		})
	}
}

func BenchmarkStrEqualFold(b *testing.B) {
	for i, bench := range equalFoldCases {
		b.Run(fmt.Sprintf("%s#%d", bench.label, i), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				_ = strEqualFold(bench.a, bench.b)
			}
		})
	}
	if *compareWithStd {
		for i, bench := range equalFoldCases {
			b.Run(fmt.Sprintf("%s#%d_std", bench.label, i), func(b *testing.B) {
				for i := 0; i < b.N; i++ {
					_ = strings.EqualFold(bench.a, bench.b)
				}
			})
		}
	}
}

func BenchmarkBtsEqualFold(b *testing.B) {
	for i, bench := range equalFoldCases {
		ab, bb := []byte(bench.a), []byte(bench.b)
		b.Run(fmt.Sprintf("%s#%d", bench.label, i), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				_ = btsEqualFold(ab, bb)
			}
		})
	}
	if *compareWithStd {
		for i, bench := range equalFoldCases {
			ab, bb := []byte(bench.a), []byte(bench.b)
			b.Run(fmt.Sprintf("%s#%d_std", bench.label, i), func(b *testing.B) {
				for i := 0; i < b.N; i++ {
					_ = bytes.EqualFold(ab, bb)
				}
			})
		}
	}
}

func TestAsciiToInt(t *testing.T) {
	for _, test := range []struct {
		bts []byte
		exp int
		err bool
	}{
		{[]byte{'0'}, 0, false},
		{[]byte{'1'}, 1, false},
		{[]byte("42"), 42, false},
		{[]byte("420"), 420, false},
		{[]byte("010050042"), 10050042, false},
	} {
		t.Run(fmt.Sprintf("%s", string(test.bts)), func(t *testing.T) {
			act, err := asciiToInt(test.bts)
			if (test.err && err == nil) || (!test.err && err != nil) {
				t.Errorf("unexpected error: %v", err)
			}
			if act != test.exp {
				t.Errorf("asciiToInt(%v) = %v; want %v", test.bts, act, test.exp)
			}
		})
	}
}

func TestBtrim(t *testing.T) {
	for _, test := range []struct {
		bts []byte
		exp []byte
	}{
		{[]byte("abc"), []byte("abc")},
		{[]byte(" abc"), []byte("abc")},
		{[]byte("abc "), []byte("abc")},
		{[]byte(" abc "), []byte("abc")},
	} {
		t.Run(fmt.Sprintf("%s", string(test.bts)), func(t *testing.T) {
			if act := btrim(test.bts); !bytes.Equal(act, test.exp) {
				t.Errorf("btrim(%v) = %v; want %v", test.bts, act, test.exp)
			}
		})
	}
}

func TestBSplit3(t *testing.T) {
	for _, test := range []struct {
		bts  []byte
		sep  byte
		exp1 []byte
		exp2 []byte
		exp3 []byte
	}{
		{[]byte(""), ' ', []byte{}, nil, nil},
		{[]byte("GET / HTTP/1.1"), ' ', []byte("GET"), []byte("/"), []byte("HTTP/1.1")},
	} {
		t.Run(fmt.Sprintf("%s", string(test.bts)), func(t *testing.T) {
			b1, b2, b3 := bsplit3(test.bts, test.sep)
			if !bytes.Equal(b1, test.exp1) || !bytes.Equal(b2, test.exp2) || !bytes.Equal(b3, test.exp3) {
				t.Errorf(
					"bsplit3(%q) = %q, %q, %q; want %q, %q, %q",
					string(test.bts), string(b1), string(b2), string(b3),
					string(test.exp1), string(test.exp2), string(test.exp3),
				)
			}
		})
	}
}

var canonicalHeaderCases = [][]byte{
	[]byte("foo-"),
	[]byte("-foo"),
	[]byte("-"),
	[]byte("foo----bar"),
	[]byte("foo-bar"),
	[]byte("FoO-BaR"),
	[]byte("Foo-Bar"),
	[]byte("sec-websocket-extensions"),
}

func TestCanonicalizeHeaderKey(t *testing.T) {
	for _, bts := range canonicalHeaderCases {
		t.Run(fmt.Sprintf("%s", string(bts)), func(t *testing.T) {
			act := append([]byte(nil), bts...)
			canonicalizeHeaderKey(act)

			exp := strToBytes(textproto.CanonicalMIMEHeaderKey(string(bts)))

			if !bytes.Equal(act, exp) {
				t.Errorf(
					"canonicalizeHeaderKey(%v) = %v; want %v",
					string(bts), string(act), string(exp),
				)
			}
		})
	}
}

func BenchmarkCanonicalizeHeaderKey(b *testing.B) {
	for _, bts := range canonicalHeaderCases {
		b.Run(fmt.Sprintf("%s", string(bts)), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				canonicalizeHeaderKey(bts)
			}
		})
	}
}

func randomEqualLetters(n int) (c equalFoldCase) {
	c.label = fmt.Sprintf("rnd_eq_%d", n)

	a, b := make([]byte, n), make([]byte, n)

	for i := 0; i < n; i++ {
		c := byte(rand.Intn('Z'-'A'+1) + 'A') // Random character from 'A' to 'Z'.
		a[i] = c
		b[i] = c | ('a' - 'A') // Swap fold.
	}

	c.a = string(a)
	c.b = string(b)

	return
}

func inequalAt(c equalFoldCase, i int) equalFoldCase {
	bts := make([]byte, len(c.a))
	copy(bts, c.a)
	for {
		b := byte(rand.Intn('z'-'a'+1) + 'a')
		if bts[i] != b {
			bts[i] = b
			c.a = string(bts)
			c.label = fmt.Sprintf("rnd_ineq_%d_%d", len(c.a), i)
			return c
		}
	}
}
<commit_msg>readLine fix, test<commit_after>package ws

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"math/rand"
	"net/textproto"
	"strings"
	"testing"
)

var compareWithStd = flag.Bool("std", false, "compare with standard library implementation (if exists)")

func TestReadLine(t *testing.T) {
	for _, test := range []struct {
		label   string
		in      string
		bufSize int
	}{
		{
			label:   "simple",
			in:      "hello, world!",
			bufSize: 1024,
		},
	} {
		t.Run(test.label, func(t *testing.T) {
			br := bufio.NewReaderSize(strings.NewReader(test.in), test.bufSize)
			bts, err := readLine(br)
			if err != nil {
				t.Errorf("unexpected error: %s", err)
			}
			if act, exp := string(bts), test.in; act != exp {
				t.Errorf("readLine() returned %#q; want %#q", act, exp)
			}
		})
	}
}

func TestHasToken(t *testing.T) {
	for i, test := range []struct {
		header string
		token  string
		exp    bool
	}{
		{"Keep-Alive, Close, Upgrade", "upgrade", true},
		{"Keep-Alive, Close, upgrade, hello", "upgrade", true},
		{"Keep-Alive, Close,  hello", "upgrade", false},
	} {
		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
			if has := strHasToken(test.header, test.token); has != test.exp {
				t.Errorf("hasToken(%q, %q) = %v; want %v", test.header, test.token, has, test.exp)
			}
		})
	}
}

func BenchmarkHasToken(b *testing.B) {
	for i, bench := range []struct {
		header string
		token  string
	}{
		{"Keep-Alive, Close, Upgrade", "upgrade"},
		{"Keep-Alive, Close, upgrade, hello", "upgrade"},
		{"Keep-Alive, Close,  hello", "upgrade"},
	} {
		b.Run(fmt.Sprintf("#%d", i), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				_ = strHasToken(bench.header, bench.token)
			}
		})
	}
}

type equalFoldCase struct {
	label string
	a, b  string
}

var equalFoldCases = []equalFoldCase{
	{"websocket", "WebSocket", "websocket"},
	{"upgrade", "Upgrade", "upgrade"},
	randomEqualLetters(512),
	inequalAt(randomEqualLetters(512), 256),
}

func TestStrEqualFold(t *testing.T) {
	for i, test := range equalFoldCases {
		t.Run(fmt.Sprintf("%s#%d", test.label, i), func(t *testing.T) {
			if len(test.a) < 100 && len(test.b) < 100 {
				t.Logf("\n\ta: %s\n\tb: %s\n", test.a, test.b)
			}
			exp := strings.EqualFold(test.a, test.b)
			if act := strEqualFold(test.a, test.b); act != exp {
				t.Errorf("strEqualFold(%q, %q) = %v; want %v", test.a, test.b, act, exp)
			}
		})
	}
}

func BenchmarkStrEqualFold(b *testing.B) {
	for i, bench := range equalFoldCases {
		b.Run(fmt.Sprintf("%s#%d", bench.label, i), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				_ = strEqualFold(bench.a, bench.b)
			}
		})
	}
	if *compareWithStd {
		for i, bench := range equalFoldCases {
			b.Run(fmt.Sprintf("%s#%d_std", bench.label, i), func(b *testing.B) {
				for i := 0; i < b.N; i++ {
					_ = strings.EqualFold(bench.a, bench.b)
				}
			})
		}
	}
}

func BenchmarkBtsEqualFold(b *testing.B) {
	for i, bench := range equalFoldCases {
		ab, bb := []byte(bench.a), []byte(bench.b)
		b.Run(fmt.Sprintf("%s#%d", bench.label, i), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				_ = btsEqualFold(ab, bb)
			}
		})
	}
	if *compareWithStd {
		for i, bench := range equalFoldCases {
			ab, bb := []byte(bench.a), []byte(bench.b)
			b.Run(fmt.Sprintf("%s#%d_std", bench.label, i), func(b *testing.B) {
				for i := 0; i < b.N; i++ {
					_ = bytes.EqualFold(ab, bb)
				}
			})
		}
	}
}

func TestAsciiToInt(t *testing.T) {
	for _, test := range []struct {
		bts []byte
		exp int
		err bool
	}{
		{[]byte{'0'}, 0, false},
		{[]byte{'1'}, 1, false},
		{[]byte("42"), 42, false},
		{[]byte("420"), 420, false},
		{[]byte("010050042"), 10050042, false},
	} {
		t.Run(fmt.Sprintf("%s", string(test.bts)), func(t *testing.T) {
			act, err := asciiToInt(test.bts)
			if (test.err && err == nil) || (!test.err && err != nil) {
				t.Errorf("unexpected error: %v", err)
			}
			if act != test.exp {
				t.Errorf("asciiToInt(%v) = %v; want %v", test.bts, act, test.exp)
			}
		})
	}
}

func TestBtrim(t *testing.T) {
	for _, test := range []struct {
		bts []byte
		exp []byte
	}{
		{[]byte("abc"), []byte("abc")},
		{[]byte(" abc"), []byte("abc")},
		{[]byte("abc "), []byte("abc")},
		{[]byte(" abc "), []byte("abc")},
	} {
		t.Run(fmt.Sprintf("%s", string(test.bts)), func(t *testing.T) {
			if act := btrim(test.bts); !bytes.Equal(act, test.exp) {
				t.Errorf("btrim(%v) = %v; want %v", test.bts, act, test.exp)
			}
		})
	}
}

func TestBSplit3(t *testing.T) {
	for _, test := range []struct {
		bts  []byte
		sep  byte
		exp1 []byte
		exp2 []byte
		exp3 []byte
	}{
		{[]byte(""), ' ', []byte{}, nil, nil},
		{[]byte("GET / HTTP/1.1"), ' ', []byte("GET"), []byte("/"), []byte("HTTP/1.1")},
	} {
		t.Run(fmt.Sprintf("%s", string(test.bts)), func(t *testing.T) {
			b1, b2, b3 := bsplit3(test.bts, test.sep)
			if !bytes.Equal(b1, test.exp1) || !bytes.Equal(b2, test.exp2) || !bytes.Equal(b3, test.exp3) {
				t.Errorf(
					"bsplit3(%q) = %q, %q, %q; want %q, %q, %q",
					string(test.bts), string(b1), string(b2), string(b3),
					string(test.exp1), string(test.exp2), string(test.exp3),
				)
			}
		})
	}
}

var canonicalHeaderCases = [][]byte{
	[]byte("foo-"),
	[]byte("-foo"),
	[]byte("-"),
	[]byte("foo----bar"),
	[]byte("foo-bar"),
	[]byte("FoO-BaR"),
	[]byte("Foo-Bar"),
	[]byte("sec-websocket-extensions"),
}

func TestCanonicalizeHeaderKey(t *testing.T) {
	for _, bts := range canonicalHeaderCases {
		t.Run(fmt.Sprintf("%s", string(bts)), func(t *testing.T) {
			act := append([]byte(nil), bts...)
			canonicalizeHeaderKey(act)

			exp := strToBytes(textproto.CanonicalMIMEHeaderKey(string(bts)))

			if !bytes.Equal(act, exp) {
				t.Errorf(
					"canonicalizeHeaderKey(%v) = %v; want %v",
					string(bts), string(act), string(exp),
				)
			}
		})
	}
}

func BenchmarkCanonicalizeHeaderKey(b *testing.B) {
	for _, bts := range canonicalHeaderCases {
		b.Run(fmt.Sprintf("%s", string(bts)), func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				canonicalizeHeaderKey(bts)
			}
		})
	}
}

func randomEqualLetters(n int) (c equalFoldCase) {
	c.label = fmt.Sprintf("rnd_eq_%d", n)

	a, b := make([]byte, n), make([]byte, n)

	for i := 0; i < n; i++ {
		c := byte(rand.Intn('Z'-'A'+1) + 'A') // Random character from 'A' to 'Z'.
		a[i] = c
		b[i] = c | ('a' - 'A') // Swap fold.
	}

	c.a = string(a)
	c.b = string(b)

	return
}

func inequalAt(c equalFoldCase, i int) equalFoldCase {
	bts := make([]byte, len(c.a))
	copy(bts, c.a)
	for {
		b := byte(rand.Intn('z'-'a'+1) + 'a')
		if bts[i] != b {
			bts[i] = b
			c.a = string(bts)
			c.label = fmt.Sprintf("rnd_ineq_%d_%d", len(c.a), i)
			return c
		}
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"log"
	"os"
	"os/exec"

	"github.com/urfave/cli"
)

// Run ...
func Run(cmd string) ([]byte, error) {
	command := exec.Command("sh", "-c", cmd)
	return command.Output()
}

// RunInteractive ...
func RunInteractive(cmd string) error {
	if os.Getenv("DEBUG") != "" {
		log.Println(cmd)
	}
	command := exec.Command("sh", "-c", cmd)
	command.Stdout = os.Stdout
	command.Stdin = os.Stdin
	command.Stderr = os.Stderr
	// command.Dir = cwd
	err := command.Run()
	if err != nil {
		return cli.NewExitError(err, 1)
	}
	return nil
}
<commit_msg>Fix utils.Run to return ExitError<commit_after>package utils

import (
	"log"
	"os"
	"os/exec"

	"github.com/urfave/cli"
)

// Run ...
func Run(cmd string) ([]byte, error) {
	command := exec.Command("sh", "-c", cmd)
	data, err := command.Output()
	if err != nil {
		return data, cli.NewExitError(err, 1)
	}
	return data, err
}

// RunInteractive ...
func RunInteractive(cmd string) error {
	if os.Getenv("DEBUG") != "" {
		log.Println(cmd)
	}
	command := exec.Command("sh", "-c", cmd)
	command.Stdout = os.Stdout
	command.Stdin = os.Stdin
	command.Stderr = os.Stderr
	// command.Dir = cwd
	err := command.Run()
	if err != nil {
		return cli.NewExitError(err, 1)
	}
	return nil
}
<|endoftext|>


<commit_before>package slogger

import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"time"
)

type Log struct {
	Prefix     string
	Level      Level
	Filename   string
	Line       int
	Timestamp  time.Time
	messageFmt string
	args       []interface{}
}

func (self *Log) Message() string {
	return fmt.Sprintf(self.messageFmt, self.args...)
}

type Logger struct {
	Prefix    string
	Appenders []Appender
}

// Log a message and a level to a logger instance. This returns a
// pointer to a Log and a slice of errors that were gathered from every
// Appender (nil errors included).
func (self *Logger) Logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	return self.logf(level, messageFmt, args...)
}

// Log and return a formatted error string.
// Example:
//
// if whatIsExpected != whatIsReturned {
//     return slogger.Errorf(slogger.WARN, "Unexpected return value. Expected: %v Received: %v",
//         whatIsExpected, whatIsReturned)
// }
//
func (self *Logger) Errorf(level Level, messageFmt string, args ...interface{}) error {
	log, _ := self.logf(level, messageFmt, args...)
	return errors.New(log.Message())
}

// Stackf is designed to work in tandem with `NewStackError`. This
// function is similar to `Logf`, but takes a `stackErr`
// parameter. `stackErr` is expected to be of type StackError, but does
// not have to be.
func (self *Logger) Stackf(level Level, stackErr error, messageFmt string, args ...interface{}) (*Log, []error) {
	messageFmt = fmt.Sprintf("%v\n%v", messageFmt, stackErr.Error())
	return self.logf(level, messageFmt, args...)
}

func (self *Logger) logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	var errors []error

	_, file, line, ok := runtime.Caller(2)
	if ok == false {
		return nil, []error{fmt.Errorf("Failed to find the calling method.")}
	}

	file = stripDirectories(file, 2)

	log := &Log{
		Prefix:     self.Prefix,
		Level:      level,
		Filename:   file,
		Line:       line,
		Timestamp:  time.Now(),
		messageFmt: messageFmt,
		args:       args,
	}

	Cache.Add(log)

	for _, appender := range self.Appenders {
		if err := appender.Append(log); err != nil {
			error := fmt.Errorf("Error appending. Appender: %T Error: %v", appender, err)
			errors = append(errors, error)
		}
	}

	return log, errors
}

type Level uint8

// The level is in an order such that the expressions
// `level < WARN`, `level >= INFO` have intuitive meaning.
const (
	OFF Level = iota
	DEBUG
	INFO
	WARN
	ERROR
)

func (self Level) Type() string {
	switch self {
	case ERROR:
		return "error"
	case WARN:
		return "warn"
	case INFO:
		return "info"
	case DEBUG:
		return "debug"
	}

	return "off?"
}

func stacktrace() []string {
	ret := make([]string, 0, 2)
	for skip := 2; true; skip++ {
		_, file, line, ok := runtime.Caller(skip)
		if ok == false {
			break
		}

		ret = append(ret, fmt.Sprintf("at %s:%d", stripDirectories(file, 2), line))
	}

	return ret
}

type StackError struct {
	Message    string
	Stacktrace []string
}

func NewStackError(messageFmt string, args ...interface{}) *StackError {
	return &StackError{
		Message:    fmt.Sprintf(messageFmt, args...),
		Stacktrace: stacktrace(),
	}
}

func (self *StackError) Error() string {
	return fmt.Sprintf("%s\n\t%s", self.Message, strings.Join(self.Stacktrace, "\n\t"))
}

func stripDirectories(filepath string, toKeep int) string {
	var idxCutoff int
	if idxCutoff = strings.LastIndex(filepath, "/"); idxCutoff == -1 {
		return filepath
	}

	for dirToKeep := 0; dirToKeep < toKeep; dirToKeep++ {
		switch idx := strings.LastIndex(filepath[:idxCutoff], "/"); idx {
		case -1:
			break
		default:
			idxCutoff = idx
		}
	}

	return filepath[idxCutoff+1:]
}
<commit_msg>Apply Bard's changes.<commit_after>package slogger

import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"time"
)

type Log struct {
	Prefix     string
	Level      Level
	Filename   string
	Line       int
	Timestamp  time.Time
	messageFmt string
	args       []interface{}
}

func (self *Log) Message() string {
	return fmt.Sprintf(self.messageFmt, self.args...)
}

type Logger struct {
	Prefix    string
	Appenders []Appender
	StripDirs int
}

// Log a message and a level to a logger instance. This returns a
// pointer to a Log and a slice of errors that were gathered from every
// Appender (nil errors included).
func (self *Logger) Logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	return self.logf(level, messageFmt, args...)
}

// Log and return a formatted error string.
// Example:
//
// if whatIsExpected != whatIsReturned {
//     return slogger.Errorf(slogger.WARN, "Unexpected return value. Expected: %v Received: %v",
//         whatIsExpected, whatIsReturned)
// }
//
func (self *Logger) Errorf(level Level, messageFmt string, args ...interface{}) error {
	log, _ := self.logf(level, messageFmt, args...)
	return errors.New(log.Message())
}

// Stackf is designed to work in tandem with `NewStackError`. This
// function is similar to `Logf`, but takes a `stackErr`
// parameter. `stackErr` is expected to be of type StackError, but does
// not have to be.
func (self *Logger) Stackf(level Level, stackErr error, messageFmt string, args ...interface{}) (*Log, []error) {
	messageFmt = fmt.Sprintf("%v\n%v", messageFmt, stackErr.Error())
	return self.logf(level, messageFmt, args...)
}

var ignoredFileNames = []string{"logger.go"}

// Add a file to the list of file names that slogger will skip when it identifies the source
// of a message.  This is useful if you have a logging library built on top of slogger.
// If you IgnoreThisFilenameToo(...) on the files of that library, logging messages
// will be marked as coming from your code that calls your library, rather than from your library.
func IgnoreThisFilenameToo(fn string) {
	ignoredFileNames = append(ignoredFileNames, fn)
}

func containsAnyIgnoredFilename(s string) bool {
	for _, ign := range ignoredFileNames {
		if strings.Contains(s, ign)  {
			return true
		}
	}
	return false
}

func nonSloggerCaller() (pc uintptr, file string, line int, ok bool) {
	for skip := 0; skip < 100; skip++ {
		a,b,c,ok := runtime.Caller(skip)
		shouldBeIgnored := containsAnyIgnoredFilename(b)
		if !ok || !shouldBeIgnored  {
			return a,b,c,ok
		}
	}
	return 0, "", 0, false
}

func (self *Logger) logf(level Level, messageFmt string, args ...interface{}) (*Log, []error) {
	var errors []error

	_, file, line, ok := nonSloggerCaller()
//	_, file, line, ok := runtime.Caller(2+offset)
	if ok == false {
		return nil, []error{fmt.Errorf("Failed to find the calling method.")}
	}

	file = stripDirectories(file, self.StripDirs)

	log := &Log{
		Prefix:     self.Prefix,
		Level:      level,
		Filename:   file,
		Line:       line,
		Timestamp:  time.Now(),
		messageFmt: messageFmt,
		args:       args,
	}

	Cache.Add(log)

	for _, appender := range self.Appenders {
		if err := appender.Append(log); err != nil {
			error := fmt.Errorf("Error appending. Appender: %T Error: %v", appender, err)
			errors = append(errors, error)
		}
	}

	return log, errors
}

type Level uint8

// The level is in an order such that the expressions
// `level < WARN`, `level >= INFO` have intuitive meaning.
const (
	OFF Level = iota
	DEBUG
	ROUTINE
	INFO
	WARN
	ERROR
	DOOM
)

func (self Level) Type() string {
	switch self {
	case ERROR:
		return "error"
	case WARN:
		return "warn"
	case INFO:
		return "info"
	case ROUTINE:
		return "routine"
	case DEBUG:
		return "debug"
	case DOOM:
		return "doom"
	}

	return "off?"
}

func stacktrace() []string {
	ret := make([]string, 0, 2)
	for skip := 2; true; skip++ {
		_, file, line, ok := runtime.Caller(skip)
		if ok == false {
			break
		}

		ret = append(ret, fmt.Sprintf("at %s:%d", stripDirectories(file, 2), line))
	}

	return ret
}

type StackError struct {
	Message    string
	Stacktrace []string
}

func NewStackError(messageFmt string, args ...interface{}) *StackError {
	return &StackError{
		Message:    fmt.Sprintf(messageFmt, args...),
		Stacktrace: stacktrace(),
	}
}

func (self *StackError) Error() string {
	return fmt.Sprintf("%s\n\t%s", self.Message, strings.Join(self.Stacktrace, "\n\t"))
}

func stripDirectories(filepath string, toKeep int) string {
	var idxCutoff int
	if idxCutoff = strings.LastIndex(filepath, "/"); idxCutoff == -1 {
		return filepath
	}

	for dirToKeep := 0; dirToKeep < toKeep; dirToKeep++ {
		switch idx := strings.LastIndex(filepath[:idxCutoff], "/"); idx {
		case -1:
			break
		default:
			idxCutoff = idx
		}
	}

	return filepath[idxCutoff+1:]
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package utils

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/loggo/loggo"

	"launchpad.net/juju-core/juju/osenv"
)

var (
	aptLogger  = loggo.GetLogger("juju.utils.apt")
	aptProxyRE = regexp.MustCompile(`(?im)^\s*Acquire::(?P<protocol>[a-z]+)::Proxy\s+"(?P<proxy>[^"]+)";\s*$`)

	// AptConfFile is the full file path for the proxy settings that are
	// written by cloud-init and the machine environ worker.
	AptConfFile = "/etc/apt/apt.conf.d/42-juju-proxy-settings"
)

// Some helpful functions for running apt in a sane way

// AptCommandOutput calls cmd.Output, this is used as an overloading point so we
// can test what *would* be run without actually executing another program
var AptCommandOutput = (*exec.Cmd).CombinedOutput

// This is the default apt-get command used in cloud-init, the various settings
// mean that apt won't actually block waiting for a prompt from the user.
var aptGetCommand = []string{
	"apt-get", "--option=Dpkg::Options::=--force-confold",
	"--option=Dpkg::options::=--force-unsafe-io", "--assume-yes", "--quiet",
}

// aptEnvOptions are options we need to pass to apt-get to not have it prompt
// the user
var aptGetEnvOptions = []string{"DEBIAN_FRONTEND=noninteractive"}

// AptGetInstall runs 'apt-get install packages' for the packages listed here
func AptGetInstall(packages ...string) error {
	cmdArgs := append([]string(nil), aptGetCommand...)
	cmdArgs = append(cmdArgs, "install")
	cmdArgs = append(cmdArgs, packages...)
	aptLogger.Infof("Running: %s", cmdArgs)
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Env = append(os.Environ(), aptGetEnvOptions...)
	out, err := AptCommandOutput(cmd)
	if err != nil {
		aptLogger.Errorf("apt-get command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return fmt.Errorf("apt-get failed: %v", err)
	}
	return nil
}

// AptConfigProxy will consult apt-config about the configured proxy
// settings. If there are no proxy settings configured, an empty string is
// returned.
func AptConfigProxy() (string, error) {
	cmdArgs := []string{
		"apt-config",
		"dump",
		"Acquire::http::Proxy",
		"Acquire::https::Proxy",
		"Acquire::ftp::Proxy",
	}
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	out, err := AptCommandOutput(cmd)
	if err != nil {
		aptLogger.Errorf("apt-config command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return "", fmt.Errorf("apt-config failed: %v", err)
	}
	return string(bytes.Join(aptProxyRE.FindAll(out, -1), []byte("\n"))), nil
}

// DetectAptProxies will parse the results of AptConfigProxy to return a
// ProxySettings instance.
func DetectAptProxies() (result osenv.ProxySettings, err error) {
	output, err := AptConfigProxy()
	if err != nil {
		return result, err
	}
	for _, match := range aptProxyRE.FindAllStringSubmatch(output, -1) {
		switch match[1] {
		case "http":
			result.Http = match[2]
		case "https":
			result.Https = match[2]
		case "ftp":
			result.Ftp = match[2]
		}
	}
	return result, nil
}

// AptProxyContent produces the format expected by the apt config files
// from the ProxySettings struct.
func AptProxyContent(proxy osenv.ProxySettings) string {
	lines := []string{}
	addLine := func(proxy, value string) {
		if value != "" {
			lines = append(lines, fmt.Sprintf(
				"Acquire::%s::Proxy %q;", proxy, value))
		}
	}
	addLine("http", proxy.Http)
	addLine("https", proxy.Https)
	addLine("ftp", proxy.Ftp)
	return strings.Join(lines, "\n")
}

// IsUbuntu executes lxb_release to see if the host OS is Ubuntu.
func IsUbuntu() bool {
	out, err := RunCommand("lsb_release", "-i", "-s")
	if err != nil {
		return false
	}
	return strings.TrimSpace(out) == "Ubuntu"
}

// UbuntuRelease executes lsb_release to see if the host OS is Ubuntu.
func UbuntuRelease() (float, error) {
	if !IsUbuntu() {
		return 0, errors.New("not ubuntu")
	}
	out, err := RunCommand("lsb_release", "-r", "-s")
	if err != nil {
		return 0, err
	}
	release, err := strconv.ParseFloat(out, 10)
	if err != nil {
		return 0, err
	}
	return release, nil
}

// IsPackageInstalled uses dpkg-query to determine if the `packageName`
// package is installed.
func IsPackageInstalled(packageName string) bool {
	_, err := RunCommand("dpkg-query", "--status", packageName)
	return err == nil
}
<commit_msg>tests and cleanup<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package utils

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/loggo/loggo"

	"launchpad.net/juju-core/juju/osenv"
)

var (
	aptLogger  = loggo.GetLogger("juju.utils.apt")
	aptProxyRE = regexp.MustCompile(`(?im)^\s*Acquire::(?P<protocol>[a-z]+)::Proxy\s+"(?P<proxy>[^"]+)";\s*$`)

	// AptConfFile is the full file path for the proxy settings that are
	// written by cloud-init and the machine environ worker.
	AptConfFile = "/etc/apt/apt.conf.d/42-juju-proxy-settings"
)

// Some helpful functions for running apt in a sane way

// AptCommandOutput calls cmd.Output, this is used as an overloading point so we
// can test what *would* be run without actually executing another program
var AptCommandOutput = (*exec.Cmd).CombinedOutput

// This is the default apt-get command used in cloud-init, the various settings
// mean that apt won't actually block waiting for a prompt from the user.
var aptGetCommand = []string{
	"apt-get", "--option=Dpkg::Options::=--force-confold",
	"--option=Dpkg::options::=--force-unsafe-io", "--assume-yes", "--quiet",
}

// aptEnvOptions are options we need to pass to apt-get to not have it prompt
// the user
var aptGetEnvOptions = []string{"DEBIAN_FRONTEND=noninteractive"}

// AptGetInstall runs 'apt-get install packages' for the packages listed here
func AptGetInstall(packages ...string) error {
	cmdArgs := append([]string(nil), aptGetCommand...)
	cmdArgs = append(cmdArgs, "install")
	cmdArgs = append(cmdArgs, packages...)
	aptLogger.Infof("Running: %s", cmdArgs)
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.Env = append(os.Environ(), aptGetEnvOptions...)
	out, err := AptCommandOutput(cmd)
	if err != nil {
		aptLogger.Errorf("apt-get command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return fmt.Errorf("apt-get failed: %v", err)
	}
	return nil
}

// AptConfigProxy will consult apt-config about the configured proxy
// settings. If there are no proxy settings configured, an empty string is
// returned.
func AptConfigProxy() (string, error) {
	cmdArgs := []string{
		"apt-config",
		"dump",
		"Acquire::http::Proxy",
		"Acquire::https::Proxy",
		"Acquire::ftp::Proxy",
	}
	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	out, err := AptCommandOutput(cmd)
	if err != nil {
		aptLogger.Errorf("apt-config command failed: %v\nargs: %#v\n%s",
			err, cmdArgs, string(out))
		return "", fmt.Errorf("apt-config failed: %v", err)
	}
	return string(bytes.Join(aptProxyRE.FindAll(out, -1), []byte("\n"))), nil
}

// DetectAptProxies will parse the results of AptConfigProxy to return a
// ProxySettings instance.
func DetectAptProxies() (result osenv.ProxySettings, err error) {
	output, err := AptConfigProxy()
	if err != nil {
		return result, err
	}
	for _, match := range aptProxyRE.FindAllStringSubmatch(output, -1) {
		switch match[1] {
		case "http":
			result.Http = match[2]
		case "https":
			result.Https = match[2]
		case "ftp":
			result.Ftp = match[2]
		}
	}
	return result, nil
}

// AptProxyContent produces the format expected by the apt config files
// from the ProxySettings struct.
func AptProxyContent(proxy osenv.ProxySettings) string {
	lines := []string{}
	addLine := func(proxy, value string) {
		if value != "" {
			lines = append(lines, fmt.Sprintf(
				"Acquire::%s::Proxy %q;", proxy, value))
		}
	}
	addLine("http", proxy.Http)
	addLine("https", proxy.Https)
	addLine("ftp", proxy.Ftp)
	return strings.Join(lines, "\n")
}

// IsUbuntu executes lxb_release to see if the host OS is Ubuntu.
func IsUbuntu() bool {
	out, err := RunCommand("lsb_release", "-i", "-s")
	if err != nil {
		return false
	}
	return strings.TrimSpace(out) == "Ubuntu"
}

// IsPackageInstalled uses dpkg-query to determine if the `packageName`
// package is installed.
func IsPackageInstalled(packageName string) bool {
	_, err := RunCommand("dpkg-query", "--status", packageName)
	return err == nil
}
<|endoftext|>


<commit_before>package ngram

import (
	"errors"
	"github.com/reusee/mmh3"
	"math"
)

const maxN = 8

const defaultPad = "$"

const defaultN = 3

type TokenId int

type nGramValue struct {
	items map[TokenId]int
	count int64
}

// N-gram index, can be initialized by default (zeroed) or created with "NewNgramIndex"
type NGramIndex struct {
	pad   string
	n     int
	spool stringPool
	index map[uint32]nGramValue
	warp  float64
}

// Search result, contains token id and similarity - value in range from 0.0 to 1.0
type SearchResult struct {
	TokenId    TokenId
	Similarity float64
}

func (ngram *NGramIndex) split_input(str string) ([]uint32, error) {
	if len(str) == 0 {
		return nil, errors.New("Empty string")
	}
	pad := ngram.pad
	n := ngram.n
	input := pad + str + pad
	prev_indexes := make([]int, maxN)
	counter := 0
	results := make([]uint32, 0)

	for index, _ := range input {
		counter++
		if counter > n {
			top := prev_indexes[(counter-n)%len(prev_indexes)]
			substr := input[top:index]
			hash := mmh3.Hash32([]byte(substr))
			results = append(results, hash)
		}
		prev_indexes[counter%len(prev_indexes)] = index
	}

	for i := n - 1; i > 1; i-- {
		if len(input) >= i {
			top := prev_indexes[(len(input)-i)%len(prev_indexes)]
			substr := input[top:]
			hash := mmh3.Hash32([]byte(substr))
			results = append(results, hash)
		}
	}

	return results, nil
}

func (ngram *NGramIndex) init() {
	ngram.index = make(map[uint32]nGramValue)
	if ngram.pad == "" {
		ngram.pad = defaultPad
	}
	if ngram.n == 0 {
		ngram.n = defaultN
	}
	if ngram.warp == 0.0 {
		ngram.warp = 1.0
	}
}

type padArgTrait struct {
	pad rune
}

type nArgTrait struct {
	n int
}

type warpArgTrait struct {
	warp float64
}

// This function must be used to pass padding character to NGramIndex c-tor
func SetPad(c rune) padArgTrait {
	return padArgTrait{pad: c}
}

// This function must be used to pass N (gram size) to NGramIndex c-tor
func SetN(n int) nArgTrait {
	return nArgTrait{n: n}
}

// This function must be used to pass warp to NGramIndex c-tor
func SetWarp(warp float64) warpArgTrait {
	return warpArgTrait{warp: warp}
}

// N-gram index c-tor. In most cases must be used withot parameters.
// You can pass parameters to c-tor using functions SetPad and SetN.
func NewNGramIndex(args ...interface{}) (*NGramIndex, error) {
	ngram := new(NGramIndex)
	for _, arg := range args {
		switch i := arg.(type) {
		case padArgTrait:
			ngram.pad = string(i.pad)
		case nArgTrait:
			if i.n < 2 || i.n > maxN {
				return nil, errors.New("Bad 'n' value for n-gram index")
			}
			ngram.n = i.n
		case warpArgTrait:
			if i.warp < 0.0 || i.warp > 1.0 {
				return nil, errors.New("Bad 'warp' value for n-gram index")
			}
		default:
			return nil, errors.New("Invalid argument")
		}
	}
	ngram.init()
	return ngram, nil
}

// Add token to index. Function returns token id, this id can be converted
// to string with function "GetString".
func (ngram *NGramIndex) Add(input string) (TokenId, error) {
	if ngram.index == nil {
		ngram.init()
	}
	results, error := ngram.split_input(input)
	if error != nil {
		return -1, error
	}
	ixstr, error := ngram.spool.Append(input)
	if error != nil {
		return -1, error
	}
	for _, hash := range results {
		var val nGramValue
		var ok bool
		if val, ok = ngram.index[hash]; !ok {
			ngram.index[hash] = nGramValue{items: make(map[TokenId]int)}
		}
		val.count++
		// insert string and counter
        ngram.index[hash].items[ixstr]++
	}
	return ixstr, nil
}

// Converts token-id to string.
func (ngram *NGramIndex) GetString(id TokenId) (string, error) {
	return ngram.spool.ReadAt(id)
}

// Map matched tokens to the number of ngrams, shared with input string
func (ngram *NGramIndex) count_ngrams(input_ngrams []uint32) map[TokenId]int {
	counters := make(map[TokenId]int)
	for _, ngram_hash := range input_ngrams {
		if tokmap, ok := ngram.index[ngram_hash]; ok {
			for tok, _ := range tokmap.items {
				counters[tok] += 1
			}
		}
	}
	return counters
}

func validate_threshold_values(thresholds []float64) (float64, error) {
	var tval float64
	if len(thresholds) == 1 {
		tval = thresholds[0]
		if tval < 0.0 || tval > 1.0 {
			return 0.0, errors.New("Threshold must be in range (0, 1)")
		}
	} else if len(thresholds) > 1 {
		return 0.0, errors.New("Too many arguments")
	}
	return tval, nil
}

func (ngram *NGramIndex) match(input string, tval float64) ([]SearchResult, error) {
	input_ngrams, error := ngram.split_input(input)
	if error != nil {
		return nil, error
	}
	output := make([]SearchResult, 0)
	token_count := ngram.count_ngrams(input_ngrams)
	for token, count := range token_count {
		var sim float64
		allngrams := float64(len(input_ngrams))
		matchngrams := float64(count)
		if ngram.warp == 1.0 {
			sim = matchngrams / allngrams
		} else {
			diffngrams := allngrams - matchngrams
			sim = math.Pow(allngrams, ngram.warp) - math.Pow(diffngrams, ngram.warp)
			sim /= math.Pow(allngrams, ngram.warp)
		}
		if sim >= tval {
			res := SearchResult{Similarity: sim, TokenId: token}
			output = append(output, res)
		}
	}
	return output, nil
}

// Search for matches between query string (input) and indexed strings.
// First parameter - threshold is optional and can be used to set minimal similarity
// between input string and matching string. You can pass only one threshold value.
// Results is an unordered array of 'SearchResult' structs. This struct contains similarity
// value (float32 value from threshold to 1.0) and token-id.
func (ngram *NGramIndex) Search(input string, threshold ...float64) ([]SearchResult, error) {
	if ngram.index == nil {
		ngram.init()
	}
	tval, error := validate_threshold_values(threshold)
	if error != nil {
		return nil, error
	}
	return ngram.match(input, tval)
}

func (ngram *NGramIndex) BestMatch(input string, threshold ...float64) (*SearchResult, error) {
	if ngram.index == nil {
		ngram.init()
	}
	tval, error := validate_threshold_values(threshold)
	if error != nil {
		return nil, error
	}
	variants, error := ngram.match(input, tval)
	if error != nil {
		return nil, error
	}
	if len(variants) == 0 {
		return nil, errors.New("No matches found")
	}
	var result SearchResult
	maxsim := -1.0
	for _, val := range variants {
		if val.Similarity > maxsim {
			maxsim = val.Similarity
			result = val
		}
	}
	return &result, nil
}
<commit_msg>Fix formatting and remove counter from nGramValue<commit_after>package ngram

import (
	"errors"
	"github.com/reusee/mmh3"
	"math"
)

const maxN = 8

const defaultPad = "$"

const defaultN = 3

type TokenId int

type nGramValue struct {
	items map[TokenId]int
}

// N-gram index, can be initialized by default (zeroed) or created with "NewNgramIndex"
type NGramIndex struct {
	pad   string
	n     int
	spool stringPool
	index map[uint32]nGramValue
	warp  float64
}

// Search result, contains token id and similarity - value in range from 0.0 to 1.0
type SearchResult struct {
	TokenId    TokenId
	Similarity float64
}

func (ngram *NGramIndex) split_input(str string) ([]uint32, error) {
	if len(str) == 0 {
		return nil, errors.New("Empty string")
	}
	pad := ngram.pad
	n := ngram.n
	input := pad + str + pad
	prev_indexes := make([]int, maxN)
	counter := 0
	results := make([]uint32, 0)

	for index, _ := range input {
		counter++
		if counter > n {
			top := prev_indexes[(counter-n)%len(prev_indexes)]
			substr := input[top:index]
			hash := mmh3.Hash32([]byte(substr))
			results = append(results, hash)
		}
		prev_indexes[counter%len(prev_indexes)] = index
	}

	for i := n - 1; i > 1; i-- {
		if len(input) >= i {
			top := prev_indexes[(len(input)-i)%len(prev_indexes)]
			substr := input[top:]
			hash := mmh3.Hash32([]byte(substr))
			results = append(results, hash)
		}
	}

	return results, nil
}

func (ngram *NGramIndex) init() {
	ngram.index = make(map[uint32]nGramValue)
	if ngram.pad == "" {
		ngram.pad = defaultPad
	}
	if ngram.n == 0 {
		ngram.n = defaultN
	}
	if ngram.warp == 0.0 {
		ngram.warp = 1.0
	}
}

type padArgTrait struct {
	pad rune
}

type nArgTrait struct {
	n int
}

type warpArgTrait struct {
	warp float64
}

// This function must be used to pass padding character to NGramIndex c-tor
func SetPad(c rune) padArgTrait {
	return padArgTrait{pad: c}
}

// This function must be used to pass N (gram size) to NGramIndex c-tor
func SetN(n int) nArgTrait {
	return nArgTrait{n: n}
}

// This function must be used to pass warp to NGramIndex c-tor
func SetWarp(warp float64) warpArgTrait {
	return warpArgTrait{warp: warp}
}

// N-gram index c-tor. In most cases must be used withot parameters.
// You can pass parameters to c-tor using functions SetPad and SetN.
func NewNGramIndex(args ...interface{}) (*NGramIndex, error) {
	ngram := new(NGramIndex)
	for _, arg := range args {
		switch i := arg.(type) {
		case padArgTrait:
			ngram.pad = string(i.pad)
		case nArgTrait:
			if i.n < 2 || i.n > maxN {
				return nil, errors.New("Bad 'n' value for n-gram index")
			}
			ngram.n = i.n
		case warpArgTrait:
			if i.warp < 0.0 || i.warp > 1.0 {
				return nil, errors.New("Bad 'warp' value for n-gram index")
			}
		default:
			return nil, errors.New("Invalid argument")
		}
	}
	ngram.init()
	return ngram, nil
}

// Add token to index. Function returns token id, this id can be converted
// to string with function "GetString".
func (ngram *NGramIndex) Add(input string) (TokenId, error) {
	if ngram.index == nil {
		ngram.init()
	}
	results, error := ngram.split_input(input)
	if error != nil {
		return -1, error
	}
	ixstr, error := ngram.spool.Append(input)
	if error != nil {
		return -1, error
	}
	for _, hash := range results {
		var ok bool
		if val, ok = ngram.index[hash]; !ok {
			ngram.index[hash] = nGramValue{items: make(map[TokenId]int)}
		}
		// insert string and counter
		ngram.index[hash].items[ixstr]++
	}
	return ixstr, nil
}

// Converts token-id to string.
func (ngram *NGramIndex) GetString(id TokenId) (string, error) {
	return ngram.spool.ReadAt(id)
}

// Map matched tokens to the number of ngrams, shared with input string
func (ngram *NGramIndex) count_ngrams(input_ngrams []uint32) map[TokenId]int {
	counters := make(map[TokenId]int)
	for _, ngram_hash := range input_ngrams {
		if tokmap, ok := ngram.index[ngram_hash]; ok {
			for tok, _ := range tokmap.items {
				counters[tok] += 1
			}
		}
	}
	return counters
}

func validate_threshold_values(thresholds []float64) (float64, error) {
	var tval float64
	if len(thresholds) == 1 {
		tval = thresholds[0]
		if tval < 0.0 || tval > 1.0 {
			return 0.0, errors.New("Threshold must be in range (0, 1)")
		}
	} else if len(thresholds) > 1 {
		return 0.0, errors.New("Too many arguments")
	}
	return tval, nil
}

func (ngram *NGramIndex) match(input string, tval float64) ([]SearchResult, error) {
	input_ngrams, error := ngram.split_input(input)
	if error != nil {
		return nil, error
	}
	output := make([]SearchResult, 0)
	token_count := ngram.count_ngrams(input_ngrams)
	for token, count := range token_count {
		var sim float64
		allngrams := float64(len(input_ngrams))
		matchngrams := float64(count)
		if ngram.warp == 1.0 {
			sim = matchngrams / allngrams
		} else {
			diffngrams := allngrams - matchngrams
			sim = math.Pow(allngrams, ngram.warp) - math.Pow(diffngrams, ngram.warp)
			sim /= math.Pow(allngrams, ngram.warp)
		}
		if sim >= tval {
			res := SearchResult{Similarity: sim, TokenId: token}
			output = append(output, res)
		}
	}
	return output, nil
}

// Search for matches between query string (input) and indexed strings.
// First parameter - threshold is optional and can be used to set minimal similarity
// between input string and matching string. You can pass only one threshold value.
// Results is an unordered array of 'SearchResult' structs. This struct contains similarity
// value (float32 value from threshold to 1.0) and token-id.
func (ngram *NGramIndex) Search(input string, threshold ...float64) ([]SearchResult, error) {
	if ngram.index == nil {
		ngram.init()
	}
	tval, error := validate_threshold_values(threshold)
	if error != nil {
		return nil, error
	}
	return ngram.match(input, tval)
}

func (ngram *NGramIndex) BestMatch(input string, threshold ...float64) (*SearchResult, error) {
	if ngram.index == nil {
		ngram.init()
	}
	tval, error := validate_threshold_values(threshold)
	if error != nil {
		return nil, error
	}
	variants, error := ngram.match(input, tval)
	if error != nil {
		return nil, error
	}
	if len(variants) == 0 {
		return nil, errors.New("No matches found")
	}
	var result SearchResult
	maxsim := -1.0
	for _, val := range variants {
		if val.Similarity > maxsim {
			maxsim = val.Similarity
			result = val
		}
	}
	return &result, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package pacman

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
)

// NotFoundError is returned when a package could not be found on AUR.
//
// The error message returned is different dependent on the number of packages
// that could not be found.
type NotFoundError struct {
	Names []string
}

func (e NotFoundError) Error() string {
	n := len(e.Names)
	if n == 1 {
		return fmt.Sprintf("package %q could not be found on AUR", e.Names[0])
	} else if n == 2 {
		return fmt.Sprintf("packages %q and %q could not be found on AUR", e.Names[0], e.Names[1])
	}

	// We have three or more packages that we could not find.
	var buf bytes.Buffer
	buf.WriteString("packages ")
	for _, nam := range e.Names[:n-1] {
		buf.WriteRune('"')
		buf.WriteString(nam)
		buf.WriteString(`", `)
	}
	buf.WriteString(`and "`)
	buf.WriteString(e.Names[n-1])
	buf.WriteString(`" could not be found on AUR`)
	return buf.String()
}

type aurResponse struct {
	ResultCount int
	Results     []*AURPackage
}

// AURPackage is the information that we can retrieve about a package that is
// hosted on the Arch Linux User Repository (AUR), version 4.
//
// JSON Example:
//
//	{
//		"ID": 213309,
//		"Name": "repoctl",
//		"PackageBaseID": 96153,
//		"PackageBase": "repoctl",
//		"Version": "0.13-2",
//		"Description": "A supplement to repo-add and repo-remove which simplifies managing local repositories",
//		"URL": "https:\/\/github.com\/cassava\/repoctl",
//		"NumVotes": 1,
//		"OutOfDate": 0,
//		"Maintainer": "cassava",
//		"FirstSubmitted": 1437296687,
//		"LastModified": 1437298275,
//		"License": "MIT",
//		"URLPath": "\/cgit\/aur.git\/snapshot\/repoctl.tar.gz",
//		"CategoryID": 1,
//		"Popularity": 0
//	}
type AURPackage struct {
	ID             uint64
	Name           string
	PackageBaseID  uint64
	PackageBase    string
	Version        string
	Description    string
	URL            string
	NumVotes       int
	OutOfDate      int
	Maintainer     string
	FirstSubmitted uint64
	LastModified   uint64
	License        string
	URLPath        string
	CategoryID     int
	Popularity     float64
}

// Package converts an AURPackage into a Package.
//
// Note that only a few fields in the resulting Package are actually filled in,
// namely Origin, Name, Version, Description, URL, and License. This is all the
// information that we are able to retrieve.
func (ap *AURPackage) Package() *Package {
	return &Package{
		Origin:      AUROrigin,
		Name:        ap.Name,
		Version:     ap.Version,
		Description: ap.Description,
		URL:         ap.URL,
		License:     ap.License,
	}
}

// DownloadURL returns the URL for downloading the PKGBUILD tarball.
func (ap *AURPackage) DownloadURL() string {
	return fmt.Sprintf("https://aur4.archlinux.org%s", ap.URLPath)
}

const (
	apiURL = "https://aur4.archlinux.org/rpc.php?type=multiinfo&arg[]=%s"
	apiArg = "&arg[]="
)

// generateURL creates a URL that gets the package information from AUR.
func generateURL(args []string) string {
	na := make([]string, len(args))
	for i, s := range args {
		na[i] = url.QueryEscape(s)
	}
	return fmt.Sprintf(apiURL, strings.Join(na, apiArg))
}

// ReadAUR reads package information from the Arch Linux User Repository (AUR)
// online.
//
// If a package cannot be found, (nil, *NotFoundError) is returned.
func ReadAUR(pkgname string) (*AURPackage, error) {
	q := generateURL([]string{pkgname})
	resp, err := http.Get(q)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var msg aurResponse
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(&msg)
	if err != nil {
		return nil, err
	}
	if msg.ResultCount == 0 {
		return nil, &NotFoundError{Names: []string{pkgname}}
	}
	return msg.Results[0], nil
}

type AURPackages []*AURPackage

func (pkgs AURPackages) Len() int      { return len(pkgs) }
func (pkgs AURPackages) Swap(i, j int) { pkgs[i], pkgs[j] = pkgs[j], pkgs[i] }
func (pkgs AURPackages) Less(i, j int) bool {
	if pkgs[i].Name != pkgs[j].Name {
		return pkgs[i].Name < pkgs[j].Name
	}
	return VerCmp(pkgs[i].Version, pkgs[j].Version) == -1
}

// ReadAllAUR reads multiple packages from the Arch Linux User Repository (AUR)
// at once.
//
// If any packages cannot be found, (AURPackages, *NotFoundError) is returned.
// That is, all successfully read packages are returned.
func ReadAllAUR(pkgnames []string) (AURPackages, error) {
	q := generateURL(pkgnames)
	resp, err := http.Get(q)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var msg aurResponse
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(&msg)
	if err != nil {
		return nil, err
	}
	if msg.ResultCount != len(pkgnames) {
		m := make(map[string]bool)
		nfe := &NotFoundError{
			Names: make([]string, 0, len(pkgnames)-msg.ResultCount),
		}
		for _, i := range msg.Results {
			m[i.Name] = true
		}
		for _, s := range pkgnames {
			if !m[s] {
				nfe.Names = append(nfe.Names, s)
			}
		}
		return msg.Results, nfe
	}
	return msg.Results, nil
}
<commit_msg>Updating AUR URL<commit_after>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package pacman

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
)

// NotFoundError is returned when a package could not be found on AUR.
//
// The error message returned is different dependent on the number of packages
// that could not be found.
type NotFoundError struct {
	Names []string
}

func (e NotFoundError) Error() string {
	n := len(e.Names)
	if n == 1 {
		return fmt.Sprintf("package %q could not be found on AUR", e.Names[0])
	} else if n == 2 {
		return fmt.Sprintf("packages %q and %q could not be found on AUR", e.Names[0], e.Names[1])
	}

	// We have three or more packages that we could not find.
	var buf bytes.Buffer
	buf.WriteString("packages ")
	for _, nam := range e.Names[:n-1] {
		buf.WriteRune('"')
		buf.WriteString(nam)
		buf.WriteString(`", `)
	}
	buf.WriteString(`and "`)
	buf.WriteString(e.Names[n-1])
	buf.WriteString(`" could not be found on AUR`)
	return buf.String()
}

type aurResponse struct {
	ResultCount int
	Results     []*AURPackage
}

// AURPackage is the information that we can retrieve about a package that is
// hosted on the Arch Linux User Repository (AUR), version 4.
//
// JSON Example:
//
//	{
//		"ID": 213309,
//		"Name": "repoctl",
//		"PackageBaseID": 96153,
//		"PackageBase": "repoctl",
//		"Version": "0.13-2",
//		"Description": "A supplement to repo-add and repo-remove which simplifies managing local repositories",
//		"URL": "https:\/\/github.com\/cassava\/repoctl",
//		"NumVotes": 1,
//		"OutOfDate": 0,
//		"Maintainer": "cassava",
//		"FirstSubmitted": 1437296687,
//		"LastModified": 1437298275,
//		"License": "MIT",
//		"URLPath": "\/cgit\/aur.git\/snapshot\/repoctl.tar.gz",
//		"CategoryID": 1,
//		"Popularity": 0
//	}
type AURPackage struct {
	ID             uint64
	Name           string
	PackageBaseID  uint64
	PackageBase    string
	Version        string
	Description    string
	URL            string
	NumVotes       int
	OutOfDate      int
	Maintainer     string
	FirstSubmitted uint64
	LastModified   uint64
	License        string
	URLPath        string
	CategoryID     int
	Popularity     float64
}

// Package converts an AURPackage into a Package.
//
// Note that only a few fields in the resulting Package are actually filled in,
// namely Origin, Name, Version, Description, URL, and License. This is all the
// information that we are able to retrieve.
func (ap *AURPackage) Package() *Package {
	return &Package{
		Origin:      AUROrigin,
		Name:        ap.Name,
		Version:     ap.Version,
		Description: ap.Description,
		URL:         ap.URL,
		License:     ap.License,
	}
}

// DownloadURL returns the URL for downloading the PKGBUILD tarball.
func (ap *AURPackage) DownloadURL() string {
	return fmt.Sprintf("https://aur.archlinux.org%s", ap.URLPath)
}

const (
	apiURL = "https://aur.archlinux.org/rpc.php?type=multiinfo&arg[]=%s"
	apiArg = "&arg[]="
)

// generateURL creates a URL that gets the package information from AUR.
func generateURL(args []string) string {
	na := make([]string, len(args))
	for i, s := range args {
		na[i] = url.QueryEscape(s)
	}
	return fmt.Sprintf(apiURL, strings.Join(na, apiArg))
}

// ReadAUR reads package information from the Arch Linux User Repository (AUR)
// online.
//
// If a package cannot be found, (nil, *NotFoundError) is returned.
func ReadAUR(pkgname string) (*AURPackage, error) {
	q := generateURL([]string{pkgname})
	resp, err := http.Get(q)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var msg aurResponse
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(&msg)
	if err != nil {
		return nil, err
	}
	if msg.ResultCount == 0 {
		return nil, &NotFoundError{Names: []string{pkgname}}
	}
	return msg.Results[0], nil
}

type AURPackages []*AURPackage

func (pkgs AURPackages) Len() int      { return len(pkgs) }
func (pkgs AURPackages) Swap(i, j int) { pkgs[i], pkgs[j] = pkgs[j], pkgs[i] }
func (pkgs AURPackages) Less(i, j int) bool {
	if pkgs[i].Name != pkgs[j].Name {
		return pkgs[i].Name < pkgs[j].Name
	}
	return VerCmp(pkgs[i].Version, pkgs[j].Version) == -1
}

// ReadAllAUR reads multiple packages from the Arch Linux User Repository (AUR)
// at once.
//
// If any packages cannot be found, (AURPackages, *NotFoundError) is returned.
// That is, all successfully read packages are returned.
func ReadAllAUR(pkgnames []string) (AURPackages, error) {
	q := generateURL(pkgnames)
	resp, err := http.Get(q)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var msg aurResponse
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(&msg)
	if err != nil {
		return nil, err
	}
	if msg.ResultCount != len(pkgnames) {
		m := make(map[string]bool)
		nfe := &NotFoundError{
			Names: make([]string, 0, len(pkgnames)-msg.ResultCount),
		}
		for _, i := range msg.Results {
			m[i.Name] = true
		}
		for _, s := range pkgnames {
			if !m[s] {
				nfe.Names = append(nfe.Names, s)
			}
		}
		return msg.Results, nfe
	}
	return msg.Results, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/vaughan0/go-ini"
	"launchpad.net/goamz/aws"
	"os"
	"strings"
)

type AWSConf struct {
	aws.Auth
	Region aws.Region
}

var ErrNoAccessKeyGiven = errors.New("no access key given")
var ErrUnknownRegion = errors.New("unknown region given")

func getAWSConf(projectName string, lookupEnv bool) (conf AWSConf, err error) {
	confFn := os.Getenv("AWS_CONFIG_FILE")
	if confFn == "" {
		confFn = os.Getenv("HOME") + "/.aws/credentials"
		if _, err = os.Stat(confFn); os.IsNotExist(err) {
			confFn = os.Getenv("HOME") + "/.aws/config"
		}
	}

	awsAuth, err := aws.EnvAuth()
	if err == nil {
		conf.Auth = awsAuth

	} else if _, err = os.Stat(confFn); os.IsNotExist(err) {
		return

	} else {
		profiles := make([]string, 0)

		if lookupEnv {
			envProfile := os.Getenv("AWS_DEFAULT_PROFILE")
			if envProfile != "" {
				profiles = append(profiles, envProfile)
			}
		}
		profiles = append(profiles, projectName)
		lowerProjectName := strings.ToLower(projectName)
		profiles = append(profiles, strings.Replace(lowerProjectName, " ", "_", -1))
		profiles = append(profiles, strings.Replace(lowerProjectName, " ", "-", -1))

		var iniFile ini.File
		iniFile, err = ini.LoadFile(confFn)
		if err != nil {
			return
		}

		var fileConf ini.Section
		for _, profile := range profiles {
			fileConf = iniFile["profile "+profile]
			if fileConf != nil {
				break
			}
		}

		if fileConf == nil {
			err = errors.New(
				fmt.Sprintf("Couldn't find a suitable AWS config profile; looked for profiles named '%s'. Please add one to your AWS config file at %s",
					strings.Join(profiles, "', '"), confFn))
			return
		}

		conf.AccessKey = fileConf["aws_access_key_id"]
		conf.SecretKey = fileConf["aws_secret_access_key"]
		conf.Region = aws.Regions[fileConf["region"]]
	}

	envRegion := os.Getenv("AWS_DEFAULT_REGION")
	if envRegion != "" {
		conf.Region = aws.Regions[envRegion]
	}

	if conf.AccessKey == "" {
		err = ErrNoAccessKeyGiven
	}

	if conf.Region.Name == "" {
		err = ErrUnknownRegion
	}

	return
}
<commit_msg>Fixed where profiles in credentials file were being looked up with config-style prefix<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/vaughan0/go-ini"
	"launchpad.net/goamz/aws"
	"os"
	"strings"
)

type AWSConf struct {
	aws.Auth
	Region aws.Region
}

var ErrNoAccessKeyGiven = errors.New("no access key given")
var ErrUnknownRegion = errors.New("unknown region given")

func getAWSConf(projectName string, lookupEnv bool) (conf AWSConf, err error) {
	profileNameHasPrefix := false
	confFn := os.Getenv("AWS_CONFIG_FILE")
	if confFn == "" {
		confFn = os.Getenv("HOME") + "/.aws/credentials"
		if _, err = os.Stat(confFn); os.IsNotExist(err) {
			confFn = os.Getenv("HOME") + "/.aws/config"
			profileNameHasPrefix = true
		}
	}

	awsAuth, err := aws.EnvAuth()
	if err == nil {
		conf.Auth = awsAuth

	} else if _, err = os.Stat(confFn); os.IsNotExist(err) {
		return

	} else {
		profiles := make([]string, 0)

		if lookupEnv {
			envProfile := os.Getenv("AWS_DEFAULT_PROFILE")
			if envProfile != "" {
				profiles = append(profiles, envProfile)
			}
		}
		profiles = append(profiles, projectName)
		lowerProjectName := strings.ToLower(projectName)
		profiles = append(profiles, strings.Replace(lowerProjectName, " ", "_", -1))
		profiles = append(profiles, strings.Replace(lowerProjectName, " ", "-", -1))

		var iniFile ini.File
		iniFile, err = ini.LoadFile(confFn)
		if err != nil {
			return
		}

		var fileConf ini.Section
		for _, profile := range profiles {
			n := profile
			if profileNameHasPrefix {
				n = "profile "+n
			}
			fileConf = iniFile[n]
			if fileConf != nil {
				break
			}
		}

		if fileConf == nil {
			err = errors.New(
				fmt.Sprintf("Couldn't find a suitable AWS config profile; looked for profiles named '%s'. Please add one to your AWS config file at %s",
					strings.Join(profiles, "', '"), confFn))
			return
		}

		conf.AccessKey = fileConf["aws_access_key_id"]
		conf.SecretKey = fileConf["aws_secret_access_key"]
		conf.Region = aws.Regions[fileConf["region"]]
	}

	envRegion := os.Getenv("AWS_DEFAULT_REGION")
	if envRegion != "" {
		conf.Region = aws.Regions[envRegion]
	}

	if conf.AccessKey == "" {
		err = ErrNoAccessKeyGiven
	}

	if conf.Region.Name == "" {
		err = ErrUnknownRegion
	}

	return
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package auth

import (
	"context"
	"encoding/json"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/containerd/containerd/log"
	remoteserrors "github.com/containerd/containerd/remotes/errors"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

var (
	// ErrNoToken is returned if a request is successful but the body does not
	// contain an authorization token.
	ErrNoToken = errors.New("authorization server did not include a token in the response")
)

// GenerateTokenOptions generates options for fetching a token based on a challenge
func GenerateTokenOptions(ctx context.Context, host, username, secret string, c Challenge) (TokenOptions, error) {
	realm, ok := c.Parameters["realm"]
	if !ok {
		return TokenOptions{}, errors.New("no realm specified for token auth challenge")
	}

	realmURL, err := url.Parse(realm)
	if err != nil {
		return TokenOptions{}, errors.Wrap(err, "invalid token auth challenge realm")
	}

	to := TokenOptions{
		Realm:    realmURL.String(),
		Service:  c.Parameters["service"],
		Username: username,
		Secret:   secret,
	}

	scope, ok := c.Parameters["scope"]
	if ok {
		to.Scopes = append(to.Scopes, scope)
	} else {
		log.G(ctx).WithField("host", host).Debug("no scope specified for token auth challenge")
	}

	return to, nil
}

// TokenOptions are options for requesting a token
type TokenOptions struct {
	Realm    string
	Service  string
	Scopes   []string
	Username string
	Secret   string
}

// OAuthTokenResponse is response from fetching token with a OAuth POST request
type OAuthTokenResponse struct {
	AccessToken  string    `json:"access_token"`
	RefreshToken string    `json:"refresh_token"`
	ExpiresIn    int       `json:"expires_in"`
	IssuedAt     time.Time `json:"issued_at"`
	Scope        string    `json:"scope"`
}

// FetchTokenWithOAuth fetches a token using a POST request
func FetchTokenWithOAuth(ctx context.Context, client *http.Client, headers http.Header, clientID string, to TokenOptions) (*OAuthTokenResponse, error) {
	form := url.Values{}
	if len(to.Scopes) > 0 {
		form.Set("scope", strings.Join(to.Scopes, " "))
	}
	form.Set("service", to.Service)
	form.Set("client_id", clientID)

	if to.Username == "" {
		form.Set("grant_type", "refresh_token")
		form.Set("refresh_token", to.Secret)
	} else {
		form.Set("grant_type", "password")
		form.Set("username", to.Username)
		form.Set("password", to.Secret)
	}

	req, err := http.NewRequest("POST", to.Realm, strings.NewReader(form.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
	for k, v := range headers {
		req.Header[k] = append(req.Header[k], v...)
	}

	resp, err := ctxhttp.Do(ctx, client, req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		return nil, errors.WithStack(remoteserrors.NewUnexpectedStatusErr(resp))
	}

	decoder := json.NewDecoder(resp.Body)

	var tr OAuthTokenResponse
	if err = decoder.Decode(&tr); err != nil {
		return nil, errors.Wrap(err, "unable to decode token response")
	}

	if tr.AccessToken == "" {
		return nil, errors.WithStack(ErrNoToken)
	}

	return &tr, nil
}

// FetchTokenResponse is response from fetching token with GET request
type FetchTokenResponse struct {
	Token        string    `json:"token"`
	AccessToken  string    `json:"access_token"`
	ExpiresIn    int       `json:"expires_in"`
	IssuedAt     time.Time `json:"issued_at"`
	RefreshToken string    `json:"refresh_token"`
}

// FetchToken fetches a token using a GET request
func FetchToken(ctx context.Context, client *http.Client, headers http.Header, to TokenOptions) (*FetchTokenResponse, error) {
	req, err := http.NewRequest("GET", to.Realm, nil)
	if err != nil {
		return nil, err
	}

	for k, v := range headers {
		req.Header[k] = append(req.Header[k], v...)
	}

	reqParams := req.URL.Query()

	if to.Service != "" {
		reqParams.Add("service", to.Service)
	}

	for _, scope := range to.Scopes {
		reqParams.Add("scope", scope)
	}

	if to.Secret != "" {
		req.SetBasicAuth(to.Username, to.Secret)
	}

	req.URL.RawQuery = reqParams.Encode()

	resp, err := ctxhttp.Do(ctx, client, req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		return nil, errors.WithStack(remoteserrors.NewUnexpectedStatusErr(resp))
	}

	decoder := json.NewDecoder(resp.Body)

	var tr FetchTokenResponse
	if err = decoder.Decode(&tr); err != nil {
		return nil, errors.Wrap(err, "unable to decode token response")
	}

	// `access_token` is equivalent to `token` and if both are specified
	// the choice is undefined.  Canonicalize `access_token` by sticking
	// things in `token`.
	if tr.AccessToken != "" {
		tr.Token = tr.AccessToken
	}

	if tr.Token == "" {
		return nil, errors.WithStack(ErrNoToken)
	}

	return &tr, nil
}
<commit_msg>Fix incorrect UA used for registry authentication<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package auth

import (
	"context"
	"encoding/json"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/containerd/containerd/log"
	remoteserrors "github.com/containerd/containerd/remotes/errors"
	"github.com/containerd/containerd/version"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

var (
	// ErrNoToken is returned if a request is successful but the body does not
	// contain an authorization token.
	ErrNoToken = errors.New("authorization server did not include a token in the response")
)

// GenerateTokenOptions generates options for fetching a token based on a challenge
func GenerateTokenOptions(ctx context.Context, host, username, secret string, c Challenge) (TokenOptions, error) {
	realm, ok := c.Parameters["realm"]
	if !ok {
		return TokenOptions{}, errors.New("no realm specified for token auth challenge")
	}

	realmURL, err := url.Parse(realm)
	if err != nil {
		return TokenOptions{}, errors.Wrap(err, "invalid token auth challenge realm")
	}

	to := TokenOptions{
		Realm:    realmURL.String(),
		Service:  c.Parameters["service"],
		Username: username,
		Secret:   secret,
	}

	scope, ok := c.Parameters["scope"]
	if ok {
		to.Scopes = append(to.Scopes, scope)
	} else {
		log.G(ctx).WithField("host", host).Debug("no scope specified for token auth challenge")
	}

	return to, nil
}

// TokenOptions are options for requesting a token
type TokenOptions struct {
	Realm    string
	Service  string
	Scopes   []string
	Username string
	Secret   string
}

// OAuthTokenResponse is response from fetching token with a OAuth POST request
type OAuthTokenResponse struct {
	AccessToken  string    `json:"access_token"`
	RefreshToken string    `json:"refresh_token"`
	ExpiresIn    int       `json:"expires_in"`
	IssuedAt     time.Time `json:"issued_at"`
	Scope        string    `json:"scope"`
}

// FetchTokenWithOAuth fetches a token using a POST request
func FetchTokenWithOAuth(ctx context.Context, client *http.Client, headers http.Header, clientID string, to TokenOptions) (*OAuthTokenResponse, error) {
	form := url.Values{}
	if len(to.Scopes) > 0 {
		form.Set("scope", strings.Join(to.Scopes, " "))
	}
	form.Set("service", to.Service)
	form.Set("client_id", clientID)

	if to.Username == "" {
		form.Set("grant_type", "refresh_token")
		form.Set("refresh_token", to.Secret)
	} else {
		form.Set("grant_type", "password")
		form.Set("username", to.Username)
		form.Set("password", to.Secret)
	}

	req, err := http.NewRequest("POST", to.Realm, strings.NewReader(form.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
	for k, v := range headers {
		req.Header[k] = append(req.Header[k], v...)
	}
	if len(req.Header.Get("User-Agent")) == 0 {
		req.Header.Set("User-Agent", "containerd/"+version.Version)
	}

	resp, err := ctxhttp.Do(ctx, client, req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		return nil, errors.WithStack(remoteserrors.NewUnexpectedStatusErr(resp))
	}

	decoder := json.NewDecoder(resp.Body)

	var tr OAuthTokenResponse
	if err = decoder.Decode(&tr); err != nil {
		return nil, errors.Wrap(err, "unable to decode token response")
	}

	if tr.AccessToken == "" {
		return nil, errors.WithStack(ErrNoToken)
	}

	return &tr, nil
}

// FetchTokenResponse is response from fetching token with GET request
type FetchTokenResponse struct {
	Token        string    `json:"token"`
	AccessToken  string    `json:"access_token"`
	ExpiresIn    int       `json:"expires_in"`
	IssuedAt     time.Time `json:"issued_at"`
	RefreshToken string    `json:"refresh_token"`
}

// FetchToken fetches a token using a GET request
func FetchToken(ctx context.Context, client *http.Client, headers http.Header, to TokenOptions) (*FetchTokenResponse, error) {
	req, err := http.NewRequest("GET", to.Realm, nil)
	if err != nil {
		return nil, err
	}

	for k, v := range headers {
		req.Header[k] = append(req.Header[k], v...)
	}
	if len(req.Header.Get("User-Agent")) == 0 {
		req.Header.Set("User-Agent", "containerd/"+version.Version)
	}

	reqParams := req.URL.Query()

	if to.Service != "" {
		reqParams.Add("service", to.Service)
	}

	for _, scope := range to.Scopes {
		reqParams.Add("scope", scope)
	}

	if to.Secret != "" {
		req.SetBasicAuth(to.Username, to.Secret)
	}

	req.URL.RawQuery = reqParams.Encode()

	resp, err := ctxhttp.Do(ctx, client, req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		return nil, errors.WithStack(remoteserrors.NewUnexpectedStatusErr(resp))
	}

	decoder := json.NewDecoder(resp.Body)

	var tr FetchTokenResponse
	if err = decoder.Decode(&tr); err != nil {
		return nil, errors.Wrap(err, "unable to decode token response")
	}

	// `access_token` is equivalent to `token` and if both are specified
	// the choice is undefined.  Canonicalize `access_token` by sticking
	// things in `token`.
	if tr.AccessToken != "" {
		tr.Token = tr.AccessToken
	}

	if tr.Token == "" {
		return nil, errors.WithStack(ErrNoToken)
	}

	return &tr, nil
}
<|endoftext|>


<commit_before>package gopiper

import (
	"errors"
	"fmt"
	"html"
	"log"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

func init() {
	RegisterFilter("preadd", preadd)
	RegisterFilter("postadd", postadd)
	RegisterFilter("replace", replace)
	RegisterFilter("split", split)
	RegisterFilter("join", join)
	RegisterFilter("trim", trim)
	RegisterFilter("trimspace", trimspace)
	RegisterFilter("substr", substr)
	RegisterFilter("intval", intval)
	RegisterFilter("floatval", floatval)
	RegisterFilter("hrefreplace", hrefreplace)
	RegisterFilter("wraphtml", wraphtml)
	RegisterFilter("tosbc", tosbc)
	RegisterFilter("unescape", unescape)
	RegisterFilter("escape", escape)
}

type FilterFunction func(src *reflect.Value, params *reflect.Value) (interface{}, error)

var filters = make(map[string]FilterFunction)

func RegisterFilter(name string, fn FilterFunction) {
	_, existing := filters[name]
	if existing {
		panic(fmt.Sprintf("Filter with name '%s' is already registered.", name))
	}
	filters[name] = fn
}

func ReplaceFilter(name string, fn FilterFunction) {
	_, existing := filters[name]
	if !existing {
		panic(fmt.Sprintf("Filter with name '%s' does not exist (therefore cannot be overridden).", name))
	}
	filters[name] = fn
}

func applyFilter(name string, src *reflect.Value, params *reflect.Value) (interface{}, error) {
	fn, existing := filters[name]
	if !existing {
		return nil, errors.New(fmt.Sprintf("Filter with name '%s' not found.", name))
	}
	return fn(src, params)
}

func callFilter(src interface{}, value string) (interface{}, error) {

	if src == nil || len(value) == 0 {
		return src, nil
	}

	exp, _ := regexp.Compile(`([a-zA-Z0-9\-_]+)\(([\w\W]*?)\)(\||$)`)
	vt := exp.FindAllStringSubmatch(value, -1)

	for _, v := range vt {
		if len(v) < 3 {
			continue
		}
		name := v[1]
		params := v[2]

		src_value := reflect.ValueOf(src)
		param_value := reflect.ValueOf(params)
		next, err := applyFilter(name, &src_value, &param_value)
		if err != nil {
			continue
		}
		src = next

	}

	return src, nil
}

func preadd(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return params.String() + src.String(), nil
}
func postadd(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return src.String() + params.String(), nil
}
func substr(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	vt := strings.Split(params.String(), ",")
	if len(vt) == 1 {
		start, _ := strconv.Atoi(vt[0])
		return src.String()[start:], nil
	} else if len(vt) == 2 {
		start, _ := strconv.Atoi(vt[0])
		end, _ := strconv.Atoi(vt[1])
		return src.String()[start:end], nil
	}
	return src.Interface(), nil
}
func replace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	vt := strings.Split(params.String(), ",")
	if len(vt) == 1 {
		return strings.Replace(src.String(), vt[0], "", -1), nil
	} else if len(vt) == 2 {
		return strings.Replace(src.String(), vt[0], vt[1], -1), nil
	} else if len(vt) == 3 {
		n, _ := strconv.Atoi(vt[2])
		return strings.Replace(src.String(), vt[0], vt[1], n), nil
	}
	return src.Interface(), nil
}
func trim(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		log.Println(src.String())
		return src.Interface(), errors.New("filter split nil params")
	}

	switch src.Interface().(type) {
	case string:
		return strings.Trim(src.String(), params.String()), nil
	case []string:
		vt, _ := src.Interface().([]string)
		for i := 0; i < len(vt); i++ {
			vt[i] = strings.Trim(vt[i], params.String())
		}
		return vt, nil
	}

	return src.Interface(), nil
}

func trimspace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		log.Println(src.String())
		return src.Interface(), errors.New("filter split nil params")
	}

	switch src.Interface().(type) {
	case string:
		return strings.TrimSpace(src.String()), nil
	case []string:
		vt, _ := src.Interface().([]string)
		for i := 0; i < len(vt); i++ {
			vt[i] = strings.TrimSpace(vt[i])
		}
		return vt, nil
	}

	return src.Interface(), nil
}

func split(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter split nil params")
	}
	return strings.Split(src.String(), params.String()), nil
}

func join(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter split nil params")
	}
	switch src.Interface().(type) {
	case []string:
		vt, _ := src.Interface().([]string)
		rs := make([]string, 0)
		for _, v := range vt {
			if len(v) > 0 {
				rs = append(rs, v)
			}
		}
		return strings.Join(rs, params.String()), nil
	}

	return src.Interface(), nil
}

func intval(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if src.Interface() == nil {
		return 0, nil
	}
	v, _ := strconv.Atoi(src.String())
	return v, nil
}

func floatval(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if src.Interface() == nil {
		return 0.0, nil
	}
	v, _ := strconv.ParseFloat(src.String(), 64)
	return v, nil
}

func hrefreplace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	href_filter_regexp, _ := regexp.Compile(`href(\s*)=(\s*)([\w\W]+?)"`)
	return href_filter_regexp.ReplaceAllString(src.String(), params.String()), nil
}

func regexpreplace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return src.Interface(), nil
}

func tosbc(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	res := ""
	for _, t := range src.String() {
		if t == 12288 {
			t = 32
		} else if t > 65280 && t < 65375 {
			t = t - 65248
		}
		res += string(t)
	}
	return res, nil
}

func unescape(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return html.UnescapeString(src.String()), nil
}

func escape(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return html.EscapeString(src.String()), nil
}

func wraphtml(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter wraphtml nil params")
	}

	switch src.Interface().(type) {
	case string:
		return fmt.Sprintf("<%s>%s</%s>", params.String(), src.String(), params.String()), nil
	case []string:
		vt, _ := src.Interface().([]string)
		for i := 0; i < len(vt); i++ {
			if len(vt[i]) <= 0 {
				continue
			}
			vt[i] = fmt.Sprintf("<%s>%s</%s>", params.String(), vt[i], params.String())
		}
		return vt, nil
	}

	return src.Interface(), nil
}
<commit_msg>add sprintf function<commit_after>package gopiper

import (
	"errors"
	"fmt"
	"html"
	"log"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

func init() {
	RegisterFilter("preadd", preadd)
	RegisterFilter("postadd", postadd)
	RegisterFilter("replace", replace)
	RegisterFilter("split", split)
	RegisterFilter("join", join)
	RegisterFilter("trim", trim)
	RegisterFilter("trimspace", trimspace)
	RegisterFilter("substr", substr)
	RegisterFilter("intval", intval)
	RegisterFilter("floatval", floatval)
	RegisterFilter("hrefreplace", hrefreplace)
	RegisterFilter("wraphtml", wraphtml)
	RegisterFilter("tosbc", tosbc)
	RegisterFilter("unescape", unescape)
	RegisterFilter("escape", escape)
	RegisterFilter("sprintf", sprintf)
	RegisterFilter("sprintfmap", sprintfmap)
}

type FilterFunction func(src *reflect.Value, params *reflect.Value) (interface{}, error)

var filters = make(map[string]FilterFunction)

func RegisterFilter(name string, fn FilterFunction) {
	_, existing := filters[name]
	if existing {
		panic(fmt.Sprintf("Filter with name '%s' is already registered.", name))
	}
	filters[name] = fn
}

func ReplaceFilter(name string, fn FilterFunction) {
	_, existing := filters[name]
	if !existing {
		panic(fmt.Sprintf("Filter with name '%s' does not exist (therefore cannot be overridden).", name))
	}
	filters[name] = fn
}

func applyFilter(name string, src *reflect.Value, params *reflect.Value) (interface{}, error) {
	fn, existing := filters[name]
	if !existing {
		return nil, errors.New(fmt.Sprintf("Filter with name '%s' not found.", name))
	}
	return fn(src, params)
}

func callFilter(src interface{}, value string) (interface{}, error) {

	if src == nil || len(value) == 0 {
		return src, nil
	}

	exp, _ := regexp.Compile(`([a-zA-Z0-9\-_]+)\(([\w\W]*?)\)(\||$)`)
	vt := exp.FindAllStringSubmatch(value, -1)

	for _, v := range vt {
		if len(v) < 3 {
			continue
		}
		name := v[1]
		params := v[2]

		src_value := reflect.ValueOf(src)
		param_value := reflect.ValueOf(params)
		next, err := applyFilter(name, &src_value, &param_value)
		if err != nil {
			continue
		}
		src = next

	}

	return src, nil
}

func preadd(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return params.String() + src.String(), nil
}
func postadd(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return src.String() + params.String(), nil
}
func substr(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	vt := strings.Split(params.String(), ",")
	if len(vt) == 1 {
		start, _ := strconv.Atoi(vt[0])
		return src.String()[start:], nil
	} else if len(vt) == 2 {
		start, _ := strconv.Atoi(vt[0])
		end, _ := strconv.Atoi(vt[1])
		return src.String()[start:end], nil
	}
	return src.Interface(), nil
}
func replace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	vt := strings.Split(params.String(), ",")
	if len(vt) == 1 {
		return strings.Replace(src.String(), vt[0], "", -1), nil
	} else if len(vt) == 2 {
		return strings.Replace(src.String(), vt[0], vt[1], -1), nil
	} else if len(vt) == 3 {
		n, _ := strconv.Atoi(vt[2])
		return strings.Replace(src.String(), vt[0], vt[1], n), nil
	}
	return src.Interface(), nil
}
func trim(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		log.Println(src.String())
		return src.Interface(), errors.New("filter split nil params")
	}

	switch src.Interface().(type) {
	case string:
		return strings.Trim(src.String(), params.String()), nil
	case []string:
		vt, _ := src.Interface().([]string)
		for i := 0; i < len(vt); i++ {
			vt[i] = strings.Trim(vt[i], params.String())
		}
		return vt, nil
	}

	return src.Interface(), nil
}

func trimspace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		log.Println(src.String())
		return src.Interface(), errors.New("filter split nil params")
	}

	switch src.Interface().(type) {
	case string:
		return strings.TrimSpace(src.String()), nil
	case []string:
		vt, _ := src.Interface().([]string)
		for i := 0; i < len(vt); i++ {
			vt[i] = strings.TrimSpace(vt[i])
		}
		return vt, nil
	}

	return src.Interface(), nil
}

func split(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter split nil params")
	}
	return strings.Split(src.String(), params.String()), nil
}

func join(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter split nil params")
	}
	switch src.Interface().(type) {
	case []string:
		vt, _ := src.Interface().([]string)
		rs := make([]string, 0)
		for _, v := range vt {
			if len(v) > 0 {
				rs = append(rs, v)
			}
		}
		return strings.Join(rs, params.String()), nil
	}

	return src.Interface(), nil
}

func intval(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if src.Interface() == nil {
		return 0, nil
	}
	v, _ := strconv.Atoi(src.String())
	return v, nil
}

func floatval(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if src.Interface() == nil {
		return 0.0, nil
	}
	v, _ := strconv.ParseFloat(src.String(), 64)
	return v, nil
}

func hrefreplace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	href_filter_regexp, _ := regexp.Compile(`href(\s*)=(\s*)([\w\W]+?)"`)
	return href_filter_regexp.ReplaceAllString(src.String(), params.String()), nil
}

func regexpreplace(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return src.Interface(), nil
}

func tosbc(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	res := ""
	for _, t := range src.String() {
		if t == 12288 {
			t = 32
		} else if t > 65280 && t < 65375 {
			t = t - 65248
		}
		res += string(t)
	}
	return res, nil
}

func unescape(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return html.UnescapeString(src.String()), nil
}

func escape(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	return html.EscapeString(src.String()), nil
}

func wraphtml(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter wraphtml nil params")
	}

	switch src.Interface().(type) {
	case string:
		return fmt.Sprintf("<%s>%s</%s>", params.String(), src.String(), params.String()), nil
	case []string:
		vt, _ := src.Interface().([]string)
		for i := 0; i < len(vt); i++ {
			if len(vt[i]) <= 0 {
				continue
			}
			vt[i] = fmt.Sprintf("<%s>%s</%s>", params.String(), vt[i], params.String())
		}
		return vt, nil
	}

	return src.Interface(), nil
}

func sprintf(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter split nil params")
	}
	return fmt.Sprintf(params.String(), src.Interface()), nil
}
func sprintfmap(src *reflect.Value, params *reflect.Value) (interface{}, error) {
	if params == nil {
		return src.Interface(), errors.New("filter split nil params")
	}
	msrc, ok := src.Interface().(map[string]interface{})
	if ok == false {
		return src.Interface(), errors.New("value is not map[string]interface{}")
	}
	vt := strings.Split(params.String(), ",")
	if len(vt) <= 1 {
		return src.Interface(), errors.New("params length must > 1")
	}
	p_array := []interface{}{}
	for _, x := range vt[1:] {
		if vm, ok := msrc[x]; ok {
			p_array = append(p_array, vm)
		}
	}
	return fmt.Sprintf(vt[0], p_array), nil
}
<|endoftext|>


<commit_before>package sqlstorage

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/facette/logger"
	"github.com/facette/maputil"
	"github.com/jinzhu/gorm"
	"github.com/pkg/errors"
)

// Storage represents a storage instance.
type Storage struct {
	name         string
	config       *maputil.Map
	log          *logger.Logger
	driver       sqlDriver
	db           *gorm.DB
	associations map[reflect.Type]string
}

// NewStorage creates a new storage instance.
func NewStorage(name string, config *maputil.Map, log *logger.Logger) (*Storage, error) {
	// Open database connection
	driver, err := newSQLDriver(name, config)
	if err != nil {
		return nil, err
	}

	sqlDB, err := driver.Open()
	if err != nil {
		return nil, errors.Wrap(err, "failed to open database")
	}

	db, err := gorm.Open(driver.Name(), sqlDB)
	if err != nil {
		return nil, errors.Wrap(err, "failed to open database")
	}

	// Set ORM logger and debugging mode
	db.SetLogger(&ormLogger{
		log.Context(log.CurrentContext() + "[sql]").Logger(logger.LevelDebug),
	})

	if debug, _ := config.GetBool("debug", false); debug {
		db.LogMode(true)
	}

	// Execute driver-specific commands
	if err := driver.Init(db); err != nil {
		return nil, errors.Wrap(err, "failed to initialize driver")
	}

	return &Storage{
		name:         name,
		config:       config,
		log:          log,
		driver:       driver,
		db:           db,
		associations: make(map[reflect.Type]string),
	}, nil
}

// Close closes the storage database connection.
func (s *Storage) Close() error {
	return s.db.Close()
}

// Migrate handles automatic migration of given item database models.
func (s *Storage) Migrate(v ...interface{}) error {
	return s.db.AutoMigrate(v...).Error
}

// AddForeignKey defines a new foreign key for a given item.
func (s *Storage) AddForeignKey(v interface{}, field, dest, onDelete, onUpdate string) *Storage {
	s.db.Model(v).AddForeignKey(field, dest, onDelete, onUpdate)
	return s
}

// Association registers a new item association field.
func (s *Storage) Association(v interface{}, field string) *Storage {
	s.associations[reflect.TypeOf(v)] = field
	return s
}

// Create stores a new item into the storage.
func (s *Storage) Create(v interface{}) error {
	tx := s.db.Begin()
	defer tx.Commit()

	if err := tx.Create(v).Error; err != nil {
		return s.driver.NormalizeError(err)
	}

	return nil
}

// Get retrieves an existing item from the storage.
func (s *Storage) Get(column string, values interface{}, v interface{}) error {
	tx := s.db.Begin()
	defer tx.Commit()

	// Retrieve item from database
	whereClause := tx.Dialect().Quote(column)
	if reflect.TypeOf(values).Kind() == reflect.Slice {
		whereClause += " IN (?)"
	} else {
		whereClause += " = ?"
	}

	if err := tx.Where(whereClause, values).Find(v).Error; err == gorm.ErrRecordNotFound {
		return ErrItemNotFound
	} else if err != nil {
		return s.driver.NormalizeError(err)
	}

	// Retrieve item-specific associations
	if field, ok := s.associations[reflect.TypeOf(v)]; ok {
		tx.Model(v).Association(field).Find(reflect.Indirect(reflect.ValueOf(v)).FieldByName(field).Addr().Interface())
	}

	return nil
}

// Update modifies an existing item from the storage.
func (s *Storage) Update(v interface{}) error {
	tx := s.db.Begin()
	defer tx.Commit()

	if err := tx.Save(v).Error; err != nil {
		return s.driver.NormalizeError(err)
	}

	return nil
}

// Delete removes an existing item from the storage.
func (s *Storage) Delete(v interface{}) error {
	tx := s.db.Begin()
	defer tx.Commit()

	// Delete item from database
	if count := tx.Delete(v).RowsAffected; count == 0 {
		return ErrItemNotFound
	}

	return nil
}

// Count returns the count of existing items from the storage.
func (s *Storage) Count(v interface{}) (int, error) {
	tx := s.db.Begin()
	defer tx.Commit()

	count := 0
	if err := tx.Model(v).Count(&count).Error; err != nil {
		return 0, err
	}

	return count, nil
}

// List retrieves a list of existing items from the storage.
func (s *Storage) List(v interface{}, filters map[string]interface{}, sort []string, offset, limit int) (int, error) {
	tx := s.db.Begin()
	defer tx.Commit()

	tx = tx.Model(v)

	for k, v := range filters {
		k, v = s.driver.WhereClause(k, v)
		tx = tx.Where(k, v)
	}

	count := 0
	if err := tx.Count(&count).Error; err != nil {
		return 0, err
	}

	for _, field := range sort {
		var desc bool

		if strings.HasPrefix(field, "-") {
			field = field[1:]
			desc = true
		}

		if !tx.Dialect().HasColumn(tx.NewScope(v).TableName(), field) {
			return 0, ErrUnknownColumn
		}

		if desc {
			tx = tx.Order(field + " DESC")
		} else {
			tx = tx.Order(field)
		}
	}

	if limit > 0 {
		tx = tx.Offset(offset).Limit(limit)
	}

	if err := tx.Find(v).Error; err != nil {
		return 0, s.driver.NormalizeError(err)
	}

	// Retrieve item-specific associations
	rv := reflect.ValueOf(v)
	if rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}

	if field, ok := s.associations[rv.Type().Elem()]; ok {
		// Reset filters, orders and limits
		tx = tx.New()

		for i, n := 0, rv.Len(); i < n; i++ {
			cur := rv.Index(i)

			tx.Model(cur.Interface()).Association(field).
				Find(reflect.Indirect(cur).FieldByName(field).Addr().Interface())
		}
	}

	return count, nil
}

// Search searches for existing items given multiple types in the storage.
func (s *Storage) Search(values []interface{}, v interface{}, filters map[string]interface{}, sort []string, offset,
	limit int) (int, error) {

	tx := s.db.Begin()
	defer tx.Commit()

	// Get columns list
	scope := tx.NewScope(reflect.Indirect(reflect.New(reflect.TypeOf(v).Elem())).Interface())

	columns := []string{fmt.Sprintf("? AS type")}
	for _, field := range scope.Fields() {
		if !field.IsIgnored {
			columns = append(columns, field.DBName)
		}
	}

	tx = tx.Select(columns)

	// Generate sub-queries
	queries := []string{}
	args := []interface{}{}

	for _, v := range values {
		scope := tx.NewScope(v)
		args = append(args, scope.TableName())

		for k, v := range filters {
			if scope.Dialect().HasColumn(scope.TableName(), k) {
				k, v = s.driver.WhereClause(k, v)
				scope.Search.Where(k, v)
				args = append(args, v)
			}
		}

		queries = append(queries, fmt.Sprintf(
			"SELECT %s FROM %s %s",
			strings.Join(scope.SelectAttrs(), ", "),
			scope.QuotedTableName(),
			scope.CombinedConditionSql(),
		))
	}

	tx1 := tx.Raw(fmt.Sprintf("SELECT COUNT(*) FROM (%s) AS a", strings.Join(queries, " UNION ALL ")), args...)

	count := 0
	if err := tx1.Count(&count).Error; err != nil {
		return 0, err
	}

	scope = tx.NewScope(nil)

	for _, field := range sort {
		var desc bool

		if strings.HasPrefix(field, "-") {
			field = field[1:]
			desc = true
		}

		if desc {
			scope.Search.Order(field + " DESC")
		} else {
			scope.Search.Order(field)
		}
	}

	if limit > 0 {
		tx = tx.Offset(offset).Limit(limit)
	}

	tx2 := tx.Raw(strings.Join(queries, " UNION ALL ")+scope.CombinedConditionSql(), args...)

	if err := tx2.Scan(v).Error; err != nil {
		return 0, s.driver.NormalizeError(err)
	}

	return count, nil
}
<commit_msg>Improve item handling<commit_after>package sqlstorage

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/facette/logger"
	"github.com/facette/maputil"
	"github.com/jinzhu/gorm"
	"github.com/pkg/errors"
)

// Storage represents a storage instance.
type Storage struct {
	name         string
	config       *maputil.Map
	log          *logger.Logger
	driver       sqlDriver
	db           *gorm.DB
	associations map[reflect.Type][]string
}

// NewStorage creates a new storage instance.
func NewStorage(name string, config *maputil.Map, log *logger.Logger) (*Storage, error) {
	// Open database connection
	driver, err := newSQLDriver(name, config)
	if err != nil {
		return nil, err
	}

	sqlDB, err := driver.Open()
	if err != nil {
		return nil, errors.Wrap(err, "failed to open database")
	}

	db, err := gorm.Open(driver.Name(), sqlDB)
	if err != nil {
		return nil, errors.Wrap(err, "failed to open database")
	}

	// Set ORM logger and debugging mode
	db.SetLogger(&ormLogger{
		log.Context(log.CurrentContext() + "[sql]").Logger(logger.LevelDebug),
	})

	if debug, _ := config.GetBool("debug", false); debug {
		db.LogMode(true)
	}

	// Execute driver-specific commands
	if err := driver.Init(db); err != nil {
		return nil, errors.Wrap(err, "failed to initialize driver")
	}

	return &Storage{
		name:         name,
		config:       config,
		log:          log,
		driver:       driver,
		db:           db,
		associations: make(map[reflect.Type][]string),
	}, nil
}

// Close closes the storage database connection.
func (s *Storage) Close() error {
	return s.db.Close()
}

// Migrate handles automatic migration of given item database models.
func (s *Storage) Migrate(v ...interface{}) error {
	return s.db.AutoMigrate(v...).Error
}

// AddForeignKey defines a new foreign key for a given item.
func (s *Storage) AddForeignKey(v interface{}, field, dest, onDelete, onUpdate string) *Storage {
	s.db.Model(v).AddForeignKey(field, dest, onDelete, onUpdate)
	return s
}

// Association registers a new item association field.
func (s *Storage) Association(v interface{}, fields ...string) *Storage {
	rt := reflect.TypeOf(v)
	for rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
	}

	if _, ok := s.associations[rt]; !ok {
		s.associations[rt] = []string{}
	}
	s.associations[rt] = append(s.associations[rt], fields...)

	return s
}

// Save stores a new item or modifies an existing item into the storage.
func (s *Storage) Save(v interface{}) error {
	var err error

	tx := s.db.Begin()
	defer tx.Commit()

	// Check if item with the given primary field already exists
	field := tx.NewScope(v).PrimaryField()
	value := reflect.Indirect(reflect.ValueOf(v)).FieldByName(field.Name).Interface()

	count := 0
	if err := tx.Model(v).Where(field.DBName+" = ?", value).Count(&count).Error; err != nil {
		return s.driver.NormalizeError(err)
	}

	if count == 0 {
		err = tx.Create(v).Error
	} else {
		err = tx.Save(v).Error
	}

	if err != nil {
		return s.driver.NormalizeError(err)
	}

	return nil
}

// Get retrieves an existing item from the storage.
func (s *Storage) Get(column string, values interface{}, v interface{}) error {
	tx := s.db.Begin()
	defer tx.Commit()

	// Retrieve item from database
	whereClause := tx.Dialect().Quote(column)
	if reflect.TypeOf(values).Kind() == reflect.Slice {
		whereClause += " IN (?)"
	} else {
		whereClause += " = ?"
	}

	if err := tx.Where(whereClause, values).Find(v).Error; err == gorm.ErrRecordNotFound {
		return ErrItemNotFound
	} else if err != nil {
		return s.driver.NormalizeError(err)
	}

	// Retrieve item-specific associations
	s.applyAssociations(tx, v)

	return nil
}

// Delete removes an existing item from the storage.
func (s *Storage) Delete(v interface{}) error {
	tx := s.db.Begin()
	defer tx.Commit()

	// Delete item from database
	if count := tx.Delete(v).RowsAffected; count == 0 {
		return ErrItemNotFound
	}

	return nil
}

// Count returns the count of existing items from the storage.
func (s *Storage) Count(v interface{}) (int, error) {
	tx := s.db.Begin()
	defer tx.Commit()

	count := 0
	if err := tx.Model(v).Count(&count).Error; err != nil {
		return 0, err
	}

	return count, nil
}

// List retrieves a list of existing items from the storage.
func (s *Storage) List(v interface{}, filters map[string]interface{}, sort []string, offset, limit int) (int, error) {
	tx := s.db.Begin()
	defer tx.Commit()

	tx = tx.Model(v)

	for k, v := range filters {
		k, v = s.driver.WhereClause(k, v)
		tx = tx.Where(k, v)
	}

	count := 0
	if err := tx.Count(&count).Error; err != nil {
		return 0, err
	}

	for _, field := range sort {
		var desc bool

		if strings.HasPrefix(field, "-") {
			field = field[1:]
			desc = true
		}

		if !tx.Dialect().HasColumn(tx.NewScope(v).TableName(), field) {
			return 0, ErrUnknownColumn
		}

		if desc {
			tx = tx.Order(field + " DESC")
		} else {
			tx = tx.Order(field)
		}
	}

	if limit > 0 {
		tx = tx.Offset(offset).Limit(limit)
	}

	if err := tx.Find(v).Error; err != nil {
		return 0, s.driver.NormalizeError(err)
	}

	// Retrieve item-specific associations
	rv := reflect.ValueOf(v)
	for rv.Kind() == reflect.Ptr {
		rv = rv.Elem()
	}

	if _, ok := s.associations[rv.Type().Elem()]; ok {
		// Reset filters, orders and limits
		tx = tx.New()

		for i, n := 0, rv.Len(); i < n; i++ {
			s.applyAssociations(tx, rv.Index(i).Interface())
		}
	}

	return count, nil
}

// Search searches for existing items given multiple types in the storage.
func (s *Storage) Search(values []interface{}, v interface{}, filters map[string]interface{}, sort []string, offset,
	limit int) (int, error) {

	tx := s.db.Begin()
	defer tx.Commit()

	// Get columns list
	scope := tx.NewScope(reflect.Indirect(reflect.New(reflect.TypeOf(v).Elem())).Interface())

	columns := []string{fmt.Sprintf("? AS type")}
	for _, field := range scope.Fields() {
		if !field.IsIgnored {
			columns = append(columns, field.DBName)
		}
	}

	tx = tx.Select(columns)

	// Generate sub-queries
	queries := []string{}
	args := []interface{}{}

	for _, v := range values {
		scope := tx.NewScope(v)
		args = append(args, scope.TableName())

		for k, v := range filters {
			if scope.Dialect().HasColumn(scope.TableName(), k) {
				k, v = s.driver.WhereClause(k, v)
				scope.Search.Where(k, v)
				args = append(args, v)
			}
		}

		queries = append(queries, fmt.Sprintf(
			"SELECT %s FROM %s %s",
			strings.Join(scope.SelectAttrs(), ", "),
			scope.QuotedTableName(),
			scope.CombinedConditionSql(),
		))
	}

	tx1 := tx.Raw(fmt.Sprintf("SELECT COUNT(*) FROM (%s) AS a", strings.Join(queries, " UNION ALL ")), args...)

	count := 0
	if err := tx1.Count(&count).Error; err != nil {
		return 0, err
	}

	scope = tx.NewScope(nil)

	for _, field := range sort {
		var desc bool

		if strings.HasPrefix(field, "-") {
			field = field[1:]
			desc = true
		}

		if desc {
			scope.Search.Order(field + " DESC")
		} else {
			scope.Search.Order(field)
		}
	}

	if limit > 0 {
		tx = tx.Offset(offset).Limit(limit)
	}

	tx2 := tx.Raw(strings.Join(queries, " UNION ALL ")+scope.CombinedConditionSql(), args...)

	if err := tx2.Scan(v).Error; err != nil {
		return 0, s.driver.NormalizeError(err)
	}

	return count, nil
}

func (s *Storage) applyAssociations(tx *gorm.DB, v interface{}) {
	rv := reflect.ValueOf(v)

	rt := rv.Type()
	for rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
	}

	if fieldNames, ok := s.associations[rt]; ok {
		for _, name := range fieldNames {
			tx.Model(v).Association(name).Find(reflect.Indirect(rv).FieldByName(name).Addr().Interface())
		}
	}
}
<|endoftext|>