<commit_before>package gherkin

import (
	"regexp"
	"strings"
)

const (
	DEFAULT_DIALECT                 = "en"
	COMMENT_PREFIX                  = "#"
	TAG_PREFIX                      = "@"
	TITLE_KEYWORD_SEPARATOR         = ":"
	TABLE_CELL_SEPARATOR            = "|"
	DOCSTRING_SEPARATOR             = "\"\"\""
	DOCSTRING_ALTERNATIVE_SEPARATOR = "```"
)

type matcher struct {
	gdp                      GherkinDialectProvider
	lang                     string
	dialect                  *GherkinDialect
	activeDocStringSeparator string
	indentToRemove           int
	languagePattern          *regexp.Regexp
}

func NewMatcher(gdp GherkinDialectProvider) Matcher {
	return &matcher{
		gdp:             gdp,
		lang:            DEFAULT_DIALECT,
		dialect:         gdp.GetDialect(DEFAULT_DIALECT),
		languagePattern: regexp.MustCompile("^\\s*#\\s*language\\s*:\\s*([a-zA-Z\\-_]+)\\s*$"),
	}
}

func (m *matcher) newTokenAtLocation(line, index int) (token *Token) {
	column := index + 1
	token = new(Token)
	token.GherkinDialect = m.lang
	token.Location = &Location{line, column}
	return
}

func (m *matcher) MatchEOF(line *Line) (ok bool, token *Token, err error) {
	if line.IsEof() {
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_EOF
	}
	return
}

func (m *matcher) MatchEmpty(line *Line) (ok bool, token *Token, err error) {
	if line.IsEmpty() {
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_Empty
	}
	return
}

func (m *matcher) MatchComment(line *Line) (ok bool, token *Token, err error) {
	if line.StartsWith(COMMENT_PREFIX) {
		token, ok = m.newTokenAtLocation(line.LineNumber, 0), true
		token.Type = TokenType_Comment
		token.Text = line.LineText
	}
	return
}

func (m *matcher) MatchTagLine(line *Line) (ok bool, token *Token, err error) {
	if line.StartsWith(TAG_PREFIX) {
		var tags []*LineSpan
		var column = line.Indent()
		splits := strings.Split(line.TrimmedLineText, TAG_PREFIX)
		for i := range splits {
			txt := strings.Trim(splits[i], " ")
			if txt != "" {
				tags = append(tags, &LineSpan{column, TAG_PREFIX + txt})
			}
			column = column + len(splits[i]) + 1
		}

		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_TagLine
		token.Items = tags
	}
	return
}

func (m *matcher) matchTitleLine(line *Line, tokenType TokenType, keywords []string) (ok bool, token *Token, err error) {
	for i := range keywords {
		keyword := keywords[i]
		if line.StartsWith(keyword + TITLE_KEYWORD_SEPARATOR) {
			token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
			token.Type = tokenType
			token.Keyword = keyword
			token.Text = strings.Trim(line.TrimmedLineText[len(keyword)+1:], " ")
			return
		}
	}
	return
}

func (m *matcher) MatchFeatureLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_FeatureLine, m.dialect.FeatureKeywords())
}
func (m *matcher) MatchBackgroundLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_BackgroundLine, m.dialect.BackgroundKeywords())
}
func (m *matcher) MatchScenarioLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_ScenarioLine, m.dialect.ScenarioKeywords())
}
func (m *matcher) MatchScenarioOutlineLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_ScenarioOutlineLine, m.dialect.ScenarioOutlineKeywords())
}
func (m *matcher) MatchExamplesLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_ExamplesLine, m.dialect.ExamplesKeywords())
}
func (m *matcher) MatchStepLine(line *Line) (ok bool, token *Token, err error) {
	keywords := m.dialect.StepKeywords()
	for i := range keywords {
		keyword := keywords[i]
		if line.StartsWith(keyword) {
			token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
			token.Type = TokenType_StepLine
			token.Keyword = keyword
			token.Text = strings.Trim(line.TrimmedLineText[len(keyword):], " ")
			return
		}
	}
	return
}

func (m *matcher) MatchDocStringSeparator(line *Line) (ok bool, token *Token, err error) {
	if m.activeDocStringSeparator != "" {
		if line.StartsWith(m.activeDocStringSeparator) {
			// close
			token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
			token.Type = TokenType_DocStringSeparator

			m.indentToRemove = 0
			m.activeDocStringSeparator = ""
		}
		return
	}
	if line.StartsWith(DOCSTRING_SEPARATOR) {
		m.activeDocStringSeparator = DOCSTRING_SEPARATOR
	} else if line.StartsWith(DOCSTRING_ALTERNATIVE_SEPARATOR) {
		m.activeDocStringSeparator = DOCSTRING_ALTERNATIVE_SEPARATOR
	}
	if m.activeDocStringSeparator != "" {
		// open
		contentType := line.TrimmedLineText[len(m.activeDocStringSeparator):]
		m.indentToRemove = line.Indent()
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_DocStringSeparator
		token.Text = contentType
	}
	return
}

func (m *matcher) MatchTableRow(line *Line) (ok bool, token *Token, err error) {
	if line.StartsWith(TABLE_CELL_SEPARATOR) {
		var cells []*LineSpan
		var column = line.Indent() + 1
		ttxt := strings.Trim(line.TrimmedLineText, " ")
		splits := strings.Split(ttxt[1:len(ttxt)-1], TABLE_CELL_SEPARATOR)
		for i := range splits {
			element := splits[i]
			txt := strings.TrimLeft(element, " ")
			ind := len(element) - len(txt)
			cells = append(cells, &LineSpan{column + ind + 1, strings.TrimRight(txt, " ")})
			column = column + len(element) + 1
		}

		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_TableRow
		token.Items = cells
	}
	return
}

func (m *matcher) MatchLanguage(line *Line) (ok bool, token *Token, err error) {
	matches := m.languagePattern.FindStringSubmatch(line.TrimmedLineText)
	if len(matches) > 0 {
		lang := matches[1]
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_Language
		token.Text = lang

		dialect := m.gdp.GetDialect(lang)
		if dialect == nil {
			err = &parseError{"Language not supported: " + lang, token.Location}
		} else {
			m.lang = lang
			m.dialect = dialect
		}
	}
	return
}

func (m *matcher) MatchOther(line *Line) (ok bool, token *Token, err error) {
	token, ok = m.newTokenAtLocation(line.LineNumber, 0), true
	token.Type = TokenType_Other

	element := line.LineText
	txt := strings.TrimLeft(element, " ")

	if len(element)-len(txt) > m.indentToRemove {
		token.Text = element[m.indentToRemove:]
	} else {
		token.Text = txt
	}
	return
}
<commit_msg>Implement DocString escaping for all languages<commit_after>package gherkin

import (
	"regexp"
	"strings"
)

const (
	DEFAULT_DIALECT                 = "en"
	COMMENT_PREFIX                  = "#"
	TAG_PREFIX                      = "@"
	TITLE_KEYWORD_SEPARATOR         = ":"
	TABLE_CELL_SEPARATOR            = "|"
	DOCSTRING_SEPARATOR             = "\"\"\""
	DOCSTRING_ALTERNATIVE_SEPARATOR = "```"
)

type matcher struct {
	gdp                      GherkinDialectProvider
	lang                     string
	dialect                  *GherkinDialect
	activeDocStringSeparator string
	indentToRemove           int
	languagePattern          *regexp.Regexp
}

func NewMatcher(gdp GherkinDialectProvider) Matcher {
	return &matcher{
		gdp:             gdp,
		lang:            DEFAULT_DIALECT,
		dialect:         gdp.GetDialect(DEFAULT_DIALECT),
		languagePattern: regexp.MustCompile("^\\s*#\\s*language\\s*:\\s*([a-zA-Z\\-_]+)\\s*$"),
	}
}

func (m *matcher) newTokenAtLocation(line, index int) (token *Token) {
	column := index + 1
	token = new(Token)
	token.GherkinDialect = m.lang
	token.Location = &Location{line, column}
	return
}

func (m *matcher) MatchEOF(line *Line) (ok bool, token *Token, err error) {
	if line.IsEof() {
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_EOF
	}
	return
}

func (m *matcher) MatchEmpty(line *Line) (ok bool, token *Token, err error) {
	if line.IsEmpty() {
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_Empty
	}
	return
}

func (m *matcher) MatchComment(line *Line) (ok bool, token *Token, err error) {
	if line.StartsWith(COMMENT_PREFIX) {
		token, ok = m.newTokenAtLocation(line.LineNumber, 0), true
		token.Type = TokenType_Comment
		token.Text = line.LineText
	}
	return
}

func (m *matcher) MatchTagLine(line *Line) (ok bool, token *Token, err error) {
	if line.StartsWith(TAG_PREFIX) {
		var tags []*LineSpan
		var column = line.Indent()
		splits := strings.Split(line.TrimmedLineText, TAG_PREFIX)
		for i := range splits {
			txt := strings.Trim(splits[i], " ")
			if txt != "" {
				tags = append(tags, &LineSpan{column, TAG_PREFIX + txt})
			}
			column = column + len(splits[i]) + 1
		}

		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_TagLine
		token.Items = tags
	}
	return
}

func (m *matcher) matchTitleLine(line *Line, tokenType TokenType, keywords []string) (ok bool, token *Token, err error) {
	for i := range keywords {
		keyword := keywords[i]
		if line.StartsWith(keyword + TITLE_KEYWORD_SEPARATOR) {
			token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
			token.Type = tokenType
			token.Keyword = keyword
			token.Text = strings.Trim(line.TrimmedLineText[len(keyword)+1:], " ")
			return
		}
	}
	return
}

func (m *matcher) MatchFeatureLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_FeatureLine, m.dialect.FeatureKeywords())
}
func (m *matcher) MatchBackgroundLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_BackgroundLine, m.dialect.BackgroundKeywords())
}
func (m *matcher) MatchScenarioLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_ScenarioLine, m.dialect.ScenarioKeywords())
}
func (m *matcher) MatchScenarioOutlineLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_ScenarioOutlineLine, m.dialect.ScenarioOutlineKeywords())
}
func (m *matcher) MatchExamplesLine(line *Line) (ok bool, token *Token, err error) {
	return m.matchTitleLine(line, TokenType_ExamplesLine, m.dialect.ExamplesKeywords())
}
func (m *matcher) MatchStepLine(line *Line) (ok bool, token *Token, err error) {
	keywords := m.dialect.StepKeywords()
	for i := range keywords {
		keyword := keywords[i]
		if line.StartsWith(keyword) {
			token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
			token.Type = TokenType_StepLine
			token.Keyword = keyword
			token.Text = strings.Trim(line.TrimmedLineText[len(keyword):], " ")
			return
		}
	}
	return
}

func (m *matcher) MatchDocStringSeparator(line *Line) (ok bool, token *Token, err error) {
	if m.activeDocStringSeparator != "" {
		if line.StartsWith(m.activeDocStringSeparator) {
			// close
			token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
			token.Type = TokenType_DocStringSeparator

			m.indentToRemove = 0
			m.activeDocStringSeparator = ""
		}
		return
	}
	if line.StartsWith(DOCSTRING_SEPARATOR) {
		m.activeDocStringSeparator = DOCSTRING_SEPARATOR
	} else if line.StartsWith(DOCSTRING_ALTERNATIVE_SEPARATOR) {
		m.activeDocStringSeparator = DOCSTRING_ALTERNATIVE_SEPARATOR
	}
	if m.activeDocStringSeparator != "" {
		// open
		contentType := line.TrimmedLineText[len(m.activeDocStringSeparator):]
		m.indentToRemove = line.Indent()
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_DocStringSeparator
		token.Text = contentType
	}
	return
}

func (m *matcher) MatchTableRow(line *Line) (ok bool, token *Token, err error) {
	if line.StartsWith(TABLE_CELL_SEPARATOR) {
		var cells []*LineSpan
		var column = line.Indent() + 1
		ttxt := strings.Trim(line.TrimmedLineText, " ")
		splits := strings.Split(ttxt[1:len(ttxt)-1], TABLE_CELL_SEPARATOR)
		for i := range splits {
			element := splits[i]
			txt := strings.TrimLeft(element, " ")
			ind := len(element) - len(txt)
			cells = append(cells, &LineSpan{column + ind + 1, strings.TrimRight(txt, " ")})
			column = column + len(element) + 1
		}

		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_TableRow
		token.Items = cells
	}
	return
}

func (m *matcher) MatchLanguage(line *Line) (ok bool, token *Token, err error) {
	matches := m.languagePattern.FindStringSubmatch(line.TrimmedLineText)
	if len(matches) > 0 {
		lang := matches[1]
		token, ok = m.newTokenAtLocation(line.LineNumber, line.Indent()), true
		token.Type = TokenType_Language
		token.Text = lang

		dialect := m.gdp.GetDialect(lang)
		if dialect == nil {
			err = &parseError{"Language not supported: " + lang, token.Location}
		} else {
			m.lang = lang
			m.dialect = dialect
		}
	}
	return
}

func (m *matcher) MatchOther(line *Line) (ok bool, token *Token, err error) {
	token, ok = m.newTokenAtLocation(line.LineNumber, 0), true
	token.Type = TokenType_Other

	element := line.LineText
	txt := strings.TrimLeft(element, " ")

	if len(element)-len(txt) > m.indentToRemove {
		token.Text = unescapeDocString(element[m.indentToRemove:])
	} else {
		token.Text = unescapeDocString(txt)
	}
	return
}

func unescapeDocString(text string) string {
	return strings.Replace(text, "\\\"\\\"\\\"", "\"\"\"", -1)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/davecgh/go-spew/spew"
	"github.com/gophergala2016/Pulse/LogPulse/api"
	"github.com/gophergala2016/Pulse/LogPulse/config"
	"github.com/gophergala2016/Pulse/LogPulse/email"
	"github.com/gophergala2016/Pulse/LogPulse/file"
	"github.com/gophergala2016/Pulse/pulse"
)

var (
	runAPI      bool
	outputFile  string
	buffStrings []string
	logList     []string
)

func init() {
	flag.BoolVar(&runAPI, "api", false, "Turn on API mode")
	flag.Parse()

	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(0)
		}
	}()

	cfg, err := config.Load()
	if err != nil {
		panic(fmt.Errorf("main.init: Could not load the config.\n %v", err))
	}

	logList = cfg.LogList
	outputFile = cfg.OutputFile
}

func main() {
	//uncomment for production

	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(0)
		}
	}()

	if len(flag.Args()) == 0 && !runAPI {
		if len(logList) == 0 {
			panic(fmt.Errorf("main.main: Must supply a list of log files in the config"))
		}
		startPulse(logList)
	} else if runAPI {
		startAPI()
	} else {
		startPulse(flag.Args())
	}
}

func startAPI() {
	api.Start()
}

func startPulse(filenames []string) {
	spew.Dump(filenames)
	checkList(filenames)
	spew.Dump(filenames)
	stdIn := make(chan string)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	// On keyboard interrup cleanup the program
	go func() {
		for _ = range c {
			fmt.Println("Exiting for Keyboard Interupt")
			cleanUp()
			os.Exit(0)
		}
	}()

	defer cleanUp()

	pulse.Run(stdIn, email.Send)
	for _, filename := range filenames {
		line := make(chan string)
		file.Read(filename, line)
		for l := range line {
			stdIn <- l
		}
	}
	close(stdIn)
}

func cleanUp() {
	email.DumpBuffer()
}

func checkList(filenames []string) {
	for i, filename := range filenames {
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			panic(fmt.Errorf("main.checkList: %s", err))
		}
		if len(filename) > 3 && filename[len(filename)-3:len(filename)] == ".gz" {
			if err := file.UnGZip(filename); err != nil {
				panic(fmt.Errorf("main.checkList: %s", err))
			}
			if _, err := os.Stat(filename[:len(filename)-3]); os.IsNotExist(err) {
				panic(fmt.Errorf("main.checkList: %s", err))
			}
			filenames[i] = filename[:len(filename)-3]
		}
	}
}
<commit_msg>Remove Debug messages<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/gophergala2016/Pulse/LogPulse/api"
	"github.com/gophergala2016/Pulse/LogPulse/config"
	"github.com/gophergala2016/Pulse/LogPulse/email"
	"github.com/gophergala2016/Pulse/LogPulse/file"
	"github.com/gophergala2016/Pulse/pulse"
)

var (
	runAPI      bool
	outputFile  string
	buffStrings []string
	logList     []string
)

func init() {
	flag.BoolVar(&runAPI, "api", false, "Turn on API mode")
	flag.Parse()

	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(0)
		}
	}()

	cfg, err := config.Load()
	if err != nil {
		panic(fmt.Errorf("main.init: Could not load the config.\n %v", err))
	}

	logList = cfg.LogList
	outputFile = cfg.OutputFile
}

func main() {
	//uncomment for production

	defer func() {
		if r := recover(); r != nil {
			fmt.Println(r)
			os.Exit(0)
		}
	}()

	if len(flag.Args()) == 0 && !runAPI {
		if len(logList) == 0 {
			panic(fmt.Errorf("main.main: Must supply a list of log files in the config"))
		}
		startPulse(logList)
	} else if runAPI {
		startAPI()
	} else {
		startPulse(flag.Args())
	}
}

func startAPI() {
	api.Start()
}

func startPulse(filenames []string) {
	checkList(filenames)
	stdIn := make(chan string)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	// On keyboard interrup cleanup the program
	go func() {
		for _ = range c {
			fmt.Println("Exiting for Keyboard Interupt")
			cleanUp()
			os.Exit(0)
		}
	}()

	defer cleanUp()

	pulse.Run(stdIn, email.Send)
	for _, filename := range filenames {
		line := make(chan string)
		file.Read(filename, line)
		for l := range line {
			stdIn <- l
		}
	}
	close(stdIn)
}

func cleanUp() {
	email.DumpBuffer()
}

func checkList(filenames []string) {
	for i, filename := range filenames {
		if _, err := os.Stat(filename); os.IsNotExist(err) {
			panic(fmt.Errorf("main.checkList: %s", err))
		}
		if len(filename) > 3 && filename[len(filename)-3:len(filename)] == ".gz" {
			if err := file.UnGZip(filename); err != nil {
				panic(fmt.Errorf("main.checkList: %s", err))
			}
			if _, err := os.Stat(filename[:len(filename)-3]); os.IsNotExist(err) {
				panic(fmt.Errorf("main.checkList: %s", err))
			}
			filenames[i] = filename[:len(filename)-3]
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"fmt"
	"flag"
	"time"
	"os"
	"net/http"
	"crypto/tls"
	"strings"
	"io"
)

// scope github.com/vgeshel/s3-parallel-download github.com/mitchellh/goamz/aws github.com/mitchellh/goamz/s3 fmt flag time os net/http crypto/tls

func main() {
	// dir := flag.String("dir", ".", "output directory")
	region := flag.String("region", aws.USEast.Name, "region")
	verbose := flag.Bool("v", false, "verbose")
	insecure := flag.Bool("insecure", false, "turn on InsecureSkipVerify: http://golang.org/pkg/crypto/tls/#Config")
	//max_attempts := flag.Int64("attempts", 5, "the maximum number of attempts to read the object")
	//delay := flag.Int64("delay", 5, "seconds to sleep between attempts")

	flag.Parse()

	auth, err := aws.GetAuth("", "")

	if err != nil {
		fmt.Fprintf(os.Stderr, "auth error: %#v\n", err)
		os.Exit(10)
	}

	if *verbose {
		fmt.Fprintf(os.Stderr, "auth: %#v\n", auth)
	}

	reg, found := aws.Regions[*region]

	if ! found {
		fmt.Fprintf(os.Stderr, "invalid region %s\n", *region)
		os.Exit(11)
	}

	s3c := s3.New(auth, reg)

	done := make(chan string)
	before := time.Now()
	cnt := 0

	for i := 0; i < flag.NArg(); i ++ {
		path := flag.Arg(i)
		split := strings.SplitN(path, "/", 2)
		bucket := split[0]
		key := split[1]
		cnt ++

		go doGet(s3c, bucket, key, done, *insecure, before, *verbose)
	}

	if cnt > 0 {
		for {
			donePath := <-done;
			fmt.Printf("done %s in %v\n", donePath, time.Now().Sub(before))
			cnt --

			if cnt <= 0 {
				break
			}
		}
	}

	fmt.Printf("done ALL in %v\n", time.Now().Sub(before))

}

func doGet(s3c *s3.S3, bucket string, key string, done chan string, insecure bool, before time.Time, verbose bool) {
	path := bucket + "/" + key

	if verbose {
		fmt.Printf("starting %s in %v\n", path, time.Now().Sub(before))
	}

	if insecure {
		tr := &http.Transport{
			TLSClientConfig:    &tls.Config{InsecureSkipVerify: true},
		}
		http.DefaultClient.Transport = tr
	}

	buck := s3c.Bucket(bucket)
	reader, err := buck.GetReader(key)

	if err != nil {
		done <- "ERROR " + fmt.Sprintf("%v", err) + ": " + bucket + "/" + key
	} else {
		writer, err := os.Create("/dev/null")

		if err != nil {
			panic(err)
		}

		if verbose {
			fmt.Printf("starting copy %s in %v\n", path, time.Now().Sub(before))
		}

		io.Copy(writer, reader)

		done <- bucket + "/" + key
	}

}
<commit_msg>less output<commit_after>package main

import (
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"fmt"
	"flag"
	"time"
	"os"
	"net/http"
	"crypto/tls"
	"strings"
	"io"
)

// scope github.com/vgeshel/s3-parallel-download github.com/mitchellh/goamz/aws github.com/mitchellh/goamz/s3 fmt flag time os net/http crypto/tls

func main() {
	// dir := flag.String("dir", ".", "output directory")
	region := flag.String("region", aws.USEast.Name, "region")
	verbose := flag.Bool("v", false, "verbose")
	insecure := flag.Bool("insecure", false, "turn on InsecureSkipVerify: http://golang.org/pkg/crypto/tls/#Config")
	//max_attempts := flag.Int64("attempts", 5, "the maximum number of attempts to read the object")
	//delay := flag.Int64("delay", 5, "seconds to sleep between attempts")

	flag.Parse()

	auth, err := aws.GetAuth("", "")

	if err != nil {
		fmt.Fprintf(os.Stderr, "auth error: %#v\n", err)
		os.Exit(10)
	}

	if *verbose {
		fmt.Fprintf(os.Stderr, "auth: %#v\n", auth)
	}

	reg, found := aws.Regions[*region]

	if ! found {
		fmt.Fprintf(os.Stderr, "invalid region %s\n", *region)
		os.Exit(11)
	}

	s3c := s3.New(auth, reg)

	done := make(chan string)
	before := time.Now()
	cnt := 0

	for i := 0; i < flag.NArg(); i ++ {
		path := flag.Arg(i)
		split := strings.SplitN(path, "/", 2)
		bucket := split[0]
		key := split[1]
		cnt ++

		go doGet(s3c, bucket, key, done, *insecure, before, *verbose)
	}

	if cnt > 0 {
		for {
			donePath := <-done;

			if *verbose {
				fmt.Printf("done %s in %v\n", donePath, time.Now().Sub(before))
			}

			cnt --

			if cnt <= 0 {
				break
			}
		}
	}

	fmt.Printf("done ALL in %v\n", time.Now().Sub(before))

}

func doGet(s3c *s3.S3, bucket string, key string, done chan string, insecure bool, before time.Time, verbose bool) {
	path := bucket + "/" + key

	if verbose {
		fmt.Printf("starting %s in %v\n", path, time.Now().Sub(before))
	}

	if insecure {
		tr := &http.Transport{
			TLSClientConfig:    &tls.Config{InsecureSkipVerify: true},
		}
		http.DefaultClient.Transport = tr
	}

	buck := s3c.Bucket(bucket)
	reader, err := buck.GetReader(key)

	if err != nil {
		done <- "ERROR " + fmt.Sprintf("%v", err) + ": " + bucket + "/" + key
	} else {
		writer, err := os.Create("/dev/null")

		if err != nil {
			panic(err)
		}

		if verbose {
			fmt.Printf("starting copy %s in %v\n", path, time.Now().Sub(before))
		}

		io.Copy(writer, reader)

		done <- bucket + "/" + key
	}

}
<|endoftext|>


<commit_before>package downloader

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"time"

	"github.com/macarrie/flemzerd/configuration"
	"github.com/macarrie/flemzerd/db"
	log "github.com/macarrie/flemzerd/logging"
	"github.com/macarrie/flemzerd/notifiers"
	. "github.com/macarrie/flemzerd/objects"

	"github.com/rs/xid"
)

var downloadersCollection []Downloader

func AddDownloader(d Downloader) {
	downloadersCollection = append(downloadersCollection, d)
}

func Status() ([]Module, error) {
	var modList []Module
	var aggregatedErrorMessage bytes.Buffer

	for _, downloader := range downloadersCollection {
		mod, downloaderAliveError := downloader.Status()
		if downloaderAliveError != nil {
			log.WithFields(log.Fields{
				"error": downloaderAliveError,
			}).Warning("Downloader is not alive")
			aggregatedErrorMessage.WriteString(downloaderAliveError.Error())
			aggregatedErrorMessage.WriteString("\n")
		}
		modList = append(modList, mod)
	}

	var retError error
	if aggregatedErrorMessage.Len() == 0 {
		retError = nil
	} else {
		retError = errors.New(aggregatedErrorMessage.String())
	}
	return modList, retError
}

func Reset() {
	downloadersCollection = []Downloader{}
}

func AddTorrent(t Torrent) (string, error) {
	if len(downloadersCollection) == 0 {
		return "", errors.New("Cannot add torrents, no downloaders are configured")
	}

	id, err := downloadersCollection[0].AddTorrent(t)
	if err != nil {
		return "", err
	}

	return id, nil
}

func AddTorrentMapping(flemzerId string, downloaderId string) {
	downloadersCollection[0].AddTorrentMapping(flemzerId, downloaderId)
}

func StartTorrent(t Torrent) error {
	return nil
}

func RemoveTorrent(t Torrent) error {
	if len(downloadersCollection) == 0 {
		return errors.New("Cannot remove torrents, no downloaders are configured")
	}

	return downloadersCollection[0].RemoveTorrent(t)
}

func GetTorrentStatus(t Torrent) (int, error) {
	return downloadersCollection[0].GetTorrentStatus(t)
}

func EpisodeHandleTorrentDownload(e *Episode, recovery bool) error {
	torrent := e.DownloadingItem.CurrentTorrent
	if !recovery {
		torrentId, err := AddTorrent(torrent)
		if err != nil {
			return errors.New("Couldn't add torrent in downloader. Skipping to next torrent in list")
		}
		e.DownloadingItem.CurrentTorrent = torrent
		e.DownloadingItem.CurrentDownloaderId = torrentId
		db.Client.Save(&e)
	}

	StartTorrent(torrent)

	retryCount := 0

	// Try twice to download a torrent before marking it as rubbish
	downloadErr := WaitForDownload(torrent)
	if downloadErr != nil {
		log.WithFields(log.Fields{
			"error":   downloadErr,
			"torrent": torrent.Name,
		}).Debug("Error during torrent download. Retrying download")

		RemoveTorrent(torrent)
		AddTorrent(torrent)
		retryCount++
		retryErr := WaitForDownload(torrent)
		if retryErr != nil {
			RemoveTorrent(torrent)
			e.DownloadingItem.FailedTorrents = append(e.DownloadingItem.FailedTorrents, torrent)
			db.Client.Save(&e)

			log.WithFields(log.Fields{
				"error":   downloadErr,
				"torrent": torrent.Name,
			}).Debug("Error during torrent download. Finish current torrent download")

			return retryErr
		}
	}

	// If function has not returned yet, download ended with no errors !
	log.WithFields(log.Fields{
		"show":   e.TvShow.Name,
		"season": e.Season,
		"number": e.Number,
		"name":   e.Name,
	}).Info("Episode successfully downloaded")
	notifier.NotifyDownloadedEpisode(e)

	e.Downloaded = true
	e.DownloadingItem.Downloading = false
	err := MoveEpisodeToLibrary(e)
	if err != nil {
		log.WithFields(log.Fields{
			"show":           e.TvShow.Name,
			"episode":        e.Name,
			"season":         e.Season,
			"number":         e.Number,
			"temporary_path": e.DownloadingItem.CurrentTorrent.DownloadDir,
			"library_path":   configuration.Config.Library.ShowPath,
			"error":          err,
		}).Error("Could not move episode from temporay download path to library folder")
	}
	db.Client.Save(e)

	RemoveTorrent(torrent)

	return nil
}

func MovieHandleTorrentDownload(m *Movie, recovery bool) error {
	torrent := m.DownloadingItem.CurrentTorrent
	if !recovery {
		torrentId, err := AddTorrent(torrent)
		if err != nil {
			return errors.New("Couldn't add torrent in downloader. Skipping to next torrent in list")
		}

		m.DownloadingItem.CurrentTorrent = torrent
		m.DownloadingItem.CurrentDownloaderId = torrentId
		db.Client.Save(&m)
	}

	StartTorrent(torrent)

	retryCount := 0

	// Try twice to download a torrent before marking it as rubbish
	downloadErr := WaitForDownload(torrent)
	if downloadErr != nil {
		log.WithFields(log.Fields{
			"error":   downloadErr,
			"torrent": torrent.Name,
		}).Debug("Error during torrent download. Retrying download")

		RemoveTorrent(torrent)
		AddTorrent(torrent)
		retryCount++
		retryErr := WaitForDownload(torrent)
		if retryErr != nil {
			RemoveTorrent(torrent)
			m.DownloadingItem.FailedTorrents = append(m.DownloadingItem.FailedTorrents, torrent)
			db.Client.Save(&m)

			log.WithFields(log.Fields{
				"error":   downloadErr,
				"torrent": torrent.Name,
			}).Debug("Error during torrent download. Finish current torrent download")

			return retryErr
		}
	}

	// If function has not returned yet, download ended with no errors !
	log.WithFields(log.Fields{
		"name": m.Title,
	}).Info("Movie successfully downloaded")
	notifier.NotifyDownloadedMovie(m)

	m.Downloaded = true
	m.DownloadingItem.Downloading = false
	err := MoveMovieToLibrary(m)
	if err != nil {
		log.WithFields(log.Fields{
			"movie":          m.Title,
			"temporary_path": m.DownloadingItem.CurrentTorrent.DownloadDir,
			"library_path":   configuration.Config.Library.ShowPath,
			"error":          err,
		}).Error("Could not move movie from temporay download path to library folder")
	}
	db.Client.Save(m)

	RemoveTorrent(torrent)

	return nil
}

func WaitForDownload(t Torrent) error {
	downloadLoopTicker := time.NewTicker(1 * time.Minute)
	for {
		log.WithFields(log.Fields{
			"torrent": t.Name,
		}).Debug("Checking torrent download progress")

		status, err := GetTorrentStatus(t)
		if err != nil {
			return err
		}

		switch status {
		case TORRENT_STOPPED:
			return errors.New("Torrent stopped in download client")
		case TORRENT_SEEDING:
			// Download complete ! Return with no error
			return nil
		}
		<-downloadLoopTicker.C
	}
}

func DownloadEpisode(show TvShow, e Episode, torrentList []Torrent) error {
	if e.Downloaded || e.DownloadingItem.Downloading {
		return errors.New("Episode downloading or already downloaded. Skipping")
	}

	log.WithFields(log.Fields{
		"show":   show.Name,
		"season": e.Season,
		"number": e.Number,
		"name":   e.Name,
	}).Info("Starting download process")

	e.DownloadingItem.Downloading = true
	db.Client.Save(&e)

	for _, torrent := range torrentList {
		torrent.DownloadDir = fmt.Sprintf("%s/%s/", DOWNLOAD_TMP_DIR, xid.New())

		if db.TorrentHasFailed(e.DownloadingItem, torrent) {
			continue
		}

		e.DownloadingItem.CurrentTorrent = torrent
		db.Client.Save(&e)

		torrentDownload := EpisodeHandleTorrentDownload(&e, false)
		if torrentDownload != nil {
			log.WithFields(log.Fields{
				"err":     torrentDownload,
				"torrent": torrent.Name,
			}).Warning("Couldn't download torrent. Skipping to next torrent in list")
		} else {
			return nil
		}

		continue
	}

	// If function has not returned yet, it means the download failed
	if len(e.DownloadingItem.FailedTorrents) > configuration.Config.System.TorrentDownloadAttemptsLimit {
		MarkEpisodeFailedDownload(&show, &e)
		return errors.New("Download failed, no torrents could be downloaded")
	}

	return errors.New("No torrents in current torrent list could be downloaded")
}

func DownloadMovie(m Movie, torrentList []Torrent) error {
	if m.Downloaded || m.DownloadingItem.Downloading {
		return errors.New("Movie downloading or already downloaded. Skipping")
	}

	log.WithFields(log.Fields{
		"name": m.Title,
	}).Info("Starting download process")

	m.DownloadingItem.Downloading = true
	db.Client.Save(&m)

	for _, torrent := range torrentList {
		torrent.DownloadDir = fmt.Sprintf("%s/%s/", DOWNLOAD_TMP_DIR, xid.New())

		if db.TorrentHasFailed(m.DownloadingItem, torrent) {
			continue
		}

		m.DownloadingItem.CurrentTorrent = torrent
		db.Client.Save(&m)

		torrentDownload := MovieHandleTorrentDownload(&m, false)
		if torrentDownload != nil {
			log.WithFields(log.Fields{
				"err":     torrentDownload,
				"torrent": torrent.Name,
			}).Warning("Couldn't download torrent. Skipping to next torrent in list")
		} else {
			return nil
		}
		continue
	}

	// If function has not returned yet, it means the download failed
	if len(m.DownloadingItem.FailedTorrents) > configuration.Config.System.TorrentDownloadAttemptsLimit {
		MarkMovieFailedDownload(&m)
		return errors.New("Download failed, no torrents could be downloaded")
	}

	return errors.New("No torrents in current torrent list could be downloaded")
}

func MarkEpisodeFailedDownload(show *TvShow, e *Episode) {
	log.WithFields(log.Fields{
		"show":   show.Name,
		"season": e.Season,
		"number": e.Number,
		"name":   e.Name,
	}).Error("Download failed, no torrents could be downloaded")

	notifier.NotifyFailedEpisode(e)

	e.DownloadingItem.DownloadFailed = true
	e.DownloadingItem.Downloading = false
	db.Client.Save(&e)
}

func MarkMovieFailedDownload(m *Movie) {
	log.WithFields(log.Fields{
		"movie": m.Title,
	}).Error("Download failed, no torrents could be downloaded")

	notifier.NotifyFailedMovie(m)

	m.DownloadingItem.DownloadFailed = true
	m.DownloadingItem.Downloading = false
	db.Client.Save(&m)
}

func MoveEpisodeToLibrary(episode *Episode) error {
	log.WithFields(log.Fields{
		"show":           episode.TvShow.Name,
		"episode":        episode.Name,
		"season":         episode.Season,
		"number":         episode.Number,
		"temporary_path": episode.DownloadingItem.CurrentTorrent.DownloadDir,
		"library_path":   configuration.Config.Library.ShowPath,
	}).Debug("Moving episode to library")

	destinationPath := fmt.Sprintf("%s/%s/Season %d/s%de%d", configuration.Config.Library.ShowPath, episode.TvShow.Name, episode.Season, episode.Season, episode.Number)
	err := os.Rename(episode.DownloadingItem.CurrentTorrent.DownloadDir, destinationPath)
	if err != nil {
		return err
	}

	err = os.Remove(episode.DownloadingItem.CurrentTorrent.DownloadDir)
	if err != nil {
		log.WithFields(log.Fields{
			"path": episode.DownloadingItem.CurrentTorrent.DownloadDir,
		}).Warning("Could not remove temporary folder for download")
	}

	episode.DownloadingItem.CurrentTorrent.DownloadDir = destinationPath
	db.Client.Save(episode)

	return nil
}

func MoveMovieToLibrary(movie *Movie) error {
	log.WithFields(log.Fields{
		"movie":          movie.Title,
		"temporary_path": movie.DownloadingItem.CurrentTorrent.DownloadDir,
		"library_path":   configuration.Config.Library.ShowPath,
	}).Debug("Moving episode to library")

	destinationPath := fmt.Sprintf("%s/%s", configuration.Config.Library.MoviePath, movie.Title)
	err := os.Rename(movie.DownloadingItem.CurrentTorrent.DownloadDir, destinationPath)
	if err != nil {
		return err
	}

	err = os.Remove(movie.DownloadingItem.CurrentTorrent.DownloadDir)
	if err != nil {
		log.WithFields(log.Fields{
			"path": movie.DownloadingItem.CurrentTorrent.DownloadDir,
		}).Warning("Could not remove temporary folder for download")
	}

	movie.DownloadingItem.CurrentTorrent.DownloadDir = destinationPath
	db.Client.Save(movie)

	return nil
}

func FillEpisodeToDownloadTorrentList(e *Episode, list []Torrent) []Torrent {
	var torrentList []Torrent
	for _, torrent := range list {
		if !db.TorrentHasFailed(e.DownloadingItem, torrent) {
			torrentList = append(torrentList, torrent)
		}
	}

	if len(torrentList) < 10 {
		return torrentList
	} else {
		return torrentList[:10]
	}
}

func FillMovieToDownloadTorrentList(m *Movie, list []Torrent) []Torrent {
	var torrentList []Torrent
	for _, torrent := range list {
		if !db.TorrentHasFailed(m.DownloadingItem, torrent) {
			torrentList = append(torrentList, torrent)
		}
	}

	if len(torrentList) < 10 {
		return torrentList
	} else {
		return torrentList[:10]
	}
}

func RecoverFromRetention() {
	downloadingEpisodesFromRetention, err := db.GetDownloadingEpisodes()
	if err != nil {
		log.Error(err)
		return
	}
	downloadingMoviesFromRetention, err := db.GetDownloadingMovies()
	if err != nil {
		log.Error(err)
		return
	}

	if len(downloadingEpisodesFromRetention) != 0 {
		log.Debug("Launching watch threads for downloading episodes found in retention")
	}
	for _, ep := range downloadingEpisodesFromRetention {
		AddTorrentMapping(ep.DownloadingItem.CurrentTorrent.TorrentId, ep.DownloadingItem.CurrentDownloaderId)

		log.WithFields(log.Fields{
			"episode": ep.Name,
			"season":  ep.Season,
			"number":  ep.Number,
		}).Debug("Launched download processing recovery")

		go EpisodeHandleTorrentDownload(&ep, true)

	}

	if len(downloadingMoviesFromRetention) != 0 {
		log.Debug("Launching watch threads for downloading movies found in retention")
	}
	for _, m := range downloadingMoviesFromRetention {
		AddTorrentMapping(m.DownloadingItem.CurrentTorrent.TorrentId, m.DownloadingItem.CurrentDownloaderId)

		log.WithFields(log.Fields{
			"name": m.Title,
		}).Debug("Launched download processing recovery")

		go MovieHandleTorrentDownload(&m, true)
	}
}
<commit_msg>Added error in logs when removing tmp dir fter successful download<commit_after>package downloader

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"time"

	"github.com/macarrie/flemzerd/configuration"
	"github.com/macarrie/flemzerd/db"
	log "github.com/macarrie/flemzerd/logging"
	"github.com/macarrie/flemzerd/notifiers"
	. "github.com/macarrie/flemzerd/objects"

	"github.com/rs/xid"
)

var downloadersCollection []Downloader

func AddDownloader(d Downloader) {
	downloadersCollection = append(downloadersCollection, d)
}

func Status() ([]Module, error) {
	var modList []Module
	var aggregatedErrorMessage bytes.Buffer

	for _, downloader := range downloadersCollection {
		mod, downloaderAliveError := downloader.Status()
		if downloaderAliveError != nil {
			log.WithFields(log.Fields{
				"error": downloaderAliveError,
			}).Warning("Downloader is not alive")
			aggregatedErrorMessage.WriteString(downloaderAliveError.Error())
			aggregatedErrorMessage.WriteString("\n")
		}
		modList = append(modList, mod)
	}

	var retError error
	if aggregatedErrorMessage.Len() == 0 {
		retError = nil
	} else {
		retError = errors.New(aggregatedErrorMessage.String())
	}
	return modList, retError
}

func Reset() {
	downloadersCollection = []Downloader{}
}

func AddTorrent(t Torrent) (string, error) {
	if len(downloadersCollection) == 0 {
		return "", errors.New("Cannot add torrents, no downloaders are configured")
	}

	id, err := downloadersCollection[0].AddTorrent(t)
	if err != nil {
		return "", err
	}

	return id, nil
}

func AddTorrentMapping(flemzerId string, downloaderId string) {
	downloadersCollection[0].AddTorrentMapping(flemzerId, downloaderId)
}

func StartTorrent(t Torrent) error {
	return nil
}

func RemoveTorrent(t Torrent) error {
	if len(downloadersCollection) == 0 {
		return errors.New("Cannot remove torrents, no downloaders are configured")
	}

	return downloadersCollection[0].RemoveTorrent(t)
}

func GetTorrentStatus(t Torrent) (int, error) {
	return downloadersCollection[0].GetTorrentStatus(t)
}

func EpisodeHandleTorrentDownload(e *Episode, recovery bool) error {
	torrent := e.DownloadingItem.CurrentTorrent
	if !recovery {
		torrentId, err := AddTorrent(torrent)
		if err != nil {
			return errors.New("Couldn't add torrent in downloader. Skipping to next torrent in list")
		}
		e.DownloadingItem.CurrentTorrent = torrent
		e.DownloadingItem.CurrentDownloaderId = torrentId
		db.Client.Save(&e)
	}

	StartTorrent(torrent)

	retryCount := 0

	// Try twice to download a torrent before marking it as rubbish
	downloadErr := WaitForDownload(torrent)
	if downloadErr != nil {
		log.WithFields(log.Fields{
			"error":   downloadErr,
			"torrent": torrent.Name,
		}).Debug("Error during torrent download. Retrying download")

		RemoveTorrent(torrent)
		AddTorrent(torrent)
		retryCount++
		retryErr := WaitForDownload(torrent)
		if retryErr != nil {
			RemoveTorrent(torrent)
			e.DownloadingItem.FailedTorrents = append(e.DownloadingItem.FailedTorrents, torrent)
			db.Client.Save(&e)

			log.WithFields(log.Fields{
				"error":   downloadErr,
				"torrent": torrent.Name,
			}).Debug("Error during torrent download. Finish current torrent download")

			return retryErr
		}
	}

	// If function has not returned yet, download ended with no errors !
	log.WithFields(log.Fields{
		"show":   e.TvShow.Name,
		"season": e.Season,
		"number": e.Number,
		"name":   e.Name,
	}).Info("Episode successfully downloaded")
	notifier.NotifyDownloadedEpisode(e)

	e.Downloaded = true
	e.DownloadingItem.Downloading = false
	err := MoveEpisodeToLibrary(e)
	if err != nil {
		log.WithFields(log.Fields{
			"show":           e.TvShow.Name,
			"episode":        e.Name,
			"season":         e.Season,
			"number":         e.Number,
			"temporary_path": e.DownloadingItem.CurrentTorrent.DownloadDir,
			"library_path":   configuration.Config.Library.ShowPath,
			"error":          err,
		}).Error("Could not move episode from temporay download path to library folder")
	}
	db.Client.Save(e)

	RemoveTorrent(torrent)

	return nil
}

func MovieHandleTorrentDownload(m *Movie, recovery bool) error {
	torrent := m.DownloadingItem.CurrentTorrent
	if !recovery {
		torrentId, err := AddTorrent(torrent)
		if err != nil {
			return errors.New("Couldn't add torrent in downloader. Skipping to next torrent in list")
		}

		m.DownloadingItem.CurrentTorrent = torrent
		m.DownloadingItem.CurrentDownloaderId = torrentId
		db.Client.Save(&m)
	}

	StartTorrent(torrent)

	retryCount := 0

	// Try twice to download a torrent before marking it as rubbish
	downloadErr := WaitForDownload(torrent)
	if downloadErr != nil {
		log.WithFields(log.Fields{
			"error":   downloadErr,
			"torrent": torrent.Name,
		}).Debug("Error during torrent download. Retrying download")

		RemoveTorrent(torrent)
		AddTorrent(torrent)
		retryCount++
		retryErr := WaitForDownload(torrent)
		if retryErr != nil {
			RemoveTorrent(torrent)
			m.DownloadingItem.FailedTorrents = append(m.DownloadingItem.FailedTorrents, torrent)
			db.Client.Save(&m)

			log.WithFields(log.Fields{
				"error":   downloadErr,
				"torrent": torrent.Name,
			}).Debug("Error during torrent download. Finish current torrent download")

			return retryErr
		}
	}

	// If function has not returned yet, download ended with no errors !
	log.WithFields(log.Fields{
		"name": m.Title,
	}).Info("Movie successfully downloaded")
	notifier.NotifyDownloadedMovie(m)

	m.Downloaded = true
	m.DownloadingItem.Downloading = false
	err := MoveMovieToLibrary(m)
	if err != nil {
		log.WithFields(log.Fields{
			"movie":          m.Title,
			"temporary_path": m.DownloadingItem.CurrentTorrent.DownloadDir,
			"library_path":   configuration.Config.Library.ShowPath,
			"error":          err,
		}).Error("Could not move movie from temporay download path to library folder")
	}
	db.Client.Save(m)

	RemoveTorrent(torrent)

	return nil
}

func WaitForDownload(t Torrent) error {
	downloadLoopTicker := time.NewTicker(1 * time.Minute)
	for {
		log.WithFields(log.Fields{
			"torrent": t.Name,
		}).Debug("Checking torrent download progress")

		status, err := GetTorrentStatus(t)
		if err != nil {
			return err
		}

		switch status {
		case TORRENT_STOPPED:
			return errors.New("Torrent stopped in download client")
		case TORRENT_SEEDING:
			// Download complete ! Return with no error
			return nil
		}
		<-downloadLoopTicker.C
	}
}

func DownloadEpisode(show TvShow, e Episode, torrentList []Torrent) error {
	if e.Downloaded || e.DownloadingItem.Downloading {
		return errors.New("Episode downloading or already downloaded. Skipping")
	}

	log.WithFields(log.Fields{
		"show":   show.Name,
		"season": e.Season,
		"number": e.Number,
		"name":   e.Name,
	}).Info("Starting download process")

	e.DownloadingItem.Downloading = true
	db.Client.Save(&e)

	for _, torrent := range torrentList {
		torrent.DownloadDir = fmt.Sprintf("%s/%s/", DOWNLOAD_TMP_DIR, xid.New())

		if db.TorrentHasFailed(e.DownloadingItem, torrent) {
			continue
		}

		e.DownloadingItem.CurrentTorrent = torrent
		db.Client.Save(&e)

		torrentDownload := EpisodeHandleTorrentDownload(&e, false)
		if torrentDownload != nil {
			log.WithFields(log.Fields{
				"err":     torrentDownload,
				"torrent": torrent.Name,
			}).Warning("Couldn't download torrent. Skipping to next torrent in list")
		} else {
			return nil
		}

		continue
	}

	// If function has not returned yet, it means the download failed
	if len(e.DownloadingItem.FailedTorrents) > configuration.Config.System.TorrentDownloadAttemptsLimit {
		MarkEpisodeFailedDownload(&show, &e)
		return errors.New("Download failed, no torrents could be downloaded")
	}

	return errors.New("No torrents in current torrent list could be downloaded")
}

func DownloadMovie(m Movie, torrentList []Torrent) error {
	if m.Downloaded || m.DownloadingItem.Downloading {
		return errors.New("Movie downloading or already downloaded. Skipping")
	}

	log.WithFields(log.Fields{
		"name": m.Title,
	}).Info("Starting download process")

	m.DownloadingItem.Downloading = true
	db.Client.Save(&m)

	for _, torrent := range torrentList {
		torrent.DownloadDir = fmt.Sprintf("%s/%s/", DOWNLOAD_TMP_DIR, xid.New())

		if db.TorrentHasFailed(m.DownloadingItem, torrent) {
			continue
		}

		m.DownloadingItem.CurrentTorrent = torrent
		db.Client.Save(&m)

		torrentDownload := MovieHandleTorrentDownload(&m, false)
		if torrentDownload != nil {
			log.WithFields(log.Fields{
				"err":     torrentDownload,
				"torrent": torrent.Name,
			}).Warning("Couldn't download torrent. Skipping to next torrent in list")
		} else {
			return nil
		}
		continue
	}

	// If function has not returned yet, it means the download failed
	if len(m.DownloadingItem.FailedTorrents) > configuration.Config.System.TorrentDownloadAttemptsLimit {
		MarkMovieFailedDownload(&m)
		return errors.New("Download failed, no torrents could be downloaded")
	}

	return errors.New("No torrents in current torrent list could be downloaded")
}

func MarkEpisodeFailedDownload(show *TvShow, e *Episode) {
	log.WithFields(log.Fields{
		"show":   show.Name,
		"season": e.Season,
		"number": e.Number,
		"name":   e.Name,
	}).Error("Download failed, no torrents could be downloaded")

	notifier.NotifyFailedEpisode(e)

	e.DownloadingItem.DownloadFailed = true
	e.DownloadingItem.Downloading = false
	db.Client.Save(&e)
}

func MarkMovieFailedDownload(m *Movie) {
	log.WithFields(log.Fields{
		"movie": m.Title,
	}).Error("Download failed, no torrents could be downloaded")

	notifier.NotifyFailedMovie(m)

	m.DownloadingItem.DownloadFailed = true
	m.DownloadingItem.Downloading = false
	db.Client.Save(&m)
}

func MoveEpisodeToLibrary(episode *Episode) error {
	log.WithFields(log.Fields{
		"show":           episode.TvShow.Name,
		"episode":        episode.Name,
		"season":         episode.Season,
		"number":         episode.Number,
		"temporary_path": episode.DownloadingItem.CurrentTorrent.DownloadDir,
		"library_path":   configuration.Config.Library.ShowPath,
	}).Debug("Moving episode to library")

	destinationPath := fmt.Sprintf("%s/%s/Season %d/s%de%d", configuration.Config.Library.ShowPath, episode.TvShow.Name, episode.Season, episode.Season, episode.Number)
	err := os.Rename(episode.DownloadingItem.CurrentTorrent.DownloadDir, destinationPath)
	if err != nil {
		return err
	}

	err = os.Remove(episode.DownloadingItem.CurrentTorrent.DownloadDir)
	if err != nil {
		log.WithFields(log.Fields{
			"path":  episode.DownloadingItem.CurrentTorrent.DownloadDir,
			"error": err,
		}).Warning("Could not remove temporary folder for download")
	}

	episode.DownloadingItem.CurrentTorrent.DownloadDir = destinationPath
	db.Client.Save(episode)

	return nil
}

func MoveMovieToLibrary(movie *Movie) error {
	log.WithFields(log.Fields{
		"movie":          movie.Title,
		"temporary_path": movie.DownloadingItem.CurrentTorrent.DownloadDir,
		"library_path":   configuration.Config.Library.ShowPath,
	}).Debug("Moving episode to library")

	destinationPath := fmt.Sprintf("%s/%s", configuration.Config.Library.MoviePath, movie.Title)
	err := os.Rename(movie.DownloadingItem.CurrentTorrent.DownloadDir, destinationPath)
	if err != nil {
		return err
	}

	err = os.Remove(movie.DownloadingItem.CurrentTorrent.DownloadDir)
	if err != nil {
		log.WithFields(log.Fields{
			"path":  movie.DownloadingItem.CurrentTorrent.DownloadDir,
			"error": err,
		}).Warning("Could not remove temporary folder for download")
	}

	movie.DownloadingItem.CurrentTorrent.DownloadDir = destinationPath
	db.Client.Save(movie)

	return nil
}

func FillEpisodeToDownloadTorrentList(e *Episode, list []Torrent) []Torrent {
	var torrentList []Torrent
	for _, torrent := range list {
		if !db.TorrentHasFailed(e.DownloadingItem, torrent) {
			torrentList = append(torrentList, torrent)
		}
	}

	if len(torrentList) < 10 {
		return torrentList
	} else {
		return torrentList[:10]
	}
}

func FillMovieToDownloadTorrentList(m *Movie, list []Torrent) []Torrent {
	var torrentList []Torrent
	for _, torrent := range list {
		if !db.TorrentHasFailed(m.DownloadingItem, torrent) {
			torrentList = append(torrentList, torrent)
		}
	}

	if len(torrentList) < 10 {
		return torrentList
	} else {
		return torrentList[:10]
	}
}

func RecoverFromRetention() {
	downloadingEpisodesFromRetention, err := db.GetDownloadingEpisodes()
	if err != nil {
		log.Error(err)
		return
	}
	downloadingMoviesFromRetention, err := db.GetDownloadingMovies()
	if err != nil {
		log.Error(err)
		return
	}

	if len(downloadingEpisodesFromRetention) != 0 {
		log.Debug("Launching watch threads for downloading episodes found in retention")
	}
	for _, ep := range downloadingEpisodesFromRetention {
		AddTorrentMapping(ep.DownloadingItem.CurrentTorrent.TorrentId, ep.DownloadingItem.CurrentDownloaderId)

		log.WithFields(log.Fields{
			"episode": ep.Name,
			"season":  ep.Season,
			"number":  ep.Number,
		}).Debug("Launched download processing recovery")

		go EpisodeHandleTorrentDownload(&ep, true)

	}

	if len(downloadingMoviesFromRetention) != 0 {
		log.Debug("Launching watch threads for downloading movies found in retention")
	}
	for _, m := range downloadingMoviesFromRetention {
		AddTorrentMapping(m.DownloadingItem.CurrentTorrent.TorrentId, m.DownloadingItem.CurrentDownloaderId)

		log.WithFields(log.Fields{
			"name": m.Title,
		}).Debug("Launched download processing recovery")

		go MovieHandleTorrentDownload(&m, true)
	}
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"testing"
)

func TestLoadConfig(t *testing.T) {
	c, err := New("../tests/awsnycast.yaml")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	if c == nil {
		t.Fail()
	}
}

func TestLoadConfigFails(t *testing.T) {
	_, err := New("../tests/doesnotexist.yaml")
	if err == nil {
		t.Fail()
	}
}

func TestLoadConfigHealthchecks(t *testing.T) {
	c, _ := New("../tests/awsnycast.yaml")
	if c.Healthchecks == nil {
		t.Log("c.Healthchecks == nil")
		t.Fail()
	}
	h, ok := c.Healthchecks["public"]
	if !ok {
		t.Log("c.Healthchecks['public'] not ok")
		t.Fail()
	}
	if h.Type != "ping" {
		t.Log("type not ping")
		t.Fail()
	}
	if h.Destination != "8.8.8.8" {
		t.Log("Destination not 8.8.8.8")
		t.Fail()
	}
	if h.Rise != 2 {
		t.Log("Rise not 2")
		t.Fail()
	}
	if h.Fall != 10 {
		t.Log("fall not 10")
		t.Fail()
	}
	if h.Every != 1 {
		t.Log("every not 1")
		t.Fail()
	}
	a, ok := c.RouteTables["a"]
	if !ok {
		t.Log("RouteTables a not ok")
		t.Fail()
	}
	if a.Find.Type != "by_tag" {
		t.Log("Not by_tag")
		t.Fail()
	}
	if v, ok := a.Find.Config["key"]; ok {
		if v != "Name" {
			t.Log("Config key Name not found")
			t.Fail()
		}
	} else {
		t.Log(fmt.Sprintf("Config key not found: %+v", a.Find.Config))
		t.Fail()
	}
	if v, ok := a.Find.Config["value"]; ok {
		if v != "private a" {
			t.Log("Config value not private a")
			t.Fail()
		}
	} else {
		t.Log("Config value not present")
		t.Fail()
	}
	routes := a.UpsertRoutes
	if len(routes) != 2 {
		t.Log("Route len not 2")
		t.Fail()
	}
	for _, route := range routes {
		if route.Cidr == "0.0.0.0/0" || route.Cidr == "192.168.1.1/32" {
			if route.Instance != "SELF" {
				t.Log("route.Instance not SELF")
				t.Fail()
			}
			if route.Cidr == "0.0.0.0/0" {
				if route.Healthcheck != "public" {
					t.Log("Healthcheck not public")
					t.Fail()
				}
			} else {
				if route.Healthcheck != "localservice" {
					t.Fail()
				}
			}
		} else {
			t.Fail()
		}
	}
	b, ok := c.RouteTables["b"]
	if !ok {
		t.Fail()
	}
	if b.Find.Type != "by_tag" {
		t.Fail()
	}
}

func TestConfigDefault(t *testing.T) {
	r := make(map[string]*RouteTable)
	r["a"] = &RouteTable{
		UpsertRoutes: []*UpsertRoutesSpec{&UpsertRoutesSpec{Cidr: "127.0.0.1"}},
	}
	c := Config{
		RouteTables: r,
	}
	c.Default()
	if c.Healthchecks == nil {
		t.Fail()
	}
	if c.RouteTables["a"].UpsertRoutes[0].Cidr != "127.0.0.1/32" {
		t.Fail()
	}
}

func TestConfigValidateNoRouteTables(t *testing.T) {
	c := Config{}
	c.Default()
	err := c.Validate()
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No route_tables defined in config" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestConfigValidate(t *testing.T) {
	u := make([]*UpsertRoutesSpec, 1)
	u[0] = &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	r := make(map[string]*RouteTable)
	r["a"] = &RouteTable{
		UpsertRoutes: u,
	}
	c := Config{
		RouteTables: r,
	}
	c.Default()
	err := c.Validate()
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	rt := c.RouteTables["a"]
	ur := rt.UpsertRoutes[0]
	if ur.Cidr != "127.0.0.1/32" {
		t.Fail()
	}
}

func TestConfigValidateEmpty(t *testing.T) {
	c := Config{}
	c.Default()
	err := c.Validate()
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No route_tables defined in config" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestConfigValidateEmptyRouteTables(t *testing.T) {
	r := make(map[string]*RouteTable)
	c := Config{
		RouteTables: r,
	}
	c.Default()
	err := c.Validate()
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No route_tables defined in config" {
		t.Log(err.Error())
		t.Fail()
	}
}

// FIXME - need tests for each part of config failing, and check errors.

func TestUpsertRoutesSpecDefault(t *testing.T) {
	u := &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	u.Default()
	if u.Cidr != "127.0.0.1/32" {
		t.Log("Not canonicalized in UpsertRoutesSpecDefault")
		t.Fail()
	}
	if u.Instance != "SELF" {
		t.Log("Instance not defaulted to SELF")
	}
}

func TestUpsertRoutesSpecValidateBadInstance(t *testing.T) {
	r := &UpsertRoutesSpec{
		Instance: "vpc-1234",
		Cidr:     "127.0.0.1",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: 127.0.0.1 in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateMissingCidr(t *testing.T) {
	r := UpsertRoutesSpec{
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "cidr is not defined in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateBadCidr1(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "300.0.0.0/16",
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: 300.0.0.0/16 in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateBadCidr2(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "3.0.0.0/160",
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: 3.0.0.0/160 in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateBadCidr3(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "foo",
		Instance: "SELF",
	}
	err := r.Validate("bar")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: foo in bar" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidate(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "0.0.0.0/0",
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
}

func TestRouteTableFindSpecDefault(t *testing.T) {
	r := RouteTableFindSpec{}
	r.Default()
	if r.Config == nil {
		t.Fail()
	}
}
func TestRouteTableFindSpecValidate(t *testing.T) {
	c := make(map[string]string)
	c["key"] = "Name"
	c["value"] = "private a"
	r := RouteTableFindSpec{
		Type:   "by_tag",
		Config: c,
	}
	err := r.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
}

func TestRouteTableFindSpecValidateNoType(t *testing.T) {
	c := make(map[string]string)
	c["key"] = "Name"
	c["value"] = "private a"
	r := RouteTableFindSpec{
		Config: c,
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Route find spec foo needs a type key" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableFindSpecValidateNoConfig(t *testing.T) {
	r := RouteTableFindSpec{
		Type: "by_tag",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No config supplied" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableDefaultEmpty(t *testing.T) {
	r := RouteTable{}
	r.Default()
}

func TestRouteTableDefault(t *testing.T) {
	routes := make([]*UpsertRoutesSpec, 1)
	routes[0] = &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	r := RouteTable{
		UpsertRoutes: routes,
	}
	r.Default()
	if len(r.UpsertRoutes) != 1 {
		t.Fail()
	}
	routeSpec := r.UpsertRoutes[0]
	if routeSpec.Cidr != "127.0.0.1/32" {
		t.Fail()
	}
}

func TestRouteTableValidateNullRoutes(t *testing.T) {
	r := RouteTable{}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No upsert_routes key in route table 'foo'" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableValidateNoRoutes(t *testing.T) {
	r := RouteTable{
		UpsertRoutes: make([]*UpsertRoutesSpec, 0),
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No upsert_routes key in route table 'foo'" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableValidate(t *testing.T) {
	routes := make([]*UpsertRoutesSpec, 1)
	routes[0] = &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	r := RouteTable{
		UpsertRoutes: routes,
	}
	r.Default()
	err := r.Validate("foo")
	if err != nil {
		t.Fail()
	}
}

func TestUpsertRouteSpecGetInstanceSELF(t *testing.T) {
	urs := UpsertRoutesSpec{
		Cidr:     "127.0.0.1",
		Instance: "SELF",
	}
	if urs.GetInstance("i-other") != "i-other" {
		t.Fail()
	}
}

func TestUpsertRouteSpecGetInstanceOther(t *testing.T) {
	urs := UpsertRoutesSpec{
		Cidr:     "127.0.0.1",
		Instance: "i-foo",
	}
	if urs.GetInstance("i-other") != "i-foo" {
		t.Fail()
	}
}
<commit_msg>More tests<commit_after>package config

import (
	"fmt"
	"testing"
)

func TestLoadConfig(t *testing.T) {
	c, err := New("../tests/awsnycast.yaml")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	if c == nil {
		t.Fail()
	}
}

func TestLoadConfigFails(t *testing.T) {
	_, err := New("../tests/doesnotexist.yaml")
	if err == nil {
		t.Fail()
	}
}

func TestLoadConfigHealthchecks(t *testing.T) {
	c, _ := New("../tests/awsnycast.yaml")
	if c.Healthchecks == nil {
		t.Log("c.Healthchecks == nil")
		t.Fail()
	}
	h, ok := c.Healthchecks["public"]
	if !ok {
		t.Log("c.Healthchecks['public'] not ok")
		t.Fail()
	}
	if h.Type != "ping" {
		t.Log("type not ping")
		t.Fail()
	}
	if h.Destination != "8.8.8.8" {
		t.Log("Destination not 8.8.8.8")
		t.Fail()
	}
	if h.Rise != 2 {
		t.Log("Rise not 2")
		t.Fail()
	}
	if h.Fall != 10 {
		t.Log("fall not 10")
		t.Fail()
	}
	if h.Every != 1 {
		t.Log("every not 1")
		t.Fail()
	}
	a, ok := c.RouteTables["a"]
	if !ok {
		t.Log("RouteTables a not ok")
		t.Fail()
	}
	if a.Find.Type != "by_tag" {
		t.Log("Not by_tag")
		t.Fail()
	}
	if v, ok := a.Find.Config["key"]; ok {
		if v != "Name" {
			t.Log("Config key Name not found")
			t.Fail()
		}
	} else {
		t.Log(fmt.Sprintf("Config key not found: %+v", a.Find.Config))
		t.Fail()
	}
	if v, ok := a.Find.Config["value"]; ok {
		if v != "private a" {
			t.Log("Config value not private a")
			t.Fail()
		}
	} else {
		t.Log("Config value not present")
		t.Fail()
	}
	routes := a.UpsertRoutes
	if len(routes) != 2 {
		t.Log("Route len not 2")
		t.Fail()
	}
	for _, route := range routes {
		if route.Cidr == "0.0.0.0/0" || route.Cidr == "192.168.1.1/32" {
			if route.Instance != "SELF" {
				t.Log("route.Instance not SELF")
				t.Fail()
			}
			if route.Cidr == "0.0.0.0/0" {
				if route.Healthcheck != "public" {
					t.Log("Healthcheck not public")
					t.Fail()
				}
			} else {
				if route.Healthcheck != "localservice" {
					t.Fail()
				}
			}
		} else {
			t.Fail()
		}
	}
	b, ok := c.RouteTables["b"]
	if !ok {
		t.Fail()
	}
	if b.Find.Type != "by_tag" {
		t.Fail()
	}
}

func TestConfigDefault(t *testing.T) {
	r := make(map[string]*RouteTable)
	r["a"] = &RouteTable{
		UpsertRoutes: []*UpsertRoutesSpec{&UpsertRoutesSpec{Cidr: "127.0.0.1"}},
	}
	c := Config{
		RouteTables: r,
	}
	c.Default()
	if c.Healthchecks == nil {
		t.Fail()
	}
	if c.RouteTables["a"].UpsertRoutes[0].Cidr != "127.0.0.1/32" {
		t.Fail()
	}
}

func TestConfigValidateNoRouteTables(t *testing.T) {
	c := Config{}
	c.Default()
	err := c.Validate()
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No route_tables defined in config" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestConfigValidate(t *testing.T) {
	u := make([]*UpsertRoutesSpec, 1)
	u[0] = &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	r := make(map[string]*RouteTable)
	r["a"] = &RouteTable{
		UpsertRoutes: u,
	}
	c := Config{
		RouteTables: r,
	}
	c.Default()
	err := c.Validate()
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	rt := c.RouteTables["a"]
	ur := rt.UpsertRoutes[0]
	if ur.Cidr != "127.0.0.1/32" {
		t.Fail()
	}
}

func TestConfigValidateEmpty(t *testing.T) {
	c := Config{}
	c.Default()
	err := c.Validate()
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No route_tables defined in config" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestConfigValidateEmptyRouteTables(t *testing.T) {
	r := make(map[string]*RouteTable)
	c := Config{
		RouteTables: r,
	}
	c.Default()
	err := c.Validate()
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No route_tables defined in config" {
		t.Log(err.Error())
		t.Fail()
	}
}

// FIXME - need tests for each part of config failing, and check errors.

func TestUpsertRoutesSpecDefault(t *testing.T) {
	u := &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	u.Default()
	if u.Cidr != "127.0.0.1/32" {
		t.Log("Not canonicalized in UpsertRoutesSpecDefault")
		t.Fail()
	}
	if u.Instance != "SELF" {
		t.Log("Instance not defaulted to SELF")
	}
}

func TestUpsertRoutesSpecValidateBadInstance(t *testing.T) {
	r := &UpsertRoutesSpec{
		Instance: "vpc-1234",
		Cidr:     "127.0.0.1",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: 127.0.0.1 in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateMissingCidr(t *testing.T) {
	r := UpsertRoutesSpec{
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "cidr is not defined in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateBadCidr1(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "300.0.0.0/16",
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: 300.0.0.0/16 in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateBadCidr2(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "3.0.0.0/160",
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: 3.0.0.0/160 in foo" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidateBadCidr3(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "foo",
		Instance: "SELF",
	}
	err := r.Validate("bar")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Could not parse invalid CIDR address: foo in bar" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestUpsertRoutesSpecValidate(t *testing.T) {
	r := UpsertRoutesSpec{
		Cidr:     "0.0.0.0/0",
		Instance: "SELF",
	}
	err := r.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
}

func TestRouteTableFindSpecDefault(t *testing.T) {
	r := RouteTableFindSpec{}
	r.Default()
	if r.Config == nil {
		t.Fail()
	}
}
func TestRouteTableFindSpecValidate(t *testing.T) {
	c := make(map[string]string)
	c["key"] = "Name"
	c["value"] = "private a"
	r := RouteTableFindSpec{
		Type:   "by_tag",
		Config: c,
	}
	err := r.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
}

func TestRouteTableFindSpecValidateNoType(t *testing.T) {
	c := make(map[string]string)
	c["key"] = "Name"
	c["value"] = "private a"
	r := RouteTableFindSpec{
		Config: c,
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "Route find spec foo needs a type key" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableFindSpecValidateNoConfig(t *testing.T) {
	r := RouteTableFindSpec{
		Type: "by_tag",
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No config supplied" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableDefaultEmpty(t *testing.T) {
	r := RouteTable{}
	r.Default()
}

func TestRouteTableDefault(t *testing.T) {
	routes := make([]*UpsertRoutesSpec, 1)
	routes[0] = &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	r := RouteTable{
		UpsertRoutes: routes,
	}
	r.Default()
	if len(r.UpsertRoutes) != 1 {
		t.Fail()
	}
	routeSpec := r.UpsertRoutes[0]
	if routeSpec.Cidr != "127.0.0.1/32" {
		t.Fail()
	}
}

func TestRouteTableValidateNullRoutes(t *testing.T) {
	r := RouteTable{}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No upsert_routes key in route table 'foo'" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableValidateNoRoutes(t *testing.T) {
	r := RouteTable{
		UpsertRoutes: make([]*UpsertRoutesSpec, 0),
	}
	err := r.Validate("foo")
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No upsert_routes key in route table 'foo'" {
		t.Log(err.Error())
		t.Fail()
	}
}

func TestRouteTableValidate(t *testing.T) {
	routes := make([]*UpsertRoutesSpec, 1)
	routes[0] = &UpsertRoutesSpec{
		Cidr: "127.0.0.1",
	}
	r := RouteTable{
		UpsertRoutes: routes,
	}
	r.Default()
	err := r.Validate("foo")
	if err != nil {
		t.Fail()
	}
}

func TestUpsertRouteSpecGetInstanceSELF(t *testing.T) {
	urs := UpsertRoutesSpec{
		Cidr:     "127.0.0.1",
		Instance: "SELF",
	}
	if urs.GetInstance("i-other") != "i-other" {
		t.Fail()
	}
}

func TestUpsertRouteSpecGetInstanceOther(t *testing.T) {
	urs := UpsertRoutesSpec{
		Cidr:     "127.0.0.1",
		Instance: "i-foo",
	}
	if urs.GetInstance("i-other") != "i-foo" {
		t.Fail()
	}
}

func TestByTagRouteTableFindMissingKey(t *testing.T) {
	c := make(map[string]string)
	rts := RouteTableFindSpec{
		Type:   "by_tag",
		Config: c,
	}
	rtf, err := rts.GetFilter()
	if rtf != nil {
		t.Fail()
	}
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No key in config for by_tag route table finder" {
		t.Fail()
	}
}

func TestByTagRouteTableFindMissingValue(t *testing.T) {
	c := make(map[string]string)
	c["key"] = "Name"
	rts := RouteTableFindSpec{
		Type:   "by_tag",
		Config: c,
	}
	rtf, err := rts.GetFilter()
	if rtf != nil {
		t.Fail()
	}
	if err == nil {
		t.Fail()
	}
	if err.Error() != "No value in config for by_tag route table finder" {
		t.Fail()
	}
}

func TestByTagRouteTableFind(t *testing.T) {
	c := make(map[string]string)
	c["key"] = "Name"
	c["value"] = "private b"
	rts := RouteTableFindSpec{
		Type:   "by_tag",
		Config: c,
	}
	rtf, err := rts.GetFilter()
	if rtf == nil {
		t.Fail()
	}
	if err != nil {
		t.Fail()
	}
}

func TestRouteTableFindUnknownType(t *testing.T) {
	c := make(map[string]string)
	rts := RouteTableFindSpec{
		Type:   "unknown",
		Config: c,
	}
	rtf, err := rts.GetFilter()
	if rtf != nil {
		t.Fail()
	}
	if err == nil {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package virtualbox

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"strings"

	"strconv"

	"github.com/docker/machine/libmachine/log"
)

var (
	reColonLine       = regexp.MustCompile(`(.+):\s+(.*)`)
	reEqualLine       = regexp.MustCompile(`(.+)=(.*)`)
	reEqualQuoteLine  = regexp.MustCompile(`"(.+)"="(.*)"`)
	reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`)

	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVBMNotFound     = errors.New("VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path")

	vboxManageCmd = detectVBoxManageCmd()
)

// VBoxManager defines the interface to communicate to VirtualBox.
type VBoxManager interface {
	vbm(args ...string) error

	vbmOut(args ...string) (string, error)

	vbmOutErr(args ...string) (string, string, error)
}

// VBoxCmdManager communicates with VirtualBox through the commandline using `VBoxManage`.
type VBoxCmdManager struct{}

func (v *VBoxCmdManager) vbm(args ...string) error {
	_, _, err := v.vbmOutErr(args...)
	return err
}

func (v *VBoxCmdManager) vbmOut(args ...string) (string, error) {
	stdout, _, err := v.vbmOutErr(args...)
	return stdout, err
}

func (v *VBoxCmdManager) vbmOutErr(args ...string) (string, string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	log.Debugf("COMMAND: %v %v", vboxManageCmd, strings.Join(args, " "))
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	stderrStr := stderr.String()
	if len(args) > 0 {
		log.Debugf("STDOUT:\n{\n%v}", stdout.String())
		log.Debugf("STDERR:\n{\n%v}", stderrStr)
	}

	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}

	if err == nil || strings.HasPrefix(err.Error(), "exit status ") {
		// VBoxManage will sometimes not set the return code, but has a fatal error
		// such as VBoxManage.exe: error: VT-x is not available. (VERR_VMX_NO_VMX)
		if strings.Contains(stderrStr, "error:") {
			err = fmt.Errorf("%v %v failed:\n%v", vboxManageCmd, strings.Join(args, " "), stderrStr)
		}
	}

	return stdout.String(), stderrStr, err
}

func checkVBoxManageVersion(version string) error {
	major, minor, err := parseVersion(version)
	if (err != nil) || (major < 4) || (major == 4 && minor <= 2) {
		return fmt.Errorf("We support Virtualbox starting with version 5. Your VirtualBox install is %q. Please upgrade at https://www.virtualbox.org", version)
	}

	if major < 5 {
		log.Warnf("You are using version %s of VirtualBox. If you encouter issues, you might want to upgrade to version 5 at https://www.virtualbox.org", version)
	}

	return nil
}

func parseVersion(version string) (int, int, error) {
	parts := strings.Split(version, ".")
	if len(parts) < 2 {
		return 0, 0, fmt.Errorf("Invalid version: %q", version)
	}

	major, err := strconv.Atoi(parts[0])
	if err != nil {
		return 0, 0, fmt.Errorf("Invalid version: %q", version)
	}

	minor, err := strconv.Atoi(parts[1])
	if err != nil {
		return 0, 0, fmt.Errorf("Invalid version: %q", version)
	}

	return major, minor, err
}

func parseKeyValues(stdOut string, regexp *regexp.Regexp, callback func(key, val string) error) error {
	r := strings.NewReader(stdOut)
	s := bufio.NewScanner(r)

	for s.Scan() {
		line := s.Text()
		if line == "" {
			continue
		}

		res := regexp.FindStringSubmatch(line)
		if res == nil {
			continue
		}

		key, val := res[1], res[2]
		if err := callback(key, val); err != nil {
			return err
		}
	}

	return s.Err()
}
<commit_msg>FIX #2762 retry Virtualbox commands<commit_after>package virtualbox

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"strings"

	"strconv"

	"github.com/docker/machine/libmachine/log"
)

const (
	retryCountOnObjectNotReadyError = 5
)

var (
	reColonLine       = regexp.MustCompile(`(.+):\s+(.*)`)
	reEqualLine       = regexp.MustCompile(`(.+)=(.*)`)
	reEqualQuoteLine  = regexp.MustCompile(`"(.+)"="(.*)"`)
	reMachineNotFound = regexp.MustCompile(`Could not find a registered machine named '(.+)'`)

	ErrMachineNotExist = errors.New("machine does not exist")
	ErrVBMNotFound     = errors.New("VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path")

	vboxManageCmd = detectVBoxManageCmd()
)

// VBoxManager defines the interface to communicate to VirtualBox.
type VBoxManager interface {
	vbm(args ...string) error

	vbmOut(args ...string) (string, error)

	vbmOutErr(args ...string) (string, string, error)
}

// VBoxCmdManager communicates with VirtualBox through the commandline using `VBoxManage`.
type VBoxCmdManager struct{}

func (v *VBoxCmdManager) vbm(args ...string) error {
	_, _, err := v.vbmOutErr(args...)
	return err
}

func (v *VBoxCmdManager) vbmOut(args ...string) (string, error) {
	stdout, _, err := v.vbmOutErr(args...)
	return stdout, err
}

func (v *VBoxCmdManager) vbmOutErr(args ...string) (string, string, error) {
	return v.vbmOutErrRetry(retryCountOnObjectNotReadyError, args...)
}

func (v *VBoxCmdManager) vbmOutErrRetry(retry int, args ...string) (string, string, error) {
	cmd := exec.Command(vboxManageCmd, args...)
	log.Debugf("COMMAND: %v %v", vboxManageCmd, strings.Join(args, " "))
	var stdout bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	stderrStr := stderr.String()
	if len(args) > 0 {
		log.Debugf("STDOUT:\n{\n%v}", stdout.String())
		log.Debugf("STDERR:\n{\n%v}", stderrStr)
	}

	if err != nil {
		if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
			err = ErrVBMNotFound
		}
	}

	// Sometimes, we just need to retry...
	if retry > 0 {
		if strings.Contains(stderrStr, "error: The object is not ready") {
			return v.vbmOutErrRetry(retry-1, args...)
		}
	}

	if err == nil || strings.HasPrefix(err.Error(), "exit status ") {
		// VBoxManage will sometimes not set the return code, but has a fatal error
		// such as VBoxManage.exe: error: VT-x is not available. (VERR_VMX_NO_VMX)
		if strings.Contains(stderrStr, "error:") {
			err = fmt.Errorf("%v %v failed:\n%v", vboxManageCmd, strings.Join(args, " "), stderrStr)
		}
	}

	return stdout.String(), stderrStr, err
}

func checkVBoxManageVersion(version string) error {
	major, minor, err := parseVersion(version)
	if (err != nil) || (major < 4) || (major == 4 && minor <= 2) {
		return fmt.Errorf("We support Virtualbox starting with version 5. Your VirtualBox install is %q. Please upgrade at https://www.virtualbox.org", version)
	}

	if major < 5 {
		log.Warnf("You are using version %s of VirtualBox. If you encouter issues, you might want to upgrade to version 5 at https://www.virtualbox.org", version)
	}

	return nil
}

func parseVersion(version string) (int, int, error) {
	parts := strings.Split(version, ".")
	if len(parts) < 2 {
		return 0, 0, fmt.Errorf("Invalid version: %q", version)
	}

	major, err := strconv.Atoi(parts[0])
	if err != nil {
		return 0, 0, fmt.Errorf("Invalid version: %q", version)
	}

	minor, err := strconv.Atoi(parts[1])
	if err != nil {
		return 0, 0, fmt.Errorf("Invalid version: %q", version)
	}

	return major, minor, err
}

func parseKeyValues(stdOut string, regexp *regexp.Regexp, callback func(key, val string) error) error {
	r := strings.NewReader(stdOut)
	s := bufio.NewScanner(r)

	for s.Scan() {
		line := s.Text()
		if line == "" {
			continue
		}

		res := regexp.FindStringSubmatch(line)
		if res == nil {
			continue
		}

		key, val := res[1], res[2]
		if err := callback(key, val); err != nil {
			return err
		}
	}

	return s.Err()
}
<|endoftext|>


<commit_before>package easyvk

// WallUploadServer describes the server address
// for photo upload onto a user's wall.
type WallUploadServer struct {
	Response struct {
		UploadURL string `json:"upload_url"`
		AlbumID int `json:"album_id"`
		UserID int `json:"user_id"`
	} `json:"response"`
}

// SavedWallPhoto describes info about
// saved photo on wall after being uploaded.
type SavedWallPhoto struct {
	Response []struct {
		Pid int `json:"pid"`
		ID string `json:"id"`
		Aid int `json:"aid"`
		OwnerID int `json:"owner_id"`
		Src string `json:"src"`
		SrcBig string `json:"src_big"`
		SrcSmall string `json:"src_small"`
		SrcXbig string `json:"src_xbig"`
		Width int `json:"width"`
		Height int `json:"height"`
		Text string `json:"text"`
		Created int `json:"created"`
	} `json:"response"`
}<commit_msg>Photos.SaveWallPhoto update<commit_after>package easyvk

// WallUploadServer describes the server address
// for photo upload onto a user's wall.
type WallUploadServer struct {
	Response struct {
		UploadURL string `json:"upload_url"`
		AlbumID int `json:"album_id"`
		UserID int `json:"user_id"`
	} `json:"response"`
}

// SavedWallPhoto describes info about
// saved photo on wall after being uploaded.
type SavedWallPhoto struct {
	Response []struct {
		ID int `json:"id"`
		AlbumID int `json:"album_id"`
		OwnerID int `json:"owner_id"`
		Photo75 string `json:"photo_75"`
		Photo130 string `json:"photo_130"`
		Photo604 string `json:"photo_604"`
		Photo807 string `json:"photo_807"`
		Width int `json:"width"`
		Height int `json:"height"`
		Text string `json:"text"`
		Date int `json:"date"`
	} `json:"response"`
}<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package database

import (
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"sync"
	"time"
)

var (
	log = logging.MustGetLogger("main")
)

type PubsubEvent string

const (
	CLUSTER            PubsubEvent = "_mirrorbits_cluster"
	FILE_UPDATE        PubsubEvent = "_mirrorbits_file_update"
	MIRROR_UPDATE      PubsubEvent = "_mirrorbits_mirror_update"
	MIRROR_FILE_UPDATE PubsubEvent = "_mirrorbits_mirror_file_update"

	PUBSUB_RECONNECTED PubsubEvent = "_mirrorbits_pubsub_reconnected"
)

type Pubsub struct {
	r                  *Redis
	rconn              redis.Conn
	connlock           sync.Mutex
	extSubscribers     map[string][]chan string
	extSubscribersLock sync.RWMutex
	stop               chan bool
	wg                 sync.WaitGroup
}

func NewPubsub(r *Redis) *Pubsub {
	pubsub := new(Pubsub)
	pubsub.r = r
	pubsub.stop = make(chan bool)
	pubsub.extSubscribers = make(map[string][]chan string)
	go pubsub.updateEvents()
	return pubsub
}

func (p *Pubsub) Close() {
	close(p.stop)
	p.connlock.Lock()
	if p.rconn != nil {
		// FIXME Calling p.rconn.Close() here will block indefinitely in redigo
		p.rconn.Send("UNSUBSCRIBE")
		p.rconn.Send("QUIT")
		p.rconn.Flush()
	}
	p.connlock.Unlock()
	p.wg.Wait()
}

// SubscribeEvent allows subscription to a particular kind of events and receive a
// notification when an event is dispatched on the given channel.
func (p *Pubsub) SubscribeEvent(event PubsubEvent, channel chan string) {
	p.extSubscribersLock.Lock()
	defer p.extSubscribersLock.Unlock()

	listeners := p.extSubscribers[string(event)]
	listeners = append(listeners, channel)
	p.extSubscribers[string(event)] = listeners
}

func (p *Pubsub) updateEvents() {
	p.wg.Add(1)
	defer p.wg.Done()
	var disconnected bool = false
connect:
	for {
		select {
		case <-p.stop:
			return
		default:
		}
		p.connlock.Lock()
		p.rconn = p.r.Get()
		if _, err := p.rconn.Do("PING"); err != nil {
			disconnected = true
			p.rconn.Close()
			p.rconn = nil
			p.connlock.Unlock()
			if RedisIsLoading(err) {
				// Doing a PING after (re-connection) prevents cases where redis
				// is currently loading the dataset and is still not ready.
				log.Warning("Redis is still loading the dataset in memory")
			}
			time.Sleep(500 * time.Millisecond)
			continue
		}
		p.connlock.Unlock()
		log.Info("Subscribing pubsub")
		psc := redis.PubSubConn{Conn: p.rconn}

		psc.Subscribe(CLUSTER)
		psc.Subscribe(FILE_UPDATE)
		psc.Subscribe(MIRROR_UPDATE)
		psc.Subscribe(MIRROR_FILE_UPDATE)

		if disconnected == true {
			// This is a way to keep the cache active while disconnected
			// from redis but still clear the cache (possibly outdated)
			// after a successful reconnection.
			disconnected = false
			p.handleMessage(string(PUBSUB_RECONNECTED), nil)
		}
		for {
			switch v := psc.Receive().(type) {
			case redis.Message:
				//log.Debug("Redis message on channel %s: message: %s", v.Channel, v.Data)
				p.handleMessage(v.Channel, v.Data)
			case redis.Subscription:
				log.Debug("Redis subscription on channel %s: %s (%d)", v.Channel, v.Kind, v.Count)
			case error:
				select {
				case <-p.stop:
					return
				default:
				}
				log.Error("Pubsub disconnected: %s", v)
				psc.Close()
				p.rconn.Close()
				time.Sleep(50 * time.Millisecond)
				disconnected = true
				goto connect
			}
		}
	}
}

// Notify subscribers of the new message
func (p *Pubsub) handleMessage(channel string, data []byte) {
	p.extSubscribersLock.RLock()
	defer p.extSubscribersLock.RUnlock()

	listeners := p.extSubscribers[channel]
	for _, listener := range listeners {
		select {
		case listener <- string(data):
		default:
			// Don't block if the listener is not available
			// and drop the message.
		}
	}
}

func Publish(r redis.Conn, event PubsubEvent, message string) error {
	_, err := r.Do("PUBLISH", string(event), message)
	return err
}

func SendPublish(r redis.Conn, event PubsubEvent, message string) error {
	err := r.Send("PUBLISH", string(event), message)
	return err
}
<commit_msg>pubsub: downgrade a log message from info to debug<commit_after>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package database

import (
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"sync"
	"time"
)

var (
	log = logging.MustGetLogger("main")
)

type PubsubEvent string

const (
	CLUSTER            PubsubEvent = "_mirrorbits_cluster"
	FILE_UPDATE        PubsubEvent = "_mirrorbits_file_update"
	MIRROR_UPDATE      PubsubEvent = "_mirrorbits_mirror_update"
	MIRROR_FILE_UPDATE PubsubEvent = "_mirrorbits_mirror_file_update"

	PUBSUB_RECONNECTED PubsubEvent = "_mirrorbits_pubsub_reconnected"
)

type Pubsub struct {
	r                  *Redis
	rconn              redis.Conn
	connlock           sync.Mutex
	extSubscribers     map[string][]chan string
	extSubscribersLock sync.RWMutex
	stop               chan bool
	wg                 sync.WaitGroup
}

func NewPubsub(r *Redis) *Pubsub {
	pubsub := new(Pubsub)
	pubsub.r = r
	pubsub.stop = make(chan bool)
	pubsub.extSubscribers = make(map[string][]chan string)
	go pubsub.updateEvents()
	return pubsub
}

func (p *Pubsub) Close() {
	close(p.stop)
	p.connlock.Lock()
	if p.rconn != nil {
		// FIXME Calling p.rconn.Close() here will block indefinitely in redigo
		p.rconn.Send("UNSUBSCRIBE")
		p.rconn.Send("QUIT")
		p.rconn.Flush()
	}
	p.connlock.Unlock()
	p.wg.Wait()
}

// SubscribeEvent allows subscription to a particular kind of events and receive a
// notification when an event is dispatched on the given channel.
func (p *Pubsub) SubscribeEvent(event PubsubEvent, channel chan string) {
	p.extSubscribersLock.Lock()
	defer p.extSubscribersLock.Unlock()

	listeners := p.extSubscribers[string(event)]
	listeners = append(listeners, channel)
	p.extSubscribers[string(event)] = listeners
}

func (p *Pubsub) updateEvents() {
	p.wg.Add(1)
	defer p.wg.Done()
	var disconnected bool = false
connect:
	for {
		select {
		case <-p.stop:
			return
		default:
		}
		p.connlock.Lock()
		p.rconn = p.r.Get()
		if _, err := p.rconn.Do("PING"); err != nil {
			disconnected = true
			p.rconn.Close()
			p.rconn = nil
			p.connlock.Unlock()
			if RedisIsLoading(err) {
				// Doing a PING after (re-connection) prevents cases where redis
				// is currently loading the dataset and is still not ready.
				log.Warning("Redis is still loading the dataset in memory")
			}
			time.Sleep(500 * time.Millisecond)
			continue
		}
		p.connlock.Unlock()
		log.Debug("Subscribing pubsub")
		psc := redis.PubSubConn{Conn: p.rconn}

		psc.Subscribe(CLUSTER)
		psc.Subscribe(FILE_UPDATE)
		psc.Subscribe(MIRROR_UPDATE)
		psc.Subscribe(MIRROR_FILE_UPDATE)

		if disconnected == true {
			// This is a way to keep the cache active while disconnected
			// from redis but still clear the cache (possibly outdated)
			// after a successful reconnection.
			disconnected = false
			p.handleMessage(string(PUBSUB_RECONNECTED), nil)
		}
		for {
			switch v := psc.Receive().(type) {
			case redis.Message:
				//log.Debug("Redis message on channel %s: message: %s", v.Channel, v.Data)
				p.handleMessage(v.Channel, v.Data)
			case redis.Subscription:
				log.Debug("Redis subscription on channel %s: %s (%d)", v.Channel, v.Kind, v.Count)
			case error:
				select {
				case <-p.stop:
					return
				default:
				}
				log.Error("Pubsub disconnected: %s", v)
				psc.Close()
				p.rconn.Close()
				time.Sleep(50 * time.Millisecond)
				disconnected = true
				goto connect
			}
		}
	}
}

// Notify subscribers of the new message
func (p *Pubsub) handleMessage(channel string, data []byte) {
	p.extSubscribersLock.RLock()
	defer p.extSubscribersLock.RUnlock()

	listeners := p.extSubscribers[channel]
	for _, listener := range listeners {
		select {
		case listener <- string(data):
		default:
			// Don't block if the listener is not available
			// and drop the message.
		}
	}
}

func Publish(r redis.Conn, event PubsubEvent, message string) error {
	_, err := r.Do("PUBLISH", string(event), message)
	return err
}

func SendPublish(r redis.Conn, event PubsubEvent, message string) error {
	err := r.Send("PUBLISH", string(event), message)
	return err
}
<|endoftext|>


<commit_before>package caddytls

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"io"
	"strings"
	"testing"
	"time"

	"github.com/xenolf/lego/acme"
	"os"
)

func TestUser(t *testing.T) {
	defer testStorage.clean()

	privateKey, err := rsa.GenerateKey(rand.Reader, 128)
	if err != nil {
		t.Fatalf("Could not generate test private key: %v", err)
	}
	u := User{
		Email:        "me@mine.com",
		Registration: new(acme.RegistrationResource),
		key:          privateKey,
	}

	if expected, actual := "me@mine.com", u.GetEmail(); actual != expected {
		t.Errorf("Expected email '%s' but got '%s'", expected, actual)
	}
	if u.GetRegistration() == nil {
		t.Error("Expected a registration resource, but got nil")
	}
	if expected, actual := privateKey, u.GetPrivateKey(); actual != expected {
		t.Errorf("Expected the private key at address %p but got one at %p instead ", expected, actual)
	}
}

func TestNewUser(t *testing.T) {
	email := "me@foobar.com"
	user, err := newUser(email)
	if err != nil {
		t.Fatalf("Error creating user: %v", err)
	}
	if user.key == nil {
		t.Error("Private key is nil")
	}
	if user.Email != email {
		t.Errorf("Expected email to be %s, but was %s", email, user.Email)
	}
	if user.Registration != nil {
		t.Error("New user already has a registration resource; it shouldn't")
	}
}

func TestSaveUser(t *testing.T) {
	defer testStorage.clean()

	email := "me@foobar.com"
	user, err := newUser(email)
	if err != nil {
		t.Fatalf("Error creating user: %v", err)
	}

	err = saveUser(testStorage, user)
	if err != nil {
		t.Fatalf("Error saving user: %v", err)
	}
	_, err = testStorage.LoadUser(email)
	if err != nil {
		t.Errorf("Cannot access user data, error: %v", err)
	}
}

func TestGetUserDoesNotAlreadyExist(t *testing.T) {
	defer testStorage.clean()

	user, err := getUser(testStorage, "user_does_not_exist@foobar.com")
	if err != nil {
		t.Fatalf("Error getting user: %v", err)
	}

	if user.key == nil {
		t.Error("Expected user to have a private key, but it was nil")
	}
}

func TestGetUserAlreadyExists(t *testing.T) {
	defer testStorage.clean()

	email := "me@foobar.com"

	// Set up test
	user, err := newUser(email)
	if err != nil {
		t.Fatalf("Error creating user: %v", err)
	}
	err = saveUser(testStorage, user)
	if err != nil {
		t.Fatalf("Error saving user: %v", err)
	}

	// Expect to load user from disk
	user2, err := getUser(testStorage, email)
	if err != nil {
		t.Fatalf("Error getting user: %v", err)
	}

	// Assert keys are the same
	if !PrivateKeysSame(user.key, user2.key) {
		t.Error("Expected private key to be the same after loading, but it wasn't")
	}

	// Assert emails are the same
	if user.Email != user2.Email {
		t.Errorf("Expected emails to be equal, but was '%s' before and '%s' after loading", user.Email, user2.Email)
	}
}

func TestGetEmail(t *testing.T) {
	storageBasePath = string(testStorage) // to contain calls that create a new Storage...

	// let's not clutter up the output
	origStdout := os.Stdout
	os.Stdout = nil
	defer func() { os.Stdout = origStdout }()

	defer testStorage.clean()
	DefaultEmail = "test2@foo.com"

	// Test1: Use default email from flag (or user previously typing it)
	actual := getEmail(testStorage, true)
	if actual != DefaultEmail {
		t.Errorf("Did not get correct email from memory; expected '%s' but got '%s'", DefaultEmail, actual)
	}

	// Test2: Get input from user
	DefaultEmail = ""
	stdin = new(bytes.Buffer)
	_, err := io.Copy(stdin, strings.NewReader("test3@foo.com\n"))
	if err != nil {
		t.Fatalf("Could not simulate user input, error: %v", err)
	}
	actual = getEmail(testStorage, true)
	if actual != "test3@foo.com" {
		t.Errorf("Did not get correct email from user input prompt; expected '%s' but got '%s'", "test3@foo.com", actual)
	}

	// Test3: Get most recent email from before
	DefaultEmail = ""
	for i, eml := range []string{
		"TEST4-3@foo.com", // test case insensitivity
		"test4-2@foo.com",
		"test4-1@foo.com",
	} {
		u, err := newUser(eml)
		if err != nil {
			t.Fatalf("Error creating user %d: %v", i, err)
		}
		err = saveUser(testStorage, u)
		if err != nil {
			t.Fatalf("Error saving user %d: %v", i, err)
		}

		// Change modified time so they're all different, so the test becomes deterministic
		f, err := os.Stat(testStorage.user(eml))
		if err != nil {
			t.Fatalf("Could not access user folder for '%s': %v", eml, err)
		}
		chTime := f.ModTime().Add(-(time.Duration(i) * time.Second))
		if err := os.Chtimes(testStorage.user(eml), chTime, chTime); err != nil {
			t.Fatalf("Could not change user folder mod time for '%s': %v", eml, err)
		}
	}
	actual = getEmail(testStorage, true)
	if actual != "test4-3@foo.com" {
		t.Errorf("Did not get correct email from storage; expected '%s' but got '%s'", "test4-3@foo.com", actual)
	}
}

var testStorage = FileStorage("./testdata")

func (s FileStorage) clean() error {
	return os.RemoveAll(string(s))
}
<commit_msg>Use P384 for TestUser (privateKey) (#1009)<commit_after>package caddytls

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"io"
	"strings"
	"testing"
	"time"

	"github.com/xenolf/lego/acme"
	"os"
)

func TestUser(t *testing.T) {
	defer testStorage.clean()

	privateKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
	if err != nil {
		t.Fatalf("Could not generate test private key: %v", err)
	}
	u := User{
		Email:        "me@mine.com",
		Registration: new(acme.RegistrationResource),
		key:          privateKey,
	}

	if expected, actual := "me@mine.com", u.GetEmail(); actual != expected {
		t.Errorf("Expected email '%s' but got '%s'", expected, actual)
	}
	if u.GetRegistration() == nil {
		t.Error("Expected a registration resource, but got nil")
	}
	if expected, actual := privateKey, u.GetPrivateKey(); actual != expected {
		t.Errorf("Expected the private key at address %p but got one at %p instead ", expected, actual)
	}
}

func TestNewUser(t *testing.T) {
	email := "me@foobar.com"
	user, err := newUser(email)
	if err != nil {
		t.Fatalf("Error creating user: %v", err)
	}
	if user.key == nil {
		t.Error("Private key is nil")
	}
	if user.Email != email {
		t.Errorf("Expected email to be %s, but was %s", email, user.Email)
	}
	if user.Registration != nil {
		t.Error("New user already has a registration resource; it shouldn't")
	}
}

func TestSaveUser(t *testing.T) {
	defer testStorage.clean()

	email := "me@foobar.com"
	user, err := newUser(email)
	if err != nil {
		t.Fatalf("Error creating user: %v", err)
	}

	err = saveUser(testStorage, user)
	if err != nil {
		t.Fatalf("Error saving user: %v", err)
	}
	_, err = testStorage.LoadUser(email)
	if err != nil {
		t.Errorf("Cannot access user data, error: %v", err)
	}
}

func TestGetUserDoesNotAlreadyExist(t *testing.T) {
	defer testStorage.clean()

	user, err := getUser(testStorage, "user_does_not_exist@foobar.com")
	if err != nil {
		t.Fatalf("Error getting user: %v", err)
	}

	if user.key == nil {
		t.Error("Expected user to have a private key, but it was nil")
	}
}

func TestGetUserAlreadyExists(t *testing.T) {
	defer testStorage.clean()

	email := "me@foobar.com"

	// Set up test
	user, err := newUser(email)
	if err != nil {
		t.Fatalf("Error creating user: %v", err)
	}
	err = saveUser(testStorage, user)
	if err != nil {
		t.Fatalf("Error saving user: %v", err)
	}

	// Expect to load user from disk
	user2, err := getUser(testStorage, email)
	if err != nil {
		t.Fatalf("Error getting user: %v", err)
	}

	// Assert keys are the same
	if !PrivateKeysSame(user.key, user2.key) {
		t.Error("Expected private key to be the same after loading, but it wasn't")
	}

	// Assert emails are the same
	if user.Email != user2.Email {
		t.Errorf("Expected emails to be equal, but was '%s' before and '%s' after loading", user.Email, user2.Email)
	}
}

func TestGetEmail(t *testing.T) {
	storageBasePath = string(testStorage) // to contain calls that create a new Storage...

	// let's not clutter up the output
	origStdout := os.Stdout
	os.Stdout = nil
	defer func() { os.Stdout = origStdout }()

	defer testStorage.clean()
	DefaultEmail = "test2@foo.com"

	// Test1: Use default email from flag (or user previously typing it)
	actual := getEmail(testStorage, true)
	if actual != DefaultEmail {
		t.Errorf("Did not get correct email from memory; expected '%s' but got '%s'", DefaultEmail, actual)
	}

	// Test2: Get input from user
	DefaultEmail = ""
	stdin = new(bytes.Buffer)
	_, err := io.Copy(stdin, strings.NewReader("test3@foo.com\n"))
	if err != nil {
		t.Fatalf("Could not simulate user input, error: %v", err)
	}
	actual = getEmail(testStorage, true)
	if actual != "test3@foo.com" {
		t.Errorf("Did not get correct email from user input prompt; expected '%s' but got '%s'", "test3@foo.com", actual)
	}

	// Test3: Get most recent email from before
	DefaultEmail = ""
	for i, eml := range []string{
		"TEST4-3@foo.com", // test case insensitivity
		"test4-2@foo.com",
		"test4-1@foo.com",
	} {
		u, err := newUser(eml)
		if err != nil {
			t.Fatalf("Error creating user %d: %v", i, err)
		}
		err = saveUser(testStorage, u)
		if err != nil {
			t.Fatalf("Error saving user %d: %v", i, err)
		}

		// Change modified time so they're all different, so the test becomes deterministic
		f, err := os.Stat(testStorage.user(eml))
		if err != nil {
			t.Fatalf("Could not access user folder for '%s': %v", eml, err)
		}
		chTime := f.ModTime().Add(-(time.Duration(i) * time.Second))
		if err := os.Chtimes(testStorage.user(eml), chTime, chTime); err != nil {
			t.Fatalf("Could not change user folder mod time for '%s': %v", eml, err)
		}
	}
	actual = getEmail(testStorage, true)
	if actual != "test4-3@foo.com" {
		t.Errorf("Did not get correct email from storage; expected '%s' but got '%s'", "test4-3@foo.com", actual)
	}
}

var testStorage = FileStorage("./testdata")

func (s FileStorage) clean() error {
	return os.RemoveAll(string(s))
}
<|endoftext|>