<commit_before>package php

import "stephensearles.com/php/ast"

/*

Valid Expression Patterns
Expr [Binary Op] Expr
[Unary Op] Expr
Expr [Unary Op]
Expr [Tertiary Op 1] Expr [Tertiary Op 2] Expr
Identifier
Literal
Function Call

Parentesis always triggers sub-expression

non-associative clone new clone and new
left  [ array()
right ++ -- ~ (int) (float) (string) (array) (object) (bool) @  types and increment/decrement
non-associative instanceof  types
right ! logical
left  * / % arithmetic
left  + - . arithmetic and string
left  << >> bitwise
non-associative < <= > >= comparison
non-associative == != === !== <>  comparison
left  & bitwise and references
left  ^ bitwise
left  | bitwise
left  &&  logical
left  ||  logical
left  ? : ternary
right = += -= *= /= .= %= &= |= ^= <<= >>= => assignment
left  and logical
left  xor logical
left  or  logical
left  , many uses

*/

var operatorPrecedence = map[ItemType]int{
	itemArrayLookupOperatorLeft:  19,
	itemArrayLookupOperatorRight: 19,
	itemUnaryOperator:            18,
	itemCastOperator:             18,
	itemInstanceofOperator:       17,
	itemNegationOperator:         16,
	itemMultOperator:             15,
	itemAdditionOperator:         14,
	itemSubtractionOperator:      14,
	itemConcatenationOperator:    14,

	itemBitwiseShiftOperator: 13,
	itemComparisonOperator:   12,
	itemEqualityOperator:     11,

	itemAmpersandOperator:  10,
	itemBitwiseXorOperator: 9,
	itemBitwiseOrOperator:  8,
	itemAndOperator:        7,
	itemOrOperator:         6,
	itemTernaryOperator1:   5,
	itemTernaryOperator2:   5,
	itemAssignmentOperator: 4,
	itemWrittenAndOperator: 3,
	itemWrittenXorOperator: 2,
	itemWrittenOrOperator:  1,
}

func (p *parser) parseExpression() (expr ast.Expression) {
	// consume expression
	originalParenLev := p.parenLevel
	switch p.current.typ {
	case itemNewOperator:
		return &ast.NewExpression{
			Expression: p.parseNextExpression(),
		}
	case itemUnaryOperator, itemNegationOperator, itemAmpersandOperator:
		op := p.current
		expr = p.parseUnaryExpressionRight(p.parseNextExpression(), op)
	case itemArray:
		return p.parseArrayDeclaration()
	case itemIdentifier:
		if p.peek().typ == itemAssignmentOperator {
			assignee := p.parseIdentifier().(ast.Assignable)
			p.next()
			return ast.AssignmentExpression{
				Assignee: assignee,
				Operator: p.current.val,
				Value:    p.parseNextExpression(),
			}
		}
		fallthrough
	case itemNonVariableIdentifier, itemStringLiteral, itemNumberLiteral, itemBooleanLiteral:
		expr = p.parseOperation(originalParenLev, p.expressionize())
	case itemOpenParen:
		p.parenLevel += 1
		p.next()
		expr = p.parseExpression()
		p.expect(itemCloseParen)
		p.parenLevel -= 1
		expr = p.parseOperation(originalParenLev, expr)
	default:
		p.errorf("Expected expression. Found %s", p.current)
		return nil
	}
	if p.parenLevel != originalParenLev {
		p.errorf("unbalanced parens: %d prev: %d", p.parenLevel, originalParenLev)
		return nil
	}
	return
}

func (p *parser) parseOperation(originalParenLevel int, lhs ast.Expression) (expr ast.Expression) {
	p.next()
	switch p.current.typ {
	case itemUnaryOperator:
		expr = p.parseUnaryExpressionLeft(lhs, p.current)
	case itemAdditionOperator, itemSubtractionOperator, itemConcatenationOperator, itemComparisonOperator, itemMultOperator, itemAndOperator, itemOrOperator, itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator, itemWrittenAndOperator, itemWrittenXorOperator, itemWrittenOrOperator:
		expr = p.parseBinaryOperation(lhs, p.current, originalParenLevel)
	case itemTernaryOperator1:
		expr = p.parseTernaryOperation(lhs)
	case itemCloseParen:
		if p.parenLevel <= originalParenLevel {
			p.backup()
			return lhs
		}
		p.parenLevel -= 1
		return p.parseOperation(originalParenLevel, lhs)
	default:
		p.backup()
		return lhs
	}
	return p.parseOperation(originalParenLevel, expr)
}

func (p *parser) parseBinaryOperation(lhs ast.Expression, operator Item, originalParenLevel int) ast.Expression {
	p.next()
	rhs := p.expressionize()
	for {
		p.next()
		nextOperator := p.current
		p.backup()
		nextOperatorPrecedence, ok := operatorPrecedence[nextOperator.typ]
		if ok && nextOperatorPrecedence > operatorPrecedence[operator.typ] {
			rhs = p.parseOperation(originalParenLevel, rhs)
		} else {
			break
		}
	}
	return newBinaryOperation(operator, lhs, rhs)
}

func (p *parser) parseTernaryOperation(lhs ast.Expression) ast.Expression {
	truthy := p.parseNextExpression()
	p.expect(itemTernaryOperator2)
	falsy := p.parseNextExpression()
	return &ast.OperatorExpression{
		Operand1: lhs,
		Operand2: truthy,
		Operand3: falsy,
		Type:     truthy.EvaluatesTo() | falsy.EvaluatesTo(),
		Operator: "?:",
	}
}

func (p *parser) parseUnaryExpressionRight(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

func (p *parser) parseUnaryExpressionLeft(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

// expressionize takes the current token and returns it as the simplest
// expression for that token. That means an expression with no operators
// except for the object operator.
func (p *parser) expressionize() ast.Expression {
	switch p.current.typ {
	case itemIdentifier:
		return p.parseIdentifier()
	case itemStringLiteral, itemBooleanLiteral, itemNumberLiteral:
		return p.parseLiteral()
	case itemNonVariableIdentifier:
		if p.peek().typ == itemOpenParen {
			expr := p.parseFunctionCall()
			if p.peek().typ == itemObjectOperator {
				return p.parseObjectLookup(expr)
			}
			return expr
		}
		if p.peek().typ == itemScopeResolutionOperator {
			p.expect(itemScopeResolutionOperator)
			return &ast.ClassExpression{
				Receiver:   p.current.val,
				Expression: p.parseNextExpression(),
			}
		}
		return ast.ConstantExpression{
			Identifier: ast.NewIdentifier(p.current.val),
		}
	case itemOpenParen:
		return p.parseExpression()
	}
	// error?
	return nil
}

func (p *parser) parseLiteral() *ast.Literal {
	switch p.current.typ {
	case itemStringLiteral:
		return &ast.Literal{Type: ast.String}
	case itemBooleanLiteral:
		return &ast.Literal{Type: ast.Boolean}
	case itemNumberLiteral:
		return &ast.Literal{Type: ast.Float}
	}
	p.errorf("Unknown literal type")
	return nil
}

func (p *parser) parseIdentifier() ast.Expression {
	ident := ast.NewIdentifier(p.current.val)
	switch pk := p.peek(); pk.typ {
	case itemObjectOperator:
		return p.parseObjectLookup(ident)
	case itemArrayLookupOperatorLeft:
		return p.parseArrayLookup(ident)
	}
	return ident
}

func (p *parser) parseObjectLookup(r ast.Expression) ast.Expression {
	p.expect(itemObjectOperator)
	p.expect(itemNonVariableIdentifier)
	if pk := p.peek(); pk.typ == itemOpenParen {
		expr := &ast.MethodCallExpression{
			Receiver:               r,
			FunctionCallExpression: p.parseFunctionCall(),
		}
		return expr
	}
	return &ast.PropertyExpression{
		Receiver: r,
		Name:     p.current.val,
	}
}

func (p *parser) parseArrayLookup(e ast.Expression) ast.Expression {
	p.expect(itemArrayLookupOperatorLeft)
	if p.peek().typ == itemArrayLookupOperatorRight {
		p.expect(itemArrayLookupOperatorRight)
		return ast.ArrayAppendExpression{Array: e}
	}
	p.next()
	expr := &ast.ArrayLookupExpression{
		Array: e,
		Index: p.parseExpression(),
	}
	p.expect(itemArrayLookupOperatorRight)
	if p.peek().typ == itemArrayLookupOperatorLeft {
		return p.parseArrayLookup(expr)
	}
	return expr
}

func (p *parser) parseArrayDeclaration() ast.Expression {
	pairs := make([]ast.ArrayPair, 0)
	p.expect(itemOpenParen)
	var key, val ast.Expression
ArrayLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemArrayKeyOperator:
			if val == nil {
				p.errorf("expected array key before =>.")
				return nil
			}
			key = val
			p.next()
			val = p.parseExpression()
		case itemCloseParen:
			if val != nil {
				pairs = append(pairs, ast.ArrayPair{key, val})
			}
			break ArrayLoop
		case itemArgumentSeparator:
			pairs = append(pairs, ast.ArrayPair{key, val})
			key = nil
			val = nil
		default:
			val = p.parseExpression()
		}
	}
	return &ast.ArrayExpression{Pairs: pairs}
}
<commit_msg>Ignoring the ignore-error operator at the start of an expression<commit_after>package php

import "stephensearles.com/php/ast"

/*

Valid Expression Patterns
Expr [Binary Op] Expr
[Unary Op] Expr
Expr [Unary Op]
Expr [Tertiary Op 1] Expr [Tertiary Op 2] Expr
Identifier
Literal
Function Call

Parentesis always triggers sub-expression

non-associative clone new clone and new
left  [ array()
right ++ -- ~ (int) (float) (string) (array) (object) (bool) @  types and increment/decrement
non-associative instanceof  types
right ! logical
left  * / % arithmetic
left  + - . arithmetic and string
left  << >> bitwise
non-associative < <= > >= comparison
non-associative == != === !== <>  comparison
left  & bitwise and references
left  ^ bitwise
left  | bitwise
left  &&  logical
left  ||  logical
left  ? : ternary
right = += -= *= /= .= %= &= |= ^= <<= >>= => assignment
left  and logical
left  xor logical
left  or  logical
left  , many uses

*/

var operatorPrecedence = map[ItemType]int{
	itemArrayLookupOperatorLeft:  19,
	itemArrayLookupOperatorRight: 19,
	itemUnaryOperator:            18,
	itemCastOperator:             18,
	itemInstanceofOperator:       17,
	itemNegationOperator:         16,
	itemMultOperator:             15,
	itemAdditionOperator:         14,
	itemSubtractionOperator:      14,
	itemConcatenationOperator:    14,

	itemBitwiseShiftOperator: 13,
	itemComparisonOperator:   12,
	itemEqualityOperator:     11,

	itemAmpersandOperator:  10,
	itemBitwiseXorOperator: 9,
	itemBitwiseOrOperator:  8,
	itemAndOperator:        7,
	itemOrOperator:         6,
	itemTernaryOperator1:   5,
	itemTernaryOperator2:   5,
	itemAssignmentOperator: 4,
	itemWrittenAndOperator: 3,
	itemWrittenXorOperator: 2,
	itemWrittenOrOperator:  1,
}

func (p *parser) parseExpression() (expr ast.Expression) {
	// consume expression
	originalParenLev := p.parenLevel
	switch p.current.typ {
	case itemIgnoreErrorOperator:
		p.next()
		return p.parseExpression()
	case itemNewOperator:
		return &ast.NewExpression{
			Expression: p.parseNextExpression(),
		}
	case itemUnaryOperator, itemNegationOperator, itemAmpersandOperator:
		op := p.current
		expr = p.parseUnaryExpressionRight(p.parseNextExpression(), op)
	case itemArray:
		return p.parseArrayDeclaration()
	case itemIdentifier:
		if p.peek().typ == itemAssignmentOperator {
			assignee := p.parseIdentifier().(ast.Assignable)
			p.next()
			return ast.AssignmentExpression{
				Assignee: assignee,
				Operator: p.current.val,
				Value:    p.parseNextExpression(),
			}
		}
		fallthrough
	case itemNonVariableIdentifier, itemStringLiteral, itemNumberLiteral, itemBooleanLiteral:
		expr = p.parseOperation(originalParenLev, p.expressionize())
	case itemOpenParen:
		p.parenLevel += 1
		p.next()
		expr = p.parseExpression()
		p.expect(itemCloseParen)
		p.parenLevel -= 1
		expr = p.parseOperation(originalParenLev, expr)
	default:
		p.errorf("Expected expression. Found %s", p.current)
		return nil
	}
	if p.parenLevel != originalParenLev {
		p.errorf("unbalanced parens: %d prev: %d", p.parenLevel, originalParenLev)
		return nil
	}
	return
}

func (p *parser) parseOperation(originalParenLevel int, lhs ast.Expression) (expr ast.Expression) {
	p.next()
	switch p.current.typ {
	case itemUnaryOperator:
		expr = p.parseUnaryExpressionLeft(lhs, p.current)
	case itemAdditionOperator, itemSubtractionOperator, itemConcatenationOperator, itemComparisonOperator, itemMultOperator, itemAndOperator, itemOrOperator, itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator, itemWrittenAndOperator, itemWrittenXorOperator, itemWrittenOrOperator:
		expr = p.parseBinaryOperation(lhs, p.current, originalParenLevel)
	case itemTernaryOperator1:
		expr = p.parseTernaryOperation(lhs)
	case itemCloseParen:
		if p.parenLevel <= originalParenLevel {
			p.backup()
			return lhs
		}
		p.parenLevel -= 1
		return p.parseOperation(originalParenLevel, lhs)
	default:
		p.backup()
		return lhs
	}
	return p.parseOperation(originalParenLevel, expr)
}

func (p *parser) parseBinaryOperation(lhs ast.Expression, operator Item, originalParenLevel int) ast.Expression {
	p.next()
	rhs := p.expressionize()
	for {
		p.next()
		nextOperator := p.current
		p.backup()
		nextOperatorPrecedence, ok := operatorPrecedence[nextOperator.typ]
		if ok && nextOperatorPrecedence > operatorPrecedence[operator.typ] {
			rhs = p.parseOperation(originalParenLevel, rhs)
		} else {
			break
		}
	}
	return newBinaryOperation(operator, lhs, rhs)
}

func (p *parser) parseTernaryOperation(lhs ast.Expression) ast.Expression {
	truthy := p.parseNextExpression()
	p.expect(itemTernaryOperator2)
	falsy := p.parseNextExpression()
	return &ast.OperatorExpression{
		Operand1: lhs,
		Operand2: truthy,
		Operand3: falsy,
		Type:     truthy.EvaluatesTo() | falsy.EvaluatesTo(),
		Operator: "?:",
	}
}

func (p *parser) parseUnaryExpressionRight(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

func (p *parser) parseUnaryExpressionLeft(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

// expressionize takes the current token and returns it as the simplest
// expression for that token. That means an expression with no operators
// except for the object operator.
func (p *parser) expressionize() ast.Expression {
	switch p.current.typ {
	case itemIdentifier:
		return p.parseIdentifier()
	case itemStringLiteral, itemBooleanLiteral, itemNumberLiteral:
		return p.parseLiteral()
	case itemNonVariableIdentifier:
		if p.peek().typ == itemOpenParen {
			expr := p.parseFunctionCall()
			if p.peek().typ == itemObjectOperator {
				return p.parseObjectLookup(expr)
			}
			return expr
		}
		if p.peek().typ == itemScopeResolutionOperator {
			p.expect(itemScopeResolutionOperator)
			return &ast.ClassExpression{
				Receiver:   p.current.val,
				Expression: p.parseNextExpression(),
			}
		}
		return ast.ConstantExpression{
			Identifier: ast.NewIdentifier(p.current.val),
		}
	case itemOpenParen:
		return p.parseExpression()
	}
	// error?
	return nil
}

func (p *parser) parseLiteral() *ast.Literal {
	switch p.current.typ {
	case itemStringLiteral:
		return &ast.Literal{Type: ast.String}
	case itemBooleanLiteral:
		return &ast.Literal{Type: ast.Boolean}
	case itemNumberLiteral:
		return &ast.Literal{Type: ast.Float}
	}
	p.errorf("Unknown literal type")
	return nil
}

func (p *parser) parseIdentifier() ast.Expression {
	ident := ast.NewIdentifier(p.current.val)
	switch pk := p.peek(); pk.typ {
	case itemObjectOperator:
		return p.parseObjectLookup(ident)
	case itemArrayLookupOperatorLeft:
		return p.parseArrayLookup(ident)
	}
	return ident
}

func (p *parser) parseObjectLookup(r ast.Expression) ast.Expression {
	p.expect(itemObjectOperator)
	p.expect(itemNonVariableIdentifier)
	if pk := p.peek(); pk.typ == itemOpenParen {
		expr := &ast.MethodCallExpression{
			Receiver:               r,
			FunctionCallExpression: p.parseFunctionCall(),
		}
		return expr
	}
	return &ast.PropertyExpression{
		Receiver: r,
		Name:     p.current.val,
	}
}

func (p *parser) parseArrayLookup(e ast.Expression) ast.Expression {
	p.expect(itemArrayLookupOperatorLeft)
	if p.peek().typ == itemArrayLookupOperatorRight {
		p.expect(itemArrayLookupOperatorRight)
		return ast.ArrayAppendExpression{Array: e}
	}
	p.next()
	expr := &ast.ArrayLookupExpression{
		Array: e,
		Index: p.parseExpression(),
	}
	p.expect(itemArrayLookupOperatorRight)
	if p.peek().typ == itemArrayLookupOperatorLeft {
		return p.parseArrayLookup(expr)
	}
	return expr
}

func (p *parser) parseArrayDeclaration() ast.Expression {
	pairs := make([]ast.ArrayPair, 0)
	p.expect(itemOpenParen)
	var key, val ast.Expression
ArrayLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemArrayKeyOperator:
			if val == nil {
				p.errorf("expected array key before =>.")
				return nil
			}
			key = val
			p.next()
			val = p.parseExpression()
		case itemCloseParen:
			if val != nil {
				pairs = append(pairs, ast.ArrayPair{key, val})
			}
			break ArrayLoop
		case itemArgumentSeparator:
			pairs = append(pairs, ast.ArrayPair{key, val})
			key = nil
			val = nil
		default:
			val = p.parseExpression()
		}
	}
	return &ast.ArrayExpression{Pairs: pairs}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Makoto Ito
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

// Config stores the common config.
type Config struct {
	Dimension        int
	Window           int
	InitLearningRate float64
	Thread           int
	ToLower          bool
	Verbose          bool
}

// NewConfig creates *Config
func NewConfig(dimension, window int, initlr float64, thread int, toLower, verbose bool) *Config {
	return &Config{
		Dimension:        dimension,
		Window:           window,
		InitLearningRate: initlr,
		Thread:           thread,
		ToLower:          toLower,
		Verbose:          verbose,
	}
}
<commit_msg>Add iter, minCount<commit_after>// Copyright © 2017 Makoto Ito
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

// Config stores the common config.
type Config struct {
	Dimension        int
	Iteration        int
	MinCount         int
	Thread           int
	Window           int
	InitLearningRate float64
	ToLower          bool
	Verbose          bool
}

// NewConfig creates *Config
func NewConfig(dimension, iteration, minCount, thread, window int, initlr float64, toLower, verbose bool) *Config {
	return &Config{
		Dimension:        dimension,
		Iteration:        iteration,
		MinCount:         minCount,
		Thread:           thread,
		Window:           window,
		InitLearningRate: initlr,
		ToLower:          toLower,
		Verbose:          verbose,
	}
}
<|endoftext|>


<commit_before>package model

import (
	"appengine"
	"appengine/datastore"
	"log"
)

type Player struct {
	Name string
	Email string
	Faction string
}

func playerKey(c appengine.Context, s *Season, email string) *datastore.Key {
	var sKey *datastore.Key
	sKey = nil
	if s != nil {
		sKey = seasonKey(c, s.Name, s.Year)
	}
	return datastore.NewKey(c, "Player", email, 0, sKey)
}

func SavePlayer(c appengine.Context, s *Season, name string, email string, faction string) error {
	p := Player {
		Name: name,
		Email: email,
		Faction: faction,
	}
	key := playerKey(c, s, email)
	_, err := datastore.Put(c, key, &p)
	return err
}

func LoadPlayer(c appengine.Context, s *Season, email string) *Player {
	key := playerKey(c, s, email)
	var p Player
	err := datastore.Get(c, key, &p)
	if err == datastore.ErrNoSuchEntity {
		return nil
	} else if err != nil {
		log.Printf("Got an unexpected error looking up a season: %v", err)
	}
	return &p
}
<commit_msg>Added in implementation of reading player csv data and saving it to a player object.  No unit tests for it yet.<commit_after>package model

import (
	"appengine"
	"appengine/datastore"
	"log"
	"encoding/csv"
	"strings"
)

type Player struct {
	Name string
	Email string
	Faction string
}

func playerKey(c appengine.Context, s *Season, email string) *datastore.Key {
	var sKey *datastore.Key
	sKey = nil
	if s != nil {
		sKey = seasonKey(c, s.Name, s.Year)
	}
	return datastore.NewKey(c, "Player", email, 0, sKey)
}

func SavePlayer(c appengine.Context, s *Season, name string, email string, faction string) error {
	p := Player {
		Name: name,
		Email: email,
		Faction: faction,
	}
	key := playerKey(c, s, email)
	_, err := datastore.Put(c, key, &p)
	return err
}

func LoadPlayer(c appengine.Context, s *Season, email string) *Player {
	key := playerKey(c, s, email)
	var p Player
	err := datastore.Get(c, key, &p)
	if err == datastore.ErrNoSuchEntity {
		return nil
	} else if err != nil {
		log.Printf("Got an unexpected error looking up a season: %v", err)
	}
	return &p
}

func CreatePlayersFromCsv(c appengine.Context, owningSeason *Season, csvData string) {
	strReader := strings.NewReader(csvData)
	csvReader := csv.NewReader(strReader)
	records, err := csvReader.ReadAll()
	if err != nil {
		log.Printf("Got an unexpected error [%v] reading csv data:\n%v", err, csvData)
		panic(err)
	}
	for _, row := range records {
		SavePlayer(c, owningSeason, row[0], row[2], row[1])
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/dutchcoders/goftp"
	"github.com/garyburd/redigo/redis"
)

var (
	sshKey    = flag.String("key", "", "SSH key to use for cloning")
	ftpUrl    = flag.String("ftp", "", "FTP server to save backups to")
	redisUrl  = flag.String("redis", "", "Address of redis")
	frequency = flag.Duration("frequency", 24*time.Hour, "Frequency of backups")
	force     = flag.Bool("force", false, "Force download")
	help      = flag.Bool("help", false, "Show this help")
)

var (
	badCharacters = regexp.MustCompilePOSIX("[/@:!?*\\&]")
)

func main() {
	flag.Parse()
	if *help {
		flag.PrintDefaults()
		return
	}
	if *ftpUrl == "" || *redisUrl == "" {
		log.Fatalf("-ftp and -redis have to be set")
	}

	if *sshKey != "" {
		if err := addSshKey(*sshKey); err != nil {
			log.Fatalf("Could not add SSH key: %s", err)
		}
	}

	redisConn, err := connectRedis(*redisUrl)
	if err != nil {
		log.Fatalf("Could not connect to redis: %s", err)
	}
	defer redisConn.Close()

	ftpConn, ftpUrl, err := connectFtp(*ftpUrl)
	if err != nil {
		log.Fatalf("Could not connect to FTP server: %s", err)
	}
	defer ftpConn.Close()
	if err := ftpConn.Cwd(ftpUrl.Path); err != nil {
		log.Fatalf("Could not cd to target directory: %s", err)
	}

	for {
		if !*force {
			nextRun := lastRun(redisConn).Add(*frequency)
			if nextRun.After(time.Now()) {
				time.Sleep(nextRun.Sub(time.Now()))
				continue
			}
		}
		*force = false

		log.Printf("Downloading all the repos...")
		repos := repos(redisConn)
		for _, repo := range repos {
			log.Printf("Downloading %s...", repo)
			safeName := badCharacters.ReplaceAllString(repo, "_")
			buf, err := downloadRepository(repo)
			if err != nil {
				log.Printf("Error downloading repository: %s", err)
				continue
			}

			if err := ftpConn.Stor(safeName+".tar.gz", buf); err != nil {
				log.Printf("Error uploading: %s", err)
			}
		}
		log.Printf("Finished.")
		timestampLastRun(redisConn)
	}
}

func lastRun(conn redis.Conn) time.Time {
	ok, err := redis.Bool(conn.Do("EXISTS", "github-backup:lastrun"))
	if err != nil {
		log.Fatalf("Error querying database: %s", err)
	}
	if !ok {
		return time.Unix(0, 0)
	}

	ts, err := redis.String(conn.Do("GET", "github-backup:lastrun"))
	if err != nil {
		log.Fatalf("Error retrieving timestamp: %s", err)
	}
	t, err := time.Parse(time.RFC3339, ts)
	if err != nil {
		log.Fatalf("Error parsing timestamp: %s", err)
	}
	return t
}

func timestampLastRun(conn redis.Conn) {
	_, err := conn.Do("SET", "github-backup:lastrun", time.Now().Format(time.RFC3339))
	if err != nil {
		log.Fatalf("Error writing timestamp: %s", err)
	}
}

func repos(conn redis.Conn) []string {
	repos, err := redis.Values(conn.Do("LRANGE", "github-backup:repos", 0, 1000))
	if err == redis.ErrNil {
		return []string{}
	}
	if err != nil {
		log.Fatalf("Error retrieving repo list: %s", err)
	}
	r := make([]string, 0, len(repos))
	if err := redis.ScanSlice(repos, &r); err != nil {
		log.Fatalf("Error parsing repo list: %s", err)
	}
	return r
}

func connectRedis(s string) (redis.Conn, error) {
	redisUrl, err := url.Parse(s)
	if err != nil {
		return nil, fmt.Errorf("Could not parse redis url: %s", err)
	}
	if redisUrl.Scheme != "redis" {
		return nil, fmt.Errorf("Unsupported redis scheme %s", redisUrl.Scheme)
	}

	conn, err := redis.Dial("tcp", redisUrl.Host)
	if err != nil {
		return conn, err
	}
	if redisUrl.User != nil {
		pass, ok := redisUrl.User.Password()
		if !ok {
			pass = redisUrl.User.Username()
		}
		_, err := conn.Do("AUTH", pass)
		if err != nil {
			return conn, err
		}
	}
	_, err = conn.Do("EXISTS", "github-backup:lastrun")
	return conn, err
}

func connectFtp(s string) (*goftp.FTP, *url.URL, error) {
	ftpUrl, err := url.Parse(s)
	if err != nil {
		log.Fatalf("Invalid ftp url: %s", err)
	}
	if ftpUrl.Scheme != "ftp" {
		log.Fatalf("Unsupported target scheme %s", ftpUrl.Scheme)
	}
	if !strings.Contains(ftpUrl.Host, ":") {
		ftpUrl.Host += ":21"
	}

	ftp, err := goftp.Connect(ftpUrl.Host)
	if err != nil {
		return ftp, ftpUrl, err
	}
	if ftpUrl.User == nil {
		return ftp, ftpUrl, err
	}
	user := ftpUrl.User.Username()
	pass, _ := ftpUrl.User.Password()
	return ftp, ftpUrl, ftp.Login(user, pass)
}

func downloadRepository(path string) (*bytes.Buffer, error) {
	repo := os.TempDir() + "github-backup"

	if err := os.MkdirAll(repo, os.FileMode(0700)); err != nil {
		return nil, err
	}
	defer os.RemoveAll(repo)

	cmd := exec.Command("git", "clone", "--bare", path)
	cmd.Dir = repo
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}

	return tarDir(repo)
}

func tarDir(root string) (*bytes.Buffer, error) {
	buf := &bytes.Buffer{}
	gzbuf := gzip.NewWriter(buf)
	defer gzbuf.Close()
	defer gzbuf.Flush()
	archive := tar.NewWriter(gzbuf)
	defer archive.Close()
	defer archive.Flush()
	err := filepath.Walk(root, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
		if path == root {
			return nil
		}
		relPath := strings.TrimPrefix(path, root)
		hdr := &tar.Header{
			Name:     strings.TrimPrefix(relPath, "/"),
			Mode:     int64(info.Mode() & os.ModePerm),
			Uid:      1000,
			Gid:      1000,
			Size:     info.Size(),
			Typeflag: tar.TypeReg,
		}
		if info.IsDir() {
			hdr.Typeflag = tar.TypeDir
			hdr.Size = 0
		}

		if err := archive.WriteHeader(hdr); err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()
		if _, err := io.Copy(archive, f); err != nil && err != io.EOF {
			return err
		}
		return nil
	}))
	if err != nil {
		return nil, err
	}
	return buf, nil
}

const (
	sshConfig = `
	IdentityFile ~/.ssh/id-rsa
	StrictHostKeyChecking no
	`
)

func writeFile(path string, content []byte) error {
	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(0700))
	if err != nil {
		return fmt.Errorf("Error creating file %s: %s", path, err)
	}
	defer f.Close()
	if _, err := f.Write(content); err != nil {
		return fmt.Errorf("Error writing file %s: %s", path, err)
	}
	return nil
}

func addSshKey(encKey string) error {
	key, err := base64.StdEncoding.DecodeString(encKey)
	if err != nil {
		return fmt.Errorf("Error decoding key: %s", err)
	}
	if err := os.MkdirAll(os.ExpandEnv("$HOME/.ssh"), os.FileMode(0700)); err != nil {
		return fmt.Errorf("Error creating .ssh folder: %s", err)
	}

	if err := writeFile(os.ExpandEnv("$HOME/.ssh/id-rsa"), key); err != nil {
		return err
	}
	if err := writeFile(os.ExpandEnv("$HOME/.ssh/config"), []byte(sshConfig)); err != nil {
		return err
	}
	return nil
}
<commit_msg>Use hard-coded paths for now<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/dutchcoders/goftp"
	"github.com/garyburd/redigo/redis"
)

var (
	sshKey    = flag.String("key", "", "SSH key to use for cloning")
	ftpUrl    = flag.String("ftp", "", "FTP server to save backups to")
	redisUrl  = flag.String("redis", "", "Address of redis")
	frequency = flag.Duration("frequency", 24*time.Hour, "Frequency of backups")
	force     = flag.Bool("force", false, "Force download")
	help      = flag.Bool("help", false, "Show this help")
)

var (
	badCharacters = regexp.MustCompilePOSIX("[/@:!?*\\&]")
)

func main() {
	flag.Parse()
	if *help {
		flag.PrintDefaults()
		return
	}
	if *ftpUrl == "" || *redisUrl == "" {
		log.Fatalf("-ftp and -redis have to be set")
	}

	if *sshKey != "" {
		if err := addSshKey(*sshKey); err != nil {
			log.Fatalf("Could not add SSH key: %s", err)
		}
	}

	redisConn, err := connectRedis(*redisUrl)
	if err != nil {
		log.Fatalf("Could not connect to redis: %s", err)
	}
	defer redisConn.Close()

	ftpConn, ftpUrl, err := connectFtp(*ftpUrl)
	if err != nil {
		log.Fatalf("Could not connect to FTP server: %s", err)
	}
	defer ftpConn.Close()
	if err := ftpConn.Cwd(ftpUrl.Path); err != nil {
		log.Fatalf("Could not cd to target directory: %s", err)
	}

	for {
		if !*force {
			nextRun := lastRun(redisConn).Add(*frequency)
			if nextRun.After(time.Now()) {
				time.Sleep(nextRun.Sub(time.Now()))
				continue
			}
		}
		*force = false

		log.Printf("Downloading all the repos...")
		repos := repos(redisConn)
		for _, repo := range repos {
			log.Printf("Downloading %s...", repo)
			safeName := badCharacters.ReplaceAllString(repo, "_")
			buf, err := downloadRepository(repo)
			if err != nil {
				log.Printf("Error downloading repository: %s", err)
				continue
			}

			if err := ftpConn.Stor(safeName+".tar.gz", buf); err != nil {
				log.Printf("Error uploading: %s", err)
			}
		}
		log.Printf("Finished.")
		timestampLastRun(redisConn)
	}
}

func lastRun(conn redis.Conn) time.Time {
	ok, err := redis.Bool(conn.Do("EXISTS", "github-backup:lastrun"))
	if err != nil {
		log.Fatalf("Error querying database: %s", err)
	}
	if !ok {
		return time.Unix(0, 0)
	}

	ts, err := redis.String(conn.Do("GET", "github-backup:lastrun"))
	if err != nil {
		log.Fatalf("Error retrieving timestamp: %s", err)
	}
	t, err := time.Parse(time.RFC3339, ts)
	if err != nil {
		log.Fatalf("Error parsing timestamp: %s", err)
	}
	return t
}

func timestampLastRun(conn redis.Conn) {
	_, err := conn.Do("SET", "github-backup:lastrun", time.Now().Format(time.RFC3339))
	if err != nil {
		log.Fatalf("Error writing timestamp: %s", err)
	}
}

func repos(conn redis.Conn) []string {
	repos, err := redis.Values(conn.Do("LRANGE", "github-backup:repos", 0, 1000))
	if err == redis.ErrNil {
		return []string{}
	}
	if err != nil {
		log.Fatalf("Error retrieving repo list: %s", err)
	}
	r := make([]string, 0, len(repos))
	if err := redis.ScanSlice(repos, &r); err != nil {
		log.Fatalf("Error parsing repo list: %s", err)
	}
	return r
}

func connectRedis(s string) (redis.Conn, error) {
	redisUrl, err := url.Parse(s)
	if err != nil {
		return nil, fmt.Errorf("Could not parse redis url: %s", err)
	}
	if redisUrl.Scheme != "redis" {
		return nil, fmt.Errorf("Unsupported redis scheme %s", redisUrl.Scheme)
	}

	conn, err := redis.Dial("tcp", redisUrl.Host)
	if err != nil {
		return conn, err
	}
	if redisUrl.User != nil {
		pass, ok := redisUrl.User.Password()
		if !ok {
			pass = redisUrl.User.Username()
		}
		_, err := conn.Do("AUTH", pass)
		if err != nil {
			return conn, err
		}
	}
	_, err = conn.Do("EXISTS", "github-backup:lastrun")
	return conn, err
}

func connectFtp(s string) (*goftp.FTP, *url.URL, error) {
	ftpUrl, err := url.Parse(s)
	if err != nil {
		log.Fatalf("Invalid ftp url: %s", err)
	}
	if ftpUrl.Scheme != "ftp" {
		log.Fatalf("Unsupported target scheme %s", ftpUrl.Scheme)
	}
	if !strings.Contains(ftpUrl.Host, ":") {
		ftpUrl.Host += ":21"
	}

	ftp, err := goftp.Connect(ftpUrl.Host)
	if err != nil {
		return ftp, ftpUrl, err
	}
	if ftpUrl.User == nil {
		return ftp, ftpUrl, err
	}
	user := ftpUrl.User.Username()
	pass, _ := ftpUrl.User.Password()
	return ftp, ftpUrl, ftp.Login(user, pass)
}

func downloadRepository(path string) (*bytes.Buffer, error) {
	repo := os.TempDir() + "github-backup"

	if err := os.MkdirAll(repo, os.FileMode(0700)); err != nil {
		return nil, err
	}
	defer os.RemoveAll(repo)

	cmd := exec.Command("git", "clone", "--bare", path)
	cmd.Dir = repo
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, err
	}

	return tarDir(repo)
}

func tarDir(root string) (*bytes.Buffer, error) {
	buf := &bytes.Buffer{}
	gzbuf := gzip.NewWriter(buf)
	defer gzbuf.Close()
	defer gzbuf.Flush()
	archive := tar.NewWriter(gzbuf)
	defer archive.Close()
	defer archive.Flush()
	err := filepath.Walk(root, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error {
		if path == root {
			return nil
		}
		relPath := strings.TrimPrefix(path, root)
		hdr := &tar.Header{
			Name:     strings.TrimPrefix(relPath, "/"),
			Mode:     int64(info.Mode() & os.ModePerm),
			Uid:      1000,
			Gid:      1000,
			Size:     info.Size(),
			Typeflag: tar.TypeReg,
		}
		if info.IsDir() {
			hdr.Typeflag = tar.TypeDir
			hdr.Size = 0
		}

		if err := archive.WriteHeader(hdr); err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()
		if _, err := io.Copy(archive, f); err != nil && err != io.EOF {
			return err
		}
		return nil
	}))
	if err != nil {
		return nil, err
	}
	return buf, nil
}

const (
	sshConfig = `
	IdentityFile /root/.ssh/github-backup
	StrictHostKeyChecking no
	`
)

func writeFile(path string, content []byte) error {
	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(0700))
	if err != nil {
		return fmt.Errorf("Error creating file %s: %s", path, err)
	}
	defer f.Close()
	if _, err := f.Write(content); err != nil {
		return fmt.Errorf("Error writing file %s: %s", path, err)
	}
	return nil
}

func addSshKey(encKey string) error {
	key, err := base64.StdEncoding.DecodeString(encKey)
	if err != nil {
		return fmt.Errorf("Error decoding key: %s", err)
	}
	if err := os.MkdirAll("/root/.ssh", os.FileMode(0700)); err != nil {
		return fmt.Errorf("Error creating .ssh folder: %s", err)
	}

	if err := writeFile("/root/.ssh/github-backup", key); err != nil {
		return err
	}
	if err := writeFile("/root/.ssh/config", []byte(sshConfig)); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"sync"
	"time"

	"github.com/emirpasic/gods/sets/linkedhashset"
	"github.com/emirpasic/gods/trees/binaryheap"

	"github.com/zitryss/aye-and-nay/domain/model"
	"github.com/zitryss/aye-and-nay/pkg/errors"
)

func NewMem(opts ...options) *Mem {
	conf := newMemConfig()
	m := &Mem{
		conf:        conf,
		syncQueues:  syncQueues{queues: map[uint64]*linkedhashset.Set{}},
		syncPQueues: syncPQueues{pqueues: map[uint64]*binaryheap.Heap{}},
		syncPairs:   syncPairs{pairs: map[uint64]*pairsTime{}},
		syncTokens:  syncTokens{tokens: map[uint64]*tokenTime{}},
	}
	for _, opt := range opts {
		opt(m)
	}
	return m
}

type options func(*Mem)

func WithHeartbeatPair(ch chan<- interface{}) options {
	return func(m *Mem) {
		m.heartbeat.pair = ch
	}
}

func WithHeartbeatToken(ch chan<- interface{}) options {
	return func(m *Mem) {
		m.heartbeat.token = ch
	}
}

type Mem struct {
	conf memConfig
	syncQueues
	syncPQueues
	syncPairs
	syncTokens
	heartbeat struct {
		pair  chan<- interface{}
		token chan<- interface{}
	}
}

type syncQueues struct {
	sync.Mutex
	queues map[uint64]*linkedhashset.Set
}

type syncPQueues struct {
	sync.Mutex
	pqueues map[uint64]*binaryheap.Heap
}

type syncPairs struct {
	sync.Mutex
	pairs map[uint64]*pairsTime
}

type pairsTime struct {
	pairs [][2]uint64
	seen  time.Time
}

type syncTokens struct {
	sync.Mutex
	tokens map[uint64]*tokenTime
}

type tokenTime struct {
	token uint64
	seen  time.Time
}

type elem struct {
	album   uint64
	expires time.Time
}

func timeComparator(a, b interface{}) int {
	tA := a.(elem).expires
	tB := b.(elem).expires
	switch {
	case tA.After(tB):
		return 1
	case tA.Before(tB):
		return -1
	default:
		return 0
	}
}

func (m *Mem) Monitor() {
	go func() {
		for {
			if m.heartbeat.pair != nil {
				m.heartbeat.pair <- struct{}{}
			}
			now := time.Now()
			m.syncPairs.Lock()
			for k, v := range m.pairs {
				if now.Sub(v.seen) >= m.conf.timeToLive {
					delete(m.pairs, k)
				}
			}
			m.syncPairs.Unlock()
			time.Sleep(m.conf.cleanupInterval)
			if m.heartbeat.pair != nil {
				m.heartbeat.pair <- struct{}{}
			}
		}
	}()
	go func() {
		for {
			if m.heartbeat.token != nil {
				m.heartbeat.token <- struct{}{}
			}
			now := time.Now()
			m.syncTokens.Lock()
			for k, v := range m.tokens {
				if now.Sub(v.seen) >= m.conf.timeToLive {
					delete(m.tokens, k)
				}
			}
			m.syncTokens.Unlock()
			time.Sleep(m.conf.cleanupInterval)
			if m.heartbeat.token != nil {
				m.heartbeat.token <- struct{}{}
			}
		}
	}()
}

func (m *Mem) Add(_ context.Context, queue uint64, album uint64) error {
	m.syncQueues.Lock()
	defer m.syncQueues.Unlock()
	q, ok := m.queues[queue]
	if !ok {
		q = linkedhashset.New()
		m.queues[queue] = q
	}
	q.Add(album)
	return nil
}

func (m *Mem) Poll(_ context.Context, queue uint64) (uint64, error) {
	m.syncQueues.Lock()
	defer m.syncQueues.Unlock()
	q, ok := m.queues[queue]
	if !ok {
		return 0x0, errors.Wrap(model.ErrUnknown)
	}
	it := q.Iterator()
	if !it.Next() {
		return 0x0, errors.Wrap(model.ErrUnknown)
	}
	album := it.Value().(uint64)
	q.Remove(album)
	return album, nil
}

func (m *Mem) Size(_ context.Context, queue uint64) (int, error) {
	m.syncQueues.Lock()
	defer m.syncQueues.Unlock()
	q, ok := m.queues[queue]
	if !ok {
		return 0, nil
	}
	n := q.Size()
	return n, nil
}

func (m *Mem) PAdd(_ context.Context, pqueue uint64, album uint64, expires time.Time) error {
	m.syncPQueues.Lock()
	defer m.syncPQueues.Unlock()
	pq, ok := m.pqueues[pqueue]
	if !ok {
		pq = binaryheap.NewWith(timeComparator)
		m.pqueues[pqueue] = pq
	}
	pq.Push(elem{album, expires})
	return nil
}

func (m *Mem) PPoll(_ context.Context, pqueue uint64) (uint64, time.Time, error) {
	m.syncPQueues.Lock()
	defer m.syncPQueues.Unlock()
	pq, ok := m.pqueues[pqueue]
	if !ok {
		return 0x0, time.Time{}, errors.Wrap(model.ErrUnknown)
	}
	e, ok := pq.Pop()
	if !ok {
		return 0x0, time.Time{}, errors.Wrap(model.ErrUnknown)
	}
	return e.(elem).album, e.(elem).expires, nil
}

func (m *Mem) PSize(_ context.Context, pqueue uint64) (int, error) {
	m.syncPQueues.Lock()
	defer m.syncPQueues.Unlock()
	pq, ok := m.pqueues[pqueue]
	if !ok {
		return 0, nil
	}
	n := pq.Size()
	return n, nil
}

func (m *Mem) Push(_ context.Context, album uint64, pairs [][2]uint64) error {
	m.syncPairs.Lock()
	defer m.syncPairs.Unlock()
	p, ok := m.pairs[album]
	if !ok {
		p = &pairsTime{}
		p.pairs = [][2]uint64{}
		m.pairs[album] = p
	}
	for _, images := range pairs {
		p.pairs = append(p.pairs, [2]uint64{images[0], images[1]})
	}
	p.seen = time.Now()
	return nil
}

func (m *Mem) Pop(_ context.Context, album uint64) (uint64, uint64, error) {
	m.syncPairs.Lock()
	defer m.syncPairs.Unlock()
	p, ok := m.pairs[album]
	if !ok {
		return 0x0, 0x0, errors.Wrap(model.ErrPairNotFound)
	}
	if len(p.pairs) == 0 {
		return 0x0, 0x0, errors.Wrap(model.ErrPairNotFound)
	}
	images := (p.pairs)[0]
	p.pairs = (p.pairs)[1:]
	p.seen = time.Now()
	return images[0], images[1], nil
}

func (m *Mem) Set(_ context.Context, _ uint64, token uint64, image uint64) error {
	m.syncTokens.Lock()
	defer m.syncTokens.Unlock()
	_, ok := m.tokens[token]
	if ok {
		return errors.Wrap(model.ErrTokenAlreadyExists)
	}
	t := &tokenTime{}
	t.token = image
	t.seen = time.Now()
	m.tokens[token] = t
	return nil
}

func (m *Mem) Get(_ context.Context, _ uint64, token uint64) (uint64, error) {
	m.syncTokens.Lock()
	defer m.syncTokens.Unlock()
	image, ok := m.tokens[token]
	if !ok {
		return 0x0, errors.Wrap(model.ErrTokenNotFound)
	}
	delete(m.tokens, token)
	return image.token, nil
}
<commit_msg>Preallocate memory<commit_after>package cache

import (
	"context"
	"sync"
	"time"

	"github.com/emirpasic/gods/sets/linkedhashset"
	"github.com/emirpasic/gods/trees/binaryheap"

	"github.com/zitryss/aye-and-nay/domain/model"
	"github.com/zitryss/aye-and-nay/pkg/errors"
)

func NewMem(opts ...options) *Mem {
	conf := newMemConfig()
	m := &Mem{
		conf:        conf,
		syncQueues:  syncQueues{queues: map[uint64]*linkedhashset.Set{}},
		syncPQueues: syncPQueues{pqueues: map[uint64]*binaryheap.Heap{}},
		syncPairs:   syncPairs{pairs: map[uint64]*pairsTime{}},
		syncTokens:  syncTokens{tokens: map[uint64]*tokenTime{}},
	}
	for _, opt := range opts {
		opt(m)
	}
	return m
}

type options func(*Mem)

func WithHeartbeatPair(ch chan<- interface{}) options {
	return func(m *Mem) {
		m.heartbeat.pair = ch
	}
}

func WithHeartbeatToken(ch chan<- interface{}) options {
	return func(m *Mem) {
		m.heartbeat.token = ch
	}
}

type Mem struct {
	conf memConfig
	syncQueues
	syncPQueues
	syncPairs
	syncTokens
	heartbeat struct {
		pair  chan<- interface{}
		token chan<- interface{}
	}
}

type syncQueues struct {
	sync.Mutex
	queues map[uint64]*linkedhashset.Set
}

type syncPQueues struct {
	sync.Mutex
	pqueues map[uint64]*binaryheap.Heap
}

type syncPairs struct {
	sync.Mutex
	pairs map[uint64]*pairsTime
}

type pairsTime struct {
	pairs [][2]uint64
	seen  time.Time
}

type syncTokens struct {
	sync.Mutex
	tokens map[uint64]*tokenTime
}

type tokenTime struct {
	token uint64
	seen  time.Time
}

type elem struct {
	album   uint64
	expires time.Time
}

func timeComparator(a, b interface{}) int {
	tA := a.(elem).expires
	tB := b.(elem).expires
	switch {
	case tA.After(tB):
		return 1
	case tA.Before(tB):
		return -1
	default:
		return 0
	}
}

func (m *Mem) Monitor() {
	go func() {
		for {
			if m.heartbeat.pair != nil {
				m.heartbeat.pair <- struct{}{}
			}
			now := time.Now()
			m.syncPairs.Lock()
			for k, v := range m.pairs {
				if now.Sub(v.seen) >= m.conf.timeToLive {
					delete(m.pairs, k)
				}
			}
			m.syncPairs.Unlock()
			time.Sleep(m.conf.cleanupInterval)
			if m.heartbeat.pair != nil {
				m.heartbeat.pair <- struct{}{}
			}
		}
	}()
	go func() {
		for {
			if m.heartbeat.token != nil {
				m.heartbeat.token <- struct{}{}
			}
			now := time.Now()
			m.syncTokens.Lock()
			for k, v := range m.tokens {
				if now.Sub(v.seen) >= m.conf.timeToLive {
					delete(m.tokens, k)
				}
			}
			m.syncTokens.Unlock()
			time.Sleep(m.conf.cleanupInterval)
			if m.heartbeat.token != nil {
				m.heartbeat.token <- struct{}{}
			}
		}
	}()
}

func (m *Mem) Add(_ context.Context, queue uint64, album uint64) error {
	m.syncQueues.Lock()
	defer m.syncQueues.Unlock()
	q, ok := m.queues[queue]
	if !ok {
		q = linkedhashset.New()
		m.queues[queue] = q
	}
	q.Add(album)
	return nil
}

func (m *Mem) Poll(_ context.Context, queue uint64) (uint64, error) {
	m.syncQueues.Lock()
	defer m.syncQueues.Unlock()
	q, ok := m.queues[queue]
	if !ok {
		return 0x0, errors.Wrap(model.ErrUnknown)
	}
	it := q.Iterator()
	if !it.Next() {
		return 0x0, errors.Wrap(model.ErrUnknown)
	}
	album := it.Value().(uint64)
	q.Remove(album)
	return album, nil
}

func (m *Mem) Size(_ context.Context, queue uint64) (int, error) {
	m.syncQueues.Lock()
	defer m.syncQueues.Unlock()
	q, ok := m.queues[queue]
	if !ok {
		return 0, nil
	}
	n := q.Size()
	return n, nil
}

func (m *Mem) PAdd(_ context.Context, pqueue uint64, album uint64, expires time.Time) error {
	m.syncPQueues.Lock()
	defer m.syncPQueues.Unlock()
	pq, ok := m.pqueues[pqueue]
	if !ok {
		pq = binaryheap.NewWith(timeComparator)
		m.pqueues[pqueue] = pq
	}
	pq.Push(elem{album, expires})
	return nil
}

func (m *Mem) PPoll(_ context.Context, pqueue uint64) (uint64, time.Time, error) {
	m.syncPQueues.Lock()
	defer m.syncPQueues.Unlock()
	pq, ok := m.pqueues[pqueue]
	if !ok {
		return 0x0, time.Time{}, errors.Wrap(model.ErrUnknown)
	}
	e, ok := pq.Pop()
	if !ok {
		return 0x0, time.Time{}, errors.Wrap(model.ErrUnknown)
	}
	return e.(elem).album, e.(elem).expires, nil
}

func (m *Mem) PSize(_ context.Context, pqueue uint64) (int, error) {
	m.syncPQueues.Lock()
	defer m.syncPQueues.Unlock()
	pq, ok := m.pqueues[pqueue]
	if !ok {
		return 0, nil
	}
	n := pq.Size()
	return n, nil
}

func (m *Mem) Push(_ context.Context, album uint64, pairs [][2]uint64) error {
	m.syncPairs.Lock()
	defer m.syncPairs.Unlock()
	p, ok := m.pairs[album]
	if !ok {
		p = &pairsTime{}
		p.pairs = make([][2]uint64, 0, len(pairs))
		m.pairs[album] = p
	}
	for _, images := range pairs {
		p.pairs = append(p.pairs, [2]uint64{images[0], images[1]})
	}
	p.seen = time.Now()
	return nil
}

func (m *Mem) Pop(_ context.Context, album uint64) (uint64, uint64, error) {
	m.syncPairs.Lock()
	defer m.syncPairs.Unlock()
	p, ok := m.pairs[album]
	if !ok {
		return 0x0, 0x0, errors.Wrap(model.ErrPairNotFound)
	}
	if len(p.pairs) == 0 {
		return 0x0, 0x0, errors.Wrap(model.ErrPairNotFound)
	}
	images := (p.pairs)[0]
	p.pairs = (p.pairs)[1:]
	p.seen = time.Now()
	return images[0], images[1], nil
}

func (m *Mem) Set(_ context.Context, _ uint64, token uint64, image uint64) error {
	m.syncTokens.Lock()
	defer m.syncTokens.Unlock()
	_, ok := m.tokens[token]
	if ok {
		return errors.Wrap(model.ErrTokenAlreadyExists)
	}
	t := &tokenTime{}
	t.token = image
	t.seen = time.Now()
	m.tokens[token] = t
	return nil
}

func (m *Mem) Get(_ context.Context, _ uint64, token uint64) (uint64, error) {
	m.syncTokens.Lock()
	defer m.syncTokens.Unlock()
	image, ok := m.tokens[token]
	if !ok {
		return 0x0, errors.Wrap(model.ErrTokenNotFound)
	}
	delete(m.tokens, token)
	return image.token, nil
}
<|endoftext|>


<commit_before>package setup

import (
	"context"

	corev1alpha1 "github.com/giantswarm/apiextensions/pkg/apis/core/v1alpha1"
	"github.com/giantswarm/e2e-harness/pkg/release"
	"github.com/giantswarm/e2etemplates/pkg/chartvalues"
	"github.com/giantswarm/microerror"

	"github.com/giantswarm/azure-operator/integration/env"
	"github.com/giantswarm/azure-operator/integration/key"
)

// common installs components required to run the operator.
func common(ctx context.Context, config Config) error {
	{
		err := config.K8s.EnsureNamespaceCreated(ctx, namespace)
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		c := chartvalues.E2ESetupVaultConfig{
			Vault: chartvalues.E2ESetupVaultConfigVault{
				Token: env.VaultToken(),
			},
		}

		values, err := chartvalues.NewE2ESetupVault(c)
		if err != nil {
			return microerror.Mask(err)
		}

		err = config.Release.Install(ctx, key.VaultReleaseName(), release.NewStableVersion(), values, config.Release.Condition().PodExists(ctx, "default", "app=vault"))
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		c := chartvalues.CertOperatorConfig{
			CommonDomain:       env.CommonDomain(),
			RegistryPullSecret: env.RegistryPullSecret(),
			Vault: chartvalues.CertOperatorVault{
				Token: env.VaultToken(),
			},
		}

		values, err := chartvalues.NewCertOperator(c)
		if err != nil {
			return microerror.Mask(err)
		}

		err = config.Release.InstallOperator(ctx, key.CertOperatorReleaseName(), release.NewStableVersion(), values, corev1alpha1.NewCertConfigCRD())
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		err := ensureCertConfigsInstalled(ctx, env.ClusterID(), config)
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		c := chartvalues.NodeOperatorConfig{
			RegistryPullSecret: env.RegistryPullSecret(),
		}

		values, err := chartvalues.NewNodeOperator(c)
		if err != nil {
			return microerror.Mask(err)
		}

		err = config.Release.InstallOperator(ctx, key.NodeOperatorReleaseName(), release.NewStableVersion(), values, corev1alpha1.NewNodeConfigCRD())
		if err != nil {
			return microerror.Mask(err)
		}
	}

	return nil
}
<commit_msg>fix waiting for node-operator being installed (#504)<commit_after>package setup

import (
	"context"

	corev1alpha1 "github.com/giantswarm/apiextensions/pkg/apis/core/v1alpha1"
	"github.com/giantswarm/e2e-harness/pkg/release"
	"github.com/giantswarm/e2etemplates/pkg/chartvalues"
	"github.com/giantswarm/microerror"

	"github.com/giantswarm/azure-operator/integration/env"
	"github.com/giantswarm/azure-operator/integration/key"
)

// common installs components required to run the operator.
func common(ctx context.Context, config Config) error {
	{
		err := config.K8s.EnsureNamespaceCreated(ctx, namespace)
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		c := chartvalues.E2ESetupVaultConfig{
			Vault: chartvalues.E2ESetupVaultConfigVault{
				Token: env.VaultToken(),
			},
		}

		values, err := chartvalues.NewE2ESetupVault(c)
		if err != nil {
			return microerror.Mask(err)
		}

		err = config.Release.Install(ctx, key.VaultReleaseName(), release.NewStableVersion(), values, config.Release.Condition().PodExists(ctx, "default", "app=vault"))
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		c := chartvalues.CertOperatorConfig{
			CommonDomain:       env.CommonDomain(),
			RegistryPullSecret: env.RegistryPullSecret(),
			Vault: chartvalues.CertOperatorVault{
				Token: env.VaultToken(),
			},
		}

		values, err := chartvalues.NewCertOperator(c)
		if err != nil {
			return microerror.Mask(err)
		}

		err = config.Release.InstallOperator(ctx, key.CertOperatorReleaseName(), release.NewStableVersion(), values, corev1alpha1.NewCertConfigCRD())
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		err := ensureCertConfigsInstalled(ctx, env.ClusterID(), config)
		if err != nil {
			return microerror.Mask(err)
		}
	}

	{
		c := chartvalues.NodeOperatorConfig{
			RegistryPullSecret: env.RegistryPullSecret(),
		}

		values, err := chartvalues.NewNodeOperator(c)
		if err != nil {
			return microerror.Mask(err)
		}

		err = config.Release.InstallOperator(ctx, key.NodeOperatorReleaseName(), release.NewStableVersion(), values, corev1alpha1.NewDrainerConfigCRD())
		if err != nil {
			return microerror.Mask(err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/containers/image/signature"
	"github.com/go-check/check"
)

const (
	gpgBinary = "gpg"
)

func init() {
	check.Suite(&SigningSuite{})
}

type SigningSuite struct {
	gpgHome     string
	fingerprint string
}

func findFingerprint(lineBytes []byte) (string, error) {
	lines := string(lineBytes)
	for _, line := range strings.Split(lines, "\n") {
		fields := strings.Split(line, ":")
		if len(fields) >= 10 && fields[0] == "fpr" {
			return fields[9], nil
		}
	}
	return "", errors.New("No fingerprint found")
}

func (s *SigningSuite) SetUpSuite(c *check.C) {
	_, err := exec.LookPath(skopeoBinary)
	c.Assert(err, check.IsNil)
}

func (s *SigningSuite) SetUpTest(c *check.C) {
	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
	c.Assert(err, check.IsNil)
	defer mech.Close()
	if err := mech.SupportsSigning(); err != nil { // FIXME? Test that verification and policy enforcement works, using signatures from fixtures
		c.Skip(fmt.Sprintf("Signing not supported: %v", err))
	}

	s.gpgHome, err = ioutil.TempDir("", "skopeo-gpg")
	c.Assert(err, check.IsNil)
	os.Setenv("GNUPGHOME", s.gpgHome)

	runCommandWithInput(c, "Key-Type: RSA\nName-Real: Testing user\n%commit\n", gpgBinary, "--homedir", s.gpgHome, "--batch", "--gen-key")

	lines, err := exec.Command(gpgBinary, "--homedir", s.gpgHome, "--with-colons", "--no-permission-warning", "--fingerprint").Output()
	c.Assert(err, check.IsNil)
	s.fingerprint, err = findFingerprint(lines)
	c.Assert(err, check.IsNil)
}

func (s *SigningSuite) TearDownTest(c *check.C) {
	if s.gpgHome != "" {
		err := os.RemoveAll(s.gpgHome)
		c.Assert(err, check.IsNil)
	}
	s.gpgHome = ""

	os.Unsetenv("GNUPGHOME")
}

func (s *SigningSuite) TestSignVerifySmoke(c *check.C) {
	manifestPath := "fixtures/image.manifest.json"
	dockerReference := "testing/smoketest"

	sigOutput, err := ioutil.TempFile("", "sig")
	c.Assert(err, check.IsNil)
	defer os.Remove(sigOutput.Name())
	assertSkopeoSucceeds(c, "^$", "standalone-sign", "-o", sigOutput.Name(),
		manifestPath, dockerReference, s.fingerprint)

	expected := fmt.Sprintf("^Signature verified, digest %s\n$", TestImageManifestDigest)
	assertSkopeoSucceeds(c, expected, "standalone-verify", manifestPath,
		dockerReference, s.fingerprint, sigOutput.Name())
}
<commit_msg>Clean up SigningSuite test initialization<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/containers/image/signature"
	"github.com/go-check/check"
)

const (
	gpgBinary = "gpg"
)

func init() {
	check.Suite(&SigningSuite{})
}

type SigningSuite struct {
	gpgHome     string
	fingerprint string
}

func findFingerprint(lineBytes []byte) (string, error) {
	lines := string(lineBytes)
	for _, line := range strings.Split(lines, "\n") {
		fields := strings.Split(line, ":")
		if len(fields) >= 10 && fields[0] == "fpr" {
			return fields[9], nil
		}
	}
	return "", errors.New("No fingerprint found")
}

func (s *SigningSuite) SetUpSuite(c *check.C) {
	_, err := exec.LookPath(skopeoBinary)
	c.Assert(err, check.IsNil)

	s.gpgHome, err = ioutil.TempDir("", "skopeo-gpg")
	c.Assert(err, check.IsNil)
	os.Setenv("GNUPGHOME", s.gpgHome)

	runCommandWithInput(c, "Key-Type: RSA\nName-Real: Testing user\n%commit\n", gpgBinary, "--homedir", s.gpgHome, "--batch", "--gen-key")

	lines, err := exec.Command(gpgBinary, "--homedir", s.gpgHome, "--with-colons", "--no-permission-warning", "--fingerprint").Output()
	c.Assert(err, check.IsNil)
	s.fingerprint, err = findFingerprint(lines)
	c.Assert(err, check.IsNil)
}

func (s *SigningSuite) TearDownSuite(c *check.C) {
	if s.gpgHome != "" {
		err := os.RemoveAll(s.gpgHome)
		c.Assert(err, check.IsNil)
	}
	s.gpgHome = ""

	os.Unsetenv("GNUPGHOME")
}

func (s *SigningSuite) TestSignVerifySmoke(c *check.C) {
	mech, _, err := signature.NewEphemeralGPGSigningMechanism([]byte{})
	c.Assert(err, check.IsNil)
	defer mech.Close()
	if err := mech.SupportsSigning(); err != nil { // FIXME? Test that verification and policy enforcement works, using signatures from fixtures
		c.Skip(fmt.Sprintf("Signing not supported: %v", err))
	}

	manifestPath := "fixtures/image.manifest.json"
	dockerReference := "testing/smoketest"

	sigOutput, err := ioutil.TempFile("", "sig")
	c.Assert(err, check.IsNil)
	defer os.Remove(sigOutput.Name())
	assertSkopeoSucceeds(c, "^$", "standalone-sign", "-o", sigOutput.Name(),
		manifestPath, dockerReference, s.fingerprint)

	expected := fmt.Sprintf("^Signature verified, digest %s\n$", TestImageManifestDigest)
	assertSkopeoSucceeds(c, expected, "standalone-verify", manifestPath,
		dockerReference, s.fingerprint, sigOutput.Name())
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"strings"
	"time"
)

type Archive struct {
	UUID           string `json:"uuid"            mbus:"uuid"`
	TenantUUID     string `json:"tenant_uuid"     mbus:"tenant_uuid"`
	TargetUUID     string `json:"target_uuid"     mbus:"target_uuid"`
	StoreUUID      string `json:"store_uuid"      mbus:"store_uuid"`
	StoreKey       string `json:"key"             mbus:"key"`
	TakenAt        int64  `json:"taken_at"        mbus:"taken_at"`
	ExpiresAt      int64  `json:"expires_at"      mbus:"expires_at"`
	Notes          string `json:"notes"           mbus:"notes"`
	Status         string `json:"status"          mbus:"status"`
	PurgeReason    string `json:"purge_reason"    mbus:"purge_reason"`
	EncryptionType string `json:"encryption_type" mbus:"encryption_type"`
	Compression    string `json:"compression"     mbus:"compression"`
	Size           int64  `json:"size"            mbus:"size"`

	TargetName     string `json:"target_name"`
	TargetPlugin   string `json:"target_plugin"`
	TargetEndpoint string `json:"target_endpoint"`
	StoreName      string `json:"store_name"`
	StorePlugin    string `json:"store_plugin"`
	StoreEndpoint  string `json:"store_endpoint"`
	StoreAgent     string `json:"store_agent"`
	Job            string `json:"job"`
}

type ArchiveFilter struct {
	ForTarget     string
	ForStore      string
	Before        *time.Time
	After         *time.Time
	ExpiresBefore *time.Time
	ExpiresAfter  *time.Time
	WithStatus    []string
	WithOutStatus []string
	ForTenant     string
	Limit         int
}

func (f *ArchiveFilter) Query() (string, []interface{}) {
	wheres := []string{"a.uuid = a.uuid"}
	var args []interface{}
	if f.ForTarget != "" {
		wheres = append(wheres, "target_uuid = ?")
		args = append(args, f.ForTarget)
	}
	if f.ForStore != "" {
		wheres = append(wheres, "store_uuid = ?")
		args = append(args, f.ForStore)
	}
	if f.Before != nil {
		wheres = append(wheres, "taken_at <= ?")
		args = append(args, f.Before.Unix())
	}
	if f.After != nil {
		wheres = append(wheres, "taken_at >= ?")
		args = append(args, f.After.Unix())
	}
	if len(f.WithStatus) > 0 {
		var params []string
		for _, e := range f.WithStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status IN (%s)", strings.Join(params, ", ")))
	}
	if len(f.WithOutStatus) > 0 {
		var params []string
		for _, e := range f.WithOutStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status NOT IN (%s)", strings.Join(params, ", ")))
	}
	if f.ExpiresBefore != nil {
		wheres = append(wheres, "expires_at < ?")
		args = append(args, f.ExpiresBefore.Unix())
	}

	if f.ForTenant != "" {
		wheres = append(wheres, "a.tenant_uuid = ?")
		args = append(args, f.ForTenant)
	}
	limit := ""
	if f.Limit > 0 {
		limit = " LIMIT ?"
		args = append(args, f.Limit)
	}

	return `
		SELECT a.uuid, a.store_key,
		       a.taken_at, a.expires_at, a.notes,
		       t.uuid, t.name, t.plugin, t.endpoint,
		       s.uuid, s.name, s.plugin, s.endpoint,
		       a.status, a.purge_reason, a.job, a.encryption_type,
		       a.compression, a.tenant_uuid, a.size

		FROM archives a
		   INNER JOIN targets t   ON t.uuid = a.target_uuid
		   INNER JOIN stores  s   ON s.uuid = a.store_uuid

		WHERE ` + strings.Join(wheres, " AND ") + `
		ORDER BY a.taken_at DESC, a.uuid ASC
	` + limit, args
}

func (db *DB) CountArchives(filter *ArchiveFilter) (int, error) {
	if filter == nil {
		filter = &ArchiveFilter{}
	}

	query, args := filter.Query()
	r, err := db.query(query, args...)
	if err != nil {
		return 0, err
	}
	defer r.Close()

	i := 0
	for r.Next() {
		i++
	}
	return i, nil
}

func (db *DB) GetAllArchives(filter *ArchiveFilter) ([]*Archive, error) {
	if filter == nil {
		filter = &ArchiveFilter{}
	}

	l := []*Archive{}
	query, args := filter.Query()
	r, err := db.query(query, args...)
	if err != nil {
		return l, err
	}
	defer r.Close()

	for r.Next() {
		ann := &Archive{}

		var takenAt, expiresAt, size *int64
		var targetName, storeName *string
		if err = r.Scan(
			&ann.UUID, &ann.StoreKey, &takenAt, &expiresAt, &ann.Notes,
			&ann.TargetUUID, &targetName, &ann.TargetPlugin, &ann.TargetEndpoint,
			&ann.StoreUUID, &storeName, &ann.StorePlugin, &ann.StoreEndpoint,
			&ann.Status, &ann.PurgeReason, &ann.Job, &ann.EncryptionType,
			&ann.Compression, &ann.TenantUUID, &size); err != nil {

			return l, err
		}
		if takenAt != nil {
			ann.TakenAt = *takenAt
		}
		if expiresAt != nil {
			ann.ExpiresAt = *expiresAt
		}
		if targetName != nil {
			ann.TargetName = *targetName
		}
		if storeName != nil {
			ann.StoreName = *storeName
		}
		if size != nil {
			ann.Size = *size
		}

		l = append(l, ann)
	}

	return l, nil
}

func (db *DB) GetArchive(id string) (*Archive, error) {
	r, err := db.query(`
		SELECT a.uuid, a.store_key,
		       a.taken_at, a.expires_at, a.notes,
		       t.uuid, t.name, t.plugin, t.endpoint,
		       s.uuid, s.name, s.plugin, s.endpoint, s.agent,
		       a.status, a.purge_reason, a.job, a.encryption_type,
		       a.compression, a.tenant_uuid, a.size

		FROM archives a
		   INNER JOIN targets t   ON t.uuid = a.target_uuid
		   INNER JOIN stores  s   ON s.uuid = a.store_uuid

		WHERE a.uuid = ?`, id)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	if !r.Next() {
		return nil, nil
	}
	ann := &Archive{}

	var takenAt, expiresAt, size *int64
	var targetName, storeName *string
	if err = r.Scan(
		&ann.UUID, &ann.StoreKey, &takenAt, &expiresAt, &ann.Notes,
		&ann.TargetUUID, &targetName, &ann.TargetPlugin, &ann.TargetEndpoint,
		&ann.StoreUUID, &storeName, &ann.StorePlugin, &ann.StoreEndpoint, &ann.StoreAgent,
		&ann.Status, &ann.PurgeReason, &ann.Job, &ann.EncryptionType,
		&ann.Compression, &ann.TenantUUID, &size); err != nil {

		return nil, err
	}
	if takenAt != nil {
		ann.TakenAt = *takenAt
	}
	if expiresAt != nil {
		ann.ExpiresAt = *expiresAt
	}
	if targetName != nil {
		ann.TargetName = *targetName
	}
	if storeName != nil {
		ann.StoreName = *storeName
	}
	if size != nil {
		ann.Size = *size
	}

	return ann, nil
}

func (db *DB) UpdateArchive(update *Archive) error {
	return db.exec(
		`UPDATE archives SET notes = ? WHERE uuid = ?`,
		update.Notes, update.UUID,
	)
}

func (db *DB) AnnotateTargetArchive(target, id, notes string) error {
	return db.exec(
		`UPDATE archives SET notes = ? WHERE uuid = ? AND target_uuid = ?`,
		notes, id, target,
	)
}

func (db *DB) GetArchivesNeedingPurge() ([]*Archive, error) {
	filter := &ArchiveFilter{
		WithOutStatus: []string{"purged", "valid"},
	}
	return db.GetAllArchives(filter)
}

func (db *DB) GetExpiredArchives() ([]*Archive, error) {
	now := time.Now()
	filter := &ArchiveFilter{
		ExpiresBefore: &now,
		WithStatus:    []string{"valid"},
	}
	return db.GetAllArchives(filter)
}

func (db *DB) InvalidateArchive(id string) error {
	return db.exec(`UPDATE archives SET status = 'invalid' WHERE uuid = ?`, id)
}

func (db *DB) PurgeArchive(id string) error {
	a, err := db.GetArchive(id)
	if err != nil {
		return err
	}

	if a.Status == "valid" {
		return fmt.Errorf("Invalid attempt to purge a 'valid' archive detected")
	}

	err = db.exec(`UPDATE archives SET purge_reason = status WHERE uuid = ?`, id)
	if err != nil {
		return err
	}

	return db.exec(`UPDATE archives SET status = 'purged' WHERE uuid = ?`, id)
}

func (db *DB) ExpireArchive(id string) error {
	return db.exec(`UPDATE archives SET status = 'expired' WHERE uuid = ?`, id)
}

func (db *DB) DeleteArchive(id string) (bool, error) {
	return true, db.exec(`DELETE FROM archives WHERE uuid = ?`, id)
}

func (db *DB) ArchiveStorageFootprint(filter *ArchiveFilter) (int64, error) {
	var i int64

	if filter == nil {
		filter = &ArchiveFilter{}
	}

	wheres := []string{"a.uuid = a.uuid"}
	var args []interface{}
	if filter.ForTarget != "" {
		wheres = append(wheres, "target_uuid = ?")
		args = append(args, filter.ForTarget)
	}
	if filter.ForStore != "" {
		wheres = append(wheres, "store_uuid = ?")
		args = append(args, filter.ForStore)
	}
	if filter.Before != nil {
		wheres = append(wheres, "taken_at <= ?")
		args = append(args, filter.Before.Unix())
	}
	if filter.After != nil {
		wheres = append(wheres, "taken_at >= ?")
		args = append(args, filter.After.Unix())
	}
	if len(filter.WithStatus) > 0 {
		var params []string
		for _, e := range filter.WithStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status IN (%s)", strings.Join(params, ", ")))
	}
	if len(filter.WithOutStatus) > 0 {
		var params []string
		for _, e := range filter.WithOutStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status NOT IN (%s)", strings.Join(params, ", ")))
	}
	if filter.ExpiresBefore != nil {
		wheres = append(wheres, "expires_at <= ?")
		args = append(args, filter.ExpiresBefore.Unix())
	}
	if filter.ExpiresAfter != nil {
		wheres = append(wheres, "expires_at >= ?")
		args = append(args, filter.ExpiresAfter.Unix())
	}
	if filter.ForTenant != "" {
		wheres = append(wheres, "a.tenant_uuid = ?")
		args = append(args, filter.ForTenant)
	}
	limit := ""
	if filter.Limit > 0 {
		limit = " LIMIT ?"
		args = append(args, filter.Limit)
	}

	r, err := db.query(`
		SELECT SUM(a.size)
		FROM archives a
			INNER JOIN targets t   ON t.uuid = a.target_uuid
			INNER JOIN stores  s   ON s.uuid = a.store_uuid
		WHERE `+strings.Join(wheres, " AND ")+limit, args...)
	if err != nil {
		return i, err
	}
	defer r.Close()

	var p *int64
	if !r.Next() {
		return 0, fmt.Errorf("no results from SUM(size) query...")
	}

	if err = r.Scan(&p); err != nil {
		return 0, err
	}
	if p != nil {
		i = *p
	}
	return i, nil
}
<commit_msg>Rename final `ann` var to `a` (archives)<commit_after>package db

import (
	"fmt"
	"strings"
	"time"
)

type Archive struct {
	UUID           string `json:"uuid"            mbus:"uuid"`
	TenantUUID     string `json:"tenant_uuid"     mbus:"tenant_uuid"`
	TargetUUID     string `json:"target_uuid"     mbus:"target_uuid"`
	StoreUUID      string `json:"store_uuid"      mbus:"store_uuid"`
	StoreKey       string `json:"key"             mbus:"key"`
	TakenAt        int64  `json:"taken_at"        mbus:"taken_at"`
	ExpiresAt      int64  `json:"expires_at"      mbus:"expires_at"`
	Notes          string `json:"notes"           mbus:"notes"`
	Status         string `json:"status"          mbus:"status"`
	PurgeReason    string `json:"purge_reason"    mbus:"purge_reason"`
	EncryptionType string `json:"encryption_type" mbus:"encryption_type"`
	Compression    string `json:"compression"     mbus:"compression"`
	Size           int64  `json:"size"            mbus:"size"`

	TargetName     string `json:"target_name"`
	TargetPlugin   string `json:"target_plugin"`
	TargetEndpoint string `json:"target_endpoint"`
	StoreName      string `json:"store_name"`
	StorePlugin    string `json:"store_plugin"`
	StoreEndpoint  string `json:"store_endpoint"`
	StoreAgent     string `json:"store_agent"`
	Job            string `json:"job"`
}

type ArchiveFilter struct {
	ForTarget     string
	ForStore      string
	Before        *time.Time
	After         *time.Time
	ExpiresBefore *time.Time
	ExpiresAfter  *time.Time
	WithStatus    []string
	WithOutStatus []string
	ForTenant     string
	Limit         int
}

func (f *ArchiveFilter) Query() (string, []interface{}) {
	wheres := []string{"a.uuid = a.uuid"}
	var args []interface{}
	if f.ForTarget != "" {
		wheres = append(wheres, "target_uuid = ?")
		args = append(args, f.ForTarget)
	}
	if f.ForStore != "" {
		wheres = append(wheres, "store_uuid = ?")
		args = append(args, f.ForStore)
	}
	if f.Before != nil {
		wheres = append(wheres, "taken_at <= ?")
		args = append(args, f.Before.Unix())
	}
	if f.After != nil {
		wheres = append(wheres, "taken_at >= ?")
		args = append(args, f.After.Unix())
	}
	if len(f.WithStatus) > 0 {
		var params []string
		for _, e := range f.WithStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status IN (%s)", strings.Join(params, ", ")))
	}
	if len(f.WithOutStatus) > 0 {
		var params []string
		for _, e := range f.WithOutStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status NOT IN (%s)", strings.Join(params, ", ")))
	}
	if f.ExpiresBefore != nil {
		wheres = append(wheres, "expires_at < ?")
		args = append(args, f.ExpiresBefore.Unix())
	}

	if f.ForTenant != "" {
		wheres = append(wheres, "a.tenant_uuid = ?")
		args = append(args, f.ForTenant)
	}
	limit := ""
	if f.Limit > 0 {
		limit = " LIMIT ?"
		args = append(args, f.Limit)
	}

	return `
		SELECT a.uuid, a.store_key,
		       a.taken_at, a.expires_at, a.notes,
		       t.uuid, t.name, t.plugin, t.endpoint,
		       s.uuid, s.name, s.plugin, s.endpoint,
		       a.status, a.purge_reason, a.job, a.encryption_type,
		       a.compression, a.tenant_uuid, a.size

		FROM archives a
		   INNER JOIN targets t   ON t.uuid = a.target_uuid
		   INNER JOIN stores  s   ON s.uuid = a.store_uuid

		WHERE ` + strings.Join(wheres, " AND ") + `
		ORDER BY a.taken_at DESC, a.uuid ASC
	` + limit, args
}

func (db *DB) CountArchives(filter *ArchiveFilter) (int, error) {
	if filter == nil {
		filter = &ArchiveFilter{}
	}

	query, args := filter.Query()
	r, err := db.query(query, args...)
	if err != nil {
		return 0, err
	}
	defer r.Close()

	i := 0
	for r.Next() {
		i++
	}
	return i, nil
}

func (db *DB) GetAllArchives(filter *ArchiveFilter) ([]*Archive, error) {
	if filter == nil {
		filter = &ArchiveFilter{}
	}

	l := []*Archive{}
	query, args := filter.Query()
	r, err := db.query(query, args...)
	if err != nil {
		return l, err
	}
	defer r.Close()

	for r.Next() {
		a := &Archive{}

		var takenAt, expiresAt, size *int64
		var targetName, storeName *string
		if err = r.Scan(
			&a.UUID, &a.StoreKey, &takenAt, &expiresAt, &a.Notes,
			&a.TargetUUID, &targetName, &a.TargetPlugin, &a.TargetEndpoint,
			&a.StoreUUID, &storeName, &a.StorePlugin, &a.StoreEndpoint,
			&a.Status, &a.PurgeReason, &a.Job, &a.EncryptionType,
			&a.Compression, &a.TenantUUID, &size); err != nil {

			return l, err
		}
		if takenAt != nil {
			a.TakenAt = *takenAt
		}
		if expiresAt != nil {
			a.ExpiresAt = *expiresAt
		}
		if targetName != nil {
			a.TargetName = *targetName
		}
		if storeName != nil {
			a.StoreName = *storeName
		}
		if size != nil {
			a.Size = *size
		}

		l = append(l, a)
	}

	return l, nil
}

func (db *DB) GetArchive(id string) (*Archive, error) {
	r, err := db.query(`
		SELECT a.uuid, a.store_key,
		       a.taken_at, a.expires_at, a.notes,
		       t.uuid, t.name, t.plugin, t.endpoint,
		       s.uuid, s.name, s.plugin, s.endpoint, s.agent,
		       a.status, a.purge_reason, a.job, a.encryption_type,
		       a.compression, a.tenant_uuid, a.size

		FROM archives a
		   INNER JOIN targets t   ON t.uuid = a.target_uuid
		   INNER JOIN stores  s   ON s.uuid = a.store_uuid

		WHERE a.uuid = ?`, id)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	if !r.Next() {
		return nil, nil
	}
	a := &Archive{}

	var takenAt, expiresAt, size *int64
	var targetName, storeName *string
	if err = r.Scan(
		&a.UUID, &a.StoreKey, &takenAt, &expiresAt, &a.Notes,
		&a.TargetUUID, &targetName, &a.TargetPlugin, &a.TargetEndpoint,
		&a.StoreUUID, &storeName, &a.StorePlugin, &a.StoreEndpoint, &a.StoreAgent,
		&a.Status, &a.PurgeReason, &a.Job, &a.EncryptionType,
		&a.Compression, &a.TenantUUID, &size); err != nil {

		return nil, err
	}
	if takenAt != nil {
		a.TakenAt = *takenAt
	}
	if expiresAt != nil {
		a.ExpiresAt = *expiresAt
	}
	if targetName != nil {
		a.TargetName = *targetName
	}
	if storeName != nil {
		a.StoreName = *storeName
	}
	if size != nil {
		a.Size = *size
	}

	return a, nil
}

func (db *DB) UpdateArchive(update *Archive) error {
	return db.exec(
		`UPDATE archives SET notes = ? WHERE uuid = ?`,
		update.Notes, update.UUID,
	)
}

func (db *DB) AnnotateTargetArchive(target, id, notes string) error {
	return db.exec(
		`UPDATE archives SET notes = ? WHERE uuid = ? AND target_uuid = ?`,
		notes, id, target,
	)
}

func (db *DB) GetArchivesNeedingPurge() ([]*Archive, error) {
	filter := &ArchiveFilter{
		WithOutStatus: []string{"purged", "valid"},
	}
	return db.GetAllArchives(filter)
}

func (db *DB) GetExpiredArchives() ([]*Archive, error) {
	now := time.Now()
	filter := &ArchiveFilter{
		ExpiresBefore: &now,
		WithStatus:    []string{"valid"},
	}
	return db.GetAllArchives(filter)
}

func (db *DB) InvalidateArchive(id string) error {
	return db.exec(`UPDATE archives SET status = 'invalid' WHERE uuid = ?`, id)
}

func (db *DB) PurgeArchive(id string) error {
	a, err := db.GetArchive(id)
	if err != nil {
		return err
	}

	if a.Status == "valid" {
		return fmt.Errorf("Invalid attempt to purge a 'valid' archive detected")
	}

	err = db.exec(`UPDATE archives SET purge_reason = status WHERE uuid = ?`, id)
	if err != nil {
		return err
	}

	return db.exec(`UPDATE archives SET status = 'purged' WHERE uuid = ?`, id)
}

func (db *DB) ExpireArchive(id string) error {
	return db.exec(`UPDATE archives SET status = 'expired' WHERE uuid = ?`, id)
}

func (db *DB) DeleteArchive(id string) (bool, error) {
	return true, db.exec(`DELETE FROM archives WHERE uuid = ?`, id)
}

func (db *DB) ArchiveStorageFootprint(filter *ArchiveFilter) (int64, error) {
	var i int64

	if filter == nil {
		filter = &ArchiveFilter{}
	}

	wheres := []string{"a.uuid = a.uuid"}
	var args []interface{}
	if filter.ForTarget != "" {
		wheres = append(wheres, "target_uuid = ?")
		args = append(args, filter.ForTarget)
	}
	if filter.ForStore != "" {
		wheres = append(wheres, "store_uuid = ?")
		args = append(args, filter.ForStore)
	}
	if filter.Before != nil {
		wheres = append(wheres, "taken_at <= ?")
		args = append(args, filter.Before.Unix())
	}
	if filter.After != nil {
		wheres = append(wheres, "taken_at >= ?")
		args = append(args, filter.After.Unix())
	}
	if len(filter.WithStatus) > 0 {
		var params []string
		for _, e := range filter.WithStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status IN (%s)", strings.Join(params, ", ")))
	}
	if len(filter.WithOutStatus) > 0 {
		var params []string
		for _, e := range filter.WithOutStatus {
			params = append(params, "?")
			args = append(args, e)
		}
		wheres = append(wheres, fmt.Sprintf("status NOT IN (%s)", strings.Join(params, ", ")))
	}
	if filter.ExpiresBefore != nil {
		wheres = append(wheres, "expires_at <= ?")
		args = append(args, filter.ExpiresBefore.Unix())
	}
	if filter.ExpiresAfter != nil {
		wheres = append(wheres, "expires_at >= ?")
		args = append(args, filter.ExpiresAfter.Unix())
	}
	if filter.ForTenant != "" {
		wheres = append(wheres, "a.tenant_uuid = ?")
		args = append(args, filter.ForTenant)
	}
	limit := ""
	if filter.Limit > 0 {
		limit = " LIMIT ?"
		args = append(args, filter.Limit)
	}

	r, err := db.query(`
		SELECT SUM(a.size)
		FROM archives a
			INNER JOIN targets t   ON t.uuid = a.target_uuid
			INNER JOIN stores  s   ON s.uuid = a.store_uuid
		WHERE `+strings.Join(wheres, " AND ")+limit, args...)
	if err != nil {
		return i, err
	}
	defer r.Close()

	var p *int64
	if !r.Next() {
		return 0, fmt.Errorf("no results from SUM(size) query...")
	}

	if err = r.Scan(&p); err != nil {
		return 0, err
	}
	if p != nil {
		i = *p
	}
	return i, nil
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"strings"
	"testing"
)

// TODO: maybe we also want to test "# *foo*" headings? Would we test that they
// parse the markdown?
func TestHeaders(t *testing.T) {
	mdParser := NewMarkdownParser(`
# One
##One # Two

Two
===

# # Three # Four
`)
	headers := mdParser.Headers()
	if len(headers) < 3 {
		t.Errorf("Length is less than 3: %#v", headers)
		return
	}

	results := []string{"One", "Two", "# Three # Four"}
	for i := 0; i < len(results); i++ {
		if headers[i] != results[i] {
			t.Errorf("%s is missing: %#v", results[i], headers)
		}
	}
}

func TestSubHeadersOf(t *testing.T) {
	mdParser := NewMarkdownParser(`
# Doo-doo

## Dee-dee
### Dum-dum
## Boo-boo

# Bla-bla

One-two
=======

Three-four
-------
`)
	subHeaders := mdParser.SubHeadersOf("Doo-doo")

	if len(subHeaders) < 2 {
		t.Errorf("Length is less than 2: %#v", subHeaders)
		return
	}

	expecteds := []string{"Dee-dee", "Boo-boo"}
	for i, expected := range expecteds {
		if subHeaders[i] != expected {
			t.Errorf("%s is missing: %#v", expected, subHeaders)
		}
	}

	subHeaders = mdParser.SubHeadersOf("Bla-bla")
	if len(subHeaders) != 0 {
		t.Errorf("Length is not 0: %#v", subHeaders)
		return
	}

	subHeaders = mdParser.SubHeadersOf("One-two")

	if len(subHeaders) != 1 {
		t.Errorf("Length is less than 2: %#v", subHeaders)
		return
	}

	expectations := []string{"Three-four"}
	for i, expected := range expectations {
		if subHeaders[i] != expected {
			t.Errorf("%s is missing: %#v", expected, subHeaders)
		}
	}
}

func TestNames(t *testing.T) {
	mdParser := NewMarkdownParser("Beginning Of Line. Аз съм Иван Петров. He Used to play games. His favourite browser is Firefox. That's Mozilla Firefox.")
	names := mdParser.Names()
	expected := []string{"Beginning Of", "Иван Петров", "Mozilla Firefox"}

	if !reflect.DeepEqual(names, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", names, expected)
	}
}

func TestPhoneNumbers(t *testing.T) {
	mdParser := NewMarkdownParser("sometext 0889123456 alabala, 0 (889) 123 baba - 456, +45-(31), foobar")
	numbers := mdParser.PhoneNumbers()
	expected := []string{"0889123456", "0 (889) 123", "456", "+45-(31)"}

	for i, x := range numbers {
		numbers[i] = strings.TrimSpace(x)
	}

	if !reflect.DeepEqual(numbers, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", numbers, expected)
	}
}

func TestLinks(t *testing.T) {
	mdParser := NewMarkdownParser("sometext http://somelink.com:230 ignore this 123 https://www.google.bg/search?q=4531&ie=utf-8&oe=utf-8&rls=org.mozilla:en-US:official&client=%20firefox-a&gws_rd=asd&ei=some#somefragment endoflink junk")
	links := mdParser.Links()
	expected := []string{
		"http://somelink.com:230",
		"https://www.google.bg/search?q=4531&ie=utf-8&oe=utf-8&rls=org.mozilla:en-US:official&client=%20firefox-a&gws_rd=asd&ei=some#somefragment",
	}
	if !reflect.DeepEqual(links, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", links, expected)
	}
}

func TestEmails(t *testing.T) {
	mdParser := NewMarkdownParser("ignore validMail12@foobar.com sometext@ _invalidmail@google.com toolongmailhereaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@gmail.com 12mail@gmail.com ")
	emails := mdParser.Emails()
	expected := []string{"validMail12@foobar.com", "12mail@gmail.com"}
	if !reflect.DeepEqual(emails, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", emails, expected)
	}
}

func getSplitContents(tableOfContents string) []string {
	splitContents := strings.Split(tableOfContents, "\n")
	lengthMinusOne := len(splitContents) - 1
	if splitContents[lengthMinusOne] == "" {
		splitContents = splitContents[:lengthMinusOne]
	}

	for i, x := range splitContents {
		splitContents[i] = strings.TrimSpace(x)
	}

	return splitContents
}

func TestTableOfContents(t *testing.T) {
	println("Pending: TestTableOfContents")
	return

	mdParser := NewMarkdownParser("")
	tableOfContents := mdParser.GenerateTableOfContents()
	splitContents := getSplitContents(tableOfContents)
	if !reflect.DeepEqual(splitContents, []string{"1. Path", "1.1 Примери:"}) {
		t.Fail()
	}

	mdParser = NewMarkdownParser("")
	tableOfContents = mdParser.GenerateTableOfContents()
	splitContents = getSplitContents(tableOfContents)
	if len(splitContents) != 7 || splitContents[3] != "1.3 Colors" {
		t.Fail()
	}

	mdParser = NewMarkdownParser("")
	tableOfContents = mdParser.GenerateTableOfContents()
	splitContents = getSplitContents(tableOfContents)
	if len(splitContents) != 11 || splitContents[9] != "1.1.8 `func (mp *MarkdownParser) GenerateTableOfContents() string`" {
		t.Fail()
	}
}
<commit_msg>Full test for homework 03<commit_after>package main

import (
	"reflect"
	"strings"
	"testing"
)

func TestHeaders(t *testing.T) {
	mdParser := NewMarkdownParser(`
# One
##One # Two

Two
===

# # Three # Four
`)
	// let's not test whether the solution trims or not
	headers := []string{}
	for _, h := range mdParser.Headers() {
		headers = append(headers, strings.TrimSpace(h))
	}
	expected := []string{"One", "Two", "# Three # Four"}

	if !reflect.DeepEqual(headers, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", headers, expected)
	}
}

func TestSubHeadersOf(t *testing.T) {
	mdParser := NewMarkdownParser(`
# Doo-doo

## Dee-dee
### Dum-dum
## Boo-boo

# Bla-bla

One-two
=======

Three-four
-------
`)
	subHeaders := mdParser.SubHeadersOf("Doo-doo")
	expected := []string{"Dee-dee", "Boo-boo"}
	if !reflect.DeepEqual(subHeaders, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", subHeaders, expected)
	}

	subHeaders = mdParser.SubHeadersOf("Bla-bla")
	expected = nil
	if !reflect.DeepEqual(subHeaders, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", subHeaders, expected)
	}

	subHeaders = mdParser.SubHeadersOf("One-two")
	expected = []string{"Three-four"}
	if !reflect.DeepEqual(subHeaders, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", subHeaders, expected)
	}
}

func TestNames(t *testing.T) {
	mdParser := NewMarkdownParser("Beginning Of Line. Аз съм Иван Петров. He Used to play games. His favourite browser is Firefox. That's Mozilla Firefox.")
	names := mdParser.Names()
	expected := []string{"Of Line", "Иван Петров", "Mozilla Firefox"}

	if !reflect.DeepEqual(names, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", names, expected)
	}
}

func TestPhoneNumbers(t *testing.T) {
	mdParser := NewMarkdownParser("sometext 0889123456 alabala, 0 (889) 123 baba - 456, +45-(31), foobar")
	numbers := mdParser.PhoneNumbers()
	expected := []string{"0889123456", "0 (889) 123", "456", "+45-(31)"}

	for i, x := range numbers {
		numbers[i] = strings.TrimSpace(x)
	}

	if !reflect.DeepEqual(numbers, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", numbers, expected)
	}
}

func TestLinks(t *testing.T) {
	mdParser := NewMarkdownParser("sometext http://somelink.com:230 ignore this 123 https://www.google.bg/search?q=4531&ie=utf-8&oe=utf-8&rls=org.mozilla:en-US:official&client=%20firefox-a&gws_rd=asd&ei=some#somefragment endoflink junk")
	links := mdParser.Links()
	expected := []string{
		"http://somelink.com:230",
		"https://www.google.bg/search?q=4531&ie=utf-8&oe=utf-8&rls=org.mozilla:en-US:official&client=%20firefox-a&gws_rd=asd&ei=some#somefragment",
	}
	if !reflect.DeepEqual(links, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", links, expected)
	}
}

func TestEmails(t *testing.T) {
	mdParser := NewMarkdownParser("ignore validMail12@foobar.com sometext@ _invalidmail@google.com toolongmailhereaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@gmail.com 12mail@gmail.com ")
	emails := mdParser.Emails()
	expected := []string{"validMail12@foobar.com", "12mail@gmail.com"}
	if !reflect.DeepEqual(emails, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", emails, expected)
	}
}

func getSplitContents(tableOfContents string) []string {
	splitContents := strings.Split(tableOfContents, "\n")
	lengthMinusOne := len(splitContents) - 1
	if splitContents[lengthMinusOne] == "" {
		splitContents = splitContents[:lengthMinusOne]
	}

	for i, x := range splitContents {
		splitContents[i] = strings.TrimSpace(x)
	}

	return splitContents
}

func TestTableOfContents(t *testing.T) {
	mdParser := NewMarkdownParser(`
# Path
Нещо
## Примери:
Още нещо
`)
	tableOfContents := mdParser.GenerateTableOfContents()
	splitContents := getSplitContents(tableOfContents)
	expected := []string{
		"1. Path",
		"1.1 Примери:",
	}

	if !reflect.DeepEqual(splitContents, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", splitContents, expected)
	}

	mdParser = NewMarkdownParser(`
One
====
Two
====
## Three

# Four
Five
----
	`)
	tableOfContents = mdParser.GenerateTableOfContents()
	splitContents = getSplitContents(tableOfContents)
	expected = []string{
		"1. One",
		"2. Two",
		"2.1 Three",
		"3. Four",
		"3.1 Five",
	}

	if !reflect.DeepEqual(splitContents, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", splitContents, expected)
	}

	mdParser = NewMarkdownParser(`
# One
## Two
### Three
#### Four
##### Five
###### Six
	`)
	tableOfContents = mdParser.GenerateTableOfContents()
	splitContents = getSplitContents(tableOfContents)
	expected = []string{
		"1. One",
		"1.1 Two",
		"1.1.1 Three",
		"1.1.1.1 Four",
		"1.1.1.1.1 Five",
		"1.1.1.1.1.1 Six",
	}

	if !reflect.DeepEqual(splitContents, expected) {
		t.Errorf("Not equal:\n  %#v\n  %#v", splitContents, expected)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/base64"
	"flag"
	"fmt"
	"golang.org/x/crypto/ssh/terminal"
	"log"
	"os"
)

func readStdin() []byte {
	r := bufio.NewReader(os.Stdin)
	bytes, _ := r.ReadBytes('\x00')

	return bytes
}

func readFile(fname string) []byte {
	var bytes []byte

	if fname == "stdin" {
		bytes = readStdin()
	} else {
		f, err := os.Open(fname)
		defer f.Close()
		if err != nil {
			log.Fatal(err)
		}

		r := bufio.NewReader(f)
		bytes, _ = r.ReadBytes('\x00')
	}

	return bytes
}

func parseArgs() (decrypt bool, printKey bool, suffix string, fnames []string) {
	d := flag.Bool("d", false, "decrypt")
	p := flag.Bool("p", false, "print key")
	s := flag.String("s", "", "suffix (ignored when if no files are specified)")

	var usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: rc4crypt [options] [FILE1 [FILE2 ..]] \noptions:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "output is printed stdout, pass-phrase is read from /dev/tty (or stdin as backup)\n")
		fmt.Fprintf(os.Stderr, "if no files are specified stream is read from stdin\n")
	}

	flag.Usage = usage

	flag.Parse()

	fnames = flag.Args()

	decrypt = *d
	printKey = *p
	suffix = *s

	return
}

func readPassPhrase(decrypt bool) []byte {
	f, err := os.Open("/dev/tty")
	defer f.Close()
	if err != nil {
		f = os.Stdin
	}

	fmt.Fprintf(os.Stderr, "Enter pass-phrase: ")
	try1, _ := terminal.ReadPassword(int(f.Fd()))

	if !decrypt {
		fmt.Fprintf(os.Stderr, "\nEnter pass-phrase again: ")
		try2, _ := terminal.ReadPassword(int(f.Fd()))

		if string(try1) != string(try2) {
			log.Fatal("Error: passphrases dont match")
		}
	}

	fmt.Fprintf(os.Stderr, "\n")

	return try1
}

func makeKey(passPhrase []byte, printKey bool) []byte {
	key := make([]byte, 256)

	for i, _ := range key {
		key[i] = byte(i)
	}

	//var x int
	x := 0

	for i, _ := range key {
		x = int(byte(x) + passPhrase[(i%len(passPhrase))] + (key[i] & '\xFF'))
		swap := key[i]
		key[i] = key[x]
		key[x] = swap
	}

	if printKey {
		fmt.Println("key: ", base64.StdEncoding.EncodeToString(key))
	}

	return key
}

func applyEncryption(input []byte, keyOrig []byte) []byte {
	// copy the key so it isn't changed
	key := make([]byte, len(keyOrig))
	for i, v := range keyOrig {
		key[i] = v
	}

	output := make([]byte, len(input))

	x := 0
	y := 0

	for i, _ := range input {
		x = (x + 1) % 256
		y = int(key[x] + byte(y)&'\xFF')
		swap := key[x]
		key[x] = key[y]
		key[y] = swap
		r := key[(key[x] + key[y]&'\xFF')]
		output[i] = byte(input[i] ^ r)
	}

	return output
}

func printOrWrite(fname string, suffix string, output []byte) {
	if suffix == "" || fname == "stdin" {
		fmt.Println(string(output))
	} else {
		fnameNew := fname + suffix

		f, err := os.Create(fnameNew)
		if err != nil {
			log.Fatal(err)
		}

		f.Write(output)
		f.Close()
	}
}

func main() {
	decrypt, printKey, suffix, fnames := parseArgs()

	passPhrase := readPassPhrase(decrypt)

	key := makeKey(passPhrase, printKey)

	if len(fnames) == 0 {
		fnames = append(fnames, "stdin")
	}

	for _, fname := range fnames {
		input := readFile(fname)

		if decrypt {
			input, _ = base64.StdEncoding.DecodeString(string(input))
		}

		output := applyEncryption(input, key)

		if !decrypt {
			output = []byte(base64.StdEncoding.EncodeToString(output))
		}

		printOrWrite(fname, suffix, output)
	}
}
<commit_msg>suffixed files are placed in PWD, not in original source dir<commit_after>package main

import (
	"bufio"
	"encoding/base64"
	"flag"
	"fmt"
	"golang.org/x/crypto/ssh/terminal"
	"log"
	"os"
	"path"
)

func readStdin() []byte {
	r := bufio.NewReader(os.Stdin)
	bytes, _ := r.ReadBytes('\x00')

	return bytes
}

func readFile(fname string) []byte {
	var bytes []byte

	if fname == "stdin" {
		bytes = readStdin()
	} else {
		f, err := os.Open(fname)
		defer f.Close()
		if err != nil {
			log.Fatal(err)
		}

		r := bufio.NewReader(f)
		bytes, _ = r.ReadBytes('\x00')
	}

	return bytes
}

func parseArgs() (decrypt bool, printKey bool, suffix string, fnames []string) {
	d := flag.Bool("d", false, "decrypt")
	p := flag.Bool("p", false, "print key")
	s := flag.String("s", "", "suffix (ignored when if no files are specified)")

	var usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: rc4crypt [options] [FILE1 [FILE2 ..]] \noptions:\n")
		flag.PrintDefaults()
		fmt.Fprintf(os.Stderr, "output is printed stdout, pass-phrase is read from /dev/tty (or stdin as backup)\n")
		fmt.Fprintf(os.Stderr, "if no files are specified stream is read from stdin\n")
	}

	flag.Usage = usage

	flag.Parse()

	fnames = flag.Args()

	decrypt = *d
	printKey = *p
	suffix = *s

	return
}

func readPassPhrase(decrypt bool) []byte {
	f, err := os.Open("/dev/tty")
	defer f.Close()
	if err != nil {
		f = os.Stdin
	}

	fmt.Fprintf(os.Stderr, "Enter pass-phrase: ")
	try1, _ := terminal.ReadPassword(int(f.Fd()))

	if !decrypt {
		fmt.Fprintf(os.Stderr, "\nEnter pass-phrase again: ")
		try2, _ := terminal.ReadPassword(int(f.Fd()))

		if string(try1) != string(try2) {
			log.Fatal("Error: passphrases dont match")
		}
	}

	fmt.Fprintf(os.Stderr, "\n")

	return try1
}

func makeKey(passPhrase []byte, printKey bool) []byte {
	key := make([]byte, 256)

	for i, _ := range key {
		key[i] = byte(i)
	}

	//var x int
	x := 0

	for i, _ := range key {
		x = int(byte(x) + passPhrase[(i%len(passPhrase))] + (key[i] & '\xFF'))
		swap := key[i]
		key[i] = key[x]
		key[x] = swap
	}

	if printKey {
		fmt.Println("key: ", base64.StdEncoding.EncodeToString(key))
	}

	return key
}

func applyEncryption(input []byte, keyOrig []byte) []byte {
	// copy the key so it isn't changed
	key := make([]byte, len(keyOrig))
	for i, v := range keyOrig {
		key[i] = v
	}

	output := make([]byte, len(input))

	x := 0
	y := 0

	for i, _ := range input {
		x = (x + 1) % 256
		y = int(key[x] + byte(y)&'\xFF')
		swap := key[x]
		key[x] = key[y]
		key[y] = swap
		r := key[(key[x] + key[y]&'\xFF')]
		output[i] = byte(input[i] ^ r)
	}

	return output
}

func printOrWrite(fname string, suffix string, output []byte) {
	if suffix == "" || fname == "stdin" {
		fmt.Println(string(output))
	} else {
		fnameNew := path.Base(fname + suffix)

		f, err := os.Create(fnameNew)
		if err != nil {
			log.Fatal(err)
		}

		f.Write(output)
		f.Close()
	}
}

func main() {
	decrypt, printKey, suffix, fnames := parseArgs()

	passPhrase := readPassPhrase(decrypt)

	key := makeKey(passPhrase, printKey)

	if len(fnames) == 0 {
		fnames = append(fnames, "stdin")
	}

	for _, fname := range fnames {
		input := readFile(fname)

		if decrypt {
			input, _ = base64.StdEncoding.DecodeString(string(input))
		}

		output := applyEncryption(input, key)

		if !decrypt {
			output = []byte(base64.StdEncoding.EncodeToString(output))
		}

		printOrWrite(fname, suffix, output)
	}
}
<|endoftext|>


<commit_before>package db

import (
	"bs3stat/migration"
	"fmt"

	"github.com/jinzhu/gorm"
	// Needs to be blank import because of sqlite init function
	_ "github.com/mattn/go-sqlite3"
)

// InitDB creates an database and opens a connection
func InitDB(filepath string, runMigrations bool, seedDatabase bool) *gorm.DB {
	var err error
	db, err := gorm.Open("sqlite3", filepath)
	if err != nil {
		panic(fmt.Sprintf("failed to connect database. err=%+v", err))
	}

	if runMigrations == true {
		migration.Migrate(db)
	}

	if seedDatabase == true {
		migration.Seed(db)
	}

	db.LogMode(true)
	return db
}
<commit_msg>🔥 Remove log mode<commit_after>package db

import (
	"bs3stat/migration"
	"fmt"

	"github.com/jinzhu/gorm"
	// Needs to be blank import because of sqlite init function
	_ "github.com/jinzhu/gorm/dialects/sqlite"
)

// InitDB creates an database and opens a connection
func InitDB(filepath string, runMigrations bool, seedDatabase bool) *gorm.DB {
	var err error
	db, err := gorm.Open("sqlite3", filepath)
	if err != nil {
		panic(fmt.Sprintf("failed to connect database. err=%+v", err))
	}

	if runMigrations == true {
		migration.Migrate(db)
	}

	if seedDatabase == true {
		migration.Seed(db)
	}

	return db
}
<|endoftext|>