<commit_before>package bagman_test

import (
	"testing"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagins"
	"errors"
	"os"
	"fmt"
	"io/ioutil"
	"path/filepath"
)

var gopath string = os.Getenv("GOPATH")
var testDataPath = filepath.Join(gopath, "src/github.com/APTrust/bagman/testdata")
var sampleBadChecksums string = filepath.Join(testDataPath, "sample_bad_checksums.tar")
var sampleGood string = filepath.Join(testDataPath, "sample_good.tar")
var sampleMissingDataFile string = filepath.Join(testDataPath, "sample_missing_data_file.tar")
var sampleNoBagInfo string = filepath.Join(testDataPath, "sample_no_bag_info.tar")
var sampleNoBagit string = filepath.Join(testDataPath, "sample_no_bagit.tar")
var badFiles []string = []string{
	sampleBadChecksums,
	sampleMissingDataFile,
	sampleNoBagInfo,
	sampleNoBagit,
}
var goodFiles []string = []string{
	sampleGood,
}
var allFiles []string = append(badFiles, goodFiles ...)

func setup() {

}

func teardown() {
	files, err := ioutil.ReadDir(testDataPath)
	if err != nil {
		fmt.Errorf("Can't cleanup %s: %s", testDataPath, err.Error())
		return
	}
	for _, fileInfo := range files {
		if fileInfo.IsDir() {
			subDir := filepath.Join(testDataPath, fileInfo.Name())
			err := os.RemoveAll(subDir)
			if err != nil {
				fmt.Errorf("Test cleanup was unable delete %s", subDir)
			}
		}
	}
}

func assertTagMatch(tag bagins.TagField, expectedLabel string, expectedValue string) (err error) {
	if tag.Label() != expectedLabel || tag.Value() != expectedValue {
		return errors.New(fmt.Sprintf("Expected tag '%s: %s', got '%s: %s'",
			expectedLabel, expectedValue, tag.Label(), tag.Value()))
	}
	return nil
}

// Make sure we can parse a bag that is known to be good, and that we
// get the right data in the results. This is not a strict unit test,
// since it depends on bagman.Untar succeeding.
func TestGoodBagParsesCorrectly(t *testing.T) {
	setup()
	defer teardown()
	tarResult := bagman.Untar(sampleGood)
	result := bagman.ReadBag(tarResult.OutputDir)
	if result.Path != tarResult.OutputDir {
		t.Errorf("Result path %s is incorrect, expected %s", result.Path, tarResult.OutputDir)
	}
	if len(result.Files) != 8 {
		t.Errorf("Unpacked %d files, expected %d", len(result.Files), 8)
	}
	if result.Error != nil {
		t.Errorf("Unexpected error in read result: %v", result.Error)
	}
	// Note that we're testing to see not only that the tags are present,
	// but that they are in the correct order.
	//
	// TODO: Bagins is returning one extra empty tag as the first element in every tag list
	// from every bag file. We need to fix that.
	if len(result.Tags) != 6 {
		t.Errorf("Expected 6 tags, got %d", len(result.Tags))
	}
	// TODO: This empty tag should not be here.
	if err := assertTagMatch(result.Tags[0], "", ""); err != nil { t.Error(err) }

	err := assertTagMatch(result.Tags[1], "Source-Organization", "virginia.edu")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[2], "Bagging-Date", "2014-04-14T11:55:26.17-0400")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[3], "Bag-Count", "1 of 1")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[4], "Bag-Group-Identifier", "")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[5], "Internal-Sender-Description", "")
	if err != nil { t.Error(err) }

	if len(result.ChecksumErrors) != 0 {
		t.Errorf("Bag read result contained %d checksum errors; it should have none",
			len(result.ChecksumErrors))
	}
}

func TestBadBagReturnsError(t *testing.T) {
	setup()
	defer teardown()
	for _, tarFile := range badFiles {
		tarResult := bagman.Untar(tarFile)
		result := bagman.ReadBag(tarResult.OutputDir)
		if result.Error == nil {
			t.Errorf("Bag unpacked from %s should have produced an error, but did not",
				tarResult.OutputDir)
		}
	}
}
<commit_msg>Updated tests to work with fixes to bagins tag parser<commit_after>package bagman_test

import (
	"testing"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagins"
	"errors"
	"os"
	"fmt"
	"io/ioutil"
	"path/filepath"
)

var gopath string = os.Getenv("GOPATH")
var testDataPath = filepath.Join(gopath, "src/github.com/APTrust/bagman/testdata")
var sampleBadChecksums string = filepath.Join(testDataPath, "sample_bad_checksums.tar")
var sampleGood string = filepath.Join(testDataPath, "sample_good.tar")
var sampleMissingDataFile string = filepath.Join(testDataPath, "sample_missing_data_file.tar")
var sampleNoBagInfo string = filepath.Join(testDataPath, "sample_no_bag_info.tar")
var sampleNoBagit string = filepath.Join(testDataPath, "sample_no_bagit.tar")
var badFiles []string = []string{
	sampleBadChecksums,
	sampleMissingDataFile,
	sampleNoBagInfo,
	sampleNoBagit,
}
var goodFiles []string = []string{
	sampleGood,
}
var allFiles []string = append(badFiles, goodFiles ...)

// Setup to run before tests
func setup() {

}

// Teardown to run after tests. This deletes the directories
// that were created when tar files were unpacked.
func teardown() {
	files, err := ioutil.ReadDir(testDataPath)
	if err != nil {
		fmt.Errorf("Can't cleanup %s: %s", testDataPath, err.Error())
		return
	}
	for _, fileInfo := range files {
		if fileInfo.IsDir() {
			subDir := filepath.Join(testDataPath, fileInfo.Name())
			err := os.RemoveAll(subDir)
			if err != nil {
				fmt.Errorf("Test cleanup was unable delete %s", subDir)
			}
		}
	}
}

// Check to see if the label and value of a tag match what
// we're expecting. If the label or value does not match
// what's expected, return an error. Otherwise return nil.
func assertTagMatch(tag bagins.TagField, expectedLabel string, expectedValue string) (err error) {
	if tag.Label() != expectedLabel || tag.Value() != expectedValue {
		return errors.New(fmt.Sprintf("Expected tag '%s: %s', got '%s: %s'",
			expectedLabel, expectedValue, tag.Label(), tag.Value()))
	}
	return nil
}

// Make sure we can parse a bag that is known to be good, and that we
// get the right data in the results. This is not a strict unit test,
// since it depends on bagman.Untar succeeding.
func TestGoodBagParsesCorrectly(t *testing.T) {
	setup()
	defer teardown()
	tarResult := bagman.Untar(sampleGood)
	result := bagman.ReadBag(tarResult.OutputDir)
	if result.Path != tarResult.OutputDir {
		t.Errorf("Result path %s is incorrect, expected %s", result.Path, tarResult.OutputDir)
	}
	if len(result.Files) != 8 {
		t.Errorf("Unpacked %d files, expected %d", len(result.Files), 8)
	}
	if result.Error != nil {
		t.Errorf("Unexpected error in read result: %v", result.Error)
	}
	// Note that we're testing to see not only that the tags are present,
	// but that they are in the correct order.
	//
	// TODO: Bagins is returning one extra empty tag as the first element in every tag list
	// from every bag file. We need to fix that.
	if len(result.Tags) != 6 {
		t.Errorf("Expected 6 tags, got %d", len(result.Tags))
	}
	// TODO: This empty tag should not be here.
	// if err := assertTagMatch(result.Tags[0], "", ""); err != nil { t.Error(err) }

	err := assertTagMatch(result.Tags[0], "Source-Organization", "virginia.edu")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[1], "Bagging-Date", "2014-04-14T11:55:26.17-0400")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[2], "Bag-Count", "1 of 1")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[3], "Bag-Group-Identifier", "")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[4], "Internal-Sender-Description", "")
	if err != nil { t.Error(err) }

	err = assertTagMatch(result.Tags[5], "Internal-Sender-Identifier", "")
	if err != nil { t.Error(err) }

	if len(result.ChecksumErrors) != 0 {
		t.Errorf("Bag read result contained %d checksum errors; it should have none",
			len(result.ChecksumErrors))
	}
}

func TestBadBagReturnsError(t *testing.T) {
	setup()
	defer teardown()
	for _, tarFile := range badFiles {
		tarResult := bagman.Untar(tarFile)
		result := bagman.ReadBag(tarResult.OutputDir)
		if result.Error == nil {
			t.Errorf("Bag unpacked from %s should have produced an error, but did not",
				tarResult.OutputDir)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/abourget/slack"
	"sync"
	"time"
)

type Standup struct {
	Questions         []string
	Finished          bool
	Channel           slack.Channel
	Duration          time.Duration
	client            *AuthedSlack
	userIds           []string
	userManager       *UserManager
	userReplies       map[*User]userReply
	userRepliesMutex  sync.Mutex
	finishedChan      chan struct{}
	reportedWaitGroup *sync.WaitGroup
}

type userReply interface {
	isUserReply()
}

type userAbsentReply struct{}
type userAnswersReply []string
type userSkippedReply struct{}
type userErrorReply struct{}

func (r userAbsentReply) isUserReply() {
}

func (r userAnswersReply) isUserReply() {
}

func (r userAnswersReply) isCompleted() bool {
	for _, a := range r {
		if a == "" {
			return false
		}
	}
	return true
}

func (r userSkippedReply) isUserReply() {
}

func (r userErrorReply) isUserReply() {
}

func NewStandup(client *AuthedSlack, channel slack.Channel, userManager *UserManager, reportedWaitGroup *sync.WaitGroup) (s *Standup) {

	reportedWaitGroup.Add(1)

	s = &Standup{
		client:            client,
		Channel:           channel,
		userManager:       userManager,
		userReplies:       make(map[*User]userReply),
		Questions:         Questions,
		finishedChan:      make(chan struct{}, 1),
		Duration:          StandupTimeMinutes * time.Minute,
		reportedWaitGroup: reportedWaitGroup,
	}

	return s
}

func (self *Standup) Run() {
	self.userIds = make([]string, 0, len(self.Channel.Members))

	for _, userId := range self.Channel.Members {
		if userId != self.client.UserId && self.userManager.StartStandup(self, userId) {
			self.userIds = append(self.userIds, userId)
		}
	}

	go self.startTheClock()

	_ = <-self.finishedChan
	self.Finished = true
	DebugLog.Print("sending summary...")

	var msg bytes.Buffer

	msg.WriteString("<!everyone>: *BARKBARKBARK Stand-up done!*\nQuestions were:\n")
	for _, q := range self.Questions {
		msg.WriteString("• ")
		msg.WriteString(q)
		msg.WriteString("\n")
	}
	msg.WriteString("\n")

	for user, anyReply := range self.userReplies {
		userName := fmt.Sprintf("<@%s|%s>", user.Info.Id, user.Info.Name)
		switch reply := anyReply.(type) {
		case userAnswersReply:
			msg.WriteString(userName)
			msg.WriteString(" answered:\n")
			for _, a := range reply {
				if a == "" {
					msg.WriteString("but didn't respond to the rest.\n")
					break
				}
				msg.WriteString("• ")
				msg.WriteString(a)
				msg.WriteString("\n")
			}
		case userAbsentReply:
			msg.WriteString(userName)
			msg.WriteString(" never replied to me :disappointed:")
		case userSkippedReply:
			msg.WriteString(userName)
			msg.WriteString(" skipped this stand-up.")
		case userErrorReply:
			msg.WriteString("There was an error when trying to chat with ")
			msg.WriteString(userName)
		default:
			msg.WriteString("I don't know what ")
			msg.WriteString(userName)
			msg.WriteString(" did. It is a mystery to me. :no_mouth:")
		}
		msg.WriteString("\n")
	}

	var params slack.PostMessageParameters
	params = DefaultMessageParameters
	params.Parse = "none"
	params.LinkNames = 0
	params.EscapeText = false

	_, _, err := self.client.PostMessage(self.Channel.Id, msg.String(), params)
	if err == nil {
		DebugLog.Print("summary sent")
	} else {
		DebugLog.Printf("error posting summary: %s", err.Error())
	}

	self.reportedWaitGroup.Done()
}

func (self *Standup) ReportUserAcknowledged(u *User) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	self.userReplies[u] = userAbsentReply{}
	// don't check for completion, we're only just starting
}

func (self *Standup) ReportUserAnswer(u *User, qidx int, answer string) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	DebugLog.Printf("got answer from user %s: %s", u.Info.Name, answer)
	reply, replyExists := self.userReplies[u]
	if _, isAbsent := reply.(userAbsentReply); !replyExists || isAbsent {
		reply = make(userAnswersReply, len(self.Questions))
		self.userReplies[u] = reply
	}
	if answers, ok := reply.(userAnswersReply); ok {
		answers[qidx] = answer
	}

	self.checkFinished()
}

func (self *Standup) ReportUserError(u *User) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	self.userReplies[u] = userErrorReply{}
	self.checkFinished()
}

func (self *Standup) ReportUserSkip(u *User) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	self.userReplies[u] = userSkippedReply{}
	self.checkFinished()
}

func (self *Standup) IsLastQuestion(i int) bool {
	return i >= len(self.Questions)-1
}

func (self *Standup) startTheClock() {
	time.Sleep(self.Duration)

	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	for user, _ := range self.userReplies {
		user.StandupTimeUp(self)
	}

	self.finish()
}

func (self *Standup) finish() {
	DebugLog.Print("finishing standup...")
	self.finishedChan <- struct{}{}
}

func (self *Standup) isFinished() bool {
	if len(self.userIds) != len(self.userReplies) {
		return false
	}
	for _, reply := range self.userReplies {
		switch r := reply.(type) {
		case userAnswersReply:
			if !r.isCompleted() {
				return false
			}
		case userAbsentReply:
			return false
		}
	}
	return true
}

func (self *Standup) checkFinished() {
	if self.isFinished() {
		self.finish()
	}
}
<commit_msg>Use unrestricted at-here rather than at-channel in summary message<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/abourget/slack"
	"sync"
	"time"
)

type Standup struct {
	Questions         []string
	Finished          bool
	Channel           slack.Channel
	Duration          time.Duration
	client            *AuthedSlack
	userIds           []string
	userManager       *UserManager
	userReplies       map[*User]userReply
	userRepliesMutex  sync.Mutex
	finishedChan      chan struct{}
	reportedWaitGroup *sync.WaitGroup
}

type userReply interface {
	isUserReply()
}

type userAbsentReply struct{}
type userAnswersReply []string
type userSkippedReply struct{}
type userErrorReply struct{}

func (r userAbsentReply) isUserReply() {
}

func (r userAnswersReply) isUserReply() {
}

func (r userAnswersReply) isCompleted() bool {
	for _, a := range r {
		if a == "" {
			return false
		}
	}
	return true
}

func (r userSkippedReply) isUserReply() {
}

func (r userErrorReply) isUserReply() {
}

func NewStandup(client *AuthedSlack, channel slack.Channel, userManager *UserManager, reportedWaitGroup *sync.WaitGroup) (s *Standup) {

	reportedWaitGroup.Add(1)

	s = &Standup{
		client:            client,
		Channel:           channel,
		userManager:       userManager,
		userReplies:       make(map[*User]userReply),
		Questions:         Questions,
		finishedChan:      make(chan struct{}, 1),
		Duration:          StandupTimeMinutes * time.Minute,
		reportedWaitGroup: reportedWaitGroup,
	}

	return s
}

func (self *Standup) Run() {
	self.userIds = make([]string, 0, len(self.Channel.Members))

	for _, userId := range self.Channel.Members {
		if userId != self.client.UserId && self.userManager.StartStandup(self, userId) {
			self.userIds = append(self.userIds, userId)
		}
	}

	go self.startTheClock()

	_ = <-self.finishedChan
	self.Finished = true
	DebugLog.Print("sending summary...")

	var msg bytes.Buffer

	msg.WriteString("<!here>: *BARKBARKBARK Stand-up done!*\nQuestions were:\n")
	for _, q := range self.Questions {
		msg.WriteString("• ")
		msg.WriteString(q)
		msg.WriteString("\n")
	}
	msg.WriteString("\n")

	for user, anyReply := range self.userReplies {
		userName := fmt.Sprintf("<@%s|%s>", user.Info.Id, user.Info.Name)
		switch reply := anyReply.(type) {
		case userAnswersReply:
			msg.WriteString(userName)
			msg.WriteString(" answered:\n")
			for _, a := range reply {
				if a == "" {
					msg.WriteString("but didn't respond to the rest.\n")
					break
				}
				msg.WriteString("• ")
				msg.WriteString(a)
				msg.WriteString("\n")
			}
		case userAbsentReply:
			msg.WriteString(userName)
			msg.WriteString(" never replied to me :disappointed:")
		case userSkippedReply:
			msg.WriteString(userName)
			msg.WriteString(" skipped this stand-up.")
		case userErrorReply:
			msg.WriteString("There was an error when trying to chat with ")
			msg.WriteString(userName)
		default:
			msg.WriteString("I don't know what ")
			msg.WriteString(userName)
			msg.WriteString(" did. It is a mystery to me. :no_mouth:")
		}
		msg.WriteString("\n")
	}

	var params slack.PostMessageParameters
	params = DefaultMessageParameters
	params.Parse = "none"
	params.LinkNames = 0
	params.EscapeText = false

	_, _, err := self.client.PostMessage(self.Channel.Id, msg.String(), params)
	if err == nil {
		DebugLog.Print("summary sent")
	} else {
		DebugLog.Printf("error posting summary: %s", err.Error())
	}

	self.reportedWaitGroup.Done()
}

func (self *Standup) ReportUserAcknowledged(u *User) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	self.userReplies[u] = userAbsentReply{}
	// don't check for completion, we're only just starting
}

func (self *Standup) ReportUserAnswer(u *User, qidx int, answer string) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	DebugLog.Printf("got answer from user %s: %s", u.Info.Name, answer)
	reply, replyExists := self.userReplies[u]
	if _, isAbsent := reply.(userAbsentReply); !replyExists || isAbsent {
		reply = make(userAnswersReply, len(self.Questions))
		self.userReplies[u] = reply
	}
	if answers, ok := reply.(userAnswersReply); ok {
		answers[qidx] = answer
	}

	self.checkFinished()
}

func (self *Standup) ReportUserError(u *User) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	self.userReplies[u] = userErrorReply{}
	self.checkFinished()
}

func (self *Standup) ReportUserSkip(u *User) {
	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	self.userReplies[u] = userSkippedReply{}
	self.checkFinished()
}

func (self *Standup) IsLastQuestion(i int) bool {
	return i >= len(self.Questions)-1
}

func (self *Standup) startTheClock() {
	time.Sleep(self.Duration)

	self.userRepliesMutex.Lock()
	defer self.userRepliesMutex.Unlock()

	for user, _ := range self.userReplies {
		user.StandupTimeUp(self)
	}

	self.finish()
}

func (self *Standup) finish() {
	DebugLog.Print("finishing standup...")
	self.finishedChan <- struct{}{}
}

func (self *Standup) isFinished() bool {
	if len(self.userIds) != len(self.userReplies) {
		return false
	}
	for _, reply := range self.userReplies {
		switch r := reply.(type) {
		case userAnswersReply:
			if !r.isCompleted() {
				return false
			}
		case userAbsentReply:
			return false
		}
	}
	return true
}

func (self *Standup) checkFinished() {
	if self.isFinished() {
		self.finish()
	}
}
<|endoftext|>


<commit_before>package goldb

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type Storage struct {
	Context
	dir string
	db  *leveldb.DB
	op  *opt.Options
	seq map[Entity]uint64
	mx  sync.Mutex
}

func NewStorage(dir string, op *opt.Options) (s *Storage) {
	dir = strings.TrimSuffix(dir, "/")

	s = &Storage{
		dir: dir,
		op:  op,
		seq: map[Entity]uint64{},
	}

	if err := s.Open(); err != nil {
		if errors.IsCorrupted(err) {
			// try to recover files
			if err := s.Recover(); err != nil {
				log.Println("!!! db.Storage.Recover-ERROR: ", err)
			}
			if err := s.Open(); err != nil {
				panic(err)
			}
		} else {
			panic(err)
		}
	}
	return
}

func (s *Storage) Open() error {
	db, err := leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return err
	}
	s.db = db
	s.Context.qCtx = db
	return nil
}

func (s *Storage) Recover() error {
	if db, err := leveldb.RecoverFile(s.dir, nil); err != nil {
		return err
	} else {
		return db.Close()
	}
}

func (s *Storage) Close() error {
	if s.db != nil {
		if err := s.db.Close(); err != leveldb.ErrClosed {
			return err
		}
	}
	return nil
}

func (s *Storage) Drop() error {
	if err := s.Close(); err != nil {
		return err
	}
	return os.RemoveAll(s.dir)
}

func (s *Storage) Size() (size uint64) {
	filepath.Walk(s.dir, func(_ string, info os.FileInfo, err error) error {
		if info != nil && !info.IsDir() {
			size += uint64(info.Size())
		}
		return err
	})
	return
}

func (s *Storage) Truncate() error {
	s.mx.Lock()
	defer s.mx.Unlock()

	if err := s.Drop(); err != nil {
		return err
	}
	return s.Open()
}

// Exec executes transaction.
// The executing transaction can be discard by methods tx.Fail(err) or by panic(err)
func (s *Storage) Exec(fn func(tx *Transaction)) (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	t := newTransaction(s)
	defer func() {
		if e, _ := recover().(error); e != nil {
			t.Discard()
			err = e
		}
	}()
	if t.err != nil {
		return t.err
	}
	fn(t)
	if t.err == nil {
		t.Commit()
	} else {
		t.Discard()
	}
	return t.err
}

func (s *Storage) Vacuum() (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	tmpDir := s.dir + ".reindex"
	oldDir := s.dir + ".old"

	defer os.RemoveAll(tmpDir)
	os.RemoveAll(tmpDir)
	os.RemoveAll(oldDir)

	dbOld := s.db
	dbNew, err := leveldb.OpenFile(tmpDir, s.op)
	if err != nil {
		return
	}

	iterator := dbOld.NewIterator(&util.Range{}, s.ReadOptions)

	var tr *leveldb.Transaction
	defer func() {
		iterator.Release()
		if err == nil {
			err = iterator.Error()
		}
		if tr != nil {
			tr.Discard()
		}
	}()
	for i := 0; iterator.Next(); i++ {
		if err = iterator.Error(); err != nil {
			return
		}
		if i%10000 == 0 {
			if tr != nil {
				if err = tr.Commit(); err != nil {
					return
				}
			}
			if tr, err = dbNew.OpenTransaction(); err != nil {
				return
			}
		}
		// put values to new DB
		key := iterator.Key()
		val := iterator.Value()
		if err = tr.Put(key, val, s.WriteOptions); err != nil {
			return
		}
	}
	if tr != nil {
		if err = tr.Commit(); err != nil {
			return
		}
		tr = nil
	}

	if err = dbNew.Close(); err != nil {
		return
	}

	if err = os.Rename(s.dir, oldDir); err != nil {
		return
	}
	if err = os.Rename(tmpDir, s.dir); err != nil {
		return
	}

	// reopen db
	dbNew, err = leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return
	}
	s.Context.qCtx = dbNew
	s.db = dbNew
	dbOld.Close()

	os.RemoveAll(oldDir)

	return
}

func (s *Storage) Put(key, data []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Put(key, data)
	})
}

func (s *Storage) PutID(key []byte, id uint64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutID(key, id)
	})
}

func (s *Storage) PutInt(key []byte, num int64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutInt(key, num)
	})
}

func (s *Storage) PutVar(key []byte, v interface{}) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutVar(key, v)
	})
}

func (s *Storage) Del(key []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Del(key)
	})
}

func (s *Storage) RemoveByQuery(q *Query) error {
	return s.Exec(func(tr *Transaction) {
		tr.Fetch(q, func(key, value []byte) error {
			tr.Del(key)
			return nil
		})
	})
}
<commit_msg>change Size func<commit_after>package goldb

import (
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

type Storage struct {
	Context
	dir string
	db  *leveldb.DB
	op  *opt.Options
	seq map[Entity]uint64
	mx  sync.Mutex
}

func NewStorage(dir string, op *opt.Options) (s *Storage) {
	dir = strings.TrimSuffix(dir, "/")

	s = &Storage{
		dir: dir,
		op:  op,
		seq: map[Entity]uint64{},
	}

	if err := s.Open(); err != nil {
		if errors.IsCorrupted(err) {
			// try to recover files
			if err := s.Recover(); err != nil {
				log.Println("!!! db.Storage.Recover-ERROR: ", err)
			}
			if err := s.Open(); err != nil {
				panic(err)
			}
		} else {
			panic(err)
		}
	}
	return
}

func (s *Storage) Open() error {
	db, err := leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return err
	}
	s.db = db
	s.Context.qCtx = db
	return nil
}

func (s *Storage) Recover() error {
	if db, err := leveldb.RecoverFile(s.dir, nil); err != nil {
		return err
	} else {
		return db.Close()
	}
}

func (s *Storage) Close() error {
	if s.db != nil {
		if err := s.db.Close(); err != leveldb.ErrClosed {
			return err
		}
	}
	return nil
}

func (s *Storage) Drop() error {
	if err := s.Close(); err != nil {
		return err
	}
	return os.RemoveAll(s.dir)
}

func (s *Storage) Size() (size int64) {
	filepath.Walk(s.dir, func(_ string, info os.FileInfo, err error) error {
		if info != nil && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return
}

func (s *Storage) Truncate() error {
	s.mx.Lock()
	defer s.mx.Unlock()

	if err := s.Drop(); err != nil {
		return err
	}
	return s.Open()
}

// Exec executes transaction.
// The executing transaction can be discard by methods tx.Fail(err) or by panic(err)
func (s *Storage) Exec(fn func(tx *Transaction)) (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	t := newTransaction(s)
	defer func() {
		if e, _ := recover().(error); e != nil {
			t.Discard()
			err = e
		}
	}()
	if t.err != nil {
		return t.err
	}
	fn(t)
	if t.err == nil {
		t.Commit()
	} else {
		t.Discard()
	}
	return t.err
}

func (s *Storage) Vacuum() (err error) {
	s.mx.Lock()
	defer s.mx.Unlock()

	tmpDir := s.dir + ".reindex"
	oldDir := s.dir + ".old"

	defer os.RemoveAll(tmpDir)
	os.RemoveAll(tmpDir)
	os.RemoveAll(oldDir)

	dbOld := s.db
	dbNew, err := leveldb.OpenFile(tmpDir, s.op)
	if err != nil {
		return
	}

	iterator := dbOld.NewIterator(&util.Range{}, s.ReadOptions)

	var tr *leveldb.Transaction
	defer func() {
		iterator.Release()
		if err == nil {
			err = iterator.Error()
		}
		if tr != nil {
			tr.Discard()
		}
	}()
	for i := 0; iterator.Next(); i++ {
		if err = iterator.Error(); err != nil {
			return
		}
		if i%10000 == 0 {
			if tr != nil {
				if err = tr.Commit(); err != nil {
					return
				}
			}
			if tr, err = dbNew.OpenTransaction(); err != nil {
				return
			}
		}
		// put values to new DB
		key := iterator.Key()
		val := iterator.Value()
		if err = tr.Put(key, val, s.WriteOptions); err != nil {
			return
		}
	}
	if tr != nil {
		if err = tr.Commit(); err != nil {
			return
		}
		tr = nil
	}

	if err = dbNew.Close(); err != nil {
		return
	}

	if err = os.Rename(s.dir, oldDir); err != nil {
		return
	}
	if err = os.Rename(tmpDir, s.dir); err != nil {
		return
	}

	// reopen db
	dbNew, err = leveldb.OpenFile(s.dir, s.op)
	if err != nil {
		return
	}
	s.Context.qCtx = dbNew
	s.db = dbNew
	dbOld.Close()

	os.RemoveAll(oldDir)

	return
}

func (s *Storage) Put(key, data []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Put(key, data)
	})
}

func (s *Storage) PutID(key []byte, id uint64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutID(key, id)
	})
}

func (s *Storage) PutInt(key []byte, num int64) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutInt(key, num)
	})
}

func (s *Storage) PutVar(key []byte, v interface{}) error {
	return s.Exec(func(tr *Transaction) {
		tr.PutVar(key, v)
	})
}

func (s *Storage) Del(key []byte) error {
	return s.Exec(func(tr *Transaction) {
		tr.Del(key)
	})
}

func (s *Storage) RemoveByQuery(q *Query) error {
	return s.Exec(func(tr *Transaction) {
		tr.Fetch(q, func(key, value []byte) error {
			tr.Del(key)
			return nil
		})
	})
}
<|endoftext|>


<commit_before>package mpb_test

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"
	"unicode/utf8"

	"github.com/vbauerster/mpb"
	"github.com/vbauerster/mpb/decor"
)

func TestBarSetWidth(t *testing.T) {
	var buf bytes.Buffer
	// overwrite default width 80
	customWidth := 60
	p := mpb.New(mpb.Output(&buf), mpb.WithWidth(customWidth))
	bar := p.AddBar(100, mpb.BarTrim())

	for i := 0; i < 100; i++ {
		bar.Incr(1)
	}

	p.Stop()

	gotWidth := len(buf.Bytes())
	if gotWidth != customWidth+1 { // +1 for new line
		t.Errorf("Expected width: %d, got: %d\n", customWidth, gotWidth)
	}
}

func TestBarSetInvalidWidth(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New(mpb.Output(&buf), mpb.WithWidth(1))
	bar := p.AddBar(100, mpb.BarTrim())

	for i := 0; i < 100; i++ {
		bar.Incr(1)
	}

	p.Stop()

	wantWidth := 80
	gotWidth := len(buf.Bytes())
	if gotWidth != wantWidth+1 { // +1 for new line
		t.Errorf("Expected width: %d, got: %d\n", wantWidth, gotWidth)
	}
}

func TestBarFormat(t *testing.T) {
	var buf bytes.Buffer
	cancel := make(chan struct{})
	customFormat := "(#>_)"
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithCancel(cancel),
		mpb.WithFormat(customFormat),
	)
	bar := p.AddBar(100, mpb.BarTrim())

	go func() {
		for i := 0; i < 100; i++ {
			time.Sleep(10 * time.Millisecond)
			bar.Incr(1)
		}
	}()

	time.Sleep(250 * time.Millisecond)
	close(cancel)
	p.Stop()

	barAsStr := strings.Trim(buf.String(), "\n")

	seen := make(map[rune]bool)
	for _, r := range barAsStr {
		if !seen[r] {
			seen[r] = true
		}
	}
	for _, r := range customFormat {
		if !seen[r] {
			t.Errorf("Rune %#U not found in bar\n", r)
		}
	}
}

func TestBarInvalidFormat(t *testing.T) {
	var buf bytes.Buffer
	customWidth := 60
	customFormat := "(#>=_)"
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithWidth(customWidth),
		mpb.WithFormat(customFormat),
	)
	bar := p.AddBar(100, mpb.BarTrim())

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	got := buf.String()
	want := fmt.Sprintf("[%s]", strings.Repeat("=", customWidth-2))
	if !strings.Contains(got, want) {
		t.Errorf("Expected format: %s, got %s\n", want, got)
	}
}

func TestBarInProgress(t *testing.T) {
	var buf bytes.Buffer
	cancel := make(chan struct{})
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithCancel(cancel),
	)
	bar := p.AddBar(100, mpb.BarTrim())

	stopped := make(chan struct{})

	go func() {
		defer close(stopped)
		for bar.InProgress() {
			time.Sleep(10 * time.Millisecond)
			bar.Incr(1)
		}
	}()

	time.Sleep(250 * time.Millisecond)
	close(cancel)
	p.Stop()

	select {
	case <-stopped:
	case <-time.After(300 * time.Millisecond):
		t.Error("bar.InProgress returns true after cancel")
	}
}

func TestBarGetID(t *testing.T) {
	var wg sync.WaitGroup
	var buf bytes.Buffer
	p := mpb.New(mpb.Output(&buf))

	numBars := 3
	wg.Add(numBars)

	bars := make([]*mpb.Bar, numBars)
	for i := 0; i < numBars; i++ {
		bars[i] = p.AddBar(100, mpb.BarID(i))

		go func(bar *mpb.Bar) {
			defer wg.Done()
			for i := 0; i < 100; i++ {
				time.Sleep(10 * time.Millisecond)
				bar.Incr(1)
			}
		}(bars[i])
	}

	for wantID, bar := range bars {
		gotID := bar.ID()
		if gotID != wantID {
			t.Errorf("Expected bar id: %d, got %d\n", wantID, gotID)
		}
	}

	wg.Wait()
	p.Stop()
}

func TestBarIncrWithReFill(t *testing.T) {
	var buf bytes.Buffer

	width := 100
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithWidth(width),
	)

	total := 100
	till := 30
	refillChar := '+'

	bar := p.AddBar(100, mpb.BarTrim())

	bar.ResumeFill(refillChar, int64(till))

	for i := 0; i < total; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	bytes := removeLastRune(buf.Bytes())

	gotBar := string(bytes[len(bytes)-width:])
	wantBar := fmt.Sprintf("[%s%s]",
		strings.Repeat(string(refillChar), till-1),
		strings.Repeat("=", total-till-1))
	if gotBar != wantBar {
		t.Errorf("Want bar: %s, got bar: %s\n", wantBar, gotBar)
	}
}

func TestBarPanics(t *testing.T) {
	var wg sync.WaitGroup
	var buf bytes.Buffer
	p := mpb.New(mpb.Output(&buf), mpb.WithWaitGroup(&wg))

	wantPanic := "Upps!!!"
	numBars := 3
	wg.Add(numBars)

	for i := 0; i < numBars; i++ {
		name := fmt.Sprintf("b#%02d:", i)
		bar := p.AddBar(100, mpb.BarID(i), mpb.PrependDecorators(
			func(s *decor.Statistics, _ chan<- int, _ <-chan int) string {
				if s.ID == 2 && s.Current >= 42 {
					panic(wantPanic)
				}
				return name
			},
		))

		go func() {
			defer wg.Done()
			for i := 0; i < 100; i++ {
				time.Sleep(10 * time.Millisecond)
				bar.Increment()
			}
		}()
	}

	p.Stop()

	out := bytes.Split(removeLastRune(buf.Bytes()), []byte("\n"))
	gotPanic := out[len(out)-1]
	wantPanic = fmt.Sprintf("b#%02d panic: %v", 2, wantPanic)

	if string(gotPanic) != wantPanic {
		t.Errorf("Want: %q, got: %q\n", wantPanic, gotPanic)
	}
}

func removeLastRune(bytes []byte) []byte {
	_, size := utf8.DecodeLastRune(bytes)
	return bytes[:len(bytes)-size]
}
<commit_msg>TestWithWidth<commit_after>package mpb_test

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"
	"unicode/utf8"

	"github.com/vbauerster/mpb"
	"github.com/vbauerster/mpb/decor"
)

func TestWithWidth(t *testing.T) {
	cases := map[string]struct{ w, expected int }{
		"WithWidth-1": {-1, 81},
		"WithWidth0":  {0, 3},
		"WithWidth1":  {1, 3},
		"WithWidth2":  {2, 3},
		"WithWidth3":  {3, 4},
		"WithWidth60": {60, 61},
	}

	var buf bytes.Buffer
	for k, tc := range cases {
		buf.Reset()
		p := mpb.New(
			mpb.Output(&buf),
			mpb.WithWidth(tc.w),
		)
		bar := p.AddBar(10, mpb.BarTrim())

		for i := 0; i < 10; i++ {
			bar.Increment()
		}

		p.Stop()

		gotWidth := len(buf.Bytes())
		if gotWidth != tc.expected {
			t.Errorf("%s: Expected width: %d, got: %d\n", k, tc.expected, gotWidth)
		}
	}
}

func TestBarFormat(t *testing.T) {
	var buf bytes.Buffer
	cancel := make(chan struct{})
	customFormat := "(#>_)"
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithCancel(cancel),
		mpb.WithFormat(customFormat),
	)
	bar := p.AddBar(100, mpb.BarTrim())

	go func() {
		for i := 0; i < 100; i++ {
			time.Sleep(10 * time.Millisecond)
			bar.Increment()
		}
	}()

	time.Sleep(250 * time.Millisecond)
	close(cancel)
	p.Stop()

	barAsStr := strings.Trim(buf.String(), "\n")

	seen := make(map[rune]bool)
	for _, r := range barAsStr {
		if !seen[r] {
			seen[r] = true
		}
	}
	for _, r := range customFormat {
		if !seen[r] {
			t.Errorf("Rune %#U not found in bar\n", r)
		}
	}
}

func TestBarInvalidFormat(t *testing.T) {
	var buf bytes.Buffer
	customWidth := 60
	customFormat := "(#>=_)"
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithWidth(customWidth),
		mpb.WithFormat(customFormat),
	)
	bar := p.AddBar(100, mpb.BarTrim())

	for i := 0; i < 100; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	got := buf.String()
	want := fmt.Sprintf("[%s]", strings.Repeat("=", customWidth-2))
	if !strings.Contains(got, want) {
		t.Errorf("Expected format: %s, got %s\n", want, got)
	}
}

func TestBarInProgress(t *testing.T) {
	var buf bytes.Buffer
	cancel := make(chan struct{})
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithCancel(cancel),
	)
	bar := p.AddBar(100, mpb.BarTrim())

	stopped := make(chan struct{})

	go func() {
		defer close(stopped)
		for bar.InProgress() {
			time.Sleep(10 * time.Millisecond)
			bar.Incr(1)
		}
	}()

	time.Sleep(250 * time.Millisecond)
	close(cancel)
	p.Stop()

	select {
	case <-stopped:
	case <-time.After(300 * time.Millisecond):
		t.Error("bar.InProgress returns true after cancel")
	}
}

func TestBarGetID(t *testing.T) {
	var wg sync.WaitGroup
	var buf bytes.Buffer
	p := mpb.New(mpb.Output(&buf))

	numBars := 3
	wg.Add(numBars)

	bars := make([]*mpb.Bar, numBars)
	for i := 0; i < numBars; i++ {
		bars[i] = p.AddBar(100, mpb.BarID(i))

		go func(bar *mpb.Bar) {
			defer wg.Done()
			for i := 0; i < 100; i++ {
				time.Sleep(10 * time.Millisecond)
				bar.Incr(1)
			}
		}(bars[i])
	}

	for wantID, bar := range bars {
		gotID := bar.ID()
		if gotID != wantID {
			t.Errorf("Expected bar id: %d, got %d\n", wantID, gotID)
		}
	}

	wg.Wait()
	p.Stop()
}

func TestBarIncrWithReFill(t *testing.T) {
	var buf bytes.Buffer

	width := 100
	p := mpb.New(
		mpb.Output(&buf),
		mpb.WithWidth(width),
	)

	total := 100
	till := 30
	refillChar := '+'

	bar := p.AddBar(100, mpb.BarTrim())

	bar.ResumeFill(refillChar, int64(till))

	for i := 0; i < total; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()

	bytes := removeLastRune(buf.Bytes())

	gotBar := string(bytes[len(bytes)-width:])
	wantBar := fmt.Sprintf("[%s%s]",
		strings.Repeat(string(refillChar), till-1),
		strings.Repeat("=", total-till-1))
	if gotBar != wantBar {
		t.Errorf("Want bar: %s, got bar: %s\n", wantBar, gotBar)
	}
}

func TestBarPanics(t *testing.T) {
	var wg sync.WaitGroup
	var buf bytes.Buffer
	p := mpb.New(mpb.Output(&buf), mpb.WithWaitGroup(&wg))

	wantPanic := "Upps!!!"
	numBars := 3
	wg.Add(numBars)

	for i := 0; i < numBars; i++ {
		name := fmt.Sprintf("b#%02d:", i)
		bar := p.AddBar(100, mpb.BarID(i), mpb.PrependDecorators(
			func(s *decor.Statistics, _ chan<- int, _ <-chan int) string {
				if s.ID == 2 && s.Current >= 42 {
					panic(wantPanic)
				}
				return name
			},
		))

		go func() {
			defer wg.Done()
			for i := 0; i < 100; i++ {
				time.Sleep(10 * time.Millisecond)
				bar.Increment()
			}
		}()
	}

	p.Stop()

	out := bytes.Split(removeLastRune(buf.Bytes()), []byte("\n"))
	gotPanic := out[len(out)-1]
	wantPanic = fmt.Sprintf("b#%02d panic: %v", 2, wantPanic)

	if string(gotPanic) != wantPanic {
		t.Errorf("Want: %q, got: %q\n", wantPanic, gotPanic)
	}
}

func removeLastRune(bytes []byte) []byte {
	_, size := utf8.DecodeLastRune(bytes)
	return bytes[:len(bytes)-size]
}
<|endoftext|>


<commit_before>package base

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// ParseCSVGetRows returns the number of rows in a given file.
func ParseCSVGetRows(filepath string) int {
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	reader := csv.NewReader(file)
	counter := 0
	for {
		_, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
		counter++
	}
	return counter
}

// ParseCSVGetAttributes returns an ordered slice of appropriate-ly typed
// and named Attributes.
func ParseCSVGetAttributes(filepath string, hasHeaders bool) []Attribute {
	attrs := ParseCSVSniffAttributeTypes(filepath, hasHeaders)
	names := ParseCSVSniffAttributeNames(filepath, hasHeaders)
	for i, attr := range attrs {
		attr.SetName(names[i])
	}
	return attrs
}

// ParseCsvSniffAttributeNames returns a slice containing the top row
// of a given CSV file, or placeholders if hasHeaders is false.
func ParseCSVSniffAttributeNames(filepath string, hasHeaders bool) []string {
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	reader := csv.NewReader(file)
	headers, err := reader.Read()
	if err != nil {
		panic(err)
	}

	if hasHeaders {
		for i, h := range headers {
			headers[i] = strings.TrimSpace(h)
		}
		return headers
	}

	for i := range headers {
		headers[i] = fmt.Sprintf("%d", i)
	}
	return headers

}

// ParseCSVSniffAttributeTypes returns a slice of appropriately-typed Attributes.
//
// The type of a given attribute is determined by looking at the first data row
// of the CSV.
func ParseCSVSniffAttributeTypes(filepath string, hasHeaders bool) []Attribute {
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	reader := csv.NewReader(file)
	attrs := make([]Attribute, 0)
	if hasHeaders {
		_, err := reader.Read()
		if err != nil {
			panic(err)
		}
	}
	columns, err := reader.Read()
	if err != nil {
		panic(err)
	}

	for _, entry := range columns {
		matched, err := regexp.MatchString("^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?$", entry)
		if err != nil {
			panic(err)
		}
		if matched {
			attrs = append(attrs, NewFloatAttribute())
		} else {
			attrs = append(attrs, new(CategoricalAttribute))
		}
	}

	return attrs
}

// ParseCSVToInstances reads the CSV file given by filepath and returns
// the read Instances.
func ParseCSVToInstances(filepath string, hasHeaders bool) (instances *Instances, err error) {

	defer func() {
		if r := recover(); r != nil {
			var ok bool
			if err, ok = r.(error); !ok {
				err = fmt.Errorf("golearn: ParseCSVToInstances: %v", r)
			}
		}
	}()

	// Read the number of rows in the file
	rowCount := ParseCSVGetRows(filepath)
	if hasHeaders {
		rowCount--
	}

	// Read the row headers
	attrs := ParseCSVGetAttributes(filepath, hasHeaders)

	// Allocate the Instances to return
	instances = NewInstances(attrs, rowCount)

	// Read the input
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	reader := csv.NewReader(file)

	rowCounter := 0
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
		if rowCounter == 0 {
			if hasHeaders {
				hasHeaders = false
				continue
			}
		}
		for i := range attrs {
			instances.SetAttrStr(rowCounter, i, record[i])
		}
		rowCounter++
	}

	return
}

//ParseCSV parses a CSV file and returns the number of columns and rows, the headers, the labels associated with
//classification, and the data that will be used for training.
func ParseCSV(filepath string, label int, columns []int) (int, int, []string, []string, []float64) {
	labels := make([]string, 0)
	data := make([]float64, 0)
	headers := make([]string, 0)
	rows := 0

	file, err := os.Open(filepath)
	if err != nil {
		fmt.Println("Error:", err)
	}
	defer file.Close()

	reader := csv.NewReader(file)

	headerrow, _ := reader.Read()

	for _, col := range columns {
		entry := headerrow[col]
		headers = append(headers, entry)
	}

	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Error:", err)
		}

		//
		labels = append(labels, record[label])

		//Iterate over our rows and append the values to a slice
		for _, col := range columns {
			entry := record[col]
			number, _ := strconv.ParseFloat(entry, 64)
			data = append(data, number)
		}
		rows++
	}
	cols := len(columns)
	return cols, rows, headers, labels, data
}
<commit_msg>base: handling spaces between entries<commit_after>package base

import (
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// ParseCSVGetRows returns the number of rows in a given file.
func ParseCSVGetRows(filepath string) int {
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	reader := csv.NewReader(file)
	counter := 0
	for {
		_, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
		counter++
	}
	return counter
}

// ParseCSVGetAttributes returns an ordered slice of appropriate-ly typed
// and named Attributes.
func ParseCSVGetAttributes(filepath string, hasHeaders bool) []Attribute {
	attrs := ParseCSVSniffAttributeTypes(filepath, hasHeaders)
	names := ParseCSVSniffAttributeNames(filepath, hasHeaders)
	for i, attr := range attrs {
		attr.SetName(names[i])
	}
	return attrs
}

// ParseCsvSniffAttributeNames returns a slice containing the top row
// of a given CSV file, or placeholders if hasHeaders is false.
func ParseCSVSniffAttributeNames(filepath string, hasHeaders bool) []string {
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	reader := csv.NewReader(file)
	headers, err := reader.Read()
	if err != nil {
		panic(err)
	}

	if hasHeaders {
		for i, h := range headers {
			headers[i] = strings.TrimSpace(h)
		}
		return headers
	}

	for i := range headers {
		headers[i] = fmt.Sprintf("%d", i)
	}
	return headers

}

// ParseCSVSniffAttributeTypes returns a slice of appropriately-typed Attributes.
//
// The type of a given attribute is determined by looking at the first data row
// of the CSV.
func ParseCSVSniffAttributeTypes(filepath string, hasHeaders bool) []Attribute {
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	reader := csv.NewReader(file)
	attrs := make([]Attribute, 0)
	if hasHeaders {
		_, err := reader.Read()
		if err != nil {
			panic(err)
		}
	}
	columns, err := reader.Read()
	if err != nil {
		panic(err)
	}

	for _, entry := range columns {
		entry = strings.Trim(entry, " ")
		matched, err := regexp.MatchString("^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?$", entry)
		fmt.Println(entry, matched)
		if err != nil {
			panic(err)
		}
		if matched {
			attrs = append(attrs, NewFloatAttribute())
		} else {
			attrs = append(attrs, new(CategoricalAttribute))
		}
	}

	return attrs
}

// ParseCSVToInstances reads the CSV file given by filepath and returns
// the read Instances.
func ParseCSVToInstances(filepath string, hasHeaders bool) (instances *Instances, err error) {

	defer func() {
		if r := recover(); r != nil {
			var ok bool
			if err, ok = r.(error); !ok {
				err = fmt.Errorf("golearn: ParseCSVToInstances: %v", r)
			}
		}
	}()

	// Read the number of rows in the file
	rowCount := ParseCSVGetRows(filepath)
	if hasHeaders {
		rowCount--
	}

	// Read the row headers
	attrs := ParseCSVGetAttributes(filepath, hasHeaders)

	// Allocate the Instances to return
	instances = NewInstances(attrs, rowCount)

	// Read the input
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	reader := csv.NewReader(file)

	rowCounter := 0
	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			panic(err)
		}
		if rowCounter == 0 {
			if hasHeaders {
				hasHeaders = false
				continue
			}
		}
		for i := range attrs {
			instances.SetAttrStr(rowCounter, i, strings.Trim(record[i], " "))
		}
		rowCounter++
	}

	return
}

//ParseCSV parses a CSV file and returns the number of columns and rows, the headers, the labels associated with
//classification, and the data that will be used for training.
func ParseCSV(filepath string, label int, columns []int) (int, int, []string, []string, []float64) {
	labels := make([]string, 0)
	data := make([]float64, 0)
	headers := make([]string, 0)
	rows := 0

	file, err := os.Open(filepath)
	if err != nil {
		fmt.Println("Error:", err)
	}
	defer file.Close()

	reader := csv.NewReader(file)

	headerrow, _ := reader.Read()

	for _, col := range columns {
		entry := headerrow[col]
		headers = append(headers, entry)
	}

	for {
		record, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Error:", err)
		}

		//
		labels = append(labels, record[label])

		//Iterate over our rows and append the values to a slice
		for _, col := range columns {
			entry := record[col]
			number, _ := strconv.ParseFloat(entry, 64)
			data = append(data, number)
		}
		rows++
	}
	cols := len(columns)
	return cols, rows, headers, labels, data
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"fmt"
	"math/rand"
	"testing"
	"time"

	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"

	"github.com/stretchr/testify/assert"
)

var snapshots = map[string]func() ClusterSnapshot{
	"basic": func() ClusterSnapshot { return NewBasicClusterSnapshot() },
	"delta": func() ClusterSnapshot { return NewDeltaClusterSnapshot() },
}

func nodeNames(nodes []*apiv1.Node) []string {
	names := make([]string, len(nodes), len(nodes))
	for i, node := range nodes {
		names[i] = node.Name
	}
	return names
}

func extractNodes(nodeInfos []*schedulernodeinfo.NodeInfo) []*apiv1.Node {
	nodes := []*apiv1.Node{}
	for _, ni := range nodeInfos {
		nodes = append(nodes, ni.Node())
	}
	return nodes
}

type snapshotState struct {
	nodes []*apiv1.Node
	pods  []*apiv1.Pod
}

func compareStates(t *testing.T, a, b snapshotState) {
	assert.ElementsMatch(t, a.nodes, b.nodes)
	assert.ElementsMatch(t, a.pods, b.pods)
}

func getSnapshotState(t *testing.T, snapshot ClusterSnapshot) snapshotState {
	nodes, err := snapshot.NodeInfos().List()
	assert.NoError(t, err)
	pods, err := snapshot.Pods().List(labels.Everything())
	assert.NoError(t, err)
	return snapshotState{extractNodes(nodes), pods}
}

func startSnapshot(t *testing.T, snapshotFactory func() ClusterSnapshot, state snapshotState) ClusterSnapshot {
	snapshot := snapshotFactory()
	err := snapshot.AddNodes(state.nodes)
	assert.NoError(t, err)
	for _, pod := range state.pods {
		err := snapshot.AddPod(pod, pod.Spec.NodeName)
		assert.NoError(t, err)
	}
	return snapshot
}

func TestForking(t *testing.T) {
	node := BuildTestNode("specialNode", 10, 100)
	pod := BuildTestPod("specialPod", 1, 1)
	pod.Spec.NodeName = node.Name

	testCases := []struct {
		name          string
		op            func(ClusterSnapshot)
		state         snapshotState
		modifiedState snapshotState
	}{
		{
			name: "add node",
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.AddNode(node)
				assert.NoError(t, err)
			},
			modifiedState: snapshotState{
				nodes: []*apiv1.Node{node},
			},
		},
		{
			name: "add node with pods",
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.AddNodeWithPods(node, []*apiv1.Pod{pod})
				assert.NoError(t, err)
			},
			modifiedState: snapshotState{
				nodes: []*apiv1.Node{node},
				pods:  []*apiv1.Pod{pod},
			},
		},
		{
			name: "remove node",
			state: snapshotState{
				nodes: []*apiv1.Node{node},
			},
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.RemoveNode(node.Name)
				assert.NoError(t, err)
			},
		},
		{
			name: "remove node, then add it back",
			state: snapshotState{
				nodes: []*apiv1.Node{node},
			},
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.RemoveNode(node.Name)
				assert.NoError(t, err)

				err = snapshot.AddNode(node)
				assert.NoError(t, err)
			},
			modifiedState: snapshotState{
				nodes: []*apiv1.Node{node},
			},
		},
		{
			name: "add pod, then remove node",
			state: snapshotState{
				nodes: []*apiv1.Node{node},
			},
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.AddPod(pod, node.Name)
				assert.NoError(t, err)
				err = snapshot.RemoveNode(node.Name)
				assert.NoError(t, err)
			},
		},
	}

	for name, snapshotFactory := range snapshots {
		for _, tc := range testCases {
			t.Run(fmt.Sprintf("%s: %s base", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)
				tc.op(snapshot)

				// Modifications should be applied.
				compareStates(t, tc.modifiedState, getSnapshotState(t, snapshot))
			})
			t.Run(fmt.Sprintf("%s: %s fork", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)

				err := snapshot.Fork()
				assert.NoError(t, err)

				tc.op(snapshot)

				// Modifications should be applied.
				compareStates(t, tc.modifiedState, getSnapshotState(t, snapshot))
			})
			t.Run(fmt.Sprintf("%s: %s fork & revert", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)

				err := snapshot.Fork()
				assert.NoError(t, err)

				tc.op(snapshot)

				err = snapshot.Revert()
				assert.NoError(t, err)

				// Modifications should no longer be applied.
				compareStates(t, tc.state, getSnapshotState(t, snapshot))
			})
			t.Run(fmt.Sprintf("%s: %s fork & commit", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)

				err := snapshot.Fork()
				assert.NoError(t, err)

				tc.op(snapshot)

				err = snapshot.Commit()
				assert.NoError(t, err)

				// Modifications should be applied.
				compareStates(t, tc.modifiedState, getSnapshotState(t, snapshot))
			})
		}
	}
}

func TestClear(t *testing.T) {
	// Run with -count=1 to avoid caching.
	localRand := rand.New(rand.NewSource(time.Now().Unix()))

	nodeCount := localRand.Intn(100)
	podCount := localRand.Intn(1000)
	extraNodeCount := localRand.Intn(100)
	extraPodCount := localRand.Intn(1000)

	nodes := createTestNodes(nodeCount)
	pods := createTestPods(podCount)
	assignPodsToNodes(pods, nodes)

	state := snapshotState{nodes, pods}

	extraNodes := createTestNodesWithPrefix("extra", extraNodeCount)

	allNodes := make([]*apiv1.Node, len(nodes)+len(extraNodes), len(nodes)+len(extraNodes))
	copy(allNodes, nodes)
	copy(allNodes[len(nodes):], extraNodes)

	extraPods := createTestPodsWithPrefix("extra", extraPodCount)
	assignPodsToNodes(extraPods, allNodes)

	allPods := make([]*apiv1.Pod, len(pods)+len(extraPods), len(pods)+len(extraPods))
	copy(allPods, pods)
	copy(allPods[len(pods):], extraPods)

	for name, snapshotFactory := range snapshots {
		t.Run(fmt.Sprintf("%s: clear base %d nodes %d pods", name, nodeCount, podCount),
			func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, state)
				compareStates(t, state, getSnapshotState(t, snapshot))

				snapshot.Clear()

				compareStates(t, snapshotState{}, getSnapshotState(t, snapshot))
			})
		t.Run(fmt.Sprintf("%s: clear fork %d nodes %d pods %d extra nodes %d extra pods", name, nodeCount, podCount, extraNodeCount, extraPodCount),
			func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, state)
				compareStates(t, state, getSnapshotState(t, snapshot))

				err := snapshot.Fork()
				assert.NoError(t, err)

				err = snapshot.AddNodes(extraNodes)
				assert.NoError(t, err)

				for _, pod := range extraPods {
					err := snapshot.AddPod(pod, pod.Spec.NodeName)
					assert.NoError(t, err)
				}

				compareStates(t, snapshotState{allNodes, allPods}, getSnapshotState(t, snapshot))

				// Fork()ing twice is not allowed.
				err = snapshot.Fork()
				assert.Error(t, err)

				snapshot.Clear()

				compareStates(t, snapshotState{}, getSnapshotState(t, snapshot))

				// Clear() should break out of forked state.
				err = snapshot.Fork()
				assert.NoError(t, err)
			})
	}
}

func TestNode404(t *testing.T) {
	// Anything and everything that returns errNodeNotFound should be tested here.
	ops := []struct {
		name string
		op   func(ClusterSnapshot) error
	}{
		{"add pod", func(snapshot ClusterSnapshot) error {
			return snapshot.AddPod(BuildTestPod("p1", 0, 0), "node")
		}},
		{"remove pod", func(snapshot ClusterSnapshot) error {
			return snapshot.RemovePod("default", "p1", "node")
		}},
		{"get node", func(snapshot ClusterSnapshot) error {
			_, err := snapshot.NodeInfos().Get("node")
			return err
		}},
		{"remove node", func(snapshot ClusterSnapshot) error {
			return snapshot.RemoveNode("node")
		}},
	}

	for name, snapshotFactory := range snapshots {
		for _, op := range ops {
			t.Run(fmt.Sprintf("%s: %s empty", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					// Empty snapshot - shouldn't be able to operate on nodes that are not here.
					err := op.op(snapshot)
					assert.Error(t, err)
				})

			t.Run(fmt.Sprintf("%s: %s fork", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					node := BuildTestNode("node", 10, 100)
					err := snapshot.AddNode(node)
					assert.NoError(t, err)

					err = snapshot.Fork()
					assert.NoError(t, err)

					err = snapshot.RemoveNode("node")
					assert.NoError(t, err)

					// Node deleted after fork - shouldn't be able to operate on it.
					err = op.op(snapshot)
					assert.Error(t, err)

					err = snapshot.Commit()
					assert.NoError(t, err)

					// Node deleted before commit - shouldn't be able to operate on it.
					err = op.op(snapshot)
					assert.Error(t, err)
				})

			t.Run(fmt.Sprintf("%s: %s base", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					node := BuildTestNode("node", 10, 100)
					err := snapshot.AddNode(node)
					assert.NoError(t, err)

					err = snapshot.RemoveNode("node")
					assert.NoError(t, err)

					// Node deleted from base - shouldn't be able to operate on it.
					err = op.op(snapshot)
					assert.Error(t, err)
				})
		}
	}
}
<commit_msg>test adding node error when node already exists<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package simulator

import (
	"fmt"
	"math/rand"
	"testing"
	"time"

	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"

	"github.com/stretchr/testify/assert"
)

var snapshots = map[string]func() ClusterSnapshot{
	"basic": func() ClusterSnapshot { return NewBasicClusterSnapshot() },
	"delta": func() ClusterSnapshot { return NewDeltaClusterSnapshot() },
}

func nodeNames(nodes []*apiv1.Node) []string {
	names := make([]string, len(nodes), len(nodes))
	for i, node := range nodes {
		names[i] = node.Name
	}
	return names
}

func extractNodes(nodeInfos []*schedulernodeinfo.NodeInfo) []*apiv1.Node {
	nodes := []*apiv1.Node{}
	for _, ni := range nodeInfos {
		nodes = append(nodes, ni.Node())
	}
	return nodes
}

type snapshotState struct {
	nodes []*apiv1.Node
	pods  []*apiv1.Pod
}

func compareStates(t *testing.T, a, b snapshotState) {
	assert.ElementsMatch(t, a.nodes, b.nodes)
	assert.ElementsMatch(t, a.pods, b.pods)
}

func getSnapshotState(t *testing.T, snapshot ClusterSnapshot) snapshotState {
	nodes, err := snapshot.NodeInfos().List()
	assert.NoError(t, err)
	pods, err := snapshot.Pods().List(labels.Everything())
	assert.NoError(t, err)
	return snapshotState{extractNodes(nodes), pods}
}

func startSnapshot(t *testing.T, snapshotFactory func() ClusterSnapshot, state snapshotState) ClusterSnapshot {
	snapshot := snapshotFactory()
	err := snapshot.AddNodes(state.nodes)
	assert.NoError(t, err)
	for _, pod := range state.pods {
		err := snapshot.AddPod(pod, pod.Spec.NodeName)
		assert.NoError(t, err)
	}
	return snapshot
}

func TestForking(t *testing.T) {
	node := BuildTestNode("specialNode", 10, 100)
	pod := BuildTestPod("specialPod", 1, 1)
	pod.Spec.NodeName = node.Name

	testCases := []struct {
		name          string
		op            func(ClusterSnapshot)
		state         snapshotState
		modifiedState snapshotState
	}{
		{
			name: "add node",
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.AddNode(node)
				assert.NoError(t, err)
			},
			modifiedState: snapshotState{
				nodes: []*apiv1.Node{node},
			},
		},
		{
			name: "add node with pods",
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.AddNodeWithPods(node, []*apiv1.Pod{pod})
				assert.NoError(t, err)
			},
			modifiedState: snapshotState{
				nodes: []*apiv1.Node{node},
				pods:  []*apiv1.Pod{pod},
			},
		},
		{
			name: "remove node",
			state: snapshotState{
				nodes: []*apiv1.Node{node},
			},
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.RemoveNode(node.Name)
				assert.NoError(t, err)
			},
		},
		{
			name: "remove node, then add it back",
			state: snapshotState{
				nodes: []*apiv1.Node{node},
			},
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.RemoveNode(node.Name)
				assert.NoError(t, err)

				err = snapshot.AddNode(node)
				assert.NoError(t, err)
			},
			modifiedState: snapshotState{
				nodes: []*apiv1.Node{node},
			},
		},
		{
			name: "add pod, then remove node",
			state: snapshotState{
				nodes: []*apiv1.Node{node},
			},
			op: func(snapshot ClusterSnapshot) {
				err := snapshot.AddPod(pod, node.Name)
				assert.NoError(t, err)
				err = snapshot.RemoveNode(node.Name)
				assert.NoError(t, err)
			},
		},
	}

	for name, snapshotFactory := range snapshots {
		for _, tc := range testCases {
			t.Run(fmt.Sprintf("%s: %s base", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)
				tc.op(snapshot)

				// Modifications should be applied.
				compareStates(t, tc.modifiedState, getSnapshotState(t, snapshot))
			})
			t.Run(fmt.Sprintf("%s: %s fork", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)

				err := snapshot.Fork()
				assert.NoError(t, err)

				tc.op(snapshot)

				// Modifications should be applied.
				compareStates(t, tc.modifiedState, getSnapshotState(t, snapshot))
			})
			t.Run(fmt.Sprintf("%s: %s fork & revert", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)

				err := snapshot.Fork()
				assert.NoError(t, err)

				tc.op(snapshot)

				err = snapshot.Revert()
				assert.NoError(t, err)

				// Modifications should no longer be applied.
				compareStates(t, tc.state, getSnapshotState(t, snapshot))
			})
			t.Run(fmt.Sprintf("%s: %s fork & commit", name, tc.name), func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, tc.state)

				err := snapshot.Fork()
				assert.NoError(t, err)

				tc.op(snapshot)

				err = snapshot.Commit()
				assert.NoError(t, err)

				// Modifications should be applied.
				compareStates(t, tc.modifiedState, getSnapshotState(t, snapshot))
			})
		}
	}
}

func TestClear(t *testing.T) {
	// Run with -count=1 to avoid caching.
	localRand := rand.New(rand.NewSource(time.Now().Unix()))

	nodeCount := localRand.Intn(100)
	podCount := localRand.Intn(1000)
	extraNodeCount := localRand.Intn(100)
	extraPodCount := localRand.Intn(1000)

	nodes := createTestNodes(nodeCount)
	pods := createTestPods(podCount)
	assignPodsToNodes(pods, nodes)

	state := snapshotState{nodes, pods}

	extraNodes := createTestNodesWithPrefix("extra", extraNodeCount)

	allNodes := make([]*apiv1.Node, len(nodes)+len(extraNodes), len(nodes)+len(extraNodes))
	copy(allNodes, nodes)
	copy(allNodes[len(nodes):], extraNodes)

	extraPods := createTestPodsWithPrefix("extra", extraPodCount)
	assignPodsToNodes(extraPods, allNodes)

	allPods := make([]*apiv1.Pod, len(pods)+len(extraPods), len(pods)+len(extraPods))
	copy(allPods, pods)
	copy(allPods[len(pods):], extraPods)

	for name, snapshotFactory := range snapshots {
		t.Run(fmt.Sprintf("%s: clear base %d nodes %d pods", name, nodeCount, podCount),
			func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, state)
				compareStates(t, state, getSnapshotState(t, snapshot))

				snapshot.Clear()

				compareStates(t, snapshotState{}, getSnapshotState(t, snapshot))
			})
		t.Run(fmt.Sprintf("%s: clear fork %d nodes %d pods %d extra nodes %d extra pods", name, nodeCount, podCount, extraNodeCount, extraPodCount),
			func(t *testing.T) {
				snapshot := startSnapshot(t, snapshotFactory, state)
				compareStates(t, state, getSnapshotState(t, snapshot))

				err := snapshot.Fork()
				assert.NoError(t, err)

				err = snapshot.AddNodes(extraNodes)
				assert.NoError(t, err)

				for _, pod := range extraPods {
					err := snapshot.AddPod(pod, pod.Spec.NodeName)
					assert.NoError(t, err)
				}

				compareStates(t, snapshotState{allNodes, allPods}, getSnapshotState(t, snapshot))

				// Fork()ing twice is not allowed.
				err = snapshot.Fork()
				assert.Error(t, err)

				snapshot.Clear()

				compareStates(t, snapshotState{}, getSnapshotState(t, snapshot))

				// Clear() should break out of forked state.
				err = snapshot.Fork()
				assert.NoError(t, err)
			})
	}
}

func TestNode404(t *testing.T) {
	// Anything and everything that returns errNodeNotFound should be tested here.
	ops := []struct {
		name string
		op   func(ClusterSnapshot) error
	}{
		{"add pod", func(snapshot ClusterSnapshot) error {
			return snapshot.AddPod(BuildTestPod("p1", 0, 0), "node")
		}},
		{"remove pod", func(snapshot ClusterSnapshot) error {
			return snapshot.RemovePod("default", "p1", "node")
		}},
		{"get node", func(snapshot ClusterSnapshot) error {
			_, err := snapshot.NodeInfos().Get("node")
			return err
		}},
		{"remove node", func(snapshot ClusterSnapshot) error {
			return snapshot.RemoveNode("node")
		}},
	}

	for name, snapshotFactory := range snapshots {
		for _, op := range ops {
			t.Run(fmt.Sprintf("%s: %s empty", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					// Empty snapshot - shouldn't be able to operate on nodes that are not here.
					err := op.op(snapshot)
					assert.Error(t, err)
				})

			t.Run(fmt.Sprintf("%s: %s fork", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					node := BuildTestNode("node", 10, 100)
					err := snapshot.AddNode(node)
					assert.NoError(t, err)

					err = snapshot.Fork()
					assert.NoError(t, err)

					err = snapshot.RemoveNode("node")
					assert.NoError(t, err)

					// Node deleted after fork - shouldn't be able to operate on it.
					err = op.op(snapshot)
					assert.Error(t, err)

					err = snapshot.Commit()
					assert.NoError(t, err)

					// Node deleted before commit - shouldn't be able to operate on it.
					err = op.op(snapshot)
					assert.Error(t, err)
				})

			t.Run(fmt.Sprintf("%s: %s base", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					node := BuildTestNode("node", 10, 100)
					err := snapshot.AddNode(node)
					assert.NoError(t, err)

					err = snapshot.RemoveNode("node")
					assert.NoError(t, err)

					// Node deleted from base - shouldn't be able to operate on it.
					err = op.op(snapshot)
					assert.Error(t, err)
				})
		}
	}
}

func TestNodeAlreadyExists(t *testing.T) {
	node := BuildTestNode("node", 10, 100)
	pod := BuildTestPod("pod", 1, 1)
	pod.Spec.NodeName = node.Name

	ops := []struct {
		name string
		op   func(ClusterSnapshot) error
	}{
		{"add node", func(snapshot ClusterSnapshot) error {
			return snapshot.AddNode(node)
		}},
		{"add node with pod", func(snapshot ClusterSnapshot) error {
			return snapshot.AddNodeWithPods(node, []*apiv1.Pod{pod})
		}},
	}

	for name, snapshotFactory := range snapshots {
		for _, op := range ops {
			t.Run(fmt.Sprintf("%s: %s base", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					err := snapshot.AddNode(node)
					assert.NoError(t, err)

					// Node already in base.
					err = op.op(snapshot)
					assert.Error(t, err)
				})

			t.Run(fmt.Sprintf("%s: %s base, forked", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					err := snapshot.AddNode(node)
					assert.NoError(t, err)

					err = snapshot.Fork()
					assert.NoError(t, err)

					// Node already in base, shouldn't be able to add in fork.
					err = op.op(snapshot)
					assert.Error(t, err)
				})

			t.Run(fmt.Sprintf("%s: %s fork", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					err := snapshot.Fork()
					assert.NoError(t, err)

					err = snapshot.AddNode(node)
					assert.NoError(t, err)

					// Node already in fork.
					err = op.op(snapshot)
					assert.Error(t, err)
				})
			t.Run(fmt.Sprintf("%s: %s committed", name, op.name),
				func(t *testing.T) {
					snapshot := snapshotFactory()

					err := snapshot.Fork()
					assert.NoError(t, err)

					err = snapshot.AddNode(node)
					assert.NoError(t, err)

					err = snapshot.Commit()
					assert.NoError(t, err)

					// Node already in new base.
					err = op.op(snapshot)
					assert.Error(t, err)
				})
		}
	}
}
<|endoftext|>


<commit_before>package xkcd_test

import (
	"encoding/json"
	"github.com/rkoesters/xkcd"
	"io"
	"reflect"
	"testing"
	"unicode/utf8"
)

func TestNew(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	r, w := io.Pipe()

	go func() {
		e := json.NewEncoder(w)
		err = e.Encode(comic1)
		if err != nil {
			t.Fatal(err)
		}
	}()

	comic2, err := xkcd.New(r)
	if err != nil {
		t.Fatal(err)
	}

	t.Log("comic1: ", comic1)
	t.Log("comic2: ", comic2)

	if !comicValidUtf8(comic1) {
		t.Errorf("%+q isn't valid utf-8", comic1)
	}

	if !comicValidUtf8(comic2) {
		t.Errorf("%+q isn't valid utf-8", comic2)
	}

	if !reflect.DeepEqual(comic1, comic2) {
		t.Fatal("comic1 and comic2 don't match")
	}
}

func TestGetCurrent(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	comic2, err := xkcd.Get(comic1.Num)
	if err != nil {
		t.Fatal(err)
	}

	t.Log("comic1: ", comic1)
	t.Log("comic2: ", comic2)

	if !comicValidUtf8(comic1) {
		t.Errorf("%+q isn't valid utf-8", comic1)
	}

	if !comicValidUtf8(comic2) {
		t.Errorf("%+q isn't valid utf-8", comic2)
	}

	if !reflect.DeepEqual(comic1, comic2) {
		t.Fatal("comic1 and comic2 don't match")
	}
}

func TestGet4(t *testing.T) {
	expect := &xkcd.Comic{
		Num:       221,
		Title:     "Random Number",
		SafeTitle: "Random Number",
		Img:       "https://imgs.xkcd.com/comics/random_number.png",
		Alt:       "RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.",
		Year:      "2007",
		Month:     "2",
		Day:       "9",
		News:      "",
		Link:      "",
		Transcript: `int getRandomNumber()
{
  return 4;
 chosen by fair dice roll.

 guarenteed to be random.
}
{{title text: RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.}}`,
	}

	testGet(t, expect)
}

func TestGet404(t *testing.T) {
	_, err := xkcd.Get(404)
	if err != xkcd.ErrNotFound {
		t.Fatal(err)
	}
}

func TestGet1953(t *testing.T) {
	expect := &xkcd.Comic{
		Num:        1953,
		Title:      "The History of Unicode",
		SafeTitle:  "The History of Unicode",
		Img:        "https://imgs.xkcd.com/comics/the_history_of_unicode.png",
		Alt:        "2048: \"Great news for Maine—we're once again an independent state!!! Thanks, @unicode, for ruling in our favor and sending troops to end New Hampshire's annexation. 🙏🚁🎖\"",
		Year:       "2018",
		Month:      "2",
		Day:        "9",
		News:       "",
		Link:       "",
		Transcript: "",
	}

	testGet(t, expect)
}

func TestGet1956(t *testing.T) {
	expect := &xkcd.Comic{
		Num:        1956,
		Title:      "Unification",
		SafeTitle:  "Unification",
		Img:        "https://imgs.xkcd.com/comics/unification.png",
		Alt:        "For a while, some physicists worked on a theory unifying the other forces with both the force of gravity and the film \"Gravity,\" but even after Alfonso Cuarón was held in a deep underground chamber of water for 10^31 years he refused to sell his film to Disney.",
		Year:       "2018",
		Month:      "2",
		Day:        "16",
		News:       "",
		Link:       "",
		Transcript: "",
	}

	testGet(t, expect)
}

func TestGet2038(t *testing.T) {
	expect := &xkcd.Comic{
		Num:        2038,
		Title:      "Hazard Symbol",
		SafeTitle:  "Hazard Symbol",
		Img:        "https://imgs.xkcd.com/comics/hazard_symbol.png",
		Alt:        "The warning diamond on the Materials Safety Data Sheet for this stuff just has the \"😰\" emoji in all four fields.",
		Year:       "2018",
		Month:      "8",
		Day:        "27",
		News:       "",
		Link:       "",
		Transcript: "",
	}

	testGet(t, expect)
}

func testGet(t *testing.T, expect *xkcd.Comic) {
	actual, err := xkcd.Get(expect.Num)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("actual=%v", actual)
	t.Logf("expect=%v", expect)

	if !comicValidUtf8(actual) {
		t.Errorf("%+q isn't valid utf-8", actual)
	}

	if !reflect.DeepEqual(actual, expect) {
		t.Fail()
	}
}

func comicValidUtf8(c *xkcd.Comic) bool {
	return utf8.ValidString(c.Title) &&
		utf8.ValidString(c.SafeTitle) &&
		utf8.ValidString(c.Img) &&
		utf8.ValidString(c.Alt) &&
		utf8.ValidString(c.Year) &&
		utf8.ValidString(c.Month) &&
		utf8.ValidString(c.Day) &&
		utf8.ValidString(c.News) &&
		utf8.ValidString(c.Link) &&
		utf8.ValidString(c.Transcript)
}
<commit_msg>Further refractor xkcd_test.go.<commit_after>package xkcd_test

import (
	"encoding/json"
	"github.com/rkoesters/xkcd"
	"io"
	"reflect"
	"testing"
	"unicode/utf8"
)

func TestNew(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	r, w := io.Pipe()

	go func() {
		e := json.NewEncoder(w)
		err = e.Encode(comic1)
		if err != nil {
			t.Fatal(err)
		}
	}()

	comic2, err := xkcd.New(r)
	if err != nil {
		t.Fatal(err)
	}

	t.Log("comic1: ", comic1)
	t.Log("comic2: ", comic2)

	if !comicValidUtf8(comic1) {
		t.Errorf("%+q isn't valid utf-8", comic1)
	}

	if !comicValidUtf8(comic2) {
		t.Errorf("%+q isn't valid utf-8", comic2)
	}

	if !reflect.DeepEqual(comic1, comic2) {
		t.Fatal("comic1 and comic2 don't match")
	}
}

func TestGetCurrent(t *testing.T) {
	comic1, err := xkcd.GetCurrent()
	if err != nil {
		t.Fatal(err)
	}

	comic2, err := xkcd.Get(comic1.Num)
	if err != nil {
		t.Fatal(err)
	}

	t.Log("comic1: ", comic1)
	t.Log("comic2: ", comic2)

	if !comicValidUtf8(comic1) {
		t.Errorf("%+q isn't valid utf-8", comic1)
	}

	if !comicValidUtf8(comic2) {
		t.Errorf("%+q isn't valid utf-8", comic2)
	}

	if !reflect.DeepEqual(comic1, comic2) {
		t.Fatal("comic1 and comic2 don't match")
	}
}

func TestGet4(t *testing.T) {
	testGet(t, &xkcd.Comic{
		Num:       221,
		Title:     "Random Number",
		SafeTitle: "Random Number",
		Img:       "https://imgs.xkcd.com/comics/random_number.png",
		Alt:       "RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.",
		Year:      "2007",
		Month:     "2",
		Day:       "9",
		News:      "",
		Link:      "",
		Transcript: `int getRandomNumber()
{
  return 4;
 chosen by fair dice roll.

 guarenteed to be random.
}
{{title text: RFC 1149.5 specifies 4 as the standard IEEE-vetted random number.}}`,
	})
}

func TestGet404(t *testing.T) {
	_, err := xkcd.Get(404)
	if err != xkcd.ErrNotFound {
		t.Fatal(err)
	}
}

func TestGet1953(t *testing.T) {
	testGet(t, &xkcd.Comic{
		Num:        1953,
		Title:      "The History of Unicode",
		SafeTitle:  "The History of Unicode",
		Img:        "https://imgs.xkcd.com/comics/the_history_of_unicode.png",
		Alt:        "2048: \"Great news for Maine—we're once again an independent state!!! Thanks, @unicode, for ruling in our favor and sending troops to end New Hampshire's annexation. 🙏🚁🎖\"",
		Year:       "2018",
		Month:      "2",
		Day:        "9",
		News:       "",
		Link:       "",
		Transcript: "",
	})
}

func TestGet1956(t *testing.T) {
	testGet(t, &xkcd.Comic{
		Num:        1956,
		Title:      "Unification",
		SafeTitle:  "Unification",
		Img:        "https://imgs.xkcd.com/comics/unification.png",
		Alt:        "For a while, some physicists worked on a theory unifying the other forces with both the force of gravity and the film \"Gravity,\" but even after Alfonso Cuarón was held in a deep underground chamber of water for 10^31 years he refused to sell his film to Disney.",
		Year:       "2018",
		Month:      "2",
		Day:        "16",
		News:       "",
		Link:       "",
		Transcript: "",
	})
}

func TestGet2038(t *testing.T) {
	testGet(t, &xkcd.Comic{
		Num:        2038,
		Title:      "Hazard Symbol",
		SafeTitle:  "Hazard Symbol",
		Img:        "https://imgs.xkcd.com/comics/hazard_symbol.png",
		Alt:        "The warning diamond on the Materials Safety Data Sheet for this stuff just has the \"😰\" emoji in all four fields.",
		Year:       "2018",
		Month:      "8",
		Day:        "27",
		News:       "",
		Link:       "",
		Transcript: "",
	})
}

func testGet(t *testing.T, expect *xkcd.Comic) {
	actual, err := xkcd.Get(expect.Num)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("actual=%v", actual)
	t.Logf("expect=%v", expect)

	if !comicValidUtf8(actual) {
		t.Errorf("%+q isn't valid utf-8", actual)
	}

	if !reflect.DeepEqual(actual, expect) {
		t.Fail()
	}
}

func comicValidUtf8(c *xkcd.Comic) bool {
	return utf8.ValidString(c.Title) &&
		utf8.ValidString(c.SafeTitle) &&
		utf8.ValidString(c.Img) &&
		utf8.ValidString(c.Alt) &&
		utf8.ValidString(c.Year) &&
		utf8.ValidString(c.Month) &&
		utf8.ValidString(c.Day) &&
		utf8.ValidString(c.News) &&
		utf8.ValidString(c.Link) &&
		utf8.ValidString(c.Transcript)
}
<|endoftext|>


<commit_before>// Converts a string in an arbitrary base to any other arbitrary base.
package baseconv

import (
	"errors"
	"fmt"
	"unicode/utf8"
)

// Convert num from specified base to a different base.
func Convert(num, fromBase, toBase string) (string, error) {
	if num == "" {
		return "", errors.New("invalid number")
	}

	if len(fromBase) < 2 {
		return "", errors.New("invalid fromBase")
	}

	if len(toBase) < 2 {
		return "", errors.New("invalid toBase")
	}

	// rune counts
	fromLenRunes := utf8.RuneCountInString(fromBase)
	toLenRunes := utf8.RuneCountInString(toBase)
	numLen := utf8.RuneCountInString(num)

	// loop over unicode runes in original string and store representative
	// values in number -- number[i] = index(num[i], fromBase)
	number, ipos := make([]int, numLen), 0
	for i, r := range num {
		jpos, found := 0, false
		for _, s := range fromBase {
			if r == s {
				number[ipos] = jpos
				found = true
				break
			}

			jpos++
		}

		// if character wasn't found in fromBase, then error
		if !found {
			return "", fmt.Errorf("invalid character '%c' at position %d (%d)", r, ipos, i)
		}

		ipos++
	}

	// split the runes in toBase
	todigits, idx := make([]rune, toLenRunes), 0
	for _, r := range toBase {
		todigits[idx] = r
		idx++
	}

	// loop until whole number is converted
	result := make([]rune, 0)
	for {
		divide, newlen := 0, 0

		// perform division manually (which is why this works with big numbers)
		for i := 0; i < numLen; i++ {
			divide = divide*fromLenRunes + number[i]
			if divide >= toLenRunes {
				number[newlen] = int(divide / toLenRunes)
				divide = divide % toLenRunes
				newlen++
			} else if newlen > 0 {
				number[newlen] = 0
				newlen++
			}
		}

		numLen = newlen
		result = append(result, todigits[divide])

		if newlen == 0 {
			break
		}
	}

	// reverse result
	for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
		result[i], result[j] = result[j], result[i]
	}

	return string(result), nil
}

const (
	DigitsBin = "01"
	DigitsOct = "01234567"
	DigitsDec = "0123456789"
	DigitsHex = "0123456789abcdef"
	Digits36  = "0123456789abcdefghijklmnopqrstuvwxyz"
	Digits62  = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	Digits64  = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_"
)

// Encode a string into DigitsBin with optional specified base (default: DigitsDec).
func EncodeBin(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, DigitsBin)
}

// Decode a string from DigitsBin with optional specified base (default: DigitsDec).
func DecodeBin(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, DigitsBin, to)
}

// Encode a string into DigitsOct with optional specified base (default: DigitsDec).
func EncodeOct(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, DigitsOct)
}

// Decode a string from DigitsOct with optional specified base (default: DigitsDec).
func DecodeOct(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, DigitsOct, to)
}

// Encode a string into DigitsHex with optional specified base (default: DigitsDec).
func EncodeHex(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, DigitsHex)
}

// Decode a string from DigitsHex with optional specified base (default: DigitsDec).
func DecodeHex(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, DigitsHex, to)
}

// Encode a string into Digits36 with optional specified base (default: DigitsDec).
func Encode36(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, Digits36)
}

// Decode a string from Digits36 with optional specified base (default: DigitsDec).
func Decode36(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, Digits36, to)
}

// Encode a string into Digits62 with optional specified base (default: DigitsDec).
func Encode62(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, Digits62)
}

// Decode a string from Digits62 with optional specified base (default: DigitsDec).
func Decode62(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, Digits62, to)
}

// Encode a string into Digits64 with optional specified base (default: DigitsDec).
func Encode64(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, Digits64)
}

// Decode a string from Digits64 with optional specified base (default: DigitsDec).
func Decode64(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, Digits64, to)
}
<commit_msg>Fixing errors from golint tool<commit_after>// Package baseconv converts a string in an arbitrary base to any other
// arbitrary base.
package baseconv

import (
	"errors"
	"fmt"
	"unicode/utf8"
)

// Convert num from specified base to a different base.
func Convert(num, fromBase, toBase string) (string, error) {
	if num == "" {
		return "", errors.New("invalid number")
	}

	if len(fromBase) < 2 {
		return "", errors.New("invalid fromBase")
	}

	if len(toBase) < 2 {
		return "", errors.New("invalid toBase")
	}

	// rune counts
	fromLenRunes := utf8.RuneCountInString(fromBase)
	toLenRunes := utf8.RuneCountInString(toBase)
	numLen := utf8.RuneCountInString(num)

	// loop over unicode runes in original string and store representative
	// values in number -- number[i] = index(num[i], fromBase)
	number, ipos := make([]int, numLen), 0
	for i, r := range num {
		jpos, found := 0, false
		for _, s := range fromBase {
			if r == s {
				number[ipos] = jpos
				found = true
				break
			}

			jpos++
		}

		// if character wasn't found in fromBase, then error
		if !found {
			return "", fmt.Errorf("invalid character '%c' at position %d (%d)", r, ipos, i)
		}

		ipos++
	}

	// split the runes in toBase
	todigits, idx := make([]rune, toLenRunes), 0
	for _, r := range toBase {
		todigits[idx] = r
		idx++
	}

	// loop until whole number is converted
	var result []rune
	for {
		divide, newlen := 0, 0

		// perform division manually (which is why this works with big numbers)
		for i := 0; i < numLen; i++ {
			divide = divide*fromLenRunes + number[i]
			if divide >= toLenRunes {
				number[newlen] = int(divide / toLenRunes)
				divide = divide % toLenRunes
				newlen++
			} else if newlen > 0 {
				number[newlen] = 0
				newlen++
			}
		}

		numLen = newlen
		result = append(result, todigits[divide])

		if newlen == 0 {
			break
		}
	}

	// reverse result
	for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
		result[i], result[j] = result[j], result[i]
	}

	return string(result), nil
}

const (
	// DigitsBin represents binary digits
	DigitsBin = "01"

	// DigitsOct represents octal Digits
	DigitsOct = "01234567"

	// DigitsDec represents decimal digits
	DigitsDec = "0123456789"

	// DigitsHex represents hex digits
	DigitsHex = "0123456789abcdef"

	// Digits36 represents base36 digits
	Digits36 = "0123456789abcdefghijklmnopqrstuvwxyz"

	// Digits62 represents base62 digits
	Digits62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

	// Digits64 represents base64 digits
	Digits64 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_"
)

// EncodeBin encodes a string into DigitsBin with optional specified base (default: DigitsDec).
func EncodeBin(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, DigitsBin)
}

// DecodeBin decodes a string from DigitsBin with optional specified base (default: DigitsDec).
func DecodeBin(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, DigitsBin, to)
}

// EncodeOct encodes a string into DigitsOct with optional specified base (default: DigitsDec).
func EncodeOct(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, DigitsOct)
}

// DecodeOct decodes a string from DigitsOct with optional specified base (default: DigitsDec).
func DecodeOct(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, DigitsOct, to)
}

// EncodeHex encodes a string into DigitsHex with optional specified base (default: DigitsDec).
func EncodeHex(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, DigitsHex)
}

// DecodeHex decodes a string from DigitsHex with optional specified base (default: DigitsDec).
func DecodeHex(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, DigitsHex, to)
}

// Encode36 encodes a string into Digits36 with optional specified base (default: DigitsDec).
func Encode36(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, Digits36)
}

// Decode36 decodes a string from Digits36 with optional specified base (default: DigitsDec).
func Decode36(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, Digits36, to)
}

// Encode62 encodes a string into Digits62 with optional specified base (default: DigitsDec).
func Encode62(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, Digits62)
}

// Decode62 decodes a string from Digits62 with optional specified base (default: DigitsDec).
func Decode62(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, Digits62, to)
}

// Encode64 encodes a string into Digits64 with optional specified base (default: DigitsDec).
func Encode64(num string, base ...string) (string, error) {
	from := DigitsDec
	if len(base) > 0 {
		from = base[0]
	}

	return Convert(num, from, Digits64)
}

// Decode64 decodes a string from Digits64 with optional specified base (default: DigitsDec).
func Decode64(num string, base ...string) (string, error) {
	to := DigitsDec
	if len(base) > 0 {
		to = base[0]
	}

	return Convert(num, Digits64, to)
}
<|endoftext|>


<commit_before>// +build !jenkins

package brands

import (
	//"fmt"
	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

var validSkeletonBrand = Brand{
	UUID:      "92f4ec09-436d-4092-a88c-96f54e34007d",
	PrefLabel: "validSkeletonBrand",
	Identifiers: []identifier{
		identifier{
			Authority:       tmeAuthority,
			IdentifierValue: "111",
		},
	},
}

var validSimpleBrand = Brand{
	UUID:           "92f4ec09-436d-4092-a88c-96f54e34007c",
	PrefLabel:      "validSimpleBrand",
	Strapline:      "Keeping it simple",
	Description:    "This brand has no parent but otherwise has valid values for all fields",
	DescriptionXML: "This brand has no parent but otherwise has valid values for all fields",
	ImageURL:       "http://media.ft.com/validSimpleBrand.png",
	Identifiers: []identifier{
		identifier{
			Authority:       tmeAuthority,
			IdentifierValue: "123",
		},
	},
}

var validChildBrand = Brand{
	UUID:           "a806e270-edbc-423f-b8db-d21ae90e06c8",
	ParentUUID:     "92f4ec09-436d-4092-a88c-96f54e34007c",
	PrefLabel:      "validChildBrand",
	Strapline:      "My parent is simple",
	Description:    "This brand has a parent and valid values for all fields",
	DescriptionXML: "This brand has a parent and valid values for all fields",
	ImageURL:       "http://media.ft.com/validChildBrand.png",
	Identifiers: []identifier{
		identifier{
			Authority:       tmeAuthority,
			IdentifierValue: "123123",
		},
	},
}

func TestCreateNotAllValuesPresent(t *testing.T) {
	err := getCypherDriver(t).Write(validSkeletonBrand)
	assert.NoError(t, err)
	readBrandAndCompare(validSkeletonBrand, t)
	cleanUp(validSkeletonBrand.UUID, t)
}

func TestDeleteExistingBrand(t *testing.T) {
	driver := getCypherDriver(t)
	err := driver.Write(validSimpleBrand)
	assert.NoError(t, err)

	done, err := getCypherDriver(t).Delete(validSimpleBrand.UUID)
	assert.NoError(t, err)
	assert.True(t, done)

	person, found, err := getCypherDriver(t).Read(validSimpleBrand.UUID)
	assert.NoError(t, err)
	assert.EqualValues(t, Brand{}, person)
	assert.False(t, found)
}

func TestCreateAllValuesPresent(t *testing.T) {
	err := getCypherDriver(t).Write(validChildBrand)
	assert.NoError(t, err)
	readBrandAndCompare(validChildBrand, t)
	cleanUp(validChildBrand.UUID, t)
}

func TestCreateHandlesSpecialCharacters(t *testing.T) {
	specialCharBrand := Brand{
		UUID:        "327af339-39d4-4c7b-8c06-9f80211ea93d",
		PrefLabel:   "specialCharBrand",
		Description: "This brand has a heart \u2665 and smiley \u263A",
		Identifiers: []identifier{
			identifier{
				Authority:       tmeAuthority,
				IdentifierValue: "1111",
			},
		},
	}
	err := getCypherDriver(t).Write(specialCharBrand)
	assert.NoError(t, err)
	readBrandAndCompare(specialCharBrand, t)
	cleanUp(specialCharBrand.UUID, t)
}

func TestUpdateWillRemovePropertiesNoLongerPresent(t *testing.T) {
	err := getCypherDriver(t).Write(validSimpleBrand)
	assert.NoError(t, err)
	err = getCypherDriver(t).Write(validSkeletonBrand)
	assert.NoError(t, err)
	readBrandAndCompare(validSkeletonBrand, t)
	cleanUp(validSkeletonBrand.UUID, t)
}

func TestConnectivityCheck(t *testing.T) {
	driver := getCypherDriver(t)
	err := driver.Check()
	assert.NoError(t, err)
}

func getCypherDriver(t *testing.T) (service baseftrwapp.Service) {
	url := os.Getenv("NEO4J_TEST_URL")
	if url == "" {
		url = "http://localhost:7474/db/data"
	}
	db, err := neoism.Connect(url)
	assert.NoError(t, err, "Error setting up connection to %s", url)
	return NewCypherBrandsService(neoutils.StringerDb{db}, db)
}

func readBrandAndCompare(expected Brand, t *testing.T) {
	//fmt.Printf("Looking for %+v\n", expected.UUID)
	actual, found, err := getCypherDriver(t).Read(expected.UUID)
	assert.NoError(t, err)
	assert.True(t, found)
	//fmt.Printf("Found %+v\n", actual)
	assert.EqualValues(t, expected, actual)
}

func cleanUp(uuid string, t *testing.T) {
	found, err := getCypherDriver(t).Delete(uuid)
	assert.True(t, found, "Didn't manage to delete brand for uuid %s", uuid)
	assert.NoError(t, err, "Error deleting brand for uuid %s", uuid)
}
<commit_msg>Fix TestUpdateWillRemovePropertiesNoLongerPresent case<commit_after>// +build !jenkins

package brands

import (
	//"fmt"
	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

var validSkeletonBrand = Brand{
	UUID:      "92f4ec09-436d-4092-a88c-96f54e34007d",
	PrefLabel: "validSkeletonBrand",
	Identifiers: []identifier{
		identifier{
			Authority:       tmeAuthority,
			IdentifierValue: "111",
		},
	},
}

var validSimpleBrand = Brand{
	UUID:           "92f4ec09-436d-4092-a88c-96f54e34007c",
	PrefLabel:      "validSimpleBrand",
	Strapline:      "Keeping it simple",
	Description:    "This brand has no parent but otherwise has valid values for all fields",
	DescriptionXML: "This brand has no parent but otherwise has valid values for all fields",
	ImageURL:       "http://media.ft.com/validSimpleBrand.png",
	Identifiers: []identifier{
		identifier{
			Authority:       tmeAuthority,
			IdentifierValue: "123",
		},
	},
}

var validChildBrand = Brand{
	UUID:           "a806e270-edbc-423f-b8db-d21ae90e06c8",
	ParentUUID:     "92f4ec09-436d-4092-a88c-96f54e34007c",
	PrefLabel:      "validChildBrand",
	Strapline:      "My parent is simple",
	Description:    "This brand has a parent and valid values for all fields",
	DescriptionXML: "This brand has a parent and valid values for all fields",
	ImageURL:       "http://media.ft.com/validChildBrand.png",
	Identifiers: []identifier{
		identifier{
			Authority:       tmeAuthority,
			IdentifierValue: "123123",
		},
	},
}

func TestCreateNotAllValuesPresent(t *testing.T) {
	err := getCypherDriver(t).Write(validSkeletonBrand)
	assert.NoError(t, err)
	readBrandAndCompare(validSkeletonBrand, t)
	cleanUp(validSkeletonBrand.UUID, t)
}

func TestDeleteExistingBrand(t *testing.T) {
	driver := getCypherDriver(t)
	err := driver.Write(validSimpleBrand)
	assert.NoError(t, err)

	done, err := getCypherDriver(t).Delete(validSimpleBrand.UUID)
	assert.NoError(t, err)
	assert.True(t, done)

	person, found, err := getCypherDriver(t).Read(validSimpleBrand.UUID)
	assert.NoError(t, err)
	assert.EqualValues(t, Brand{}, person)
	assert.False(t, found)
}

func TestCreateAllValuesPresent(t *testing.T) {
	err := getCypherDriver(t).Write(validChildBrand)
	assert.NoError(t, err)
	readBrandAndCompare(validChildBrand, t)
	cleanUp(validChildBrand.UUID, t)
}

func TestCreateHandlesSpecialCharacters(t *testing.T) {
	specialCharBrand := Brand{
		UUID:        "327af339-39d4-4c7b-8c06-9f80211ea93d",
		PrefLabel:   "specialCharBrand",
		Description: "This brand has a heart \u2665 and smiley \u263A",
		Identifiers: []identifier{
			identifier{
				Authority:       tmeAuthority,
				IdentifierValue: "1111",
			},
		},
	}
	err := getCypherDriver(t).Write(specialCharBrand)
	assert.NoError(t, err)
	readBrandAndCompare(specialCharBrand, t)
	cleanUp(specialCharBrand.UUID, t)
}

func TestUpdateWillRemovePropertiesNoLongerPresent(t *testing.T) {
	myBrand := validSimpleBrand
	err := getCypherDriver(t).Write(myBrand)
	readBrandAndCompare(myBrand, t)
	assert.NoError(t, err)
	myBrand.Description = ""
	err = getCypherDriver(t).Write(myBrand)
	assert.NoError(t, err)
	readBrandAndCompare(myBrand, t)
	cleanUp(myBrand.UUID, t)
}

func TestConnectivityCheck(t *testing.T) {
	driver := getCypherDriver(t)
	err := driver.Check()
	assert.NoError(t, err)
}

func getCypherDriver(t *testing.T) (service baseftrwapp.Service) {
	url := os.Getenv("NEO4J_TEST_URL")
	if url == "" {
		url = "http://localhost:7474/db/data"
	}
	db, err := neoism.Connect(url)
	assert.NoError(t, err, "Error setting up connection to %s", url)
	return NewCypherBrandsService(neoutils.StringerDb{db}, db)
}

func readBrandAndCompare(expected Brand, t *testing.T) {
	//fmt.Printf("Looking for %+v\n", expected.UUID)
	actual, found, err := getCypherDriver(t).Read(expected.UUID)
	assert.NoError(t, err)
	assert.True(t, found)
	//fmt.Printf("Found %+v\n", actual)
	assert.EqualValues(t, expected, actual)
}

func cleanUp(uuid string, t *testing.T) {
	found, err := getCypherDriver(t).Delete(uuid)
	assert.True(t, found, "Didn't manage to delete brand for uuid %s", uuid)
	assert.NoError(t, err, "Error deleting brand for uuid %s", uuid)
}
<|endoftext|>