<commit_before>package yiigo

import (
	"fmt"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

var dbmap map[string]*sqlx.DB

// SQL expression
type expr struct {
	expr string
	args []interface{}
}

type grammar struct {
	table   string
	columns []string
	clauses []string
	binds   []interface{}
}

type MySQL struct {
	Error        error
	LastInsertId int64
	RowsAffected int64

	db      *sqlx.DB
	tx      *sqlx.Tx
	grammar *grammar
	sql     string
}

// initMySQL init db connections
func initMySQL() error {
	dbmap = make(map[string]*sqlx.DB)

	sections := childSections("mysql")

	for _, v := range sections {
		host := v.Key("host").MustString("localhost")
		port := v.Key("post").MustInt(3306)
		username := v.Key("username").MustString("root")
		password := v.Key("password").MustString("")
		database := v.Key("database").MustString("test")
		charset := v.Key("charset").MustString("utf8mb4")
		collection := v.Key("collection").MustString("utf8mb4_general_ci")

		dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&collation=%s&parseTime=True&loc=Local", username, password, host, port, database, charset, collection)

		db, err := sqlx.Open("mysql", dsn)

		if err != nil {
			db.Close()
			return err
		}

		db.SetMaxOpenConns(EnvInt(k, "maxOpenConns", 20))
		db.SetMaxIdleConns(EnvInt(k, "maxIdleConns", 10))

		err = db.Ping()

		if err != nil {
			db.Close()
			return err
		}

		dbmap[v.Name()] = db
	}

	return nil
}

// DB get a db connection
func DB(connection ...string) (*MySQL, error) {
	conn := "default"

	if len(connection) > 0 {
		conn = connection[0]
	}

	db, ok := dbmap[fmt.Sprintf("mysql.%s", conn)]

	if !ok {
		return nil, fmt.Errorf("database %s is not connected", conn)
	}

	return &MySQL{db: db}, nil
}

// Expr build sql expression, eg: yiigo.Expr("price * ? + ?", 2, 100)
func Expr(expression string, args ...interface{}) *expr {
	return &expr{expr: expression, args: args}
}

func (m *MySQL) Table(table string) *MySQL {
	m.grammar.table = table
	return m
}

func (m *MySQL) Select(columns ...string) *MySQL {
	m.grammar.columns = columns
	return m
}

func (m *MySQL) InnerJoin(table string, on string) *MySQL {
	clause := fmt.Sprintf("INNER JOIN %s ON %s", table, on)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) LeftJoin(table string, on string) *MySQL {
	clause := fmt.Sprintf("LEFT JOIN %s ON %s", table, on)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) RightJoin(table string, on string) *MySQL {
	clause := fmt.Sprintf("RIGHT JOIN %s ON %s", table, on)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Where(where string) *MySQL {
	clause := fmt.Sprintf("WHERE %s", where)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) GroupBy(group string) *MySQL {
	clause := fmt.Sprintf("GROUP BY %s", group)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Having(having string) *MySQL {
	clause := fmt.Sprintf("HAVING %s", having)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) OrderBy(order string) *MySQL {
	clause := fmt.Sprintf("ORDER BY %s", order)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Limit(limit int) *MySQL {
	clause := fmt.Sprintf("LIMIT %d", limit)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Offset(offset int) *MySQL {
	join := fmt.Sprintf("OFFSET %d", offset)
	m.grammar.clauses = append(m.grammar.clauses, join)

	return m
}

func (m *MySQL) Insert(data X) {
	defer m.grammar = nil

	columns := []string{}
	placeholders := []string{}
	binds := []interface{}{}

	for k, v := range data {
		columns = append(columns, k)
		placeholders = append(placeholders, "?")
		binds = append(binds, v)
	}

	sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", m.table, strings.Join(columns, ","), strings.Join(placeholders, ","))

	result, err := m.db.Exec(sql, binds...)

	if err != nil {
		m.Error = fmt.Errorf("%v, SQL: %s, Args: %v", err, sql, binds)
		return
	}

	id, err := result.LastInsertId()

	if err != nil {
		return 0, fmt.Errorf("%v, SQL: %s, args: %v", err, sql, binds)
	}

	return id, nil
}

// BuildInsert build insert sql
func BuildInsert(table string, data X) (string, []interface{}) {
	columns := []string{}
	placeholders := []string{}
	binds := []interface{}{}

	for k, v := range data {
		columns = append(columns, k)
		placeholders = append(placeholders, "?")
		binds = append(binds, v)
	}

	sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(columns, ","), strings.Join(placeholders, ","))

	return sql, binds
}

// BuildBatchInsert build batch insert sql
func BuildBatchInsert(table string, columns []string, data X) (string, []interface{}) {
	placeholders := []string{}
	binds := []interface{}{}

	for _, v := range data {
		bindvars := []string{}

		for _, c := range columns {
			binds = append(binds, v[c])
			bindvars = append(bindvars, "?")
		}

		placeholders = append(placeholders, fmt.Sprintf("(%s)", strings.Join(bindvars, ",")))
	}

	sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", table, strings.Join(columns, ","), strings.Join(placeholders, ","))

	return sql, binds
}

// BuildUpdate build update sql
func BuildUpdate(table string, data X, condition ...interface{}) (string, []interface{}) {
	clauses := []string{}
	sets := []string{}
	binds := []interface{}{}

	clauses = append(clauses, fmt.Sprintf("UPDATE %s", table))

	for k, v := range data {
		if expr, ok := v.(*expr); ok {
			sets = append(sets, fmt.Sprintf("%s = %s", k, expr.expr))
			binds = append(binds, expr.args...)
		} else {
			sets = append(sets, fmt.Sprintf("%s = ?", k))
			binds = append(binds, v)
		}
	}

	clauses = append(clauses, fmt.Sprintf("SET %s", strings.Join(sets, ",")))

	length := len(condition)

	if length > 0 {
		clauses = append(clauses, fmt.Sprintf("WHERE %s", condition[0].(string)))
	}

	if length > 1 {
		binds = append(binds, condition[1:]...)
	}

	sql := strings.Join(clauses, " ")

	return sql, binds
}
<commit_msg>update<commit_after>package yiigo

import (
	"fmt"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

var dbmap map[string]*sqlx.DB

// SQL expression
type expr struct {
	expr string
	args []interface{}
}

type grammar struct {
	table   string
	columns []string
	clauses []string
	binds   []interface{}
}

type MySQL struct {
	db      *sqlx.DB
	tx      *sqlx.Tx
	grammar *grammar
	sql     string
}

// initMySQL init db connections
func initMySQL() error {
	dbmap = make(map[string]*sqlx.DB)

	sections := childSections("mysql")

	for _, v := range sections {
		host := v.Key("host").MustString("localhost")
		port := v.Key("post").MustInt(3306)
		username := v.Key("username").MustString("root")
		password := v.Key("password").MustString("")
		database := v.Key("database").MustString("test")
		charset := v.Key("charset").MustString("utf8mb4")
		collection := v.Key("collection").MustString("utf8mb4_general_ci")

		dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&collation=%s&parseTime=True&loc=Local", username, password, host, port, database, charset, collection)

		db, err := sqlx.Open("mysql", dsn)

		if err != nil {
			db.Close()
			return err
		}

		db.SetMaxOpenConns(EnvInt(k, "maxOpenConns", 20))
		db.SetMaxIdleConns(EnvInt(k, "maxIdleConns", 10))

		err = db.Ping()

		if err != nil {
			db.Close()
			return err
		}

		dbmap[v.Name()] = db
	}

	return nil
}

// DB get a db connection
func DB(connection ...string) (*MySQL, error) {
	conn := "default"

	if len(connection) > 0 {
		conn = connection[0]
	}

	db, ok := dbmap[fmt.Sprintf("mysql.%s", conn)]

	if !ok {
		return nil, fmt.Errorf("database %s is not connected", conn)
	}

	return &MySQL{
		db:      db,
		grammar: &grammar{},
	}, nil
}

// Expr build sql expression, eg: yiigo.Expr("price * ? + ?", 2, 100)
func Expr(expression string, args ...interface{}) *expr {
	return &expr{expr: expression, args: args}
}

func (m *MySQL) Table(table string) *MySQL {
	m.grammar.table = table
	return m
}

func (m *MySQL) Select(columns ...string) *MySQL {
	m.grammar.columns = columns
	return m
}

func (m *MySQL) InnerJoin(table string, on string) *MySQL {
	clause := fmt.Sprintf("INNER JOIN %s ON %s", table, on)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) LeftJoin(table string, on string) *MySQL {
	clause := fmt.Sprintf("LEFT JOIN %s ON %s", table, on)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) RightJoin(table string, on string) *MySQL {
	clause := fmt.Sprintf("RIGHT JOIN %s ON %s", table, on)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Where(where string) *MySQL {
	clause := fmt.Sprintf("WHERE %s", where)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) GroupBy(group string) *MySQL {
	clause := fmt.Sprintf("GROUP BY %s", group)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Having(having string) *MySQL {
	clause := fmt.Sprintf("HAVING %s", having)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) OrderBy(order string) *MySQL {
	clause := fmt.Sprintf("ORDER BY %s", order)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Limit(limit int) *MySQL {
	clause := fmt.Sprintf("LIMIT %d", limit)
	m.grammar.clauses = append(m.grammar.clauses, clause)

	return m
}

func (m *MySQL) Offset(offset int) *MySQL {
	join := fmt.Sprintf("OFFSET %d", offset)
	m.grammar.clauses = append(m.grammar.clauses, join)

	return m
}

func (m *MySQL) Insert(data X) {
	defer m.reset(false)

	columns := []string{}
	placeholders := []string{}
	binds := []interface{}{}

	for k, v := range data {
		columns = append(columns, k)
		placeholders = append(placeholders, "?")
		binds = append(binds, v)
	}

	sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", m.table, strings.Join(columns, ","), strings.Join(placeholders, ","))

	result, err := m.db.Exec(sql, binds...)

	if err != nil {
		m.Error = fmt.Errorf("%v, SQL: %s, Args: %v", err, sql, binds)
		return
	}

	id, err := result.LastInsertId()

	if err != nil {
		return 0, fmt.Errorf("%v, SQL: %s, args: %v", err, sql, binds)
	}

	return id, nil
}

// BuildInsert build insert sql
func BuildInsert(table string, data X) (string, []interface{}) {
	columns := []string{}
	placeholders := []string{}
	binds := []interface{}{}

	for k, v := range data {
		columns = append(columns, k)
		placeholders = append(placeholders, "?")
		binds = append(binds, v)
	}

	sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(columns, ","), strings.Join(placeholders, ","))

	return sql, binds
}

// BuildBatchInsert build batch insert sql
func BuildBatchInsert(table string, columns []string, data X) (string, []interface{}) {
	placeholders := []string{}
	binds := []interface{}{}

	for _, v := range data {
		bindvars := []string{}

		for _, c := range columns {
			binds = append(binds, v[c])
			bindvars = append(bindvars, "?")
		}

		placeholders = append(placeholders, fmt.Sprintf("(%s)", strings.Join(bindvars, ",")))
	}

	sql := fmt.Sprintf("INSERT INTO %s (%s) VALUES %s", table, strings.Join(columns, ","), strings.Join(placeholders, ","))

	return sql, binds
}

// BuildUpdate build update sql
func BuildUpdate(table string, data X, condition ...interface{}) (string, []interface{}) {
	clauses := []string{}
	sets := []string{}
	binds := []interface{}{}

	clauses = append(clauses, fmt.Sprintf("UPDATE %s", table))

	for k, v := range data {
		if expr, ok := v.(*expr); ok {
			sets = append(sets, fmt.Sprintf("%s = %s", k, expr.expr))
			binds = append(binds, expr.args...)
		} else {
			sets = append(sets, fmt.Sprintf("%s = ?", k))
			binds = append(binds, v)
		}
	}

	clauses = append(clauses, fmt.Sprintf("SET %s", strings.Join(sets, ",")))

	length := len(condition)

	if length > 0 {
		clauses = append(clauses, fmt.Sprintf("WHERE %s", condition[0].(string)))
	}

	if length > 1 {
		binds = append(binds, condition[1:]...)
	}

	sql := strings.Join(clauses, " ")

	return sql, binds
}

func (m *MySQL) reset(tx bool) {
	if tx {
		m.tx = nil
	}

	m.grammar = &grammar{}
	m.sql = ""
}
<|endoftext|>


<commit_before>package archive

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"
)

type ChangeType int

const (
	ChangeModify = iota
	ChangeAdd
	ChangeDelete
)

type Change struct {
	Path string
	Kind ChangeType
}

func (change *Change) String() string {
	var kind string
	switch change.Kind {
	case ChangeModify:
		kind = "C"
	case ChangeAdd:
		kind = "A"
	case ChangeDelete:
		kind = "D"
	}
	return fmt.Sprintf("%s %s", kind, change.Path)
}

func Changes(layers []string, rw string) ([]Change, error) {
	var changes []Change
	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		path, err = filepath.Rel(rw, path)
		if err != nil {
			return err
		}
		path = filepath.Join("/", path)

		// Skip root
		if path == "/" {
			return nil
		}

		// Skip AUFS metadata
		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
			return err
		}

		change := Change{
			Path: path,
		}

		// Find out what kind of modification happened
		file := filepath.Base(path)
		// If there is a whiteout, then the file was removed
		if strings.HasPrefix(file, ".wh.") {
			originalFile := file[len(".wh."):]
			change.Path = filepath.Join(filepath.Dir(path), originalFile)
			change.Kind = ChangeDelete
		} else {
			// Otherwise, the file was added
			change.Kind = ChangeAdd

			// ...Unless it already existed in a top layer, in which case, it's a modification
			for _, layer := range layers {
				stat, err := os.Stat(filepath.Join(layer, path))
				if err != nil && !os.IsNotExist(err) {
					return err
				}
				if err == nil {
					// The file existed in the top layer, so that's a modification

					// However, if it's a directory, maybe it wasn't actually modified.
					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
					if stat.IsDir() && f.IsDir() {
						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() {
							// Both directories are the same, don't record the change
							return nil
						}
					}
					change.Kind = ChangeModify
					break
				}
			}
		}

		// Record change
		changes = append(changes, change)
		return nil
	})
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	return changes, nil
}

type FileInfo struct {
	parent   *FileInfo
	name     string
	stat     syscall.Stat_t
	children map[string]*FileInfo
}

func (root *FileInfo) LookUp(path string) *FileInfo {
	parent := root
	if path == "/" {
		return root
	}

	pathElements := strings.Split(path, "/")
	for _, elem := range pathElements {
		if elem != "" {
			child := parent.children[elem]
			if child == nil {
				return nil
			}
			parent = child
		}
	}
	return parent
}

func (info *FileInfo) path() string {
	if info.parent == nil {
		return "/"
	}
	return filepath.Join(info.parent.path(), info.name)
}

func (info *FileInfo) isDir() bool {
	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
}

func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
	if oldInfo == nil {
		// add
		change := Change{
			Path: info.path(),
			Kind: ChangeAdd,
		}
		*changes = append(*changes, change)
	}

	// We make a copy so we can modify it to detect additions
	// also, we only recurse on the old dir if the new info is a directory
	// otherwise any previous delete/change is considered recursive
	oldChildren := make(map[string]*FileInfo)
	if oldInfo != nil && info.isDir() {
		for k, v := range oldInfo.children {
			oldChildren[k] = v
		}
	}

	for name, newChild := range info.children {
		oldChild, _ := oldChildren[name]
		if oldChild != nil {
			// change?
			oldStat := &oldChild.stat
			newStat := &newChild.stat
			// Note: We can't compare inode or ctime or blocksize here, because these change
			// when copying a file into a container. However, that is not generally a problem
			// because any content change will change mtime, and any status change should
			// be visible when actually comparing the stat fields. The only time this
			// breaks down is if some code intentionally hides a change by setting
			// back mtime
			if oldStat.Mode != newStat.Mode ||
				oldStat.Uid != newStat.Uid ||
				oldStat.Gid != newStat.Gid ||
				oldStat.Rdev != newStat.Rdev ||
				// Don't look at size for dirs, its not a good measure of change
				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
				getLastModification(oldStat) != getLastModification(newStat) {
				change := Change{
					Path: newChild.path(),
					Kind: ChangeModify,
				}
				*changes = append(*changes, change)
			}

			// Remove from copy so we can detect deletions
			delete(oldChildren, name)
		}

		newChild.addChanges(oldChild, changes)
	}
	for _, oldChild := range oldChildren {
		// delete
		change := Change{
			Path: oldChild.path(),
			Kind: ChangeDelete,
		}
		*changes = append(*changes, change)
	}

}

func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
	var changes []Change

	info.addChanges(oldInfo, &changes)

	return changes
}

func newRootFileInfo() *FileInfo {
	root := &FileInfo{
		name:     "/",
		children: make(map[string]*FileInfo),
	}
	return root
}

func collectFileInfo(sourceDir string) (*FileInfo, error) {
	root := newRootFileInfo()

	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		relPath, err := filepath.Rel(sourceDir, path)
		if err != nil {
			return err
		}
		relPath = filepath.Join("/", relPath)

		if relPath == "/" {
			return nil
		}

		parent := root.LookUp(filepath.Dir(relPath))
		if parent == nil {
			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
		}

		info := &FileInfo{
			name:     filepath.Base(relPath),
			children: make(map[string]*FileInfo),
			parent:   parent,
		}

		if err := syscall.Lstat(path, &info.stat); err != nil {
			return err
		}

		parent.children[info.name] = info

		return nil
	})
	if err != nil {
		return nil, err
	}
	return root, nil
}

// Compare two directories and generate an array of Change objects describing the changes
func ChangesDirs(newDir, oldDir string) ([]Change, error) {
	oldRoot, err := collectFileInfo(oldDir)
	if err != nil {
		return nil, err
	}
	newRoot, err := collectFileInfo(newDir)
	if err != nil {
		return nil, err
	}

	return newRoot.Changes(oldRoot), nil
}

func ChangesSize(newDir string, changes []Change) int64 {
	var size int64
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			file := filepath.Join(newDir, change.Path)
			fileInfo, _ := os.Lstat(file)
			if fileInfo != nil && !fileInfo.IsDir() {
				size += fileInfo.Size()
			}
		}
	}
	return size
}

func ExportChanges(dir string, changes []Change) (Archive, error) {
	files := make([]string, 0)
	deletions := make([]string, 0)
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			files = append(files, change.Path)
		}
		if change.Kind == ChangeDelete {
			base := filepath.Base(change.Path)
			dir := filepath.Dir(change.Path)
			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
		}
	}
	// FIXME: Why do we create whiteout files inside Tar code ?
	return TarFilter(dir, &TarOptions{
		Compression: Uncompressed,
		Includes:    files,
		Recursive:   false,
		CreateFiles: deletions,
	})
}
<commit_msg>Fix change detection when applying tar layers<commit_after>package archive

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

type ChangeType int

const (
	ChangeModify = iota
	ChangeAdd
	ChangeDelete
)

type Change struct {
	Path string
	Kind ChangeType
}

func (change *Change) String() string {
	var kind string
	switch change.Kind {
	case ChangeModify:
		kind = "C"
	case ChangeAdd:
		kind = "A"
	case ChangeDelete:
		kind = "D"
	}
	return fmt.Sprintf("%s %s", kind, change.Path)
}

// Gnu tar and the go tar writer don't have sub-second mtime
// precision, which is problematic when we apply changes via tar
// files, we handle this by comparing for exact times, *or* same
// second count and either a or b having exactly 0 nanoseconds
func sameFsTime(a, b time.Time) bool {
	return a == b ||
		(a.Unix() == b.Unix() &&
			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
}

func sameFsTimeSpec(a, b syscall.Timespec) bool {
	return a.Sec == b.Sec &&
		(a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
}

func Changes(layers []string, rw string) ([]Change, error) {
	var changes []Change
	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		path, err = filepath.Rel(rw, path)
		if err != nil {
			return err
		}
		path = filepath.Join("/", path)

		// Skip root
		if path == "/" {
			return nil
		}

		// Skip AUFS metadata
		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
			return err
		}

		change := Change{
			Path: path,
		}

		// Find out what kind of modification happened
		file := filepath.Base(path)
		// If there is a whiteout, then the file was removed
		if strings.HasPrefix(file, ".wh.") {
			originalFile := file[len(".wh."):]
			change.Path = filepath.Join(filepath.Dir(path), originalFile)
			change.Kind = ChangeDelete
		} else {
			// Otherwise, the file was added
			change.Kind = ChangeAdd

			// ...Unless it already existed in a top layer, in which case, it's a modification
			for _, layer := range layers {
				stat, err := os.Stat(filepath.Join(layer, path))
				if err != nil && !os.IsNotExist(err) {
					return err
				}
				if err == nil {
					// The file existed in the top layer, so that's a modification

					// However, if it's a directory, maybe it wasn't actually modified.
					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
					if stat.IsDir() && f.IsDir() {
						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
							// Both directories are the same, don't record the change
							return nil
						}
					}
					change.Kind = ChangeModify
					break
				}
			}
		}

		// Record change
		changes = append(changes, change)
		return nil
	})
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	return changes, nil
}

type FileInfo struct {
	parent   *FileInfo
	name     string
	stat     syscall.Stat_t
	children map[string]*FileInfo
}

func (root *FileInfo) LookUp(path string) *FileInfo {
	parent := root
	if path == "/" {
		return root
	}

	pathElements := strings.Split(path, "/")
	for _, elem := range pathElements {
		if elem != "" {
			child := parent.children[elem]
			if child == nil {
				return nil
			}
			parent = child
		}
	}
	return parent
}

func (info *FileInfo) path() string {
	if info.parent == nil {
		return "/"
	}
	return filepath.Join(info.parent.path(), info.name)
}

func (info *FileInfo) isDir() bool {
	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
}

func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
	if oldInfo == nil {
		// add
		change := Change{
			Path: info.path(),
			Kind: ChangeAdd,
		}
		*changes = append(*changes, change)
	}

	// We make a copy so we can modify it to detect additions
	// also, we only recurse on the old dir if the new info is a directory
	// otherwise any previous delete/change is considered recursive
	oldChildren := make(map[string]*FileInfo)
	if oldInfo != nil && info.isDir() {
		for k, v := range oldInfo.children {
			oldChildren[k] = v
		}
	}

	for name, newChild := range info.children {
		oldChild, _ := oldChildren[name]
		if oldChild != nil {
			// change?
			oldStat := &oldChild.stat
			newStat := &newChild.stat
			// Note: We can't compare inode or ctime or blocksize here, because these change
			// when copying a file into a container. However, that is not generally a problem
			// because any content change will change mtime, and any status change should
			// be visible when actually comparing the stat fields. The only time this
			// breaks down is if some code intentionally hides a change by setting
			// back mtime
			if oldStat.Mode != newStat.Mode ||
				oldStat.Uid != newStat.Uid ||
				oldStat.Gid != newStat.Gid ||
				oldStat.Rdev != newStat.Rdev ||
				// Don't look at size for dirs, its not a good measure of change
				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
				!sameFsTimeSpec(getLastModification(oldStat), getLastModification(newStat)) {
				change := Change{
					Path: newChild.path(),
					Kind: ChangeModify,
				}
				*changes = append(*changes, change)
			}

			// Remove from copy so we can detect deletions
			delete(oldChildren, name)
		}

		newChild.addChanges(oldChild, changes)
	}
	for _, oldChild := range oldChildren {
		// delete
		change := Change{
			Path: oldChild.path(),
			Kind: ChangeDelete,
		}
		*changes = append(*changes, change)
	}

}

func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
	var changes []Change

	info.addChanges(oldInfo, &changes)

	return changes
}

func newRootFileInfo() *FileInfo {
	root := &FileInfo{
		name:     "/",
		children: make(map[string]*FileInfo),
	}
	return root
}

func collectFileInfo(sourceDir string) (*FileInfo, error) {
	root := newRootFileInfo()

	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		relPath, err := filepath.Rel(sourceDir, path)
		if err != nil {
			return err
		}
		relPath = filepath.Join("/", relPath)

		if relPath == "/" {
			return nil
		}

		parent := root.LookUp(filepath.Dir(relPath))
		if parent == nil {
			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
		}

		info := &FileInfo{
			name:     filepath.Base(relPath),
			children: make(map[string]*FileInfo),
			parent:   parent,
		}

		if err := syscall.Lstat(path, &info.stat); err != nil {
			return err
		}

		parent.children[info.name] = info

		return nil
	})
	if err != nil {
		return nil, err
	}
	return root, nil
}

// Compare two directories and generate an array of Change objects describing the changes
func ChangesDirs(newDir, oldDir string) ([]Change, error) {
	oldRoot, err := collectFileInfo(oldDir)
	if err != nil {
		return nil, err
	}
	newRoot, err := collectFileInfo(newDir)
	if err != nil {
		return nil, err
	}

	return newRoot.Changes(oldRoot), nil
}

func ChangesSize(newDir string, changes []Change) int64 {
	var size int64
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			file := filepath.Join(newDir, change.Path)
			fileInfo, _ := os.Lstat(file)
			if fileInfo != nil && !fileInfo.IsDir() {
				size += fileInfo.Size()
			}
		}
	}
	return size
}

func ExportChanges(dir string, changes []Change) (Archive, error) {
	files := make([]string, 0)
	deletions := make([]string, 0)
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			files = append(files, change.Path)
		}
		if change.Kind == ChangeDelete {
			base := filepath.Base(change.Path)
			dir := filepath.Dir(change.Path)
			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
		}
	}
	// FIXME: Why do we create whiteout files inside Tar code ?
	return TarFilter(dir, &TarOptions{
		Compression: Uncompressed,
		Includes:    files,
		Recursive:   false,
		CreateFiles: deletions,
	})
}
<|endoftext|>


<commit_before>/* ***************************************************************
*
* THIS SECTION IS FOR STRINGS
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectString(args ...[]string) []string {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctString(arg []string) []string {
	tempMap := make(map[string]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}

/* ***************************************************************
*
* THIS SECTION IS FOR uint64's
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectUint64(args ...[]uint64) []uint64 {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctUint64(arg []uint64) []uint64 {
	tempMap := make(map[uint64]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}
<commit_msg>fixed bug in two uint64 functions<commit_after>/* ***************************************************************
*
* THIS SECTION IS FOR STRINGS
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectString(args ...[]string) []string {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctString(arg []string) []string {
	tempMap := make(map[string]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}

/* ***************************************************************
*
* THIS SECTION IS FOR uint64's
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectUint64(args ...[]uint64) []uint64 {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctUint64(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctUint64(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctUint64(arg []uint64) []uint64 {
	tempMap := make(map[uint64]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}
<|endoftext|>


<commit_before>/* ***************************************************************
*
* THIS SECTION IS FOR STRINGS
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectString(args ...[]string) []string {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctString(arg []string) []string {
	tempMap := make(map[string]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}

/* ***************************************************************
*
* THIS SECTION IS FOR uint64's
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectUint64(args ...[]uint64) []uint64 {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctUint64(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectUint64Arr(arr [][]uint64) []uint64 {
	// create a map to count all the instances of the strings
	arrLength := len(arr)
	tempMap := make(map[uint64]int)
	for idx1 := range arr {
		tempArr := DistinctUint64(arr[idx1])
		for idx2 := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx2]]; ok {
				tempMap[tempArr[idx2]]++
			} else {
				tempMap[tempArr[idx2]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctUint64(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctUint64(arg []uint64) []uint64 {
	tempMap := make(map[uint64]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}
<commit_msg>added support for multiDimensional array input<commit_after>/* ***************************************************************
*
* THIS SECTION IS FOR STRINGS
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectString(args ...[]string) []string {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectStringArr(arr [][]string) []string {
	// create a map to count all the instances of the strings
	arrLength := len(arr)
	tempMap := make(map[string]int)
	for idx1 := range arr {
		tempArr := DistinctString(arr[idx1])
		for idx2 := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx2]]; ok {
				tempMap[tempArr[idx2]]++
			} else {
				tempMap[tempArr[idx2]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union([a1, a2]) >> [1 2 4 5 6]
func UnionStringArr(arr [][]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for idx1 := range arr {
		for idx2 := range arr[idx1] {
			tempMap[arr[idx1][idx2]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceString(args ...[]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]int)
	for _, arg := range args {
		tempArr := DistinctString(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference([a1, a2]) >> [5 6]
func DifferenceStringArr(arr [][]string) []string {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[string]int)
	for idx1 := range arr {
		tempArr := DistinctString(arr[idx1])
		for idx2 := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx2]]; ok {
				tempMap[tempArr[idx2]]++
			} else {
				tempMap[tempArr[idx2]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]string, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctString(arg []string) []string {
	tempMap := make(map[string]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]string, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}

/* ***************************************************************
*
* THIS SECTION IS FOR uint64's
*
/* *************************************************************** */

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect(a1, a2) >> [2 4]
func IntersectUint64(args ...[]uint64) []uint64 {
	// create a map to count all the instances of the strings
	arrLength := len(args)
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctUint64(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the intersection of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Intersect([a1, a2]) >> [2 4]
func IntersectUint64Arr(arr [][]uint64) []uint64 {
	// create a map to count all the instances of the strings
	arrLength := len(arr)
	tempMap := make(map[uint64]int)
	for idx1 := range arr {
		tempArr := DistinctUint64(arr[idx1])
		for idx2 := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx2]]; ok {
				tempMap[tempArr[idx2]]++
			} else {
				tempMap[tempArr[idx2]] = 1
			}
		}
	}

	// find the keys equal to the length of the input args
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == arrLength {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union(a1, a2) >> [1 2 4 5 6]
func UnionUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for _, arg := range args {
		for idx := range arg {
			tempMap[arg[idx]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the union of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Union([a1, a2]) >> [1 2 4 5 6]
func UnionUint64Arr(arr [][]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]uint8)

	// write the contents of the arrays as keys to the map. The map values don't matter
	for idx1 := range arr {
		for idx2 := range arr[idx1] {
			tempMap[arr[idx1][idx2]] = 0
		}
	}

	// the map keys are now unique instances of all of the array contents
	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference(a1, a2) >> [5 6]
func DifferenceUint64(args ...[]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]int)
	for _, arg := range args {
		tempArr := DistinctUint64(arg)
		for idx := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx]]; ok {
				tempMap[tempArr[idx]]++
			} else {
				tempMap[tempArr[idx]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// find the difference of two arrays.
// e.g. a1 = [1 2 2 4 6]; a2 = [2 4 5]
// Difference([a1, a2]) >> [5 6]
func DifferenceUint64Arr(arr [][]uint64) []uint64 {
	// create a temporary map to hold the contents of the arrays
	tempMap := make(map[uint64]int)
	for idx1 := range arr {
		tempArr := DistinctUint64(arr[idx1])
		for idx2 := range tempArr {
			// how many times have we encountered this elem?
			if _, ok := tempMap[tempArr[idx2]]; ok {
				tempMap[tempArr[idx2]]++
			} else {
				tempMap[tempArr[idx2]] = 1
			}
		}
	}

	// write the final val of the diffMap to an array and return
	tempArray := make([]uint64, 0)
	for key, val := range tempMap {
		if val == 1 {
			tempArray = append(tempArray, key)
		}
	}

	return tempArray
}

// Remove duplicate values from one array.
// e.g. a1 = [1 2 2 4 6]
// Distinct(a1) >> [1 2 4 6]
func DistinctUint64(arg []uint64) []uint64 {
	tempMap := make(map[uint64]uint8)

	for idx := range arg {
		tempMap[arg[idx]] = 0
	}

	tempArray := make([]uint64, 0)
	for key := range tempMap {
		tempArray = append(tempArray, key)
	}
	return tempArray
}
<|endoftext|>


<commit_before>package forces

import "github.com/hAWKdv/go-gravity/vectors/vectors"

// Drag force
type Drag struct {
	mover *vectors.Mover
	coef  float64
}

// CreateDrag creates a drag force
func CreateDrag(mover *vectors.Mover, coef float64) {
	// Fd = -1/2 * ro * v^2 * A * Cd * v(^)
	// Fd - Drag force vector
	// ro - density of the environment (air/liquid)
	// v - velocity of the object
	// A - frontal area
	// Cd - drag coefficient
	// v(^) - velocity unit vector
	//
	// Simplified: Fd = -1 * m.velocity.mag() ^ 2 * Cd * m.velocity.norm()
	return &Drag{mover}
}

func (d *Drag) GetForce() *vectors.Vector {
	force := mover.GetVelocity().Copy()
	speed := force.Magnitude()
	direction.Normalize()
	direction.Multiply(-1)

	vUnit = mover.GET
}
<commit_msg>Fix drag force<commit_after>package forces

import "github.com/hAWKdv/go-gravity/vectors/vectors"

const defaultDragCoef = 0.02

// Drag force
type Drag struct {
	mover *vectors.Mover
	coef  float64
}

// CreateDrag creates a drag force
func CreateDrag(mover *vectors.Mover, coef float64) *Drag {
	// Fd = -1/2 * ro * v^2 * A * Cd * v(^)
	// Fd - Drag force vector
	// ro - density of the environment (air/liquid)
	// v - velocity of the object
	// A - frontal area
	// Cd - drag coefficient
	// v(^) - velocity unit vector
	//
	// Simplified: Fd = -1 * m.velocity.mag() ^ 2 * Cd * m.velocity.norm()

	var dragC float64

	if coef > 0 {
		dragC = coef
	} else {
		dragC = defaultDragCoef
	}

	return &Drag{mover, dragC}
}

// GetForce returns the force vector of gravity
func (d *Drag) GetForce() *vectors.Vector {
	force := d.mover.GetVelocity().Copy()
	speed := force.Magnitude()

	force.Normalize()
	force.Multiply(-1)
	force.Multiply(d.coef)
	force.Multiply(speed * speed)

	return force
}
<|endoftext|>


<commit_before><commit_msg>Remove KV entry for level 0 cache when leaf is deleted<commit_after><|endoftext|>


<commit_before><commit_msg>Defined save.List.<commit_after><|endoftext|>


<commit_before><commit_msg>add a PANIC command (enabled via an environment variable)<commit_after><|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package packet

import "errors"

type MarketSide byte

const (
	MarketSideUnknown MarketSide = 0
	MarketSideBid     MarketSide = 'B'
	MarketSideAsk     MarketSide = 'A'
)

var MarketSideUnknownError = errors.New("MarketSide unknown")

func (ms MarketSide) String() string {
	switch ms {
	case MarketSideBid:
		return "B"
	case MarketSideAsk:
		return "A"
	default:
		return "?"
	}
}
func (ms MarketSide) ToByte() (byte, error) {
	switch ms {
	case MarketSideBid:
		return 'B', nil
	case MarketSideAsk:
		return 'S', nil
	default:
		return 0, MarketSideUnknownError
	}
}
func MarketSideFromByte(b byte) MarketSide {
	switch b {
	case 'B':
		return MarketSideBid
	case 'A', 'S':
		return MarketSideAsk
	default:
		return MarketSideUnknown
	}
}

type OptionId struct {
	raw uint64
}

var OptionIdUnknown = OptionId{}

func OptionIdFromUint32(v uint32) OptionId {
	return OptionId{raw: uint64(v)}
}
func OptionIdFromUint64(v uint64) OptionId {
	return OptionId{raw: v}
}
func (oid OptionId) Valid() bool {
	return oid.raw != 0
}
func (oid OptionId) Invalid() bool {
	return !oid.Valid()
}

type Price int

const (
	PriceScale4Dec Price = 10000
	PriceScale2Dec Price = 100
)

func PriceFrom2Dec(price2d int) Price {
	return Price(price2d) * PriceScale4Dec / PriceScale2Dec
}
func PriceFrom4Dec(price4d int) Price {
	return Price(price4d)
}
func PriceTo2Dec(price Price) int {
	return int(price * PriceScale2Dec / PriceScale4Dec)
}
func PriceTo4Dec(price Price) int {
	return int(price)
}
<commit_msg>packet: add OptionId.String()<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package packet

import (
	"errors"
	"fmt"
)

type MarketSide byte

const (
	MarketSideUnknown MarketSide = 0
	MarketSideBid     MarketSide = 'B'
	MarketSideAsk     MarketSide = 'A'
)

var MarketSideUnknownError = errors.New("MarketSide unknown")

func (ms MarketSide) String() string {
	switch ms {
	case MarketSideBid:
		return "B"
	case MarketSideAsk:
		return "A"
	default:
		return "?"
	}
}
func (ms MarketSide) ToByte() (byte, error) {
	switch ms {
	case MarketSideBid:
		return 'B', nil
	case MarketSideAsk:
		return 'S', nil
	default:
		return 0, MarketSideUnknownError
	}
}
func MarketSideFromByte(b byte) MarketSide {
	switch b {
	case 'B':
		return MarketSideBid
	case 'A', 'S':
		return MarketSideAsk
	default:
		return MarketSideUnknown
	}
}

type OptionId struct {
	raw uint64
}

var OptionIdUnknown = OptionId{}

func OptionIdFromUint32(v uint32) OptionId {
	return OptionId{raw: uint64(v)}
}
func OptionIdFromUint64(v uint64) OptionId {
	return OptionId{raw: v}
}
func (oid OptionId) Valid() bool {
	return oid.raw != 0
}
func (oid OptionId) Invalid() bool {
	return !oid.Valid()
}
func (oid OptionId) String() string {
	return fmt.Sprintf("%#x", oid.raw)
}

type Price int

const (
	PriceScale4Dec Price = 10000
	PriceScale2Dec Price = 100
)

func PriceFrom2Dec(price2d int) Price {
	return Price(price2d) * PriceScale4Dec / PriceScale2Dec
}
func PriceFrom4Dec(price4d int) Price {
	return Price(price4d)
}
func PriceTo2Dec(price Price) int {
	return int(price * PriceScale2Dec / PriceScale4Dec)
}
func PriceTo4Dec(price Price) int {
	return int(price)
}
<|endoftext|>


<commit_before>package util

import (
	"go.polydawn.net/go-timeless-api"
)

/*
	Return a first, second, and remaining chunk of a ware's hash as strings.

	These are the first three, second three, and remaining bytes of the string.
	For base58 encoded values, these first two chunks used as dir prefixes are a
	cozy density for storing many many thousands of objects:
*/
func ChunkifyHash(wareID api.WareID) (string, string, string) {
	return wareID.Hash[0:3], wareID.Hash[3:6], wareID.Hash[6:]
}
<commit_msg>warehouse: define spitting for short hashes.<commit_after>package util

import (
	"go.polydawn.net/go-timeless-api"
)

/*
	Return a first, second, and remaining chunk of a ware's hash as strings.

	These are the first three, second three, and remaining bytes of the string.
	For base58 encoded values, these first two chunks used as dir prefixes are a
	cozy density for storing many many thousands of objects:

	If the hash is too short, we return a bunch of dashes.  (The hash is probably
	invalid semantically anyway, but we're not going to error about that here.)
	A hash of empty string will result in a return of `"---", "---", "-"` (in other
	words, as if the hash had been padded to a min of 7 characts, all dashes).
*/
func ChunkifyHash(wareID api.WareID) (string, string, string) {
	hash := wareID.Hash
	if len(hash) < 7 {
		hash = hash + "-------"[:7-len(hash)]
	}
	return hash[0:3], hash[3:6], hash[6:]
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type Request struct {
	a, b	int;
	replyc	chan int;
}

type BinOp (a, b int) int;

func Run(op *BinOp, request *Request) {
	result := op(request.a, request.b);
	request.replyc <- result;
}

func Server(op *BinOp, service chan *Request) {
	for {
		request := <-service;
		go Run(op, request);  // don't wait for it
	}
}

func StartServer(op *BinOp) chan *Request {
	req := new(chan *Request);
	go Server(op, req);
	return req;
}

func main() {
	adder := StartServer(func(a, b int) int { return a + b });
	const N = 100;
	var reqs [N]Request;
	for i := 0; i < N; i++ {
		req := &reqs[i];
		req.a = i;
		req.b = i + N;
		req.replyc = new(chan int);
		adder <- req;
	}
	for i := N-1; i >= 0; i-- {   // doesn't matter what order
		if <-reqs[i].replyc != N + 2*i {
			print("fail at ", i, "\n");
		}
	}
	print("done\n");
}
<commit_msg>Use make instead of new to allocate a channel.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

type Request struct {
	a, b	int;
	replyc	chan int;
}

type BinOp (a, b int) int;

func Run(op *BinOp, request *Request) {
	result := op(request.a, request.b);
	request.replyc <- result;
}

func Server(op *BinOp, service chan *Request) {
	for {
		request := <-service;
		go Run(op, request);  // don't wait for it
	}
}

func StartServer(op *BinOp) chan *Request {
	req := make(chan *Request);
	go Server(op, req);
	return req;
}

func main() {
	adder := StartServer(func(a, b int) int { return a + b });
	const N = 100;
	var reqs [N]Request;
	for i := 0; i < N; i++ {
		req := &reqs[i];
		req.a = i;
		req.b = i + N;
		req.replyc = make(chan int);
		adder <- req;
	}
	for i := N-1; i >= 0; i-- {   // doesn't matter what order
		if <-reqs[i].replyc != N + 2*i {
			print("fail at ", i, "\n");
		}
	}
	print("done\n");
}
<|endoftext|>


<commit_before>package databaseAccessObj

import (
	"database/sql"
	"math"
	"strconv"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

type Update struct {
	db *sql.DB
}

type Pair struct {
	// pair struct that holds user_id:[emails]
	id     int
	emails []string
}

type SqlStrVal struct {
	sqlStr string
	val    []interface{}
}

type GraphValue struct {
	graphType string
	X         float32
	Y         float32
}

func modId(userid int) int {
	// mod user_id by 15
	return int(math.Mod(float64(userid), 15.0))
}

func New(dsn string) *Update {
	// input: dsn = 'username:password@/database'
	// create new connection
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		panic(err.Error())
	}
	return &Update{db}
}

func (update *Update) hasTable(databaseName, tableName string) bool {
	// For testing purposes
	// Checks if the specify tablename exist in the specified database
	db := update.db
	var check string
	err := db.QueryRow("SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_name = ? LIMIT 1", databaseName, tableName).Scan(&check)
	if err == sql.ErrNoRows {
		return false
	} else {
		checkErr(err)
	}
	return true
}

func (update *Update) dropTable(tableName string) {
	// for testing purposes
	// Delete the table in the db if exists
	db := update.db
	_, err := db.Exec("DROP TABLE IF EXISTS " + tableName)
	checkErr(err)
}

func (update *Update) CloseConnection() {
	// Update method, closes connection
	update.db.Close()
}

func (update *Update) MkTbl(tablename, schema string) {
	db := update.db
	_, err := db.Exec("CREATE TABLE IF NOT EXISTS " + tablename + schema + ";")
	checkErr(err)
}

func (update *Update) SelectRandSubset(tblNum, size int) map[int][]string {
	db := update.db
	result := make(map[int][]string)

	stmt, err := db.Prepare("SELECT user_id, email FROM unsub_" + strconv.Itoa(tblNum) + " ORDER BY RAND() LIMIT ?;")
	checkErr(err)

	rows, err := stmt.Query(strconv.Itoa(size))
	checkErr(err)

	defer rows.Close()
	var user_id int
	var email string
	for rows.Next() {
		err = rows.Scan(&user_id, &email)
		checkErr(err)
		result[user_id] = append(result[user_id], email)
	}
	return result
}

func (update *Update) SelectLegacy(dataSet map[int][]string) map[int][]string {
	// Return items that exist both in input dataSet and database
	db := update.db
	result := make(map[int][]string)
	for userid, emails := range dataSet {
		tableName := "unsub_" + strconv.Itoa(modId(userid))
		sqlStr := "SELECT user_id, email FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
		var vals []interface{}
		for i := range emails {
			sqlStr += "email = ? OR "
			vals = append(vals, dataSet[userid][i])
		}
		sqlStr = sqlStr[0 : len(sqlStr)-4]
		sqlStr += ")"
		rows, err := db.Query(sqlStr, vals...)
		if err == sql.ErrNoRows {
			continue
		}
		checkErr(err)

		for rows.Next() {
			var user_id int
			var email string
			err = rows.Scan(&user_id, &email)
			checkErr(err)
			result[user_id] = append(result[user_id], email)
		}
	}
	return result
}

/*
func (update *Update) SelectByShard(dataSet map[int][]string) map[int][]string{
	//db := update.db
	result := make(map[int][]string)
	shardMap := make(map[int][]Pair)
	for userid, emails := range dataSet {
		shardMap[modId(userid)] = append(shardMap[modId(userid)], Pair{userid, emails})
	}
	int pairFlag = 0;
	int emailFlag = 0;
	int counter = 0;
	for shardNum := range shardMap{
		sqlStr := "SELECT user_id, email FROM unsub_" + shardNum + "WHERE "
		for p := range pairs{
			for e := range pairs[p].emails{
				while(counter < 32000){
					sqlStr += "(user_id = ? AND email = ?) OR "
					counter += 2
				}
			}
		}
	}
	return result
}
*/

func (update *Update) Select(dataSet map[int][]string) map[int][]string {
	// Return items that exist both in input dataSet and database
	db := update.db
	result := make(map[int][]string)
	var sqlStrings []SqlStrVal
	for userid, emails := range dataSet {
		counter := 0
		tableName := "unsub_" + strconv.Itoa(modId(userid))
		sqlStr := "SELECT user_id, email FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
		var vals []interface{}
		for i := range emails {
			sqlStr += "email = ? OR "
			vals = append(vals, dataSet[userid][i])
			counter += 1
			if counter >= 64000 {
				sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
				sqlStr = "SELECT user_id, email FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
				vals = make([]interface{}, 0)
				counter = 0
			}
		}
		if len(vals) != 0 {
			sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
		}
		for i := range sqlStrings {
			rows, err := db.Query(sqlStrings[i].sqlStr[0:len(sqlStrings[i].sqlStr)-4]+")", sqlStrings[i].val[0:len(sqlStrings[i].val)]...)
			if err == sql.ErrNoRows {
				continue
			}
			checkErr(err)
			defer rows.Close()
			for rows.Next() {
				var user_id int
				var email string
				err = rows.Scan(&user_id, &email)
				checkErr(err)
				result[user_id] = append(result[user_id], email)
			}
		}
	}
	return result
}

func (update *Update) SelectByTimestamp(ts time.Time) map[int][]string {
	// Select all items from database where input time after item's timestamp
	db := update.db
	result := make(map[int][]string)
	for i := 0; i < 15; i++ {
		tableName := "unsub_" + strconv.Itoa(i)
		sqlStr := "SELECT user_id, email FROM " + tableName + " WHERE ts >= ?"
		rows, err := db.Query(sqlStr, ts.String())
		if err == sql.ErrNoRows {
			continue
		}
		checkErr(err)
		defer rows.Close()
		for rows.Next() {
			var user_id int
			var email string
			err = rows.Scan(&user_id, &email)
			checkErr(err)
			result[user_id] = append(result[user_id], email)
		}
	}
	return result
}

func (update *Update) SelectTable(tableNum int) map[int][]string {
	// Select all items from a single table
	db := update.db
	result := make(map[int][]string)
	tableName := "unsub_" + strconv.Itoa(tableNum)
	sqlStr := "SELECT user_id, email FROM " + tableName
	rows, err := db.Query(sqlStr)
	checkErr(err)
	defer rows.Close()
	for rows.Next() {
		var user_id int
		var email string
		err = rows.Scan(&user_id, &email)
		checkErr(err)
		result[user_id] = append(result[user_id], email)
	}
	return result
}

func (update *Update) InsertLegacy(dataSet map[int][]string) {
	// Takes (int, string[])map of data and inserts
	// listed items into database
	db := update.db
	shardMap := make(map[int][]Pair)
	for userid, emails := range dataSet {
		shardMap[modId(userid)] = append(shardMap[modId(userid)], Pair{userid, emails})
	}
	for tabNum, pairs := range shardMap {
		tableName := "unsub_" + strconv.Itoa(tabNum)
		sqlStr := "INSERT INTO " + tableName + "(user_id, email, ts) VALUES "
		var vals []interface{}
		counter := 0
		for p := range pairs {
			for e := range pairs[p].emails {
				sqlStr += "(?, ?, CURRENT_TIMESTAMP), "
				vals = append(vals, pairs[p].id, pairs[p].emails[e])
				counter += 1
			}
		}
		sqlStr = sqlStr[0 : len(sqlStr)-2]
		stmt, err := db.Prepare(sqlStr)
		checkErr(err)
		_, err = stmt.Exec(vals...)
		checkErr(err)
	}
}

func (update *Update) Insert(dataSet map[int][]string) {
	// Takes (int, string[])map of data and inserts
	// listed items into database
	db := update.db
	shardMap := make(map[int][]Pair)
	for userid, emails := range dataSet {
		shardMap[modId(userid)] = append(shardMap[modId(userid)], Pair{userid, emails})
	}

	var sqlStrings []SqlStrVal

	for tabNum, pairs := range shardMap {
		tableName := "unsub_" + strconv.Itoa(tabNum)
		sqlStr := "INSERT INTO " + tableName + "(user_id, email, ts) VALUES "
		var vals []interface{}
		counter := 0
		for p := range pairs {
			for e := range pairs[p].emails {
				sqlStr += "(?, ?, CURRENT_TIMESTAMP), "
				vals = append(vals, pairs[p].id, pairs[p].emails[e])
				counter += 1
				if counter >= 32000 {
					sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
					sqlStr = "INSERT INTO " + tableName + "(user_id, email, ts) VALUES "
					vals = make([]interface{}, 0)
					counter = 0
				}
			}
		}
		if len(vals) != 0 {
			sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
		}

	}
	for i := range sqlStrings {
		stmt, err := db.Prepare(sqlStrings[i].sqlStr[0 : len(sqlStrings[i].sqlStr)-2])
		checkErr(err)
		_, err = stmt.Exec(sqlStrings[i].val...)
		checkErr(err)
	}
}

func (update *Update) LogTestResult(resultType string, x, y float64) {
	db := update.db
	sqlStr := "INSERT INTO test_results (result_type, x_axis, y_axis) VALUES (?, ?, ?)"
	stmt, err := db.Prepare(sqlStr)
	checkErr(err)
	_, err = stmt.Exec(resultType, x, y)
	checkErr(err)
}

func (update *Update) SelectTestResults() []GraphValue {
	db := update.db
	sqlStr := "SELECT result_type, x_axis, y_axis FROM test_results"
	stmt, err := db.Prepare(sqlStr)
	checkErr(err)
	rows, err := stmt.Query()
	checkErr(err)

	var resultType string
	var x float32
	var y float32
	result := make([]GraphValue, 0)

	defer rows.Close()
	for rows.Next() {
		err = rows.Scan(&resultType, &x, &y)
		checkErr(err)
		result = append(result, GraphValue{resultType, x, y})
	}
	return result[0:len(result)]

}

func (update *Update) Delete(dataSet map[int][]string) {
	// Takes (int, string[])map of data and removes
	// listed items from database
	db := update.db
	for userid, emails := range dataSet {
		tableName := "unsub_" + strconv.Itoa(modId(userid))
		sqlStr := "DELETE FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
		var vals []interface{}
		for i := range emails {
			sqlStr += "email = ? OR "
			vals = append(vals, dataSet[userid][i])
		}
		sqlStr = sqlStr[0 : len(sqlStr)-4]
		sqlStr += ")"
		stmt, err := db.Prepare(sqlStr)
		checkErr(err)
		_, err = stmt.Exec(vals...)
		checkErr(err)
	}
}

func (update *Update) Clear() {
	// Delete all rows from all tables in the database
	// Be very careful when using this function! It can
	// take a while to repopulate the db
	db := update.db
	for i := 0; i < 15; i++ {
		_, err := db.Exec("TRUNCATE TABLE unsub_" + strconv.Itoa(i))
		checkErr(err)
	}
}

func (update *Update) ClearTestResults() {
	db := update.db
	_, err := db.Exec("TRUNCATE TABLE test_results")
	checkErr(err)
}
<commit_msg>Bug fixes<commit_after>package databaseAccessObj

import (
	"database/sql"
	"math"
	"strconv"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

type Update struct {
	db *sql.DB
}

type Pair struct {
	// pair struct that holds user_id:[emails]
	id     int
	emails []string
}

type SqlStrVal struct {
	sqlStr string
	val    []interface{}
}

type GraphValue struct {
	graphType string
	X         float64
	Y         float64
}

func modId(userid int) int {
	// mod user_id by 15
	return int(math.Mod(float64(userid), 15.0))
}

func New(dsn string) *Update {
	// input: dsn = 'username:password@/database'
	// create new connection
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		panic(err.Error())
	}
	return &Update{db}
}

func (update *Update) hasTable(databaseName, tableName string) bool {
	// For testing purposes
	// Checks if the specify tablename exist in the specified database
	db := update.db
	var check string
	err := db.QueryRow("SELECT table_name FROM information_schema.tables WHERE table_schema = ? AND table_name = ? LIMIT 1", databaseName, tableName).Scan(&check)
	if err == sql.ErrNoRows {
		return false
	} else {
		checkErr(err)
	}
	return true
}

func (update *Update) dropTable(tableName string) {
	// for testing purposes
	// Delete the table in the db if exists
	db := update.db
	_, err := db.Exec("DROP TABLE IF EXISTS " + tableName)
	checkErr(err)
}

func (update *Update) CloseConnection() {
	// Update method, closes connection
	update.db.Close()
}

func (update *Update) MkTbl(tablename, schema string) {
	db := update.db
	_, err := db.Exec("CREATE TABLE IF NOT EXISTS " + tablename + schema + ";")
	checkErr(err)
}

func (update *Update) SelectRandSubset(tblNum, size int) map[int][]string {
	db := update.db
	result := make(map[int][]string)

	stmt, err := db.Prepare("SELECT user_id, email FROM unsub_" + strconv.Itoa(tblNum) + " ORDER BY RAND() LIMIT ?;")
	checkErr(err)

	rows, err := stmt.Query(strconv.Itoa(size))
	checkErr(err)

	defer rows.Close()
	var user_id int
	var email string
	for rows.Next() {
		err = rows.Scan(&user_id, &email)
		checkErr(err)
		result[user_id] = append(result[user_id], email)
	}
	return result
}

func (update *Update) SelectLegacy(dataSet map[int][]string) map[int][]string {
	// Return items that exist both in input dataSet and database
	db := update.db
	result := make(map[int][]string)
	for userid, emails := range dataSet {
		tableName := "unsub_" + strconv.Itoa(modId(userid))
		sqlStr := "SELECT user_id, email FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
		var vals []interface{}
		for i := range emails {
			sqlStr += "email = ? OR "
			vals = append(vals, dataSet[userid][i])
		}
		sqlStr = sqlStr[0 : len(sqlStr)-4]
		sqlStr += ")"
		rows, err := db.Query(sqlStr, vals...)
		if err == sql.ErrNoRows {
			continue
		}
		checkErr(err)

		for rows.Next() {
			var user_id int
			var email string
			err = rows.Scan(&user_id, &email)
			checkErr(err)
			result[user_id] = append(result[user_id], email)
		}
	}
	return result
}

/*
func (update *Update) SelectByShard(dataSet map[int][]string) map[int][]string{
	//db := update.db
	result := make(map[int][]string)
	shardMap := make(map[int][]Pair)
	for userid, emails := range dataSet {
		shardMap[modId(userid)] = append(shardMap[modId(userid)], Pair{userid, emails})
	}
	int pairFlag = 0;
	int emailFlag = 0;
	int counter = 0;
	for shardNum := range shardMap{
		sqlStr := "SELECT user_id, email FROM unsub_" + shardNum + "WHERE "
		for p := range pairs{
			for e := range pairs[p].emails{
				while(counter < 32000){
					sqlStr += "(user_id = ? AND email = ?) OR "
					counter += 2
				}
			}
		}
	}
	return result
}
*/

func (update *Update) Select(dataSet map[int][]string) map[int][]string {
	// Return items that exist both in input dataSet and database
	db := update.db
	result := make(map[int][]string)
	var sqlStrings []SqlStrVal
	for userid, emails := range dataSet {
		counter := 0
		tableName := "unsub_" + strconv.Itoa(modId(userid))
		sqlStr := "SELECT user_id, email FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
		var vals []interface{}
		for i := range emails {
			sqlStr += "email = ? OR "
			vals = append(vals, dataSet[userid][i])
			counter += 1
			if counter >= 64000 {
				sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
				sqlStr = "SELECT user_id, email FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
				vals = make([]interface{}, 0)
				counter = 0
			}
		}
		if len(vals) != 0 {
			sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
		}
		for i := range sqlStrings {
			rows, err := db.Query(sqlStrings[i].sqlStr[0:len(sqlStrings[i].sqlStr)-4]+")", sqlStrings[i].val[0:len(sqlStrings[i].val)]...)
			if err == sql.ErrNoRows {
				continue
			}
			checkErr(err)
			defer rows.Close()
			for rows.Next() {
				var user_id int
				var email string
				err = rows.Scan(&user_id, &email)
				checkErr(err)
				result[user_id] = append(result[user_id], email)
			}
		}
	}
	return result
}

func (update *Update) SelectByTimestamp(ts time.Time) map[int][]string {
	// Select all items from database where input time after item's timestamp
	db := update.db
	result := make(map[int][]string)
	for i := 0; i < 15; i++ {
		tableName := "unsub_" + strconv.Itoa(i)
		sqlStr := "SELECT user_id, email FROM " + tableName + " WHERE ts >= ?"
		rows, err := db.Query(sqlStr, ts.String())
		if err == sql.ErrNoRows {
			continue
		}
		checkErr(err)
		defer rows.Close()
		for rows.Next() {
			var user_id int
			var email string
			err = rows.Scan(&user_id, &email)
			checkErr(err)
			result[user_id] = append(result[user_id], email)
		}
	}
	return result
}

func (update *Update) SelectTable(tableNum int) map[int][]string {
	// Select all items from a single table
	db := update.db
	result := make(map[int][]string)
	tableName := "unsub_" + strconv.Itoa(tableNum)
	sqlStr := "SELECT user_id, email FROM " + tableName
	rows, err := db.Query(sqlStr)
	checkErr(err)
	defer rows.Close()
	for rows.Next() {
		var user_id int
		var email string
		err = rows.Scan(&user_id, &email)
		checkErr(err)
		result[user_id] = append(result[user_id], email)
	}
	return result
}

func (update *Update) InsertLegacy(dataSet map[int][]string) {
	// Takes (int, string[])map of data and inserts
	// listed items into database
	db := update.db
	shardMap := make(map[int][]Pair)
	for userid, emails := range dataSet {
		shardMap[modId(userid)] = append(shardMap[modId(userid)], Pair{userid, emails})
	}
	for tabNum, pairs := range shardMap {
		tableName := "unsub_" + strconv.Itoa(tabNum)
		sqlStr := "INSERT INTO " + tableName + "(user_id, email, ts) VALUES "
		var vals []interface{}
		counter := 0
		for p := range pairs {
			for e := range pairs[p].emails {
				sqlStr += "(?, ?, CURRENT_TIMESTAMP), "
				vals = append(vals, pairs[p].id, pairs[p].emails[e])
				counter += 1
			}
		}
		sqlStr = sqlStr[0 : len(sqlStr)-2]
		stmt, err := db.Prepare(sqlStr)
		checkErr(err)
		_, err = stmt.Exec(vals...)
		checkErr(err)
	}
}

func (update *Update) Insert(dataSet map[int][]string) {
	// Takes (int, string[])map of data and inserts
	// listed items into database
	db := update.db
	shardMap := make(map[int][]Pair)
	for userid, emails := range dataSet {
		shardMap[modId(userid)] = append(shardMap[modId(userid)], Pair{userid, emails})
	}

	var sqlStrings []SqlStrVal

	for tabNum, pairs := range shardMap {
		tableName := "unsub_" + strconv.Itoa(tabNum)
		sqlStr := "INSERT INTO " + tableName + "(user_id, email, ts) VALUES "
		var vals []interface{}
		counter := 0
		for p := range pairs {
			for e := range pairs[p].emails {
				sqlStr += "(?, ?, CURRENT_TIMESTAMP), "
				vals = append(vals, pairs[p].id, pairs[p].emails[e])
				counter += 1
				if counter >= 32000 {
					sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
					sqlStr = "INSERT INTO " + tableName + "(user_id, email, ts) VALUES "
					vals = make([]interface{}, 0)
					counter = 0
				}
			}
		}
		if len(vals) != 0 {
			sqlStrings = append(sqlStrings, SqlStrVal{sqlStr, vals[0:len(vals)]})
		}

	}
	for i := range sqlStrings {
		stmt, err := db.Prepare(sqlStrings[i].sqlStr[0 : len(sqlStrings[i].sqlStr)-2])
		checkErr(err)
		_, err = stmt.Exec(sqlStrings[i].val...)
		checkErr(err)
	}
}

func (update *Update) LogTestResult(resultType string, x, y float64) {
	db := update.db
	sqlStr := "INSERT INTO test_results (result_type, x_axis, y_axis) VALUES (?, ?, ?)"
	stmt, err := db.Prepare(sqlStr)
	checkErr(err)
	_, err = stmt.Exec(resultType, x, y)
	checkErr(err)
}

func (update *Update) SelectTestResults() []GraphValue {
	db := update.db
	sqlStr := "SELECT result_type, x_axis, y_axis FROM test_results"
	stmt, err := db.Prepare(sqlStr)
	checkErr(err)
	rows, err := stmt.Query()
	checkErr(err)

	var resultType string
	var x float64
	var y float64
	result := make([]GraphValue, 0)

	defer rows.Close()
	for rows.Next() {
		err = rows.Scan(&resultType, &x, &y)
		checkErr(err)
		result = append(result, GraphValue{resultType, x, y})
	}
	return result[0:len(result)]

}

func (update *Update) Delete(dataSet map[int][]string) {
	// Takes (int, string[])map of data and removes
	// listed items from database
	db := update.db
	for userid, emails := range dataSet {
		tableName := "unsub_" + strconv.Itoa(modId(userid))
		sqlStr := "DELETE FROM " + tableName + " WHERE user_id = " + strconv.Itoa(userid) + " AND ("
		var vals []interface{}
		for i := range emails {
			sqlStr += "email = ? OR "
			vals = append(vals, dataSet[userid][i])
		}
		sqlStr = sqlStr[0 : len(sqlStr)-4]
		sqlStr += ")"
		stmt, err := db.Prepare(sqlStr)
		checkErr(err)
		_, err = stmt.Exec(vals...)
		checkErr(err)
	}
}

func (update *Update) Clear() {
	// Delete all rows from all tables in the database
	// Be very careful when using this function! It can
	// take a while to repopulate the db
	db := update.db
	for i := 0; i < 15; i++ {
		_, err := db.Exec("TRUNCATE TABLE unsub_" + strconv.Itoa(i))
		checkErr(err)
	}
}

func (update *Update) ClearTestResults() {
	db := update.db
	_, err := db.Exec("TRUNCATE TABLE test_results")
	checkErr(err)
}
<|endoftext|>