<commit_before>package main

import (
	"fmt"

	"github.com/spf13/cobra"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <limits.h>

#include "include/macro.h"
#include "include/memory_utils.h"
#include <lxc/attach_options.h>
#include <lxc/lxccontainer.h>

extern char *advance_arg(bool required);

static bool write_nointr(int fd, const void *buf, size_t count)
{
	ssize_t ret;

	do {
		ret = write(fd, buf, count);
	} while (ret < 0 && errno == EINTR);

	if (ret < 0)
		return false;

	return (size_t)ret == count;
}

define_cleanup_function(struct lxc_container *, lxc_container_put);

static int wait_for_pid_status_nointr(pid_t pid)
{
	int status, ret;

again:
	ret = waitpid(pid, &status, 0);
	if (ret == -1) {
		if (errno == EINTR)
			goto again;

		return -1;
	}

	if (ret != pid)
		goto again;

	return status;
}

static int append_null_to_list(void ***list)
{
	int newentry = 0;
	void **new_list;

	if (*list)
		for (; (*list)[newentry]; newentry++)
			;

	new_list = realloc(*list, (newentry + 2) * sizeof(void **));
	if (!new_list)
		return ret_errno(ENOMEM);

	*list = new_list;
	(*list)[newentry + 1] = NULL;
	return newentry;
}

static int push_vargs(char ***list, char *entry)
{
	__do_free char *copy = NULL;
	int newentry;

	copy = strdup(entry);
	if (!copy)
		return ret_errno(ENOMEM);

	newentry = append_null_to_list((void ***)list);
	if (newentry < 0)
		return newentry;

	(*list)[newentry] = move_ptr(copy);

	return 0;
}

// We use a separate function because cleanup macros are called during stack
// unwinding if I'm not mistaken and if the compiler knows it exits it won't
// call them. That's not a problem since we're exiting but I just like to be on
// the safe side in case we ever call this from a different context. We also
// tell the compiler to not inline us.
__attribute__ ((noinline)) static int __forkexec(void)
{
	__do_close int status_pipe = 6;
	__do_free_string_list char **argvp = NULL, **envvp = NULL;
	call_cleaner(lxc_container_put) struct lxc_container *c = NULL;
	const char *config_path = NULL, *lxcpath = NULL, *name = NULL;
	char *cwd = NULL;
	lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT;
	lxc_attach_command_t command = {
		.program = NULL,
	};
	int ret;
	pid_t pid;
	uid_t uid;
	gid_t gid;

	if (geteuid() != 0)
		return log_error(EXIT_FAILURE, "Error: forkexec requires root privileges");

	name = advance_arg(false);
	if (name == NULL ||
	    (strcmp(name, "--help") == 0 ||
	     strcmp(name, "--version") == 0 || strcmp(name, "-h") == 0))
		return 0;

	lxcpath = advance_arg(true);
	config_path = advance_arg(true);
	cwd = advance_arg(true);
	uid = atoi(advance_arg(true));
	if (uid < 0)
		uid = (uid_t) - 1;
	gid = atoi(advance_arg(true));
	if (gid < 0)
		gid = (gid_t) - 1;

	for (char *arg = NULL, *section = NULL; (arg = advance_arg(false)); ) {
		if (!strcmp(arg, "--") && (!section || strcmp(section, "cmd"))) {
			section = NULL;
			continue;
		}

		if (!section) {
			section = arg;
			continue;
		}

		if (!strcmp(section, "env")) {
			if (!strncmp(arg, "HOME=", STRLITERALLEN("HOME=")))
				attach_options.initial_cwd = arg + STRLITERALLEN("HOME=");
			ret = push_vargs(&envvp, arg);
			if (ret < 0)
				return log_error(ret, "Failed to add %s to env array", arg);
		} else if (!strcmp(section, "cmd")) {
			ret = push_vargs(&argvp, arg);
			if (ret < 0)
				return log_error(ret, "Failed to add %s to arg array", arg);
		} else {
			return log_error(EXIT_FAILURE, "Invalid exec section %s", section);
		}
	}

	if (!argvp || !*argvp)
		return log_error(EXIT_FAILURE, "No command specified");

	c = lxc_container_new(name, lxcpath);
	if (!c)
		return EXIT_FAILURE;

	c->clear_config(c);

	if (!c->load_config(c, config_path))
		return EXIT_FAILURE;

	if (strcmp(cwd, ""))
		attach_options.initial_cwd = cwd;
	attach_options.env_policy = LXC_ATTACH_CLEAR_ENV;
	attach_options.extra_env_vars = envvp;
	attach_options.stdin_fd = 3;
	attach_options.stdout_fd = 4;
	attach_options.stderr_fd = 5;
	attach_options.uid = uid;
	attach_options.gid = gid;
	command.program = argvp[0];
	command.argv = argvp;

	ret = c->attach(c, lxc_attach_run_command, &command, &attach_options, &pid);
	if (ret < 0)
		return EXIT_FAILURE;

	if (!write_nointr(status_pipe, &pid, sizeof(pid))) {
		// Kill the child just to be safe.
		fprintf(stderr, "Failed to send pid %d of executing child to LXD. Killing child\n", pid);
		kill(pid, SIGKILL);
	}

	ret = wait_for_pid_status_nointr(pid);
	if (ret < 0)
		return log_error(EXIT_FAILURE, "Failed to wait for child process %d", pid);

	if (WIFEXITED(ret))
		return WEXITSTATUS(ret);

	if (WIFSIGNALED(ret))
		return 128 + WTERMSIG(ret);

	return EXIT_FAILURE;
}

void forkexec(void)
{
	_exit(__forkexec());
}
*/
import "C"

type cmdForkexec struct {
	global *cmdGlobal
}

func (c *cmdForkexec) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forkexec <container name> <containers path> <config> <cwd> <uid> <gid> -- env [key=value...] -- cmd <args...>"
	cmd.Short = "Execute a task inside the container"
	cmd.Long = `Description:
  Execute a task inside the container

  This internal command is used to spawn a task inside the container and
  allow LXD to interact with it.
`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdForkexec) Run(cmd *cobra.Command, args []string) error {
	return fmt.Errorf("This command should have been intercepted in cgo")
}
<commit_msg>forkexec: mark fd cloexec so the attaching process doesn't inherit it<commit_after>package main

import (
	"fmt"

	"github.com/spf13/cobra"
)

/*
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <limits.h>

#include "include/macro.h"
#include "include/memory_utils.h"
#include <lxc/attach_options.h>
#include <lxc/lxccontainer.h>

extern char *advance_arg(bool required);

static bool write_nointr(int fd, const void *buf, size_t count)
{
	ssize_t ret;

	do {
		ret = write(fd, buf, count);
	} while (ret < 0 && errno == EINTR);

	if (ret < 0)
		return false;

	return (size_t)ret == count;
}

define_cleanup_function(struct lxc_container *, lxc_container_put);

static int wait_for_pid_status_nointr(pid_t pid)
{
	int status, ret;

again:
	ret = waitpid(pid, &status, 0);
	if (ret == -1) {
		if (errno == EINTR)
			goto again;

		return -1;
	}

	if (ret != pid)
		goto again;

	return status;
}

static int append_null_to_list(void ***list)
{
	int newentry = 0;
	void **new_list;

	if (*list)
		for (; (*list)[newentry]; newentry++)
			;

	new_list = realloc(*list, (newentry + 2) * sizeof(void **));
	if (!new_list)
		return ret_errno(ENOMEM);

	*list = new_list;
	(*list)[newentry + 1] = NULL;
	return newentry;
}

static int push_vargs(char ***list, char *entry)
{
	__do_free char *copy = NULL;
	int newentry;

	copy = strdup(entry);
	if (!copy)
		return ret_errno(ENOMEM);

	newentry = append_null_to_list((void ***)list);
	if (newentry < 0)
		return newentry;

	(*list)[newentry] = move_ptr(copy);

	return 0;
}

static int fd_cloexec(int fd, bool cloexec)
{
	int oflags, nflags;

	oflags = fcntl(fd, F_GETFD, 0);
	if (oflags < 0)
		return -errno;

	if (cloexec)
		nflags = oflags | FD_CLOEXEC;
	else
		nflags = oflags & ~FD_CLOEXEC;

	if (nflags == oflags)
		return 0;

	if (fcntl(fd, F_SETFD, nflags) < 0)
		return -errno;

	return 0;
}

// We use a separate function because cleanup macros are called during stack
// unwinding if I'm not mistaken and if the compiler knows it exits it won't
// call them. That's not a problem since we're exiting but I just like to be on
// the safe side in case we ever call this from a different context. We also
// tell the compiler to not inline us.
__attribute__ ((noinline)) static int __forkexec(void)
{
	__do_close int status_pipe = 6;
	__do_free_string_list char **argvp = NULL, **envvp = NULL;
	call_cleaner(lxc_container_put) struct lxc_container *c = NULL;
	const char *config_path = NULL, *lxcpath = NULL, *name = NULL;
	char *cwd = NULL;
	lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT;
	lxc_attach_command_t command = {
		.program = NULL,
	};
	int ret;
	pid_t pid;
	uid_t uid;
	gid_t gid;

	if (geteuid() != 0)
		return log_error(EXIT_FAILURE, "Error: forkexec requires root privileges");

	name = advance_arg(false);
	if (name == NULL ||
	    (strcmp(name, "--help") == 0 ||
	     strcmp(name, "--version") == 0 || strcmp(name, "-h") == 0))
		return 0;

	lxcpath = advance_arg(true);
	config_path = advance_arg(true);
	cwd = advance_arg(true);
	uid = atoi(advance_arg(true));
	if (uid < 0)
		uid = (uid_t) - 1;
	gid = atoi(advance_arg(true));
	if (gid < 0)
		gid = (gid_t) - 1;

	for (char *arg = NULL, *section = NULL; (arg = advance_arg(false)); ) {
		if (!strcmp(arg, "--") && (!section || strcmp(section, "cmd"))) {
			section = NULL;
			continue;
		}

		if (!section) {
			section = arg;
			continue;
		}

		if (!strcmp(section, "env")) {
			if (!strncmp(arg, "HOME=", STRLITERALLEN("HOME=")))
				attach_options.initial_cwd = arg + STRLITERALLEN("HOME=");
			ret = push_vargs(&envvp, arg);
			if (ret < 0)
				return log_error(ret, "Failed to add %s to env array", arg);
		} else if (!strcmp(section, "cmd")) {
			ret = push_vargs(&argvp, arg);
			if (ret < 0)
				return log_error(ret, "Failed to add %s to arg array", arg);
		} else {
			return log_error(EXIT_FAILURE, "Invalid exec section %s", section);
		}
	}

	if (!argvp || !*argvp)
		return log_error(EXIT_FAILURE, "No command specified");

	ret = fd_cloexec(status_pipe, true);
	if (ret)
		return EXIT_FAILURE;

	c = lxc_container_new(name, lxcpath);
	if (!c)
		return EXIT_FAILURE;

	c->clear_config(c);

	if (!c->load_config(c, config_path))
		return EXIT_FAILURE;

	if (strcmp(cwd, ""))
		attach_options.initial_cwd = cwd;
	attach_options.env_policy = LXC_ATTACH_CLEAR_ENV;
	attach_options.extra_env_vars = envvp;
	attach_options.stdin_fd = 3;
	attach_options.stdout_fd = 4;
	attach_options.stderr_fd = 5;
	attach_options.uid = uid;
	attach_options.gid = gid;
	command.program = argvp[0];
	command.argv = argvp;

	ret = c->attach(c, lxc_attach_run_command, &command, &attach_options, &pid);
	if (ret < 0)
		return EXIT_FAILURE;

	if (!write_nointr(status_pipe, &pid, sizeof(pid))) {
		// Kill the child just to be safe.
		fprintf(stderr, "Failed to send pid %d of executing child to LXD. Killing child\n", pid);
		kill(pid, SIGKILL);
	}

	ret = wait_for_pid_status_nointr(pid);
	if (ret < 0)
		return log_error(EXIT_FAILURE, "Failed to wait for child process %d", pid);

	if (WIFEXITED(ret))
		return WEXITSTATUS(ret);

	if (WIFSIGNALED(ret))
		return 128 + WTERMSIG(ret);

	return EXIT_FAILURE;
}

void forkexec(void)
{
	_exit(__forkexec());
}
*/
import "C"

type cmdForkexec struct {
	global *cmdGlobal
}

func (c *cmdForkexec) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forkexec <container name> <containers path> <config> <cwd> <uid> <gid> -- env [key=value...] -- cmd <args...>"
	cmd.Short = "Execute a task inside the container"
	cmd.Long = `Description:
  Execute a task inside the container

  This internal command is used to spawn a task inside the container and
  allow LXD to interact with it.
`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdForkexec) Run(cmd *cobra.Command, args []string) error {
	return fmt.Errorf("This command should have been intercepted in cgo")
}
<|endoftext|>


<commit_before>package migrate

import (
	"bytes"
	"database/sql"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/rubenv/sql-migrate/sqlparse"
	"gopkg.in/gorp.v1"
)

type MigrationDirection int

const (
	Up MigrationDirection = iota
	Down
)

var tableName = "gorp_migrations"
var schemaName = ""
var numberPrefixRegex = regexp.MustCompile(`^(\d+).*$`)

// TxError is returned when any error is encountered during a database
// transaction. It contains the relevant *Migration and notes it's Id in the
// Error function output.
type TxError struct {
	Migration *Migration
	Err       error
}

func newTxError(migration *PlannedMigration, err error) error {
	return &TxError{
		Migration: migration.Migration,
		Err:       err,
	}
}

func (e *TxError) Error() string {
	return e.Err.Error() + " handling " + e.Migration.Id
}

// Set the name of the table used to store migration info.
//
// Should be called before any other call such as (Exec, ExecMax, ...).
func SetTable(name string) {
	if name != "" {
		tableName = name
	}
}

// SetSchema sets the name of a schema that the migration table be referenced.
func SetSchema(name string) {
	if name != "" {
		schemaName = name
	}
}

type Migration struct {
	Id   string
	Up   []string
	Down []string

	DisableTransactionUp   bool
	DisableTransactionDown bool
}

func (m Migration) Less(other *Migration) bool {
	switch {
	case m.isNumeric() && other.isNumeric() && m.VersionInt() != other.VersionInt():
		return m.VersionInt() < other.VersionInt()
	case m.isNumeric() && !other.isNumeric():
		return true
	case !m.isNumeric() && other.isNumeric():
		return false
	default:
		return m.Id < other.Id
	}
}

func (m Migration) isNumeric() bool {
	return len(m.NumberPrefixMatches()) > 0
}

func (m Migration) NumberPrefixMatches() []string {
	return numberPrefixRegex.FindStringSubmatch(m.Id)
}

func (m Migration) VersionInt() int64 {
	v := m.NumberPrefixMatches()[1]
	value, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		panic(fmt.Sprintf("Could not parse %q into int64: %s", v, err))
	}
	return value
}

type PlannedMigration struct {
	*Migration

	DisableTransaction bool
	Queries            []string
}

type byId []*Migration

func (b byId) Len() int           { return len(b) }
func (b byId) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b byId) Less(i, j int) bool { return b[i].Less(b[j]) }

type MigrationRecord struct {
	Id        string    `db:"id"`
	AppliedAt time.Time `db:"applied_at"`
}

var MigrationDialects = map[string]gorp.Dialect{
	"sqlite3":  gorp.SqliteDialect{},
	"postgres": gorp.PostgresDialect{},
	"mysql":    gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"},
	"mssql":    gorp.SqlServerDialect{},
	"oci8":     gorp.OracleDialect{},
}

type MigrationSource interface {
	// Finds the migrations.
	//
	// The resulting slice of migrations should be sorted by Id.
	FindMigrations() ([]*Migration, error)
}

// A hardcoded set of migrations, in-memory.
type MemoryMigrationSource struct {
	Migrations []*Migration
}

var _ MigrationSource = (*MemoryMigrationSource)(nil)

func (m MemoryMigrationSource) FindMigrations() ([]*Migration, error) {
	// Make sure migrations are sorted
	sort.Sort(byId(m.Migrations))

	return m.Migrations, nil
}

// A set of migrations loaded from a directory.
type FileMigrationSource struct {
	Dir string
}

var _ MigrationSource = (*FileMigrationSource)(nil)

func (f FileMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	file, err := os.Open(f.Dir)
	if err != nil {
		return nil, err
	}

	files, err := file.Readdir(0)
	if err != nil {
		return nil, err
	}

	for _, info := range files {
		if strings.HasSuffix(info.Name(), ".sql") {
			file, err := os.Open(path.Join(f.Dir, info.Name()))
			if err != nil {
				return nil, fmt.Errorf("Error while opening %s : %s", info.Name(), err)
			}

			migration, err := ParseMigration(info.Name(), file)
			if err != nil {
				return nil, fmt.Errorf("Error while parsing %s : %s", info.Name(), err)
			}

			migrations = append(migrations, migration)
		}
	}

	// Make sure migrations are sorted
	sort.Sort(byId(migrations))

	return migrations, nil
}

// Migrations from a bindata asset set.
type AssetMigrationSource struct {
	// Asset should return content of file in path if exists
	Asset func(path string) ([]byte, error)

	// AssetDir should return list of files in the path
	AssetDir func(path string) ([]string, error)

	// Path in the bindata to use.
	Dir string
}

var _ MigrationSource = (*AssetMigrationSource)(nil)

func (a AssetMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	files, err := a.AssetDir(a.Dir)
	if err != nil {
		return nil, err
	}

	for _, name := range files {
		if strings.HasSuffix(name, ".sql") {
			file, err := a.Asset(path.Join(a.Dir, name))
			if err != nil {
				return nil, err
			}

			migration, err := ParseMigration(name, bytes.NewReader(file))
			if err != nil {
				return nil, err
			}

			migrations = append(migrations, migration)
		}
	}

	// Make sure migrations are sorted
	sort.Sort(byId(migrations))

	return migrations, nil
}

// Migration parsing
func ParseMigration(id string, r io.ReadSeeker) (*Migration, error) {
	m := &Migration{
		Id: id,
	}

	parsed, err := sqlparse.ParseMigration(r)
	if err != nil {
		return nil, err
	}

	m.Up = parsed.UpStatements
	m.Down = parsed.DownStatements

	m.DisableTransactionUp = parsed.DisableTransactionUp
	m.DisableTransactionDown = parsed.DisableTransactionDown

	return m, nil
}

type SqlExecutor interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Insert(list ...interface{}) error
	Delete(list ...interface{}) (int64, error)
}

// Execute a set of migrations
//
// Returns the number of applied migrations.
func Exec(db *sql.DB, dialect string, m MigrationSource, dir MigrationDirection) (int, error) {
	return ExecMax(db, dialect, m, dir, 0)
}

// Execute a set of migrations
//
// Will apply at most `max` migrations. Pass 0 for no limit (or use Exec).
//
// Returns the number of applied migrations.
func ExecMax(db *sql.DB, dialect string, m MigrationSource, dir MigrationDirection, max int) (int, error) {
	migrations, dbMap, err := PlanMigration(db, dialect, m, dir, max)
	if err != nil {
		return 0, err
	}

	// Apply migrations
	applied := 0
	for _, migration := range migrations {
		var executor SqlExecutor

		if migration.DisableTransaction {
			executor = dbMap
		} else {
			executor, err = dbMap.Begin()
			if err != nil {
				return applied, newTxError(migration, err)
			}
		}

		for _, stmt := range migration.Queries {
			if _, err := executor.Exec(stmt); err != nil {
				if trans, ok := executor.(*gorp.Transaction); ok {
					trans.Rollback()
				}

				return applied, newTxError(migration, err)
			}
		}

		if dir == Up {
			err = executor.Insert(&MigrationRecord{
				Id:        migration.Id,
				AppliedAt: time.Now(),
			})
			if err != nil {
				return applied, newTxError(migration, err)
			}
		} else if dir == Down {
			_, err := executor.Delete(&MigrationRecord{
				Id: migration.Id,
			})
			if err != nil {
				return applied, newTxError(migration, err)
			}
		} else {
			panic("Not possible")
		}

		if trans, ok := executor.(*gorp.Transaction); ok {
			if err := trans.Commit(); err != nil {
				return applied, newTxError(migration, err)
			}
		}

		applied++
	}

	return applied, nil
}

// Plan a migration.
func PlanMigration(db *sql.DB, dialect string, m MigrationSource, dir MigrationDirection, max int) ([]*PlannedMigration, *gorp.DbMap, error) {
	dbMap, err := getMigrationDbMap(db, dialect)
	if err != nil {
		return nil, nil, err
	}

	migrations, err := m.FindMigrations()
	if err != nil {
		return nil, nil, err
	}

	var migrationRecords []MigrationRecord
	_, err = dbMap.Select(&migrationRecords, fmt.Sprintf("SELECT * FROM %s", dbMap.Dialect.QuotedTableForQuery(schemaName, tableName)))
	if err != nil {
		return nil, nil, err
	}

	// Sort migrations that have been run by Id.
	var existingMigrations []*Migration
	for _, migrationRecord := range migrationRecords {
		existingMigrations = append(existingMigrations, &Migration{
			Id: migrationRecord.Id,
		})
	}
	sort.Sort(byId(existingMigrations))

	// Get last migration that was run
	record := &Migration{}
	if len(existingMigrations) > 0 {
		record = existingMigrations[len(existingMigrations)-1]
	}

	result := make([]*PlannedMigration, 0)

	// Add missing migrations up to the last run migration.
	// This can happen for example when merges happened.
	if len(existingMigrations) > 0 {
		result = append(result, ToCatchup(migrations, existingMigrations, record)...)
	}

	// Figure out which migrations to apply
	toApply := ToApply(migrations, record.Id, dir)
	toApplyCount := len(toApply)
	if max > 0 && max < toApplyCount {
		toApplyCount = max
	}
	for _, v := range toApply[0:toApplyCount] {

		if dir == Up {
			result = append(result, &PlannedMigration{
				Migration:          v,
				Queries:            v.Up,
				DisableTransaction: v.DisableTransactionUp,
			})
		} else if dir == Down {
			result = append(result, &PlannedMigration{
				Migration:          v,
				Queries:            v.Down,
				DisableTransaction: v.DisableTransactionDown,
			})
		}
	}

	return result, dbMap, nil
}

// Filter a slice of migrations into ones that should be applied.
func ToApply(migrations []*Migration, current string, direction MigrationDirection) []*Migration {
	var index = -1
	if current != "" {
		for index < len(migrations)-1 {
			index++
			if migrations[index].Id == current {
				break
			}
		}
	}

	if direction == Up {
		return migrations[index+1:]
	} else if direction == Down {
		if index == -1 {
			return []*Migration{}
		}

		// Add in reverse order
		toApply := make([]*Migration, index+1)
		for i := 0; i < index+1; i++ {
			toApply[index-i] = migrations[i]
		}
		return toApply
	}

	panic("Not possible")
}

func ToCatchup(migrations, existingMigrations []*Migration, lastRun *Migration) []*PlannedMigration {
	missing := make([]*PlannedMigration, 0)
	for _, migration := range migrations {
		found := false
		for _, existing := range existingMigrations {
			if existing.Id == migration.Id {
				found = true
				break
			}
		}
		if !found && migration.Less(lastRun) {
			missing = append(missing, &PlannedMigration{
				Migration:          migration,
				Queries:            migration.Up,
				DisableTransaction: migration.DisableTransactionUp,
			})
		}
	}
	return missing
}

func GetMigrationRecords(db *sql.DB, dialect string) ([]*MigrationRecord, error) {
	dbMap, err := getMigrationDbMap(db, dialect)
	if err != nil {
		return nil, err
	}

	var records []*MigrationRecord
	query := fmt.Sprintf("SELECT * FROM %s ORDER BY id ASC", dbMap.Dialect.QuotedTableForQuery(schemaName, tableName))
	_, err = dbMap.Select(&records, query)
	if err != nil {
		return nil, err
	}

	return records, nil
}

func getMigrationDbMap(db *sql.DB, dialect string) (*gorp.DbMap, error) {
	d, ok := MigrationDialects[dialect]
	if !ok {
		return nil, fmt.Errorf("Unknown dialect: %s", dialect)
	}

	// When using the mysql driver, make sure that the parseTime option is
	// configured, otherwise it won't map time columns to time.Time. See
	// https://github.com/rubenv/sql-migrate/issues/2
	if dialect == "mysql" {
		var out *time.Time
		err := db.QueryRow("SELECT NOW()").Scan(&out)
		if err != nil {
			if err.Error() == "sql: Scan error on column index 0: unsupported driver -> Scan pair: []uint8 -> *time.Time" ||
				err.Error() == "sql: Scan error on column index 0: unsupported Scan, storing driver.Value type []uint8 into type *time.Time" {
				return nil, errors.New(`Cannot parse dates.

Make sure that the parseTime option is supplied to your database connection.
Check https://github.com/go-sql-driver/mysql#parsetime for more info.`)
			} else {
				return nil, err
			}
		}
	}

	// Create migration database map
	dbMap := &gorp.DbMap{Db: db, Dialect: d}
	dbMap.AddTableWithNameAndSchema(MigrationRecord{}, schemaName, tableName).SetKeys(false, "Id")
	//dbMap.TraceOn("", log.New(os.Stdout, "migrate: ", log.Lmicroseconds))

	err := dbMap.CreateTablesIfNotExists()
	if err != nil {
		return nil, err
	}

	return dbMap, nil
}

// TODO: Run migration + record insert in transaction.
<commit_msg>Move colons<commit_after>package migrate

import (
	"bytes"
	"database/sql"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/rubenv/sql-migrate/sqlparse"
	"gopkg.in/gorp.v1"
)

type MigrationDirection int

const (
	Up MigrationDirection = iota
	Down
)

var tableName = "gorp_migrations"
var schemaName = ""
var numberPrefixRegex = regexp.MustCompile(`^(\d+).*$`)

// TxError is returned when any error is encountered during a database
// transaction. It contains the relevant *Migration and notes it's Id in the
// Error function output.
type TxError struct {
	Migration *Migration
	Err       error
}

func newTxError(migration *PlannedMigration, err error) error {
	return &TxError{
		Migration: migration.Migration,
		Err:       err,
	}
}

func (e *TxError) Error() string {
	return e.Err.Error() + " handling " + e.Migration.Id
}

// Set the name of the table used to store migration info.
//
// Should be called before any other call such as (Exec, ExecMax, ...).
func SetTable(name string) {
	if name != "" {
		tableName = name
	}
}

// SetSchema sets the name of a schema that the migration table be referenced.
func SetSchema(name string) {
	if name != "" {
		schemaName = name
	}
}

type Migration struct {
	Id   string
	Up   []string
	Down []string

	DisableTransactionUp   bool
	DisableTransactionDown bool
}

func (m Migration) Less(other *Migration) bool {
	switch {
	case m.isNumeric() && other.isNumeric() && m.VersionInt() != other.VersionInt():
		return m.VersionInt() < other.VersionInt()
	case m.isNumeric() && !other.isNumeric():
		return true
	case !m.isNumeric() && other.isNumeric():
		return false
	default:
		return m.Id < other.Id
	}
}

func (m Migration) isNumeric() bool {
	return len(m.NumberPrefixMatches()) > 0
}

func (m Migration) NumberPrefixMatches() []string {
	return numberPrefixRegex.FindStringSubmatch(m.Id)
}

func (m Migration) VersionInt() int64 {
	v := m.NumberPrefixMatches()[1]
	value, err := strconv.ParseInt(v, 10, 64)
	if err != nil {
		panic(fmt.Sprintf("Could not parse %q into int64: %s", v, err))
	}
	return value
}

type PlannedMigration struct {
	*Migration

	DisableTransaction bool
	Queries            []string
}

type byId []*Migration

func (b byId) Len() int           { return len(b) }
func (b byId) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b byId) Less(i, j int) bool { return b[i].Less(b[j]) }

type MigrationRecord struct {
	Id        string    `db:"id"`
	AppliedAt time.Time `db:"applied_at"`
}

var MigrationDialects = map[string]gorp.Dialect{
	"sqlite3":  gorp.SqliteDialect{},
	"postgres": gorp.PostgresDialect{},
	"mysql":    gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"},
	"mssql":    gorp.SqlServerDialect{},
	"oci8":     gorp.OracleDialect{},
}

type MigrationSource interface {
	// Finds the migrations.
	//
	// The resulting slice of migrations should be sorted by Id.
	FindMigrations() ([]*Migration, error)
}

// A hardcoded set of migrations, in-memory.
type MemoryMigrationSource struct {
	Migrations []*Migration
}

var _ MigrationSource = (*MemoryMigrationSource)(nil)

func (m MemoryMigrationSource) FindMigrations() ([]*Migration, error) {
	// Make sure migrations are sorted
	sort.Sort(byId(m.Migrations))

	return m.Migrations, nil
}

// A set of migrations loaded from a directory.
type FileMigrationSource struct {
	Dir string
}

var _ MigrationSource = (*FileMigrationSource)(nil)

func (f FileMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	file, err := os.Open(f.Dir)
	if err != nil {
		return nil, err
	}

	files, err := file.Readdir(0)
	if err != nil {
		return nil, err
	}

	for _, info := range files {
		if strings.HasSuffix(info.Name(), ".sql") {
			file, err := os.Open(path.Join(f.Dir, info.Name()))
			if err != nil {
				return nil, fmt.Errorf("Error while opening %s: %s", info.Name(), err)
			}

			migration, err := ParseMigration(info.Name(), file)
			if err != nil {
				return nil, fmt.Errorf("Error while parsing %s: %s", info.Name(), err)
			}

			migrations = append(migrations, migration)
		}
	}

	// Make sure migrations are sorted
	sort.Sort(byId(migrations))

	return migrations, nil
}

// Migrations from a bindata asset set.
type AssetMigrationSource struct {
	// Asset should return content of file in path if exists
	Asset func(path string) ([]byte, error)

	// AssetDir should return list of files in the path
	AssetDir func(path string) ([]string, error)

	// Path in the bindata to use.
	Dir string
}

var _ MigrationSource = (*AssetMigrationSource)(nil)

func (a AssetMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	files, err := a.AssetDir(a.Dir)
	if err != nil {
		return nil, err
	}

	for _, name := range files {
		if strings.HasSuffix(name, ".sql") {
			file, err := a.Asset(path.Join(a.Dir, name))
			if err != nil {
				return nil, err
			}

			migration, err := ParseMigration(name, bytes.NewReader(file))
			if err != nil {
				return nil, err
			}

			migrations = append(migrations, migration)
		}
	}

	// Make sure migrations are sorted
	sort.Sort(byId(migrations))

	return migrations, nil
}

// Migration parsing
func ParseMigration(id string, r io.ReadSeeker) (*Migration, error) {
	m := &Migration{
		Id: id,
	}

	parsed, err := sqlparse.ParseMigration(r)
	if err != nil {
		return nil, err
	}

	m.Up = parsed.UpStatements
	m.Down = parsed.DownStatements

	m.DisableTransactionUp = parsed.DisableTransactionUp
	m.DisableTransactionDown = parsed.DisableTransactionDown

	return m, nil
}

type SqlExecutor interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Insert(list ...interface{}) error
	Delete(list ...interface{}) (int64, error)
}

// Execute a set of migrations
//
// Returns the number of applied migrations.
func Exec(db *sql.DB, dialect string, m MigrationSource, dir MigrationDirection) (int, error) {
	return ExecMax(db, dialect, m, dir, 0)
}

// Execute a set of migrations
//
// Will apply at most `max` migrations. Pass 0 for no limit (or use Exec).
//
// Returns the number of applied migrations.
func ExecMax(db *sql.DB, dialect string, m MigrationSource, dir MigrationDirection, max int) (int, error) {
	migrations, dbMap, err := PlanMigration(db, dialect, m, dir, max)
	if err != nil {
		return 0, err
	}

	// Apply migrations
	applied := 0
	for _, migration := range migrations {
		var executor SqlExecutor

		if migration.DisableTransaction {
			executor = dbMap
		} else {
			executor, err = dbMap.Begin()
			if err != nil {
				return applied, newTxError(migration, err)
			}
		}

		for _, stmt := range migration.Queries {
			if _, err := executor.Exec(stmt); err != nil {
				if trans, ok := executor.(*gorp.Transaction); ok {
					trans.Rollback()
				}

				return applied, newTxError(migration, err)
			}
		}

		if dir == Up {
			err = executor.Insert(&MigrationRecord{
				Id:        migration.Id,
				AppliedAt: time.Now(),
			})
			if err != nil {
				return applied, newTxError(migration, err)
			}
		} else if dir == Down {
			_, err := executor.Delete(&MigrationRecord{
				Id: migration.Id,
			})
			if err != nil {
				return applied, newTxError(migration, err)
			}
		} else {
			panic("Not possible")
		}

		if trans, ok := executor.(*gorp.Transaction); ok {
			if err := trans.Commit(); err != nil {
				return applied, newTxError(migration, err)
			}
		}

		applied++
	}

	return applied, nil
}

// Plan a migration.
func PlanMigration(db *sql.DB, dialect string, m MigrationSource, dir MigrationDirection, max int) ([]*PlannedMigration, *gorp.DbMap, error) {
	dbMap, err := getMigrationDbMap(db, dialect)
	if err != nil {
		return nil, nil, err
	}

	migrations, err := m.FindMigrations()
	if err != nil {
		return nil, nil, err
	}

	var migrationRecords []MigrationRecord
	_, err = dbMap.Select(&migrationRecords, fmt.Sprintf("SELECT * FROM %s", dbMap.Dialect.QuotedTableForQuery(schemaName, tableName)))
	if err != nil {
		return nil, nil, err
	}

	// Sort migrations that have been run by Id.
	var existingMigrations []*Migration
	for _, migrationRecord := range migrationRecords {
		existingMigrations = append(existingMigrations, &Migration{
			Id: migrationRecord.Id,
		})
	}
	sort.Sort(byId(existingMigrations))

	// Get last migration that was run
	record := &Migration{}
	if len(existingMigrations) > 0 {
		record = existingMigrations[len(existingMigrations)-1]
	}

	result := make([]*PlannedMigration, 0)

	// Add missing migrations up to the last run migration.
	// This can happen for example when merges happened.
	if len(existingMigrations) > 0 {
		result = append(result, ToCatchup(migrations, existingMigrations, record)...)
	}

	// Figure out which migrations to apply
	toApply := ToApply(migrations, record.Id, dir)
	toApplyCount := len(toApply)
	if max > 0 && max < toApplyCount {
		toApplyCount = max
	}
	for _, v := range toApply[0:toApplyCount] {

		if dir == Up {
			result = append(result, &PlannedMigration{
				Migration:          v,
				Queries:            v.Up,
				DisableTransaction: v.DisableTransactionUp,
			})
		} else if dir == Down {
			result = append(result, &PlannedMigration{
				Migration:          v,
				Queries:            v.Down,
				DisableTransaction: v.DisableTransactionDown,
			})
		}
	}

	return result, dbMap, nil
}

// Filter a slice of migrations into ones that should be applied.
func ToApply(migrations []*Migration, current string, direction MigrationDirection) []*Migration {
	var index = -1
	if current != "" {
		for index < len(migrations)-1 {
			index++
			if migrations[index].Id == current {
				break
			}
		}
	}

	if direction == Up {
		return migrations[index+1:]
	} else if direction == Down {
		if index == -1 {
			return []*Migration{}
		}

		// Add in reverse order
		toApply := make([]*Migration, index+1)
		for i := 0; i < index+1; i++ {
			toApply[index-i] = migrations[i]
		}
		return toApply
	}

	panic("Not possible")
}

func ToCatchup(migrations, existingMigrations []*Migration, lastRun *Migration) []*PlannedMigration {
	missing := make([]*PlannedMigration, 0)
	for _, migration := range migrations {
		found := false
		for _, existing := range existingMigrations {
			if existing.Id == migration.Id {
				found = true
				break
			}
		}
		if !found && migration.Less(lastRun) {
			missing = append(missing, &PlannedMigration{
				Migration:          migration,
				Queries:            migration.Up,
				DisableTransaction: migration.DisableTransactionUp,
			})
		}
	}
	return missing
}

func GetMigrationRecords(db *sql.DB, dialect string) ([]*MigrationRecord, error) {
	dbMap, err := getMigrationDbMap(db, dialect)
	if err != nil {
		return nil, err
	}

	var records []*MigrationRecord
	query := fmt.Sprintf("SELECT * FROM %s ORDER BY id ASC", dbMap.Dialect.QuotedTableForQuery(schemaName, tableName))
	_, err = dbMap.Select(&records, query)
	if err != nil {
		return nil, err
	}

	return records, nil
}

func getMigrationDbMap(db *sql.DB, dialect string) (*gorp.DbMap, error) {
	d, ok := MigrationDialects[dialect]
	if !ok {
		return nil, fmt.Errorf("Unknown dialect: %s", dialect)
	}

	// When using the mysql driver, make sure that the parseTime option is
	// configured, otherwise it won't map time columns to time.Time. See
	// https://github.com/rubenv/sql-migrate/issues/2
	if dialect == "mysql" {
		var out *time.Time
		err := db.QueryRow("SELECT NOW()").Scan(&out)
		if err != nil {
			if err.Error() == "sql: Scan error on column index 0: unsupported driver -> Scan pair: []uint8 -> *time.Time" ||
				err.Error() == "sql: Scan error on column index 0: unsupported Scan, storing driver.Value type []uint8 into type *time.Time" {
				return nil, errors.New(`Cannot parse dates.

Make sure that the parseTime option is supplied to your database connection.
Check https://github.com/go-sql-driver/mysql#parsetime for more info.`)
			} else {
				return nil, err
			}
		}
	}

	// Create migration database map
	dbMap := &gorp.DbMap{Db: db, Dialect: d}
	dbMap.AddTableWithNameAndSchema(MigrationRecord{}, schemaName, tableName).SetKeys(false, "Id")
	//dbMap.TraceOn("", log.New(os.Stdout, "migrate: ", log.Lmicroseconds))

	err := dbMap.CreateTablesIfNotExists()
	if err != nil {
		return nil, err
	}

	return dbMap, nil
}

// TODO: Run migration + record insert in transaction.
<|endoftext|>


<commit_before>package mailjet

import (
	"net/http"
	"net/textproto"
	"sync"

	"encoding/json"
)

/*
** API structures
 */

// Client bundles data needed by a large number
// of methods in order to interact with the Mailjet API.
type Client struct {
	apiBase    string
	httpClient HTTPClientInterface
	smtpClient SMTPClientInterface
	sync.Mutex
}

// Request bundles data needed to build the URL.
type Request struct {
	Resource string
	ID       int64
	AltID    string
	Action   string
	ActionID int64
}

// DataRequest bundles data needed to build the DATA URL.
type DataRequest struct {
	SourceType   string
	SourceTypeID int64
	DataType     string
	MimeType     string
	DataTypeID   int64
	LastID       bool
}

// FullRequest is the same as a Request but with a payload.
type FullRequest struct {
	Info    *Request
	Payload interface{}
}

// FullDataRequest is the same as a DataRequest but with a payload.
type FullDataRequest struct {
	Info    *DataRequest
	Payload interface{}
}

// RequestOptions are functional options that modify the specified request.
type RequestOptions func(*http.Request)

// RequestResult is the JSON result sent by the API.
type RequestResult struct {
	Count int
	Data  interface{}
	Total int
}

// RequestError is the error returned by the API.
type RequestError struct {
	ErrorInfo    string
	ErrorMessage string
	StatusCode   int
}

// RequestErrorV31 is the error returned by the API.
type RequestErrorV31 struct {
	ErrorInfo       string
	ErrorMessage    string
	StatusCode      int
	ErrorIdentifier string
}

/*
** Send API structures
 */

// InfoSendMail bundles data used by the Send API.
type InfoSendMail struct {
	FromEmail                string
	FromName                 string
	Sender                   string      `json:",omitempty"`
	Recipients               []Recipient `json:",omitempty"`
	To                       string      `json:",omitempty"`
	Cc                       string      `json:",omitempty"`
	Bcc                      string      `json:",omitempty"`
	Subject                  string
	TextPart                 string            `json:"Text-part,omitempty"`
	HTMLPart                 string            `json:"Html-part,omitempty"`
	Attachments              []Attachment      `json:",omitempty"`
	InlineAttachments        []Attachment      `json:"Inline_attachments,omitempty"`
	MjPrio                   int               `json:"Mj-prio,omitempty"`
	MjCampaign               string            `json:"Mj-campaign,omitempty"`
	MjDeduplicateCampaign    bool              `json:"Mj-deduplicatecampaign,omitempty"`
	MjCustomID               string            `json:"Mj-CustomID,omitempty"`
	MjTemplateID             string            `json:"Mj-TemplateID,omitempty"`
	MjTemplateErrorReporting string            `json:"MJ-TemplateErrorReporting,omitempty"`
	MjTemplateLanguage       string            `json:"Mj-TemplateLanguage,omitempty"`
	MjTemplateErrorDeliver   string            `json:"MJ-TemplateErrorDeliver,omitempty"`
	MjEventPayLoad           string            `json:"Mj-EventPayLoad,omitempty"`
	Headers                  map[string]string `json:",omitempty"`
	Vars                     interface{}       `json:",omitempty"`
	Messages                 []InfoSendMail    `json:",omitempty"`
}

// Recipient bundles data on the target of the mail.
type Recipient struct {
	Email string
	Name  string
	Vars  interface{} `json:",omitempty"`
}

// Attachment bundles data on the file attached to the mail.
type Attachment struct {
	ContentType string `json:"Content-Type"`
	Content     string
	Filename    string
}

// SentResult is the JSON result sent by the Send API.
type SentResult struct {
	Sent []struct {
		Email     string
		MessageID int64
	}
}

/*
** SMTP mail sending structures
 */

// InfoSMTP contains mandatory informations to send a mail via SMTP.
type InfoSMTP struct {
	From       string
	Recipients []string
	Header     textproto.MIMEHeader
	Content    []byte
}

/*
** Send API v3.1 structures
 */

// MessagesV31 definition
type MessagesV31 struct {
	Info        []InfoMessagesV31 `json:"Messages,omitempty"`
	SandBoxMode bool              `json:",omitempty"`
}

// InfoMessagesV31 represents the payload input taken by send API v3.1
type InfoMessagesV31 struct {
	From                     *RecipientV31          `json:",omitempty"`
	ReplyTo                  *RecipientV31          `json:",omitempty"`
	Sender                   *RecipientV31          `json:",omitempty"`
	To                       *RecipientsV31         `json:",omitempty"`
	Cc                       *RecipientsV31         `json:",omitempty"`
	Bcc                      *RecipientsV31         `json:",omitempty"`
	Attachments              *AttachmentsV31        `json:",omitempty"`
	InlinedAttachments       *InlinedAttachmentsV31 `json:",omitempty"`
	Subject                  string                 `json:",omitempty"`
	TextPart                 string                 `json:",omitempty"`
	HTMLPart                 string                 `json:",omitempty"`
	Priority                 int                    `json:",omitempty"`
	CustomCampaign           string                 `json:",omitempty"`
	StatisticsContactsListID int                    `json:",omitempty"`
	MonitoringCategory       string                 `json:",omitempty"`
	DeduplicateCampaign      bool                   `json:",omitempty"`
	TrackClicks              string                 `json:",omitempty"`
	TrackOpens               string                 `json:",omitempty"`
	CustomID                 string                 `json:",omitempty"`
	Variables                map[string]interface{} `json:",omitempty"`
	EventPayload             string                 `json:",omitempty"`
	TemplateID               interface{}            `json:",omitempty"`
	TemplateLanguage         bool                   `json:",omitempty"`
	TemplateErrorReporting   *RecipientV31          `json:",omitempty"`
	TemplateErrorDeliver     bool                   `json:",omitempty"`
	Headers                  map[string]interface{} `json:",omitempty"`
}

// RecipientV31 struct handle users input
type RecipientV31 struct {
	Email string `json:",omitempty"`
	Name  string `json:",omitempty"`
}

// RecipientsV31 is a collection of emails
type RecipientsV31 []RecipientV31

// AttachmentV31 struct represent a content attachment
type AttachmentV31 struct {
	ContentType   string `json:"ContentType,omitempty"`
	Base64Content string `json:"Base64Content,omitempty"`
	Filename      string `json:"Filename,omitempty"`
}

// AttachmentsV31 collection
type AttachmentsV31 []AttachmentV31

// InlinedAttachmentV31 struct represent the content of an inline attachement
type InlinedAttachmentV31 struct {
	AttachmentV31 `json:",omitempty"`
	ContentID     string `json:"ContentID,omitempty"`
}

// InlinedAttachmentsV31 collection
type InlinedAttachmentsV31 []InlinedAttachmentV31

// ErrorInfoV31 struct
type ErrorInfoV31 struct {
	Identifier string `json:"ErrorIdentifier,omitempty"`
	Info       string `json:"ErrorInfo"`
	Message    string `json:"ErrorMessage"`
	StatusCode int    `json:"StatusCode"`
}

func (err *ErrorInfoV31) Error() string {
	raw, _ := json.Marshal(err)
	return string(raw)
}

// APIErrorDetailsV31 contains the information details describing a specific error
type APIErrorDetailsV31 struct {
	ErrorClass     string
	ErrorMessage   string
	ErrorRelatedTo []string
	StatusCode     int
}

// APIFeedbackErrorV31 struct is composed of an error definition and the payload associated
type APIFeedbackErrorV31 struct {
	Errors []APIErrorDetailsV31
}

// APIFeedbackErrorsV31 defines the error when a validation error is being sent by the API
type APIFeedbackErrorsV31 struct {
	Messages []APIFeedbackErrorV31
}

func (api *APIFeedbackErrorsV31) Error() string {
	raw, _ := json.Marshal(api)
	return string(raw)
}

// GeneratedMessageV31 contains info to retrieve a generated email
type GeneratedMessageV31 struct {
	Email       string
	MessageUUID string
	MessageID   int64
	MessageHref string
}

// ResultV31 bundles the results of a sent email
type ResultV31 struct {
	Status   string
	CustomID string `json:",omitempty"`
	To       []GeneratedMessageV31
	Cc       []GeneratedMessageV31
	Bcc      []GeneratedMessageV31
}

// ResultsV31 bundles several results when several mails are sent
type ResultsV31 struct {
	ResultsV31 []ResultV31 `json:"Messages"`
}
<commit_msg>Update mailjet_resources.go<commit_after>package mailjet

import (
	"net/http"
	"net/textproto"
	"sync"

	"encoding/json"
)

/*
** API structures
 */

// Client bundles data needed by a large number
// of methods in order to interact with the Mailjet API.
type Client struct {
	apiBase    string
	httpClient HTTPClientInterface
	smtpClient SMTPClientInterface
	sync.Mutex
}

// Request bundles data needed to build the URL.
type Request struct {
	Resource string
	ID       int64
	AltID    string
	Action   string
	ActionID int64
}

// DataRequest bundles data needed to build the DATA URL.
type DataRequest struct {
	SourceType   string
	SourceTypeID int64
	DataType     string
	MimeType     string
	DataTypeID   int64
	LastID       bool
}

// FullRequest is the same as a Request but with a payload.
type FullRequest struct {
	Info    *Request
	Payload interface{}
}

// FullDataRequest is the same as a DataRequest but with a payload.
type FullDataRequest struct {
	Info    *DataRequest
	Payload interface{}
}

// RequestOptions are functional options that modify the specified request.
type RequestOptions func(*http.Request)

// RequestResult is the JSON result sent by the API.
type RequestResult struct {
	Count int
	Data  interface{}
	Total int
}

// RequestError is the error returned by the API.
type RequestError struct {
	ErrorInfo    string
	ErrorMessage string
	StatusCode   int
}

// RequestErrorV31 is the error returned by the API.
type RequestErrorV31 struct {
	ErrorInfo       string
	ErrorMessage    string
	StatusCode      int
	ErrorIdentifier string
}

/*
** Send API structures
 */

// InfoSendMail bundles data used by the Send API.
type InfoSendMail struct {
	FromEmail                string
	FromName                 string
	Sender                   string      `json:",omitempty"`
	Recipients               []Recipient `json:",omitempty"`
	To                       string      `json:",omitempty"`
	Cc                       string      `json:",omitempty"`
	Bcc                      string      `json:",omitempty"`
	Subject                  string
	TextPart                 string            `json:"Text-part,omitempty"`
	HTMLPart                 string            `json:"Html-part,omitempty"`
	Attachments              []Attachment      `json:",omitempty"`
	InlineAttachments        []Attachment      `json:"Inline_attachments,omitempty"`
	MjPrio                   int               `json:"Mj-prio,omitempty"`
	MjCampaign               string            `json:"Mj-campaign,omitempty"`
	MjDeduplicateCampaign    bool              `json:"Mj-deduplicatecampaign,omitempty"`
	MjCustomID               string            `json:"Mj-CustomID,omitempty"`
	MjTemplateID             string            `json:"Mj-TemplateID,omitempty"`
	MjTemplateErrorReporting string            `json:"MJ-TemplateErrorReporting,omitempty"`
	MjTemplateLanguage       string            `json:"Mj-TemplateLanguage,omitempty"`
	MjTemplateErrorDeliver   string            `json:"MJ-TemplateErrorDeliver,omitempty"`
	MjEventPayLoad           string            `json:"Mj-EventPayLoad,omitempty"`
	Headers                  map[string]string `json:",omitempty"`
	Vars                     interface{}       `json:",omitempty"`
	Messages                 []InfoSendMail    `json:",omitempty"`
}

// Recipient bundles data on the target of the mail.
type Recipient struct {
	Email string
	Name  string
	Vars  interface{} `json:",omitempty"`
}

// Attachment bundles data on the file attached to the mail.
type Attachment struct {
	ContentType string `json:"Content-Type"`
	Content     string
	Filename    string
}

// SentResult is the JSON result sent by the Send API.
type SentResult struct {
	Sent []struct {
		Email     string
		MessageID int64
	}
}

/*
** SMTP mail sending structures
 */

// InfoSMTP contains mandatory informations to send a mail via SMTP.
type InfoSMTP struct {
	From       string
	Recipients []string
	Header     textproto.MIMEHeader
	Content    []byte
}

/*
** Send API v3.1 structures
 */

// MessagesV31 definition
type MessagesV31 struct {
	Info        []InfoMessagesV31 `json:"Messages,omitempty"`
	SandBoxMode bool              `json:",omitempty"`
}

// InfoMessagesV31 represents the payload input taken by send API v3.1
type InfoMessagesV31 struct {
	From                     *RecipientV31          `json:",omitempty"`
	ReplyTo                  *RecipientV31          `json:",omitempty"`
	Sender                   *RecipientV31          `json:",omitempty"`
	To                       *RecipientsV31         `json:",omitempty"`
	Cc                       *RecipientsV31         `json:",omitempty"`
	Bcc                      *RecipientsV31         `json:",omitempty"`
	Attachments              *AttachmentsV31        `json:",omitempty"`
	InlinedAttachments       *InlinedAttachmentsV31 `json:",omitempty"`
	Subject                  string                 `json:",omitempty"`
	TextPart                 string                 `json:",omitempty"`
	HTMLPart                 string                 `json:",omitempty"`
	Priority                 int                    `json:",omitempty"`
	CustomCampaign           string                 `json:",omitempty"`
	StatisticsContactsListID int                    `json:",omitempty"`
	MonitoringCategory       string                 `json:",omitempty"`
	DeduplicateCampaign      bool                   `json:",omitempty"`
	TrackClicks              string                 `json:",omitempty"`
	TrackOpens               string                 `json:",omitempty"`
	CustomID                 string                 `json:",omitempty"`
	Variables                map[string]interface{} `json:",omitempty"`
	EventPayload             string                 `json:",omitempty"`
	TemplateID               int                    `json:",omitempty"`
	TemplateLanguage         bool                   `json:",omitempty"`
	TemplateErrorReporting   *RecipientV31          `json:",omitempty"`
	TemplateErrorDeliver     bool                   `json:",omitempty"`
	Headers                  map[string]interface{} `json:",omitempty"`
}

// RecipientV31 struct handle users input
type RecipientV31 struct {
	Email string `json:",omitempty"`
	Name  string `json:",omitempty"`
}

// RecipientsV31 is a collection of emails
type RecipientsV31 []RecipientV31

// AttachmentV31 struct represent a content attachment
type AttachmentV31 struct {
	ContentType   string `json:"ContentType,omitempty"`
	Base64Content string `json:"Base64Content,omitempty"`
	Filename      string `json:"Filename,omitempty"`
}

// AttachmentsV31 collection
type AttachmentsV31 []AttachmentV31

// InlinedAttachmentV31 struct represent the content of an inline attachement
type InlinedAttachmentV31 struct {
	AttachmentV31 `json:",omitempty"`
	ContentID     string `json:"ContentID,omitempty"`
}

// InlinedAttachmentsV31 collection
type InlinedAttachmentsV31 []InlinedAttachmentV31

// ErrorInfoV31 struct
type ErrorInfoV31 struct {
	Identifier string `json:"ErrorIdentifier,omitempty"`
	Info       string `json:"ErrorInfo"`
	Message    string `json:"ErrorMessage"`
	StatusCode int    `json:"StatusCode"`
}

func (err *ErrorInfoV31) Error() string {
	raw, _ := json.Marshal(err)
	return string(raw)
}

// APIErrorDetailsV31 contains the information details describing a specific error
type APIErrorDetailsV31 struct {
	ErrorClass     string
	ErrorMessage   string
	ErrorRelatedTo []string
	StatusCode     int
}

// APIFeedbackErrorV31 struct is composed of an error definition and the payload associated
type APIFeedbackErrorV31 struct {
	Errors []APIErrorDetailsV31
}

// APIFeedbackErrorsV31 defines the error when a validation error is being sent by the API
type APIFeedbackErrorsV31 struct {
	Messages []APIFeedbackErrorV31
}

func (api *APIFeedbackErrorsV31) Error() string {
	raw, _ := json.Marshal(api)
	return string(raw)
}

// GeneratedMessageV31 contains info to retrieve a generated email
type GeneratedMessageV31 struct {
	Email       string
	MessageUUID string
	MessageID   int64
	MessageHref string
}

// ResultV31 bundles the results of a sent email
type ResultV31 struct {
	Status   string
	CustomID string `json:",omitempty"`
	To       []GeneratedMessageV31
	Cc       []GeneratedMessageV31
	Bcc      []GeneratedMessageV31
}

// ResultsV31 bundles several results when several mails are sent
type ResultsV31 struct {
	ResultsV31 []ResultV31 `json:"Messages"`
}
<|endoftext|>


<commit_before>package levenshtein

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/smartystreets/mafsa"
)

type MinTree struct {
	mafsa.MinTree
	root *MinTreeNode
}

type MinTreeNode struct {
	mafsa.MinTreeNode
	r rune
}

type mtstackNode struct {
	vec  sparseVector
	str  string
	node *MinTreeNode
}

// Creates a new MinTree from a sorted list of strings. The list must be
// sorted because that is what the mafsa package expects.
func NewMinTree(words []string) (*MinTree, error) {
	bt := mafsa.New()
	for _, w := range words {
		err := bt.Insert(w)
		if err != nil {
			return nil, fmt.Errorf("Could not insert value into MinTree: %q\n", err)
		}
	}

	bt.Finish()
	me := mafsa.Encoder{}
	bytes, err := me.Encode(bt)
	if err != nil {
		return nil, err
	}

	de := mafsa.Decoder{}
	mmt, err := de.Decode(bytes)
	if err != nil {
		return nil, err
	}

	return &MinTree{*mmt, &MinTreeNode{*mmt.Root, rune(0)}}, nil
}

// Creates a new MinTree from a sorted list of strings. The list must be
// sorted because that is what the mafsa package expects.
// After the MinTree has been successfully created, the function also
// writes it to the io.Writer.
func NewMinTreeWrite(words []string, wr io.Writer) (*MinTree, error) {
	bt := mafsa.New()
	for _, w := range words {
		bt.Insert(w)
	}

	bt.Finish()
	me := mafsa.Encoder{}
	bs, err := me.Encode(bt)
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(wr, bytes.NewReader(bs))
	if err != nil {
		return nil, err
	}

	de := mafsa.Decoder{}
	mmt, err := de.Decode(bs)
	if err != nil {
		return nil, err
	}

	return &MinTree{*mmt, &MinTreeNode{*mmt.Root, rune(0)}}, nil
}

// LoadMinTree loads a MinTree from an io.Reader.
func LoadMinTree(r io.Reader) (*MinTree, error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	mt, err := new(mafsa.Decoder).Decode(data)
	if err != nil {
		return nil, err
	}

	return &MinTree{*mt, &MinTreeNode{*mt.Root, rune(0)}}, nil
}

func (n *MinTreeNode) traverse(a *SparseAutomatonRune, vec sparseVector) []string {
	ret := []string{}

	stack := make([]*mtstackNode, len(n.Edges))
	var i int

	for r, mt := range n.Edges {
		nmt := MinTreeNode{*mt, r}
		stack[i] = &mtstackNode{vec, "", &nmt}
		i++
	}

	var top *mtstackNode
	newVec := vec

	for len(stack) > 0 {
		top, stack = stack[len(stack)-1], stack[:len(stack)-1]
		n = top.node
		if n.r != 0 {
			newVec = a.Step(top.vec, n.r)
		}

		// if this is a terminal node - just check if we have
		// a match and add it to the results
		if n.Final && len(newVec) > 0 && a.IsMatch(newVec) {
			ret = append(ret, top.str+string(n.r))
		}

		if n.Edges != nil && a.CanMatch(newVec) {
			if n.r != 0 {
				top.str += string(n.r)
			}

			for r, child := range n.Edges {
				stack = append(stack, &mtstackNode{newVec, top.str, &MinTreeNode{*child, r}})
			}
		}
	}

	return ret
}

// FuzzyMatches returns all the words in the MinTree that are with
// maxDist edit distance from s
func (mt *MinTree) FuzzyMatches(s string, maxDist int) []string {
	a := NewSparseAutomatonRune(s, maxDist)

	state := a.Start()
	return mt.root.traverse(a, state)
}
<commit_msg>switch to fork for https://github.com/smartystreets-archives/mafsa/pull/8<commit_after>package levenshtein

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"

	"github.com/rubenv/mafsa"
)

type MinTree struct {
	mafsa.MinTree
	root *MinTreeNode
}

type MinTreeNode struct {
	mafsa.MinTreeNode
	r rune
}

type mtstackNode struct {
	vec  sparseVector
	str  string
	node *MinTreeNode
}

// Creates a new MinTree from a sorted list of strings. The list must be
// sorted because that is what the mafsa package expects.
func NewMinTree(words []string) (*MinTree, error) {
	bt := mafsa.New()
	for _, w := range words {
		err := bt.Insert(w)
		if err != nil {
			return nil, fmt.Errorf("Could not insert value into MinTree: %q\n", err)
		}
	}

	bt.Finish()
	me := mafsa.Encoder{}
	bytes, err := me.Encode(bt)
	if err != nil {
		return nil, err
	}

	de := mafsa.Decoder{}
	mmt, err := de.Decode(bytes)
	if err != nil {
		return nil, err
	}

	return &MinTree{*mmt, &MinTreeNode{*mmt.Root, rune(0)}}, nil
}

// Creates a new MinTree from a sorted list of strings. The list must be
// sorted because that is what the mafsa package expects.
// After the MinTree has been successfully created, the function also
// writes it to the io.Writer.
func NewMinTreeWrite(words []string, wr io.Writer) (*MinTree, error) {
	bt := mafsa.New()
	for _, w := range words {
		bt.Insert(w)
	}

	bt.Finish()
	me := mafsa.Encoder{}
	bs, err := me.Encode(bt)
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(wr, bytes.NewReader(bs))
	if err != nil {
		return nil, err
	}

	de := mafsa.Decoder{}
	mmt, err := de.Decode(bs)
	if err != nil {
		return nil, err
	}

	return &MinTree{*mmt, &MinTreeNode{*mmt.Root, rune(0)}}, nil
}

// LoadMinTree loads a MinTree from an io.Reader.
func LoadMinTree(r io.Reader) (*MinTree, error) {
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}

	mt, err := new(mafsa.Decoder).Decode(data)
	if err != nil {
		return nil, err
	}

	return &MinTree{*mt, &MinTreeNode{*mt.Root, rune(0)}}, nil
}

func (n *MinTreeNode) traverse(a *SparseAutomatonRune, vec sparseVector) []string {
	ret := []string{}

	stack := make([]*mtstackNode, len(n.Edges))
	var i int

	for r, mt := range n.Edges {
		nmt := MinTreeNode{*mt, r}
		stack[i] = &mtstackNode{vec, "", &nmt}
		i++
	}

	var top *mtstackNode
	newVec := vec

	for len(stack) > 0 {
		top, stack = stack[len(stack)-1], stack[:len(stack)-1]
		n = top.node
		if n.r != 0 {
			newVec = a.Step(top.vec, n.r)
		}

		// if this is a terminal node - just check if we have
		// a match and add it to the results
		if n.Final && len(newVec) > 0 && a.IsMatch(newVec) {
			ret = append(ret, top.str+string(n.r))
		}

		if n.Edges != nil && a.CanMatch(newVec) {
			if n.r != 0 {
				top.str += string(n.r)
			}

			for r, child := range n.Edges {
				stack = append(stack, &mtstackNode{newVec, top.str, &MinTreeNode{*child, r}})
			}
		}
	}

	return ret
}

// FuzzyMatches returns all the words in the MinTree that are with
// maxDist edit distance from s
func (mt *MinTree) FuzzyMatches(s string, maxDist int) []string {
	a := NewSparseAutomatonRune(s, maxDist)

	state := a.Start()
	return mt.root.traverse(a, state)
}
<|endoftext|>


<commit_before>package minhash

import (
	"math"
)

// MinWise is a data structure for generating a parametric family of
// hash functions of the form h1 + i*h2 for i=1, ..., k to compute
// a MinHash signature.  Each instance is tied to a single
// streamed set and hence single signature.  As it ingests
// elements it will update the current signature by calculating
// the ith hash function on the input and replacing the current
// ith element of the signature with the minimum of the hash
// and current value.

// The ith element of the signature is the current minimal value
// of the ith hash function.
type MinWise struct {
	minimums Signature
	h1       HashFunc
	h2       HashFunc
}

// NOTE MinWise constructors.

// NewMinWise constructs a new instance and pushes the optional elements.
func NewMinWise(h1, h2 HashFunc, size int, elements ...interface{}) *MinWise {
	mw := &MinWise{
		minimums: defaultSignature(size), // running set of min values
		h1:       h1,
		h2:       h2,
	}

	for _, e := range elements {
		mw.Push(e)
	}

	return mw
}

func NewMinWiseFromSignature(h1, h2 HashFunc, sig []uint64) *MinWise {
	csig := make([]uint64, len(sig))
	copy(csig, sig)
	mw := MinWise{
		minimums: csig,
		h1:       h1,
		h2:       h2,
	}
	return &mw
}

// InitStringIntMinWise creates a new MinWise instance and pushes a
// set of integers (represented as strings).  The returned instance
// contains the signature for the input set.
// func NewMinWiseFromStringInts(h1, h2 HashFunc, size int, xs []string) *MinWise {
// 	mw := NewMinWise(h1, h2, size)
// 	for _, x := range xs {
// 		mw.PushStringInt(x)
// 	}
// 	return mw
// }

// NOTE MinWise methods

// IsEmpty reports whether the MinWise instance represents a signature
// of the empty set.  Note that it's possible for a signature of a
// non-empty set to equal the signature for the empty set in rare
// circumstances (e.g., when the hash family is not min-wise independent).
func (m *MinWise) IsEmpty() bool {
	return IsEmpty(m)
}

// Copy returns a new MinWise instance with the same type and data.
func (m *MinWise) Copy() *MinWise {
	return NewMinWiseFromSignature(m.h1, m.h2, m.Signature())
}

// PushBytes updates the set's signature.  It hashes the input
// with each function in the family and compares these values
// with the current set of minimums, replacing them as necessary.
func (m *MinWise) PushBytes(b []byte) {

	v1 := m.h1(b)
	v2 := m.h2(b)

	// Compare minimal values
	for i, min := range m.minimums {
		// Compute hi(b) for ith hash function hi
		hb := v1 + uint64(i)*v2
		if hb < min {
			m.minimums[i] = hb
		}
	}
}

// PushStringInt first converts a string into a uint64 before pushing.
func (m *MinWise) PushStringInt(s string) {
	m.PushBytes(stringIntToBytes(s))
}

// Push deals with generic data by handling byte conversion.
// It first hashes the input with each function in the instance's family,
// and then compares these values to the set of current minimums,
// updating them as necessary.
func (m *MinWise) Push(x interface{}) {
	m.PushBytes(toBytes(x))
}

// Signature returns  the  current signature.
func (m *MinWise) Signature() []uint64 {
	return m.minimums
}

// Similarity computes the similarity of two signatures represented
// as MinWise instances.  This estimates the Jaccard index of the
// two underlying sets.
func (m *MinWise) Similarity(m2 MinHash) float64 {
	return MinWiseSimilarity(m.Signature(), m2.Signature())
}

// Merge combines the signatures of the second set,
// creating the signature of their union.
func (m *MinWise) Merge(m2 MinHash) {

	for i, v := range m2.Signature() {

		if v < m.minimums[i] {
			m.minimums[i] = v
		}
	}
}

// Cardinality estimates the cardinality of the set. Both the
// signature for the empty set and the zero signature have 0 cardinality.
func (m *MinWise) Cardinality() int {

	// http://www.cohenwang.com/edith/Papers/tcest.pdf
	// Let h be a hash function.
	// With M as the the maximum uint64, the value
	// z := -ln( (M - min h(A)) / M)
	// is a sample from Z := min { X1 ~ Exp(1), ... , Xn ~ Exp(1)}.
	// where n := |A|.  But Z ~ Exp(n) has mean 1/n.
	// Thus the inverse of the mean of all the transformed signature values
	// is an estimate for |A|.
	var cardinality int
	sum := 0.0 // running sum

	// Use inverse transform to obtain a sample from Z for each sig element.
	for _, v := range m.minimums {

		d := infinity - v
		// Make sure the zero signature and sig for empty set return 0.
		if d == 0 || d == infinity {
			continue
		}

		d := float64(infinity - v)
		sum += -math.Log(d / float64(infinity))
	}

	if sum != 0.0 {
		cardinality = int(float64(len(m.minimums)) / sum)
	}

	return cardinality
}

// UnionCardinality estimates the cardinality of the union.
func (m *MinWise) UnionCardinality(m2 MinHash) int {
	union := m.Copy()
	union.Merge(m2)
	return union.Cardinality()
}

// IntersectionCardinality estimates the cardinality of the intersection.
func (m *MinWise) IntersectionCardinality(m2 MinHash) int {
	// Estimate size of the union.
	u := m.UnionCardinality(m2)

	// |A & B| + |A || B| = |A| +|B|
	est := m.Cardinality() + m2.Cardinality() - u
	// Take absolute value.
	if est < 0 {
		est = 0
	}

	return est
}

// SymmetricDifferenceCardinality estimates the difference between
// the cardinality of the union and intersection.
func (m *MinWise) SymmetricDifferenceCardinality(m2 MinHash) int {
	est := m.UnionCardinality(m2) - m.IntersectionCardinality(m2)
	if est < 0 {
		est = 0
	}

	return est
}

// LessCardinality estimates the cardinality of the left set minus
// the right set. This operator is not symmetric.
func (m *MinWise) LessCardinality(m2 MinHash) int {
	est := m.Cardinality() - m.IntersectionCardinality(m2)
	if est < 0 {
		est = 0
	}

	return est
}

// SignatureBbit returns a b-bit reduction of the signature.
// This will result in unused bits at the high-end of the words if b does not divide 64 evenly.
func (m *MinWise) SignatureBbit(b uint) []uint64 {

	var sig []uint64 // full signature
	var w uint64     // current word
	bits := uint(64) // bits free in current word

	mask := uint64(1<<b) - 1

	for _, v := range m.minimums {
		if bits >= b {
			w <<= b
			w |= v & mask
			bits -= b
		} else {
			sig = append(sig, w)
			w = 0
			bits = 64
		}
	}

	if bits != 64 {
		sig = append(sig, w)
	}

	return sig
}

// MinWiseSimilarity computes an estimate for the
// Jaccard similarity of two sets given their MinWise signatures.
func MinWiseSimilarity(sig1, sig2 []uint64) float64 {
	if len(sig1) != len(sig2) {
		panic("Signature size mismatch.")
	}

	intersect := 0 // counter for number of elements in intersection

	for i := range sig1 {
		if sig1[i] == sig2[i] {
			intersect++
		}
	}

	return float64(intersect) / float64(len(sig1))
}

// SimilarityBbit computes an estimate for the similarity between two b-bit signatures
func SimilarityBbit(sig1, sig2 []uint64, b uint) float64 {

	if len(sig1) != len(sig2) {
		panic("signature size mismatch")
	}

	intersect := 0
	count := 0

	mask := uint64(1<<b) - 1

	for i := range sig1 {
		w1 := sig1[i]
		w2 := sig2[i]

		bits := uint(64)

		for bits >= b {
			v1 := (w1 & mask)
			v2 := (w2 & mask)

			count++
			if v1 == v2 {
				intersect++
			}

			bits -= b
			w1 >>= b
			w2 >>= b
		}
	}

	return float64(intersect) / float64(count)
}
<commit_msg>Fix type error.<commit_after>package minhash

import (
	"math"
)

// MinWise is a data structure for generating a parametric family of
// hash functions of the form h1 + i*h2 for i=1, ..., k to compute
// a MinHash signature.  Each instance is tied to a single
// streamed set and hence single signature.  As it ingests
// elements it will update the current signature by calculating
// the ith hash function on the input and replacing the current
// ith element of the signature with the minimum of the hash
// and current value.

// The ith element of the signature is the current minimal value
// of the ith hash function.
type MinWise struct {
	minimums Signature
	h1       HashFunc
	h2       HashFunc
}

// NOTE MinWise constructors.

// NewMinWise constructs a new instance and pushes the optional elements.
func NewMinWise(h1, h2 HashFunc, size int, elements ...interface{}) *MinWise {
	mw := &MinWise{
		minimums: defaultSignature(size), // running set of min values
		h1:       h1,
		h2:       h2,
	}

	for _, e := range elements {
		mw.Push(e)
	}

	return mw
}

func NewMinWiseFromSignature(h1, h2 HashFunc, sig []uint64) *MinWise {
	csig := make([]uint64, len(sig))
	copy(csig, sig)
	mw := MinWise{
		minimums: csig,
		h1:       h1,
		h2:       h2,
	}
	return &mw
}

// InitStringIntMinWise creates a new MinWise instance and pushes a
// set of integers (represented as strings).  The returned instance
// contains the signature for the input set.
// func NewMinWiseFromStringInts(h1, h2 HashFunc, size int, xs []string) *MinWise {
// 	mw := NewMinWise(h1, h2, size)
// 	for _, x := range xs {
// 		mw.PushStringInt(x)
// 	}
// 	return mw
// }

// NOTE MinWise methods

// IsEmpty reports whether the MinWise instance represents a signature
// of the empty set.  Note that it's possible for a signature of a
// non-empty set to equal the signature for the empty set in rare
// circumstances (e.g., when the hash family is not min-wise independent).
func (m *MinWise) IsEmpty() bool {
	return IsEmpty(m)
}

// Copy returns a new MinWise instance with the same type and data.
func (m *MinWise) Copy() *MinWise {
	return NewMinWiseFromSignature(m.h1, m.h2, m.Signature())
}

// PushBytes updates the set's signature.  It hashes the input
// with each function in the family and compares these values
// with the current set of minimums, replacing them as necessary.
func (m *MinWise) PushBytes(b []byte) {

	v1 := m.h1(b)
	v2 := m.h2(b)

	// Compare minimal values
	for i, min := range m.minimums {
		// Compute hi(b) for ith hash function hi
		hb := v1 + uint64(i)*v2
		if hb < min {
			m.minimums[i] = hb
		}
	}
}

// PushStringInt first converts a string into a uint64 before pushing.
func (m *MinWise) PushStringInt(s string) {
	m.PushBytes(stringIntToBytes(s))
}

// Push deals with generic data by handling byte conversion.
// It first hashes the input with each function in the instance's family,
// and then compares these values to the set of current minimums,
// updating them as necessary.
func (m *MinWise) Push(x interface{}) {
	m.PushBytes(toBytes(x))
}

// Signature returns  the  current signature.
func (m *MinWise) Signature() []uint64 {
	return m.minimums
}

// Similarity computes the similarity of two signatures represented
// as MinWise instances.  This estimates the Jaccard index of the
// two underlying sets.
func (m *MinWise) Similarity(m2 MinHash) float64 {
	return MinWiseSimilarity(m.Signature(), m2.Signature())
}

// Merge combines the signatures of the second set,
// creating the signature of their union.
func (m *MinWise) Merge(m2 MinHash) {

	for i, v := range m2.Signature() {

		if v < m.minimums[i] {
			m.minimums[i] = v
		}
	}
}

// Cardinality estimates the cardinality of the set. Both the
// signature for the empty set and the zero signature have 0 cardinality.
func (m *MinWise) Cardinality() int {

	// http://www.cohenwang.com/edith/Papers/tcest.pdf
	// Let h be a hash function.
	// With M as the the maximum uint64, the value
	// z := -ln( (M - min h(A)) / M)
	// is a sample from Z := min { X1 ~ Exp(1), ... , Xn ~ Exp(1)}.
	// where n := |A|.  But Z ~ Exp(n) has mean 1/n.
	// Thus the inverse of the mean of all the transformed signature values
	// is an estimate for |A|.
	var cardinality int
	sum := 0.0 // running sum

	// Use inverse transform to obtain a sample from Z for each sig element.
	for _, v := range m.minimums {

		d := infinity - v
		// Make sure the zero signature and sig for empty set return 0.
		if d == 0 || d == infinity {
			continue
		} else {
			sum += -math.Log(float64(d) / float64(infinity))
		}

	}

	if sum != 0.0 {
		cardinality = int(float64(len(m.minimums)) / sum)
	}

	return cardinality
}

// UnionCardinality estimates the cardinality of the union.
func (m *MinWise) UnionCardinality(m2 MinHash) int {
	union := m.Copy()
	union.Merge(m2)
	return union.Cardinality()
}

// IntersectionCardinality estimates the cardinality of the intersection.
func (m *MinWise) IntersectionCardinality(m2 MinHash) int {
	// Estimate size of the union.
	u := m.UnionCardinality(m2)

	// |A & B| + |A || B| = |A| +|B|
	est := m.Cardinality() + m2.Cardinality() - u
	// Take absolute value.
	if est < 0 {
		est = 0
	}

	return est
}

// SymmetricDifferenceCardinality estimates the difference between
// the cardinality of the union and intersection.
func (m *MinWise) SymmetricDifferenceCardinality(m2 MinHash) int {
	est := m.UnionCardinality(m2) - m.IntersectionCardinality(m2)
	if est < 0 {
		est = 0
	}

	return est
}

// LessCardinality estimates the cardinality of the left set minus
// the right set. This operator is not symmetric.
func (m *MinWise) LessCardinality(m2 MinHash) int {
	est := m.Cardinality() - m.IntersectionCardinality(m2)
	if est < 0 {
		est = 0
	}

	return est
}

// SignatureBbit returns a b-bit reduction of the signature.
// This will result in unused bits at the high-end of the words if b does not divide 64 evenly.
func (m *MinWise) SignatureBbit(b uint) []uint64 {

	var sig []uint64 // full signature
	var w uint64     // current word
	bits := uint(64) // bits free in current word

	mask := uint64(1<<b) - 1

	for _, v := range m.minimums {
		if bits >= b {
			w <<= b
			w |= v & mask
			bits -= b
		} else {
			sig = append(sig, w)
			w = 0
			bits = 64
		}
	}

	if bits != 64 {
		sig = append(sig, w)
	}

	return sig
}

// MinWiseSimilarity computes an estimate for the
// Jaccard similarity of two sets given their MinWise signatures.
func MinWiseSimilarity(sig1, sig2 []uint64) float64 {
	if len(sig1) != len(sig2) {
		panic("Signature size mismatch.")
	}

	intersect := 0 // counter for number of elements in intersection

	for i := range sig1 {
		if sig1[i] == sig2[i] {
			intersect++
		}
	}

	return float64(intersect) / float64(len(sig1))
}

// SimilarityBbit computes an estimate for the similarity between two b-bit signatures
func SimilarityBbit(sig1, sig2 []uint64, b uint) float64 {

	if len(sig1) != len(sig2) {
		panic("signature size mismatch")
	}

	intersect := 0
	count := 0

	mask := uint64(1<<b) - 1

	for i := range sig1 {
		w1 := sig1[i]
		w2 := sig2[i]

		bits := uint(64)

		for bits >= b {
			v1 := (w1 & mask)
			v2 := (w2 & mask)

			count++
			if v1 == v2 {
				intersect++
			}

			bits -= b
			w1 >>= b
			w2 >>= b
		}
	}

	return float64(intersect) / float64(count)
}
<|endoftext|>


<commit_before>package maps

// MultiHashMap XXX
type MultiHashMap struct {
	data map[interface{}][]interface{}
}

// Put XXX
func (m *MultiHashMap) Put(key interface{}, value interface{}) {
	values, ok := m.data[key]
	if ok {
		values = append(values, value)
	} else {
		values = []interface{}{value}
	}
	m.data[key] = values
}

// Get XXX
func (m *MultiHashMap) Get(key interface{}) (values []interface{}, ok bool) {
	values, ok = m.data[key]
	if !ok {
		return nil, false
	}
	return values, true
}

// Count XXX
func (m *MultiHashMap) Count(key interface{}) (count int, ok bool) {
	values, ok := m.data[key]
	if !ok {
		return 0, false
	}
	return len(values), true
}

// Has XXX
func (m *MultiHashMap) Has(key interface{}) bool {
	_, ok := m.data[key]
	return ok
}

// HasKeyValue XXX
func (m *MultiHashMap) HasKeyValue(key interface{}, value interface{}) bool {
	values, ok := m.data[key]
	if !ok {
		return false
	}
	for val := range values {
		if val == value {
			return true
		}
	}
	return false
}

// Del XXX
func (m *MultiHashMap) Del(key interface{}) {
	delete(m.data, key)
}

// DelKeyValue XXX
func (m *MultiHashMap) DelKeyValue(key interface{}, value interface{}) {
	values, ok := m.data[key]
	if !ok {
		return
	}
	for i, val := range values {
		if val == value {
			values = append(values[:i], values[i+1:]...)
		}
	}
	m.data[key] = values
}

// Keys XXX
func (m *MultiHashMap) Keys() []interface{} {
	res := make([]interface{}, len(m.data))
	i := 0
	for k := range m.data {
		res[i] = k
		i++
	}
	return res
}
<commit_msg>map: MultiHashMap small cleanup (#68)<commit_after>package maps

// MultiHashMap hashmap with multiple values per key
type MultiHashMap struct {
	data map[interface{}][]interface{}
}

// NewMultiHashMap returns a pointer to the MultiHashMap
func NewMultiHashMap() *MultiHashMap {
	m := &MultiHashMap{
		data: make(map[interface{}][]interface{}),
	}
	return m
}

// Put XXX
func (m *MultiHashMap) Put(key interface{}, value interface{}) {
	values, ok := m.data[key]
	if ok {
		values = append(values, value)
	} else {
		values = []interface{}{value}
	}
	m.data[key] = values
}

// Get XXX
func (m *MultiHashMap) Get(key interface{}) (values []interface{}, ok bool) {
	values, ok = m.data[key]
	if ok {
		return values, true
	}
	return nil, false
}

// Count XXX
func (m *MultiHashMap) Count(key interface{}) (count int, ok bool) {
	values, ok := m.data[key]
	if ok {
		return len(values), true
	}
	return 0, false
}

// Has XXX
func (m *MultiHashMap) Has(key interface{}) bool {
	_, ok := m.data[key]
	return ok
}

// HasKeyValue XXX
func (m *MultiHashMap) HasKeyValue(key interface{}, value interface{}) bool {
	values, ok := m.data[key]
	if !ok {
		return false
	}
	for val := range values {
		if val == value {
			return true
		}
	}
	return false
}

// Del XXX
func (m *MultiHashMap) Del(key interface{}) {
	delete(m.data, key)
}

// DelKeyValue XXX
func (m *MultiHashMap) DelKeyValue(key interface{}, value interface{}) {
	values, ok := m.data[key]
	if !ok {
		return
	}
	for i, val := range values {
		if val == value {
			values = append(values[:i], values[i+1:]...)
		}
	}
	m.data[key] = values
}

// Keys XXX
func (m *MultiHashMap) Keys() []interface{} {
	res := make([]interface{}, len(m.data))
	i := 0
	for k := range m.data {
		res[i] = k
		i++
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2017 NDP Systèmes. All Rights Reserved.
// See LICENSE file for full licensing details.

package assets

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/hexya-erp/hexya/src/tools/logging"
)

// A Compiler can compile to CSS
type Compiler interface {
	// Compile CSS from the in stream to out.
	Compile(in io.Reader, out io.Writer, includePaths ...string) error
}

var log logging.Logger

const rxPreprocessImports = `(@import\s?['"]([^'"]+)['"](;?))`

// LessCompiler can compile Less to CSS
type LessCompiler struct{}

// Compile Less to CSS
func (lc LessCompiler) Compile(in io.Reader, out io.Writer, includePaths ...string) error {
	var includeDirective string
	if len(includePaths) > 0 {
		includeDirective = fmt.Sprintf("--include-path=%s", strings.Join(includePaths, ":"))
	}
	lessCmd := exec.Command("lessc", "-", "--no-js", "--no-color", includeDirective)
	return executeCmd(lessCmd, in, out)
}

var _ Compiler = LessCompiler{}

// ScssCompiler can compile Sass to CSS
type ScssCompiler struct{}

// Compile Sass/Scss to CSS
func (sc ScssCompiler) Compile(in io.Reader, out io.Writer, includePaths ...string) error {
	var (
		includeDirectives []string
		sanitized         bytes.Buffer
	)
	sc.sanitize(in, &sanitized)
	for _, includePath := range includePaths {
		includeDirectives = append(includeDirectives, "--load-path", includePath)
	}
	// DEBUG
	tmpFile, err := os.OpenFile("/tmp/sass", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
	if err != nil {
		panic(err)
	}
	tr := io.TeeReader(&sanitized, tmpFile)
	defer tmpFile.Close()
	// ======
	scssCmd := exec.Command("sass", "--stdin", "--no-quiet")
	scssCmd.Args = append(scssCmd.Args, includeDirectives...)
	return executeCmd(scssCmd, tr, out)
}

// sanitize the import directives of the input SCSS
func (sc ScssCompiler) sanitize(in io.Reader, out io.Writer) error {
	data, err := ioutil.ReadAll(in)
	if err != nil {
		return fmt.Errorf("error while reading input CSS: %v", err)
	}
	re, err := regexp.Compile(rxPreprocessImports)
	if err != nil {
		return err
	}
	imports := make(map[string]bool)
	re.ReplaceAllFunc(data, func(match []byte) []byte {
		line := bytes.ReplaceAll(match, []byte("'"), []byte("\""))
		if bytes.Contains(line, []byte(".")) || bytes.HasPrefix(line, []byte("@import \"/")) || bytes.HasPrefix(line, []byte("@import \"~")) {
			panic(fmt.Errorf("local import '%s' is forbidden for security reasons. "+
				"Please remove all @import {your_file} imports in your custom files. "+
				"In Hexya you have to import all files in the assets, and not through the @import statement", match))
		}
		if imports[string(line)] {
			return []byte{}
		}
		imports[string(line)] = true
		return line
	})
	out.Write(data)
	return nil
}

var _ Compiler = ScssCompiler{}

func executeCmd(cmd *exec.Cmd, in io.Reader, out io.Writer) error {
	log.Debug("Compiling assets", "cmd", cmd.String())
	lessIn, err := cmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("unable to get compile command stdin pipe: %v", err)
	}
	lessOut, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("unable to get compile command stdout pipe: %v", err)
	}
	lessErr, err := cmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("unable to get compile command stderr pipe: %v", err)
	}
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("unable to start compile command: %v", err)
	}
	if _, err := io.Copy(lessIn, in); err != nil {
		return fmt.Errorf("unable to read input: %v", err)
	}
	if err := lessIn.Close(); err != nil {
		return err
	}
	var outPut bytes.Buffer
	teeReader := io.TeeReader(lessOut, &outPut)
	if _, err := io.Copy(out, teeReader); err != nil {
		return fmt.Errorf("unable to write output: %v", err)
	}
	var msg bytes.Buffer
	if _, err := io.Copy(&msg, lessErr); err != nil {
		return fmt.Errorf("unable to write on err pipe: %v", err)
	}
	if err := cmd.Wait(); err != nil {
		msgString := msg.String()
		if msgString == "" {
			msgString = outPut.String()
		}
		return fmt.Errorf("error while executing compile command %s. Error: %s: %s\n", cmd.String(), err, msgString)
	}
	return nil
}

func init() {
	log = logging.GetLogger("assets")
}
<commit_msg>Fixed tests in go 1.12<commit_after>// Copyright 2017 NDP Systèmes. All Rights Reserved.
// See LICENSE file for full licensing details.

package assets

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/hexya-erp/hexya/src/tools/logging"
)

// A Compiler can compile to CSS
type Compiler interface {
	// Compile CSS from the in stream to out.
	Compile(in io.Reader, out io.Writer, includePaths ...string) error
}

var log logging.Logger

const rxPreprocessImports = `(@import\s?['"]([^'"]+)['"](;?))`

// LessCompiler can compile Less to CSS
type LessCompiler struct{}

// Compile Less to CSS
func (lc LessCompiler) Compile(in io.Reader, out io.Writer, includePaths ...string) error {
	var includeDirective string
	if len(includePaths) > 0 {
		includeDirective = fmt.Sprintf("--include-path=%s", strings.Join(includePaths, ":"))
	}
	lessCmd := exec.Command("lessc", "-", "--no-js", "--no-color", includeDirective)
	return executeCmd(lessCmd, in, out)
}

var _ Compiler = LessCompiler{}

// ScssCompiler can compile Sass to CSS
type ScssCompiler struct{}

// Compile Sass/Scss to CSS
func (sc ScssCompiler) Compile(in io.Reader, out io.Writer, includePaths ...string) error {
	var (
		includeDirectives []string
		sanitized         bytes.Buffer
	)
	sc.sanitize(in, &sanitized)
	for _, includePath := range includePaths {
		includeDirectives = append(includeDirectives, "--load-path", includePath)
	}
	// DEBUG
	tmpFile, err := os.OpenFile("/tmp/sass", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.ModePerm)
	if err != nil {
		panic(err)
	}
	tr := io.TeeReader(&sanitized, tmpFile)
	defer tmpFile.Close()
	// ======
	scssCmd := exec.Command("sass", "--stdin", "--no-quiet")
	scssCmd.Args = append(scssCmd.Args, includeDirectives...)
	return executeCmd(scssCmd, tr, out)
}

// sanitize the import directives of the input SCSS
func (sc ScssCompiler) sanitize(in io.Reader, out io.Writer) error {
	data, err := ioutil.ReadAll(in)
	if err != nil {
		return fmt.Errorf("error while reading input CSS: %v", err)
	}
	re, err := regexp.Compile(rxPreprocessImports)
	if err != nil {
		return err
	}
	imports := make(map[string]bool)
	re.ReplaceAllFunc(data, func(match []byte) []byte {
		line := bytes.ReplaceAll(match, []byte("'"), []byte("\""))
		if bytes.Contains(line, []byte(".")) || bytes.HasPrefix(line, []byte("@import \"/")) || bytes.HasPrefix(line, []byte("@import \"~")) {
			panic(fmt.Errorf("local import '%s' is forbidden for security reasons. "+
				"Please remove all @import {your_file} imports in your custom files. "+
				"In Hexya you have to import all files in the assets, and not through the @import statement", match))
		}
		if imports[string(line)] {
			return []byte{}
		}
		imports[string(line)] = true
		return line
	})
	out.Write(data)
	return nil
}

var _ Compiler = ScssCompiler{}

func executeCmd(cmd *exec.Cmd, in io.Reader, out io.Writer) error {
	log.Debug("Compiling assets", "cmd", cmd)
	lessIn, err := cmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("unable to get compile command stdin pipe: %v", err)
	}
	lessOut, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("unable to get compile command stdout pipe: %v", err)
	}
	lessErr, err := cmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("unable to get compile command stderr pipe: %v", err)
	}
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("unable to start compile command: %v", err)
	}
	if _, err := io.Copy(lessIn, in); err != nil {
		return fmt.Errorf("unable to read input: %v", err)
	}
	if err := lessIn.Close(); err != nil {
		return err
	}
	var outPut bytes.Buffer
	teeReader := io.TeeReader(lessOut, &outPut)
	if _, err := io.Copy(out, teeReader); err != nil {
		return fmt.Errorf("unable to write output: %v", err)
	}
	var msg bytes.Buffer
	if _, err := io.Copy(&msg, lessErr); err != nil {
		return fmt.Errorf("unable to write on err pipe: %v", err)
	}
	if err := cmd.Wait(); err != nil {
		msgString := msg.String()
		if msgString == "" {
			msgString = outPut.String()
		}
		return fmt.Errorf("error while executing compile command %s. Error: %s: %s\n", cmd, err, msgString)
	}
	return nil
}

func init() {
	log = logging.GetLogger("assets")
}
<|endoftext|>