<commit_before>package secrets

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/containers/buildah/pkg/umask"
	"github.com/containers/storage/pkg/idtools"
	rspec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/selinux/go-selinux/label"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

var (
	// DefaultMountsFile holds the default mount paths in the form
	// "host_path:container_path"
	DefaultMountsFile = "/usr/share/containers/mounts.conf"
	// OverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the user
	OverrideMountsFile = "/etc/containers/mounts.conf"
	// UserOverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the rootless user
	UserOverrideMountsFile = filepath.Join(os.Getenv("HOME"), ".config/containers/mounts.conf")
)

// secretData stores the name of the file and the content read from it
type secretData struct {
	name    string
	data    []byte
	mode    os.FileMode
	dirMode os.FileMode
}

// saveTo saves secret data to given directory
func (s secretData) saveTo(dir string) error {
	path := filepath.Join(dir, s.name)
	if err := os.MkdirAll(filepath.Dir(path), s.dirMode); err != nil && !os.IsExist(err) {
		return err
	}
	return ioutil.WriteFile(path, s.data, s.mode)
}

func readAll(root, prefix string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, prefix)

	data := []secretData{}

	files, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return data, nil
		}

		return nil, err
	}

	for _, f := range files {
		fileData, err := readFileOrDir(root, filepath.Join(prefix, f.Name()), parentMode)
		if err != nil {
			// If the file did not exist, might be a dangling symlink
			// Ignore the error
			if os.IsNotExist(err) {
				continue
			}
			return nil, err
		}
		data = append(data, fileData...)
	}

	return data, nil
}

func readFileOrDir(root, name string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, name)

	s, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if s.IsDir() {
		dirData, err := readAll(root, name, s.Mode())
		if err != nil {
			return nil, err
		}
		return dirData, nil
	}
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return []secretData{{
		name:    name,
		data:    bytes,
		mode:    s.Mode(),
		dirMode: parentMode,
	}}, nil
}

func getHostSecretData(hostDir string, mode os.FileMode) ([]secretData, error) {
	var allSecrets []secretData
	hostSecrets, err := readAll(hostDir, "", mode)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to read secrets from %q", hostDir)
	}
	return append(allSecrets, hostSecrets...), nil
}

func getMounts(filePath string) []string {
	file, err := os.Open(filePath)
	if err != nil {
		// This is expected on most systems
		logrus.Debugf("file %q not found, skipping...", filePath)
		return nil
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	if err = scanner.Err(); err != nil {
		logrus.Errorf("error reading file %q, %v skipping...", filePath, err)
		return nil
	}
	var mounts []string
	for scanner.Scan() {
		if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") {
			mounts = append(mounts, scanner.Text())
		} else {
			logrus.Debugf("skipping unrecognized mount in %v: %q",
				filePath, scanner.Text())
		}
	}
	return mounts
}

// getHostAndCtrDir separates the host:container paths
func getMountsMap(path string) (string, string, error) {
	arr := strings.SplitN(path, ":", 2)
	switch len(arr) {
	case 1:
		return arr[0], arr[0], nil
	case 2:
		return arr[0], arr[1], nil
	}
	return "", "", errors.Errorf("unable to get host and container dir from path: %s", path)
}

// SecretMounts copies, adds, and mounts the secrets to the container root filesystem
// Deprecated, Please use SecretMountWithUIDGID
func SecretMounts(mountLabel, containerWorkingDir, mountFile string, rootless, disableFips bool) []rspec.Mount {
	return SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, containerWorkingDir, 0, 0, rootless, disableFips)
}

// SecretMountsWithUIDGID copies, adds, and mounts the secrets to the container root filesystem
// mountLabel: MAC/SELinux label for container content
// containerWorkingDir: Private data for storing secrets on the host mounted in container.
// mountFile: Additional mount points required for the container.
// mountPoint: Container image mountpoint
// uid: to assign to content created for secrets
// gid: to assign to content created for secrets
// rootless: indicates whether container is running in rootless mode
// disableFips: indicates whether system should ignore fips mode
func SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, mountPoint string, uid, gid int, rootless, disableFips bool) []rspec.Mount {
	var (
		secretMounts []rspec.Mount
		mountFiles   []string
	)
	// Add secrets from paths given in the mounts.conf files
	// mountFile will have a value if the hidden --default-mounts-file flag is set
	// Note for testing purposes only
	if mountFile == "" {
		mountFiles = append(mountFiles, []string{OverrideMountsFile, DefaultMountsFile}...)
		if rootless {
			mountFiles = append([]string{UserOverrideMountsFile}, mountFiles...)
		}
	} else {
		mountFiles = append(mountFiles, mountFile)
	}
	for _, file := range mountFiles {
		if _, err := os.Stat(file); err == nil {
			mounts, err := addSecretsFromMountsFile(file, mountLabel, containerWorkingDir, uid, gid)
			if err != nil {
				logrus.Warnf("error mounting secrets, skipping entry in %s: %v", file, err)
			}
			secretMounts = mounts
			break
		}
	}

	// Only add FIPS secret mount if disableFips=false
	if disableFips {
		return secretMounts
	}
	// Add FIPS mode secret if /etc/system-fips exists on the host
	_, err := os.Stat("/etc/system-fips")
	if err == nil {
		if err := addFIPSModeSecret(&secretMounts, containerWorkingDir, mountPoint, mountLabel, uid, gid); err != nil {
			logrus.Errorf("error adding FIPS mode secret to container: %v", err)
		}
	} else if os.IsNotExist(err) {
		logrus.Debug("/etc/system-fips does not exist on host, not mounting FIPS mode secret")
	} else {
		logrus.Errorf("stat /etc/system-fips failed for FIPS mode secret: %v", err)
	}
	return secretMounts
}

func rchown(chowndir string, uid, gid int) error {
	return filepath.Walk(chowndir, func(filePath string, f os.FileInfo, err error) error {
		return os.Lchown(filePath, uid, gid)
	})
}

// addSecretsFromMountsFile copies the contents of host directory to container directory
// and returns a list of mounts
func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir string, uid, gid int) ([]rspec.Mount, error) {
	var mounts []rspec.Mount
	defaultMountsPaths := getMounts(filePath)
	for _, path := range defaultMountsPaths {
		hostDirOrFile, ctrDirOrFile, err := getMountsMap(path)
		if err != nil {
			return nil, err
		}
		// skip if the hostDirOrFile path doesn't exist
		fileInfo, err := os.Stat(hostDirOrFile)
		if err != nil {
			if os.IsNotExist(err) {
				logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath)
				continue
			}
			return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile)
		}

		ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile)

		// In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost
		_, err = os.Stat(ctrDirOrFileOnHost)
		if os.IsNotExist(err) {

			hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile)
			if err != nil {
				return nil, err
			}

			// Don't let the umask have any influence on the file and directory creation
			oldUmask := umask.SetUmask(0)
			defer umask.SetUmask(oldUmask)

			switch mode := fileInfo.Mode(); {
			case mode.IsDir():
				if err = os.MkdirAll(ctrDirOrFileOnHost, mode.Perm()); err != nil {
					return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost)
				}
				data, err := getHostSecretData(hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "getting host secret data failed")
				}
				for _, s := range data {
					if err := s.saveTo(ctrDirOrFileOnHost); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			case mode.IsRegular():
				data, err := readFileOrDir("", hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile)

				}
				for _, s := range data {
					if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), s.dirMode); err != nil {
						return nil, err
					}
					if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, s.mode); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			default:
				return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile)
			}

			err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false)
			if err != nil {
				return nil, errors.Wrap(err, "error applying correct labels")
			}
			if uid != 0 || gid != 0 {
				if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil {
					return nil, err
				}
			}
		} else if err != nil {
			return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost)
		}

		m := rspec.Mount{
			Source:      ctrDirOrFileOnHost,
			Destination: ctrDirOrFile,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}

		mounts = append(mounts, m)
	}
	return mounts, nil
}

// addFIPSModeSecret creates /run/secrets/system-fips in the container
// root filesystem if /etc/system-fips exists on hosts.
// This enables the container to be FIPS compliant and run openssl in
// FIPS mode as the host is also in FIPS mode.
func addFIPSModeSecret(mounts *[]rspec.Mount, containerWorkingDir, mountPoint, mountLabel string, uid, gid int) error {
	secretsDir := "/run/secrets"
	ctrDirOnHost := filepath.Join(containerWorkingDir, secretsDir)
	if _, err := os.Stat(ctrDirOnHost); os.IsNotExist(err) {
		if err = idtools.MkdirAllAs(ctrDirOnHost, 0755, uid, gid); err != nil {
			return errors.Wrapf(err, "making container directory %q on host failed", ctrDirOnHost)
		}
		if err = label.Relabel(ctrDirOnHost, mountLabel, false); err != nil {
			return errors.Wrapf(err, "error applying correct labels on %q", ctrDirOnHost)
		}
	}
	fipsFile := filepath.Join(ctrDirOnHost, "system-fips")
	// In the event of restart, it is possible for the FIPS mode file to already exist
	if _, err := os.Stat(fipsFile); os.IsNotExist(err) {
		file, err := os.Create(fipsFile)
		if err != nil {
			return errors.Wrapf(err, "error creating system-fips file in container for FIPS mode")
		}
		defer file.Close()
	}

	if !mountExists(*mounts, secretsDir) {
		m := rspec.Mount{
			Source:      ctrDirOnHost,
			Destination: secretsDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}

	srcBackendDir := "/usr/share/crypto-policies/back-ends/FIPS"
	destDir := "/etc/crypto-policies/back-ends"
	srcOnHost := filepath.Join(mountPoint, srcBackendDir)
	if _, err := os.Stat(srcOnHost); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrapf(err, "failed to stat FIPS Backend directory %q", ctrDirOnHost)
	}

	if !mountExists(*mounts, destDir) {
		m := rspec.Mount{
			Source:      srcOnHost,
			Destination: destDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}
	return nil
}

// mountExists checks if a mount already exists in the spec
func mountExists(mounts []rspec.Mount, dest string) bool {
	for _, mount := range mounts {
		if mount.Destination == dest {
			return true
		}
	}
	return false
}

// resolveSymbolicLink resolves a possbile symlink path. If the path is a symlink, returns resolved
// path; if not, returns the original path.
func resolveSymbolicLink(path string) (string, error) {
	info, err := os.Lstat(path)
	if err != nil {
		return "", err
	}
	if info.Mode()&os.ModeSymlink != os.ModeSymlink {
		return path, nil
	}
	return filepath.EvalSymlinks(path)
}
<commit_msg>Fix fips-mode check for RHEL8 boxes<commit_after>package secrets

import (
	"bufio"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/containers/buildah/pkg/umask"
	"github.com/containers/storage/pkg/idtools"
	rspec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/opencontainers/selinux/go-selinux/label"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

var (
	// DefaultMountsFile holds the default mount paths in the form
	// "host_path:container_path"
	DefaultMountsFile = "/usr/share/containers/mounts.conf"
	// OverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the user
	OverrideMountsFile = "/etc/containers/mounts.conf"
	// UserOverrideMountsFile holds the default mount paths in the form
	// "host_path:container_path" overridden by the rootless user
	UserOverrideMountsFile = filepath.Join(os.Getenv("HOME"), ".config/containers/mounts.conf")
)

// secretData stores the name of the file and the content read from it
type secretData struct {
	name    string
	data    []byte
	mode    os.FileMode
	dirMode os.FileMode
}

// saveTo saves secret data to given directory
func (s secretData) saveTo(dir string) error {
	path := filepath.Join(dir, s.name)
	if err := os.MkdirAll(filepath.Dir(path), s.dirMode); err != nil && !os.IsExist(err) {
		return err
	}
	return ioutil.WriteFile(path, s.data, s.mode)
}

func readAll(root, prefix string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, prefix)

	data := []secretData{}

	files, err := ioutil.ReadDir(path)
	if err != nil {
		if os.IsNotExist(err) {
			return data, nil
		}

		return nil, err
	}

	for _, f := range files {
		fileData, err := readFileOrDir(root, filepath.Join(prefix, f.Name()), parentMode)
		if err != nil {
			// If the file did not exist, might be a dangling symlink
			// Ignore the error
			if os.IsNotExist(err) {
				continue
			}
			return nil, err
		}
		data = append(data, fileData...)
	}

	return data, nil
}

func readFileOrDir(root, name string, parentMode os.FileMode) ([]secretData, error) {
	path := filepath.Join(root, name)

	s, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	if s.IsDir() {
		dirData, err := readAll(root, name, s.Mode())
		if err != nil {
			return nil, err
		}
		return dirData, nil
	}
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	return []secretData{{
		name:    name,
		data:    bytes,
		mode:    s.Mode(),
		dirMode: parentMode,
	}}, nil
}

func getHostSecretData(hostDir string, mode os.FileMode) ([]secretData, error) {
	var allSecrets []secretData
	hostSecrets, err := readAll(hostDir, "", mode)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to read secrets from %q", hostDir)
	}
	return append(allSecrets, hostSecrets...), nil
}

func getMounts(filePath string) []string {
	file, err := os.Open(filePath)
	if err != nil {
		// This is expected on most systems
		logrus.Debugf("file %q not found, skipping...", filePath)
		return nil
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	if err = scanner.Err(); err != nil {
		logrus.Errorf("error reading file %q, %v skipping...", filePath, err)
		return nil
	}
	var mounts []string
	for scanner.Scan() {
		if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") {
			mounts = append(mounts, scanner.Text())
		} else {
			logrus.Debugf("skipping unrecognized mount in %v: %q",
				filePath, scanner.Text())
		}
	}
	return mounts
}

// getHostAndCtrDir separates the host:container paths
func getMountsMap(path string) (string, string, error) {
	arr := strings.SplitN(path, ":", 2)
	switch len(arr) {
	case 1:
		return arr[0], arr[0], nil
	case 2:
		return arr[0], arr[1], nil
	}
	return "", "", errors.Errorf("unable to get host and container dir from path: %s", path)
}

// SecretMounts copies, adds, and mounts the secrets to the container root filesystem
// Deprecated, Please use SecretMountWithUIDGID
func SecretMounts(mountLabel, containerWorkingDir, mountFile string, rootless, disableFips bool) []rspec.Mount {
	return SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, containerWorkingDir, 0, 0, rootless, disableFips)
}

// SecretMountsWithUIDGID copies, adds, and mounts the secrets to the container root filesystem
// mountLabel: MAC/SELinux label for container content
// containerWorkingDir: Private data for storing secrets on the host mounted in container.
// mountFile: Additional mount points required for the container.
// mountPoint: Container image mountpoint
// uid: to assign to content created for secrets
// gid: to assign to content created for secrets
// rootless: indicates whether container is running in rootless mode
// disableFips: indicates whether system should ignore fips mode
func SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, mountPoint string, uid, gid int, rootless, disableFips bool) []rspec.Mount {
	var (
		secretMounts []rspec.Mount
		mountFiles   []string
	)
	// Add secrets from paths given in the mounts.conf files
	// mountFile will have a value if the hidden --default-mounts-file flag is set
	// Note for testing purposes only
	if mountFile == "" {
		mountFiles = append(mountFiles, []string{OverrideMountsFile, DefaultMountsFile}...)
		if rootless {
			mountFiles = append([]string{UserOverrideMountsFile}, mountFiles...)
		}
	} else {
		mountFiles = append(mountFiles, mountFile)
	}
	for _, file := range mountFiles {
		if _, err := os.Stat(file); err == nil {
			mounts, err := addSecretsFromMountsFile(file, mountLabel, containerWorkingDir, uid, gid)
			if err != nil {
				logrus.Warnf("error mounting secrets, skipping entry in %s: %v", file, err)
			}
			secretMounts = mounts
			break
		}
	}

	// Only add FIPS secret mount if disableFips=false
	if disableFips {
		return secretMounts
	}
	// Add FIPS mode secret if /etc/system-fips exists on the host
	_, err := os.Stat("/etc/system-fips")
	if err == nil {
		if err := addFIPSModeSecret(&secretMounts, containerWorkingDir, mountPoint, mountLabel, uid, gid); err != nil {
			logrus.Errorf("error adding FIPS mode secret to container: %v", err)
		}
	} else if os.IsNotExist(err) {
		logrus.Debug("/etc/system-fips does not exist on host, not mounting FIPS mode secret")
	} else {
		logrus.Errorf("stat /etc/system-fips failed for FIPS mode secret: %v", err)
	}
	return secretMounts
}

func rchown(chowndir string, uid, gid int) error {
	return filepath.Walk(chowndir, func(filePath string, f os.FileInfo, err error) error {
		return os.Lchown(filePath, uid, gid)
	})
}

// addSecretsFromMountsFile copies the contents of host directory to container directory
// and returns a list of mounts
func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir string, uid, gid int) ([]rspec.Mount, error) {
	var mounts []rspec.Mount
	defaultMountsPaths := getMounts(filePath)
	for _, path := range defaultMountsPaths {
		hostDirOrFile, ctrDirOrFile, err := getMountsMap(path)
		if err != nil {
			return nil, err
		}
		// skip if the hostDirOrFile path doesn't exist
		fileInfo, err := os.Stat(hostDirOrFile)
		if err != nil {
			if os.IsNotExist(err) {
				logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath)
				continue
			}
			return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile)
		}

		ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile)

		// In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost
		_, err = os.Stat(ctrDirOrFileOnHost)
		if os.IsNotExist(err) {

			hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile)
			if err != nil {
				return nil, err
			}

			// Don't let the umask have any influence on the file and directory creation
			oldUmask := umask.SetUmask(0)
			defer umask.SetUmask(oldUmask)

			switch mode := fileInfo.Mode(); {
			case mode.IsDir():
				if err = os.MkdirAll(ctrDirOrFileOnHost, mode.Perm()); err != nil {
					return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost)
				}
				data, err := getHostSecretData(hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "getting host secret data failed")
				}
				for _, s := range data {
					if err := s.saveTo(ctrDirOrFileOnHost); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			case mode.IsRegular():
				data, err := readFileOrDir("", hostDirOrFile, mode.Perm())
				if err != nil {
					return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile)

				}
				for _, s := range data {
					if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), s.dirMode); err != nil {
						return nil, err
					}
					if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, s.mode); err != nil {
						return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost)
					}
				}
			default:
				return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile)
			}

			err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false)
			if err != nil {
				return nil, errors.Wrap(err, "error applying correct labels")
			}
			if uid != 0 || gid != 0 {
				if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil {
					return nil, err
				}
			}
		} else if err != nil {
			return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost)
		}

		m := rspec.Mount{
			Source:      ctrDirOrFileOnHost,
			Destination: ctrDirOrFile,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}

		mounts = append(mounts, m)
	}
	return mounts, nil
}

// addFIPSModeSecret creates /run/secrets/system-fips in the container
// root filesystem if /etc/system-fips exists on hosts.
// This enables the container to be FIPS compliant and run openssl in
// FIPS mode as the host is also in FIPS mode.
func addFIPSModeSecret(mounts *[]rspec.Mount, containerWorkingDir, mountPoint, mountLabel string, uid, gid int) error {
	secretsDir := "/run/secrets"
	ctrDirOnHost := filepath.Join(containerWorkingDir, secretsDir)
	if _, err := os.Stat(ctrDirOnHost); os.IsNotExist(err) {
		if err = idtools.MkdirAllAs(ctrDirOnHost, 0755, uid, gid); err != nil {
			return errors.Wrapf(err, "making container directory %q on host failed", ctrDirOnHost)
		}
		if err = label.Relabel(ctrDirOnHost, mountLabel, false); err != nil {
			return errors.Wrapf(err, "error applying correct labels on %q", ctrDirOnHost)
		}
	}
	fipsFile := filepath.Join(ctrDirOnHost, "system-fips")
	// In the event of restart, it is possible for the FIPS mode file to already exist
	if _, err := os.Stat(fipsFile); os.IsNotExist(err) {
		file, err := os.Create(fipsFile)
		if err != nil {
			return errors.Wrapf(err, "error creating system-fips file in container for FIPS mode")
		}
		defer file.Close()
	}

	if !mountExists(*mounts, secretsDir) {
		m := rspec.Mount{
			Source:      ctrDirOnHost,
			Destination: secretsDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}

	srcBackendDir := "/usr/share/crypto-policies/FIPS"
	destDir := "/etc/crypto-policies/back-ends"
	srcOnHost := filepath.Join(mountPoint, srcBackendDir)
	if _, err := os.Stat(srcOnHost); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errors.Wrapf(err, "failed to stat FIPS Backend directory %q", ctrDirOnHost)
	}

	if !mountExists(*mounts, destDir) {
		m := rspec.Mount{
			Source:      srcOnHost,
			Destination: destDir,
			Type:        "bind",
			Options:     []string{"bind", "rprivate"},
		}
		*mounts = append(*mounts, m)
	}
	return nil
}

// mountExists checks if a mount already exists in the spec
func mountExists(mounts []rspec.Mount, dest string) bool {
	for _, mount := range mounts {
		if mount.Destination == dest {
			return true
		}
	}
	return false
}

// resolveSymbolicLink resolves a possbile symlink path. If the path is a symlink, returns resolved
// path; if not, returns the original path.
func resolveSymbolicLink(path string) (string, error) {
	info, err := os.Lstat(path)
	if err != nil {
		return "", err
	}
	if info.Mode()&os.ModeSymlink != os.ModeSymlink {
		return path, nil
	}
	return filepath.EvalSymlinks(path)
}
<|endoftext|>


<commit_before>// Copyright 2019 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"sqlflow.org/sqlflow/pkg/argo"
	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/parser"
	"sqlflow.org/sqlflow/pkg/pipe"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/couler"
	"sqlflow.org/sqlflow/pkg/sql/ir"
	"sqlflow.org/sqlflow/pkg/verifier"
)

// EndOfExecution will push to the pipe when one SQL statement execution is finished.
type EndOfExecution struct {
	StartTime int64
	EndTime   int64
	Statement string
}

// WorkflowJob indicates the Argo Workflow ID
// FIXME(tony): reuse workflow job definition in proto package
type WorkflowJob struct {
	JobID string
}

// RunSQLProgram run a SQL program.
//
// TODO(wangkuiyi): Make RunSQLProgram return an error in addition to
// *pipe.Reader, and remove the calls to log.Printf.
func RunSQLProgram(sqlProgram string, modelDir string, session *pb.Session) *pipe.Reader {
	rd, wr := pipe.Pipe()
	go func() {
		var db *database.DB
		var err error
		defer wr.Close()
		if db, err = database.OpenAndConnectDB(session.DbConnStr); err != nil {
			wr.Write(fmt.Errorf("create DB failed: %v", err))
			log.Printf("create DB failed: %v", err)
			return
		}
		err = runSQLProgram(wr, sqlProgram, db, modelDir, session)
		if err != nil {
			log.Printf("runSQLProgram error: %v", err)
			if err != pipe.ErrClosedPipe {
				if err := wr.Write(err); err != nil {
					log.Printf("runSQLProgram error(piping): %v", err)
				}
			}
		}
	}()
	return rd
}

// SubmitWorkflow submits an Argo workflow
//
// TODO(wangkuiyi): Make RunSQLProgram return an error in addition to
// *pipe.Reader, and remove the calls to log.Printf.
func SubmitWorkflow(sqlProgram string, modelDir string, session *pb.Session) *pipe.Reader {
	rd, wr := pipe.Pipe()
	go func() {
		defer wr.Close()
		err := submitWorkflow(wr, sqlProgram, modelDir, session)
		if err != nil {
			if err != pipe.ErrClosedPipe {
				if err := wr.Write(err); err != nil {
					log.Printf("submit workflow error(piping): %v", err)
				}
			}
		}
	}()
	return rd
}

func submitWorkflow(wr *pipe.Writer, sqlProgram string, modelDir string, session *pb.Session) error {
	driverName, _, err := database.ParseURL(session.DbConnStr)
	if err != nil {
		return err
	}
	sqls, err := parser.Parse(driverName, sqlProgram)
	if err != nil {
		return err
	}
	// TODO(yancey1989): separate the IR generation to multiple steps:
	// For example, a TRAIN statement:
	// 		SELECT ... TO TRAIN ...
	// the multiple ir generator steps pipeline can be:
	// sql -> parsed result -> infer columns -> load train ir from saved model ..
	spIRs := []ir.SQLStatement{}
	for _, sql := range sqls {
		var r ir.SQLStatement
		if parser.IsExtendedSyntax(sql) {
			if sql.Train {
				r, err = generateTrainStmt(sql.SQLFlowSelectStmt)
			} else if sql.Explain {
				// since getTrainStmtFromModel is false, use empty cwd is fine.
				r, err = generateExplainStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, "", false)
			} else {
				r, err = generatePredictStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, "", false)
			}
		} else {
			standardSQL := ir.StandardSQL(sql.Original)
			r = &standardSQL
		}
		if err != nil {
			return err
		}
		r.SetOriginalSQL(sql.Original)
		spIRs = append(spIRs, r)
	}

	// 1. call codegen_couler.go to generate Argo workflow YAML
	argoFileName, err := couler.RunAndWriteArgoFile(spIRs, session)
	if err != nil {
		return err
	}
	defer os.RemoveAll(argoFileName)

	// 2. submit the argo workflow
	workflowID, err := argo.Submit(argoFileName)
	if err != nil {
		return err
	}

	return wr.Write(WorkflowJob{
		JobID: workflowID,
	})
}

func runSQLProgram(wr *pipe.Writer, sqlProgram string, db *database.DB, modelDir string, session *pb.Session) error {
	sqls, err := parser.Parse(db.DriverName, sqlProgram)
	if err != nil {
		return err
	}
	// NOTE(tony): We generate IR and execute its translated program one-by-one since IR generation may depend on the execution
	// of the previous statement. For example, consider a SQL program
	//
	//   create table some_table as (select ...);
	//   select * from some_table to train ...
	//
	// The IR generation on the second statement would fail since it requires inspection the schema of some_table,
	// which depends on the execution of create table some_table as (select ...);.
	for _, sql := range sqls {
		cwd, err := ioutil.TempDir("/tmp", "sqlflow_models")
		if err != nil {
			return err
		}
		// NOTE(typhoonzero): must call "cleanCwd" when end processing current SQL or before
		// returning error, we can not use "defer" because if we have many SQL statements in
		// the SQL program, we may overflow the defer stack.
		// For more information: https://blog.learngoprogramming.com/gotchas-of-defer-in-go-1-8d070894cb01
		cleanCwd := func(cwd string) error {
			return os.RemoveAll(cwd)
		}
		var r ir.SQLStatement
		if parser.IsExtendedSyntax(sql) {
			if sql.Train {
				r, err = generateTrainStmtWithInferredColumns(sql.SQLFlowSelectStmt, session.DbConnStr, true)
			} else if sql.Explain {
				r, err = generateExplainStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, GetSubmitter(session.Submitter).GetTrainStmtFromModel())
			} else {
				r, err = generatePredictStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, GetSubmitter(session.Submitter).GetTrainStmtFromModel())
			}
		} else {
			standardSQL := ir.StandardSQL(sql.Original)
			r = &standardSQL
		}

		if err != nil {
			if e := cleanCwd(cwd); e != nil {
				return fmt.Errorf("encounter %v when dealwith error: %s", e, err)
			}
			return err
		}
		r.SetOriginalSQL(sql.Original)
		if err := runSingleSQLIR(wr, r, db, modelDir, cwd, session); err != nil {
			if e := cleanCwd(cwd); e != nil {
				return fmt.Errorf("encounter %v when dealwith error: %s", e, err)
			}
			return err
		}
		if e := cleanCwd(cwd); e != nil {
			return fmt.Errorf("encounter %v when dealwith error: %s", e, err)
		}
	}
	return nil
}

func runSingleSQLIR(wr *pipe.Writer, sqlIR ir.SQLStatement, db *database.DB, modelDir string, cwd string, session *pb.Session) (e error) {
	startTime := time.Now().UnixNano()
	var originalSQL string
	defer func() {
		if e != nil {
			wr.Write(EndOfExecution{
				StartTime: startTime,
				EndTime:   time.Now().UnixNano(),
				Statement: originalSQL,
			})
		}
	}()
	// TODO(typhoonzero): can run feature.LogDerivationResult(wr, trainStmt) here to send
	// feature derivation logs to client, yet we disable if for now so that it's less annoying.
	GetSubmitter(session.Submitter).Setup(wr, db, modelDir, cwd, session)
	return sqlIR.Execute(GetSubmitter(session.Submitter))
}

// getColumnTypes is quiet like verify but accept a SQL string as input, and returns
// an ordered list of the field types.
func getColumnTypes(slct string, db *database.DB) ([]string, []string, error) {
	rows, err := db.Query(slct)
	if err != nil {
		return nil, nil, err
	}
	defer rows.Close()

	if !rows.Next() {
		return nil, nil, fmt.Errorf("query %s gives 0 row", slct)
	}

	if rows.Err() != nil {
		return nil, nil, err
	}

	columnTypes, err := rows.ColumnTypes()
	if err != nil {
		return nil, nil, err
	}

	ft := []string{}
	flds := []string{}
	for _, ct := range columnTypes {
		_, fld := verifier.Decomp(ct.Name())
		typeName := ct.DatabaseTypeName()
		flds = append(flds, fld)
		ft = append(ft, typeName)
	}

	return flds, ft, nil
}

// Create prediction table using the `PredictStmt`.
func createPredictionTableFromIR(predStmt *ir.PredictStmt, db *database.DB, session *pb.Session) error {
	dropStmt := fmt.Sprintf("drop table if exists %s;", predStmt.ResultTable)
	if _, e := db.Exec(dropStmt); e != nil {
		return fmt.Errorf("failed executing %s: %q", dropStmt, e)
	}
	// FIXME(typhoonzero): simply add LIMIT 1 at the end to get column types.
	tmpSQL := fmt.Sprintf("%s LIMIT 1;", strings.TrimRight(strings.TrimSpace(predStmt.Select), ";"))
	flds, fts, e := getColumnTypes(tmpSQL, db)
	if e != nil {
		return e
	}

	var b bytes.Buffer
	// NOTE(typhoonzero): predStmt.TrainStmt may be nil, because the model may not loaded when
	// creating prediction table.
	trainLabelColumn := ""
	if predStmt.TrainStmt != nil {
		trainLabelColumn = predStmt.TrainStmt.Label.GetFieldDesc()[0].Name
	}
	labelColumnName := predStmt.ResultColumn
	labelColumnType := ""
	fmt.Fprintf(&b, "create table %s (", predStmt.ResultTable)
	for idx, colType := range fts {
		stype, e := fieldType(db.DriverName, colType)
		if e != nil {
			return e
		}
		fldName := flds[idx]
		// When predicting use validation table, we should find the label column type
		// using the label column name from train table.
		if fldName == labelColumnName || fldName == trainLabelColumn {
			labelColumnType = stype
			continue
		}
		fmt.Fprintf(&b, "%s %s, ", fldName, stype)
	}

	// TODO(Yancey1989): For the current implementation, the prediction result column
	// type is derivated by the pred-select-statement, the better way is derivating
	// the result column type by the prediction result.
	//
	// label column not found in predict table, create a column specified by PREDICT clause:
	if labelColumnType == "" {
		// NOTE(typhoonzero): Clustering model may not have label in select statement, default use INT type
		labelColumnType = "INT"
	}
	stype, e := fieldType(db.DriverName, labelColumnType)
	if e != nil {
		return e
	}
	if db.DriverName == "hive" {
		fmt.Fprintf(&b, "%s %s) ROW FORMAT DELIMITED FIELDS TERMINATED BY \"\\001\" STORED AS TEXTFILE;", labelColumnName, stype)
	} else {
		fmt.Fprintf(&b, "%s %s);", labelColumnName, stype)
	}

	createStmt := b.String()
	if _, e := db.Exec(createStmt); e != nil {
		return fmt.Errorf("failed executing %s: %q", createStmt, e)
	}
	return nil
}
<commit_msg>submit workflow logs (#1707)<commit_after>// Copyright 2019 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"sqlflow.org/sqlflow/pkg/argo"
	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/parser"
	"sqlflow.org/sqlflow/pkg/pipe"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/couler"
	"sqlflow.org/sqlflow/pkg/sql/ir"
	"sqlflow.org/sqlflow/pkg/verifier"
)

// EndOfExecution will push to the pipe when one SQL statement execution is finished.
type EndOfExecution struct {
	StartTime int64
	EndTime   int64
	Statement string
}

// WorkflowJob indicates the Argo Workflow ID
// FIXME(tony): reuse workflow job definition in proto package
type WorkflowJob struct {
	JobID string
}

// RunSQLProgram run a SQL program.
//
// TODO(wangkuiyi): Make RunSQLProgram return an error in addition to
// *pipe.Reader, and remove the calls to log.Printf.
func RunSQLProgram(sqlProgram string, modelDir string, session *pb.Session) *pipe.Reader {
	rd, wr := pipe.Pipe()
	go func() {
		var db *database.DB
		var err error
		defer wr.Close()
		if db, err = database.OpenAndConnectDB(session.DbConnStr); err != nil {
			wr.Write(fmt.Errorf("create DB failed: %v", err))
			log.Printf("create DB failed: %v", err)
			return
		}
		err = runSQLProgram(wr, sqlProgram, db, modelDir, session)
		if err != nil {
			log.Printf("runSQLProgram error: %v", err)
			if err != pipe.ErrClosedPipe {
				if err := wr.Write(err); err != nil {
					log.Printf("runSQLProgram error(piping): %v", err)
				}
			}
		}
	}()
	return rd
}

// SubmitWorkflow submits an Argo workflow
//
// TODO(wangkuiyi): Make RunSQLProgram return an error in addition to
// *pipe.Reader, and remove the calls to log.Printf.
func SubmitWorkflow(sqlProgram string, modelDir string, session *pb.Session) *pipe.Reader {
	rd, wr := pipe.Pipe()
	startTime := time.Now().Second()
	go func() {
		defer wr.Close()
		wfID, err := submitWorkflow(wr, sqlProgram, modelDir, session)
		defer log.Printf("Submit SQL program: %s\nuserID: %s\nworkflowID: %s\nspent: %d\nerror:%v", sqlProgram, session.UserId, wfID, time.Now().Second()-startTime, err)
		if err != nil && err != pipe.ErrClosedPipe {
			if err := wr.Write(err); err != nil {
				log.Printf("submit workflow error(piping): %v", err)
			}
		}

	}()
	return rd
}

func submitWorkflow(wr *pipe.Writer, sqlProgram string, modelDir string, session *pb.Session) (string, error) {
	driverName, _, err := database.ParseURL(session.DbConnStr)
	if err != nil {
		return "", err
	}
	sqls, err := parser.Parse(driverName, sqlProgram)
	if err != nil {
		return "", err
	}
	// TODO(yancey1989): separate the IR generation to multiple steps:
	// For example, a TRAIN statement:
	// 		SELECT ... TO TRAIN ...
	// the multiple ir generator steps pipeline can be:
	// sql -> parsed result -> infer columns -> load train ir from saved model ..
	spIRs := []ir.SQLStatement{}
	for _, sql := range sqls {
		var r ir.SQLStatement
		if parser.IsExtendedSyntax(sql) {
			if sql.Train {
				r, err = generateTrainStmt(sql.SQLFlowSelectStmt)
			} else if sql.Explain {
				// since getTrainStmtFromModel is false, use empty cwd is fine.
				r, err = generateExplainStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, "", false)
			} else {
				r, err = generatePredictStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, "", false)
			}
		} else {
			standardSQL := ir.StandardSQL(sql.Original)
			r = &standardSQL
		}
		if err != nil {
			return "", err
		}
		r.SetOriginalSQL(sql.Original)
		spIRs = append(spIRs, r)
	}

	// 1. call codegen_couler.go to generate Argo workflow YAML
	argoFileName, err := couler.RunAndWriteArgoFile(spIRs, session)
	if err != nil {
		return "", err
	}
	defer os.RemoveAll(argoFileName)

	// 2. submit the argo workflow
	workflowID, err := argo.Submit(argoFileName)
	if err != nil {
		return "", err
	}

	return workflowID, wr.Write(WorkflowJob{
		JobID: workflowID,
	})
}

func runSQLProgram(wr *pipe.Writer, sqlProgram string, db *database.DB, modelDir string, session *pb.Session) error {
	sqls, err := parser.Parse(db.DriverName, sqlProgram)
	if err != nil {
		return err
	}
	// NOTE(tony): We generate IR and execute its translated program one-by-one since IR generation may depend on the execution
	// of the previous statement. For example, consider a SQL program
	//
	//   create table some_table as (select ...);
	//   select * from some_table to train ...
	//
	// The IR generation on the second statement would fail since it requires inspection the schema of some_table,
	// which depends on the execution of create table some_table as (select ...);.
	for _, sql := range sqls {
		cwd, err := ioutil.TempDir("/tmp", "sqlflow_models")
		if err != nil {
			return err
		}
		// NOTE(typhoonzero): must call "cleanCwd" when end processing current SQL or before
		// returning error, we can not use "defer" because if we have many SQL statements in
		// the SQL program, we may overflow the defer stack.
		// For more information: https://blog.learngoprogramming.com/gotchas-of-defer-in-go-1-8d070894cb01
		cleanCwd := func(cwd string) error {
			return os.RemoveAll(cwd)
		}
		var r ir.SQLStatement
		if parser.IsExtendedSyntax(sql) {
			if sql.Train {
				r, err = generateTrainStmtWithInferredColumns(sql.SQLFlowSelectStmt, session.DbConnStr, true)
			} else if sql.Explain {
				r, err = generateExplainStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, GetSubmitter(session.Submitter).GetTrainStmtFromModel())
			} else {
				r, err = generatePredictStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, GetSubmitter(session.Submitter).GetTrainStmtFromModel())
			}
		} else {
			standardSQL := ir.StandardSQL(sql.Original)
			r = &standardSQL
		}

		if err != nil {
			if e := cleanCwd(cwd); e != nil {
				return fmt.Errorf("encounter %v when dealwith error: %s", e, err)
			}
			return err
		}
		r.SetOriginalSQL(sql.Original)
		if err := runSingleSQLIR(wr, r, db, modelDir, cwd, session); err != nil {
			if e := cleanCwd(cwd); e != nil {
				return fmt.Errorf("encounter %v when dealwith error: %s", e, err)
			}
			return err
		}
		if e := cleanCwd(cwd); e != nil {
			return fmt.Errorf("encounter %v when dealwith error: %s", e, err)
		}
	}
	return nil
}

func runSingleSQLIR(wr *pipe.Writer, sqlIR ir.SQLStatement, db *database.DB, modelDir string, cwd string, session *pb.Session) (e error) {
	startTime := time.Now().UnixNano()
	var originalSQL string
	defer func() {
		if e != nil {
			wr.Write(EndOfExecution{
				StartTime: startTime,
				EndTime:   time.Now().UnixNano(),
				Statement: originalSQL,
			})
		}
	}()
	// TODO(typhoonzero): can run feature.LogDerivationResult(wr, trainStmt) here to send
	// feature derivation logs to client, yet we disable if for now so that it's less annoying.
	GetSubmitter(session.Submitter).Setup(wr, db, modelDir, cwd, session)
	return sqlIR.Execute(GetSubmitter(session.Submitter))
}

// getColumnTypes is quiet like verify but accept a SQL string as input, and returns
// an ordered list of the field types.
func getColumnTypes(slct string, db *database.DB) ([]string, []string, error) {
	rows, err := db.Query(slct)
	if err != nil {
		return nil, nil, err
	}
	defer rows.Close()

	if !rows.Next() {
		return nil, nil, fmt.Errorf("query %s gives 0 row", slct)
	}

	if rows.Err() != nil {
		return nil, nil, err
	}

	columnTypes, err := rows.ColumnTypes()
	if err != nil {
		return nil, nil, err
	}

	ft := []string{}
	flds := []string{}
	for _, ct := range columnTypes {
		_, fld := verifier.Decomp(ct.Name())
		typeName := ct.DatabaseTypeName()
		flds = append(flds, fld)
		ft = append(ft, typeName)
	}

	return flds, ft, nil
}

// Create prediction table using the `PredictStmt`.
func createPredictionTableFromIR(predStmt *ir.PredictStmt, db *database.DB, session *pb.Session) error {
	dropStmt := fmt.Sprintf("drop table if exists %s;", predStmt.ResultTable)
	if _, e := db.Exec(dropStmt); e != nil {
		return fmt.Errorf("failed executing %s: %q", dropStmt, e)
	}
	// FIXME(typhoonzero): simply add LIMIT 1 at the end to get column types.
	tmpSQL := fmt.Sprintf("%s LIMIT 1;", strings.TrimRight(strings.TrimSpace(predStmt.Select), ";"))
	flds, fts, e := getColumnTypes(tmpSQL, db)
	if e != nil {
		return e
	}

	var b bytes.Buffer
	// NOTE(typhoonzero): predStmt.TrainStmt may be nil, because the model may not loaded when
	// creating prediction table.
	trainLabelColumn := ""
	if predStmt.TrainStmt != nil {
		trainLabelColumn = predStmt.TrainStmt.Label.GetFieldDesc()[0].Name
	}
	labelColumnName := predStmt.ResultColumn
	labelColumnType := ""
	fmt.Fprintf(&b, "create table %s (", predStmt.ResultTable)
	for idx, colType := range fts {
		stype, e := fieldType(db.DriverName, colType)
		if e != nil {
			return e
		}
		fldName := flds[idx]
		// When predicting use validation table, we should find the label column type
		// using the label column name from train table.
		if fldName == labelColumnName || fldName == trainLabelColumn {
			labelColumnType = stype
			continue
		}
		fmt.Fprintf(&b, "%s %s, ", fldName, stype)
	}

	// TODO(Yancey1989): For the current implementation, the prediction result column
	// type is derivated by the pred-select-statement, the better way is derivating
	// the result column type by the prediction result.
	//
	// label column not found in predict table, create a column specified by PREDICT clause:
	if labelColumnType == "" {
		// NOTE(typhoonzero): Clustering model may not have label in select statement, default use INT type
		labelColumnType = "INT"
	}
	stype, e := fieldType(db.DriverName, labelColumnType)
	if e != nil {
		return e
	}
	if db.DriverName == "hive" {
		fmt.Fprintf(&b, "%s %s) ROW FORMAT DELIMITED FIELDS TERMINATED BY \"\\001\" STORED AS TEXTFILE;", labelColumnName, stype)
	} else {
		fmt.Fprintf(&b, "%s %s);", labelColumnName, stype)
	}

	createStmt := b.String()
	if _, e := db.Exec(createStmt); e != nil {
		return fmt.Errorf("failed executing %s: %q", createStmt, e)
	}
	return nil
}
<|endoftext|>


<commit_before>package status

import (
	"context"
	"fmt"
	"sync"

	"github.com/coreos/etcd/clientv3"
	"github.com/elastic/gosigar"
	"github.com/taku-k/polymerase/pkg/storage"
	"github.com/taku-k/polymerase/pkg/base"
)

type StatusRecorder struct {
	mu sync.Mutex

	storeDir string

	cli *clientv3.Client

	storage storage.BackupStorage

	name string
}

func NewStatusRecorder(client *clientv3.Client, storeDir string, storage storage.BackupStorage, name string) *StatusRecorder {
	return &StatusRecorder{
		cli:      client,
		storeDir: storeDir,
		storage:  storage,
		name:     name,
	}
}

func (sr *StatusRecorder) WriteStatus(ctx context.Context) error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	fileSystemUsage := gosigar.FileSystemUsage{}
	if err := fileSystemUsage.Get(sr.storeDir); err != nil {
		return err
	}

	kvs := []struct{
		k string
		v string
	}{
		{k: base.DiskInfoTotalKey(sr.name), v: fmt.Sprintf("%v", fileSystemUsage.Total)},
		{k: base.DiskInfoAvailKey(sr.name), v: fmt.Sprintf("%v", fileSystemUsage.Avail)},
	}
	for _, kv := range kvs {
		_, err := sr.cli.KV.Put(sr.cli.Ctx(), kv.k, kv.v)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Apply gofmt<commit_after>package status

import (
	"context"
	"fmt"
	"sync"

	"github.com/coreos/etcd/clientv3"
	"github.com/elastic/gosigar"
	"github.com/taku-k/polymerase/pkg/base"
	"github.com/taku-k/polymerase/pkg/storage"
)

type StatusRecorder struct {
	mu sync.Mutex

	storeDir string

	cli *clientv3.Client

	storage storage.BackupStorage

	name string
}

func NewStatusRecorder(client *clientv3.Client, storeDir string, storage storage.BackupStorage, name string) *StatusRecorder {
	return &StatusRecorder{
		cli:      client,
		storeDir: storeDir,
		storage:  storage,
		name:     name,
	}
}

func (sr *StatusRecorder) WriteStatus(ctx context.Context) error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	fileSystemUsage := gosigar.FileSystemUsage{}
	if err := fileSystemUsage.Get(sr.storeDir); err != nil {
		return err
	}

	kvs := []struct {
		k string
		v string
	}{
		{k: base.DiskInfoTotalKey(sr.name), v: fmt.Sprintf("%v", fileSystemUsage.Total)},
		{k: base.DiskInfoAvailKey(sr.name), v: fmt.Sprintf("%v", fileSystemUsage.Avail)},
	}
	for _, kv := range kvs {
		_, err := sr.cli.KV.Put(sr.cli.Ctx(), kv.k, kv.v)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package store

import (
	"junta/assert"
	"testing"
)

var globs = [][]string{
	[]string{"/", `^/$`},
	[]string{"/a", `^/a$`},
	[]string{"/a.b", `^/a\.b$`},
	[]string{"/a世界", `^/a世界$`},
	[]string{"/a?", `^/a[^/]$`},
	[]string{"/a/", `^/a/$`},
	[]string{"/a/b", `^/a/b$`},
	[]string{"/*", `^/[^/]*$`},
	[]string{"/*/a", `^/[^/]*/a$`},
	[]string{"/*a/b", `^/[^/]*a/b$`},
	[]string{"/a*/b", `^/a[^/]*/b$`},
	[]string{"/a*a/b", `^/a[^/]*a/b$`},
	[]string{"/*a*/b", `^/[^/]*a[^/]*/b$`},
	[]string{"/**", `^/.*$`},
	[]string{"/**/a", `^/.*/a$`},
}

var matches = [][]string{
	[]string{"/a/b", "/a/b"},
	[]string{"/a?", "/ab", "/ac"},
}

var nonMatches = [][]string{
	[]string{"/a/b", "/a/c", "/a/", "/a/b/", "/a/bc"},
	[]string{"/a?", "/abc", "/a", "/a/"},
}

func TestGlobTranslate(t *testing.T) {
	for _, parts := range globs {
		pat, exp := parts[0], parts[1]
		got := translateGlob(pat)
		if got != exp {
			t.Errorf("expected %q, but got %q from %q", exp, got, pat)
		}
	}
}

func TestGlobMatches(t *testing.T) {
	for _, parts := range matches {
		pat, paths := parts[0], parts[1:]
		re, err := compileGlob(pat)
		assert.Equal(t, nil, err)
		for _, path := range paths {
			if !re.MatchString(path) {
				t.Errorf("pat %q should match %q", pat, path)
			}
		}
	}
}

func TestGlobNonMatches(t *testing.T) {
	for _, parts := range nonMatches {
		pat, paths := parts[0], parts[1:]
		re, err := compileGlob(pat)
		assert.Equal(t, nil, err)
		for _, path := range paths {
			if re.MatchString(path) {
				t.Errorf("pat %q should not match %q", pat, path)
			}
		}
	}
}
<commit_msg>more glob tests<commit_after>package store

import (
	"junta/assert"
	"testing"
)

var globs = [][]string{
	[]string{"/", `^/$`},
	[]string{"/a", `^/a$`},
	[]string{"/a.b", `^/a\.b$`},
	[]string{"/a世界", `^/a世界$`},
	[]string{"/a?", `^/a[^/]$`},
	[]string{"/a/", `^/a/$`},
	[]string{"/a/b", `^/a/b$`},
	[]string{"/*", `^/[^/]*$`},
	[]string{"/*/a", `^/[^/]*/a$`},
	[]string{"/*a/b", `^/[^/]*a/b$`},
	[]string{"/a*/b", `^/a[^/]*/b$`},
	[]string{"/a*a/b", `^/a[^/]*a/b$`},
	[]string{"/*a*/b", `^/[^/]*a[^/]*/b$`},
	[]string{"/**", `^/.*$`},
	[]string{"/**/a", `^/.*/a$`},
}

var matches = [][]string{
	[]string{"/a/b", "/a/b"},
	[]string{"/a?", "/ab", "/ac"},
	[]string{"/a*", "/a", "/ab", "/abc"},
	[]string{"/a**", "/a", "/ab", "/abc", "/a/", "/a/b", "/ab/c"},
}

var nonMatches = [][]string{
	[]string{"/a/b", "/a/c", "/a/", "/a/b/", "/a/bc"},
	[]string{"/a?", "/", "/abc", "/a", "/a/"},
	[]string{"/a*", "/", "/a/", "/ba"},
	[]string{"/a**", "/", "/ba"},
}

func TestGlobTranslate(t *testing.T) {
	for _, parts := range globs {
		pat, exp := parts[0], parts[1]
		got := translateGlob(pat)
		if got != exp {
			t.Errorf("expected %q, but got %q from %q", exp, got, pat)
		}
	}
}

func TestGlobMatches(t *testing.T) {
	for _, parts := range matches {
		pat, paths := parts[0], parts[1:]
		re, err := compileGlob(pat)
		assert.Equal(t, nil, err)
		for _, path := range paths {
			if !re.MatchString(path) {
				t.Errorf("pat %q should match %q", pat, path)
			}
		}
	}
}

func TestGlobNonMatches(t *testing.T) {
	for _, parts := range nonMatches {
		pat, paths := parts[0], parts[1:]
		re, err := compileGlob(pat)
		assert.Equal(t, nil, err)
		for _, path := range paths {
			if re.MatchString(path) {
				t.Errorf("pat %q should not match %q", pat, path)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package version

// tag is version of nvim-go.
//
// This is used for GCP profiler Sentry error reporting and so on. tag is overridden using
// `-X main.tag` during release builds.
var Tag string

// gitCommit is commit hash of nvim-go.
//
// This is used for GCP profiler Sentry error reporting and so on. gitCommit is overridden using
// `-X main.gitCommit` during release builds.
var GitCommit string

var Version = Tag + "@" + GitCommit

// var Version string
<commit_msg>pkg/version: remove Version variable<commit_after>// Copyright 2018 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package version

// tag is version of nvim-go.
//
// This is used for GCP profiler Sentry error reporting and so on. tag is overridden using
// `-X main.tag` during release builds.
var Tag string

// gitCommit is commit hash of nvim-go.
//
// This is used for GCP profiler Sentry error reporting and so on. gitCommit is overridden using
// `-X main.gitCommit` during release builds.
var GitCommit string

var Version = Tag + "@" + GitCommit
<|endoftext|>


<commit_before>// Name: loadpackage.go
// Desc:
//   A package to load metadata about a dotfile package.
package pkgfile

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/crockeo/dotfile-manager/files"
	"os"
)

// Loading a package from a given location.
func LoadPackage(name string) (PkgFile, error) {
	path := name + "/" + RelativeLocation
	fmt.Println(path)

	if !files.Exists(path) {
		return PkgFile{}, errors.New("Package file does not exist!")
	}

	file, err := os.Open(path)

	if err != nil {
		return PkgFile{}, errors.New("Failed to open the package file.")
	}

	var pkgFile PkgFile
	fmt.Println("Loading from " + path + "!")
	decoder := json.NewDecoder(file)
	decoder.Decode(&pkgFile)

	fmt.Println(pkgFile)

	return pkgFile, nil
}
<commit_msg>Removed an accidental print.<commit_after>// Name: loadpackage.go
// Desc:
//   A package to load metadata about a dotfile package.
package pkgfile

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/crockeo/dotfile-manager/files"
	"os"
)

// Loading a package from a given location.
func LoadPackage(name string) (PkgFile, error) {
	path := name + "/" + RelativeLocation
	fmt.Println(path)

	if !files.Exists(path) {
		return PkgFile{}, errors.New("Package file does not exist!")
	}

	file, err := os.Open(path)

	if err != nil {
		return PkgFile{}, errors.New("Failed to open the package file.")
	}

	var pkgFile PkgFile
	fmt.Println("Loading from " + path + "!")
	decoder := json.NewDecoder(file)
	decoder.Decode(&pkgFile)

	return pkgFile, nil
}
<|endoftext|>


<commit_before>package ogdat

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

const (
	Info    = 1 << iota
	Warning = 1 << iota
	Error   = 1 << iota
)

const (
	StructuralError  = 0x8000
	NoDataatUrlError = 0x4000
	FetchableUrl     = 0x2000
	FetchSuccess     = 0x1000
)

var isolangfilemap map[string]*ISO6392Lang = nil

type ISO6392Lang struct {
	Code, Identifier string
}

func CheckISOLanguage(lang string) bool {
	const iso639file = "ISO-639-2_utf-8.txt"
	if isolangfilemap == nil {
		var err error
		if isolangfilemap, err = loadisolanguagefile(iso639file); err != nil {
			panic(fmt.Sprintf("Can not load ISO language file '%s'", iso639file))
		}
	}
	_, ok := isolangfilemap[lang]
	return ok
}

// copmpare input against a slice of check strings.
// input will first be converted to lower-case, and any occurence of "-" is removed.
// The slice of checks will be iterated over,
// individual elements of check will not be converted or have their "-" removed.
func CheckEncodingString(input string, check []string) bool {
	input = strings.ToLower(strings.Replace(input, "-", "", -1))
	for _, val := range check {
		if input == val {
			return true
		}
	}
	return false
}

func loadisolanguagefile(filename string) (isolangfilemap map[string]*ISO6392Lang, _ error) {

	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	defer reader.Close()
	isolangfilemap = make(map[string]*ISO6392Lang)
	csvreader := csv.NewReader(reader)
	csvreader.Comma = '|'

	for record, err := csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
		isorecord := &ISO6392Lang{Code: record[0], Identifier: record[3]}
		isolangfilemap[isorecord.Code] = isorecord
		if len(record[1]) > 0 {
			isorecord = &ISO6392Lang{Code: record[1], Identifier: record[3]}
			isolangfilemap[record[1]] = isorecord
		}
	}
	log.Printf("Info: Read %d ISO language records", len(isolangfilemap))

	return
}

var ianaencmap map[string]struct{} = nil

// CheckIANAEncoding will try to match and input of enc against the specified encodings found at http://www.iana.org/assignments/character-sets/character-sets.xml
// The file at http://www.iana.org/assignments/character-sets/character-sets.xml is retrieved by a shell scripte,
// converted to all-lower case and sorted for unique entries. Thus the encoding enc against which will be checked,
// is converted to lower case and then compared to the IANA-encodings
func CheckIANAEncoding(enc string) bool {
	const ianaencfile = "character-sets.txt"
	if ianaencmap == nil {
		var err error
		if ianaencmap, err = loadianaencodingfile(ianaencfile); err != nil {
			panic(fmt.Sprintf("Can not load IANA encoding definition file '%s': %s", ianaencfile, err))
		}
	}
	enc = strings.ToLower(enc)
	_, ok := ianaencmap[enc]
	if !ok {
		enc = strings.Replace(enc, "-", "", -1)
		_, ok = ianaencmap[enc]
	}
	return ok
}

func loadianaencodingfile(filename string) (ianamap map[string]struct{}, _ error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	ianamap = make(map[string]struct{})
	bufreader := bufio.NewReader(reader)
	delim := byte('\n')

	for line, err := bufreader.ReadString(delim); err != io.EOF; line, err = bufreader.ReadString(delim) {
		// ReadString includes the delimeter, get rid of it
		line = line[:len(line)-1]
		// normalize by lower casing
		line = strings.ToLower(line)

		ianamap[line] = struct{}{}
	}
	log.Printf("Info: Read %d IANA encoding names", len(ianamap))
	return
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

type CheckInfo struct {
	Status, Position int
	Context          string
}

func (c *CheckInfo) Error() string {
	return c.Context
}

func strrange(minrange, maxrange, idx int, s string) string {
	if minrange > maxrange {
		panic("minrange > maxrange")
	}

	if idx > len(s) {
		idx = len(s)
	}

	var prepend string
	start := idx + minrange
	if start < 0 {
		start = 0
	} else {
		prepend = "..."
	}

	var postpone string
	end := idx + maxrange
	if end > len(s) {
		end = len(s)
	} else {
		postpone = "..."
	}
	return prepend + s[start:end] + postpone
}

var regexphtmlcodecheck = regexp.MustCompile(`</{0,1}\w+.*('|"|)>`)
var regexphtmlescape = regexp.MustCompile(`&\w{1,10};|&#\d{1,6};`)
var regexpurlencode = regexp.MustCompile(`%[0-9a-fA-F][0-9a-fA-F]`)
var regexpposixescape = regexp.MustCompile(`\\n|\\b|\\v|\\t|\\r`)

// return values are:
// ok = false indicates sthg. was wrong in which case error will not be nil
//
// error: if is of type CheckError:
// Status: 1 = Info, 2 = Warning, 3 = Error
// Position: beginning position of offending input
// message: An error message describing the problem
func CheckOGDTextStringForSaneCharacters(str string) (ok bool, _ error) {
	if !utf8.ValidString(str) {
		return false, &CheckInfo{Error, 0, "Zeichenfolge ist nicht durchgängig gültig als UTF8 kodiert"}
	}
	for idx, val := range str {
		if val == unicode.ReplacementChar {
			return false, &CheckInfo{Error, idx, fmt.Sprintf("Ungültige Unicode-Sequenz: '0x%x' (Bereich '%s')", val, strrange(-20, 20, idx, str))}
		}
	}

	if idx := regexphtmlcodecheck.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche HTML-Sequenz: '%s'", strrange(-10, 10, idx[0], str))}
	}
	if idx := regexphtmlescape.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche HTML-Escapes: '%s'", strrange(-8, 8, idx[0], str))}
	}
	if idx := regexpurlencode.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche Url-Escapes: '%s'", strrange(-6, 6, idx[0], str))}
	}
	if idx := regexpposixescape.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche Posix-Escapes: '%s'", strrange(-6, 6, idx[0], str))}
	}
	return true, nil
}

var regexpEMail = regexp.MustCompile(`(?i)^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}$`)

func FetchHead(url string) (bool, CheckInfo) {

	var info CheckInfo
	var err error
	var resp *http.Response

	for i := 0; i < 2; i++ {
		if i == 0 {
			resp, err = http.Head(url)
		} else {
			resp, err = http.Get(url)
		}
		if err != nil || resp.StatusCode != 200 {
			continue
		} else {
			break
		}
	}

	if err != nil {
		info = CheckInfo{Status: Error | FetchableUrl | NoDataatUrlError, Position: -1, Context: fmt.Sprintf("'%s': URL kann nicht aufgelöst werden: '%s'", url, err)}
	} else if sc := resp.StatusCode; sc != 200 {
		info = CheckInfo{Status: Error | FetchableUrl | NoDataatUrlError, Position: -1, Context: fmt.Sprintf("'%s': Request liefert nicht-OK Statuscode '%d'", url, sc)}
	} else {
		info = CheckInfo{Status: Info | FetchableUrl | FetchSuccess, Position: -1, Context: url}
	}

	return (info.Status & (Info | FetchSuccess)) == (Info | FetchSuccess), info
}

func CheckUrl(url string, followhttplink bool) (bool, []CheckInfo) {
	// it's a contact point if it's a http-link (starts with "http(s)" )
	var checkmessages []CheckInfo
	if len(url) >= 4 && url[:4] == "http" {
		urlinfo := CheckInfo{Info | FetchableUrl, -1, url}
		checkmessages = append(checkmessages, urlinfo)

		ok := true
		if followhttplink {
			ok, urlinfo = FetchHead(urlinfo.Context)
			checkmessages = append(checkmessages, urlinfo)
		}
		return ok, checkmessages
	}
	// it's a contact point if it's an email address
	if idx := regexpEMail.FindStringIndex(url); idx != nil {
		return true, nil
	}

	if len(url) == 0 {
		checkmessages = append(checkmessages, CheckInfo{Status: Error, Position: -1, Context: "kein Wert für Link angegeben (Länge 0)"})
		return false, checkmessages
	}

	checkmessages = append(checkmessages, CheckInfo{Status: Warning, Position: -1, Context: fmt.Sprintf("vermutlich keine gültige Web- oder E-Mail Adresse: '%s' (Auszug)", url[:min(20, len(url))])})

	return false, checkmessages
}

func CheckDataPortalUrl(url string, followhttplink bool) (bool, []CheckInfo) {
	const checkDataPortalUrlstrpart = `://data.`

	ok1, checkmessages := CheckUrl(url, followhttplink)
	ok2 := strings.Contains(url, checkDataPortalUrlstrpart)
	if !ok2 {
		checkmessages = append(checkmessages, CheckInfo{Status: Warning, Position: -1, Context: fmt.Sprintf("Link zum ursprünglichen Datenportal folgt nicht der Konvention ress://data.[....]: %s (Auszug)", url[:min(20, len(url))])})
	}

	return ok1 == true && ok2 == true, checkmessages
}

type CheckMessage struct {
	Type    int // 1 = Info, 2 = Warning, 4 = Error, ...
	Text    string
	OGDID   int
	Context string
}

func AppendcheckerrorTocheckmessage(msgs []CheckMessage, checkresults []CheckInfo, ID int, prepend string) []CheckMessage {
	for _, result := range checkresults {
		msgs = append(msgs, CheckMessage{
			Type:  result.Status,
			OGDID: ID,
			Text:  prepend + result.Context})
	}
	return msgs
}

func Loadogdatspec(version, filename string) (*OGDSet, error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	csvreader := csv.NewReader(reader)
	csvreader.Comma = '|'
	csvreader.LazyQuotes = true

	// Read the first line and use it as the labels for the items to load
	record, err := csvreader.Read()
	if err != nil {
		return nil, err
	}

	set := &OGDSet{Label: record}

	spec := make([]*Beschreibung, 0)
	for record, err = csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
		id, _ := strconv.Atoi(record[0])
		var occ Occurrence
		switch record[12][0] {
		case 'R':
			occ = OccRequired
		case 'O':
			occ = OccOptional
		}
		descrecord := NewBeschreibung(id, occ, version)

		descrecord.Bezeichner = record[1]
		descrecord.OGD_Kurzname = record[2]
		descrecord.CKAN_Feld = record[3]
		descrecord.Anzahl = record[4]
		descrecord.Definition_DE = record[5]
		descrecord.Erlauterung = record[6]
		descrecord.Beispiel = record[7]
		descrecord.ONA2270 = record[8]
		descrecord.ISO19115 = record[9]
		descrecord.RDFProperty = record[10]
		descrecord.Definition_EN = record[11]

		spec = append(spec, descrecord)
	}
	set.Beschreibung = spec
	log.Printf("Info: Read %d %s specification records", len(spec), version)

	return set, nil
}
<commit_msg>cutting strings in the middle might result in an invalid UTF8. check for that<commit_after>package ogdat

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

const (
	Info    = 1 << iota
	Warning = 1 << iota
	Error   = 1 << iota
)

const (
	StructuralError  = 0x8000
	NoDataatUrlError = 0x4000
	FetchableUrl     = 0x2000
	FetchSuccess     = 0x1000
)

var isolangfilemap map[string]*ISO6392Lang = nil

type ISO6392Lang struct {
	Code, Identifier string
}

func CheckISOLanguage(lang string) bool {
	const iso639file = "ISO-639-2_utf-8.txt"
	if isolangfilemap == nil {
		var err error
		if isolangfilemap, err = loadisolanguagefile(iso639file); err != nil {
			panic(fmt.Sprintf("Can not load ISO language file '%s'", iso639file))
		}
	}
	_, ok := isolangfilemap[lang]
	return ok
}

// copmpare input against a slice of check strings.
// input will first be converted to lower-case, and any occurence of "-" is removed.
// The slice of checks will be iterated over,
// individual elements of check will not be converted or have their "-" removed.
func CheckEncodingString(input string, check []string) bool {
	input = strings.ToLower(strings.Replace(input, "-", "", -1))
	for _, val := range check {
		if input == val {
			return true
		}
	}
	return false
}

func loadisolanguagefile(filename string) (isolangfilemap map[string]*ISO6392Lang, _ error) {

	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	defer reader.Close()
	isolangfilemap = make(map[string]*ISO6392Lang)
	csvreader := csv.NewReader(reader)
	csvreader.Comma = '|'

	for record, err := csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
		isorecord := &ISO6392Lang{Code: record[0], Identifier: record[3]}
		isolangfilemap[isorecord.Code] = isorecord
		if len(record[1]) > 0 {
			isorecord = &ISO6392Lang{Code: record[1], Identifier: record[3]}
			isolangfilemap[record[1]] = isorecord
		}
	}
	log.Printf("Info: Read %d ISO language records", len(isolangfilemap))

	return
}

var ianaencmap map[string]struct{} = nil

// CheckIANAEncoding will try to match and input of enc against the specified encodings found at http://www.iana.org/assignments/character-sets/character-sets.xml
// The file at http://www.iana.org/assignments/character-sets/character-sets.xml is retrieved by a shell scripte,
// converted to all-lower case and sorted for unique entries. Thus the encoding enc against which will be checked,
// is converted to lower case and then compared to the IANA-encodings
func CheckIANAEncoding(enc string) bool {
	const ianaencfile = "character-sets.txt"
	if ianaencmap == nil {
		var err error
		if ianaencmap, err = loadianaencodingfile(ianaencfile); err != nil {
			panic(fmt.Sprintf("Can not load IANA encoding definition file '%s': %s", ianaencfile, err))
		}
	}
	enc = strings.ToLower(enc)
	_, ok := ianaencmap[enc]
	if !ok {
		enc = strings.Replace(enc, "-", "", -1)
		_, ok = ianaencmap[enc]
	}
	return ok
}

func loadianaencodingfile(filename string) (ianamap map[string]struct{}, _ error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	ianamap = make(map[string]struct{})
	bufreader := bufio.NewReader(reader)
	delim := byte('\n')

	for line, err := bufreader.ReadString(delim); err != io.EOF; line, err = bufreader.ReadString(delim) {
		// ReadString includes the delimeter, get rid of it
		line = line[:len(line)-1]
		// normalize by lower casing
		line = strings.ToLower(line)

		ianamap[line] = struct{}{}
	}
	log.Printf("Info: Read %d IANA encoding names", len(ianamap))
	return
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

type CheckInfo struct {
	Status, Position int
	Context          string
}

func (c *CheckInfo) Error() string {
	return c.Context
}

func strrange(minrange, maxrange, idx int, s string) string {
	if minrange > maxrange {
		panic("minrange > maxrange")
	}

	if idx > len(s) {
		idx = len(s)
	}

	var prepend string
	start := idx + minrange
	if start < 0 {
		start = 0
	} else {
		prepend = "..."
	}

	var postpone string
	end := idx + maxrange
	if end > len(s) {
		end = len(s)
	} else {
		postpone = "..."
	}
	middle := s[start:end]
	if !utf8.ValidString(middle) {
		var s string
		for len(middle) > 0 {
			r, size := utf8.DecodeRuneInString(middle)
			s += string(r)
			middle = middle[size:]
		}
		middle = s
	}
	return prepend + middle + postpone
}

var regexphtmlcodecheck = regexp.MustCompile(`</{0,1}\w+.*('|"|)>`)
var regexphtmlescape = regexp.MustCompile(`&\w{1,10};|&#\d{1,6};`)
var regexpurlencode = regexp.MustCompile(`%[0-9a-fA-F][0-9a-fA-F]`)
var regexpposixescape = regexp.MustCompile(`\\n|\\b|\\v|\\t|\\r`)

// return values are:
// ok = false indicates sthg. was wrong in which case error will not be nil
//
// error: if is of type CheckError:
// Status: 1 = Info, 2 = Warning, 3 = Error
// Position: beginning position of offending input
// message: An error message describing the problem
func CheckOGDTextStringForSaneCharacters(str string) (ok bool, _ error) {
	if !utf8.ValidString(str) {
		return false, &CheckInfo{Error, 0, "Zeichenfolge ist nicht durchgängig gültig als UTF8 kodiert"}
	}
	for idx, val := range str {
		if val == unicode.ReplacementChar {
			return false, &CheckInfo{Error, idx, fmt.Sprintf("Ungültige Unicode-Sequenz: '0x%x' (Bereich '%s')", val, strrange(-20, 20, idx, str))}
		}
	}

	if idx := regexphtmlcodecheck.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche HTML-Sequenz: '%s'", strrange(-10, 10, idx[0], str))}
	}
	if idx := regexphtmlescape.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche HTML-Escapes: '%s'", strrange(-8, 8, idx[0], str))}
	}
	if idx := regexpurlencode.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche Url-Escapes: '%s'", strrange(-6, 6, idx[0], str))}
	}
	if idx := regexpposixescape.FindStringIndex(str); idx != nil {
		return false, &CheckInfo{Warning, idx[0], fmt.Sprintf("Mögliche Posix-Escapes: '%s'", strrange(-6, 6, idx[0], str))}
	}
	return true, nil
}

var regexpEMail = regexp.MustCompile(`(?i)^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}$`)

func FetchHead(url string) (bool, CheckInfo) {

	var info CheckInfo
	var err error
	var resp *http.Response

	for i := 0; i < 2; i++ {
		if i == 0 {
			resp, err = http.Head(url)
		} else {
			resp, err = http.Get(url)
		}
		if err != nil || resp.StatusCode != 200 {
			continue
		} else {
			break
		}
	}

	if err != nil {
		info = CheckInfo{Status: Error | FetchableUrl | NoDataatUrlError, Position: -1, Context: fmt.Sprintf("'%s': URL kann nicht aufgelöst werden: '%s'", url, err)}
	} else if sc := resp.StatusCode; sc != 200 {
		info = CheckInfo{Status: Error | FetchableUrl | NoDataatUrlError, Position: -1, Context: fmt.Sprintf("'%s': Request liefert nicht-OK Statuscode '%d'", url, sc)}
	} else {
		info = CheckInfo{Status: Info | FetchableUrl | FetchSuccess, Position: -1, Context: url}
	}

	return (info.Status & (Info | FetchSuccess)) == (Info | FetchSuccess), info
}

func CheckUrl(url string, followhttplink bool) (bool, []CheckInfo) {
	// it's a contact point if it's a http-link (starts with "http(s)" )
	var checkmessages []CheckInfo
	if len(url) >= 4 && url[:4] == "http" {
		urlinfo := CheckInfo{Info | FetchableUrl, -1, url}
		checkmessages = append(checkmessages, urlinfo)

		ok := true
		if followhttplink {
			ok, urlinfo = FetchHead(urlinfo.Context)
			checkmessages = append(checkmessages, urlinfo)
		}
		return ok, checkmessages
	}
	// it's a contact point if it's an email address
	if idx := regexpEMail.FindStringIndex(url); idx != nil {
		return true, nil
	}

	if len(url) == 0 {
		checkmessages = append(checkmessages, CheckInfo{Status: Error, Position: -1, Context: "kein Wert für Link angegeben (Länge 0)"})
		return false, checkmessages
	}

	checkmessages = append(checkmessages, CheckInfo{Status: Warning, Position: -1, Context: fmt.Sprintf("vermutlich keine gültige Web- oder E-Mail Adresse: '%s' (Auszug)", url[:min(20, len(url))])})

	return false, checkmessages
}

func CheckDataPortalUrl(url string, followhttplink bool) (bool, []CheckInfo) {
	const checkDataPortalUrlstrpart = `://data.`

	ok1, checkmessages := CheckUrl(url, followhttplink)
	ok2 := strings.Contains(url, checkDataPortalUrlstrpart)
	if !ok2 {
		checkmessages = append(checkmessages, CheckInfo{Status: Warning, Position: -1, Context: fmt.Sprintf("Link zum ursprünglichen Datenportal folgt nicht der Konvention ress://data.[....]: %s (Auszug)", url[:min(20, len(url))])})
	}

	return ok1 == true && ok2 == true, checkmessages
}

type CheckMessage struct {
	Type    int // 1 = Info, 2 = Warning, 4 = Error, ...
	Text    string
	OGDID   int
	Context string
}

func AppendcheckerrorTocheckmessage(msgs []CheckMessage, checkresults []CheckInfo, ID int, prepend string) []CheckMessage {
	for _, result := range checkresults {
		msgs = append(msgs, CheckMessage{
			Type:  result.Status,
			OGDID: ID,
			Text:  prepend + result.Context})
	}
	return msgs
}

func Loadogdatspec(version, filename string) (*OGDSet, error) {
	reader, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	csvreader := csv.NewReader(reader)
	csvreader.Comma = '|'
	csvreader.LazyQuotes = true

	// Read the first line and use it as the labels for the items to load
	record, err := csvreader.Read()
	if err != nil {
		return nil, err
	}

	set := &OGDSet{Label: record}

	spec := make([]*Beschreibung, 0)
	for record, err = csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
		id, _ := strconv.Atoi(record[0])
		var occ Occurrence
		switch record[12][0] {
		case 'R':
			occ = OccRequired
		case 'O':
			occ = OccOptional
		}
		descrecord := NewBeschreibung(id, occ, version)

		descrecord.Bezeichner = record[1]
		descrecord.OGD_Kurzname = record[2]
		descrecord.CKAN_Feld = record[3]
		descrecord.Anzahl = record[4]
		descrecord.Definition_DE = record[5]
		descrecord.Erlauterung = record[6]
		descrecord.Beispiel = record[7]
		descrecord.ONA2270 = record[8]
		descrecord.ISO19115 = record[9]
		descrecord.RDFProperty = record[10]
		descrecord.Definition_EN = record[11]

		spec = append(spec, descrecord)
	}
	set.Beschreibung = spec
	log.Printf("Info: Read %d %s specification records", len(spec), version)

	return set, nil
}
<|endoftext|>


<commit_before>package py

import (
	. "github.com/smartystreets/goconvey/convey"
	"github.com/ugorji/go/codec"
	"pfi/sensorbee/sensorbee/data"
	"testing"
)

func init() {
	// goconvey is call same function several times, so in order to call
	// `Initialize` only once, use `init`
	Initialize()
}

func TestPythonCall(t *testing.T) {
	Convey("Given an initialized python interpreter", t, func() {

		ImportSysAndAppendPath("")

		Convey("When get invalid module", func() {
			_, err := LoadModule("notexist")
			Convey("Then an error should be occurred", func() {
				So(err, ShouldNotBeNil)
			})
		})
		Convey("When get valid module", func() {
			mdl, err := LoadModule("_test")
			defer mdl.DecRef()
			Convey("Then process should get PyModule", func() {
				So(err, ShouldBeNil)
				So(mdl, ShouldNotBeNil)

				Convey("And when call invalid function", func() {
					_, err := mdl.CallIntInt("notFoundMethod", 1)
					Convey("Then an error should be occurred", func() {
						So(err, ShouldNotBeNil)
					})
				})

				Convey("And when call int-int function", func() {
					actual, err := mdl.CallIntInt("tenTimes", 3)
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, 30)
					})
				})

				Convey("And when call int-int function with Call method", func() {
					actual, err := mdl.Call("tenTimes", data.Int(4))
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, data.Int(40))
					})
				})

				Convey("And when call none-string function", func() {
					actual, err := mdl.CallNoneString("logger")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "called")
					})
				})

				Convey("And when call none-string function with Call", func() {
					actual, err := mdl.Call("logger")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, data.String("called"))
					})
				})

				Convey("And when call none-2string function", func() {
					actual1, actual2, err := mdl.CallNone2String("twoLogger")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual1, ShouldEqual, "called1")
						So(actual2, ShouldEqual, "called2")
					})
				})

				Convey("And when call string-string function", func() {
					actual, err := mdl.CallStringString("plusSuffix", "test")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "test_through_python")
					})
				})

				Convey("And when call specialized function", func() {
					h := &codec.MsgpackHandle{}
					var out []byte
					enc := codec.NewEncoderBytes(&out, h)

					dat := []float32{1.1, 1.2, 1.3, 1.4, 1.5}
					data2 := []int{9, 8, 7., 6, 5}
					ds := map[string]interface{}{}
					ds["data"] = dat
					ds["target"] = data2
					ds["model"] = []byte{}
					err := enc.Encode(ds)
					So(err, ShouldBeNil)
					So(len(out), ShouldNotEqual, 0)
					actual1, err := mdl.CallByteByte("loadMsgPack", out)
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(len(actual1), ShouldNotEqual, 0)

						var ds2 map[string]interface{}
						dec := codec.NewDecoderBytes(actual1, h)
						dec.Decode(&ds2)
						model, ok := ds2["model"]
						So(ok, ShouldBeTrue)
						modelByte, ok := model.([]byte)
						So(ok, ShouldBeTrue)
						So(len(modelByte), ShouldNotEqual, 0)

						log, ok := ds2["log"]
						So(ok, ShouldBeTrue)
						logByte, ok := log.([]byte)
						So(ok, ShouldBeTrue)
						So(string(logByte), ShouldEqual, "done")

						Convey("And when pass pickle data", func() {
							var out2 []byte
							enc2 := codec.NewEncoderBytes(&out2, h)
							err = enc2.Encode(ds2)
							So(err, ShouldBeNil)

							Convey("Then function should return model again", func() {
								actual2, err := mdl.CallByteByte("loadMsgPack", out2)
								So(err, ShouldBeNil)
								So(len(actual2), ShouldNotEqual, 0)
								So(string(actual2), ShouldResemble,
									"\x82\xa5model\xae\x80\x02U\aTEST_req\x00.\xa3log\xa4done")

							})

							Convey("Then function should return model again with Call", func() {
								actual2, err := mdl.Call("loadMsgPack", data.Blob(out2))
								So(err, ShouldBeNil)
								dat, err := data.AsString(actual2)
								So(err, ShouldBeNil)
								So(string(dat), ShouldResemble,
									"\x82\xa5model\xae\x80\x02U\aTEST_req\x00.\xa3log\xa4done")
							})
						})
					})
				})

				Convey("And when call dictionary argument function", func() {
					arg := data.Map{
						"string": data.String("test"),
						"int":    data.Int(9),
						"byte":   data.Blob([]byte("ABC")),
					}

					actual, err := mdl.CallMapString("dict", arg)
					Convey("Then function should return valid values", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "test9ABC")
					})
				})

				Convey("And when call dictionary argument function with Call", func() {
					arg := data.Map{
						"string": data.String("test"),
						"int":    data.Int(8),
						"byte":   data.Blob([]byte("ABC")),
					}

					actual, err := mdl.Call("dict", arg)
					Convey("Then function should return valid values", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, data.String("test8ABC"))
					})
				})
			})
		})
	})
}

func TestPythonInstanceCall(t *testing.T) {
	Convey("Given an initialized python module", t, func() {

		ImportSysAndAppendPath("")

		mdl, err := LoadModule("_instance_test")
		So(err, ShouldBeNil)
		So(mdl, ShouldNotBeNil)

		Convey("When get a invalid class instance", func() {
			_, err := mdl.GetInstance("NonexistentClass")
			Convey("Then an error should be occurred", func() {
				So(err, ShouldNotBeNil)
			})
		})

		Convey("When get a new test python instance", func() {
			ins, err := mdl.GetInstance("PythonTest")
			defer ins.DecRef()
			Convey("Then process should get PyModule", func() {
				So(err, ShouldBeNil)
				So(ins, ShouldNotBeNil)

				Convey("And when call a logger function", func() {
					actual, err := ins.CallStringString("logger", "test")
					Convey("Then process should get a string", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "initialized_test")
					})

					Convey("And when call a logger function again", func() {
						actual2, err := ins.CallStringString("logger", "test")
						Convey("Then process should get a string", func() {
							So(err, ShouldBeNil)
							So(actual2, ShouldEqual, "initialized_test_test")
						})

						Convey("And when get a new test python instance", func() {
							ins2, err := mdl.GetInstance("PythonTest")
							defer ins2.DecRef()
							Convey("Then process should get PyModule", func() {
								So(err, ShouldBeNil)
								So(ins2, ShouldNotBeNil)

								Convey("And when call a logger function", func() {
									actual3, err1 := ins.CallStringString("logger", "t")
									actual4, err2 := ins2.CallStringString("logger", "t")
									Convey("Then process should get a string", func() {
										So(err1, ShouldBeNil)
										So(err2, ShouldBeNil)
										So(actual3, ShouldEqual, "initialized_test_test_t")
										So(actual4, ShouldEqual, "initialized_t")
									})
								})
							})
						})
					})
				})
			})
		})
	})
}
<commit_msg>Fix typo of convey explanation<commit_after>package py

import (
	. "github.com/smartystreets/goconvey/convey"
	"github.com/ugorji/go/codec"
	"pfi/sensorbee/sensorbee/data"
	"testing"
)

func init() {
	// goconvey is call same function several times, so in order to call
	// `Initialize` only once, use `init`
	Initialize()
}

func TestPythonCall(t *testing.T) {
	Convey("Given an initialized python interpreter", t, func() {

		ImportSysAndAppendPath("")

		Convey("When get invalid module", func() {
			_, err := LoadModule("notexist")
			Convey("Then an error should be occurred", func() {
				So(err, ShouldNotBeNil)
			})
		})
		Convey("When get valid module", func() {
			mdl, err := LoadModule("_test")
			defer mdl.DecRef()
			Convey("Then process should get PyModule", func() {
				So(err, ShouldBeNil)
				So(mdl, ShouldNotBeNil)

				Convey("And when call invalid function", func() {
					_, err := mdl.CallIntInt("notFoundMethod", 1)
					Convey("Then an error should be occurred", func() {
						So(err, ShouldNotBeNil)
					})
				})

				Convey("And when call int-int function", func() {
					actual, err := mdl.CallIntInt("tenTimes", 3)
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, 30)
					})
				})

				Convey("And when call int-int function with Call method", func() {
					actual, err := mdl.Call("tenTimes", data.Int(4))
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, data.Int(40))
					})
				})

				Convey("And when call none-string function", func() {
					actual, err := mdl.CallNoneString("logger")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "called")
					})
				})

				Convey("And when call none-string function with Call", func() {
					actual, err := mdl.Call("logger")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, data.String("called"))
					})
				})

				Convey("And when call none-2string function", func() {
					actual1, actual2, err := mdl.CallNone2String("twoLogger")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual1, ShouldEqual, "called1")
						So(actual2, ShouldEqual, "called2")
					})
				})

				Convey("And when call string-string function", func() {
					actual, err := mdl.CallStringString("plusSuffix", "test")
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "test_through_python")
					})
				})

				Convey("And when call specialized function", func() {
					h := &codec.MsgpackHandle{}
					var out []byte
					enc := codec.NewEncoderBytes(&out, h)

					dat := []float32{1.1, 1.2, 1.3, 1.4, 1.5}
					data2 := []int{9, 8, 7., 6, 5}
					ds := map[string]interface{}{}
					ds["data"] = dat
					ds["target"] = data2
					ds["model"] = []byte{}
					err := enc.Encode(ds)
					So(err, ShouldBeNil)
					So(len(out), ShouldNotEqual, 0)
					actual1, err := mdl.CallByteByte("loadMsgPack", out)
					Convey("Then function should return valid result", func() {
						So(err, ShouldBeNil)
						So(len(actual1), ShouldNotEqual, 0)

						var ds2 map[string]interface{}
						dec := codec.NewDecoderBytes(actual1, h)
						dec.Decode(&ds2)
						model, ok := ds2["model"]
						So(ok, ShouldBeTrue)
						modelByte, ok := model.([]byte)
						So(ok, ShouldBeTrue)
						So(len(modelByte), ShouldNotEqual, 0)

						log, ok := ds2["log"]
						So(ok, ShouldBeTrue)
						logByte, ok := log.([]byte)
						So(ok, ShouldBeTrue)
						So(string(logByte), ShouldEqual, "done")

						Convey("And when pass pickle data", func() {
							var out2 []byte
							enc2 := codec.NewEncoderBytes(&out2, h)
							err = enc2.Encode(ds2)
							So(err, ShouldBeNil)

							Convey("Then function should return model again", func() {
								actual2, err := mdl.CallByteByte("loadMsgPack", out2)
								So(err, ShouldBeNil)
								So(len(actual2), ShouldNotEqual, 0)
								So(string(actual2), ShouldResemble,
									"\x82\xa5model\xae\x80\x02U\aTEST_req\x00.\xa3log\xa4done")

							})

							Convey("Then function should return model again with Call", func() {
								actual2, err := mdl.Call("loadMsgPack", data.Blob(out2))
								So(err, ShouldBeNil)
								dat, err := data.AsString(actual2)
								So(err, ShouldBeNil)
								So(string(dat), ShouldResemble,
									"\x82\xa5model\xae\x80\x02U\aTEST_req\x00.\xa3log\xa4done")
							})
						})
					})
				})

				Convey("And when call dictionary argument function", func() {
					arg := data.Map{
						"string": data.String("test"),
						"int":    data.Int(9),
						"byte":   data.Blob([]byte("ABC")),
					}

					actual, err := mdl.CallMapString("dict", arg)
					Convey("Then function should return valid values", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "test9ABC")
					})
				})

				Convey("And when call dictionary argument function with Call", func() {
					arg := data.Map{
						"string": data.String("test"),
						"int":    data.Int(8),
						"byte":   data.Blob([]byte("ABC")),
					}

					actual, err := mdl.Call("dict", arg)
					Convey("Then function should return valid values", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, data.String("test8ABC"))
					})
				})
			})
		})
	})
}

func TestPythonInstanceCall(t *testing.T) {
	Convey("Given an initialized python module", t, func() {

		ImportSysAndAppendPath("")

		mdl, err := LoadModule("_instance_test")
		So(err, ShouldBeNil)
		So(mdl, ShouldNotBeNil)

		Convey("When get an invalid class instance", func() {
			_, err := mdl.GetInstance("NonexistentClass")
			Convey("Then an error should be occurred", func() {
				So(err, ShouldNotBeNil)
			})
		})

		Convey("When get a new test python instance", func() {
			ins, err := mdl.GetInstance("PythonTest")
			defer ins.DecRef()
			Convey("Then process should get PyModule", func() {
				So(err, ShouldBeNil)
				So(ins, ShouldNotBeNil)

				Convey("And when call a logger function", func() {
					actual, err := ins.CallStringString("logger", "test")
					Convey("Then process should get a string", func() {
						So(err, ShouldBeNil)
						So(actual, ShouldEqual, "initialized_test")
					})

					Convey("And when call a logger function again", func() {
						actual2, err := ins.CallStringString("logger", "test")
						Convey("Then process should get a string", func() {
							So(err, ShouldBeNil)
							So(actual2, ShouldEqual, "initialized_test_test")
						})

						Convey("And when get a new test python instance", func() {
							ins2, err := mdl.GetInstance("PythonTest")
							defer ins2.DecRef()
							Convey("Then process should get PyModule", func() {
								So(err, ShouldBeNil)
								So(ins2, ShouldNotBeNil)

								Convey("And when call a logger function", func() {
									actual3, err1 := ins.CallStringString("logger", "t")
									actual4, err2 := ins2.CallStringString("logger", "t")
									Convey("Then process should get a string", func() {
										So(err1, ShouldBeNil)
										So(err2, ShouldBeNil)
										So(actual3, ShouldEqual, "initialized_test_test_t")
										So(actual4, ShouldEqual, "initialized_t")
									})
								})
							})
						})
					})
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>package taskrunner

import (
	"context"
	"sync"

	"github.com/hashicorp/nomad/client/driver/structs"
)

// handleResult multiplexes a single WaitResult to multiple waiters. Useful
// because DriverHandle.WaitCh is closed after it returns a single WaitResult.
type handleResult struct {
	doneCh <-chan struct{}

	result *structs.WaitResult
	mu     sync.RWMutex
}

func newHandleResult(waitCh <-chan *structs.WaitResult) *handleResult {
	doneCh := make(chan struct{})

	h := &handleResult{
		doneCh: doneCh,
	}

	go func() {
		// Wait for result
		res := <-waitCh

		// Set result
		h.mu.Lock()
		h.result = res
		h.mu.Unlock()

		// Notify waiters
		close(doneCh)

	}()

	return h
}

// Wait blocks until a task's result is available or the passed-in context is
// canceled. Safe for concurrent callers.
func (h *handleResult) Wait(ctx context.Context) *structs.WaitResult {
	// Block until done or canceled
	select {
	case <-h.doneCh:
	case <-ctx.Done():
	}

	h.mu.RLock()
	res := h.result
	h.mu.RUnlock()

	return res
}
<commit_msg>tr: remove unneeded lock; chan synchronizes access<commit_after>package taskrunner

import (
	"context"

	"github.com/hashicorp/nomad/client/driver/structs"
)

// handleResult multiplexes a single WaitResult to multiple waiters. Useful
// because DriverHandle.WaitCh is closed after it returns a single WaitResult.
type handleResult struct {
	doneCh <-chan struct{}
	result *structs.WaitResult
}

func newHandleResult(waitCh <-chan *structs.WaitResult) *handleResult {
	doneCh := make(chan struct{})

	h := &handleResult{
		doneCh: doneCh,
	}

	go func() {
		// Wait for result
		res := <-waitCh

		// Set result
		h.result = res

		// Notify waiters
		close(doneCh)

	}()

	return h
}

// Wait blocks until a task's result is available or the passed-in context is
// canceled. Safe for concurrent callers.
func (h *handleResult) Wait(ctx context.Context) *structs.WaitResult {
	// Block until done or canceled
	select {
	case <-h.doneCh:
	case <-ctx.Done():
		return nil
	}

	return h.result
}
<|endoftext|>