<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"syscall"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/dustin/go-humanize"
	"github.com/kolyshkin/goploop-cli"
	"github.com/virtuozzo/ploop-flexvol/vstorage"
)

const (
	parentProvisionerAnn = "vzFSParentProvisioner"
	vzShareAnn           = "vzShare"
)

type vzFSProvisioner struct {
	// Kubernetes Client. Use to retrieve secrets with Virtuozzo Storage credentials
	client kubernetes.Interface
}

func newVzFSProvisioner(client kubernetes.Interface) controller.Provisioner {
	return &vzFSProvisioner{
		client: client,
	}
}

var _ controller.Provisioner = &vzFSProvisioner{}

const provisionerDir = "/export/virtuozzo-provisioner/"
const mountDir = provisionerDir + "mnt/"

func prepareVstorage(options map[string]string, clusterName string, clusterPassword string) error {
	mount := mountDir + clusterName
	mounted, _ := vstorage.IsVstorage(mount)
	if mounted {
		return nil
	}

	if err := os.MkdirAll(mount, 0755); err != nil {
		return err
	}

	v := vstorage.Vstorage{clusterName}
	p, _ := v.Mountpoint()
	if p != "" {
		return syscall.Mount(p, mount, "", syscall.MS_BIND, "")
	}

	if err := v.Auth(clusterPassword); err != nil {
		return err
	}
	if err := v.Mount(mount); err != nil {
		return err
	}

	return nil
}

func createPloop(mount string, options map[string]string) error {
	var (
		volumePath, volumeID, size string
	)

	for k, v := range options {
		switch k {
		case "volumePath":
			volumePath = v
		case "volumeID":
			volumeID = v
		case "size":
			size = v
		case "vzsReplicas":
		case "vzsFailureDomain":
		case "vzsEncoding":
		case "vzsTier":
		case "kubernetes.io/readwrite":
		case "kubernetes.io/fsType":
		default:
		}
	}

	if volumePath == "" {
		return fmt.Errorf("volumePath isn't specified")
	}

	if volumeID == "" {
		return fmt.Errorf("volumeID isn't specified")
	}

	if size == "" {
		return fmt.Errorf("size isn't specified")
	}

	// get a human readable size from the map
	bytes, _ := humanize.ParseBytes(size)

	// ploop driver takes kilobytes, so convert it
	volumeSize := bytes / 1024

	ploopPath := mount + "/" + options["volumePath"] + "/" + options["volumeID"]

	// make the base directory where the volume will go
	err := os.MkdirAll(ploopPath, 0755)
	if err != nil {
		return err
	}

	for k, v := range options {
		var err error
		attr := ""
		switch k {
		case "vzsReplicas":
			attr = "replicas"
		case "vzsTier":
			attr = "tier"
		case "vzsEncoding":
			attr = "encoding"
		case "vzsFailureDomain":
			attr = "failure-domain"
		}
		if attr != "" {
			cmd := "vstorage"
			args := []string{"set-attr", "-R", ploopPath,
				fmt.Sprintf("%s=%s", attr, v)}
			err = exec.Command(cmd, args...).Run()
		}

		if err != nil {
			os.RemoveAll(ploopPath)
			return fmt.Errorf("Unable to set %s to %s: %v", attr, v, err)
		}
	}

	// Create the ploop volume
	cp := ploop.CreateParam{Size: volumeSize, File: ploopPath + "/" + options["volumeID"]}
	if err := ploop.Create(&cp); err != nil {
		return err
	}

	return nil
}

// Provision creates a storage asset and returns a PV object representing it.
func (p *vzFSProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	capacity := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	bytes := capacity.Value()

	if options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}
	share := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())

	glog.Infof("Add %s %s", share, humanize.Bytes(uint64(bytes)))

	storageClassOptions := map[string]string{}
	for k, v := range options.Parameters {
		storageClassOptions[k] = v
	}

	storageClassOptions["volumeID"] = share
	storageClassOptions["size"] = fmt.Sprintf("%d", bytes)
	secretName := storageClassOptions["secretName"]
	delete(storageClassOptions, "secretName")

	secret, err := p.client.Core().Secrets(options.PVC.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	if err := prepareVstorage(storageClassOptions, name, password); err != nil {
		return nil, err
	}

	if err := createPloop(mountDir+name, storageClassOptions); err != nil {
		return nil, err
	}

	storageClassOptions["clusterName"] = name
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: options.PVName,
			Annotations: map[string]string{
				parentProvisionerAnn: *provisionerID,
				vzShareAnn:           share,
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    "virtuozzo/ploop",
					SecretRef: &v1.LocalObjectReference{Name: secretName},
					Options:   storageClassOptions,
				},
			},
		},
	}

	glog.Infof("successfully created virtuozzo storage share: %s", share)

	return pv, nil
}

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *vzFSProvisioner) Delete(volume *v1.PersistentVolume) error {
	ann, ok := volume.Annotations[parentProvisionerAnn]
	if !ok {
		return errors.New("Parent provisioner name annotation not found on PV")
	}
	if ann != *provisionerID {
		return &controller.IgnoredError{"parent provisioner name annotation on PV does not match ours"}
	}
	share, ok := volume.Annotations[vzShareAnn]
	if !ok {
		return errors.New("vz share annotation not found on PV")
	}

	secretName := volume.Spec.PersistentVolumeSource.FlexVolume.SecretRef.Name
	options := volume.Spec.PersistentVolumeSource.FlexVolume.Options

	secret, err := p.client.Core().Secrets(volume.Spec.ClaimRef.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	mount := mountDir + name
	if err := prepareVstorage(options, name, password); err != nil {
		return err
	}

	path := mount + "/" + options["volumePath"] + "/" + options["volumeID"]
	glog.Infof("Delete: %s", path)
	err = os.RemoveAll(path)
	if err != nil {
		return err
	}

	glog.Infof("successfully delete virtuozzo storage share: %s", share)

	return nil
}

var (
	master          = flag.String("master", "", "Master URL")
	kubeconfig      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
	provisionerID   = flag.String("id", "", "Unique provisioner id")
	provisionerName = flag.String("name", "virtuozzo.com/virtuozzo-storage", "Unique provisioner name")
)

func main() {
	flag.Parse()
	flag.Set("logtostderr", "true")
	if *provisionerID == "" {
		glog.Fatalf("You should provide unique provisioner name!")
	}

	var config *rest.Config
	var err error
	if *master != "" || *kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}

	if err != nil {
		glog.Fatalf("Failed to create config: %v", err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	// The controller needs to know what the server version is because out-of-tree
	// provisioners aren't officially supported until 1.5
	serverVersion, err := clientset.Discovery().ServerVersion()
	if err != nil {
		glog.Fatalf("Error getting server version: %v", err)
	}

	// Create the provisioner: it implements the Provisioner interface expected by
	// the controller
	vzFSProvisioner := newVzFSProvisioner(clientset)

	// Start the provision controller which will dynamically provision Virtuozzo Storage PVs
	pc := controller.NewProvisionController(clientset,
		*provisionerName,
		vzFSProvisioner,
		serverVersion.GitVersion,
	)

	pc.Run(wait.NeverStop)
}
<commit_msg>Allow only claims with ReadWriteOnce access mode<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"syscall"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/dustin/go-humanize"
	"github.com/kolyshkin/goploop-cli"
	"github.com/virtuozzo/ploop-flexvol/vstorage"
)

const (
	parentProvisionerAnn = "vzFSParentProvisioner"
	vzShareAnn           = "vzShare"
)

type vzFSProvisioner struct {
	// Kubernetes Client. Use to retrieve secrets with Virtuozzo Storage credentials
	client kubernetes.Interface
}

func newVzFSProvisioner(client kubernetes.Interface) controller.Provisioner {
	return &vzFSProvisioner{
		client: client,
	}
}

var _ controller.Provisioner = &vzFSProvisioner{}

const provisionerDir = "/export/virtuozzo-provisioner/"
const mountDir = provisionerDir + "mnt/"

func prepareVstorage(options map[string]string, clusterName string, clusterPassword string) error {
	mount := mountDir + clusterName
	mounted, _ := vstorage.IsVstorage(mount)
	if mounted {
		return nil
	}

	if err := os.MkdirAll(mount, 0755); err != nil {
		return err
	}

	v := vstorage.Vstorage{clusterName}
	p, _ := v.Mountpoint()
	if p != "" {
		return syscall.Mount(p, mount, "", syscall.MS_BIND, "")
	}

	if err := v.Auth(clusterPassword); err != nil {
		return err
	}
	if err := v.Mount(mount); err != nil {
		return err
	}

	return nil
}

func createPloop(mount string, options map[string]string) error {
	var (
		volumePath, volumeID, size string
	)

	for k, v := range options {
		switch k {
		case "volumePath":
			volumePath = v
		case "volumeID":
			volumeID = v
		case "size":
			size = v
		case "vzsReplicas":
		case "vzsFailureDomain":
		case "vzsEncoding":
		case "vzsTier":
		case "kubernetes.io/readwrite":
		case "kubernetes.io/fsType":
		default:
		}
	}

	if volumePath == "" {
		return fmt.Errorf("volumePath isn't specified")
	}

	if volumeID == "" {
		return fmt.Errorf("volumeID isn't specified")
	}

	if size == "" {
		return fmt.Errorf("size isn't specified")
	}

	// get a human readable size from the map
	bytes, _ := humanize.ParseBytes(size)

	// ploop driver takes kilobytes, so convert it
	volumeSize := bytes / 1024

	ploopPath := mount + "/" + options["volumePath"] + "/" + options["volumeID"]

	// make the base directory where the volume will go
	err := os.MkdirAll(ploopPath, 0755)
	if err != nil {
		return err
	}

	for k, v := range options {
		var err error
		attr := ""
		switch k {
		case "vzsReplicas":
			attr = "replicas"
		case "vzsTier":
			attr = "tier"
		case "vzsEncoding":
			attr = "encoding"
		case "vzsFailureDomain":
			attr = "failure-domain"
		}
		if attr != "" {
			cmd := "vstorage"
			args := []string{"set-attr", "-R", ploopPath,
				fmt.Sprintf("%s=%s", attr, v)}
			err = exec.Command(cmd, args...).Run()
		}

		if err != nil {
			os.RemoveAll(ploopPath)
			return fmt.Errorf("Unable to set %s to %s: %v", attr, v, err)
		}
	}

	// Create the ploop volume
	cp := ploop.CreateParam{Size: volumeSize, File: ploopPath + "/" + options["volumeID"]}
	if err := ploop.Create(&cp); err != nil {
		return err
	}

	return nil
}

// Provision creates a storage asset and returns a PV object representing it.
func (p *vzFSProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	modes := options.PVC.Spec.AccessModes
	if len(modes) == 0 {
		// if AccessModes field is absent, ReadWriteOnce is used by default
		modes = append(modes, v1.ReadWriteOnce)
	} else {
		if len(modes) != 1 && modes[0] != v1.ReadWriteOnce {
			return nil, fmt.Errorf("Virtuozzo flexvolume provisioner supports only ReadWriteOnce access mode")
		}
	}
	capacity := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	bytes := capacity.Value()

	if options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}
	share := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())

	glog.Infof("Add %s %s", share, humanize.Bytes(uint64(bytes)))

	storageClassOptions := map[string]string{}
	for k, v := range options.Parameters {
		storageClassOptions[k] = v
	}

	storageClassOptions["volumeID"] = share
	storageClassOptions["size"] = fmt.Sprintf("%d", bytes)
	secretName := storageClassOptions["secretName"]
	delete(storageClassOptions, "secretName")

	secret, err := p.client.Core().Secrets(options.PVC.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	if err := prepareVstorage(storageClassOptions, name, password); err != nil {
		return nil, err
	}

	if err := createPloop(mountDir+name, storageClassOptions); err != nil {
		return nil, err
	}

	storageClassOptions["clusterName"] = name
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: options.PVName,
			Annotations: map[string]string{
				parentProvisionerAnn: *provisionerID,
				vzShareAnn:           share,
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   modes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    "virtuozzo/ploop",
					SecretRef: &v1.LocalObjectReference{Name: secretName},
					Options:   storageClassOptions,
				},
			},
		},
	}

	glog.Infof("successfully created virtuozzo storage share: %s", share)

	return pv, nil
}

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *vzFSProvisioner) Delete(volume *v1.PersistentVolume) error {
	ann, ok := volume.Annotations[parentProvisionerAnn]
	if !ok {
		return errors.New("Parent provisioner name annotation not found on PV")
	}
	if ann != *provisionerID {
		return &controller.IgnoredError{"parent provisioner name annotation on PV does not match ours"}
	}
	share, ok := volume.Annotations[vzShareAnn]
	if !ok {
		return errors.New("vz share annotation not found on PV")
	}

	secretName := volume.Spec.PersistentVolumeSource.FlexVolume.SecretRef.Name
	options := volume.Spec.PersistentVolumeSource.FlexVolume.Options

	secret, err := p.client.Core().Secrets(volume.Spec.ClaimRef.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	mount := mountDir + name
	if err := prepareVstorage(options, name, password); err != nil {
		return err
	}

	path := mount + "/" + options["volumePath"] + "/" + options["volumeID"]
	glog.Infof("Delete: %s", path)
	err = os.RemoveAll(path)
	if err != nil {
		return err
	}

	glog.Infof("successfully delete virtuozzo storage share: %s", share)

	return nil
}

var (
	master          = flag.String("master", "", "Master URL")
	kubeconfig      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
	provisionerID   = flag.String("id", "", "Unique provisioner id")
	provisionerName = flag.String("name", "virtuozzo.com/virtuozzo-storage", "Unique provisioner name")
)

func main() {
	flag.Parse()
	flag.Set("logtostderr", "true")
	if *provisionerID == "" {
		glog.Fatalf("You should provide unique provisioner name!")
	}

	var config *rest.Config
	var err error
	if *master != "" || *kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}

	if err != nil {
		glog.Fatalf("Failed to create config: %v", err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	// The controller needs to know what the server version is because out-of-tree
	// provisioners aren't officially supported until 1.5
	serverVersion, err := clientset.Discovery().ServerVersion()
	if err != nil {
		glog.Fatalf("Error getting server version: %v", err)
	}

	// Create the provisioner: it implements the Provisioner interface expected by
	// the controller
	vzFSProvisioner := newVzFSProvisioner(clientset)

	// Start the provision controller which will dynamically provision Virtuozzo Storage PVs
	pc := controller.NewProvisionController(clientset,
		*provisionerName,
		vzFSProvisioner,
		serverVersion.GitVersion,
	)

	pc.Run(wait.NeverStop)
}
<|endoftext|>


<commit_before>/*
Package nlp provides implementations of selected machine learning algorithms for natural language processing of text corpora.  The primary focus is the statistical semantics of plain-text documents supporting semantic analysis and retrieval of semantically similar documents.

The package makes use of the Gonum (http://http//www.gonum.org/) library for linear algebra and scientific computing with some inspiration taken from Python's scikit-learn (http://scikit-learn.org/stable/) and Gensim(https://radimrehurek.com/gensim/)

Overview

The primary intended use case is to support document input as text strings encoded as a matrix of numerical feature vectors called a `term document matrix`.  Each column in the matrix corresponds to a document in the corpus and each row corresponds to a unique term occurring in the corpus.  The individual elements within the matrix contain the frequency with which each term occurs within each document (referred to as `term frequency`).  Whilst textual data from document corpora are the primary intended use case, the algorithms can be used with other types of data from other sources once encoded (vectorised) into a suitable matrix e.g. image data, sound data, users/products, etc.

These matrices can be processed and manipulated through the application of additional transformations for weighting features, identifying relationships or optimising the data for analysis, information retrieval and/or predictions.

Typically the algorithms in this package implement one of three primary interfaces:

	Vectoriser - Taking document input as strings and outputting matrices of numerical features.
	Transformer - Takes matrices of numerical features and applies logic/transformation to output a new matrix.
	Comparer - Functions taking two vectors (columns from a matrix) and outputting a distance/similarity measure.

One of the implementations of Vectoriser is Pipeline which can be used to wire together pipelines composed of a Vectoriser and one or more Transformers arranged in serial so that the output from each stage forma the input of the next.  This can be used to construct a classic LSI (Latent Semantic Indexing) pipeline (vectoriser -> TF.IDF weighting -> Truncated SVD):

	pipeline := nlp.NewPipeline(
		nlp.NewCountVectoriser(true),
		nlp.NewTFIDFTransformer(),
		nlp.NewTruncatedSVD(100),
	)

Whilst they take different inputs, both Vectorisers and Transformers have 3 primary methods:

	Fit() - Trains the model based upon the supplied, input training data.
	Transform() - Transforms the input into the output matrix (requires the model to be already fitted by a previous call to Fit or FitTransform).
	FitTransform() - Convenience method combining Fit and Transform methods so that the model is both trained and used to transform the data within a single step.
*/
package nlp
<commit_msg>updated documentation<commit_after>/*
Package nlp provides implementations of selected machine learning algorithms for natural language processing of text corpora.  The primary focus is the statistical semantics of plain-text documents supporting semantic analysis and retrieval of semantically similar documents.

The package makes use of the Gonum (http://http//www.gonum.org/) library for linear algebra and scientific computing with some inspiration taken from Python's scikit-learn (http://scikit-learn.org/stable/) and Gensim(https://radimrehurek.com/gensim/)

Overview

The primary intended use case is to support document input as text strings encoded as a matrix of numerical feature vectors called a `term document matrix`.  Each column in the matrix corresponds to a document in the corpus and each row corresponds to a unique term occurring in the corpus.  The individual elements within the matrix contain the frequency with which each term occurs within each document (referred to as `term frequency`).  Whilst textual data from document corpora are the primary intended use case, the algorithms can be used with other types of data from other sources once encoded (vectorised) into a suitable matrix e.g. image data, sound data, users/products, etc.

These matrices can be processed and manipulated through the application of additional transformations for weighting features, identifying relationships or optimising the data for analysis, information retrieval and/or predictions.

Typically the algorithms in this package implement one of three primary interfaces:

	Vectoriser - Taking document input as strings and outputting matrices of numerical features e.g. term frequency.
	Transformer - Takes matrices of numerical features and applies some logic/transformation to output a new matrix.
	Comparer - Functions taking two vectors (columns from a matrix) and outputting a distance/similarity measure.

One of the implementations of Vectoriser is Pipeline which can be used to wire together pipelines composed of a Vectoriser and one or more Transformers arranged in serial so that the output from each stage forms the input of the next.  This can be used to construct a classic LSI (Latent Semantic Indexing) pipeline (vectoriser -> TF.IDF weighting -> Truncated SVD):

	pipeline := nlp.NewPipeline(
		nlp.NewCountVectoriser(true),
		nlp.NewTFIDFTransformer(),
		nlp.NewTruncatedSVD(100),
	)

Whilst they take different inputs, both Vectorisers and Transformers have 3 primary methods:

	Fit() - Trains the model based upon the supplied, input training data.
	Transform() - Transforms the input into the output matrix (requires the model to be already fitted by a previous call to Fit() or FitTransform()).
	FitTransform() - Convenience method combining Fit() and Transform() methods to transform input data, fitting the model to the input data in the process.
*/
package nlp
<|endoftext|>


<commit_before>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Gonew generates new Go project directories. Its produced project
directories contain stub files and initialized repositories (only
git/github supported now). It can be used to create new packages and
commands.

The gonew configuration file is stored at ~/.gonewrc. It is generated the
first time you run gonew. Command line options can be used to override
some details of the configuration file.

Usage:

    gonew [options] cmd NAME
    gonew [options] pkg NAME
    gonew [options] lib NAME PKG

Arguments:

    NAME
            The name of the new project/repo.

    PKG
            The package a library (.go file) belongs to.

Options:

    -target=""
            Makefile target. The executable name in case the argument
            TYPE is "cmd", package name in case of "pkg". The default
            value based on the argument NAME.

    -import=""
            Colon ':' separated list of packages to include in source
            .go files. The packages are not imported in any tests, or
            the options.go file (or doc.go) created for cmd projects.

    -repo=""
            Repository type (currently, "git" is the only supported
            repository type).

    -remote=""
            When passed a url to a remote repository, attempt to
            initialize the remote repository to the new project's
            repository. The url passed in must agree with the host
            specified in the config file (or by -host).

    -host=""
            Repository host if any (currently, "github" is the only
            supported host). The value supplied must agree with the
            value of -repo.

    -user=""
            Username for the repo host (necessary for "github").

    -v
            Print verbose output to the stdout (this intersects with
            some -debug output).

    -debug=-1
            When set to a non-negative value, debugging output will be
            printed.

    -help
            Print a usage message

Configuration:

The configuration for gonew is simple. The configuration can provide
default hosts, usernames, and repositories. However, it also contains the
setting of the {{name}} and {{email}} template function values.

The configuration file for gonew (~/.gonewrc) is generated on the spot if
one does not exist. So you do not need to worry about editing it for the
most part.

If you wish to write/edit your own configuration file. An example can be
found at the path

    $GOROOT/src/pkg/github.com/bmatsuo/gonew/gonewrc.example

Examples:

    gonew -target=mp3lib pkg go-mp3lib
    gonew lib decode mp3lib
    gonew -remote=git@github.com:bmatsuo/goplay.git cmd goplay

Custom Templates:

Custom file templates can be created and stored in a heirarchy separate
from the standard Gonew template heirarchy at

    $GOROOT/src/pkg/github.com/bmatsuo/gonew/templates

Supply a custom template heirarchy path in the ~/.gonewrc file. See the
example config file for more info about how to do this.

The custom heirarchy is parsed as a template.Set and can reference other
templates in the heirarchy (http://golang.org/pkg/template). Templates,
however, cannot reference templates in the default Gonew template
heirarchy. In order to do so, copies or symbolic links of the desired
templates must be made in the custom heirarchy.

Templates must be given a ".t" extension in order to be recognized and
parsed by Gonew. When generating a new project or library, Gonew uses
templates with specific names to generate the files. In order to use
custom templates, a template with the proper name must be found in the
custom heirarchy. Increase the Gonew debugging variable for information
about with templates are being executed.

Template Functions:

All templates used by Gonew have acces to a small library of simple
helper functions. The {{name}} and {{email}} variables have already been
discussed. Here is list of all available template functions.

    import [PACKAGE [...]]
            Produces an import statement which includes the packages
            specified in it arguments. The arguments can be either
            strings or slices of strings.

    func NAME [ARGUMENT [...]]
            Produces the definition of a function with zero return
            values. The arguments should be supplied as identifier-type
            pairs. The name should be a valid go identifier.

    date [FORMAT]
            Produces the current date-time as human readable string when
            called with no argument. A format can be supplied in the form
            of an example string formatting of a specific day (see "time").

    year
            Produces the current year in a four-digit format.

    name
            Produces the Gonew user's name as defined in ~/.gonewrc

    email
            Produces the Gonew user's email as defined in ~/.gonewrc

Template Contexts:

The data supplied to each template a type known as a Context. Contexts
have a defined method interface that can be called from inside a template.
Read about the Context interface using godoc

    godoc $GOROOT/src/pkg/github.com/bmatsuo/gonew Context | less

*/
package documentation
<commit_msg>Add -test flag to doc.go.<commit_after>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Gonew generates new Go project directories. Its produced project
directories contain stub files and initialized repositories (only
git/github supported now). It can be used to create new packages and
commands.

The gonew configuration file is stored at ~/.gonewrc. It is generated the
first time you run gonew. Command line options can be used to override
some details of the configuration file.

Usage:

    gonew [options] cmd NAME
    gonew [options] pkg NAME
    gonew [options] lib NAME PKG

Arguments:

    NAME
            The name of the new project/repo.

    PKG
            The package a library (.go file) belongs to.

Options:

	-test=false
			Test files will not be produced.

    -target=""
            Makefile target. The executable name in case the argument
            TYPE is "cmd", package name in case of "pkg". The default
            value based on the argument NAME.

    -import=""
            Colon ':' separated list of packages to include in source
            .go files. The packages are not imported in any tests, or
            the options.go file (or doc.go) created for cmd projects.

    -repo=""
            Repository type (currently, "git" is the only supported
            repository type).

    -remote=""
            When passed a url to a remote repository, attempt to
            initialize the remote repository to the new project's
            repository. The url passed in must agree with the host
            specified in the config file (or by -host).

    -host=""
            Repository host if any (currently, "github" is the only
            supported host). The value supplied must agree with the
            value of -repo.

    -user=""
            Username for the repo host (necessary for "github").

    -v
            Print verbose output to the stdout (this intersects with
            some -debug output).

    -debug=-1
            When set to a non-negative value, debugging output will be
            printed.

    -help
            Print a usage message

Configuration:

The configuration for gonew is simple. The configuration can provide
default hosts, usernames, and repositories. However, it also contains the
setting of the {{name}} and {{email}} template function values.

The configuration file for gonew (~/.gonewrc) is generated on the spot if
one does not exist. So you do not need to worry about editing it for the
most part.

If you wish to write/edit your own configuration file. An example can be
found at the path

    $GOROOT/src/pkg/github.com/bmatsuo/gonew/gonewrc.example

Examples:

    gonew -target=mp3lib pkg go-mp3lib
    gonew lib decode mp3lib
    gonew -remote=git@github.com:bmatsuo/goplay.git cmd goplay

Custom Templates:

Custom file templates can be created and stored in a heirarchy separate
from the standard Gonew template heirarchy at

    $GOROOT/src/pkg/github.com/bmatsuo/gonew/templates

Supply a custom template heirarchy path in the ~/.gonewrc file. See the
example config file for more info about how to do this.

The custom heirarchy is parsed as a template.Set and can reference other
templates in the heirarchy (http://golang.org/pkg/template). Templates,
however, cannot reference templates in the default Gonew template
heirarchy. In order to do so, copies or symbolic links of the desired
templates must be made in the custom heirarchy.

Templates must be given a ".t" extension in order to be recognized and
parsed by Gonew. When generating a new project or library, Gonew uses
templates with specific names to generate the files. In order to use
custom templates, a template with the proper name must be found in the
custom heirarchy. Increase the Gonew debugging variable for information
about with templates are being executed.

Template Functions:

All templates used by Gonew have acces to a small library of simple
helper functions. The {{name}} and {{email}} variables have already been
discussed. Here is list of all available template functions.

    import [PACKAGE [...]]
            Produces an import statement which includes the packages
            specified in it arguments. The arguments can be either
            strings or slices of strings.

    func NAME [ARGUMENT [...]]
            Produces the definition of a function with zero return
            values. The arguments should be supplied as identifier-type
            pairs. The name should be a valid go identifier.

    date [FORMAT]
            Produces the current date-time as human readable string when
            called with no argument. A format can be supplied in the form
            of an example string formatting of a specific day (see "time").

    year
            Produces the current year in a four-digit format.

    name
            Produces the Gonew user's name as defined in ~/.gonewrc

    email
            Produces the Gonew user's email as defined in ~/.gonewrc

Template Contexts:

The data supplied to each template a type known as a Context. Contexts
have a defined method interface that can be called from inside a template.
Read about the Context interface using godoc

    godoc $GOROOT/src/pkg/github.com/bmatsuo/gonew Context | less

*/
package documentation
<|endoftext|>


<commit_before>/*
Ratchet is a library for performing data pipeline / ETL tasks in Go.

The main construct in Ratchet is Pipeline. A Pipeline has a series of
PipelineStages, which will each perform some type of data processing, and
then send new data on to the next stage. Each PipelineStage consists of one
or more DataProcessors, which are responsible for receiving, processing, and
then sending data on to the next stage of processing. DataProcessors each
run in their own goroutine, and therefore all data processing can be executing
concurrently.

Here is a conceptual drawing of a fairly simple Pipeline:

        +--Pipeline------------------------------------------------------------------------------------------+
        |                                                                       PipelineStage 3              |
        |                                                                      +---------------------------+ |
        |  PipelineStage 1                 PipelineStage 2          +-JSON---> |  CSVWriter                | |
        | +------------------+           +-----------------------+  |          +---------------------------+ |
        | |  SQLReader       +-JSON----> | Custom DataProcessor  +--+                                        |
        | +------------------+           +-----------------------+  |          +---------------------------+ |
        |                                                           +-JSON---> |  SQLWriter                | |
        |                                                                      +---------------------------+ |
        +----------------------------------------------------------------------------------------------------+

In this example, we have a Pipeline consisting of 3 PipelineStages. The first stage has a DataProcessor that
runs queries on a SQL database, the second is doing custom transformation
work on that data, and the third stage branches into 2 DataProcessors, one
writing the resulting data to a CSV file, and the other inserting into another
SQL database.

In the example above, Stage 1 and Stage 3 are using built-in DataProcessors
(see the "processors" package/subdirectory). However, Stage 2 is using a custom
implementation of DataProcessor. By using a combination of built-in processors,
and supporting the writing of any Go code to process data, Ratchet makes
it possible to write very custom and fast data pipeline systems. See the
DataProcessor documentation to learn more.

Since each DataProcessor is running in it's own goroutine, SQLReader can continue pulling and sending
data while each subsequent stage is also processing data. Optimally-designed pipelines
have processors that can each run in an isolated fashion, processing data without having
to worry about what's coming next down the pipeline.

All data payloads sent between DataProcessors are of type data.JSON ([]byte). This provides
a good balance of consistency and flexibility. See the "data" package for details
and helper functions for dealing with data.JSON. Another good read for handling
JSON data in Go is http://blog.golang.org/json-and-go.

Note that many of the concepts in Ratchet were taken from the Golang blog's post on
pipelines (http://blog.golang.org/pipelines). While the details discussed in that
blog post are largely abstracted away by Ratchet, it is still an interesting read and
will help explain the general concepts being applied.

Creating and Running a Basic Pipeline

There are two ways to construct and run a Pipeline. The first is a basic, non-branching
Pipeline. For example:

        +------------+   +-------------------+   +---------------+
        | SQLReader  +---> CustomTransformer +---> SQLWriter     |
        +------------+   +-------------------+   +---------------+

This is a 3-stage Pipeline that queries some SQL data in stage 1, does some custom data
transformation in stage 2, and then writes the resulting data to a SQL table in stage 3.
The code to create and run this basic Pipeline would look something like:

        // First initalize the DataProcessors
        read := processors.NewSQLReader(db1, "SELECT * FROM source_table")
        transform := NewCustomTransformer() // (This would your own custom DataProcessor implementation)
        write := processors.NewSQLWriter(db2, "destination_table")

        // Then create a new Pipeline using them
        pipeline := ratchet.NewPipeline(read, transform, write)

        // Finally, run the Pipeline and wait for either an error or nil to be returned
        err := <-pipeline.Run()

Creating and Running a Branching Pipeline

The second way to construct a Pipeline is using a PipelineLayout. This method allows
for more complex Pipeline configurations that support branching between stages that
are running multiple DataProcessors. Here is a (fairly complex) example:

                                                                   +----------------------+
                                                            +------> SQLReader (Dynamic)  +--+
                                                            |      +----------------------+  |
                                                            |                                |
                         +---------------------------+      |      +----------------------+  |    +-----------+
                   +-----> SQLReader (Dynamic Query) +------+   +--> Custom DataProcessor +-------> CSVWriter |
    +-----------+  |     +---------------------------+      |   |  +----------------------+  |    +-----------+
    | SQLReader +--+                                     +------+                            |
    +-----------+  |     +---------------------------+   |  |      +----------------------+  |    +-----------+
                   +-----> Custom DataProcessor      +------+------> Custom DataProcessor +--+  +-> SQLWriter |
                         +---------------------------+   |         +----------------------+     | +-----------+
                                                         |                                      |
                                                         |         +----------------------+     |
                                                         +---------> Passthrough          +-----+
                                                                   +----------------------+

This Pipeline consists of 4 stages where each DataProcessor is choosing which DataProcessors
in the subsequent stage should receive the data it sends. The SQLReader in stage 2, for example,
is sending data to only 2 processors in the next stage, while the Custom DataProcessor in
stage 2 is sending it's data to 3. The code for constructing and running a Pipeline like this
would look like:

        // First, initialize all the DataProcessors that will be used in the Pipeline
        query1 := processors.NewSQLReader(db1, "SELECT * FROM source_table")
        query2 := processors.NewSQLReader(db1, sqlGenerator1) // sqlGenerator1 would be a function that generates the query at run-time. See SQLReader docs.
        custom1 := NewCustomDataProcessor1()
        query3 := processors.NewSQLReader(db2, sqlGenerator2)
        custom2 := NewCustomDataProcessor2()
        custom3 := NewCustomDataProcessor3()
        passthrough := processors.NewPassthrough()
        writeMySQL := processors.NewSQLWriter(db3, "destination_table")
        writeCSV := processors.NewCSVWriter(file)

        // Next, construct and validate the PipelineLayout. Each DataProcessor
        // is inserted into the layout via calls to ratchet.Do().
        layout, err := ratchet.NewPipelineLayout(
                ratchet.NewPipelineStage(
                        ratchet.Do(query1).Outputs(query2),
                        ratchet.Do(query1).Outputs(custom1),
                ),
                ratchet.NewPipelineStage(
                        ratchet.Do(query2).Outputs(query3, custom3),
                        ratchet.Do(custom1).Outputs(custom2, custom3, passthrough),
                ),
                ratchet.NewPipelineStage(
                        ratchet.Do(query3).Outputs(writeCSV),
                        ratchet.Do(custom2).Outputs(writeCSV),
                        ratchet.Do(custom3).Outputs(writeCSV),
                        ratchet.Do(passthrough).Outputs(writeMySQL),
                ),
                ratchet.NewPipelineStage(
                        ratchet.Do(writeCSV),
                        ratchet.Do(writeMySQL),
                ),
        )
        if err != nil {
                // layout is invalid
                panic(err.Error())
        }

        // Finally, create and run the Pipeline
        pipeline := ratchet.NewBranchingPipeline(layout)
        err = <-pipeline.Run()

This example is only conceptual, the main points being to explain the flexibility
you have when designing your Pipeline's layout and to demonstrate the syntax for
constructing a new PipelineLayout.

*/
package ratchet
<commit_msg>Use doc.go standard<commit_after>/*
Package ratchet is a library for performing data pipeline / ETL tasks in Go.

The main construct in Ratchet is Pipeline. A Pipeline has a series of
PipelineStages, which will each perform some type of data processing, and
then send new data on to the next stage. Each PipelineStage consists of one
or more DataProcessors, which are responsible for receiving, processing, and
then sending data on to the next stage of processing. DataProcessors each
run in their own goroutine, and therefore all data processing can be executing
concurrently.

Here is a conceptual drawing of a fairly simple Pipeline:

        +--Pipeline------------------------------------------------------------------------------------------+
        |                                                                       PipelineStage 3              |
        |                                                                      +---------------------------+ |
        |  PipelineStage 1                 PipelineStage 2          +-JSON---> |  CSVWriter                | |
        | +------------------+           +-----------------------+  |          +---------------------------+ |
        | |  SQLReader       +-JSON----> | Custom DataProcessor  +--+                                        |
        | +------------------+           +-----------------------+  |          +---------------------------+ |
        |                                                           +-JSON---> |  SQLWriter                | |
        |                                                                      +---------------------------+ |
        +----------------------------------------------------------------------------------------------------+

In this example, we have a Pipeline consisting of 3 PipelineStages. The first stage has a DataProcessor that
runs queries on a SQL database, the second is doing custom transformation
work on that data, and the third stage branches into 2 DataProcessors, one
writing the resulting data to a CSV file, and the other inserting into another
SQL database.

In the example above, Stage 1 and Stage 3 are using built-in DataProcessors
(see the "processors" package/subdirectory). However, Stage 2 is using a custom
implementation of DataProcessor. By using a combination of built-in processors,
and supporting the writing of any Go code to process data, Ratchet makes
it possible to write very custom and fast data pipeline systems. See the
DataProcessor documentation to learn more.

Since each DataProcessor is running in it's own goroutine, SQLReader can continue pulling and sending
data while each subsequent stage is also processing data. Optimally-designed pipelines
have processors that can each run in an isolated fashion, processing data without having
to worry about what's coming next down the pipeline.

All data payloads sent between DataProcessors are of type data.JSON ([]byte). This provides
a good balance of consistency and flexibility. See the "data" package for details
and helper functions for dealing with data.JSON. Another good read for handling
JSON data in Go is http://blog.golang.org/json-and-go.

Note that many of the concepts in Ratchet were taken from the Golang blog's post on
pipelines (http://blog.golang.org/pipelines). While the details discussed in that
blog post are largely abstracted away by Ratchet, it is still an interesting read and
will help explain the general concepts being applied.

Creating and Running a Basic Pipeline

There are two ways to construct and run a Pipeline. The first is a basic, non-branching
Pipeline. For example:

        +------------+   +-------------------+   +---------------+
        | SQLReader  +---> CustomTransformer +---> SQLWriter     |
        +------------+   +-------------------+   +---------------+

This is a 3-stage Pipeline that queries some SQL data in stage 1, does some custom data
transformation in stage 2, and then writes the resulting data to a SQL table in stage 3.
The code to create and run this basic Pipeline would look something like:

        // First initalize the DataProcessors
        read := processors.NewSQLReader(db1, "SELECT * FROM source_table")
        transform := NewCustomTransformer() // (This would your own custom DataProcessor implementation)
        write := processors.NewSQLWriter(db2, "destination_table")

        // Then create a new Pipeline using them
        pipeline := ratchet.NewPipeline(read, transform, write)

        // Finally, run the Pipeline and wait for either an error or nil to be returned
        err := <-pipeline.Run()

Creating and Running a Branching Pipeline

The second way to construct a Pipeline is using a PipelineLayout. This method allows
for more complex Pipeline configurations that support branching between stages that
are running multiple DataProcessors. Here is a (fairly complex) example:

                                                                   +----------------------+
                                                            +------> SQLReader (Dynamic)  +--+
                                                            |      +----------------------+  |
                                                            |                                |
                         +---------------------------+      |      +----------------------+  |    +-----------+
                   +-----> SQLReader (Dynamic Query) +------+   +--> Custom DataProcessor +-------> CSVWriter |
    +-----------+  |     +---------------------------+      |   |  +----------------------+  |    +-----------+
    | SQLReader +--+                                     +------+                            |
    +-----------+  |     +---------------------------+   |  |      +----------------------+  |    +-----------+
                   +-----> Custom DataProcessor      +------+------> Custom DataProcessor +--+  +-> SQLWriter |
                         +---------------------------+   |         +----------------------+     | +-----------+
                                                         |                                      |
                                                         |         +----------------------+     |
                                                         +---------> Passthrough          +-----+
                                                                   +----------------------+

This Pipeline consists of 4 stages where each DataProcessor is choosing which DataProcessors
in the subsequent stage should receive the data it sends. The SQLReader in stage 2, for example,
is sending data to only 2 processors in the next stage, while the Custom DataProcessor in
stage 2 is sending it's data to 3. The code for constructing and running a Pipeline like this
would look like:

        // First, initialize all the DataProcessors that will be used in the Pipeline
        query1 := processors.NewSQLReader(db1, "SELECT * FROM source_table")
        query2 := processors.NewSQLReader(db1, sqlGenerator1) // sqlGenerator1 would be a function that generates the query at run-time. See SQLReader docs.
        custom1 := NewCustomDataProcessor1()
        query3 := processors.NewSQLReader(db2, sqlGenerator2)
        custom2 := NewCustomDataProcessor2()
        custom3 := NewCustomDataProcessor3()
        passthrough := processors.NewPassthrough()
        writeMySQL := processors.NewSQLWriter(db3, "destination_table")
        writeCSV := processors.NewCSVWriter(file)

        // Next, construct and validate the PipelineLayout. Each DataProcessor
        // is inserted into the layout via calls to ratchet.Do().
        layout, err := ratchet.NewPipelineLayout(
                ratchet.NewPipelineStage(
                        ratchet.Do(query1).Outputs(query2),
                        ratchet.Do(query1).Outputs(custom1),
                ),
                ratchet.NewPipelineStage(
                        ratchet.Do(query2).Outputs(query3, custom3),
                        ratchet.Do(custom1).Outputs(custom2, custom3, passthrough),
                ),
                ratchet.NewPipelineStage(
                        ratchet.Do(query3).Outputs(writeCSV),
                        ratchet.Do(custom2).Outputs(writeCSV),
                        ratchet.Do(custom3).Outputs(writeCSV),
                        ratchet.Do(passthrough).Outputs(writeMySQL),
                ),
                ratchet.NewPipelineStage(
                        ratchet.Do(writeCSV),
                        ratchet.Do(writeMySQL),
                ),
        )
        if err != nil {
                // layout is invalid
                panic(err.Error())
        }

        // Finally, create and run the Pipeline
        pipeline := ratchet.NewBranchingPipeline(layout)
        err = <-pipeline.Run()

This example is only conceptual, the main points being to explain the flexibility
you have when designing your Pipeline's layout and to demonstrate the syntax for
constructing a new PipelineLayout.

*/
package ratchet
<|endoftext|>


<commit_before>/*
package mathhelper implements helper math functions
*/
package mathhelper
<commit_msg>Replace 'package' with 'Package' to remove golint warnings.<commit_after>/*
Package mathhelper implements helper math functions
*/
package mathhelper
<|endoftext|>


<commit_before>// Copyright 2014 Benjamin Borbe. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http
<commit_msg>update doc<commit_after>// Copyright (c) 2015, Benjamin Borbe <bborbe@rocketnews.de>.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http
<|endoftext|>


<commit_before>// Package ari provides a Go library for interacting with Asterisk ARI
package ari // import "github.com/CyCoreSystems/ari"
<commit_msg>remove package import comment<commit_after>// Package ari provides a Go library for interacting with Asterisk ARI
package ari
<|endoftext|>


<commit_before>/*
Gohun exposes all of hunspell's functionality and it does away with hunspell's need for files, allowing you to pass raw buffers to create dictionary objects. This, obviously, makes it easier to run a distributed spell checking program that can rely on an SSOT, like a database.

Installation

You must have pkg-config installed, and you have to include $GOPATH/pkgconfig in its paths to search. Then you have to run make just once to install the library. Thereafter, as long as pkg-config remains aware of the new path, you should only have to use the go build tools.

export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:${GOPATH}/pkgconfig
cd ./gohun
make
go install gohun
*/
package gohun
<commit_msg>adding tabs to code<commit_after>/*
Gohun exposes all of hunspell's functionality and it does away with hunspell's need for files, allowing you to pass raw buffers to create dictionary objects. This, obviously, makes it easier to run a distributed spell checking program that can rely on an SSOT, like a database.

Installation

You must have pkg-config installed, and you have to include $GOPATH/pkgconfig in its paths to search. Then you have to run make just once to install the library. Thereafter, as long as pkg-config remains aware of the new path, you should only have to use the go build tools.

	export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:${GOPATH}/pkgconfig
	cd ./gohun
	make
	go install gohun
*/
package gohun
<|endoftext|>


<commit_before>/*
 * Copyright © 2019 Hedzr Yeh.
 */

package consul_tags

const (
	AppName    = "consul-tags" //
	Version    = "0.5.5"       //
	VersionInt = 0x000505      // using as
)
<commit_msg>v0.5.6: prerelease<commit_after>/*
 * Copyright © 2019 Hedzr Yeh.
 */

package consul_tags

const (
	AppName    = "consul-tags" //
	Version    = "0.5.6"       //
	VersionInt = 0x000506      // using as
)
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

// Package otp implements both HOTP and TOTP based
// one time passcodes in a Google Authenticator compatible manner.
//
// When adding a TOTP for a user, you must store the "secret" value
// persistently. It is recommend to store the in an encrypted field in your
// datastore.  Due to how TOTP works, it is not possible to store a hash
// for the secret value like you would a password.
//
// To enroll a user, you must first generate an OTP for them.  Google
// Authenticator supports using a QR code as an enrollment method:
//
//	import (
//		"github.com/pquerna/otp/totp"
//
//		"bytes"
//		"image/png"
//	)
//
//	key, err := totp.Generate(totp.GenerateOpts{
//			Issuer: "Example.com",
//			AccountName: "alice@example.com"
//	})
//
//	// Convert TOTP key into a QR code encoded as a PNG image.
//	var buf bytes.Buffer
//	img, err := key.Image(200, 200)
//	png.Encode(&buf, img)
//
//	// display the QR code to the user.
//	display(buf.Bytes())
//
//	// Now Validate that the user's successfully added the passcode.
//	passcode := promptForPasscode()
//	valid := totp.Validate(passcode, key.Secret())
//
//	if valid {
//		// User successfully used their TOTP, save it to your backend!
//		storeSecret("alice@example.com", key.Secret())
//	}
//
// Validating a TOTP passcode is very easy, just prompt the user for a passcode
// and retrieve the associated user's previously stored secret.
//	import "github.com/pquerna/otp/totp"
//
//	passcode := promptForPasscode()
//	secret := getSecret("alice@example.com")
//
//	valid := totp.Validate(passcode, secret)
//
//	if valid {
//		// Success! continue login process.
//	}
package otp
<commit_msg>Clarify doc sentence.<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

// Package otp implements both HOTP and TOTP based
// one time passcodes in a Google Authenticator compatible manner.
//
// When adding a TOTP for a user, you must store the "secret" value
// persistently. It is recommend to store the secret in an encrypted field in your
// datastore.  Due to how TOTP works, it is not possible to store a hash
// for the secret value like you would a password.
//
// To enroll a user, you must first generate an OTP for them.  Google
// Authenticator supports using a QR code as an enrollment method:
//
//	import (
//		"github.com/pquerna/otp/totp"
//
//		"bytes"
//		"image/png"
//	)
//
//	key, err := totp.Generate(totp.GenerateOpts{
//			Issuer: "Example.com",
//			AccountName: "alice@example.com"
//	})
//
//	// Convert TOTP key into a QR code encoded as a PNG image.
//	var buf bytes.Buffer
//	img, err := key.Image(200, 200)
//	png.Encode(&buf, img)
//
//	// display the QR code to the user.
//	display(buf.Bytes())
//
//	// Now Validate that the user's successfully added the passcode.
//	passcode := promptForPasscode()
//	valid := totp.Validate(passcode, key.Secret())
//
//	if valid {
//		// User successfully used their TOTP, save it to your backend!
//		storeSecret("alice@example.com", key.Secret())
//	}
//
// Validating a TOTP passcode is very easy, just prompt the user for a passcode
// and retrieve the associated user's previously stored secret.
//	import "github.com/pquerna/otp/totp"
//
//	passcode := promptForPasscode()
//	secret := getSecret("alice@example.com")
//
//	valid := totp.Validate(passcode, secret)
//
//	if valid {
//		// Success! continue login process.
//	}
package otp
<|endoftext|>


<commit_before>package client
<commit_msg>oops, checked in empty file<commit_after><|endoftext|>


<commit_before>package ctx

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/util"
	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"

	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/dynamic"
	kscheme "k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	k8scontrollerclient "sigs.k8s.io/controller-runtime/pkg/client"

	operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
	operatorsv2 "github.com/operator-framework/api/pkg/operators/v2"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
	controllerclient "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/controller-runtime/client"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
	pversioned "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/client/clientset/versioned"
)

var ctx TestContext

// TestContext represents the environment of an executing test. It can
// be considered roughly analogous to a kubeconfig context.
type TestContext struct {
	restConfig     *rest.Config
	kubeClient     operatorclient.ClientInterface
	operatorClient versioned.Interface
	dynamicClient  dynamic.Interface
	packageClient  pversioned.Interface
	e2eClient      *util.E2EKubeClient
	ssaClient      *controllerclient.ServerSideApplier

	kubeconfigPath      string
	artifactsDir        string
	artifactsScriptPath string

	scheme *runtime.Scheme

	// client is the controller-runtime client -- we should use this from now on
	client k8scontrollerclient.Client
}

// Ctx returns a pointer to the global test context. During parallel
// test executions, Ginkgo starts one process per test "node", and
// each node will have its own context, which may or may not point to
// the same test cluster.
func Ctx() *TestContext {
	return &ctx
}

func (ctx TestContext) Logf(f string, v ...interface{}) {
	util.Logf(f, v...)
}

func (ctx TestContext) Scheme() *runtime.Scheme {
	return ctx.scheme
}

func (ctx TestContext) RESTConfig() *rest.Config {
	return rest.CopyConfig(ctx.restConfig)
}

func (ctx TestContext) KubeClient() operatorclient.ClientInterface {
	return ctx.kubeClient
}

func (ctx TestContext) OperatorClient() versioned.Interface {
	return ctx.operatorClient
}

func (ctx TestContext) DynamicClient() dynamic.Interface {
	return ctx.dynamicClient
}

func (ctx TestContext) PackageClient() pversioned.Interface {
	return ctx.packageClient
}

func (ctx TestContext) Client() k8scontrollerclient.Client {
	return ctx.client
}

func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier {
	return ctx.ssaClient
}

func (ctx TestContext) E2EClient() *util.E2EKubeClient {
	return ctx.e2eClient
}

func (ctx TestContext) NewE2EClientSession() {
	if ctx.e2eClient != nil {
		_ = ctx.e2eClient.Reset()
	}
	ctx.e2eClient = util.NewK8sResourceManager(ctx.Client())
}

func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error {
	if ctx.artifactsDir == "" {
		ctx.Logf("$ARTIFACT_DIR is unset -- not collecting failed test case logs")
		return nil
	}
	ctx.Logf("collecting logs in the %s artifacts directory", ctx.artifactsDir)

	logDir := filepath.Join(ctx.artifactsDir, namespace)
	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
		return err
	}
	kubeconfigPath := ctx.kubeconfigPath
	if kubeconfigPath == "" {
		ctx.Logf("unable to determine kubeconfig path so defaulting to the $KUBECONFIG value")
		kubeconfigPath = os.Getenv("KUBECONFIG")
	}

	envvars := []string{
		"TEST_NAMESPACE=" + namespace,
		"TEST_ARTIFACTS_DIR=" + logDir,
		"KUBECONFIG=" + kubeconfigPath,
		"KUBECTL=" + os.Getenv("KUBECTL"),
	}

	cmd := exec.Command(ctx.artifactsScriptPath)
	cmd.Env = append(cmd.Env, envvars...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func setDerivedFields(ctx *TestContext) error {
	if ctx == nil {
		return fmt.Errorf("nil test context")
	}

	if ctx.restConfig == nil {
		return fmt.Errorf("nil RESTClient")
	}

	if ctx.artifactsDir == "" {
		if artifactsDir := os.Getenv("ARTIFACT_DIR"); artifactsDir != "" {
			ctx.artifactsDir = artifactsDir
		}
	}
	if ctx.artifactsScriptPath == "" {
		if scriptPath := os.Getenv("E2E_ARTIFACT_SCRIPT"); scriptPath != "" {
			ctx.artifactsScriptPath = scriptPath
		}
	}

	kubeClient, err := operatorclient.NewClientFromRestConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.kubeClient = kubeClient

	operatorClient, err := versioned.NewForConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.operatorClient = operatorClient

	dynamicClient, err := dynamic.NewForConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.dynamicClient = dynamicClient

	packageClient, err := pversioned.NewForConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.packageClient = packageClient

	ctx.scheme = runtime.NewScheme()
	localSchemeBuilder := runtime.NewSchemeBuilder(
		apiextensions.AddToScheme,
		kscheme.AddToScheme,
		operatorsv1alpha1.AddToScheme,
		operatorsv1.AddToScheme,
		operatorsv2.AddToScheme,
		apiextensionsv1.AddToScheme,
		appsv1.AddToScheme,
		apiregistrationv1.AddToScheme,
	)
	if err := localSchemeBuilder.AddToScheme(ctx.scheme); err != nil {
		return err
	}

	client, err := k8scontrollerclient.New(ctx.restConfig, k8scontrollerclient.Options{
		Scheme: ctx.scheme,
	})
	if err != nil {
		return err
	}
	ctx.e2eClient = util.NewK8sResourceManager(client)
	ctx.client = ctx.e2eClient

	ctx.ssaClient, err = controllerclient.NewForConfig(ctx.restConfig, ctx.scheme, "test.olm.registry")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>test/e2e: Preserve the existing environment when using exec.Command (#2876)<commit_after>package ctx

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/operator-framework/operator-lifecycle-manager/test/e2e/util"
	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"

	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/dynamic"
	kscheme "k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	k8scontrollerclient "sigs.k8s.io/controller-runtime/pkg/client"

	operatorsv1 "github.com/operator-framework/api/pkg/operators/v1"
	operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
	operatorsv2 "github.com/operator-framework/api/pkg/operators/v2"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned"
	controllerclient "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/controller-runtime/client"
	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
	pversioned "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/client/clientset/versioned"
)

var ctx TestContext

// TestContext represents the environment of an executing test. It can
// be considered roughly analogous to a kubeconfig context.
type TestContext struct {
	restConfig     *rest.Config
	kubeClient     operatorclient.ClientInterface
	operatorClient versioned.Interface
	dynamicClient  dynamic.Interface
	packageClient  pversioned.Interface
	e2eClient      *util.E2EKubeClient
	ssaClient      *controllerclient.ServerSideApplier

	kubeconfigPath      string
	artifactsDir        string
	artifactsScriptPath string

	scheme *runtime.Scheme

	// client is the controller-runtime client -- we should use this from now on
	client k8scontrollerclient.Client
}

// Ctx returns a pointer to the global test context. During parallel
// test executions, Ginkgo starts one process per test "node", and
// each node will have its own context, which may or may not point to
// the same test cluster.
func Ctx() *TestContext {
	return &ctx
}

func (ctx TestContext) Logf(f string, v ...interface{}) {
	util.Logf(f, v...)
}

func (ctx TestContext) Scheme() *runtime.Scheme {
	return ctx.scheme
}

func (ctx TestContext) RESTConfig() *rest.Config {
	return rest.CopyConfig(ctx.restConfig)
}

func (ctx TestContext) KubeClient() operatorclient.ClientInterface {
	return ctx.kubeClient
}

func (ctx TestContext) OperatorClient() versioned.Interface {
	return ctx.operatorClient
}

func (ctx TestContext) DynamicClient() dynamic.Interface {
	return ctx.dynamicClient
}

func (ctx TestContext) PackageClient() pversioned.Interface {
	return ctx.packageClient
}

func (ctx TestContext) Client() k8scontrollerclient.Client {
	return ctx.client
}

func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier {
	return ctx.ssaClient
}

func (ctx TestContext) E2EClient() *util.E2EKubeClient {
	return ctx.e2eClient
}

func (ctx TestContext) NewE2EClientSession() {
	if ctx.e2eClient != nil {
		_ = ctx.e2eClient.Reset()
	}
	ctx.e2eClient = util.NewK8sResourceManager(ctx.Client())
}

func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error {
	if ctx.artifactsDir == "" {
		ctx.Logf("$ARTIFACT_DIR is unset -- not collecting failed test case logs")
		return nil
	}
	ctx.Logf("collecting logs in the %s artifacts directory", ctx.artifactsDir)

	logDir := filepath.Join(ctx.artifactsDir, namespace)
	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
		return err
	}
	kubeconfigPath := ctx.kubeconfigPath
	if kubeconfigPath == "" {
		ctx.Logf("unable to determine kubeconfig path so defaulting to the $KUBECONFIG value")
		kubeconfigPath = os.Getenv("KUBECONFIG")
	}

	envvars := []string{
		"TEST_NAMESPACE=" + namespace,
		"TEST_ARTIFACTS_DIR=" + logDir,
		"KUBECONFIG=" + kubeconfigPath,
		"KUBECTL=" + os.Getenv("KUBECTL"),
	}

	cmd := exec.Command(ctx.artifactsScriptPath)
	cmd.Env = append(os.Environ(), envvars...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func setDerivedFields(ctx *TestContext) error {
	if ctx == nil {
		return fmt.Errorf("nil test context")
	}

	if ctx.restConfig == nil {
		return fmt.Errorf("nil RESTClient")
	}

	if ctx.artifactsDir == "" {
		if artifactsDir := os.Getenv("ARTIFACT_DIR"); artifactsDir != "" {
			ctx.artifactsDir = artifactsDir
		}
	}
	if ctx.artifactsScriptPath == "" {
		if scriptPath := os.Getenv("E2E_ARTIFACT_SCRIPT"); scriptPath != "" {
			ctx.artifactsScriptPath = scriptPath
		}
	}

	kubeClient, err := operatorclient.NewClientFromRestConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.kubeClient = kubeClient

	operatorClient, err := versioned.NewForConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.operatorClient = operatorClient

	dynamicClient, err := dynamic.NewForConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.dynamicClient = dynamicClient

	packageClient, err := pversioned.NewForConfig(ctx.restConfig)
	if err != nil {
		return err
	}
	ctx.packageClient = packageClient

	ctx.scheme = runtime.NewScheme()
	localSchemeBuilder := runtime.NewSchemeBuilder(
		apiextensions.AddToScheme,
		kscheme.AddToScheme,
		operatorsv1alpha1.AddToScheme,
		operatorsv1.AddToScheme,
		operatorsv2.AddToScheme,
		apiextensionsv1.AddToScheme,
		appsv1.AddToScheme,
		apiregistrationv1.AddToScheme,
	)
	if err := localSchemeBuilder.AddToScheme(ctx.scheme); err != nil {
		return err
	}

	client, err := k8scontrollerclient.New(ctx.restConfig, k8scontrollerclient.Options{
		Scheme: ctx.scheme,
	})
	if err != nil {
		return err
	}
	ctx.e2eClient = util.NewK8sResourceManager(client)
	ctx.client = ctx.e2eClient

	ctx.ssaClient, err = controllerclient.NewForConfig(ctx.restConfig, ctx.scheme, "test.olm.registry")
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Make balanced rosters according to weighted criteria

package main

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"text/tabwriter"
	"time"

	"gopkg.in/alecthomas/kingpin.v2"
)

// Genetic algorithm constants
const (
	// Number of teams to break players into
	numTeams = 6
	// Number of times to run our genetic algorithm
	numRuns = 100
	// Percent of the time we will try to mutate. After each
	// mutation, we have a mutationChance percent chance of
	// mutating again.
	mutationChance = 5
	// We will make numSolutionsPerRun every run, and numParents carry
	// over into the next run to create the next batch of solutions.
	numSolutionsPerRun = 1000
	numParents         = 20
)

type Score float64
type Solution struct {
	players []Player
	score   Score
}

// Implement sort.Interface for []Solution, sorting based on score
type ByScore []Solution

func (a ByScore) Len() int {
	return len(a)
}
func (a ByScore) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}
func (a ByScore) Less(i, j int) bool {
	return a[i].score < a[j].score
}

type Team struct {
	players []Player
}

func splitIntoTeams(players []Player) []Team {
	teams := make([]Team, numTeams)
	for _, player := range players {
		teams[player.team].players = append(teams[player.team].players, player)
	}
	return teams
}

func randomizeTeams(players []Player) {
	for i, _ := range players {
		players[i].team = uint8(rand.Intn(numTeams))
	}
}

func PrintTeams(solution Solution) {
	teams := splitIntoTeams(solution.players)
	for i, team := range teams {
		fmt.Printf("Team #%d, %d players. Average rating: %.2f\n",
			i, len(teams[i].players), AverageRating(team))
		writer := new(tabwriter.Writer)
		writer.Init(os.Stdout, 0, 0, 1, ' ', 0)
		for _, filterFunc := range []PlayerFilter{IsMale, IsFemale} {
			filteredPlayers := Filter(team.players, filterFunc)
			sort.Sort(sort.Reverse(ByRating(filteredPlayers)))
			for _, player := range filteredPlayers {
				fmt.Fprintln(writer, player)
			}
		}
		writer.Flush()
	}
}

// Mutate the solution by moving random players to random teams, sometimes.
func mutate(players []Player) {
	for {
		// We have mutationChance of mutating. Otherwise, we break out of our loop
		if rand.Intn(100) > mutationChance {
			return
		}
		// Mutation! Move a random player to a random new team
		players[rand.Intn(len(players))].team = uint8(rand.Intn(numTeams))
	}
}

// Breed via combining the two given solutions, then randomly mutating.
func breed(solution1 Solution, solution2 Solution) Solution {
	// Create the new solution by taking crossover from both inputs
	newPlayers := make([]Player, len(solution1.players))
	for i, _ := range newPlayers {
		// Randomly take each player from solution1 or solution2
		if rand.Intn(100) < 50 {
			newPlayers[i] = solution1.players[i]
		} else {
			newPlayers[i] = solution2.players[i]
		}
	}

	// Mutate the new player list
	mutate(newPlayers)

	solutionScore, _ := ScoreSolution(newPlayers)
	return Solution{newPlayers, solutionScore}
}

// performRun creates a new solution list by breeding parents.
func performRun(parents []Solution) []Solution {
	newSolutions := make([]Solution, numSolutionsPerRun)
	for i, _ := range newSolutions {
		if i < numParents {
			// Keep the parents from last time - elitism!
			newSolutions[i] = parents[i]
		} else {
			// Make a new solution based on two random parents
			newSolutions[i] = breed(
				parents[rand.Intn(len(parents))],
				parents[rand.Intn(len(parents))])
		}
	}
	return newSolutions
}

func parseCommandLine() []Player {
	filenamePointer := kingpin.Arg("input-file",
		"filename from which to get list of players").
		Required().String()
	deterministicPointer := kingpin.Flag("deterministic",
		"makes our output deterministic by allowing the default rand.Seed").
		Short('d').Bool()
	kingpin.Parse()

	if !*deterministicPointer {
		rand.Seed(time.Now().UTC().UnixNano())
	}

	return ParsePlayers(*filenamePointer)
}

func main() {
	players := parseCommandLine()
	if len(players) == 0 {
		panic("Could not find players")
	}

	// Create random Parent solutions to start
	parentSolutions := make([]Solution, numParents)
	for i, _ := range parentSolutions {
		ourPlayers := make([]Player, len(players))
		copy(ourPlayers, players)
		randomizeTeams(ourPlayers)
		solutionScore, _ := ScoreSolution(ourPlayers)
		parentSolutions[i] = Solution{ourPlayers, solutionScore}
	}

	// Use the random starting solutions to determine the worst case for each of
	// our criteria
	PopulateWorstCases(parentSolutions)

	topScore := parentSolutions[0].score
	for i := 0; i < numRuns; i++ {
		// If we have a new best score, save and print it!
		if topScore != parentSolutions[0].score {
			topScore = parentSolutions[0].score
			log.Println("New top score! Run number ", i, "Score:", topScore)
			PrintSolutionScoring(parentSolutions[0])
		}

		// Create new solutions, and save the best ones
		newSolutions := performRun(parentSolutions)
		sort.Sort(ByScore(newSolutions))
		for i, _ := range parentSolutions {
			parentSolutions[i] = newSolutions[i]
		}
	}

	// Display our solution to the user
	topSolution := parentSolutions[0]
	log.Printf("Top score is %f, solution: %v\n", topSolution, topSolution)
	PrintTeams(topSolution)
	PrintSolutionScoring(topSolution)
}
<commit_msg>add a -profiling flag<commit_after>// Make balanced rosters according to weighted criteria

package main

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"text/tabwriter"
	"time"

	"github.com/pkg/profile"

	"gopkg.in/alecthomas/kingpin.v2"
)

// Genetic algorithm constants
const (
	// Number of teams to break players into
	numTeams = 6
	// Number of times to run our genetic algorithm
	numRuns = 100
	// Percent of the time we will try to mutate. After each
	// mutation, we have a mutationChance percent chance of
	// mutating again.
	mutationChance = 5
	// We will make numSolutionsPerRun every run, and numParents carry
	// over into the next run to create the next batch of solutions.
	numSolutionsPerRun = 1000
	numParents         = 20
)

type Score float64
type Solution struct {
	players []Player
	score   Score
}

// Implement sort.Interface for []Solution, sorting based on score
type ByScore []Solution

func (a ByScore) Len() int {
	return len(a)
}
func (a ByScore) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}
func (a ByScore) Less(i, j int) bool {
	return a[i].score < a[j].score
}

type Team struct {
	players []Player
}

func splitIntoTeams(players []Player) []Team {
	teams := make([]Team, numTeams)
	for _, player := range players {
		teams[player.team].players = append(teams[player.team].players, player)
	}
	return teams
}

func randomizeTeams(players []Player) {
	for i, _ := range players {
		players[i].team = uint8(rand.Intn(numTeams))
	}
}

func PrintTeams(solution Solution) {
	teams := splitIntoTeams(solution.players)
	for i, team := range teams {
		fmt.Printf("Team #%d, %d players. Average rating: %.2f\n",
			i, len(teams[i].players), AverageRating(team))
		writer := new(tabwriter.Writer)
		writer.Init(os.Stdout, 0, 0, 1, ' ', 0)
		for _, filterFunc := range []PlayerFilter{IsMale, IsFemale} {
			filteredPlayers := Filter(team.players, filterFunc)
			sort.Sort(sort.Reverse(ByRating(filteredPlayers)))
			for _, player := range filteredPlayers {
				fmt.Fprintln(writer, player)
			}
		}
		writer.Flush()
	}
}

// Mutate the solution by moving random players to random teams, sometimes.
func mutate(players []Player) {
	for {
		// We have mutationChance of mutating. Otherwise, we break out of our loop
		if rand.Intn(100) > mutationChance {
			return
		}
		// Mutation! Move a random player to a random new team
		players[rand.Intn(len(players))].team = uint8(rand.Intn(numTeams))
	}
}

// Breed via combining the two given solutions, then randomly mutating.
func breed(solution1 Solution, solution2 Solution) Solution {
	// Create the new solution by taking crossover from both inputs
	newPlayers := make([]Player, len(solution1.players))
	for i, _ := range newPlayers {
		// Randomly take each player from solution1 or solution2
		if rand.Intn(100) < 50 {
			newPlayers[i] = solution1.players[i]
		} else {
			newPlayers[i] = solution2.players[i]
		}
	}

	// Mutate the new player list
	mutate(newPlayers)

	solutionScore, _ := ScoreSolution(newPlayers)
	return Solution{newPlayers, solutionScore}
}

// performRun creates a new solution list by breeding parents.
func performRun(parents []Solution) []Solution {
	newSolutions := make([]Solution, numSolutionsPerRun)
	for i, _ := range newSolutions {
		if i < numParents {
			// Keep the parents from last time - elitism!
			newSolutions[i] = parents[i]
		} else {
			// Make a new solution based on two random parents
			newSolutions[i] = breed(
				parents[rand.Intn(len(parents))],
				parents[rand.Intn(len(parents))])
		}
	}
	return newSolutions
}

// parseCommandLine returns a list of players and a bool for running profiler
func parseCommandLine() ([]Player, bool) {
	filenamePointer := kingpin.Arg("input-file",
		"filename from which to get list of players").
		Required().String()
	deterministicPointer := kingpin.Flag("deterministic",
		"makes our output deterministic by allowing the default rand.Seed").
		Short('d').Bool()
	runProfilingPointer := kingpin.Flag("profiling",
		"output profiling stats when true").Short('p').Bool()
	kingpin.Parse()

	if !*deterministicPointer {
		rand.Seed(time.Now().UTC().UnixNano())
	}

	return ParsePlayers(*filenamePointer), *runProfilingPointer
}

func main() {
	players, profilingOn := parseCommandLine()
	if len(players) == 0 {
		panic("Could not find players")
	}

	// Start profiler
	if profilingOn {
		log.Println("Running profiler")
		defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
	}

	// Create random Parent solutions to start
	parentSolutions := make([]Solution, numParents)
	for i, _ := range parentSolutions {
		ourPlayers := make([]Player, len(players))
		copy(ourPlayers, players)
		randomizeTeams(ourPlayers)
		solutionScore, _ := ScoreSolution(ourPlayers)
		parentSolutions[i] = Solution{ourPlayers, solutionScore}
	}

	// Use the random starting solutions to determine the worst case for each of
	// our criteria
	PopulateWorstCases(parentSolutions)

	topScore := parentSolutions[0].score
	for i := 0; i < numRuns; i++ {
		// If we have a new best score, save and print it!
		if topScore != parentSolutions[0].score {
			topScore = parentSolutions[0].score
			log.Println("New top score! Run number ", i, "Score:", topScore)
			PrintSolutionScoring(parentSolutions[0])
		}

		// Create new solutions, and save the best ones
		newSolutions := performRun(parentSolutions)
		sort.Sort(ByScore(newSolutions))
		for i, _ := range parentSolutions {
			parentSolutions[i] = newSolutions[i]
		}
	}

	// Display our solution to the user
	topSolution := parentSolutions[0]
	log.Printf("Top score is %f, solution: %v\n", topSolution, topSolution)
	PrintTeams(topSolution)
	PrintSolutionScoring(topSolution)
}
<|endoftext|>


<commit_before>package chroot

import (
	"context"
	"fmt"
	"os/exec"
	"syscall"

	. "github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/repeatr"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/repeatr/executor/mixins"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/stitch"
)

type Executor struct {
	workspaceFs   fs.FS             // A working dir per execution will be made in here.
	assemblerTool *stitch.Assembler // Contains: unpackTool, caching cfg, and placer tools.
	packTool      rio.PackFunc
}

func NewExecutor(
	workDir fs.AbsolutePath,
	unpackTool rio.UnpackFunc,
	packTool rio.PackFunc,
) (repeatr.RunFunc, error) {
	asm, err := stitch.NewAssembler(unpackTool)
	if err != nil {
		return nil, repeatr.ReboxRioError(err)
	}
	return Executor{
		osfs.New(workDir),
		asm,
		packTool,
	}.Run, nil
}

var _ repeatr.RunFunc = Executor{}.Run

func (cfg Executor) Run(
	ctx context.Context,
	formula api.Formula,
	formulaCtx api.FormulaContext,
	input repeatr.InputControl,
	monitor repeatr.Monitor,
) (*api.RunRecord, error) {
	// Start filling out record keeping!
	//  Includes picking a random guid for the job, which we use in all temp files.
	rr := &api.RunRecord{}
	mixins.InitRunRecord(rr, formula)

	// Make work dirs.
	//  Including whole workspace dir and parents, if necessary.
	if err := fsOp.MkdirAll(osfs.New(fs.AbsolutePath{}), cfg.workspaceFs.BasePath().CoerceRelative(), 0700); err != nil {
		return nil, Errorf(repeatr.ErrLocalCacheProblem, "cannot initialize workspace dirs: %s", err)
	}
	jobPath := fs.MustRelPath(rr.Guid)
	chrootPath := jobPath.Join(fs.MustRelPath("chroot"))
	if err := cfg.workspaceFs.Mkdir(jobPath, 0700); err != nil {
		return nil, Recategorize(repeatr.ErrLocalCacheProblem, err)
	}
	if err := cfg.workspaceFs.Mkdir(chrootPath, 0755); err != nil {
		return rr, Recategorize(repeatr.ErrLocalCacheProblem, err)
	}
	chrootFs := osfs.New(cfg.workspaceFs.BasePath().Join(chrootPath))

	// Shell out to assembler.
	unpackSpecs := stitch.FormulaToUnpackSpecs(formula, formulaCtx, api.Filter_NoMutation)
	cleanupFunc, err := cfg.assemblerTool.Run(ctx, chrootFs, unpackSpecs)
	if err != nil {
		return rr, repeatr.ReboxRioError(err)
	}
	defer func() {
		if err := cleanupFunc(); err != nil {
			// TODO log it
		}
	}()

	// Invoke containment and run!
	cmdName := formula.Action.Exec[0]
	cmd := exec.Command(cmdName, formula.Action.Exec[1:]...)
	// TODO port policy concepts
	// userinfo := cradle.UserinfoForPolicy(f.Action.Policy)
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Chroot: chrootFs.BasePath().String(),
		// TODO port policy concepts
		//Credential: &syscall.Credential{
		//	Uid: uint32(userinfo.Uid),
		//	Gid: uint32(userinfo.Gid),
		//},
	}
	cmd.Dir = string(formula.Action.Cwd)
	cmd.Env = envToSlice(formula.Action.Env)
	proxy := mixins.NewOutputForwarder(ctx, monitor.Chan)
	cmd.Stdout = proxy
	cmd.Stderr = proxy
	rr.ExitCode, err = runCmd(cmd)
	if err != nil {
		return rr, err
	}

	// Pack outputs.
	packSpecs := stitch.FormulaToPackSpecs(formula, formulaCtx, api.Filter_DefaultFlatten)
	rr.Results, err = stitch.PackMulti(ctx, cfg.packTool, chrootFs, packSpecs)
	if err != nil {
		return rr, err
	}

	// Done!
	return rr, nil
}

/*
	Return an error if any part of the filesystem is invalid for running the
	formula -- e.g. the CWD setting isn't a dir; the command binary
	does not exist or is not executable; etc.

	The formula is already expected to have been syntactically validated --
	e.g. all paths have been checked to be absolute, etc.  This method will
	panic if such invarients aren't held.

	(It's better to check all these things before attempting to launch
	containment because the error codes returned by kernel exec are sometimes
	remarkably ambiguous or outright misleading in their names.)

	Currently, we require exec paths to be absolute.
*/
func sanityCheckFs(frm api.Formula, chrootFs fs.FS) error {
	// Check that the CWD exists and is a directory.
	// FIXME this needs boxed symlink traversal to give correct answers
	stat, err := chrootFs.LStat(fs.MustAbsolutePath(string(frm.Action.Cwd)).CoerceRelative())
	if err != nil {
		return Errorf(repeatr.ErrJobInvalid, "cwd invalid: %s", err)
	}
	if stat.Type != fs.Type_Dir {
		return Errorf(repeatr.ErrJobInvalid, "cwd invalid: path is a %s, must be dir", stat.Type)
	}

	// Check that the command exists and is executable.
	//  (If the format is not executable, that's another ball of wax, and
	//  not so simple to detect, so we don't.)
	stat, err = chrootFs.LStat(fs.MustAbsolutePath(frm.Action.Exec[0]).CoerceRelative())
	if err != nil {
		return Errorf(repeatr.ErrJobInvalid, "exec invalid: %s", err)
	}
	if stat.Type != fs.Type_File {
		return Errorf(repeatr.ErrJobInvalid, "exec invalid: path is a %s, must be executable file", stat.Type)
	}
	// FUTURE: ideally we could also check if the file is properly executable,
	//  and all parents have bits to be traversable (!), to the policy uid.
	//  But this is also a loooot of work: and a correct answer (for groups
	//  at least) requires *understanding the container's groups settings*,
	//  and now you're in real hot water: parsing /etc files and hoping
	//  nobody expects nsswitch to be too interesting.  Yeah.  Nuh uh.
	//  (All of these are edge conditions tools like docker Don't Have because
	//  they simply launch you with so much privilege that it doesn't matter.)

	return nil
}

func runCmd(cmd *exec.Cmd) (int, error) {
	if err := cmd.Start(); err != nil {
		return -1, Errorf(repeatr.ErrExecutor, "executor failed to launch: %s", err)
	}
	err := cmd.Wait()
	if err == nil {
		return 0, nil
	}
	exitErr, ok := err.(*exec.ExitError)
	if !ok { // This is basically an "if stdlib isn't what we thought it is" error, so panic-worthy.
		panic(fmt.Errorf("unknown exit reason: %T %s", err, err))
	}
	waitStatus, ok := exitErr.ProcessState.Sys().(syscall.WaitStatus)
	if !ok { // This is basically a "if stdlib[...]" or OS portability issue, so also panic-able.
		panic(fmt.Errorf("unknown process state implementation %T", exitErr.ProcessState.Sys()))
	}
	if waitStatus.Exited() {
		return waitStatus.ExitStatus(), nil
	} else if waitStatus.Signaled() {
		// In bash, when a processs ends from a signal, the $? variable is set to 128+SIG.
		// We follow that same convention here.
		// So, a process terminated by ctrl-C returns 130.  A script that died to kill-9 returns 137.
		return int(waitStatus.Signal()) + 128, nil
	} else {
		return -1, Errorf(repeatr.ErrExecutor, "unknown process wait status (%#v)", waitStatus)
	}

}

func envToSlice(env map[string]string) []string {
	rv := make([]string, len(env))
	i := 0
	for k, v := range env {
		rv[i] = k + "=" + v
		i++
	}
	return rv
}
<commit_msg>executor/impl/chroot: it is critical to set CWD.<commit_after>package chroot

import (
	"context"
	"fmt"
	"os/exec"
	"syscall"

	. "github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/repeatr"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/repeatr/executor/mixins"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/stitch"
)

type Executor struct {
	workspaceFs   fs.FS             // A working dir per execution will be made in here.
	assemblerTool *stitch.Assembler // Contains: unpackTool, caching cfg, and placer tools.
	packTool      rio.PackFunc
}

func NewExecutor(
	workDir fs.AbsolutePath,
	unpackTool rio.UnpackFunc,
	packTool rio.PackFunc,
) (repeatr.RunFunc, error) {
	asm, err := stitch.NewAssembler(unpackTool)
	if err != nil {
		return nil, repeatr.ReboxRioError(err)
	}
	return Executor{
		osfs.New(workDir),
		asm,
		packTool,
	}.Run, nil
}

var _ repeatr.RunFunc = Executor{}.Run

func (cfg Executor) Run(
	ctx context.Context,
	formula api.Formula,
	formulaCtx api.FormulaContext,
	input repeatr.InputControl,
	monitor repeatr.Monitor,
) (*api.RunRecord, error) {
	// Start filling out record keeping!
	//  Includes picking a random guid for the job, which we use in all temp files.
	rr := &api.RunRecord{}
	mixins.InitRunRecord(rr, formula)

	// Make work dirs.
	//  Including whole workspace dir and parents, if necessary.
	if err := fsOp.MkdirAll(osfs.New(fs.AbsolutePath{}), cfg.workspaceFs.BasePath().CoerceRelative(), 0700); err != nil {
		return nil, Errorf(repeatr.ErrLocalCacheProblem, "cannot initialize workspace dirs: %s", err)
	}
	jobPath := fs.MustRelPath(rr.Guid)
	chrootPath := jobPath.Join(fs.MustRelPath("chroot"))
	if err := cfg.workspaceFs.Mkdir(jobPath, 0700); err != nil {
		return nil, Recategorize(repeatr.ErrLocalCacheProblem, err)
	}
	if err := cfg.workspaceFs.Mkdir(chrootPath, 0755); err != nil {
		return rr, Recategorize(repeatr.ErrLocalCacheProblem, err)
	}
	chrootFs := osfs.New(cfg.workspaceFs.BasePath().Join(chrootPath))

	// Shell out to assembler.
	unpackSpecs := stitch.FormulaToUnpackSpecs(formula, formulaCtx, api.Filter_NoMutation)
	cleanupFunc, err := cfg.assemblerTool.Run(ctx, chrootFs, unpackSpecs)
	if err != nil {
		return rr, repeatr.ReboxRioError(err)
	}
	defer func() {
		if err := cleanupFunc(); err != nil {
			// TODO log it
		}
	}()

	// Invoke containment and run!
	cmdName := formula.Action.Exec[0]
	cmd := exec.Command(cmdName, formula.Action.Exec[1:]...)
	// TODO port policy concepts
	// userinfo := cradle.UserinfoForPolicy(f.Action.Policy)
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Chroot: chrootFs.BasePath().String(),
		// TODO port policy concepts
		//Credential: &syscall.Credential{
		//	Uid: uint32(userinfo.Uid),
		//	Gid: uint32(userinfo.Gid),
		//},
	}
	cmd.Dir = string(formula.Action.Cwd)
	if formula.Action.Cwd == "" {
		cmd.Dir = "/"
	}
	cmd.Env = envToSlice(formula.Action.Env)
	proxy := mixins.NewOutputForwarder(ctx, monitor.Chan)
	cmd.Stdout = proxy
	cmd.Stderr = proxy
	rr.ExitCode, err = runCmd(cmd)
	if err != nil {
		return rr, err
	}

	// Pack outputs.
	packSpecs := stitch.FormulaToPackSpecs(formula, formulaCtx, api.Filter_DefaultFlatten)
	rr.Results, err = stitch.PackMulti(ctx, cfg.packTool, chrootFs, packSpecs)
	if err != nil {
		return rr, err
	}

	// Done!
	return rr, nil
}

/*
	Return an error if any part of the filesystem is invalid for running the
	formula -- e.g. the CWD setting isn't a dir; the command binary
	does not exist or is not executable; etc.

	The formula is already expected to have been syntactically validated --
	e.g. all paths have been checked to be absolute, etc.  This method will
	panic if such invarients aren't held.

	(It's better to check all these things before attempting to launch
	containment because the error codes returned by kernel exec are sometimes
	remarkably ambiguous or outright misleading in their names.)

	Currently, we require exec paths to be absolute.
*/
func sanityCheckFs(frm api.Formula, chrootFs fs.FS) error {
	// Check that the CWD exists and is a directory.
	// FIXME this needs boxed symlink traversal to give correct answers
	stat, err := chrootFs.LStat(fs.MustAbsolutePath(string(frm.Action.Cwd)).CoerceRelative())
	if err != nil {
		return Errorf(repeatr.ErrJobInvalid, "cwd invalid: %s", err)
	}
	if stat.Type != fs.Type_Dir {
		return Errorf(repeatr.ErrJobInvalid, "cwd invalid: path is a %s, must be dir", stat.Type)
	}

	// Check that the command exists and is executable.
	//  (If the format is not executable, that's another ball of wax, and
	//  not so simple to detect, so we don't.)
	stat, err = chrootFs.LStat(fs.MustAbsolutePath(frm.Action.Exec[0]).CoerceRelative())
	if err != nil {
		return Errorf(repeatr.ErrJobInvalid, "exec invalid: %s", err)
	}
	if stat.Type != fs.Type_File {
		return Errorf(repeatr.ErrJobInvalid, "exec invalid: path is a %s, must be executable file", stat.Type)
	}
	// FUTURE: ideally we could also check if the file is properly executable,
	//  and all parents have bits to be traversable (!), to the policy uid.
	//  But this is also a loooot of work: and a correct answer (for groups
	//  at least) requires *understanding the container's groups settings*,
	//  and now you're in real hot water: parsing /etc files and hoping
	//  nobody expects nsswitch to be too interesting.  Yeah.  Nuh uh.
	//  (All of these are edge conditions tools like docker Don't Have because
	//  they simply launch you with so much privilege that it doesn't matter.)

	return nil
}

func runCmd(cmd *exec.Cmd) (int, error) {
	if err := cmd.Start(); err != nil {
		return -1, Errorf(repeatr.ErrExecutor, "executor failed to launch: %s", err)
	}
	err := cmd.Wait()
	if err == nil {
		return 0, nil
	}
	exitErr, ok := err.(*exec.ExitError)
	if !ok { // This is basically an "if stdlib isn't what we thought it is" error, so panic-worthy.
		panic(fmt.Errorf("unknown exit reason: %T %s", err, err))
	}
	waitStatus, ok := exitErr.ProcessState.Sys().(syscall.WaitStatus)
	if !ok { // This is basically a "if stdlib[...]" or OS portability issue, so also panic-able.
		panic(fmt.Errorf("unknown process state implementation %T", exitErr.ProcessState.Sys()))
	}
	if waitStatus.Exited() {
		return waitStatus.ExitStatus(), nil
	} else if waitStatus.Signaled() {
		// In bash, when a processs ends from a signal, the $? variable is set to 128+SIG.
		// We follow that same convention here.
		// So, a process terminated by ctrl-C returns 130.  A script that died to kill-9 returns 137.
		return int(waitStatus.Signal()) + 128, nil
	} else {
		return -1, Errorf(repeatr.ErrExecutor, "unknown process wait status (%#v)", waitStatus)
	}

}

func envToSlice(env map[string]string) []string {
	rv := make([]string, len(env))
	i := 0
	for k, v := range env {
		rv[i] = k + "=" + v
		i++
	}
	return rv
}
<|endoftext|>