<commit_before>/*
Copyright 2019 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resources

import (
	"context"
	"fmt"
	"hash/adler32"
	"sort"
	"strings"

	istiov1alpha3 "istio.io/api/networking/v1alpha3"
	"istio.io/client-go/pkg/apis/networking/v1alpha3"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	corev1listers "k8s.io/client-go/listers/core/v1"
	"knative.dev/pkg/kmeta"
	"knative.dev/pkg/system"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/network"
	"knative.dev/serving/pkg/reconciler/ingress/config"
)

var httpServerPortName = "http-server"

// Istio Gateway requires to have at least one server. This placeholderServer is used when
// all of the real servers are deleted.
var placeholderServer = istiov1alpha3.Server{
	Hosts: []string{"place-holder.place-holder"},
	Port: &istiov1alpha3.Port{
		Name:     "place-holder",
		Number:   9999,
		Protocol: "HTTP",
	},
}

// GetServers gets the `Servers` from `Gateway` that belongs to the given Ingress.
func GetServers(gateway *v1alpha3.Gateway, ing *v1alpha1.Ingress) []*istiov1alpha3.Server {
	servers := []*istiov1alpha3.Server{}
	for i := range gateway.Spec.Servers {
		if belongsToIngress(gateway.Spec.Servers[i], ing) {
			servers = append(servers, gateway.Spec.Servers[i])
		}
	}
	return SortServers(servers)
}

// GetHTTPServer gets the HTTP `Server` from `Gateway`.
func GetHTTPServer(gateway *v1alpha3.Gateway) *istiov1alpha3.Server {
	for _, server := range gateway.Spec.Servers {
		if server.Port.Name == httpServerPortName {
			return server
		}
	}
	return nil
}

func belongsToIngress(server *istiov1alpha3.Server, ing *v1alpha1.Ingress) bool {
	// The format of the portName should be "<namespace>/<ingress_name>:<number>".
	// For example, default/routetest:0.
	portNameSplits := strings.Split(server.Port.Name, ":")
	if len(portNameSplits) != 2 {
		return false
	}
	return portNameSplits[0] == ing.GetNamespace()+"/"+ing.GetName()
}

// SortServers sorts `Server` according to its port name.
func SortServers(servers []*istiov1alpha3.Server) []*istiov1alpha3.Server {
	sort.Slice(servers, func(i, j int) bool {
		return strings.Compare(servers[i].Port.Name, servers[j].Port.Name) < 0
	})
	return servers
}

// MakeIngressGateways creates Gateways for a given Ingress.
func MakeIngressGateways(ctx context.Context, ing *v1alpha1.Ingress, originSecrets map[string]*corev1.Secret, svcLister corev1listers.ServiceLister) ([]*v1alpha3.Gateway, error) {
	gatewayServices, err := getGatewayServices(ctx, svcLister)
	if err != nil {
		return nil, err
	}
	gateways := make([]*v1alpha3.Gateway, len(gatewayServices))
	for i, gatewayService := range gatewayServices {
		gateway, err := makeIngressGateway(ctx, ing, originSecrets, gatewayService.Spec.Selector, gatewayService)
		if err != nil {
			return nil, err
		}
		gateways[i] = gateway
	}
	return gateways, nil
}

func makeIngressGateway(ctx context.Context, ing *v1alpha1.Ingress, originSecrets map[string]*corev1.Secret, selector map[string]string, gatewayService *corev1.Service) (*v1alpha3.Gateway, error) {
	ns := ing.GetNamespace()
	if len(ns) == 0 {
		ns = system.Namespace()
	}
	servers, err := MakeTLSServers(ing, gatewayService.Namespace, originSecrets)
	if err != nil {
		return nil, err
	}
	hosts := sets.String{}
	for _, rule := range ing.Spec.Rules {
		hosts.Insert(rule.Hosts...)
	}
	servers = append(servers, MakeHTTPServer(config.FromContext(ctx).Network.HTTPProtocol, hosts.List()))
	return &v1alpha3.Gateway{
		ObjectMeta: metav1.ObjectMeta{
			Name:            GatewayName(ing, gatewayService),
			Namespace:       ns,
			OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(ing)},
			Labels: map[string]string{
				// We need this label to find out all of Gateways of a given Ingress.
				networking.IngressLabelKey: ing.GetName(),
			},
		},
		Spec: istiov1alpha3.Gateway{
			Selector: selector,
			Servers:  servers,
		},
	}, nil
}

func getGatewayServices(ctx context.Context, svcLister corev1listers.ServiceLister) ([]*corev1.Service, error) {
	ingressSvcMetas, err := getIngressGatewaySvcNameNamespaces(ctx)
	if err != nil {
		return nil, err
	}
	services := make([]*corev1.Service, len(ingressSvcMetas))
	for i, ingressSvcMeta := range ingressSvcMetas {
		svc, err := svcLister.Services(ingressSvcMeta.Namespace).Get(ingressSvcMeta.Name)
		if err != nil {
			return nil, err
		}
		services[i] = svc
	}
	return services, nil
}

// GatewayName create a name for the Gateway that is built based on the given Ingress and bonds to the
// given ingress gateway service.
func GatewayName(accessor kmeta.Accessor, gatewaySvc *corev1.Service) string {
	gatewayServiceKey := fmt.Sprintf("%s/%s", gatewaySvc.Namespace, gatewaySvc.Name)
	return fmt.Sprintf("%s-%d", accessor.GetName(), adler32.Checksum([]byte(gatewayServiceKey)))
}

// MakeTLSServers creates the expected Gateway TLS `Servers` based on the given Ingress.
func MakeTLSServers(ing *v1alpha1.Ingress, gatewayServiceNamespace string, originSecrets map[string]*corev1.Secret) ([]*istiov1alpha3.Server, error) {
	servers := make([]*istiov1alpha3.Server, len(ing.Spec.TLS))
	// TODO(zhiminx): for the hosts that does not included in the IngressTLS but listed in the IngressRule,
	// do we consider them as hosts for HTTP?
	for i, tls := range ing.Spec.TLS {
		credentialName := tls.SecretName
		// If the origin secret is not in the target namespace, then it should have been
		// copied into the target namespace. So we use the name of the copy.
		if tls.SecretNamespace != gatewayServiceNamespace {
			originSecret, ok := originSecrets[secretKey(tls)]
			if !ok {
				return nil, fmt.Errorf("unable to get the original secret %s/%s", tls.SecretNamespace, tls.SecretName)
			}
			credentialName = targetSecret(originSecret, ing)
		}

		port := ing.GetNamespace() + "/" + ing.GetName()

		servers[i] = &istiov1alpha3.Server{
			Hosts: tls.Hosts,
			Port: &istiov1alpha3.Port{
				Name:     fmt.Sprintf("%s:%d", port, i),
				Number:   443,
				Protocol: "HTTPS",
			},
			Tls: &istiov1alpha3.Server_TLSOptions{
				Mode:              istiov1alpha3.Server_TLSOptions_SIMPLE,
				ServerCertificate: corev1.TLSCertKey,
				PrivateKey:        corev1.TLSPrivateKeyKey,
				CredentialName:    credentialName,
			},
		}
	}
	return SortServers(servers), nil
}

// MakeHTTPServer creates a HTTP Gateway `Server` based on the HTTPProtocol
// configureation.
func MakeHTTPServer(httpProtocol network.HTTPProtocol, hosts []string) *istiov1alpha3.Server {
	if httpProtocol == network.HTTPDisabled {
		return nil
	}
	server := &istiov1alpha3.Server{
		Hosts: hosts,
		Port: &istiov1alpha3.Port{
			Name:     httpServerPortName,
			Number:   80,
			Protocol: "HTTP",
		},
	}
	if httpProtocol == network.HTTPRedirected {
		server.Tls = &istiov1alpha3.Server_TLSOptions{
			HttpsRedirect: true,
		}
	}
	return server
}

// ServiceNamespaceFromURL extracts the namespace part from the service URL.
// TODO(nghia):  Remove this by parsing at config parsing time.
func ServiceNamespaceFromURL(svc string) (string, error) {
	parts := strings.SplitN(svc, ".", 3)
	if len(parts) != 3 {
		return "", fmt.Errorf("unexpected service URL form: %s", svc)
	}
	return parts[1], nil
}

// TODO(nghia):  Remove this by parsing at config parsing time.
func getIngressGatewaySvcNameNamespaces(ctx context.Context) ([]metav1.ObjectMeta, error) {
	cfg := config.FromContext(ctx).Istio
	nameNamespaces := make([]metav1.ObjectMeta, len(cfg.IngressGateways))
	for i, ingressgateway := range cfg.IngressGateways {
		parts := strings.SplitN(ingressgateway.ServiceURL, ".", 3)
		if len(parts) != 3 {
			return nil, fmt.Errorf("unexpected service URL form: %s", ingressgateway.ServiceURL)
		}
		nameNamespaces[i] = metav1.ObjectMeta{
			Name:      parts[0],
			Namespace: parts[1],
		}
	}
	return nameNamespaces, nil
}

// UpdateGateway replaces the existing servers with the wanted servers.
func UpdateGateway(gateway *v1alpha3.Gateway, want []*istiov1alpha3.Server, existing []*istiov1alpha3.Server) *v1alpha3.Gateway {
	existingServers := sets.String{}
	for i := range existing {
		existingServers.Insert(existing[i].Port.Name)
	}

	servers := []*istiov1alpha3.Server{}
	for _, server := range gateway.Spec.Servers {
		// We remove
		//  1) the existing servers
		//  2) the default HTTP server and HTTPS server in the gateway because they are only used for the scenario of not reconciling gateway.
		//  3) the placeholder servers.
		if existingServers.Has(server.Port.Name) || isDefaultServer(server) || isPlaceHolderServer(server) {
			continue
		}
		servers = append(servers, server)
	}
	servers = append(servers, want...)

	// Istio Gateway requires to have at least one server. So if the final gateway does not have any server,
	// we add "placeholder" server back.
	if len(servers) == 0 {
		servers = append(servers, &placeholderServer)
	}

	SortServers(servers)
	gateway.Spec.Servers = servers
	return gateway
}

func isDefaultServer(server *istiov1alpha3.Server) bool {
	if server.Port.Name == "https" {
		return len(server.Hosts) > 0 && server.Hosts[0] == "*"
	}
	return server.Port.Name == "http"
}

func isPlaceHolderServer(server *istiov1alpha3.Server) bool {
	return equality.Semantic.DeepEqual(server, &placeholderServer)
}
<commit_msg>Fix typo 'configureation' -> 'configuration'. (#6728)<commit_after>/*
Copyright 2019 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resources

import (
	"context"
	"fmt"
	"hash/adler32"
	"sort"
	"strings"

	istiov1alpha3 "istio.io/api/networking/v1alpha3"
	"istio.io/client-go/pkg/apis/networking/v1alpha3"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/equality"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	corev1listers "k8s.io/client-go/listers/core/v1"
	"knative.dev/pkg/kmeta"
	"knative.dev/pkg/system"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/apis/networking/v1alpha1"
	"knative.dev/serving/pkg/network"
	"knative.dev/serving/pkg/reconciler/ingress/config"
)

var httpServerPortName = "http-server"

// Istio Gateway requires to have at least one server. This placeholderServer is used when
// all of the real servers are deleted.
var placeholderServer = istiov1alpha3.Server{
	Hosts: []string{"place-holder.place-holder"},
	Port: &istiov1alpha3.Port{
		Name:     "place-holder",
		Number:   9999,
		Protocol: "HTTP",
	},
}

// GetServers gets the `Servers` from `Gateway` that belongs to the given Ingress.
func GetServers(gateway *v1alpha3.Gateway, ing *v1alpha1.Ingress) []*istiov1alpha3.Server {
	servers := []*istiov1alpha3.Server{}
	for i := range gateway.Spec.Servers {
		if belongsToIngress(gateway.Spec.Servers[i], ing) {
			servers = append(servers, gateway.Spec.Servers[i])
		}
	}
	return SortServers(servers)
}

// GetHTTPServer gets the HTTP `Server` from `Gateway`.
func GetHTTPServer(gateway *v1alpha3.Gateway) *istiov1alpha3.Server {
	for _, server := range gateway.Spec.Servers {
		if server.Port.Name == httpServerPortName {
			return server
		}
	}
	return nil
}

func belongsToIngress(server *istiov1alpha3.Server, ing *v1alpha1.Ingress) bool {
	// The format of the portName should be "<namespace>/<ingress_name>:<number>".
	// For example, default/routetest:0.
	portNameSplits := strings.Split(server.Port.Name, ":")
	if len(portNameSplits) != 2 {
		return false
	}
	return portNameSplits[0] == ing.GetNamespace()+"/"+ing.GetName()
}

// SortServers sorts `Server` according to its port name.
func SortServers(servers []*istiov1alpha3.Server) []*istiov1alpha3.Server {
	sort.Slice(servers, func(i, j int) bool {
		return strings.Compare(servers[i].Port.Name, servers[j].Port.Name) < 0
	})
	return servers
}

// MakeIngressGateways creates Gateways for a given Ingress.
func MakeIngressGateways(ctx context.Context, ing *v1alpha1.Ingress, originSecrets map[string]*corev1.Secret, svcLister corev1listers.ServiceLister) ([]*v1alpha3.Gateway, error) {
	gatewayServices, err := getGatewayServices(ctx, svcLister)
	if err != nil {
		return nil, err
	}
	gateways := make([]*v1alpha3.Gateway, len(gatewayServices))
	for i, gatewayService := range gatewayServices {
		gateway, err := makeIngressGateway(ctx, ing, originSecrets, gatewayService.Spec.Selector, gatewayService)
		if err != nil {
			return nil, err
		}
		gateways[i] = gateway
	}
	return gateways, nil
}

func makeIngressGateway(ctx context.Context, ing *v1alpha1.Ingress, originSecrets map[string]*corev1.Secret, selector map[string]string, gatewayService *corev1.Service) (*v1alpha3.Gateway, error) {
	ns := ing.GetNamespace()
	if len(ns) == 0 {
		ns = system.Namespace()
	}
	servers, err := MakeTLSServers(ing, gatewayService.Namespace, originSecrets)
	if err != nil {
		return nil, err
	}
	hosts := sets.String{}
	for _, rule := range ing.Spec.Rules {
		hosts.Insert(rule.Hosts...)
	}
	servers = append(servers, MakeHTTPServer(config.FromContext(ctx).Network.HTTPProtocol, hosts.List()))
	return &v1alpha3.Gateway{
		ObjectMeta: metav1.ObjectMeta{
			Name:            GatewayName(ing, gatewayService),
			Namespace:       ns,
			OwnerReferences: []metav1.OwnerReference{*kmeta.NewControllerRef(ing)},
			Labels: map[string]string{
				// We need this label to find out all of Gateways of a given Ingress.
				networking.IngressLabelKey: ing.GetName(),
			},
		},
		Spec: istiov1alpha3.Gateway{
			Selector: selector,
			Servers:  servers,
		},
	}, nil
}

func getGatewayServices(ctx context.Context, svcLister corev1listers.ServiceLister) ([]*corev1.Service, error) {
	ingressSvcMetas, err := getIngressGatewaySvcNameNamespaces(ctx)
	if err != nil {
		return nil, err
	}
	services := make([]*corev1.Service, len(ingressSvcMetas))
	for i, ingressSvcMeta := range ingressSvcMetas {
		svc, err := svcLister.Services(ingressSvcMeta.Namespace).Get(ingressSvcMeta.Name)
		if err != nil {
			return nil, err
		}
		services[i] = svc
	}
	return services, nil
}

// GatewayName create a name for the Gateway that is built based on the given Ingress and bonds to the
// given ingress gateway service.
func GatewayName(accessor kmeta.Accessor, gatewaySvc *corev1.Service) string {
	gatewayServiceKey := fmt.Sprintf("%s/%s", gatewaySvc.Namespace, gatewaySvc.Name)
	return fmt.Sprintf("%s-%d", accessor.GetName(), adler32.Checksum([]byte(gatewayServiceKey)))
}

// MakeTLSServers creates the expected Gateway TLS `Servers` based on the given Ingress.
func MakeTLSServers(ing *v1alpha1.Ingress, gatewayServiceNamespace string, originSecrets map[string]*corev1.Secret) ([]*istiov1alpha3.Server, error) {
	servers := make([]*istiov1alpha3.Server, len(ing.Spec.TLS))
	// TODO(zhiminx): for the hosts that does not included in the IngressTLS but listed in the IngressRule,
	// do we consider them as hosts for HTTP?
	for i, tls := range ing.Spec.TLS {
		credentialName := tls.SecretName
		// If the origin secret is not in the target namespace, then it should have been
		// copied into the target namespace. So we use the name of the copy.
		if tls.SecretNamespace != gatewayServiceNamespace {
			originSecret, ok := originSecrets[secretKey(tls)]
			if !ok {
				return nil, fmt.Errorf("unable to get the original secret %s/%s", tls.SecretNamespace, tls.SecretName)
			}
			credentialName = targetSecret(originSecret, ing)
		}

		port := ing.GetNamespace() + "/" + ing.GetName()

		servers[i] = &istiov1alpha3.Server{
			Hosts: tls.Hosts,
			Port: &istiov1alpha3.Port{
				Name:     fmt.Sprintf("%s:%d", port, i),
				Number:   443,
				Protocol: "HTTPS",
			},
			Tls: &istiov1alpha3.Server_TLSOptions{
				Mode:              istiov1alpha3.Server_TLSOptions_SIMPLE,
				ServerCertificate: corev1.TLSCertKey,
				PrivateKey:        corev1.TLSPrivateKeyKey,
				CredentialName:    credentialName,
			},
		}
	}
	return SortServers(servers), nil
}

// MakeHTTPServer creates a HTTP Gateway `Server` based on the HTTPProtocol
// configuration.
func MakeHTTPServer(httpProtocol network.HTTPProtocol, hosts []string) *istiov1alpha3.Server {
	if httpProtocol == network.HTTPDisabled {
		return nil
	}
	server := &istiov1alpha3.Server{
		Hosts: hosts,
		Port: &istiov1alpha3.Port{
			Name:     httpServerPortName,
			Number:   80,
			Protocol: "HTTP",
		},
	}
	if httpProtocol == network.HTTPRedirected {
		server.Tls = &istiov1alpha3.Server_TLSOptions{
			HttpsRedirect: true,
		}
	}
	return server
}

// ServiceNamespaceFromURL extracts the namespace part from the service URL.
// TODO(nghia):  Remove this by parsing at config parsing time.
func ServiceNamespaceFromURL(svc string) (string, error) {
	parts := strings.SplitN(svc, ".", 3)
	if len(parts) != 3 {
		return "", fmt.Errorf("unexpected service URL form: %s", svc)
	}
	return parts[1], nil
}

// TODO(nghia):  Remove this by parsing at config parsing time.
func getIngressGatewaySvcNameNamespaces(ctx context.Context) ([]metav1.ObjectMeta, error) {
	cfg := config.FromContext(ctx).Istio
	nameNamespaces := make([]metav1.ObjectMeta, len(cfg.IngressGateways))
	for i, ingressgateway := range cfg.IngressGateways {
		parts := strings.SplitN(ingressgateway.ServiceURL, ".", 3)
		if len(parts) != 3 {
			return nil, fmt.Errorf("unexpected service URL form: %s", ingressgateway.ServiceURL)
		}
		nameNamespaces[i] = metav1.ObjectMeta{
			Name:      parts[0],
			Namespace: parts[1],
		}
	}
	return nameNamespaces, nil
}

// UpdateGateway replaces the existing servers with the wanted servers.
func UpdateGateway(gateway *v1alpha3.Gateway, want []*istiov1alpha3.Server, existing []*istiov1alpha3.Server) *v1alpha3.Gateway {
	existingServers := sets.String{}
	for i := range existing {
		existingServers.Insert(existing[i].Port.Name)
	}

	servers := []*istiov1alpha3.Server{}
	for _, server := range gateway.Spec.Servers {
		// We remove
		//  1) the existing servers
		//  2) the default HTTP server and HTTPS server in the gateway because they are only used for the scenario of not reconciling gateway.
		//  3) the placeholder servers.
		if existingServers.Has(server.Port.Name) || isDefaultServer(server) || isPlaceHolderServer(server) {
			continue
		}
		servers = append(servers, server)
	}
	servers = append(servers, want...)

	// Istio Gateway requires to have at least one server. So if the final gateway does not have any server,
	// we add "placeholder" server back.
	if len(servers) == 0 {
		servers = append(servers, &placeholderServer)
	}

	SortServers(servers)
	gateway.Spec.Servers = servers
	return gateway
}

func isDefaultServer(server *istiov1alpha3.Server) bool {
	if server.Port.Name == "https" {
		return len(server.Hosts) > 0 && server.Hosts[0] == "*"
	}
	return server.Port.Name == "http"
}

func isPlaceHolderServer(server *istiov1alpha3.Server) bool {
	return equality.Semantic.DeepEqual(server, &placeholderServer)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}
	}
)

func (job *Job) run(ctx context.Context) error {
	verr := job.message.Validate()
	if verr != nil {
		log.Printf("Invalid Message: MessageId: %v, Message: %v, error: %v\n", job.message.MessageId(), job.message.raw.Message, verr)
		err := job.withNotify(CANCELLING, job.message.Ack)()
		if err != nil {
			return err
		}
		return nil
	}

	go job.message.sendMADPeriodically()
	defer job.message.Done()

	job.notification.notify(PROCESSING, job.message.MessageId(), "info")

	err := job.setupWorkspace()
	if err != nil {
		return err
	}

	defer job.clearWorkspace()

	  err = job.withNotify(PREPARING, job.setupDownloadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(DOWNLOADING, job.downloadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(EXECUTING, job.execute)()
		if err != nil {
			return err
		}

		err = job.withNotify(UPLOADING, job.uploadFiles)()
		if err != nil {
			return err
		}

		err = job.withNotify(ACKSENDING, job.message.Ack)()
		if err != nil {
			return err
		}

	job.notification.notify(CLEANUP, job.message.MessageId(), "info")
	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.MessageId()
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	return os.RemoveAll(job.workspace)
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() (*exec.Cmd, error) {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return nil, err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return nil, err
			}
		}
	}
	cmd := exec.Command(values[0], values[1:]...)
	return cmd, nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, err
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	cmd, err := job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	log.Printf("EXECUTE running: %v\n", cmd)
	err = cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n%v\n", cmd, err, out.String())
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<commit_msg>:shirt: Fix indentation<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}
	}
)

func (job *Job) run(ctx context.Context) error {
	verr := job.message.Validate()
	if verr != nil {
		log.Printf("Invalid Message: MessageId: %v, Message: %v, error: %v\n", job.message.MessageId(), job.message.raw.Message, verr)
		err := job.withNotify(CANCELLING, job.message.Ack)()
		if err != nil {
			return err
		}
		return nil
	}

	go job.message.sendMADPeriodically()
	defer job.message.Done()

	job.notification.notify(PROCESSING, job.message.MessageId(), "info")

	err := job.setupWorkspace()
	if err != nil {
		return err
	}

	defer job.clearWorkspace()

	err = job.withNotify(PREPARING, job.setupDownloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	job.notification.notify(CLEANUP, job.message.MessageId(), "info")
	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.MessageId()
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	return os.RemoveAll(job.workspace)
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() (*exec.Cmd, error) {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return nil, err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return nil, err
			}
		}
	}
	cmd := exec.Command(values[0], values[1:]...)
	return cmd, nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, err
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	cmd, err := job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	log.Printf("EXECUTE running: %v\n", cmd)
	err = cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n%v\n", cmd, err, out.String())
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<|endoftext|>


<commit_before>package render

import (
	"html"
	"html/template"
	"io"
	"path/filepath"
	"strings"

	// this blank import is here because dep doesn't
	// handle transitive dependencies correctly
	_ "github.com/russross/blackfriday"
	"github.com/shurcooL/github_flavored_markdown"
)

type templateRenderer struct {
	*Engine
	contentType string
	names       []string
}

func (s templateRenderer) ContentType() string {
	return s.contentType
}

func (s templateRenderer) Render(w io.Writer, data Data) error {
	var body template.HTML
	var err error
	for _, name := range s.names {
		body, err = s.exec(name, data)
		if err != nil {
			return err
		}
		data["yield"] = body
	}
	w.Write([]byte(body))
	return nil
}

func (s templateRenderer) partial(name string, dd Data) (template.HTML, error) {
	d, f := filepath.Split(name)
	name = filepath.Join(d, "_"+f)
	return s.exec(name, dd)
}

func (s templateRenderer) exec(name string, data Data) (template.HTML, error) {
	source, err := s.TemplatesBox.MustBytes(name)
	if err != nil {
		return "", err
	}

	helpers := map[string]interface{}{
		"partial": s.partial,
	}

	helpers = s.addAssetsHelpers(helpers)

	for k, v := range s.Helpers {
		helpers[k] = v
	}

	if strings.ToLower(filepath.Ext(name)) == ".md" && strings.ToLower(s.contentType) != "text/plain" {
		source = github_flavored_markdown.Markdown(source)
		source = []byte(html.UnescapeString(string(source)))
	}

	body, err := s.TemplateEngine(string(source), data, helpers)
	if err != nil {
		return "", err
	}

	return template.HTML(body), nil
}

// Template renders the named files using the specified
// content type and the github.com/gobuffalo/plush
// package for templating. If more than 1 file is provided
// the second file will be considered a "layout" file
// and the first file will be the "content" file which will
// be placed into the "layout" using "{{yield}}".
func Template(c string, names ...string) Renderer {
	e := New(Options{})
	return e.Template(c, names...)
}

// Template renders the named files using the specified
// content type and the github.com/gobuffalo/plush
// package for templating. If more than 1 file is provided
// the second file will be considered a "layout" file
// and the first file will be the "content" file which will
// be placed into the "layout" using "{{yield}}".
func (e *Engine) Template(c string, names ...string) Renderer {
	return templateRenderer{
		Engine:      e,
		contentType: c,
		names:       names,
	}
}
<commit_msg>removing template helpers<commit_after>package render

import (
	"html"
	"html/template"
	"io"
	"path/filepath"
	"strings"

	// this blank import is here because dep doesn't
	// handle transitive dependencies correctly
	_ "github.com/russross/blackfriday"
	"github.com/shurcooL/github_flavored_markdown"
)

type templateRenderer struct {
	*Engine
	contentType string
	names       []string
}

func (s templateRenderer) ContentType() string {
	return s.contentType
}

func (s templateRenderer) Render(w io.Writer, data Data) error {
	var body template.HTML
	var err error
	for _, name := range s.names {
		body, err = s.exec(name, data)
		if err != nil {
			return err
		}
		data["yield"] = body
	}
	w.Write([]byte(body))
	return nil
}

func (s templateRenderer) partial(name string, dd Data) (template.HTML, error) {
	d, f := filepath.Split(name)
	name = filepath.Join(d, "_"+f)
	return s.exec(name, dd)
}

func (s templateRenderer) exec(name string, data Data) (template.HTML, error) {
	source, err := s.TemplatesBox.MustBytes(name)
	if err != nil {
		return "", err
	}

	helpers := map[string]interface{}{
		"partial": s.partial,
	}

	for k, v := range s.Helpers {
		helpers[k] = v
	}

	if strings.ToLower(filepath.Ext(name)) == ".md" && strings.ToLower(s.contentType) != "text/plain" {
		source = github_flavored_markdown.Markdown(source)
		source = []byte(html.UnescapeString(string(source)))
	}

	body, err := s.TemplateEngine(string(source), data, helpers)
	if err != nil {
		return "", err
	}

	return template.HTML(body), nil
}

// Template renders the named files using the specified
// content type and the github.com/gobuffalo/plush
// package for templating. If more than 1 file is provided
// the second file will be considered a "layout" file
// and the first file will be the "content" file which will
// be placed into the "layout" using "{{yield}}".
func Template(c string, names ...string) Renderer {
	e := New(Options{})
	return e.Template(c, names...)
}

// Template renders the named files using the specified
// content type and the github.com/gobuffalo/plush
// package for templating. If more than 1 file is provided
// the second file will be considered a "layout" file
// and the first file will be the "content" file which will
// be placed into the "layout" using "{{yield}}".
func (e *Engine) Template(c string, names ...string) Renderer {
	return templateRenderer{
		Engine:      e,
		contentType: c,
		names:       names,
	}
}
<|endoftext|>


<commit_before>package request

import (
	"github.com/labstack/echo"
	"github.com/carrot/go-base-api/middleware"
	"github.com/carrot/go-base-api/controllers"
	echo_middleware "github.com/labstack/echo/middleware"
)

func BuildEcho() (e *echo.Echo) {
	// ----------
	// Framework
	// ----------

	e = echo.New()

	// -----------
	// Middleware
	// -----------

	e.Use(echo_middleware.Logger())
	e.Use(middleware.Recover())

	// ------------
	// Controllers
	// ------------

	topicsController := new(controllers.TopicsController)

	// ----------
	// Endpoints
	// ----------

	e.Get("/topics", topicsController.Index)
	e.Get("/topics/:id", topicsController.Show)
	e.Post("/topics", topicsController.Create)
	e.Put("/topics/:id", topicsController.Update)
	e.Delete("/topics/:id", topicsController.Delete)

	return
}
<commit_msg>Explicitly returning echo..  Also gofmt<commit_after>package request

import (
	"github.com/carrot/go-base-api/controllers"
	"github.com/carrot/go-base-api/middleware"
	"github.com/labstack/echo"
	echo_middleware "github.com/labstack/echo/middleware"
)

func BuildEcho() *echo.Echo {
	// ----------
	// Framework
	// ----------

	e := echo.New()

	// -----------
	// Middleware
	// -----------

	e.Use(echo_middleware.Logger())
	e.Use(middleware.Recover())

	// ------------
	// Controllers
	// ------------

	topicsController := new(controllers.TopicsController)

	// ----------
	// Endpoints
	// ----------

	e.Get("/topics", topicsController.Index)
	e.Get("/topics/:id", topicsController.Show)
	e.Post("/topics", topicsController.Create)
	e.Put("/topics/:id", topicsController.Update)
	e.Delete("/topics/:id", topicsController.Delete)

	return e
}
<|endoftext|>


<commit_before>package diff

// Runs a diff on the given Interface.
// Returns the results as a slice of Diff.
func New(iface Interface) []Diff {
	table := lcs(iface)
	diff := walk(iface, table)
	reverse(diff)
	return diff
}

// Constructs a LCSLength table
// http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Computing_the_length_of_the_LCS
func lcs(iface Interface) [][]int {
	lnum, rnum := iface.Length()
	rows, cols := lnum+1, rnum+1
	table := make([][]int, rows)
	cels := make([]int, rows*cols)
	for i := 0; i < rows; i++ {
		table[i] = cels[:cols]
		cels = cels[cols:]
	}

	for i := 1; i < rows; i++ {
		for j := 1; j < cols; j++ {
			if iface.Equal(i-1, j-1) {
				table[i][j] = table[i-1][j-1] + 1
			} else {
				a := table[i-1][j]
				b := table[i][j-1]
				if b > a {
					a = b
				}
				table[i][j] = a
			}
		}
	}
	return table
}

// Walk the lcs table
// http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Example
func walk(iface Interface, table [][]int) (diff []Diff) {
	i, j := iface.Length()
	diff = make([]Diff, 0, i+j)
	for {
		if i == 0 && j == 0 {
			return
		} else if i == 0 {
			j--
			diff = append(diff, Diff{Delta: Right, Index: j})
		} else if j == 0 {
			i--
			diff = append(diff, Diff{Delta: Left, Index: i})
		} else {
			if iface.Equal(i-1, j-1) {
				i--
				j--
				diff = append(diff, Diff{Delta: Both, Index: i})
			} else {
				if table[i-1][j] > table[i][j-1] {
					i--
					diff = append(diff, Diff{Delta: Left, Index: i})
				} else {
					j--
					diff = append(diff, Diff{Delta: Right, Index: j})
				}
			}
		}
	}
}

func reverse(diff []Diff) {
	i := 0
	j := len(diff) - 1
	for i < j {
		diff[i], diff[j] = diff[j], diff[i]
		i++
		j--
	}
}
<commit_msg>Split out Diff alloc<commit_after>package diff

// Runs a diff on the given Interface.
// Returns the results as a slice of Diff.
func New(iface Interface) []Diff {
	lnum, rnum := iface.Length()
	diff := make([]Diff, 0, lnum+rnum)
	table := lcs(iface)
	diff = walk(iface, table, diff)
	reverse(diff)
	return diff
}

// Constructs a LCSLength table
// http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Computing_the_length_of_the_LCS
func lcs(iface Interface) [][]int {
	lnum, rnum := iface.Length()
	rows, cols := lnum+1, rnum+1
	table := make([][]int, rows)
	cels := make([]int, rows*cols)
	for i := 0; i < rows; i++ {
		table[i] = cels[:cols]
		cels = cels[cols:]
	}

	for i := 1; i < rows; i++ {
		for j := 1; j < cols; j++ {
			if iface.Equal(i-1, j-1) {
				table[i][j] = table[i-1][j-1] + 1
			} else {
				a := table[i-1][j]
				b := table[i][j-1]
				if b > a {
					a = b
				}
				table[i][j] = a
			}
		}
	}
	return table
}

// Walk the lcs table
// http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Example
func walk(iface Interface, table [][]int, diff []Diff) []Diff {
	i, j := iface.Length()
	for {
		if i == 0 && j == 0 {
			return diff
		} else if i == 0 {
			j--
			diff = append(diff, Diff{Delta: Right, Index: j})
		} else if j == 0 {
			i--
			diff = append(diff, Diff{Delta: Left, Index: i})
		} else {
			if iface.Equal(i-1, j-1) {
				i--
				j--
				diff = append(diff, Diff{Delta: Both, Index: i})
			} else {
				if table[i-1][j] > table[i][j-1] {
					i--
					diff = append(diff, Diff{Delta: Left, Index: i})
				} else {
					j--
					diff = append(diff, Diff{Delta: Right, Index: j})
				}
			}
		}
	}
}

func reverse(diff []Diff) {
	i := 0
	j := len(diff) - 1
	for i < j {
		diff[i], diff[j] = diff[j], diff[i]
		i++
		j--
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	//"os"
	"strings"
	"time"
	//"github.com/stianeikeland/go-rpio"
)

// var (
// 	ledRed = rpio.Pin(4)
// 	ledYellow = rpio.Pin(17)
// 	ledGreen = rpio.Pin(27)
// )

var (
	ledRed, _    = embd.NewDigitalPin(4)
	ledYellow, _ = embd.NewDigitalPin(17)
	ledGreen, _  = embd.NewDigitalPin(27)
)

func getLEDString(color string) string {
	return "Toggle " + strings.ToUpper(color)
}

func getToggledValue(pin embd.DigitalPin) int {
	val,_ := pin.Read()
	if val == embd.High {
		return embd.Low
	} else {
		return embd.High
	}
}

func toggleLED(pin embd.DigitalPin, color string) {
	fmt.Println(getLEDString(color))
	pin.Write(getToggledValue(pin))
}

// func toggleLED(pin rpio.Pin, color string)  {
// 	fmt.Println(getLEDString(color))
// 	pin.Toggle()
// }

func initLEDs() {
	embd.SetDirection(4, embd.Out)
	embd.SetDirection(17, embd.Out)
	embd.SetDirection(27, embd.Out)
	// ledRed.Output()
	// ledYellow.Output()
	// ledGreen.Output()
}

func main() {
	fmt.Println("Parsing parameters")
	num := flag.Int("num", 0, "number of blinks")
	flag.Parse()

	fmt.Println("Number of blinks:", *num)

	fmt.Println("Opening rpio access")

	//var err = rpio.Open()
	embd.InitGPIO()

	// if err != nil {
	// 	fmt.Println(err)
	// 	os.Exit(1)
	// }

	//defer rpio.Close()
	defer embd.CloseGPIO()

	fmt.Println("Pin as output")

	for i := 0; i < *num; i++ {
		if i%3 == 0 {
			toggleLED(ledRed, "red")
		} else if i%3 == 1 {
			toggleLED(ledYellow, "yellow")
		} else {
			toggleLED(ledGreen, "green")
		}
		time.Sleep(time.Second)
	}
}
<commit_msg>Added debug output<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	//"os"
	"strings"
	"time"
	//"github.com/stianeikeland/go-rpio"
)

// var (
// 	ledRed = rpio.Pin(4)
// 	ledYellow = rpio.Pin(17)
// 	ledGreen = rpio.Pin(27)
// )

var (
	ledRed, _    = embd.NewDigitalPin(4)
	ledYellow, _ = embd.NewDigitalPin(17)
	ledGreen, _  = embd.NewDigitalPin(27)
)

func getLEDString(color string) string {
	return "Toggle " + strings.ToUpper(color)
}

func getToggledValue(pin embd.DigitalPin) int {
	val,_ := pin.Read()
	if val == embd.High {
		return embd.Low
	} else {
		return embd.High
	}
}

func toggleLED(pin embd.DigitalPin, color string) {
	fmt.Println(getLEDString(color))
	toggledValue := getToggledValue(pin)
	fmt.Println("Val to write", toggledValue)
	pin.Write()
}

// func toggleLED(pin rpio.Pin, color string)  {
// 	fmt.Println(getLEDString(color))
// 	pin.Toggle()
// }

func initLEDs() {
	embd.SetDirection(4, embd.Out)
	embd.SetDirection(17, embd.Out)
	embd.SetDirection(27, embd.Out)
	// ledRed.Output()
	// ledYellow.Output()
	// ledGreen.Output()
}

func main() {
	fmt.Println("Parsing parameters")
	num := flag.Int("num", 0, "number of blinks")
	flag.Parse()

	fmt.Println("Number of blinks:", *num)

	fmt.Println("Opening rpio access")

	//var err = rpio.Open()
	embd.InitGPIO()

	// if err != nil {
	// 	fmt.Println(err)
	// 	os.Exit(1)
	// }

	//defer rpio.Close()
	defer embd.CloseGPIO()

	fmt.Println("Pin as output")

	for i := 0; i < *num; i++ {
		if i%3 == 0 {
			toggleLED(ledRed, "red")
		} else if i%3 == 1 {
			toggleLED(ledYellow, "yellow")
		} else {
			toggleLED(ledGreen, "green")
		}
		time.Sleep(time.Second)
	}
}
<|endoftext|>


<commit_before>package matcher

import (
	"fmt"
	"log"
)

const (
	charNil charType = iota
	charEscapeLiteral
	charLiteral
	charDot
	charStar
	charConcat
	charOr
)

//go:generate stringer -type=charType
type charType int

type char struct {
	typ charType
	val byte
}

func (c char) String() string {
	return fmt.Sprintf("{%s %q}", c.typ, c.val)
}

type lexer struct {
	expression string
	pos        int
	chars      []char
}

func (l *lexer) run() {
	for {
		switch l.expression[l.pos] {
		case '\\':
			l.emit(charEscapeLiteral)
		case '.':
			l.emit(charDot)
		case '*':
			l.emit(charStar)
		case '|':
			l.emit(charOr)
		default:
			l.emit(charLiteral)
		}
		if !l.next() {
			break
		}
	}
}

func (l *lexer) top() *char {
	if len(l.chars) > 0 {
		return &l.chars[len(l.chars)-1]
	}
	return nil
}

// emit validates and appends the concatenated characters
// to to a slice.
func (l *lexer) emit(t charType) {
	c := l.expression[l.pos]
	if t == charEscapeLiteral {
		if l.next() {
			c = escape(l.expression[l.pos])
		} else {
			log.Fatalln("cannot have a trailing backslash in regular expression")
		}
	}
	top := l.top()
	if t == charStar {
		if top == nil || (top.typ != charLiteral && top.typ != charDot) {
			log.Fatalln("Preceding token to star is not quantifiable")
		}
	}
	if t != charStar && t != charOr && (top == nil || top.typ != charOr) {
		l.chars = append(l.chars, char{charConcat, '.'})
	}
	l.chars = append(l.chars, char{t, c})
}

func escape(c byte) byte {
	switch c {
	case '0':
		return '\x00'
	case 'a':
		return '\x07'
	case 'b':
		return '\x08'
	case 't':
		return '\x09'
	case 'n':
		return '\x0A'
	case 'v':
		return '\x0B'
	case 'f':
		return '\x0C'
	case 'r':
		return '\x0D'
	case 'e':
		return '\x1B'
	case '\\':
		return '\x5C'
	default:
		return c
	}
}

func (l *lexer) next() bool {
	l.pos++
	if l.pos < len(l.expression) {
		return true
	}
	return false
}

// Lex parses the input regular expression, and returns
// a sequence of concatenated character tokens.
func Lex(expression string) []char {
	if len(expression) == 0 {
		return []char{}
	}
	l := &lexer{
		expression: expression,
		chars:      make([]char, 0, len(expression)),
	}
	l.run()
	return l.chars[1:]
}

// Postfix converts a sequence of character tokens
// into postfix format. For instance, in order of
// highest to lowest precedence:
// A.B*		-->		AB*.
// A.B.C	-->		AB.C.
// A.B|C.D	-->		AB.CD.|
func Postfix(chars []char) []char {
	output := []char{}
	operator := []char{}
	pop := func() *char {
		c := &operator[len(operator)-1]
		operator = operator[:len(operator)-1]
		return c
	}
	top := func() *char {
		if len(operator) > 0 {
			c := &operator[len(operator)-1]
			return c
		}
		return nil
	}
	for _, c := range chars {
		switch c.typ {
		case charStar:
			if t := top(); t != nil {
				if t.typ == charStar {
					output = append(output, *pop())
				}
			}
			operator = append(operator, c)
		case charConcat:
			if t := top(); t != nil {
				if t.typ == charConcat || t.typ == charStar {
					output = append(output, *pop())
				}
			}
			operator = append(operator, c)
		case charOr:
			if t := top(); t != nil {
				output = append(output, *pop())
			}
			operator = append(operator, c)
		default:
			output = append(output, c)
		}
	}
	oplen := len(operator)
	for i := 0; i < oplen; i++ {
		output = append(output, *pop())
	}
	return output
}
<commit_msg>formatting postfix for godoc<commit_after>package matcher

import (
	"fmt"
	"log"
)

const (
	charNil charType = iota
	charEscapeLiteral
	charLiteral
	charDot
	charStar
	charConcat
	charOr
)

//go:generate stringer -type=charType
type charType int

type char struct {
	typ charType
	val byte
}

func (c char) String() string {
	return fmt.Sprintf("{%s %q}", c.typ, c.val)
}

type lexer struct {
	expression string
	pos        int
	chars      []char
}

func (l *lexer) run() {
	for {
		switch l.expression[l.pos] {
		case '\\':
			l.emit(charEscapeLiteral)
		case '.':
			l.emit(charDot)
		case '*':
			l.emit(charStar)
		case '|':
			l.emit(charOr)
		default:
			l.emit(charLiteral)
		}
		if !l.next() {
			break
		}
	}
}

func (l *lexer) top() *char {
	if len(l.chars) > 0 {
		return &l.chars[len(l.chars)-1]
	}
	return nil
}

// emit validates and appends the concatenated characters
// to to a slice.
func (l *lexer) emit(t charType) {
	c := l.expression[l.pos]
	if t == charEscapeLiteral {
		if l.next() {
			c = escape(l.expression[l.pos])
		} else {
			log.Fatalln("cannot have a trailing backslash in regular expression")
		}
	}
	top := l.top()
	if t == charStar {
		if top == nil || (top.typ != charLiteral && top.typ != charDot) {
			log.Fatalln("Preceding token to star is not quantifiable")
		}
	}
	if t != charStar && t != charOr && (top == nil || top.typ != charOr) {
		l.chars = append(l.chars, char{charConcat, '.'})
	}
	l.chars = append(l.chars, char{t, c})
}

func escape(c byte) byte {
	switch c {
	case '0':
		return '\x00'
	case 'a':
		return '\x07'
	case 'b':
		return '\x08'
	case 't':
		return '\x09'
	case 'n':
		return '\x0A'
	case 'v':
		return '\x0B'
	case 'f':
		return '\x0C'
	case 'r':
		return '\x0D'
	case 'e':
		return '\x1B'
	case '\\':
		return '\x5C'
	default:
		return c
	}
}

func (l *lexer) next() bool {
	l.pos++
	if l.pos < len(l.expression) {
		return true
	}
	return false
}

// Lex parses the input regular expression, and returns
// a sequence of concatenated character tokens.
func Lex(expression string) []char {
	if len(expression) == 0 {
		return []char{}
	}
	l := &lexer{
		expression: expression,
		chars:      make([]char, 0, len(expression)),
	}
	l.run()
	return l.chars[1:]
}

// Postfix converts a sequence of character tokens
// into postfix format. For instance, in order of
// highest to lowest precedence:
//		A.B*	-->		AB*.
//		A.B.C	-->		AB.C.
//		A.B|C.D	-->		AB.CD.|
func Postfix(chars []char) []char {
	output := []char{}
	operator := []char{}
	pop := func() *char {
		c := &operator[len(operator)-1]
		operator = operator[:len(operator)-1]
		return c
	}
	top := func() *char {
		if len(operator) > 0 {
			c := &operator[len(operator)-1]
			return c
		}
		return nil
	}
	for _, c := range chars {
		switch c.typ {
		case charStar:
			if t := top(); t != nil {
				if t.typ == charStar {
					output = append(output, *pop())
				}
			}
			operator = append(operator, c)
		case charConcat:
			if t := top(); t != nil {
				if t.typ == charConcat || t.typ == charStar {
					output = append(output, *pop())
				}
			}
			operator = append(operator, c)
		case charOr:
			if t := top(); t != nil {
				output = append(output, *pop())
			}
			operator = append(operator, c)
		default:
			output = append(output, c)
		}
	}
	oplen := len(operator)
	for i := 0; i < oplen; i++ {
		output = append(output, *pop())
	}
	return output
}
<|endoftext|>


<commit_before>package raft

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"sync"
)

//------------------------------------------------------------------------------
//
// Typedefs
//
//------------------------------------------------------------------------------

// A log is a collection of log entries that are persisted to durable storage.
type Log struct {
	ApplyFunc    func(Command)
	file         *os.File
	entries      []*LogEntry
	commitIndex  uint64
	commandTypes map[string]Command
	mutex        sync.Mutex
}

//------------------------------------------------------------------------------
//
// Constructor
//
//------------------------------------------------------------------------------

// Creates a new log.
func NewLog() *Log {
	l := &Log{commandTypes: make(map[string]Command)}
	l.AddCommandType(&JoinCommand{})
	return l
}

//------------------------------------------------------------------------------
//
// Accessors
//
//------------------------------------------------------------------------------

//--------------------------------------
// Log Indices
//--------------------------------------

// The current index in the log.
func (l *Log) CurrentIndex() uint64 {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if len(l.entries) == 0 {
		return 0
	}
	return l.entries[len(l.entries)-1].index
}

// The next index in the log.
func (l *Log) NextIndex() uint64 {
	return l.CurrentIndex() + 1
}

// The last committed index in the log.
func (l *Log) CommitIndex() uint64 {
	return l.commitIndex
}

// Determines if the log contains zero entries.
func (l *Log) IsEmpty() bool {
	l.mutex.Lock()
	defer l.mutex.Unlock()
	return (len(l.entries) == 0)
}

//--------------------------------------
// Log Terms
//--------------------------------------

// The current term in the log.
func (l *Log) CurrentTerm() uint64 {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if len(l.entries) == 0 {
		return 0
	}
	return l.entries[len(l.entries)-1].term
}

//------------------------------------------------------------------------------
//
// Methods
//
//------------------------------------------------------------------------------

//--------------------------------------
// Commands
//--------------------------------------

// Instantiates a new command by type name. Returns an error if the command type
// has not been registered already.
func (l *Log) NewCommand(name string) (Command, error) {
	// Find the registered command.
	command := l.commandTypes[name]
	if command == nil {
		return nil, fmt.Errorf("raft.Log: Unregistered command type: %s", name)
	}

	// Make a copy of the command.
	v := reflect.New(reflect.Indirect(reflect.ValueOf(command)).Type()).Interface()
	copy, ok := v.(Command)
	if !ok {
		panic(fmt.Sprintf("raft.Log: Unable to copy command: %s (%v)", command.CommandName(), reflect.ValueOf(v).Kind().String()))
	}
	return copy, nil
}

// Adds a command type to the log. The instance passed in will be copied and
// deserialized each time a new log entry is read. This function will panic
// if a command type with the same name already exists.
func (l *Log) AddCommandType(command Command) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if command == nil {
		panic(fmt.Sprintf("raft.Log: Command type cannot be nil"))
	} else if l.commandTypes[command.CommandName()] != nil {
		panic(fmt.Sprintf("raft.Log: Command type already exists: %s", command.CommandName()))
	}
	l.commandTypes[command.CommandName()] = command
}

//--------------------------------------
// State
//--------------------------------------

// Opens the log file and reads existing entries. The log can remain open and
// continue to append entries to the end of the log.
func (l *Log) Open(path string) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Read all the entries from the log if one exists.
	var lastIndex int = 0
	if _, err := os.Stat(path); !os.IsNotExist(err) {
		// Open the log file.
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		reader := bufio.NewReader(file)

		// Read the file and decode entries.
		for {
			if _, err := reader.Peek(1); err == io.EOF {
				break
			}

			// Instantiate log entry and decode into it.
			entry := NewLogEntry(l, 0, 0, nil)
			n, err := entry.Decode(reader)
			if err != nil {
				warn("raft.Log: %v", err)
				warn("raft.Log: Recovering (%d)", lastIndex)
				file.Close()
				if err = os.Truncate(path, int64(lastIndex)); err != nil {
					return fmt.Errorf("raft.Log: Unable to recover: %v", err)
				}
				break
			}
			l.commitIndex = entry.index
			lastIndex += n

			// Append entry.
			l.entries = append(l.entries, entry)
		}

		file.Close()
	}

	// Open the file for appending.
	var err error
	l.file, err = os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	return nil
}

// Closes the log file.
func (l *Log) Close() {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.file != nil {
		l.file.Close()
		l.file = nil
	}
	l.entries = make([]*LogEntry, 0)
}

//--------------------------------------
// Entries
//--------------------------------------

// Creates a log entry associated with this log.
func (l *Log) CreateEntry(term uint64, command Command) *LogEntry {
	return NewLogEntry(l, l.NextIndex(), term, command)
}

// Checks if the log contains a given index/term combination.
func (l *Log) ContainsEntry(index uint64, term uint64) bool {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if index == 0 || index > uint64(len(l.entries)) {
		return false
	}
	return (l.entries[index-1].term == term)
}

// Retrieves a list of entries after a given index. This function also returns
// the term of the index provided.
func (l *Log) GetEntriesAfter(index uint64) ([]*LogEntry, uint64) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Return an error if the index doesn't exist.
	if index > uint64(len(l.entries)) {
		panic(fmt.Sprintf("raft.Log: Index is beyond end of log: %v", index))
	}

	// If we're going from the beginning of the log then return the whole log.
	if index == 0 {
		return l.entries, 0
	}

	// Determine the term at the given entry and return a subslice.
	term := l.entries[index-1].term
	return l.entries[index:], term
}

//--------------------------------------
// Commit
//--------------------------------------

// Retrieves the last index and term that has been committed to the log.
func (l *Log) CommitInfo() (index uint64, term uint64) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// If we don't have any entries then just return zeros.
	if l.commitIndex == 0 {
		return 0, 0
	}

	// Return the last index & term from the last committed entry.
	lastCommitEntry := l.entries[l.commitIndex-1]
	return lastCommitEntry.index, lastCommitEntry.term
}

// Updates the commit index and writes entries after that index to the stable storage.
func (l *Log) SetCommitIndex(index uint64) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Panic if we don't have any way to apply commands.
	if l.ApplyFunc == nil {
		panic("raft.Log: Apply function not set")
	}

	// Do not allow previous indices to be committed again.
	if index < l.commitIndex {
		return fmt.Errorf("raft.Log: Commit index (%d) ahead of requested commit index (%d)", l.commitIndex, index)
	}
	if index > uint64(len(l.entries)) {
		return fmt.Errorf("raft.Log: Commit index (%d) out of range (%d)", index, len(l.entries))
	}

	// Find all entries whose index is between the previous index and the current index.
	for i := l.commitIndex + 1; i <= index; i++ {
		entry := l.entries[i-1]

		// Write to storage.
		if err := entry.Encode(l.file); err != nil {
			return err
		}

		// Apply the changes to the state machine.
		l.ApplyFunc(entry.command)

		// Update commit index.
		l.commitIndex = entry.index
	}

	return nil
}

//--------------------------------------
// Truncation
//--------------------------------------

// Truncates the log to the given index and term. This only works if the log
// at the index has not been committed.
func (l *Log) Truncate(index uint64, term uint64) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Do not allow committed entries to be truncated.
	if index < l.CommitIndex() {
		return fmt.Errorf("raft.Log: Index is already committed (%v): (IDX=%v, TERM=%v)", l.CommitIndex(), index, term)
	}

	// Do not truncate past end of entries.
	if index > uint64(len(l.entries)) {
		return fmt.Errorf("raft.Log: Entry index does not exist (MAX=%v): (IDX=%v, TERM=%v)", len(l.entries), index, term)
	}

	// If we're truncating everything then just clear the entries.
	if index == 0 {
		l.entries = []*LogEntry{}
	} else {
		// Do not truncate if the entry at index does not have the matching term.
		entry := l.entries[index-1]
		if len(l.entries) > 0 && entry.term != term {
			return fmt.Errorf("raft.Log: Entry at index does not have matching term (%v): (IDX=%v, TERM=%v)", entry.term, index, term)
		}

		// Otherwise truncate up to the desired entry.
		if index < uint64(len(l.entries)) {
			l.entries = l.entries[0:index]
		}
	}

	return nil
}

//--------------------------------------
// Append
//--------------------------------------

// Appends a series of entries to the log. These entries are not written to
// disk until SetCommitIndex() is called.
func (l *Log) AppendEntries(entries []*LogEntry) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Append each entry but exit if we hit an error.
	for _, entry := range entries {
		if err := l.appendEntry(entry); err != nil {
			return err
		}
	}

	return nil
}

// Appends a single entry to the log.
func (l *Log) AppendEntry(entry *LogEntry) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()
	return l.appendEntry(entry)
}

// Writes a single log entry to the end of the log. This function does not
// obtain a lock and should only be used internally. Use AppendEntries() and
// AppendEntry() to use it externally.
func (l *Log) appendEntry(entry *LogEntry) error {
	if l.file == nil {
		return errors.New("raft.Log: Log is not open")
	}

	// Make sure the term and index are greater than the previous.
	if len(l.entries) > 0 {
		lastEntry := l.entries[len(l.entries)-1]
		if entry.term < lastEntry.term {
			return fmt.Errorf("raft.Log: Cannot append entry with earlier term (%x:%x <= %x:%x)", entry.term, entry.index, lastEntry.term, lastEntry.index)
		} else if entry.index == lastEntry.index && entry.index <= lastEntry.index {
			return fmt.Errorf("raft.Log: Cannot append entry with earlier index in the same term (%x:%x <= %x:%x)", entry.term, entry.index, lastEntry.term, lastEntry.index)
		}
	}

	// Append to entries list if stored on disk.
	l.entries = append(l.entries, entry)

	return nil
}
<commit_msg>Add log entry access for debugging.<commit_after>package raft

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"sync"
)

//------------------------------------------------------------------------------
//
// Typedefs
//
//------------------------------------------------------------------------------

// A log is a collection of log entries that are persisted to durable storage.
type Log struct {
	ApplyFunc    func(Command)
	file         *os.File
	entries      []*LogEntry
	commitIndex  uint64
	commandTypes map[string]Command
	mutex        sync.Mutex
}

//------------------------------------------------------------------------------
//
// Constructor
//
//------------------------------------------------------------------------------

// Creates a new log.
func NewLog() *Log {
	l := &Log{commandTypes: make(map[string]Command)}
	l.AddCommandType(&JoinCommand{})
	return l
}

//------------------------------------------------------------------------------
//
// Accessors
//
//------------------------------------------------------------------------------

//--------------------------------------
// Log Indices
//--------------------------------------

// The current index in the log.
func (l *Log) CurrentIndex() uint64 {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if len(l.entries) == 0 {
		return 0
	}
	return l.entries[len(l.entries)-1].index
}

// The next index in the log.
func (l *Log) NextIndex() uint64 {
	return l.CurrentIndex() + 1
}

// The last committed index in the log.
func (l *Log) CommitIndex() uint64 {
	return l.commitIndex
}

// Determines if the log contains zero entries.
func (l *Log) IsEmpty() bool {
	l.mutex.Lock()
	defer l.mutex.Unlock()
	return (len(l.entries) == 0)
}

// A list of all the log entries. This should only be used for debugging purposes.
func (l *Log) Entries() []*LogEntry {
	return l.entries
}

//--------------------------------------
// Log Terms
//--------------------------------------

// The current term in the log.
func (l *Log) CurrentTerm() uint64 {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if len(l.entries) == 0 {
		return 0
	}
	return l.entries[len(l.entries)-1].term
}

//------------------------------------------------------------------------------
//
// Methods
//
//------------------------------------------------------------------------------

//--------------------------------------
// Commands
//--------------------------------------

// Instantiates a new command by type name. Returns an error if the command type
// has not been registered already.
func (l *Log) NewCommand(name string) (Command, error) {
	// Find the registered command.
	command := l.commandTypes[name]
	if command == nil {
		return nil, fmt.Errorf("raft.Log: Unregistered command type: %s", name)
	}

	// Make a copy of the command.
	v := reflect.New(reflect.Indirect(reflect.ValueOf(command)).Type()).Interface()
	copy, ok := v.(Command)
	if !ok {
		panic(fmt.Sprintf("raft.Log: Unable to copy command: %s (%v)", command.CommandName(), reflect.ValueOf(v).Kind().String()))
	}
	return copy, nil
}

// Adds a command type to the log. The instance passed in will be copied and
// deserialized each time a new log entry is read. This function will panic
// if a command type with the same name already exists.
func (l *Log) AddCommandType(command Command) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if command == nil {
		panic(fmt.Sprintf("raft.Log: Command type cannot be nil"))
	} else if l.commandTypes[command.CommandName()] != nil {
		panic(fmt.Sprintf("raft.Log: Command type already exists: %s", command.CommandName()))
	}
	l.commandTypes[command.CommandName()] = command
}

//--------------------------------------
// State
//--------------------------------------

// Opens the log file and reads existing entries. The log can remain open and
// continue to append entries to the end of the log.
func (l *Log) Open(path string) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Read all the entries from the log if one exists.
	var lastIndex int = 0
	if _, err := os.Stat(path); !os.IsNotExist(err) {
		// Open the log file.
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()
		reader := bufio.NewReader(file)

		// Read the file and decode entries.
		for {
			if _, err := reader.Peek(1); err == io.EOF {
				break
			}

			// Instantiate log entry and decode into it.
			entry := NewLogEntry(l, 0, 0, nil)
			n, err := entry.Decode(reader)
			if err != nil {
				warn("raft.Log: %v", err)
				warn("raft.Log: Recovering (%d)", lastIndex)
				file.Close()
				if err = os.Truncate(path, int64(lastIndex)); err != nil {
					return fmt.Errorf("raft.Log: Unable to recover: %v", err)
				}
				break
			}
			l.commitIndex = entry.index
			lastIndex += n

			// Append entry.
			l.entries = append(l.entries, entry)
		}

		file.Close()
	}

	// Open the file for appending.
	var err error
	l.file, err = os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	return nil
}

// Closes the log file.
func (l *Log) Close() {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if l.file != nil {
		l.file.Close()
		l.file = nil
	}
	l.entries = make([]*LogEntry, 0)
}

//--------------------------------------
// Entries
//--------------------------------------

// Creates a log entry associated with this log.
func (l *Log) CreateEntry(term uint64, command Command) *LogEntry {
	return NewLogEntry(l, l.NextIndex(), term, command)
}

// Checks if the log contains a given index/term combination.
func (l *Log) ContainsEntry(index uint64, term uint64) bool {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	if index == 0 || index > uint64(len(l.entries)) {
		return false
	}
	return (l.entries[index-1].term == term)
}

// Retrieves a list of entries after a given index. This function also returns
// the term of the index provided.
func (l *Log) GetEntriesAfter(index uint64) ([]*LogEntry, uint64) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Return an error if the index doesn't exist.
	if index > uint64(len(l.entries)) {
		panic(fmt.Sprintf("raft.Log: Index is beyond end of log: %v", index))
	}

	// If we're going from the beginning of the log then return the whole log.
	if index == 0 {
		return l.entries, 0
	}

	// Determine the term at the given entry and return a subslice.
	term := l.entries[index-1].term
	return l.entries[index:], term
}

//--------------------------------------
// Commit
//--------------------------------------

// Retrieves the last index and term that has been committed to the log.
func (l *Log) CommitInfo() (index uint64, term uint64) {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// If we don't have any entries then just return zeros.
	if l.commitIndex == 0 {
		return 0, 0
	}

	// Return the last index & term from the last committed entry.
	lastCommitEntry := l.entries[l.commitIndex-1]
	return lastCommitEntry.index, lastCommitEntry.term
}

// Updates the commit index and writes entries after that index to the stable storage.
func (l *Log) SetCommitIndex(index uint64) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Panic if we don't have any way to apply commands.
	if l.ApplyFunc == nil {
		panic("raft.Log: Apply function not set")
	}

	// Do not allow previous indices to be committed again.
	if index < l.commitIndex {
		return fmt.Errorf("raft.Log: Commit index (%d) ahead of requested commit index (%d)", l.commitIndex, index)
	}
	if index > uint64(len(l.entries)) {
		return fmt.Errorf("raft.Log: Commit index (%d) out of range (%d)", index, len(l.entries))
	}

	// Find all entries whose index is between the previous index and the current index.
	for i := l.commitIndex + 1; i <= index; i++ {
		entry := l.entries[i-1]

		// Write to storage.
		if err := entry.Encode(l.file); err != nil {
			return err
		}

		// Apply the changes to the state machine.
		l.ApplyFunc(entry.command)

		// Update commit index.
		l.commitIndex = entry.index
	}

	return nil
}

//--------------------------------------
// Truncation
//--------------------------------------

// Truncates the log to the given index and term. This only works if the log
// at the index has not been committed.
func (l *Log) Truncate(index uint64, term uint64) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Do not allow committed entries to be truncated.
	if index < l.CommitIndex() {
		return fmt.Errorf("raft.Log: Index is already committed (%v): (IDX=%v, TERM=%v)", l.CommitIndex(), index, term)
	}

	// Do not truncate past end of entries.
	if index > uint64(len(l.entries)) {
		return fmt.Errorf("raft.Log: Entry index does not exist (MAX=%v): (IDX=%v, TERM=%v)", len(l.entries), index, term)
	}

	// If we're truncating everything then just clear the entries.
	if index == 0 {
		l.entries = []*LogEntry{}
	} else {
		// Do not truncate if the entry at index does not have the matching term.
		entry := l.entries[index-1]
		if len(l.entries) > 0 && entry.term != term {
			return fmt.Errorf("raft.Log: Entry at index does not have matching term (%v): (IDX=%v, TERM=%v)", entry.term, index, term)
		}

		// Otherwise truncate up to the desired entry.
		if index < uint64(len(l.entries)) {
			l.entries = l.entries[0:index]
		}
	}

	return nil
}

//--------------------------------------
// Append
//--------------------------------------

// Appends a series of entries to the log. These entries are not written to
// disk until SetCommitIndex() is called.
func (l *Log) AppendEntries(entries []*LogEntry) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()

	// Append each entry but exit if we hit an error.
	for _, entry := range entries {
		if err := l.appendEntry(entry); err != nil {
			return err
		}
	}

	return nil
}

// Appends a single entry to the log.
func (l *Log) AppendEntry(entry *LogEntry) error {
	l.mutex.Lock()
	defer l.mutex.Unlock()
	return l.appendEntry(entry)
}

// Writes a single log entry to the end of the log. This function does not
// obtain a lock and should only be used internally. Use AppendEntries() and
// AppendEntry() to use it externally.
func (l *Log) appendEntry(entry *LogEntry) error {
	if l.file == nil {
		return errors.New("raft.Log: Log is not open")
	}

	// Make sure the term and index are greater than the previous.
	if len(l.entries) > 0 {
		lastEntry := l.entries[len(l.entries)-1]
		if entry.term < lastEntry.term {
			return fmt.Errorf("raft.Log: Cannot append entry with earlier term (%x:%x <= %x:%x)", entry.term, entry.index, lastEntry.term, lastEntry.index)
		} else if entry.index == lastEntry.index && entry.index <= lastEntry.index {
			return fmt.Errorf("raft.Log: Cannot append entry with earlier index in the same term (%x:%x <= %x:%x)", entry.term, entry.index, lastEntry.term, lastEntry.index)
		}
	}

	// Append to entries list if stored on disk.
	l.entries = append(l.entries, entry)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pkcs11

/*
#include <stdlib.h>
#include <string.h>
#include "pkcs11go.h"

CK_ULONG Index(CK_ULONG_PTR array, CK_ULONG i)
{
	return array[i];
}

static inline void putAttributePval(CK_ATTRIBUTE_PTR a, CK_VOID_PTR pValue)
{
	a->pValue = pValue;
}

static inline void putMechanismParam(CK_MECHANISM_PTR m, CK_VOID_PTR pParameter)
{
	m->pParameter = pParameter;
}
*/
import "C"

import (
	"fmt"
	"time"
	"unsafe"
)

type arena []unsafe.Pointer

func (a *arena) Allocate(obj []byte) (C.CK_VOID_PTR, C.CK_ULONG) {
	cobj := C.calloc(C.size_t(len(obj)), 1)
	*a = append(*a, cobj)
	C.memmove(cobj, unsafe.Pointer(&obj[0]), C.size_t(len(obj)))
	return C.CK_VOID_PTR(cobj), C.CK_ULONG(len(obj))
}

func (a arena) Free() {
	for _, p := range a {
		C.free(p)
	}
}

// toList converts from a C style array to a []uint.
func toList(clist C.CK_ULONG_PTR, size C.CK_ULONG) []uint {
	l := make([]uint, int(size))
	for i := 0; i < len(l); i++ {
		l[i] = uint(C.Index(clist, C.CK_ULONG(i)))
	}
	defer C.free(unsafe.Pointer(clist))
	return l
}

// cBBool converts a bool to a CK_BBOOL.
func cBBool(x bool) C.CK_BBOOL {
	if x {
		return C.CK_BBOOL(C.CK_TRUE)
	}
	return C.CK_BBOOL(C.CK_FALSE)
}

func uintToBytes(x uint64) []byte {
	ul := C.CK_ULONG(x)
	return C.GoBytes(unsafe.Pointer(&ul), C.int(unsafe.Sizeof(ul)))
}

// Error represents an PKCS#11 error.
type Error uint

func (e Error) Error() string {
	return fmt.Sprintf("pkcs11: 0x%X: %s", uint(e), strerror[uint(e)])
}

func toError(e C.CK_RV) error {
	if e == C.CKR_OK {
		return nil
	}
	return Error(e)
}

// SessionHandle is a Cryptoki-assigned value that identifies a session.
type SessionHandle uint

// ObjectHandle is a token-specific identifier for an object.
type ObjectHandle uint

// Version represents any version information from the library.
type Version struct {
	Major byte
	Minor byte
}

func toVersion(version C.CK_VERSION) Version {
	return Version{byte(version.major), byte(version.minor)}
}

// SlotEvent holds the SlotID which for which an slot event (token insertion,
// removal, etc.) occurred.
type SlotEvent struct {
	SlotID uint
}

// Info provides information about the library and hardware used.
type Info struct {
	CryptokiVersion    Version
	ManufacturerID     string
	Flags              uint
	LibraryDescription string
	LibraryVersion     Version
}

// SlotInfo provides information about a slot.
type SlotInfo struct {
	SlotDescription string // 64 bytes.
	ManufacturerID  string // 32 bytes.
	Flags           uint
	HardwareVersion Version
	FirmwareVersion Version
}

// TokenInfo provides information about a token.
type TokenInfo struct {
	Label              string
	ManufacturerID     string
	Model              string
	SerialNumber       string
	Flags              uint
	MaxSessionCount    uint
	SessionCount       uint
	MaxRwSessionCount  uint
	RwSessionCount     uint
	MaxPinLen          uint
	MinPinLen          uint
	TotalPublicMemory  uint
	FreePublicMemory   uint
	TotalPrivateMemory uint
	FreePrivateMemory  uint
	HardwareVersion    Version
	FirmwareVersion    Version
	UTCTime            string
}

// SessionInfo provides information about a session.
type SessionInfo struct {
	SlotID      uint
	State       uint
	Flags       uint
	DeviceError uint
}

// Attribute holds an attribute type/value combination.
type Attribute struct {
	Type  uint
	Value []byte
}

// NewAttribute allocates a Attribute and returns a pointer to it.
// Note that this is merely a convenience function, as values returned
// from the HSM are not converted back to Go values, those are just raw
// byte slices.
func NewAttribute(typ uint, x interface{}) *Attribute {
	// This function nicely transforms *to* an attribute, but there is
	// no corresponding function that transform back *from* an attribute,
	// which in PKCS#11 is just an byte array.
	a := new(Attribute)
	a.Type = typ
	if x == nil {
		return a
	}
	switch v := x.(type) {
	case bool:
		if v {
			a.Value = []byte{1}
		} else {
			a.Value = []byte{0}
		}
	case int:
		a.Value = uintToBytes(uint64(v))
	case uint:
		a.Value = uintToBytes(uint64(v))
	case string:
		a.Value = []byte(v)
	case []byte:
		a.Value = v
	case time.Time: // for CKA_DATE
		a.Value = cDate(v)
	default:
		panic("pkcs11: unhandled attribute type")
	}
	return a
}

// cAttribute returns the start address and the length of an attribute list.
func cAttributeList(a []*Attribute) (arena, C.CK_ATTRIBUTE_PTR, C.CK_ULONG) {
	var arena arena
	if len(a) == 0 {
		return nil, nil, 0
	}
	pa := make([]C.CK_ATTRIBUTE, len(a))
	for i, attr := range a {
		pa[i]._type = C.CK_ATTRIBUTE_TYPE(attr.Type)
		if len(attr.Value) != 0 {
			buf, len := arena.Allocate(attr.Value)
			// field is unaligned on windows so this has to call into C
			C.putAttributePval(&pa[i], buf)
			pa[i].ulValueLen = len
		}
	}
	return arena, &pa[0], C.CK_ULONG(len(a))
}

func cDate(t time.Time) []byte {
	b := make([]byte, 8)
	year, month, day := t.Date()
	y := fmt.Sprintf("%4d", year)
	m := fmt.Sprintf("%02d", month)
	d1 := fmt.Sprintf("%02d", day)
	b[0], b[1], b[2], b[3] = y[0], y[1], y[2], y[3]
	b[4], b[5] = m[0], m[1]
	b[6], b[7] = d1[0], d1[1]
	return b
}

// Mechanism holds an mechanism type/value combination.
type Mechanism struct {
	Mechanism uint
	Parameter []byte
	generator interface{}
}

// NewMechanism returns a pointer to an initialized Mechanism.
func NewMechanism(mech uint, x interface{}) *Mechanism {
	m := new(Mechanism)
	m.Mechanism = mech
	if x == nil {
		return m
	}

	switch p := x.(type) {
	case *GCMParams, *OAEPParams, *ECDH1DeriveParams:
		// contains pointers; defer serialization until cMechanism
		m.generator = p
	case []byte:
		m.Parameter = p
	default:
		panic("parameter must be one of type: []byte, *GCMParams, *OAEPParams, *ECDH1DeriveParams")
	}

	return m
}

func cMechanism(mechList []*Mechanism) (arena, *C.CK_MECHANISM) {
	if len(mechList) != 1 {
		panic("expected exactly one mechanism")
	}
	mech := mechList[0]
	cmech := &C.CK_MECHANISM{mechanism: C.CK_MECHANISM_TYPE(mech.Mechanism)}
	// params that contain pointers are allocated here
	param := mech.Parameter
	var arena arena
	switch p := mech.generator.(type) {
	case *GCMParams:
		// uses its own arena because it has to outlive this function call (yuck)
		param = cGCMParams(p)
	case *OAEPParams:
		param, arena = cOAEPParams(p, arena)
	case *ECDH1DeriveParams:
		param, arena = cECDH1DeriveParams(p, arena)
	}
	if len(param) != 0 {
		buf, len := arena.Allocate(param)
		// field is unaligned on windows so this has to call into C
		C.putMechanismParam(cmech, buf)
		cmech.ulParameterLen = len
	}
	return arena, cmech
}

// MechanismInfo provides information about a particular mechanism.
type MechanismInfo struct {
	MinKeySize uint
	MaxKeySize uint
	Flags      uint
}

// stubData is a persistent nonempty byte array used by cMessage.
var stubData = []byte{0}

// cMessage returns the pointer/length pair corresponding to data.
func cMessage(data []byte) (dataPtr C.CK_BYTE_PTR) {
	l := len(data)
	if l == 0 {
		// &data[0] is forbidden in this case, so use a nontrivial array instead.
		data = stubData
	}
	return C.CK_BYTE_PTR(unsafe.Pointer(&data[0]))
}
<commit_msg>fix NewAttribute panic (#143)<commit_after>// Copyright 2013 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pkcs11

/*
#include <stdlib.h>
#include <string.h>
#include "pkcs11go.h"

CK_ULONG Index(CK_ULONG_PTR array, CK_ULONG i)
{
	return array[i];
}

static inline void putAttributePval(CK_ATTRIBUTE_PTR a, CK_VOID_PTR pValue)
{
	a->pValue = pValue;
}

static inline void putMechanismParam(CK_MECHANISM_PTR m, CK_VOID_PTR pParameter)
{
	m->pParameter = pParameter;
}
*/
import "C"

import (
	"fmt"
	"time"
	"unsafe"
)

type arena []unsafe.Pointer

func (a *arena) Allocate(obj []byte) (C.CK_VOID_PTR, C.CK_ULONG) {
	cobj := C.calloc(C.size_t(len(obj)), 1)
	*a = append(*a, cobj)
	C.memmove(cobj, unsafe.Pointer(&obj[0]), C.size_t(len(obj)))
	return C.CK_VOID_PTR(cobj), C.CK_ULONG(len(obj))
}

func (a arena) Free() {
	for _, p := range a {
		C.free(p)
	}
}

// toList converts from a C style array to a []uint.
func toList(clist C.CK_ULONG_PTR, size C.CK_ULONG) []uint {
	l := make([]uint, int(size))
	for i := 0; i < len(l); i++ {
		l[i] = uint(C.Index(clist, C.CK_ULONG(i)))
	}
	defer C.free(unsafe.Pointer(clist))
	return l
}

// cBBool converts a bool to a CK_BBOOL.
func cBBool(x bool) C.CK_BBOOL {
	if x {
		return C.CK_BBOOL(C.CK_TRUE)
	}
	return C.CK_BBOOL(C.CK_FALSE)
}

func uintToBytes(x uint64) []byte {
	ul := C.CK_ULONG(x)
	return C.GoBytes(unsafe.Pointer(&ul), C.int(unsafe.Sizeof(ul)))
}

// Error represents an PKCS#11 error.
type Error uint

func (e Error) Error() string {
	return fmt.Sprintf("pkcs11: 0x%X: %s", uint(e), strerror[uint(e)])
}

func toError(e C.CK_RV) error {
	if e == C.CKR_OK {
		return nil
	}
	return Error(e)
}

// SessionHandle is a Cryptoki-assigned value that identifies a session.
type SessionHandle uint

// ObjectHandle is a token-specific identifier for an object.
type ObjectHandle uint

// Version represents any version information from the library.
type Version struct {
	Major byte
	Minor byte
}

func toVersion(version C.CK_VERSION) Version {
	return Version{byte(version.major), byte(version.minor)}
}

// SlotEvent holds the SlotID which for which an slot event (token insertion,
// removal, etc.) occurred.
type SlotEvent struct {
	SlotID uint
}

// Info provides information about the library and hardware used.
type Info struct {
	CryptokiVersion    Version
	ManufacturerID     string
	Flags              uint
	LibraryDescription string
	LibraryVersion     Version
}

// SlotInfo provides information about a slot.
type SlotInfo struct {
	SlotDescription string // 64 bytes.
	ManufacturerID  string // 32 bytes.
	Flags           uint
	HardwareVersion Version
	FirmwareVersion Version
}

// TokenInfo provides information about a token.
type TokenInfo struct {
	Label              string
	ManufacturerID     string
	Model              string
	SerialNumber       string
	Flags              uint
	MaxSessionCount    uint
	SessionCount       uint
	MaxRwSessionCount  uint
	RwSessionCount     uint
	MaxPinLen          uint
	MinPinLen          uint
	TotalPublicMemory  uint
	FreePublicMemory   uint
	TotalPrivateMemory uint
	FreePrivateMemory  uint
	HardwareVersion    Version
	FirmwareVersion    Version
	UTCTime            string
}

// SessionInfo provides information about a session.
type SessionInfo struct {
	SlotID      uint
	State       uint
	Flags       uint
	DeviceError uint
}

// Attribute holds an attribute type/value combination.
type Attribute struct {
	Type  uint
	Value []byte
}

// NewAttribute allocates a Attribute and returns a pointer to it.
// Note that this is merely a convenience function, as values returned
// from the HSM are not converted back to Go values, those are just raw
// byte slices.
func NewAttribute(typ uint, x interface{}) *Attribute {
	// This function nicely transforms *to* an attribute, but there is
	// no corresponding function that transform back *from* an attribute,
	// which in PKCS#11 is just an byte array.
	a := new(Attribute)
	a.Type = typ
	if x == nil {
		return a
	}
	switch v := x.(type) {
	case bool:
		if v {
			a.Value = []byte{1}
		} else {
			a.Value = []byte{0}
		}
	case int:
		a.Value = uintToBytes(uint64(v))
	case int16:
		a.Value = uintToBytes(uint64(v))
	case int32:
		a.Value = uintToBytes(uint64(v))
	case int64:
		a.Value = uintToBytes(uint64(v))
	case uint:
		a.Value = uintToBytes(uint64(v))
	case uint16:
		a.Value = uintToBytes(uint64(v))
	case uint32:
		a.Value = uintToBytes(uint64(v))
	case uint64:
		a.Value = uintToBytes(uint64(v))
	case string:
		a.Value = []byte(v)
	case []byte:
		a.Value = v
	case time.Time: // for CKA_DATE
		a.Value = cDate(v)
	default:
		panic("pkcs11: unhandled attribute type")
	}
	return a
}

// cAttribute returns the start address and the length of an attribute list.
func cAttributeList(a []*Attribute) (arena, C.CK_ATTRIBUTE_PTR, C.CK_ULONG) {
	var arena arena
	if len(a) == 0 {
		return nil, nil, 0
	}
	pa := make([]C.CK_ATTRIBUTE, len(a))
	for i, attr := range a {
		pa[i]._type = C.CK_ATTRIBUTE_TYPE(attr.Type)
		if len(attr.Value) != 0 {
			buf, len := arena.Allocate(attr.Value)
			// field is unaligned on windows so this has to call into C
			C.putAttributePval(&pa[i], buf)
			pa[i].ulValueLen = len
		}
	}
	return arena, &pa[0], C.CK_ULONG(len(a))
}

func cDate(t time.Time) []byte {
	b := make([]byte, 8)
	year, month, day := t.Date()
	y := fmt.Sprintf("%4d", year)
	m := fmt.Sprintf("%02d", month)
	d1 := fmt.Sprintf("%02d", day)
	b[0], b[1], b[2], b[3] = y[0], y[1], y[2], y[3]
	b[4], b[5] = m[0], m[1]
	b[6], b[7] = d1[0], d1[1]
	return b
}

// Mechanism holds an mechanism type/value combination.
type Mechanism struct {
	Mechanism uint
	Parameter []byte
	generator interface{}
}

// NewMechanism returns a pointer to an initialized Mechanism.
func NewMechanism(mech uint, x interface{}) *Mechanism {
	m := new(Mechanism)
	m.Mechanism = mech
	if x == nil {
		return m
	}

	switch p := x.(type) {
	case *GCMParams, *OAEPParams, *ECDH1DeriveParams:
		// contains pointers; defer serialization until cMechanism
		m.generator = p
	case []byte:
		m.Parameter = p
	default:
		panic("parameter must be one of type: []byte, *GCMParams, *OAEPParams, *ECDH1DeriveParams")
	}

	return m
}

func cMechanism(mechList []*Mechanism) (arena, *C.CK_MECHANISM) {
	if len(mechList) != 1 {
		panic("expected exactly one mechanism")
	}
	mech := mechList[0]
	cmech := &C.CK_MECHANISM{mechanism: C.CK_MECHANISM_TYPE(mech.Mechanism)}
	// params that contain pointers are allocated here
	param := mech.Parameter
	var arena arena
	switch p := mech.generator.(type) {
	case *GCMParams:
		// uses its own arena because it has to outlive this function call (yuck)
		param = cGCMParams(p)
	case *OAEPParams:
		param, arena = cOAEPParams(p, arena)
	case *ECDH1DeriveParams:
		param, arena = cECDH1DeriveParams(p, arena)
	}
	if len(param) != 0 {
		buf, len := arena.Allocate(param)
		// field is unaligned on windows so this has to call into C
		C.putMechanismParam(cmech, buf)
		cmech.ulParameterLen = len
	}
	return arena, cmech
}

// MechanismInfo provides information about a particular mechanism.
type MechanismInfo struct {
	MinKeySize uint
	MaxKeySize uint
	Flags      uint
}

// stubData is a persistent nonempty byte array used by cMessage.
var stubData = []byte{0}

// cMessage returns the pointer/length pair corresponding to data.
func cMessage(data []byte) (dataPtr C.CK_BYTE_PTR) {
	l := len(data)
	if l == 0 {
		// &data[0] is forbidden in this case, so use a nontrivial array instead.
		data = stubData
	}
	return C.CK_BYTE_PTR(unsafe.Pointer(&data[0]))
}
<|endoftext|>