<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrade

import (
	"fmt"
	"time"

	"github.com/pkg/errors"
	apps "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/images"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
	utilpointer "k8s.io/utils/pointer"
)

const (
	prepullPrefix = "upgrade-prepull-"
)

// Prepuller defines an interface for performing a prepull operation in a create-wait-delete fashion in parallel
type Prepuller interface {
	CreateFunc(string) error
	WaitFunc(string)
	DeleteFunc(string) error
}

// DaemonSetPrepuller makes sure the control-plane images are available on all control-planes
type DaemonSetPrepuller struct {
	client clientset.Interface
	cfg    *kubeadmapi.ClusterConfiguration
	waiter apiclient.Waiter
}

// NewDaemonSetPrepuller creates a new instance of the DaemonSetPrepuller struct
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.ClusterConfiguration) *DaemonSetPrepuller {
	return &DaemonSetPrepuller{
		client: client,
		cfg:    cfg,
		waiter: waiter,
	}
}

// CreateFunc creates a DaemonSet for making the image available on every relevant node
func (d *DaemonSetPrepuller) CreateFunc(component string) error {
	var image string
	if component == constants.Etcd {
		image = images.GetEtcdImage(d.cfg)
	} else {
		image = images.GetKubernetesImage(component, d.cfg)
	}
	ds := buildPrePullDaemonSet(component, image)

	// Create the DaemonSet in the API Server
	if err := apiclient.CreateOrUpdateDaemonSet(d.client, ds); err != nil {
		return errors.Wrapf(err, "unable to create a DaemonSet for prepulling the component %q", component)
	}
	return nil
}

// WaitFunc waits for all Pods in the specified DaemonSet to be in the Running state
func (d *DaemonSetPrepuller) WaitFunc(component string) {
	fmt.Printf("[upgrade/prepull] Prepulling image for component %s.\n", component)
	d.waiter.WaitForPodsWithLabel("k8s-app=upgrade-prepull-" + component)
}

// DeleteFunc deletes the DaemonSet used for making the image available on every relevant node
func (d *DaemonSetPrepuller) DeleteFunc(component string) error {
	dsName := addPrepullPrefix(component)
	if err := apiclient.DeleteDaemonSetForeground(d.client, metav1.NamespaceSystem, dsName); err != nil {
		return errors.Wrapf(err, "unable to cleanup the DaemonSet used for prepulling %s", component)
	}
	fmt.Printf("[upgrade/prepull] Prepulled image for component %s.\n", component)
	return nil
}

// PrepullImagesInParallel creates DaemonSets synchronously but waits in parallel for the images to pull
func PrepullImagesInParallel(kubePrepuller Prepuller, timeout time.Duration, componentsToPrepull []string) error {
	fmt.Printf("[upgrade/prepull] Will prepull images for components %v\n", componentsToPrepull)

	timeoutChan := time.After(timeout)

	// Synchronously create the DaemonSets
	for _, component := range componentsToPrepull {
		if err := kubePrepuller.CreateFunc(component); err != nil {
			return err
		}
	}

	// Create a channel for streaming data from goroutines that run in parallel to a blocking for loop that cleans up
	prePulledChan := make(chan string, len(componentsToPrepull))
	for _, component := range componentsToPrepull {
		go func(c string) {
			// Wait as long as needed. This WaitFunc call should be blocking until completion
			kubePrepuller.WaitFunc(c)
			// When the task is done, go ahead and cleanup by sending the name to the channel
			prePulledChan <- c
		}(component)
	}

	// This call blocks until all expected messages are received from the channel or errors out if timeoutChan fires.
	// For every successful wait, kubePrepuller.DeleteFunc is executed
	if err := waitForItemsFromChan(timeoutChan, prePulledChan, len(componentsToPrepull), kubePrepuller.DeleteFunc); err != nil {
		return err
	}

	fmt.Println("[upgrade/prepull] Successfully prepulled the images for all the control plane components")
	return nil
}

// waitForItemsFromChan waits for n elements from stringChan with a timeout. For every item received from stringChan, cleanupFunc is executed
func waitForItemsFromChan(timeoutChan <-chan time.Time, stringChan chan string, n int, cleanupFunc func(string) error) error {
	i := 0
	for {
		select {
		case <-timeoutChan:
			return errors.New("the prepull operation timed out")
		case result := <-stringChan:
			i++
			// If the cleanup function errors; error here as well
			if err := cleanupFunc(result); err != nil {
				return err
			}
			if i == n {
				return nil
			}
		}
	}
}

// addPrepullPrefix adds the prepull prefix for this functionality; can be used in names, labels, etc.
func addPrepullPrefix(component string) string {
	return fmt.Sprintf("%s%s", prepullPrefix, component)
}

// buildPrePullDaemonSet builds the DaemonSet that ensures the control plane image is available
func buildPrePullDaemonSet(component, image string) *apps.DaemonSet {
	var gracePeriodSecs int64
	return &apps.DaemonSet{
		ObjectMeta: metav1.ObjectMeta{
			Name:      addPrepullPrefix(component),
			Namespace: metav1.NamespaceSystem,
		},
		Spec: apps.DaemonSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					"k8s-app": addPrepullPrefix(component),
				},
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"k8s-app": addPrepullPrefix(component),
					},
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Name:    component,
							Image:   image,
							Command: []string{"/bin/sleep", "3600"},
						},
					},
					NodeSelector: map[string]string{
						constants.LabelNodeRoleMaster: "",
					},
					Tolerations:                   []v1.Toleration{constants.ControlPlaneToleration},
					TerminationGracePeriodSeconds: &gracePeriodSecs,
					// Explicitly add a PodSecurityContext to allow these Pods to run as non-root.
					// This prevents restrictive PSPs from blocking the Pod creation.
					SecurityContext: &v1.PodSecurityContext{
						RunAsUser: utilpointer.Int64Ptr(999),
					},
				},
			},
		},
	}
}
<commit_msg>do not return error, when the ds is not found<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package upgrade

import (
	"fmt"
	"time"

	"github.com/pkg/errors"
	apps "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/images"
	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
	utilpointer "k8s.io/utils/pointer"
)

const (
	prepullPrefix = "upgrade-prepull-"
)

// Prepuller defines an interface for performing a prepull operation in a create-wait-delete fashion in parallel
type Prepuller interface {
	CreateFunc(string) error
	WaitFunc(string)
	DeleteFunc(string) error
}

// DaemonSetPrepuller makes sure the control-plane images are available on all control-planes
type DaemonSetPrepuller struct {
	client clientset.Interface
	cfg    *kubeadmapi.ClusterConfiguration
	waiter apiclient.Waiter
}

// NewDaemonSetPrepuller creates a new instance of the DaemonSetPrepuller struct
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.ClusterConfiguration) *DaemonSetPrepuller {
	return &DaemonSetPrepuller{
		client: client,
		cfg:    cfg,
		waiter: waiter,
	}
}

// CreateFunc creates a DaemonSet for making the image available on every relevant node
func (d *DaemonSetPrepuller) CreateFunc(component string) error {
	var image string
	if component == constants.Etcd {
		image = images.GetEtcdImage(d.cfg)
	} else {
		image = images.GetKubernetesImage(component, d.cfg)
	}
	ds := buildPrePullDaemonSet(component, image)

	// Create the DaemonSet in the API Server
	if err := apiclient.CreateOrUpdateDaemonSet(d.client, ds); err != nil {
		return errors.Wrapf(err, "unable to create a DaemonSet for prepulling the component %q", component)
	}
	return nil
}

// WaitFunc waits for all Pods in the specified DaemonSet to be in the Running state
func (d *DaemonSetPrepuller) WaitFunc(component string) {
	fmt.Printf("[upgrade/prepull] Prepulling image for component %s.\n", component)
	d.waiter.WaitForPodsWithLabel("k8s-app=upgrade-prepull-" + component)
}

// DeleteFunc deletes the DaemonSet used for making the image available on every relevant node
func (d *DaemonSetPrepuller) DeleteFunc(component string) error {
	dsName := addPrepullPrefix(component)
	// TODO: The IsNotFound() check is required in cases where the DaemonSet is missing.
	// Investigate why this happens: https://github.com/kubernetes/kubeadm/issues/1700
	if err := apiclient.DeleteDaemonSetForeground(d.client, metav1.NamespaceSystem, dsName); err != nil && !apierrors.IsNotFound(err) {
		return errors.Wrapf(err, "unable to cleanup the DaemonSet used for prepulling %s", component)
	}
	fmt.Printf("[upgrade/prepull] Prepulled image for component %s.\n", component)
	return nil
}

// PrepullImagesInParallel creates DaemonSets synchronously but waits in parallel for the images to pull
func PrepullImagesInParallel(kubePrepuller Prepuller, timeout time.Duration, componentsToPrepull []string) error {
	fmt.Printf("[upgrade/prepull] Will prepull images for components %v\n", componentsToPrepull)

	timeoutChan := time.After(timeout)

	// Synchronously create the DaemonSets
	for _, component := range componentsToPrepull {
		if err := kubePrepuller.CreateFunc(component); err != nil {
			return err
		}
	}

	// Create a channel for streaming data from goroutines that run in parallel to a blocking for loop that cleans up
	prePulledChan := make(chan string, len(componentsToPrepull))
	for _, component := range componentsToPrepull {
		go func(c string) {
			// Wait as long as needed. This WaitFunc call should be blocking until completion
			kubePrepuller.WaitFunc(c)
			// When the task is done, go ahead and cleanup by sending the name to the channel
			prePulledChan <- c
		}(component)
	}

	// This call blocks until all expected messages are received from the channel or errors out if timeoutChan fires.
	// For every successful wait, kubePrepuller.DeleteFunc is executed
	if err := waitForItemsFromChan(timeoutChan, prePulledChan, len(componentsToPrepull), kubePrepuller.DeleteFunc); err != nil {
		return err
	}

	fmt.Println("[upgrade/prepull] Successfully prepulled the images for all the control plane components")
	return nil
}

// waitForItemsFromChan waits for n elements from stringChan with a timeout. For every item received from stringChan, cleanupFunc is executed
func waitForItemsFromChan(timeoutChan <-chan time.Time, stringChan chan string, n int, cleanupFunc func(string) error) error {
	i := 0
	for {
		select {
		case <-timeoutChan:
			return errors.New("the prepull operation timed out")
		case result := <-stringChan:
			i++
			// If the cleanup function errors; error here as well
			if err := cleanupFunc(result); err != nil {
				return err
			}
			if i == n {
				return nil
			}
		}
	}
}

// addPrepullPrefix adds the prepull prefix for this functionality; can be used in names, labels, etc.
func addPrepullPrefix(component string) string {
	return fmt.Sprintf("%s%s", prepullPrefix, component)
}

// buildPrePullDaemonSet builds the DaemonSet that ensures the control plane image is available
func buildPrePullDaemonSet(component, image string) *apps.DaemonSet {
	var gracePeriodSecs int64
	return &apps.DaemonSet{
		ObjectMeta: metav1.ObjectMeta{
			Name:      addPrepullPrefix(component),
			Namespace: metav1.NamespaceSystem,
		},
		Spec: apps.DaemonSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					"k8s-app": addPrepullPrefix(component),
				},
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"k8s-app": addPrepullPrefix(component),
					},
				},
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Name:    component,
							Image:   image,
							Command: []string{"/bin/sleep", "3600"},
						},
					},
					NodeSelector: map[string]string{
						constants.LabelNodeRoleMaster: "",
					},
					Tolerations:                   []v1.Toleration{constants.ControlPlaneToleration},
					TerminationGracePeriodSeconds: &gracePeriodSecs,
					// Explicitly add a PodSecurityContext to allow these Pods to run as non-root.
					// This prevents restrictive PSPs from blocking the Pod creation.
					SecurityContext: &v1.PodSecurityContext{
						RunAsUser: utilpointer.Int64Ptr(999),
					},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package pq

import (
	"bytes"
	"database/sql"
	"strings"
	"testing"
)

func TestCopyInStmt(t *testing.T) {
	var stmt string
	stmt = CopyIn("table name")
	if stmt != `COPY "table name" () FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyIn("table name", "column 1", "column 2")
	if stmt != `COPY "table name" ("column 1", "column 2") FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyIn(`table " name """`, `co"lumn""`)
	if stmt != `COPY "table "" name """"""" ("co""lumn""""") FROM STDIN` {
		t.Fatal(stmt)
	}
}

func TestCopyInSchemaStmt(t *testing.T) {
	var stmt string
	stmt = CopyInSchema("schema name", "table name")
	if stmt != `COPY "schema name"."table name" () FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyInSchema("schema name", "table name", "column 1", "column 2")
	if stmt != `COPY "schema name"."table name" ("column 1", "column 2") FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyInSchema(`schema " name """`, `table " name """`, `co"lumn""`)
	if stmt != `COPY "schema "" name """"""".`+
		`"table "" name """"""" ("co""lumn""""") FROM STDIN` {
		t.Fatal(stmt)
	}
}

func TestCopyInMultipleValues(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a", "b"))
	if err != nil {
		t.Fatal(err)
	}

	longString := strings.Repeat("#", 500)

	for i := 0; i < 500; i++ {
		_, err = stmt.Exec(int64(i), longString)
		if err != nil {
			t.Fatal(err)
		}
	}

	_, err = stmt.Exec()
	if err != nil {
		t.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}

	var num int
	err = txn.QueryRow("SELECT COUNT(*) FROM temp").Scan(&num)
	if err != nil {
		t.Fatal(err)
	}

	if num != 500 {
		t.Fatalf("expected 500 items, not %d", num)
	}
}

func TestCopyInRaiseStmtTrigger(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	if getServerVersion(t, db) < 90000 {
		var exists int
		err := db.QueryRow("SELECT 1 FROM pg_language WHERE lanname = 'plpgsql'").Scan(&exists)
		if err == sql.ErrNoRows {
			t.Skip("language PL/PgSQL does not exist; skipping TestCopyInRaiseStmtTrigger")
		} else if err != nil {
			t.Fatal(err)
		}
	}

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)")
	if err != nil {
		t.Fatal(err)
	}

	_, err = txn.Exec(`
			CREATE OR REPLACE FUNCTION pg_temp.temptest()
			RETURNS trigger AS
			$BODY$ begin
				raise notice 'Hello world';
				return new;
			end $BODY$
			LANGUAGE plpgsql`)
	if err != nil {
		t.Fatal(err)
	}

	_, err = txn.Exec(`
			CREATE TRIGGER temptest_trigger
			BEFORE INSERT
			ON temp
			FOR EACH ROW
			EXECUTE PROCEDURE pg_temp.temptest()`)
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a", "b"))
	if err != nil {
		t.Fatal(err)
	}

	longString := strings.Repeat("#", 500)

	_, err = stmt.Exec(int64(1), longString)
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec()
	if err != nil {
		t.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}

	var num int
	err = txn.QueryRow("SELECT COUNT(*) FROM temp").Scan(&num)
	if err != nil {
		t.Fatal(err)
	}

	if num != 1 {
		t.Fatalf("expected 1 items, not %d", num)
	}
}

func TestCopyInTypes(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, text VARCHAR, blob BYTEA, nothing VARCHAR)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "num", "text", "blob", "nothing"))
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec(int64(1234567890), "Héllö\n ☃!\r\t\\", []byte{0, 255, 9, 10, 13}, nil)
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec()
	if err != nil {
		t.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}

	var num int
	var text string
	var blob []byte
	var nothing sql.NullString

	err = txn.QueryRow("SELECT * FROM temp").Scan(&num, &text, &blob, &nothing)
	if err != nil {
		t.Fatal(err)
	}

	if num != 1234567890 {
		t.Fatal("unexpected result", num)
	}
	if text != "Héllö\n ☃!\r\t\\" {
		t.Fatal("unexpected result", text)
	}
	if bytes.Compare(blob, []byte{0, 255, 9, 10, 13}) != 0 {
		t.Fatal("unexpected result", blob)
	}
	if nothing.Valid {
		t.Fatal("unexpected result", nothing.String)
	}
}

func TestCopyInWrongType(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "num"))
	if err != nil {
		t.Fatal(err)
	}
	defer stmt.Close()

	_, err = stmt.Exec("Héllö\n ☃!\r\t\\")
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec()
	if err == nil {
		t.Fatal("expected error")
	}
	if pge := err.(*Error); pge.Code.Name() != "invalid_text_representation" {
		t.Fatalf("expected 'invalid input syntax for integer' error, got %s (%+v)", pge.Code.Name(), pge)
	}
}

func TestCopyOutsideOfTxnError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	_, err := db.Prepare(CopyIn("temp", "num"))
	if err == nil {
		t.Fatal("COPY outside of transaction did not return an error")
	}
	if err != errCopyNotSupportedOutsideTxn {
		t.Fatalf("expected %s, got %s", err, err.Error())
	}
}

func TestCopyInBinaryError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)")
	if err != nil {
		t.Fatal(err)
	}
	_, err = txn.Prepare("COPY temp (num) FROM STDIN WITH binary")
	if err != errBinaryCopyNotSupported {
		t.Fatalf("expected %s, got %+v", errBinaryCopyNotSupported, err)
	}
	// check that the protocol is in a valid state
	err = txn.Rollback()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCopyFromError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)")
	if err != nil {
		t.Fatal(err)
	}
	_, err = txn.Prepare("COPY temp (num) TO STDOUT")
	if err != errCopyToNotSupported {
		t.Fatalf("expected %s, got %+v", errCopyToNotSupported, err)
	}
	// check that the protocol is in a valid state
	err = txn.Rollback()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCopySyntaxError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Prepare("COPY ")
	if err == nil {
		t.Fatal("expected error")
	}
	if pge := err.(*Error); pge.Code.Name() != "syntax_error" {
		t.Fatalf("expected syntax error, got %s (%+v)", pge.Code.Name(), pge)
	}
	// check that the protocol is in a valid state
	err = txn.Rollback()
	if err != nil {
		t.Fatal(err)
	}
}

// Tests for connection errors in copyin.resploop()
func TestCopyRespLoopConnectionError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	var pid int
	err = txn.QueryRow("SELECT pg_backend_pid()").Scan(&pid)
	if err != nil {
		t.Fatal(err)
	}

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a"))
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.Exec("SELECT pg_terminate_backend($1)", pid)
	if err != nil {
		t.Fatal(err)
	}

	if getServerVersion(t, db) < 90500 {
		// We have to try and send something over, since postgres before
		// version 9.5 won't process SIGTERMs while it's waiting for
		// CopyData/CopyEnd messages; see tcop/postgres.c.
		_, err = stmt.Exec(1)
		if err != nil {
			t.Fatal(err)
		}
	}
	_, err = stmt.Exec()
	if err == nil {
		t.Fatalf("expected error")
	}
	pge, ok := err.(*Error)
	if !ok {
		t.Fatalf("expected *pq.Error, got %+#v", err)
	} else if pge.Code.Name() != "admin_shutdown" {
		t.Fatalf("expected admin_shutdown, got %s", pge.Code.Name())
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}
}

func BenchmarkCopyIn(b *testing.B) {
	db := openTestConn(b)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		b.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)")
	if err != nil {
		b.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a", "b"))
	if err != nil {
		b.Fatal(err)
	}

	for i := 0; i < b.N; i++ {
		_, err = stmt.Exec(int64(i), "hello world!")
		if err != nil {
			b.Fatal(err)
		}
	}

	_, err = stmt.Exec()
	if err != nil {
		b.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		b.Fatal(err)
	}

	var num int
	err = txn.QueryRow("SELECT COUNT(*) FROM temp").Scan(&num)
	if err != nil {
		b.Fatal(err)
	}

	if num != b.N {
		b.Fatalf("expected %d items, not %d", b.N, num)
	}
}
<commit_msg>Fix intermittent test failure in TestCopyRespLoopConnectionError<commit_after>package pq

import (
	"bytes"
	"database/sql"
	"database/sql/driver"
	"strings"
	"testing"
)

func TestCopyInStmt(t *testing.T) {
	var stmt string
	stmt = CopyIn("table name")
	if stmt != `COPY "table name" () FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyIn("table name", "column 1", "column 2")
	if stmt != `COPY "table name" ("column 1", "column 2") FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyIn(`table " name """`, `co"lumn""`)
	if stmt != `COPY "table "" name """"""" ("co""lumn""""") FROM STDIN` {
		t.Fatal(stmt)
	}
}

func TestCopyInSchemaStmt(t *testing.T) {
	var stmt string
	stmt = CopyInSchema("schema name", "table name")
	if stmt != `COPY "schema name"."table name" () FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyInSchema("schema name", "table name", "column 1", "column 2")
	if stmt != `COPY "schema name"."table name" ("column 1", "column 2") FROM STDIN` {
		t.Fatal(stmt)
	}

	stmt = CopyInSchema(`schema " name """`, `table " name """`, `co"lumn""`)
	if stmt != `COPY "schema "" name """"""".`+
		`"table "" name """"""" ("co""lumn""""") FROM STDIN` {
		t.Fatal(stmt)
	}
}

func TestCopyInMultipleValues(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a", "b"))
	if err != nil {
		t.Fatal(err)
	}

	longString := strings.Repeat("#", 500)

	for i := 0; i < 500; i++ {
		_, err = stmt.Exec(int64(i), longString)
		if err != nil {
			t.Fatal(err)
		}
	}

	_, err = stmt.Exec()
	if err != nil {
		t.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}

	var num int
	err = txn.QueryRow("SELECT COUNT(*) FROM temp").Scan(&num)
	if err != nil {
		t.Fatal(err)
	}

	if num != 500 {
		t.Fatalf("expected 500 items, not %d", num)
	}
}

func TestCopyInRaiseStmtTrigger(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	if getServerVersion(t, db) < 90000 {
		var exists int
		err := db.QueryRow("SELECT 1 FROM pg_language WHERE lanname = 'plpgsql'").Scan(&exists)
		if err == sql.ErrNoRows {
			t.Skip("language PL/PgSQL does not exist; skipping TestCopyInRaiseStmtTrigger")
		} else if err != nil {
			t.Fatal(err)
		}
	}

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)")
	if err != nil {
		t.Fatal(err)
	}

	_, err = txn.Exec(`
			CREATE OR REPLACE FUNCTION pg_temp.temptest()
			RETURNS trigger AS
			$BODY$ begin
				raise notice 'Hello world';
				return new;
			end $BODY$
			LANGUAGE plpgsql`)
	if err != nil {
		t.Fatal(err)
	}

	_, err = txn.Exec(`
			CREATE TRIGGER temptest_trigger
			BEFORE INSERT
			ON temp
			FOR EACH ROW
			EXECUTE PROCEDURE pg_temp.temptest()`)
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a", "b"))
	if err != nil {
		t.Fatal(err)
	}

	longString := strings.Repeat("#", 500)

	_, err = stmt.Exec(int64(1), longString)
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec()
	if err != nil {
		t.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}

	var num int
	err = txn.QueryRow("SELECT COUNT(*) FROM temp").Scan(&num)
	if err != nil {
		t.Fatal(err)
	}

	if num != 1 {
		t.Fatalf("expected 1 items, not %d", num)
	}
}

func TestCopyInTypes(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, text VARCHAR, blob BYTEA, nothing VARCHAR)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "num", "text", "blob", "nothing"))
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec(int64(1234567890), "Héllö\n ☃!\r\t\\", []byte{0, 255, 9, 10, 13}, nil)
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec()
	if err != nil {
		t.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}

	var num int
	var text string
	var blob []byte
	var nothing sql.NullString

	err = txn.QueryRow("SELECT * FROM temp").Scan(&num, &text, &blob, &nothing)
	if err != nil {
		t.Fatal(err)
	}

	if num != 1234567890 {
		t.Fatal("unexpected result", num)
	}
	if text != "Héllö\n ☃!\r\t\\" {
		t.Fatal("unexpected result", text)
	}
	if bytes.Compare(blob, []byte{0, 255, 9, 10, 13}) != 0 {
		t.Fatal("unexpected result", blob)
	}
	if nothing.Valid {
		t.Fatal("unexpected result", nothing.String)
	}
}

func TestCopyInWrongType(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "num"))
	if err != nil {
		t.Fatal(err)
	}
	defer stmt.Close()

	_, err = stmt.Exec("Héllö\n ☃!\r\t\\")
	if err != nil {
		t.Fatal(err)
	}

	_, err = stmt.Exec()
	if err == nil {
		t.Fatal("expected error")
	}
	if pge := err.(*Error); pge.Code.Name() != "invalid_text_representation" {
		t.Fatalf("expected 'invalid input syntax for integer' error, got %s (%+v)", pge.Code.Name(), pge)
	}
}

func TestCopyOutsideOfTxnError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	_, err := db.Prepare(CopyIn("temp", "num"))
	if err == nil {
		t.Fatal("COPY outside of transaction did not return an error")
	}
	if err != errCopyNotSupportedOutsideTxn {
		t.Fatalf("expected %s, got %s", err, err.Error())
	}
}

func TestCopyInBinaryError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)")
	if err != nil {
		t.Fatal(err)
	}
	_, err = txn.Prepare("COPY temp (num) FROM STDIN WITH binary")
	if err != errBinaryCopyNotSupported {
		t.Fatalf("expected %s, got %+v", errBinaryCopyNotSupported, err)
	}
	// check that the protocol is in a valid state
	err = txn.Rollback()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCopyFromError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)")
	if err != nil {
		t.Fatal(err)
	}
	_, err = txn.Prepare("COPY temp (num) TO STDOUT")
	if err != errCopyToNotSupported {
		t.Fatalf("expected %s, got %+v", errCopyToNotSupported, err)
	}
	// check that the protocol is in a valid state
	err = txn.Rollback()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCopySyntaxError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Prepare("COPY ")
	if err == nil {
		t.Fatal("expected error")
	}
	if pge := err.(*Error); pge.Code.Name() != "syntax_error" {
		t.Fatalf("expected syntax error, got %s (%+v)", pge.Code.Name(), pge)
	}
	// check that the protocol is in a valid state
	err = txn.Rollback()
	if err != nil {
		t.Fatal(err)
	}
}

// Tests for connection errors in copyin.resploop()
func TestCopyRespLoopConnectionError(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer txn.Rollback()

	var pid int
	err = txn.QueryRow("SELECT pg_backend_pid()").Scan(&pid)
	if err != nil {
		t.Fatal(err)
	}

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int)")
	if err != nil {
		t.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a"))
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.Exec("SELECT pg_terminate_backend($1)", pid)
	if err != nil {
		t.Fatal(err)
	}

	if getServerVersion(t, db) < 90500 {
		// We have to try and send something over, since postgres before
		// version 9.5 won't process SIGTERMs while it's waiting for
		// CopyData/CopyEnd messages; see tcop/postgres.c.
		_, err = stmt.Exec(1)
		if err != nil {
			t.Fatal(err)
		}
	}
	_, err = stmt.Exec()
	if err == nil {
		t.Fatalf("expected error")
	}
	pge, ok := err.(*Error)
	if !ok {
		if err != driver.ErrBadConn {
			t.Fatalf("expected *pq.Error or driver.ErrBadConn, got %+#v", err)
		}
	} else if pge.Code.Name() != "admin_shutdown" {
		t.Fatalf("expected admin_shutdown, got %s", pge.Code.Name())
	}

	err = stmt.Close()
	if err != nil {
		t.Fatal(err)
	}
}

func BenchmarkCopyIn(b *testing.B) {
	db := openTestConn(b)
	defer db.Close()

	txn, err := db.Begin()
	if err != nil {
		b.Fatal(err)
	}
	defer txn.Rollback()

	_, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)")
	if err != nil {
		b.Fatal(err)
	}

	stmt, err := txn.Prepare(CopyIn("temp", "a", "b"))
	if err != nil {
		b.Fatal(err)
	}

	for i := 0; i < b.N; i++ {
		_, err = stmt.Exec(int64(i), "hello world!")
		if err != nil {
			b.Fatal(err)
		}
	}

	_, err = stmt.Exec()
	if err != nil {
		b.Fatal(err)
	}

	err = stmt.Close()
	if err != nil {
		b.Fatal(err)
	}

	var num int
	err = txn.QueryRow("SELECT COUNT(*) FROM temp").Scan(&num)
	if err != nil {
		b.Fatal(err)
	}

	if num != b.N {
		b.Fatalf("expected %d items, not %d", b.N, num)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"cloud.google.com/go/internal/fields"
	pb "google.golang.org/genproto/googleapis/datastore/v1"
)

var (
	typeOfByteSlice = reflect.TypeOf([]byte(nil))
	typeOfTime      = reflect.TypeOf(time.Time{})
	typeOfGeoPoint  = reflect.TypeOf(GeoPoint{})
	typeOfKeyPtr    = reflect.TypeOf(&Key{})
	typeOfEntityPtr = reflect.TypeOf(&Entity{})
)

// typeMismatchReason returns a string explaining why the property p could not
// be stored in an entity field of type v.Type().
func typeMismatchReason(p Property, v reflect.Value) string {
	entityType := "empty"
	switch p.Value.(type) {
	case int64:
		entityType = "int"
	case bool:
		entityType = "bool"
	case string:
		entityType = "string"
	case float64:
		entityType = "float"
	case *Key:
		entityType = "*datastore.Key"
	case *Entity:
		entityType = "*datastore.Entity"
	case GeoPoint:
		entityType = "GeoPoint"
	case time.Time:
		entityType = "time.Time"
	case []byte:
		entityType = "[]byte"
	}

	return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type())
}

type propertyLoader struct {
	// m holds the number of times a substruct field like "Foo.Bar.Baz" has
	// been seen so far. The map is constructed lazily.
	m map[string]int
}

func (l *propertyLoader) load(codec fields.List, structValue reflect.Value, p Property, prev map[string]struct{}) string {
	sl, ok := p.Value.([]interface{})
	if !ok {
		return l.loadOneElement(codec, structValue, p, prev)
	}
	for _, val := range sl {
		p.Value = val
		if errStr := l.loadOneElement(codec, structValue, p, prev); errStr != "" {
			return errStr
		}
	}
	return ""
}

// loadOneElement loads the value of Property p into structValue based on the provided
// codec. codec is used to find the field in structValue into which p should be loaded.
// prev is the set of property names already seen for structValue.
func (l *propertyLoader) loadOneElement(codec fields.List, structValue reflect.Value, p Property, prev map[string]struct{}) string {
	var sliceOk bool
	var sliceIndex int
	var v reflect.Value

	name := p.Name
	for name != "" {
		// First we try to find a field with name matching
		// the value of 'name' exactly (though case-insensitively).
		field := codec.Match(name)
		if field != nil {
			name = ""
		} else {
			// Now try for legacy flattened nested field (named eg. "A.B.C.D").

			parent := name
			child := ""

			// Cut off the last field (delimited by ".") and find its parent
			// in the codec.
			// eg. for name "A.B.C.D", split off "A.B.C" and try to
			// find a field in the codec with this name.
			// Loop again with "A.B", etc.
			for field == nil {
				i := strings.LastIndex(parent, ".")
				if i < 0 {
					return "no such struct field"
				}
				if i == len(name)-1 {
					return "field name cannot end with '.'"
				}
				parent, child = name[:i], name[i+1:]
				field = codec.Match(parent)
			}

			name = child
		}

		v = initField(structValue, field.Index)
		if !v.IsValid() {
			return "no such struct field"
		}
		if !v.CanSet() {
			return "cannot set struct field"
		}

		var err error
		if field.Type.Kind() == reflect.Struct {
			codec, err = structCache.Fields(field.Type)
			if err != nil {
				return err.Error()
			}
			structValue = v
		}

		// If the element is a slice, we need to accommodate it.
		if v.Kind() == reflect.Slice && v.Type() != typeOfByteSlice {
			if l.m == nil {
				l.m = make(map[string]int)
			}
			sliceIndex = l.m[p.Name]
			l.m[p.Name] = sliceIndex + 1
			for v.Len() <= sliceIndex {
				v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
			}
			structValue = v.Index(sliceIndex)
			if structValue.Type().Kind() == reflect.Struct {
				codec, err = structCache.Fields(structValue.Type())
				if err != nil {
					return err.Error()
				}
			}
			sliceOk = true
		}
	}

	var slice reflect.Value
	if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
		slice = v
		v = reflect.New(v.Type().Elem()).Elem()
	} else if _, ok := prev[p.Name]; ok && !sliceOk {
		// Zero the field back out that was set previously, turns out
		// it's a slice and we don't know what to do with it
		v.Set(reflect.Zero(v.Type()))
		return "multiple-valued property requires a slice field type"
	}

	prev[p.Name] = struct{}{}

	if errReason := setVal(v, p); errReason != "" {
		// Set the slice back to its zero value.
		if slice.IsValid() {
			slice.Set(reflect.Zero(slice.Type()))
		}
		return errReason
	}

	if slice.IsValid() {
		slice.Index(sliceIndex).Set(v)
	}

	return ""
}

// setVal sets 'v' to the value of the Property 'p'.
func setVal(v reflect.Value, p Property) string {
	pValue := p.Value

	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		x, ok := pValue.(int64)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		if v.OverflowInt(x) {
			return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
		}
		v.SetInt(x)
	case reflect.Bool:
		x, ok := pValue.(bool)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		v.SetBool(x)
	case reflect.String:
		x, ok := pValue.(string)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		v.SetString(x)
	case reflect.Float32, reflect.Float64:
		x, ok := pValue.(float64)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		if v.OverflowFloat(x) {
			return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
		}
		v.SetFloat(x)
	case reflect.Ptr:
		// v must be either a pointer to a Key or Entity.
		if v.Type() != typeOfKeyPtr && v.Type().Elem().Kind() != reflect.Struct {
			return typeMismatchReason(p, v)
		}

		if pValue == nil {
			// If v is populated already, set it to nil.
			if !v.IsNil() {
				v.Set(reflect.New(v.Type()).Elem())
			}
			return ""
		}

		switch x := pValue.(type) {
		case *Key:
			if _, ok := v.Interface().(*Key); !ok {
				return typeMismatchReason(p, v)
			}
			v.Set(reflect.ValueOf(x))
		case *Entity:
			if v.IsNil() {
				v.Set(reflect.New(v.Type().Elem()))
			}
			// Check if v implements PropertyLoadSaver.
			if pls, ok := v.Interface().(PropertyLoadSaver); ok {
				err := pls.Load(x.Properties)
				if err != nil {
					return err.Error()
				}
				return ""
			}
			if v.Type().Elem().Kind() != reflect.Struct {
				return typeMismatchReason(p, v)
			}

			return setVal(v.Elem(), p)
		default:
			return typeMismatchReason(p, v)
		}
	case reflect.Struct:
		switch v.Type() {
		case typeOfTime:
			x, ok := pValue.(time.Time)
			if !ok && pValue != nil {
				return typeMismatchReason(p, v)
			}
			v.Set(reflect.ValueOf(x))
		case typeOfGeoPoint:
			x, ok := pValue.(GeoPoint)
			if !ok && pValue != nil {
				return typeMismatchReason(p, v)
			}
			v.Set(reflect.ValueOf(x))
		default:
			ent, ok := pValue.(*Entity)
			if !ok {
				return typeMismatchReason(p, v)
			}

			// Check if v implements PropertyLoadSaver.
			if _, ok := v.Interface().(PropertyLoadSaver); ok {
				return fmt.Sprintf("datastore: PropertyLoadSaver methods must be implemented on a pointer to %T.", v.Interface())
			}

			// Recursively load nested struct.
			pls, err := newStructPLS(v.Addr().Interface())
			if err != nil {
				return err.Error()
			}

			// if ent has a Key value and our struct has a Key field,
			// load the Entity's Key value into the Key field on the struct.
			keyField := pls.codec.Match(keyFieldName)
			if keyField != nil && ent.Key != nil {
				pls.v.FieldByIndex(keyField.Index).Set(reflect.ValueOf(ent.Key))
			}

			err = pls.Load(ent.Properties)
			if err != nil {
				return err.Error()
			}
		}
	case reflect.Slice:
		x, ok := pValue.([]byte)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		if v.Type().Elem().Kind() != reflect.Uint8 {
			return typeMismatchReason(p, v)
		}
		v.SetBytes(x)
	default:
		return typeMismatchReason(p, v)
	}
	return ""
}

// initField is similar to reflect's Value.FieldByIndex, in that it
// returns the nested struct field corresponding to index, but it
// initialises any nil pointers encountered when traversing the structure.
func initField(val reflect.Value, index []int) reflect.Value {
	for _, i := range index[:len(index)-1] {
		val = val.Field(i)
		if val.Kind() == reflect.Ptr {
			if val.IsNil() {
				val.Set(reflect.New(val.Type().Elem()))
			}
			val = val.Elem()
		}
	}
	return val.Field(index[len(index)-1])
}

// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
func loadEntity(dst interface{}, src *pb.Entity) (err error) {
	ent, err := protoToEntity(src)
	if err != nil {
		return err
	}
	if e, ok := dst.(PropertyLoadSaver); ok {
		return e.Load(ent.Properties)
	}
	return loadEntityToStruct(dst, ent)
}

func loadEntityToStruct(dst interface{}, ent *Entity) error {
	pls, err := newStructPLS(dst)
	if err != nil {
		return err
	}
	// Load properties.
	err = pls.Load(ent.Properties)
	if err != nil {
		return err
	}
	// Load key.
	keyField := pls.codec.Match(keyFieldName)
	if keyField != nil && ent.Key != nil {
		pls.v.FieldByIndex(keyField.Index).Set(reflect.ValueOf(ent.Key))
	}

	return nil
}

func (s structPLS) Load(props []Property) error {
	var fieldName, errReason string
	var l propertyLoader

	prev := make(map[string]struct{})
	for _, p := range props {
		if errStr := l.load(s.codec, s.v, p, prev); errStr != "" {
			// We don't return early, as we try to load as many properties as possible.
			// It is valid to load an entity into a struct that cannot fully represent it.
			// That case returns an error, but the caller is free to ignore it.
			fieldName, errReason = p.Name, errStr
		}
	}
	if errReason != "" {
		return &ErrFieldMismatch{
			StructType: s.v.Type(),
			FieldName:  fieldName,
			Reason:     errReason,
		}
	}
	return nil
}

func protoToEntity(src *pb.Entity) (*Entity, error) {
	props := make([]Property, 0, len(src.Properties))
	for name, val := range src.Properties {
		v, err := propToValue(val)
		if err != nil {
			return nil, err
		}
		props = append(props, Property{
			Name:    name,
			Value:   v,
			NoIndex: val.ExcludeFromIndexes,
		})
	}
	var key *Key
	if src.Key != nil {
		// Ignore any error, since nested entity values
		// are allowed to have an invalid key.
		key, _ = protoToKey(src.Key)
	}

	return &Entity{key, props}, nil
}

// propToValue returns a Go value that represents the PropertyValue. For
// example, a TimestampValue becomes a time.Time.
func propToValue(v *pb.Value) (interface{}, error) {
	switch v := v.ValueType.(type) {
	case *pb.Value_NullValue:
		return nil, nil
	case *pb.Value_BooleanValue:
		return v.BooleanValue, nil
	case *pb.Value_IntegerValue:
		return v.IntegerValue, nil
	case *pb.Value_DoubleValue:
		return v.DoubleValue, nil
	case *pb.Value_TimestampValue:
		return time.Unix(v.TimestampValue.Seconds, int64(v.TimestampValue.Nanos)), nil
	case *pb.Value_KeyValue:
		return protoToKey(v.KeyValue)
	case *pb.Value_StringValue:
		return v.StringValue, nil
	case *pb.Value_BlobValue:
		return []byte(v.BlobValue), nil
	case *pb.Value_GeoPointValue:
		return GeoPoint{Lat: v.GeoPointValue.Latitude, Lng: v.GeoPointValue.Longitude}, nil
	case *pb.Value_EntityValue:
		return protoToEntity(v.EntityValue)
	case *pb.Value_ArrayValue:
		arr := make([]interface{}, 0, len(v.ArrayValue.Values))
		for _, v := range v.ArrayValue.Values {
			vv, err := propToValue(v)
			if err != nil {
				return nil, err
			}
			arr = append(arr, vv)
		}
		return arr, nil
	default:
		return nil, nil
	}
}
<commit_msg>datastore: simplifies load logic<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"
	"reflect"
	"strings"
	"time"

	"cloud.google.com/go/internal/fields"
	pb "google.golang.org/genproto/googleapis/datastore/v1"
)

var (
	typeOfByteSlice = reflect.TypeOf([]byte(nil))
	typeOfTime      = reflect.TypeOf(time.Time{})
	typeOfGeoPoint  = reflect.TypeOf(GeoPoint{})
	typeOfKeyPtr    = reflect.TypeOf(&Key{})
	typeOfEntityPtr = reflect.TypeOf(&Entity{})
)

// typeMismatchReason returns a string explaining why the property p could not
// be stored in an entity field of type v.Type().
func typeMismatchReason(p Property, v reflect.Value) string {
	entityType := "empty"
	switch p.Value.(type) {
	case int64:
		entityType = "int"
	case bool:
		entityType = "bool"
	case string:
		entityType = "string"
	case float64:
		entityType = "float"
	case *Key:
		entityType = "*datastore.Key"
	case *Entity:
		entityType = "*datastore.Entity"
	case GeoPoint:
		entityType = "GeoPoint"
	case time.Time:
		entityType = "time.Time"
	case []byte:
		entityType = "[]byte"
	}

	return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type())
}

type propertyLoader struct {
	// m holds the number of times a substruct field like "Foo.Bar.Baz" has
	// been seen so far. The map is constructed lazily.
	m map[string]int
}

func (l *propertyLoader) load(codec fields.List, structValue reflect.Value, p Property, prev map[string]struct{}) string {
	sl, ok := p.Value.([]interface{})
	if !ok {
		return l.loadOneElement(codec, structValue, p, prev)
	}
	for _, val := range sl {
		p.Value = val
		if errStr := l.loadOneElement(codec, structValue, p, prev); errStr != "" {
			return errStr
		}
	}
	return ""
}

// loadOneElement loads the value of Property p into structValue based on the provided
// codec. codec is used to find the field in structValue into which p should be loaded.
// prev is the set of property names already seen for structValue.
func (l *propertyLoader) loadOneElement(codec fields.List, structValue reflect.Value, p Property, prev map[string]struct{}) string {
	var sliceOk bool
	var sliceIndex int
	var v reflect.Value

	name := p.Name
	fieldNames := strings.Split(name, ".")

	for len(fieldNames) > 0 {
		var field *fields.Field

		// Start by trying to find a field with name. If none found,
		// cut off the last field (delimited by ".") and find its parent
		// in the codec.
		// eg. for name "A.B.C.D", split off "A.B.C" and try to
		// find a field in the codec with this name.
		// Loop again with "A.B", etc.
		for i := len(fieldNames); i > 0; i-- {
			parent := strings.Join(fieldNames[:i], ".")
			field = codec.Match(parent)
			if field != nil {
				fieldNames = fieldNames[i:]
				break
			}
		}

		// If we never found a matching field in the codec, return
		// error message.
		if field == nil {
			return "no such struct field"
		}

		v = initField(structValue, field.Index)
		if !v.IsValid() {
			return "no such struct field"
		}
		if !v.CanSet() {
			return "cannot set struct field"
		}

		var err error
		if field.Type.Kind() == reflect.Struct {
			codec, err = structCache.Fields(field.Type)
			if err != nil {
				return err.Error()
			}
			structValue = v
		}

		// If the element is a slice, we need to accommodate it.
		if v.Kind() == reflect.Slice && v.Type() != typeOfByteSlice {
			if l.m == nil {
				l.m = make(map[string]int)
			}
			sliceIndex = l.m[p.Name]
			l.m[p.Name] = sliceIndex + 1
			for v.Len() <= sliceIndex {
				v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
			}
			structValue = v.Index(sliceIndex)
			if structValue.Type().Kind() == reflect.Struct {
				codec, err = structCache.Fields(structValue.Type())
				if err != nil {
					return err.Error()
				}
			}
			sliceOk = true
		}
	}

	var slice reflect.Value
	if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
		slice = v
		v = reflect.New(v.Type().Elem()).Elem()
	} else if _, ok := prev[p.Name]; ok && !sliceOk {
		// Zero the field back out that was set previously, turns out
		// it's a slice and we don't know what to do with it
		v.Set(reflect.Zero(v.Type()))
		return "multiple-valued property requires a slice field type"
	}

	prev[p.Name] = struct{}{}

	if errReason := setVal(v, p); errReason != "" {
		// Set the slice back to its zero value.
		if slice.IsValid() {
			slice.Set(reflect.Zero(slice.Type()))
		}
		return errReason
	}

	if slice.IsValid() {
		slice.Index(sliceIndex).Set(v)
	}

	return ""
}

// setVal sets 'v' to the value of the Property 'p'.
func setVal(v reflect.Value, p Property) string {
	pValue := p.Value

	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		x, ok := pValue.(int64)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		if v.OverflowInt(x) {
			return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
		}
		v.SetInt(x)
	case reflect.Bool:
		x, ok := pValue.(bool)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		v.SetBool(x)
	case reflect.String:
		x, ok := pValue.(string)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		v.SetString(x)
	case reflect.Float32, reflect.Float64:
		x, ok := pValue.(float64)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		if v.OverflowFloat(x) {
			return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
		}
		v.SetFloat(x)
	case reflect.Ptr:
		// v must be either a pointer to a Key or Entity.
		if v.Type() != typeOfKeyPtr && v.Type().Elem().Kind() != reflect.Struct {
			return typeMismatchReason(p, v)
		}

		if pValue == nil {
			// If v is populated already, set it to nil.
			if !v.IsNil() {
				v.Set(reflect.New(v.Type()).Elem())
			}
			return ""
		}

		switch x := pValue.(type) {
		case *Key:
			if _, ok := v.Interface().(*Key); !ok {
				return typeMismatchReason(p, v)
			}
			v.Set(reflect.ValueOf(x))
		case *Entity:
			if v.IsNil() {
				v.Set(reflect.New(v.Type().Elem()))
			}
			// Check if v implements PropertyLoadSaver.
			if pls, ok := v.Interface().(PropertyLoadSaver); ok {
				err := pls.Load(x.Properties)
				if err != nil {
					return err.Error()
				}
				return ""
			}
			if v.Type().Elem().Kind() != reflect.Struct {
				return typeMismatchReason(p, v)
			}

			return setVal(v.Elem(), p)
		default:
			return typeMismatchReason(p, v)
		}
	case reflect.Struct:
		switch v.Type() {
		case typeOfTime:
			x, ok := pValue.(time.Time)
			if !ok && pValue != nil {
				return typeMismatchReason(p, v)
			}
			v.Set(reflect.ValueOf(x))
		case typeOfGeoPoint:
			x, ok := pValue.(GeoPoint)
			if !ok && pValue != nil {
				return typeMismatchReason(p, v)
			}
			v.Set(reflect.ValueOf(x))
		default:
			ent, ok := pValue.(*Entity)
			if !ok {
				return typeMismatchReason(p, v)
			}

			// Check if v implements PropertyLoadSaver.
			if _, ok := v.Interface().(PropertyLoadSaver); ok {
				return fmt.Sprintf("datastore: PropertyLoadSaver methods must be implemented on a pointer to %T.", v.Interface())
			}

			// Recursively load nested struct.
			pls, err := newStructPLS(v.Addr().Interface())
			if err != nil {
				return err.Error()
			}

			// if ent has a Key value and our struct has a Key field,
			// load the Entity's Key value into the Key field on the struct.
			keyField := pls.codec.Match(keyFieldName)
			if keyField != nil && ent.Key != nil {
				pls.v.FieldByIndex(keyField.Index).Set(reflect.ValueOf(ent.Key))
			}

			err = pls.Load(ent.Properties)
			if err != nil {
				return err.Error()
			}
		}
	case reflect.Slice:
		x, ok := pValue.([]byte)
		if !ok && pValue != nil {
			return typeMismatchReason(p, v)
		}
		if v.Type().Elem().Kind() != reflect.Uint8 {
			return typeMismatchReason(p, v)
		}
		v.SetBytes(x)
	default:
		return typeMismatchReason(p, v)
	}
	return ""
}

// initField is similar to reflect's Value.FieldByIndex, in that it
// returns the nested struct field corresponding to index, but it
// initialises any nil pointers encountered when traversing the structure.
func initField(val reflect.Value, index []int) reflect.Value {
	for _, i := range index[:len(index)-1] {
		val = val.Field(i)
		if val.Kind() == reflect.Ptr {
			if val.IsNil() {
				val.Set(reflect.New(val.Type().Elem()))
			}
			val = val.Elem()
		}
	}
	return val.Field(index[len(index)-1])
}

// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
func loadEntity(dst interface{}, src *pb.Entity) (err error) {
	ent, err := protoToEntity(src)
	if err != nil {
		return err
	}
	if e, ok := dst.(PropertyLoadSaver); ok {
		return e.Load(ent.Properties)
	}
	return loadEntityToStruct(dst, ent)
}

func loadEntityToStruct(dst interface{}, ent *Entity) error {
	pls, err := newStructPLS(dst)
	if err != nil {
		return err
	}
	// Load properties.
	err = pls.Load(ent.Properties)
	if err != nil {
		return err
	}
	// Load key.
	keyField := pls.codec.Match(keyFieldName)
	if keyField != nil && ent.Key != nil {
		pls.v.FieldByIndex(keyField.Index).Set(reflect.ValueOf(ent.Key))
	}

	return nil
}

func (s structPLS) Load(props []Property) error {
	var fieldName, errReason string
	var l propertyLoader

	prev := make(map[string]struct{})
	for _, p := range props {
		if errStr := l.load(s.codec, s.v, p, prev); errStr != "" {
			// We don't return early, as we try to load as many properties as possible.
			// It is valid to load an entity into a struct that cannot fully represent it.
			// That case returns an error, but the caller is free to ignore it.
			fieldName, errReason = p.Name, errStr
		}
	}
	if errReason != "" {
		return &ErrFieldMismatch{
			StructType: s.v.Type(),
			FieldName:  fieldName,
			Reason:     errReason,
		}
	}
	return nil
}

func protoToEntity(src *pb.Entity) (*Entity, error) {
	props := make([]Property, 0, len(src.Properties))
	for name, val := range src.Properties {
		v, err := propToValue(val)
		if err != nil {
			return nil, err
		}
		props = append(props, Property{
			Name:    name,
			Value:   v,
			NoIndex: val.ExcludeFromIndexes,
		})
	}
	var key *Key
	if src.Key != nil {
		// Ignore any error, since nested entity values
		// are allowed to have an invalid key.
		key, _ = protoToKey(src.Key)
	}

	return &Entity{key, props}, nil
}

// propToValue returns a Go value that represents the PropertyValue. For
// example, a TimestampValue becomes a time.Time.
func propToValue(v *pb.Value) (interface{}, error) {
	switch v := v.ValueType.(type) {
	case *pb.Value_NullValue:
		return nil, nil
	case *pb.Value_BooleanValue:
		return v.BooleanValue, nil
	case *pb.Value_IntegerValue:
		return v.IntegerValue, nil
	case *pb.Value_DoubleValue:
		return v.DoubleValue, nil
	case *pb.Value_TimestampValue:
		return time.Unix(v.TimestampValue.Seconds, int64(v.TimestampValue.Nanos)), nil
	case *pb.Value_KeyValue:
		return protoToKey(v.KeyValue)
	case *pb.Value_StringValue:
		return v.StringValue, nil
	case *pb.Value_BlobValue:
		return []byte(v.BlobValue), nil
	case *pb.Value_GeoPointValue:
		return GeoPoint{Lat: v.GeoPointValue.Latitude, Lng: v.GeoPointValue.Longitude}, nil
	case *pb.Value_EntityValue:
		return protoToEntity(v.EntityValue)
	case *pb.Value_ArrayValue:
		arr := make([]interface{}, 0, len(v.ArrayValue.Values))
		for _, v := range v.ArrayValue.Values {
			vv, err := propToValue(v)
			if err != nil {
				return nil, err
			}
			arr = append(arr, vv)
		}
		return arr, nil
	default:
		return nil, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"flag"
	"fmt"
	"http"
	"io"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"xml"
)

var (
	sem chan bool
	wg  *sync.WaitGroup

	throttle    *uint   = flag.Uint("c", 1, "pages to prime at once")
	localDir    *string = flag.String("l", "", "directory containing cached files (relative file names, i.e. /about/ -> <path>/about/index.html)")
	localSuffix *string = flag.String("ls", "index.html", "suffix of locally cached files")
	verbose     *bool   = flag.Bool("v", false, "show information about primed pages")
)

type Url struct {
	Loc string
	// Lastmod string
	// Changefreq string
	Priority float64
}

type Urlset struct {
	XMLName xml.Name "urlset"
	Url     []Url
}

// Functions needed by sort.Sort
func (u Urlset) Len() int {
	return len(u.Url)
}

func (u Urlset) Swap(i, j int) {
	u.Url[i], u.Url[j] = u.Url[j], u.Url[i]
}

func (u Urlset) Less(i, j int) bool {
	return u.Url[i].Priority > u.Url[j].Priority
}

func GetUrlsFromSitemap(path string) (*Urlset, os.Error) {
	var (
		urlset Urlset
		f      io.ReadCloser
		err    os.Error
	)
	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
		var res *http.Response
		res, err = http.DefaultClient.Get(path)
		if res.Status == "404 Not Found" {
			return &Urlset{}, os.NewError("Web server returned 404 Not Found")
		} else if res.Status != "200 OK" {
			return &Urlset{}, os.NewError("Web server did not serve the file")
		} else {
			f = res.Body
		}
	} else {
		f, err = os.Open(path)
	}
	defer f.Close()
	if err != nil {
		return &Urlset{}, err
	}
	if strings.HasSuffix(path, ".gz") {
		f, err = gzip.NewReader(f)
		defer f.Close()
		if err != nil {
			return &Urlset{}, os.NewError("Gzip decompression failed")
		}
	}
	err = xml.Unmarshal(f, &urlset)
	return &urlset, err
}

func PrimeUrlset(urlset *Urlset) {
	if *verbose {
		fmt.Println("URLs in sitemap: ", urlset.Len())
	}
	for _, url := range urlset.Url {
		sem <- true
		wg.Add(1)
		go PrimeUrl(url)
	}
	wg.Wait()
}

func PrimeUrl(url Url) os.Error {
	var (
		err   os.Error
		found bool = false
	)
	if *verbose {
		fmt.Printf("%s (weight %d)\n", url.Loc, int(url.Priority*100))
	}
	if *localDir != "" {
		parsed, err := http.ParseURLReference(url.Loc)
		joined := path.Join(*localDir, parsed.Path, *localSuffix)
		if _, err = os.Lstat(joined); err == nil {
			found = true
		}
	}
	if !found {
		_, err = http.DefaultClient.Get(url.Loc)
	}
	wg.Done()
	<-sem
	return err
}

func main() {
	var (
		urlset *Urlset
		err    os.Error
	)
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		fmt.Println("")
		fmt.Println("Examples:")
		fmt.Println(" ", os.Args[0], "sitemap.xml")
		fmt.Println(" ", os.Args[0], "http://mysite.com/sitemap.xml")
		fmt.Println(" ", os.Args[0], "-l /var/www/mysite.com/wp-content/cache/supercache/ http://mysite.com/sitemap.xml")
		fmt.Println(" ", os.Args[0], "-l /var/www/mysite.com/wp-content/w3tc/pgcache/ -ls _index.html http://mysite.com/sitemap.xml")
		fmt.Println("")
		fmt.Println("If specifying a sitemap URL, make sure to prepend http:// or https://")
		return
	}
	path := flag.Arg(0)
	urlset, err = GetUrlsFromSitemap(path)
	if err != nil {
		fmt.Println(err)
	} else {
		sort.Sort(urlset)
		sem = make(chan bool, *throttle)
		wg = &sync.WaitGroup{}
		PrimeUrlset(urlset)
	}
}
<commit_msg>Replacable http.Client for later<commit_after>package main

import (
	"compress/gzip"
	"flag"
	"fmt"
	"http"
	"io"
	"os"
	"path"
	"sort"
	"strings"
	"sync"
	"xml"
)

var (
	sem chan bool
	wg  *sync.WaitGroup
	client *http.Client

	throttle    *uint   = flag.Uint("c", 1, "pages to prime at once")
	localDir    *string = flag.String("l", "", "directory containing cached files (relative file names, i.e. /about/ -> <path>/about/index.html)")
	localSuffix *string = flag.String("ls", "index.html", "suffix of locally cached files")
	verbose     *bool   = flag.Bool("v", false, "show information about primed pages")
)

type Url struct {
	Loc string
	// Lastmod string
	// Changefreq string
	Priority float64
}

type Urlset struct {
	XMLName xml.Name "urlset"
	Url     []Url
}

// Functions needed by sort.Sort
func (u Urlset) Len() int {
	return len(u.Url)
}

func (u Urlset) Swap(i, j int) {
	u.Url[i], u.Url[j] = u.Url[j], u.Url[i]
}

func (u Urlset) Less(i, j int) bool {
	return u.Url[i].Priority > u.Url[j].Priority
}

func GetUrlsFromSitemap(path string) (*Urlset, os.Error) {
	var (
		urlset Urlset
		f      io.ReadCloser
		err    os.Error
	)
	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
		var res *http.Response
		res, err = client.Get(path)
		if res.Status == "404 Not Found" {
			return &Urlset{}, os.NewError("Web server returned 404 Not Found")
		} else if res.Status != "200 OK" {
			return &Urlset{}, os.NewError("Web server did not serve the file")
		} else {
			f = res.Body
		}
	} else {
		f, err = os.Open(path)
	}
	defer f.Close()
	if err != nil {
		return &Urlset{}, err
	}
	if strings.HasSuffix(path, ".gz") {
		f, err = gzip.NewReader(f)
		defer f.Close()
		if err != nil {
			return &Urlset{}, os.NewError("Gzip decompression failed")
		}
	}
	err = xml.Unmarshal(f, &urlset)
	return &urlset, err
}

func PrimeUrlset(urlset *Urlset) {
	if *verbose {
		fmt.Println("URLs in sitemap: ", urlset.Len())
	}
	for _, url := range urlset.Url {
		sem <- true
		wg.Add(1)
		go PrimeUrl(url)
	}
	wg.Wait()
}

func PrimeUrl(url Url) os.Error {
	var (
		err   os.Error
		found bool = false
	)
	if *verbose {
		fmt.Printf("%s (weight %d)\n", url.Loc, int(url.Priority*100))
	}
	if *localDir != "" {
		parsed, err := http.ParseURLReference(url.Loc)
		joined := path.Join(*localDir, parsed.Path, *localSuffix)
		if _, err = os.Lstat(joined); err == nil {
			found = true
		}
	}
	if !found {
		_, err = client.Get(url.Loc)
	}
	wg.Done()
	<-sem
	return err
}

func main() {
	var (
		urlset *Urlset
		err    os.Error
	)
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		fmt.Println("")
		fmt.Println("Examples:")
		fmt.Println(" ", os.Args[0], "sitemap.xml")
		fmt.Println(" ", os.Args[0], "http://mysite.com/sitemap.xml")
		fmt.Println(" ", os.Args[0], "-l /var/www/mysite.com/wp-content/cache/supercache/ http://mysite.com/sitemap.xml")
		fmt.Println(" ", os.Args[0], "-l /var/www/mysite.com/wp-content/w3tc/pgcache/ -ls _index.html http://mysite.com/sitemap.xml")
		fmt.Println("")
		fmt.Println("If specifying a sitemap URL, make sure to prepend http:// or https://")
		return
	}
	path := flag.Arg(0)
	urlset, err = GetUrlsFromSitemap(path)
	if err != nil {
		fmt.Println(err)
	} else {
		sort.Sort(urlset)
		client = http.DefaultClient
		sem = make(chan bool, *throttle)
		wg = &sync.WaitGroup{}
		PrimeUrlset(urlset)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	}
	return nil, nil
}
<commit_msg>+++++ getStringArgs<commit_after>package main

import (
	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	if err != nil {
		//do nothing
	}
}

func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	return nil, nil
}

func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {
	case "getTxID":
		txID := stub.GetTxID()
		result := []byte(txID)
		return result, nil
	case "getTxTimestamp":
		time, err := stub.GetTxTimestamp()
		result := []byte(time.String()) //时间转换为字符串，time.String()
		return result, err
	case "getStringArgs":
		strList := stub.GetStringArgs()
		var result string
		for index := 0; index < len(strList); index++ {
			result += "***" + strList[index] + "***"
		}
		return []byte(result), nil

	}
	return nil, nil
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"path"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"

	. "github.com/tendermint/tmlibs/common"
)

func init() {
	dbCreator := func(name string, dir string) (DB, error) {
		return NewGoLevelDB(name, dir)
	}
	registerDBCreator(LevelDBBackendStr, dbCreator, false)
	registerDBCreator(GoLevelDBBackendStr, dbCreator, false)
}

type GoLevelDB struct {
	db *leveldb.DB
}

func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
	dbPath := path.Join(dir, name+".db")
	db, err := leveldb.OpenFile(dbPath, nil)
	if err != nil {
		return nil, err
	}
	database := &GoLevelDB{db: db}
	return database, nil
}

func (db *GoLevelDB) Get(key []byte) []byte {
	res, err := db.db.Get(key, nil)
	if err != nil {
		if err == errors.ErrNotFound {
			return nil
		} else {
			PanicCrisis(err)
		}
	}
	return res
}

func (db *GoLevelDB) Set(key []byte, value []byte) {
	err := db.db.Put(key, value, nil)
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) SetSync(key []byte, value []byte) {
	err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) Delete(key []byte) {
	err := db.db.Delete(key, nil)
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) DeleteSync(key []byte) {
	err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) DB() *leveldb.DB {
	return db.db
}

func (db *GoLevelDB) Close() {
	db.db.Close()
}

func (db *GoLevelDB) Print() {
	str, _ := db.db.GetProperty("leveldb.stats")
	fmt.Printf("%v\n", str)

	iter := db.db.NewIterator(nil, nil)
	for iter.Next() {
		key := iter.Key()
		value := iter.Value()
		fmt.Printf("[%X]:\t[%X]\n", key, value)
	}
}

func (db *GoLevelDB) Stats() map[string]string {
	keys := []string{
		"leveldb.num-files-at-level{n}",
		"leveldb.stats",
		"leveldb.sstables",
		"leveldb.blockpool",
		"leveldb.cachedblock",
		"leveldb.openedtables",
		"leveldb.alivesnaps",
		"leveldb.aliveiters",
	}

	stats := make(map[string]string)
	for _, key := range keys {
		str, err := db.db.GetProperty(key)
		if err == nil {
			stats[key] = str
		}
	}
	return stats
}

type goLevelDBIterator struct {
	iterator.Iterator
}

func (it *goLevelDBIterator) Key() []byte {
	key := it.Key()
	k := make([]byte, len(key))
	copy(k, key)

	return k
}

func (it *goLevelDBIterator) Value() []byte {
	val := it.Value()
	v := make([]byte, len(val))
	copy(v, val)

	return v
}

func (db *GoLevelDB) Iterator() Iterator {
	return &goLevelDBIterator{db.db.NewIterator(nil, nil)}
}

func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator {
	return &goLevelDBIterator{db.db.NewIterator(util.BytesPrefix(prefix), nil)}
}

func (db *GoLevelDB) NewBatch() Batch {
	batch := new(leveldb.Batch)
	return &goLevelDBBatch{db, batch}
}

//--------------------------------------------------------------------------------

type goLevelDBBatch struct {
	db    *GoLevelDB
	batch *leveldb.Batch
}

func (mBatch *goLevelDBBatch) Set(key, value []byte) {
	mBatch.batch.Put(key, value)
}

func (mBatch *goLevelDBBatch) Delete(key []byte) {
	mBatch.batch.Delete(key)
}

func (mBatch *goLevelDBBatch) Write() {
	err := mBatch.db.db.Write(mBatch.batch, nil)
	if err != nil {
		PanicCrisis(err)
	}
}
<commit_msg>Add comments about copying<commit_after>package db

import (
	"fmt"
	"path"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"

	. "github.com/tendermint/tmlibs/common"
)

func init() {
	dbCreator := func(name string, dir string) (DB, error) {
		return NewGoLevelDB(name, dir)
	}
	registerDBCreator(LevelDBBackendStr, dbCreator, false)
	registerDBCreator(GoLevelDBBackendStr, dbCreator, false)
}

type GoLevelDB struct {
	db *leveldb.DB
}

func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
	dbPath := path.Join(dir, name+".db")
	db, err := leveldb.OpenFile(dbPath, nil)
	if err != nil {
		return nil, err
	}
	database := &GoLevelDB{db: db}
	return database, nil
}

func (db *GoLevelDB) Get(key []byte) []byte {
	res, err := db.db.Get(key, nil)
	if err != nil {
		if err == errors.ErrNotFound {
			return nil
		} else {
			PanicCrisis(err)
		}
	}
	return res
}

func (db *GoLevelDB) Set(key []byte, value []byte) {
	err := db.db.Put(key, value, nil)
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) SetSync(key []byte, value []byte) {
	err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) Delete(key []byte) {
	err := db.db.Delete(key, nil)
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) DeleteSync(key []byte) {
	err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
	if err != nil {
		PanicCrisis(err)
	}
}

func (db *GoLevelDB) DB() *leveldb.DB {
	return db.db
}

func (db *GoLevelDB) Close() {
	db.db.Close()
}

func (db *GoLevelDB) Print() {
	str, _ := db.db.GetProperty("leveldb.stats")
	fmt.Printf("%v\n", str)

	iter := db.db.NewIterator(nil, nil)
	for iter.Next() {
		key := iter.Key()
		value := iter.Value()
		fmt.Printf("[%X]:\t[%X]\n", key, value)
	}
}

func (db *GoLevelDB) Stats() map[string]string {
	keys := []string{
		"leveldb.num-files-at-level{n}",
		"leveldb.stats",
		"leveldb.sstables",
		"leveldb.blockpool",
		"leveldb.cachedblock",
		"leveldb.openedtables",
		"leveldb.alivesnaps",
		"leveldb.aliveiters",
	}

	stats := make(map[string]string)
	for _, key := range keys {
		str, err := db.db.GetProperty(key)
		if err == nil {
			stats[key] = str
		}
	}
	return stats
}

type goLevelDBIterator struct {
	iterator.Iterator
}

// Key returns a copy of the current key.
func (it *goLevelDBIterator) Key() []byte {
	key := it.Key()
	k := make([]byte, len(key))
	copy(k, key)

	return k
}

// Value returns a copy of the current value.
func (it *goLevelDBIterator) Value() []byte {
	val := it.Value()
	v := make([]byte, len(val))
	copy(v, val)

	return v
}

func (db *GoLevelDB) Iterator() Iterator {
	return &goLevelDBIterator{db.db.NewIterator(nil, nil)}
}

func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator {
	return &goLevelDBIterator{db.db.NewIterator(util.BytesPrefix(prefix), nil)}
}

func (db *GoLevelDB) NewBatch() Batch {
	batch := new(leveldb.Batch)
	return &goLevelDBBatch{db, batch}
}

//--------------------------------------------------------------------------------

type goLevelDBBatch struct {
	db    *GoLevelDB
	batch *leveldb.Batch
}

func (mBatch *goLevelDBBatch) Set(key, value []byte) {
	mBatch.batch.Put(key, value)
}

func (mBatch *goLevelDBBatch) Delete(key []byte) {
	mBatch.batch.Delete(key)
}

func (mBatch *goLevelDBBatch) Write() {
	err := mBatch.db.db.Write(mBatch.batch, nil)
	if err != nil {
		PanicCrisis(err)
	}
}
<|endoftext|>


<commit_before>/*
Package "dense" provides an implementation of mutable dense matrix.
*/
package dense

import (
	"io"
	"math"

	"github.com/mitsuse/matrix-go/internal/rewriters"
	"github.com/mitsuse/matrix-go/internal/types"
	"github.com/mitsuse/matrix-go/internal/validates"
	"github.com/mitsuse/serial-go"
)

const (
	id      string = "github.com/mitsuse/matrix-go/dense"
	version byte   = 0
)

type denseMatrix struct {
	base     types.Shape
	view     types.Shape
	offset   types.Index
	rows     int
	columns  int
	elements []float64
	rewriter rewriters.Rewriter
}

// Create a new matrix with given elements.
// When "rows" and "columns" is not positive,
// validates.NON_POSITIVE_SIZE_PANIC will be caused.
// In addition,
// when the product of "row"s and "column" doesn't equal to the size of "elements",
// validates.INVALID_ELEMENTS_PANIC will be caused.
func New(rows, columns int) func(elements ...float64) types.Matrix {
	validates.ShapeShouldBePositive(rows, columns)

	constructor := func(elements ...float64) types.Matrix {
		size := rows * columns

		if len(elements) != size {
			panic(validates.INVALID_ELEMENTS_PANIC)
		}

		shape := types.NewShape(rows, columns)
		offset := types.NewIndex(0, 0)

		m := &denseMatrix{
			base:     shape,
			view:     shape,
			offset:   offset,
			rows:     rows,
			columns:  columns,
			elements: make([]float64, size),
			rewriter: rewriters.Reflect(),
		}
		copy(m.elements, elements)

		return m
	}

	return constructor
}

// Create a new zero matrix.
// When "rows" and "columns" is not positive,
// validates.NON_POSITIVE_SIZE_PANIC will be caused.
func Zeros(rows, columns int) types.Matrix {
	return New(rows, columns)(make([]float64, rows*columns)...)
}

// Deserialize a matrix from the given reader.
// This accepts data generated with (*denseMatrix).Serialize.
func Deserialize(reader io.Reader) (types.Matrix, error) {
	r := serial.NewReader(id, version, reader)

	r.ReadId()
	r.ReadVersion()
	r.ReadArch()

	var baseRows int64
	r.Read(&baseRows)

	var baseColumns int64
	r.Read(&baseColumns)

	var viewRows int64
	r.Read(&viewRows)

	var viewColumns int64
	r.Read(&viewColumns)

	var offsetRow int64
	r.Read(&offsetRow)

	var offsetColumn int64
	r.Read(&offsetColumn)

	var rows int64
	r.Read(&rows)

	var columns int64
	r.Read(&columns)

	var size int64
	r.Read(&size)

	elements := make([]float64, int(size))
	for index := 0; index < len(elements); index++ {
		var element float64
		r.Read(&element)
		elements[index] = element
	}

	if err := r.Error(); err != nil {
		return nil, err
	}

	rewriter, err := rewriters.Deserialize(reader)
	if err != nil {
		return nil, err
	}

	m := &denseMatrix{
		base:     types.NewShape(int(baseRows), int(baseColumns)),
		view:     types.NewShape(int(viewRows), int(viewColumns)),
		offset:   types.NewIndex(int(offsetRow), int(offsetColumn)),
		rows:     int(rows),
		columns:  int(columns),
		elements: elements,
		rewriter: rewriter,
	}

	if len(elements) != m.rows*m.columns {
		panic(validates.INVALID_ELEMENTS_PANIC)
	}

	return m, nil
}

func (m *denseMatrix) Serialize(writer io.Writer) error {
	w := serial.NewWriter(id, version, writer)

	w.WriteId()
	w.WriteVersion()
	w.WriteArch()

	w.Write(int64(m.base.Rows()))
	w.Write(int64(m.base.Columns()))
	w.Write(int64(m.view.Rows()))
	w.Write(int64(m.view.Columns()))
	w.Write(int64(m.offset.Row()))
	w.Write(int64(m.offset.Column()))

	w.Write(int64(m.rows))
	w.Write(int64(m.columns))

	w.Write(int64(len(m.elements)))
	for _, element := range m.elements {
		w.Write(int64(element))
	}

	if err := w.Error(); err != nil {
		return err
	}

	if err := m.rewriter.Serialize(writer); err != nil {
		return err
	}

	return nil
}

func (m *denseMatrix) Shape() (rows, columns int) {
	return m.rewriter.Rewrite(m.view.Rows(), m.view.Columns())
}

func (m *denseMatrix) Rows() (rows int) {
	rows, _ = m.Shape()
	return rows
}

func (m *denseMatrix) Columns() (columns int) {
	_, columns = m.Shape()
	return columns
}

func (m *denseMatrix) All() types.Cursor {
	return newAllCursor(m)
}

func (m *denseMatrix) NonZeros() types.Cursor {
	return newNonZerosCursor(m)
}

func (m *denseMatrix) Diagonal() types.Cursor {
	return newDiagonalCursor(m)
}

func (m *denseMatrix) Get(row, column int) (element float64) {
	row, column = m.rewriter.Rewrite(row, column)

	validates.IndexShouldBeInRange(m.view.Rows(), m.view.Columns(), row, column)

	index := (row+m.offset.Row())*m.base.Columns() + column + m.offset.Column()

	return m.elements[index]
}

func (m *denseMatrix) Update(row, column int, element float64) types.Matrix {
	row, column = m.rewriter.Rewrite(row, column)

	validates.IndexShouldBeInRange(m.view.Rows(), m.view.Columns(), row, column)

	index := (row+m.offset.Row())*m.base.Columns() + column + m.offset.Column()
	m.elements[index] = element

	return m
}

func (m *denseMatrix) Equal(n types.Matrix) bool {
	validates.ShapeShouldBeSame(m, n)

	cursor := n.All()

	for cursor.HasNext() {
		element, row, column := cursor.Get()
		if m.Get(row, column) != element {
			return false
		}
	}

	return true
}

func (m *denseMatrix) Add(n types.Matrix) types.Matrix {
	validates.ShapeShouldBeSame(m, n)

	cursor := n.NonZeros()

	for cursor.HasNext() {
		element, row, column := cursor.Get()
		m.Update(row, column, m.Get(row, column)+element)
	}

	return m
}

func (m *denseMatrix) Subtract(n types.Matrix) types.Matrix {
	validates.ShapeShouldBeSame(m, n)

	cursor := n.NonZeros()

	for cursor.HasNext() {
		element, row, column := cursor.Get()
		m.Update(row, column, m.Get(row, column)-element)
	}

	return m
}

func (m *denseMatrix) Multiply(n types.Matrix) types.Matrix {
	validates.ShapeShouldBeMultipliable(m, n)

	rows := m.Rows()
	columns := n.Columns()

	r := Zeros(rows, columns)

	cursor := n.NonZeros()

	for cursor.HasNext() {
		element, j, k := cursor.Get()

		for i := 0; i < rows; i++ {
			r.Update(i, k, r.Get(i, k)+m.Get(i, j)*element)
		}
	}

	return r
}

func (m *denseMatrix) Scalar(s float64) types.Matrix {
	for index, element := range m.elements {
		m.elements[index] = element * s
	}

	return m
}

func (m *denseMatrix) Transpose() types.Matrix {
	n := &denseMatrix{
		base:     m.base,
		view:     m.view,
		offset:   m.offset,
		rows:     m.rows,
		columns:  m.columns,
		elements: m.elements,
		rewriter: m.rewriter.Transpose(),
	}

	return n
}

func (m *denseMatrix) Max() (element float64, row, column int) {
	max := math.Inf(-1)
	index := 0

	for i, c := range m.elements {
		if max >= c {
			continue
		}

		max = c
		index = i
	}

	row, column = m.convertToRowColumn(index)

	return max, row, column
}

func (m *denseMatrix) Min() (element float64, row, column int) {
	min := math.Inf(0)
	index := 0

	for i, c := range m.elements {
		if min <= c {
			continue
		}

		min = c
		index = i
	}

	row, column = m.convertToRowColumn(index)

	return min, row, column
}

func (m *denseMatrix) convertToRowColumn(index int) (row, column int) {
	columns := m.Columns()

	row = index / columns
	column = index - (row * columns)

	return row, column
}
<commit_msg>"rows" and "columns" become needless.<commit_after>/*
Package "dense" provides an implementation of mutable dense matrix.
*/
package dense

import (
	"io"
	"math"

	"github.com/mitsuse/matrix-go/internal/rewriters"
	"github.com/mitsuse/matrix-go/internal/types"
	"github.com/mitsuse/matrix-go/internal/validates"
	"github.com/mitsuse/serial-go"
)

const (
	id      string = "github.com/mitsuse/matrix-go/dense"
	version byte   = 0
)

type denseMatrix struct {
	base     types.Shape
	view     types.Shape
	offset   types.Index
	elements []float64
	rewriter rewriters.Rewriter
}

// Create a new matrix with given elements.
// When "rows" and "columns" is not positive,
// validates.NON_POSITIVE_SIZE_PANIC will be caused.
// In addition,
// when the product of "row"s and "column" doesn't equal to the size of "elements",
// validates.INVALID_ELEMENTS_PANIC will be caused.
func New(rows, columns int) func(elements ...float64) types.Matrix {
	validates.ShapeShouldBePositive(rows, columns)

	constructor := func(elements ...float64) types.Matrix {
		size := rows * columns

		if len(elements) != size {
			panic(validates.INVALID_ELEMENTS_PANIC)
		}

		shape := types.NewShape(rows, columns)
		offset := types.NewIndex(0, 0)

		m := &denseMatrix{
			base:     shape,
			view:     shape,
			offset:   offset,
			elements: make([]float64, size),
			rewriter: rewriters.Reflect(),
		}
		copy(m.elements, elements)

		return m
	}

	return constructor
}

// Create a new zero matrix.
// When "rows" and "columns" is not positive,
// validates.NON_POSITIVE_SIZE_PANIC will be caused.
func Zeros(rows, columns int) types.Matrix {
	return New(rows, columns)(make([]float64, rows*columns)...)
}

// Deserialize a matrix from the given reader.
// This accepts data generated with (*denseMatrix).Serialize.
func Deserialize(reader io.Reader) (types.Matrix, error) {
	r := serial.NewReader(id, version, reader)

	r.ReadId()
	r.ReadVersion()
	r.ReadArch()

	var baseRows int64
	r.Read(&baseRows)

	var baseColumns int64
	r.Read(&baseColumns)

	var viewRows int64
	r.Read(&viewRows)

	var viewColumns int64
	r.Read(&viewColumns)

	var offsetRow int64
	r.Read(&offsetRow)

	var offsetColumn int64
	r.Read(&offsetColumn)

	var size int64
	r.Read(&size)

	elements := make([]float64, int(size))
	for index := 0; index < len(elements); index++ {
		var element float64
		r.Read(&element)
		elements[index] = element
	}

	if err := r.Error(); err != nil {
		return nil, err
	}

	rewriter, err := rewriters.Deserialize(reader)
	if err != nil {
		return nil, err
	}

	m := &denseMatrix{
		base:     types.NewShape(int(baseRows), int(baseColumns)),
		view:     types.NewShape(int(viewRows), int(viewColumns)),
		offset:   types.NewIndex(int(offsetRow), int(offsetColumn)),
		elements: elements,
		rewriter: rewriter,
	}

	if len(elements) != m.base.Rows()*m.base.Columns() {
		panic(validates.INVALID_ELEMENTS_PANIC)
	}

	return m, nil
}

func (m *denseMatrix) Serialize(writer io.Writer) error {
	w := serial.NewWriter(id, version, writer)

	w.WriteId()
	w.WriteVersion()
	w.WriteArch()

	w.Write(int64(m.base.Rows()))
	w.Write(int64(m.base.Columns()))
	w.Write(int64(m.view.Rows()))
	w.Write(int64(m.view.Columns()))
	w.Write(int64(m.offset.Row()))
	w.Write(int64(m.offset.Column()))

	w.Write(int64(len(m.elements)))
	for _, element := range m.elements {
		w.Write(int64(element))
	}

	if err := w.Error(); err != nil {
		return err
	}

	if err := m.rewriter.Serialize(writer); err != nil {
		return err
	}

	return nil
}

func (m *denseMatrix) Shape() (rows, columns int) {
	return m.rewriter.Rewrite(m.view.Rows(), m.view.Columns())
}

func (m *denseMatrix) Rows() (rows int) {
	rows, _ = m.Shape()
	return rows
}

func (m *denseMatrix) Columns() (columns int) {
	_, columns = m.Shape()
	return columns
}

func (m *denseMatrix) All() types.Cursor {
	return newAllCursor(m)
}

func (m *denseMatrix) NonZeros() types.Cursor {
	return newNonZerosCursor(m)
}

func (m *denseMatrix) Diagonal() types.Cursor {
	return newDiagonalCursor(m)
}

func (m *denseMatrix) Get(row, column int) (element float64) {
	row, column = m.rewriter.Rewrite(row, column)

	validates.IndexShouldBeInRange(m.view.Rows(), m.view.Columns(), row, column)

	index := (row+m.offset.Row())*m.base.Columns() + column + m.offset.Column()

	return m.elements[index]
}

func (m *denseMatrix) Update(row, column int, element float64) types.Matrix {
	row, column = m.rewriter.Rewrite(row, column)

	validates.IndexShouldBeInRange(m.view.Rows(), m.view.Columns(), row, column)

	index := (row+m.offset.Row())*m.base.Columns() + column + m.offset.Column()
	m.elements[index] = element

	return m
}

func (m *denseMatrix) Equal(n types.Matrix) bool {
	validates.ShapeShouldBeSame(m, n)

	cursor := n.All()

	for cursor.HasNext() {
		element, row, column := cursor.Get()
		if m.Get(row, column) != element {
			return false
		}
	}

	return true
}

func (m *denseMatrix) Add(n types.Matrix) types.Matrix {
	validates.ShapeShouldBeSame(m, n)

	cursor := n.NonZeros()

	for cursor.HasNext() {
		element, row, column := cursor.Get()
		m.Update(row, column, m.Get(row, column)+element)
	}

	return m
}

func (m *denseMatrix) Subtract(n types.Matrix) types.Matrix {
	validates.ShapeShouldBeSame(m, n)

	cursor := n.NonZeros()

	for cursor.HasNext() {
		element, row, column := cursor.Get()
		m.Update(row, column, m.Get(row, column)-element)
	}

	return m
}

func (m *denseMatrix) Multiply(n types.Matrix) types.Matrix {
	validates.ShapeShouldBeMultipliable(m, n)

	rows := m.Rows()
	columns := n.Columns()

	r := Zeros(rows, columns)

	cursor := n.NonZeros()

	for cursor.HasNext() {
		element, j, k := cursor.Get()

		for i := 0; i < rows; i++ {
			r.Update(i, k, r.Get(i, k)+m.Get(i, j)*element)
		}
	}

	return r
}

func (m *denseMatrix) Scalar(s float64) types.Matrix {
	for index, element := range m.elements {
		m.elements[index] = element * s
	}

	return m
}

func (m *denseMatrix) Transpose() types.Matrix {
	n := &denseMatrix{
		base:     m.base,
		view:     m.view,
		offset:   m.offset,
		elements: m.elements,
		rewriter: m.rewriter.Transpose(),
	}

	return n
}

func (m *denseMatrix) Max() (element float64, row, column int) {
	max := math.Inf(-1)
	index := 0

	for i, c := range m.elements {
		if max >= c {
			continue
		}

		max = c
		index = i
	}

	row, column = m.convertToRowColumn(index)

	return max, row, column
}

func (m *denseMatrix) Min() (element float64, row, column int) {
	min := math.Inf(0)
	index := 0

	for i, c := range m.elements {
		if min <= c {
			continue
		}

		min = c
		index = i
	}

	row, column = m.convertToRowColumn(index)

	return min, row, column
}

func (m *denseMatrix) convertToRowColumn(index int) (row, column int) {
	columns := m.Columns()

	row = index / columns
	column = index - (row * columns)

	return row, column
}
<|endoftext|>


<commit_before>package mater

import (
	"os"
	"reflect"
	"fmt"
)

type Component interface {
	//used to identify the component
	Name () string
	//Only called when creating a new component at runtime
	//Unmarshalled Components have to call it themselves
	Init (owner *Entity)
	//Called once per frame if owner.Enabled is true
	Update (owner *Entity, dt float64)
	//Called when removed from an Entity or before the entity is destroyed
	Destroy (owner *Entity)

	//
	MarshalJSON(owner *Entity) ([]byte, os.Error)

	UnmarshalJSON(owner *Entity, data []byte) (os.Error)
}

func (entity *Entity) AddComponent(component Component) {
	//components have to add/remove themselves from entity.Components
	component.Init(entity)
}

func (entity *Entity) RemoveComponent(component Component) {
	//components have to add/remove themselves from entity.Components
	component.Destroy(entity)
}

func (entity *Entity) RemoveComponentName(name string) {
	if component, ok := entity.Components[name]; ok {
		component.Destroy(entity)
	}
}

var components = make(map[string]reflect.Type)

func RegisterComponent(component Component) {
	components[component.Name()] = reflect.Indirect(reflect.ValueOf(component)).Type()
}

func NewComponent(name string) Component {
	compType, ok := components[name]
	if ok == false {
		return nil
	}

	component := reflect.New(compType)
	return component.Interface().(Component)
}
<commit_msg>removed unused import<commit_after>package mater

import (
	"os"
	"reflect"
)

type Component interface {
	//used to identify the component
	Name () string
	//Only called when creating a new component at runtime
	//Unmarshalled Components have to call it themselves
	Init (owner *Entity)
	//Called once per frame if owner.Enabled is true
	Update (owner *Entity, dt float64)
	//Called when removed from an Entity or before the entity is destroyed
	Destroy (owner *Entity)

	//
	MarshalJSON(owner *Entity) ([]byte, os.Error)

	UnmarshalJSON(owner *Entity, data []byte) (os.Error)
}

func (entity *Entity) AddComponent(component Component) {
	//components have to add/remove themselves from entity.Components
	component.Init(entity)
}

func (entity *Entity) RemoveComponent(component Component) {
	//components have to add/remove themselves from entity.Components
	component.Destroy(entity)
}

func (entity *Entity) RemoveComponentName(name string) {
	if component, ok := entity.Components[name]; ok {
		component.Destroy(entity)
	}
}

var components = make(map[string]reflect.Type)

func RegisterComponent(component Component) {
	components[component.Name()] = reflect.Indirect(reflect.ValueOf(component)).Type()
}

func NewComponent(name string) Component {
	compType, ok := components[name]
	if ok == false {
		return nil
	}

	component := reflect.New(compType)
	return component.Interface().(Component)
}
<|endoftext|>


<commit_before>package conf

import (
	"flag"
	"os"
)

// Command line options
var (
	PORT      = flag.Int("port", 8000, "port to listen on")
	DATAROOT  = flag.String("data", "", "data directory to store on disk files")
	MONGODB   = flag.String("mongodb", "localhost", "hostname(s) of mongodb")
	SECRETKEY = flag.String("secretkey", "supersupersecret", "secret key")
	SITEPATH  = os.ExpandEnv("$GOPATH") + "/src/github.com/MG-RAST/Shock/site"
)

func init() {
	flag.Parse()
}
<commit_msg>added conf var for email for resources req<commit_after>package conf

import (
	"flag"
	"os"
)

// Command line options
var (
	PORT       = flag.Int("port", 8000, "port to listen on")
	DATAROOT   = flag.String("data", "", "data directory to store on disk files")
	MONGODB    = flag.String("mongodb", "localhost", "hostname(s) of mongodb")
	SECRETKEY  = flag.String("secretkey", "supersupersecret", "secret key")
	SITEPATH   = os.ExpandEnv("$GOPATH") + "/src/github.com/MG-RAST/Shock/site"
	ADMINEMAIL = flag.String("email", "admin@host.com", "server admin email")
)

func init() {
	flag.Parse()
}
<|endoftext|>