<commit_before>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package core provides transport-agnostic implementation of Migrillian tool.
package core

import (
	"context"
	"fmt"
	"math/rand"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/trillian/migrillian/configpb"

	"github.com/google/trillian/merkle"
	_ "github.com/google/trillian/merkle/rfc6962" // Register hasher.
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"github.com/google/trillian/util/clock"
	"github.com/google/trillian/util/election2"
)

var (
	metrics     treeMetrics
	metricsOnce sync.Once
)

// treeMetrics holds metrics keyed by Tree ID.
type treeMetrics struct {
	masterRuns       monitoring.Counter
	masterCancels    monitoring.Counter
	controllerStarts monitoring.Counter
	isMaster         monitoring.Gauge
	entriesFetched   monitoring.Counter
	entriesSeen      monitoring.Counter
	entriesStored    monitoring.Counter
	// TODO(pavelkalinnikov): Add latency histograms, latest STH, tree size, etc.
}

// initMetrics creates metrics using the factory, if not yet created.
func initMetrics(mf monitoring.MetricFactory) {
	const treeID = "tree_id"
	metricsOnce.Do(func() {
		metrics = treeMetrics{
			masterRuns:       mf.NewCounter("master_runs", "Number of mastership runs.", treeID),
			masterCancels:    mf.NewCounter("master_cancels", "Number of unexpected mastership cancelations.", treeID),
			controllerStarts: mf.NewCounter("controller_starts", "Number of Controller (re-)starts.", treeID),
			isMaster:         mf.NewGauge("is_master", "The instance is currently the master.", treeID),
			entriesFetched:   mf.NewCounter("entries_fetched", "Entries fetched from the source log.", treeID),
			entriesSeen:      mf.NewCounter("entries_seen", "Entries seen by the submitters.", treeID),
			entriesStored:    mf.NewCounter("entries_stored", "Entries successfully submitted to Trillian.", treeID),
		}
	})
}

// Options holds configuration for a Controller.
type Options struct {
	scanner.FetcherOptions
	Submitters         int
	ChannelSize        int
	NoConsistencyCheck bool
	StartDelay         time.Duration
	StopAfter          time.Duration
}

// OptionsFromConfig returns Options created from the passed in config.
func OptionsFromConfig(cfg *configpb.MigrationConfig) Options {
	opts := Options{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     int(cfg.BatchSize),
			ParallelFetch: int(cfg.NumFetchers),
			StartIndex:    cfg.StartIndex,
			EndIndex:      cfg.EndIndex,
			Continuous:    cfg.IsContinuous,
		},
		Submitters:         int(cfg.NumSubmitters),
		ChannelSize:        int(cfg.ChannelSize),
		NoConsistencyCheck: cfg.NoConsistencyCheck,
	}
	if cfg.NumFetchers == 0 {
		opts.ParallelFetch = 1
	}
	if cfg.NumSubmitters == 0 {
		opts.Submitters = 1
	}
	return opts
}

// Controller coordinates migration from a CT log to a Trillian tree.
//
// TODO(pavelkalinnikov):
// - Schedule a distributed fetch to increase throughput.
// - Store CT STHs in Trillian or make this tool stateful on its own.
// - Make fetching stateful to reduce master resigning aftermath.
type Controller struct {
	opts     Options
	batches  chan scanner.EntryBatch
	ctClient *client.LogClient
	plClient *PreorderedLogClient
	ef       election2.Factory
	label    string
}

// NewController creates a Controller configured by the passed in options, CT
// and Trillian clients, and a master election factory.
//
// The passed in MetricFactory is used to create per-tree metrics, and it
// should be the same for all instances. However, it is used only once.
func NewController(
	opts Options,
	ctClient *client.LogClient,
	plClient *PreorderedLogClient,
	ef election2.Factory,
	mf monitoring.MetricFactory,
) *Controller {
	initMetrics(mf)
	l := strconv.FormatInt(plClient.tree.TreeId, 10)
	return &Controller{opts: opts, ctClient: ctClient, plClient: plClient, ef: ef, label: l}
}

// RunWhenMasterWithRestarts calls RunWhenMaster, and, if the migration is
// configured with continuous mode, restarts it whenever it returns.
func (c *Controller) RunWhenMasterWithRestarts(ctx context.Context) {
	uri := c.ctClient.BaseURI()
	treeID := c.plClient.tree.TreeId
	for run := true; run; run = c.opts.Continuous && ctx.Err() == nil {
		glog.Infof("Starting migration Controller (%d<-%q)", treeID, uri)
		if err := c.RunWhenMaster(ctx); err != nil {
			glog.Errorf("Controller.RunWhenMaster(%d<-%q): %v", treeID, uri, err)
			continue
		}
		glog.Infof("Controller stopped (%d<-%q)", treeID, uri)
	}
}

// RunWhenMaster is a master-elected version of Run method. It executes Run
// whenever this instance captures mastership of the tree ID. As soon as the
// instance stops being the master, Run is canceled. The method returns if a
// severe error occurs, the passed in context is canceled, or fetching is
// completed (in non-Continuous mode). Releases mastership when terminates.
func (c *Controller) RunWhenMaster(ctx context.Context) error {
	// Avoid thundering herd when starting multiple tasks on the same tree.
	if err := sleepRandom(ctx, 0, c.opts.StartDelay); err != nil {
		return err // The context has been canceled.
	}

	treeID := strconv.FormatInt(c.plClient.tree.TreeId, 10)
	metrics.controllerStarts.Inc(treeID)

	el, err := c.ef.NewElection(ctx, treeID)
	if err != nil {
		return err
	}
	defer func(ctx context.Context) {
		metrics.isMaster.Set(0, c.label)
		if err := el.Close(ctx); err != nil {
			glog.Warningf("%s: Election.Close(): %v", treeID, err)
		}
	}(ctx)

	for {
		if err := el.Await(ctx); err != nil {
			return err
		}
		metrics.isMaster.Set(1, c.label)

		mctx, err := el.WithMastership(ctx)
		if err != nil {
			return err
		} else if err := mctx.Err(); err != nil {
			return err
		}

		glog.Infof("%s: running as master", treeID)
		metrics.masterRuns.Inc(c.label)

		// Run while still master (or until an error).
		err = c.Run(mctx)
		if ctx.Err() != nil {
			// We have been externally canceled, so return the current error (which
			// could be nil or a cancelation-related error).
			return err
		} else if mctx.Err() == nil {
			// We are still the master, so try to resign and emit the real error.
			if rerr := el.Resign(ctx); rerr != nil {
				glog.Errorf("%s: Election.Resign(): %v", treeID, rerr)
			}
			return err
		}

		// Otherwise the mastership has been canceled, retry.
		metrics.isMaster.Set(0, c.label)
		metrics.masterCancels.Inc(c.label)
	}
}

// Run transfers CT log entries obtained via the CT log client to a Trillian
// pre-ordered log via Trillian client. If Options.Continuous is true then the
// migration process runs continuously trying to keep up with the target CT
// log. Returns if an error occurs, the context is canceled, or all the entries
// have been transferred (in non-Continuous mode).
func (c *Controller) Run(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId

	root, err := c.plClient.getVerifiedRoot(ctx)
	if err != nil {
		return err
	}
	if c.opts.Continuous { // Ignore range parameters in Continuous mode.
		// TODO(pavelkalinnikov): Restore fetching state from storage in a better
		// way than "take the current tree size".
		c.opts.StartIndex, c.opts.EndIndex = int64(root.TreeSize), 0
		glog.Warningf("%d: updated entry range to [%d, INF)", treeID, c.opts.StartIndex)
	} else if c.opts.StartIndex < 0 {
		c.opts.StartIndex = int64(root.TreeSize)
		glog.Warningf("%d: updated start index to %d", treeID, c.opts.StartIndex)
	}

	fetcher := scanner.NewFetcher(c.ctClient, &c.opts.FetcherOptions)
	sth, err := fetcher.Prepare(ctx)
	if err != nil {
		return err
	}
	if err := c.verifyConsistency(ctx, root, sth); err != nil {
		return err
	}

	var wg sync.WaitGroup
	c.batches = make(chan scanner.EntryBatch, c.opts.ChannelSize)
	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// TODO(pavelkalinnikov): Share the submitters pool between multiple trees.
	for w, cnt := 0, c.opts.Submitters; w < cnt; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := c.runSubmitter(cctx); err != nil {
				glog.Errorf("%d: Stopping due to submitter error: %v", treeID, err)
				cancel() // Stop the other submitters and the Fetcher.
			}
		}()
	}

	if c.opts.StopAfter != 0 { // Configured with max running time.
		go func() {
			// Sleep for random duration in [StopAfter, 2*StopAfter).
			if err := sleepRandom(cctx, c.opts.StopAfter, c.opts.StopAfter); err == nil {
				fetcher.Stop() // Trigger graceful stop if not yet canceled.
			}
		}()
	}

	handler := func(b scanner.EntryBatch) {
		metrics.entriesFetched.Add(float64(len(b.Entries)), c.label)
		select {
		case c.batches <- b:
		case <-cctx.Done(): // Avoid deadlock when shutting down.
		}
	}

	result := fetcher.Run(cctx, handler)
	close(c.batches)
	wg.Wait()
	return result
}

// verifyConsistency checks that the provided verified Trillian root is
// consistent with the CT log's STH.
func (c *Controller) verifyConsistency(ctx context.Context, root *types.LogRootV1, sth *ct.SignedTreeHead) error {
	if c.opts.NoConsistencyCheck {
		glog.Warningf("%s: skipping consistency check", c.label)
		return nil
	}
	proof, err := c.ctClient.GetSTHConsistency(ctx, root.TreeSize, sth.TreeSize)
	if err != nil {
		return err
	}
	return merkle.NewLogVerifier(c.plClient.verif.Hasher).VerifyConsistencyProof(
		int64(root.TreeSize), int64(sth.TreeSize),
		root.RootHash, sth.SHA256RootHash[:], proof)
}

// runSubmitter obtains CT log entry batches from the controller's channel and
// submits them through Trillian client. Returns when the channel is closed, or
// the client returns a non-recoverable error (an example of a recoverable
// error is when Trillian write quota is exceeded).
func (c *Controller) runSubmitter(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId
	for b := range c.batches {
		entries := float64(len(b.Entries))
		metrics.entriesSeen.Add(entries, c.label)

		end := b.Start + int64(len(b.Entries))
		if err := c.plClient.addSequencedLeaves(ctx, &b); err != nil {
			// addSequencedLeaves failed to submit entries despite retries. At this
			// point there is not much we can do. Seemingly the best strategy is to
			// shut down the Controller.
			return fmt.Errorf("failed to add batch [%d, %d): %v", b.Start, end, err)
		}
		glog.Infof("%d: added batch [%d, %d)", treeID, b.Start, end)
		metrics.entriesStored.Add(entries, c.label)
	}
	return nil
}

// sleepRandom sleeps for random duration in [base, base+spread).
func sleepRandom(ctx context.Context, base, spread time.Duration) error {
	d := base
	if spread != 0 {
		d += time.Duration(rand.Int63n(int64(spread)))
	}
	if d == 0 {
		return nil
	}
	return clock.SleepContext(ctx, d)
}
<commit_msg>Migrillian: Remove obsolete TODOs (#486)<commit_after>// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package core provides transport-agnostic implementation of Migrillian tool.
package core

import (
	"context"
	"fmt"
	"math/rand"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/trillian/migrillian/configpb"

	"github.com/google/trillian/merkle"
	_ "github.com/google/trillian/merkle/rfc6962" // Register hasher.
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"github.com/google/trillian/util/clock"
	"github.com/google/trillian/util/election2"
)

var (
	metrics     treeMetrics
	metricsOnce sync.Once
)

// treeMetrics holds metrics keyed by Tree ID.
type treeMetrics struct {
	masterRuns       monitoring.Counter
	masterCancels    monitoring.Counter
	controllerStarts monitoring.Counter
	isMaster         monitoring.Gauge
	entriesFetched   monitoring.Counter
	entriesSeen      monitoring.Counter
	entriesStored    monitoring.Counter
	// TODO(pavelkalinnikov): Add latency histograms, latest STH, tree size, etc.
}

// initMetrics creates metrics using the factory, if not yet created.
func initMetrics(mf monitoring.MetricFactory) {
	const treeID = "tree_id"
	metricsOnce.Do(func() {
		metrics = treeMetrics{
			masterRuns:       mf.NewCounter("master_runs", "Number of mastership runs.", treeID),
			masterCancels:    mf.NewCounter("master_cancels", "Number of unexpected mastership cancelations.", treeID),
			controllerStarts: mf.NewCounter("controller_starts", "Number of Controller (re-)starts.", treeID),
			isMaster:         mf.NewGauge("is_master", "The instance is currently the master.", treeID),
			entriesFetched:   mf.NewCounter("entries_fetched", "Entries fetched from the source log.", treeID),
			entriesSeen:      mf.NewCounter("entries_seen", "Entries seen by the submitters.", treeID),
			entriesStored:    mf.NewCounter("entries_stored", "Entries successfully submitted to Trillian.", treeID),
		}
	})
}

// Options holds configuration for a Controller.
type Options struct {
	scanner.FetcherOptions
	Submitters         int
	ChannelSize        int
	NoConsistencyCheck bool
	StartDelay         time.Duration
	StopAfter          time.Duration
}

// OptionsFromConfig returns Options created from the passed in config.
func OptionsFromConfig(cfg *configpb.MigrationConfig) Options {
	opts := Options{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     int(cfg.BatchSize),
			ParallelFetch: int(cfg.NumFetchers),
			StartIndex:    cfg.StartIndex,
			EndIndex:      cfg.EndIndex,
			Continuous:    cfg.IsContinuous,
		},
		Submitters:         int(cfg.NumSubmitters),
		ChannelSize:        int(cfg.ChannelSize),
		NoConsistencyCheck: cfg.NoConsistencyCheck,
	}
	if cfg.NumFetchers == 0 {
		opts.ParallelFetch = 1
	}
	if cfg.NumSubmitters == 0 {
		opts.Submitters = 1
	}
	return opts
}

// Controller coordinates migration from a CT log to a Trillian tree.
type Controller struct {
	opts     Options
	batches  chan scanner.EntryBatch
	ctClient *client.LogClient
	plClient *PreorderedLogClient
	ef       election2.Factory
	label    string
}

// NewController creates a Controller configured by the passed in options, CT
// and Trillian clients, and a master election factory.
//
// The passed in MetricFactory is used to create per-tree metrics, and it
// should be the same for all instances. However, it is used only once.
func NewController(
	opts Options,
	ctClient *client.LogClient,
	plClient *PreorderedLogClient,
	ef election2.Factory,
	mf monitoring.MetricFactory,
) *Controller {
	initMetrics(mf)
	l := strconv.FormatInt(plClient.tree.TreeId, 10)
	return &Controller{opts: opts, ctClient: ctClient, plClient: plClient, ef: ef, label: l}
}

// RunWhenMasterWithRestarts calls RunWhenMaster, and, if the migration is
// configured with continuous mode, restarts it whenever it returns.
func (c *Controller) RunWhenMasterWithRestarts(ctx context.Context) {
	uri := c.ctClient.BaseURI()
	treeID := c.plClient.tree.TreeId
	for run := true; run; run = c.opts.Continuous && ctx.Err() == nil {
		glog.Infof("Starting migration Controller (%d<-%q)", treeID, uri)
		if err := c.RunWhenMaster(ctx); err != nil {
			glog.Errorf("Controller.RunWhenMaster(%d<-%q): %v", treeID, uri, err)
			continue
		}
		glog.Infof("Controller stopped (%d<-%q)", treeID, uri)
	}
}

// RunWhenMaster is a master-elected version of Run method. It executes Run
// whenever this instance captures mastership of the tree ID. As soon as the
// instance stops being the master, Run is canceled. The method returns if a
// severe error occurs, the passed in context is canceled, or fetching is
// completed (in non-Continuous mode). Releases mastership when terminates.
func (c *Controller) RunWhenMaster(ctx context.Context) error {
	// Avoid thundering herd when starting multiple tasks on the same tree.
	if err := sleepRandom(ctx, 0, c.opts.StartDelay); err != nil {
		return err // The context has been canceled.
	}

	treeID := strconv.FormatInt(c.plClient.tree.TreeId, 10)
	metrics.controllerStarts.Inc(treeID)

	el, err := c.ef.NewElection(ctx, treeID)
	if err != nil {
		return err
	}
	defer func(ctx context.Context) {
		metrics.isMaster.Set(0, c.label)
		if err := el.Close(ctx); err != nil {
			glog.Warningf("%s: Election.Close(): %v", treeID, err)
		}
	}(ctx)

	for {
		if err := el.Await(ctx); err != nil {
			return err
		}
		metrics.isMaster.Set(1, c.label)

		mctx, err := el.WithMastership(ctx)
		if err != nil {
			return err
		} else if err := mctx.Err(); err != nil {
			return err
		}

		glog.Infof("%s: running as master", treeID)
		metrics.masterRuns.Inc(c.label)

		// Run while still master (or until an error).
		err = c.Run(mctx)
		if ctx.Err() != nil {
			// We have been externally canceled, so return the current error (which
			// could be nil or a cancelation-related error).
			return err
		} else if mctx.Err() == nil {
			// We are still the master, so try to resign and emit the real error.
			if rerr := el.Resign(ctx); rerr != nil {
				glog.Errorf("%s: Election.Resign(): %v", treeID, rerr)
			}
			return err
		}

		// Otherwise the mastership has been canceled, retry.
		metrics.isMaster.Set(0, c.label)
		metrics.masterCancels.Inc(c.label)
	}
}

// Run transfers CT log entries obtained via the CT log client to a Trillian
// pre-ordered log via Trillian client. If Options.Continuous is true then the
// migration process runs continuously trying to keep up with the target CT
// log. Returns if an error occurs, the context is canceled, or all the entries
// have been transferred (in non-Continuous mode).
func (c *Controller) Run(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId

	root, err := c.plClient.getVerifiedRoot(ctx)
	if err != nil {
		return err
	}
	if c.opts.Continuous { // Ignore range parameters in Continuous mode.
		c.opts.StartIndex, c.opts.EndIndex = int64(root.TreeSize), 0
		glog.Warningf("%d: updated entry range to [%d, INF)", treeID, c.opts.StartIndex)
	} else if c.opts.StartIndex < 0 {
		c.opts.StartIndex = int64(root.TreeSize)
		glog.Warningf("%d: updated start index to %d", treeID, c.opts.StartIndex)
	}

	fetcher := scanner.NewFetcher(c.ctClient, &c.opts.FetcherOptions)
	sth, err := fetcher.Prepare(ctx)
	if err != nil {
		return err
	}
	if err := c.verifyConsistency(ctx, root, sth); err != nil {
		return err
	}

	var wg sync.WaitGroup
	c.batches = make(chan scanner.EntryBatch, c.opts.ChannelSize)
	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	for w, cnt := 0, c.opts.Submitters; w < cnt; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := c.runSubmitter(cctx); err != nil {
				glog.Errorf("%d: Stopping due to submitter error: %v", treeID, err)
				cancel() // Stop the other submitters and the Fetcher.
			}
		}()
	}

	if c.opts.StopAfter != 0 { // Configured with max running time.
		go func() {
			// Sleep for random duration in [StopAfter, 2*StopAfter).
			if err := sleepRandom(cctx, c.opts.StopAfter, c.opts.StopAfter); err == nil {
				fetcher.Stop() // Trigger graceful stop if not yet canceled.
			}
		}()
	}

	handler := func(b scanner.EntryBatch) {
		metrics.entriesFetched.Add(float64(len(b.Entries)), c.label)
		select {
		case c.batches <- b:
		case <-cctx.Done(): // Avoid deadlock when shutting down.
		}
	}

	result := fetcher.Run(cctx, handler)
	close(c.batches)
	wg.Wait()
	return result
}

// verifyConsistency checks that the provided verified Trillian root is
// consistent with the CT log's STH.
func (c *Controller) verifyConsistency(ctx context.Context, root *types.LogRootV1, sth *ct.SignedTreeHead) error {
	if c.opts.NoConsistencyCheck {
		glog.Warningf("%s: skipping consistency check", c.label)
		return nil
	}
	proof, err := c.ctClient.GetSTHConsistency(ctx, root.TreeSize, sth.TreeSize)
	if err != nil {
		return err
	}
	return merkle.NewLogVerifier(c.plClient.verif.Hasher).VerifyConsistencyProof(
		int64(root.TreeSize), int64(sth.TreeSize),
		root.RootHash, sth.SHA256RootHash[:], proof)
}

// runSubmitter obtains CT log entry batches from the controller's channel and
// submits them through Trillian client. Returns when the channel is closed, or
// the client returns a non-recoverable error (an example of a recoverable
// error is when Trillian write quota is exceeded).
func (c *Controller) runSubmitter(ctx context.Context) error {
	treeID := c.plClient.tree.TreeId
	for b := range c.batches {
		entries := float64(len(b.Entries))
		metrics.entriesSeen.Add(entries, c.label)

		end := b.Start + int64(len(b.Entries))
		if err := c.plClient.addSequencedLeaves(ctx, &b); err != nil {
			// addSequencedLeaves failed to submit entries despite retries. At this
			// point there is not much we can do. Seemingly the best strategy is to
			// shut down the Controller.
			return fmt.Errorf("failed to add batch [%d, %d): %v", b.Start, end, err)
		}
		glog.Infof("%d: added batch [%d, %d)", treeID, b.Start, end)
		metrics.entriesStored.Add(entries, c.label)
	}
	return nil
}

// sleepRandom sleeps for random duration in [base, base+spread).
func sleepRandom(ctx context.Context, base, spread time.Duration) error {
	d := base
	if spread != 0 {
		d += time.Duration(rand.Int63n(int64(spread)))
	}
	if d == 0 {
		return nil
	}
	return clock.SleepContext(ctx, d)
}
<|endoftext|>


<commit_before>//                           _       _
// __      _____  __ ___   ___  __ _| |_ ___
// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
//
//  Copyright © 2016 - 2019 SeMI Holding B.V. (registered @ Dutch Chamber of Commerce no 75221632). All rights reserved.
//  LICENSE WEAVIATE OPEN SOURCE: https://www.semi.technology/playbook/playbook/contract-weaviate-OSS.html
//  LICENSE WEAVIATE ENTERPRISE: https://www.semi.technology/playbook/contract-weaviate-enterprise.html
//  CONCEPT: Bob van Luijt (@bobvanluijt)
//  CONTACT: hello@semi.technology
//

package vectorizer

import (
	"context"
	"testing"

	"github.com/semi-technologies/weaviate/entities/models"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestVectorizingThings(t *testing.T) {
	type testCase struct {
		name               string
		input              *models.Thing
		expectedClientCall []string
	}

	tests := []testCase{
		testCase{
			name: "empty thing",
			input: &models.Thing{
				Class: "Car",
			},
			expectedClientCall: []string{"car"},
		},
		testCase{
			name: "thing with one string prop",
			input: &models.Thing{
				Class: "Car",
				Schema: map[string]interface{}{
					"brand": "Mercedes",
				},
			},
			expectedClientCall: []string{"car", "brand mercedes"},
		},

		testCase{
			name: "thing with one non-string prop",
			input: &models.Thing{
				Class: "Car",
				Schema: map[string]interface{}{
					"power": 300,
				},
			},
			expectedClientCall: []string{"car", "power"},
		},

		testCase{
			name: "thing with a mix of props",
			input: &models.Thing{
				Class: "Car",
				Schema: map[string]interface{}{
					"brand":  "best brand",
					"power":  300,
					"review": "a very great car",
				},
			},
			expectedClientCall: []string{"car", "brand best brand",
				"power", "review a very great car"},
		},

		testCase{
			name: "with compound class and prop names",
			input: &models.Thing{
				Class: "SuperCar",
				Schema: map[string]interface{}{
					"brandOfTheCar": "best brand",
					"power":         300,
					"review":        "a very great car",
				},
			},
			expectedClientCall: []string{"super car", "brand of the car best brand",
				"power", "review a very great car"},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			client := &fakeClient{}
			v := New(client)

			res, err := v.Thing(context.Background(), test.input)

			require.Nil(t, err)
			assert.Equal(t, []float32{0, 1, 2, 3}, res)
			assert.ElementsMatch(t, test.expectedClientCall, client.lastInput)

		})

	}
}

func TestVectorizingActions(t *testing.T) {
	type testCase struct {
		name               string
		input              *models.Action
		expectedClientCall []string
	}

	tests := []testCase{
		testCase{
			name: "empty thing",
			input: &models.Action{
				Class: "Flight",
			},
			expectedClientCall: []string{"flight"},
		},
		testCase{
			name: "thing with one string prop",
			input: &models.Action{
				Class: "Flight",
				Schema: map[string]interface{}{
					"brand": "Mercedes",
				},
			},
			expectedClientCall: []string{"flight", "brand mercedes"},
		},

		testCase{
			name: "thing with one non-string prop",
			input: &models.Action{
				Class: "Flight",
				Schema: map[string]interface{}{
					"length": 300,
				},
			},
			expectedClientCall: []string{"flight", "length"},
		},

		testCase{
			name: "thing with a mix of props",
			input: &models.Action{
				Class: "Flight",
				Schema: map[string]interface{}{
					"brand":  "best brand",
					"length": 300,
					"review": "a very great flight",
				},
			},
			expectedClientCall: []string{"flight", "brand best brand",
				"length", "review a very great flight"},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			client := &fakeClient{}
			v := New(client)

			res, err := v.Action(context.Background(), test.input)

			require.Nil(t, err)
			assert.Equal(t, []float32{0, 1, 2, 3}, res)
			assert.ElementsMatch(t, test.expectedClientCall, client.lastInput)

		})

	}
}

func TestVectorizingSearchTerms(t *testing.T) {
	type testCase struct {
		name               string
		input              []string
		expectedClientCall []string
	}

	tests := []testCase{
		testCase{
			name:               "single word",
			input:              []string{"car"},
			expectedClientCall: []string{"car"},
		},
		testCase{
			name:               "multiple entries with multiple words",
			input:              []string{"car", "car brand"},
			expectedClientCall: []string{"car", "car brand"},
		},
		testCase{
			name:               "multiple entries with upper casing",
			input:              []string{"Car", "Car Brand"},
			expectedClientCall: []string{"car", "car brand"},
		},
		testCase{
			name:               "with camel cased words",
			input:              []string{"Car", "CarBrand"},
			expectedClientCall: []string{"car", "car brand"},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			client := &fakeClient{}
			v := New(client)

			res, err := v.Corpi(context.Background(), test.input)

			require.Nil(t, err)
			assert.Equal(t, []float32{0, 1, 2, 3}, res)
			assert.ElementsMatch(t, test.expectedClientCall, client.lastInput)
		})
	}
}

type fakeClient struct {
	lastInput []string
}

func (c *fakeClient) VectorForCorpi(ctx context.Context, corpi []string) ([]float32, error) {
	c.lastInput = corpi
	return []float32{0, 1, 2, 3}, nil
}
<commit_msg>gh-976 fix vectorizer unit tests<commit_after>//                           _       _
// __      _____  __ ___   ___  __ _| |_ ___
// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
//
//  Copyright © 2016 - 2019 SeMI Holding B.V. (registered @ Dutch Chamber of Commerce no 75221632). All rights reserved.
//  LICENSE WEAVIATE OPEN SOURCE: https://www.semi.technology/playbook/playbook/contract-weaviate-OSS.html
//  LICENSE WEAVIATE ENTERPRISE: https://www.semi.technology/playbook/contract-weaviate-enterprise.html
//  CONCEPT: Bob van Luijt (@bobvanluijt)
//  CONTACT: hello@semi.technology
//

package vectorizer

import (
	"context"
	"strings"
	"testing"

	"github.com/semi-technologies/weaviate/entities/models"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestVectorizingThings(t *testing.T) {
	type testCase struct {
		name               string
		input              *models.Thing
		expectedClientCall []string
	}

	tests := []testCase{
		testCase{
			name: "empty thing",
			input: &models.Thing{
				Class: "Car",
			},
			expectedClientCall: []string{"car"},
		},
		testCase{
			name: "thing with one string prop",
			input: &models.Thing{
				Class: "Car",
				Schema: map[string]interface{}{
					"brand": "Mercedes",
				},
			},
			expectedClientCall: []string{"car brand mercedes"},
		},

		testCase{
			name: "thing with one non-string prop",
			input: &models.Thing{
				Class: "Car",
				Schema: map[string]interface{}{
					"power": 300,
				},
			},
			expectedClientCall: []string{"car"},
		},

		testCase{
			name: "thing with a mix of props",
			input: &models.Thing{
				Class: "Car",
				Schema: map[string]interface{}{
					"brand":  "best brand",
					"power":  300,
					"review": "a very great car",
				},
			},
			expectedClientCall: []string{"car brand best brand review a very great car"},
		},

		testCase{
			name: "with compound class and prop names",
			input: &models.Thing{
				Class: "SuperCar",
				Schema: map[string]interface{}{
					"brandOfTheCar": "best brand",
					"power":         300,
					"review":        "a very great car",
				},
			},
			expectedClientCall: []string{"super car brand of the car best brand review a very great car"},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			client := &fakeClient{}
			v := New(client)

			res, err := v.Thing(context.Background(), test.input)

			require.Nil(t, err)
			assert.Equal(t, []float32{0, 1, 2, 3}, res)
			expected := strings.Split(test.expectedClientCall[0], " ")
			actual := strings.Split(client.lastInput[0], " ")
			assert.ElementsMatch(t, expected, actual)
		})

	}
}

func TestVectorizingActions(t *testing.T) {
	type testCase struct {
		name               string
		input              *models.Action
		expectedClientCall []string
	}

	tests := []testCase{
		testCase{
			name: "empty thing",
			input: &models.Action{
				Class: "Flight",
			},
			expectedClientCall: []string{"flight"},
		},
		testCase{
			name: "thing with one string prop",
			input: &models.Action{
				Class: "Flight",
				Schema: map[string]interface{}{
					"brand": "Mercedes",
				},
			},
			expectedClientCall: []string{"flight brand mercedes"},
		},

		testCase{
			name: "thing with one non-string prop",
			input: &models.Action{
				Class: "Flight",
				Schema: map[string]interface{}{
					"length": 300,
				},
			},
			expectedClientCall: []string{"flight"},
		},

		testCase{
			name: "thing with a mix of props",
			input: &models.Action{
				Class: "Flight",
				Schema: map[string]interface{}{
					"brand":  "best brand",
					"length": 300,
					"review": "a very great flight",
				},
			},
			expectedClientCall: []string{"flight brand best brand review a very great flight"},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			client := &fakeClient{}
			v := New(client)

			res, err := v.Action(context.Background(), test.input)

			require.Nil(t, err)
			assert.Equal(t, []float32{0, 1, 2, 3}, res)
			expected := strings.Split(test.expectedClientCall[0], " ")
			actual := strings.Split(client.lastInput[0], " ")
			assert.ElementsMatch(t, expected, actual)

		})

	}
}

func TestVectorizingSearchTerms(t *testing.T) {
	type testCase struct {
		name               string
		input              []string
		expectedClientCall []string
	}

	tests := []testCase{
		testCase{
			name:               "single word",
			input:              []string{"car"},
			expectedClientCall: []string{"car"},
		},
		testCase{
			name:               "multiple entries with multiple words",
			input:              []string{"car", "car brand"},
			expectedClientCall: []string{"car", "car brand"},
		},
		testCase{
			name:               "multiple entries with upper casing",
			input:              []string{"Car", "Car Brand"},
			expectedClientCall: []string{"car", "car brand"},
		},
		testCase{
			name:               "with camel cased words",
			input:              []string{"Car", "CarBrand"},
			expectedClientCall: []string{"car", "car brand"},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			client := &fakeClient{}
			v := New(client)

			res, err := v.Corpi(context.Background(), test.input)

			require.Nil(t, err)
			assert.Equal(t, []float32{0, 1, 2, 3}, res)
			assert.ElementsMatch(t, test.expectedClientCall, client.lastInput)
		})
	}
}

type fakeClient struct {
	lastInput []string
}

func (c *fakeClient) VectorForCorpi(ctx context.Context, corpi []string) ([]float32, error) {
	c.lastInput = corpi
	return []float32{0, 1, 2, 3}, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configuration

import (
	"errors"
	selfLogger "github.com/cloudawan/cloudone/utility/logger"
	"github.com/cloudawan/cloudone_utility/configuration"
	"github.com/cloudawan/cloudone_utility/logger"
	"github.com/cloudawan/cloudone_utility/restclient"
	"io/ioutil"
	"time"
)

var log = selfLogger.GetLogManager().GetLogger("utility")

var configurationContent = `
{
	"certificate": "/etc/cloudone/development_cert.pem",
	"key": "/etc/cloudone/development_key.pem",
	"restapiPort": 8081,
	"etcdEndpoints": ["http://127.0.0.1:4001"],
	"etcdHeaderTimeoutPerRequestInMilliSecond": 2000,
	"etcdBasePath": "/cloudawan/cloudone",
	"storageTypeDefault": 3,
	"cloudoneAnalysisHost": "127.0.0.1",
	"cloudoneAnalysisPort": 8082,
	"kubeApiServerEndPoints": ["https://kubernetes.default.svc.cluster.local:443"],
	"kubeApiServerHealthCheckTimeoutInMilliSecond": 1000,
	"kubeApiServerTokenPath": "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
`

var LocalConfiguration *configuration.Configuration

const (
	KubeApiServerHealthCheckTimeoutInMilliSecond = 1000
)

func init() {
	err := Reload()
	if err != nil {
		log.Critical(err)
		panic(err)
	}
}

func Reload() error {
	localConfiguration, err := configuration.CreateConfiguration("cloudone", configurationContent)
	if err == nil {
		LocalConfiguration = localConfiguration
		if err := reloadFile(); err != nil {
			return err
		}
	}

	return err
}

const (
	StorageTypeDefault   = 0
	StorageTypeDummy     = 1
	StorageTypeCassandra = 2
	StorageTypeEtcd      = 3
)

func GetStorageTypeDefault() (int, error) {
	value, ok := LocalConfiguration.GetInt("storageTypeDefault")
	if ok == false {
		log.Critical("Can't load storageTypeDefault")
		return 0, errors.New("Can't load storageTypeDefault")
	}
	return value, nil
}

func reloadFile() error {
	// Token
	kubeApiServerTokenPath, ok := LocalConfiguration.GetString("kubeApiServerTokenPath")
	if ok == false {
		log.Error("Fail to get configuration kubeApiServerTokenPath")
		return errors.New("Fail to get configuration kubeApiServerTokenPath")
	}

	fileContent, err := ioutil.ReadFile(kubeApiServerTokenPath)
	if err != nil {
		log.Error("Fail to get the file content of kubeApiServerTokenPath %s", kubeApiServerTokenPath)
		return errors.New("Fail to get the file content of kubeApiServerTokenPath " + kubeApiServerTokenPath)
	}
	LocalConfiguration.SetNative("kubeApiServerToken", string(fileContent))

	return nil
}

func GetAvailablekubeApiServerEndPoint() (returnedEndPoint string, returnedToken string, returnedError error) {
	defer func() {
		if err := recover(); err != nil {
			returnedEndPoint = ""
			returnedToken = ""
			returnedError = err.(error)
			log.Error("GetAvailablekubeApiServerEndPoint Error: %s", err)
			log.Error(logger.GetStackTrace(4096, false))
		}
	}()

	kubeApiServerEndPointSlice, ok := LocalConfiguration.GetStringSlice("kubeApiServerEndPoints")
	if ok == false {
		log.Error("Fail to get configuration kubeApiServerEndPoints")
		return "", "", errors.New("Fail to get configuration kubeApiServerEndPoints")
	}

	kubeApiServerHealthCheckTimeoutInMilliSecond, ok := LocalConfiguration.GetInt("kubeApiServerHealthCheckTimeoutInMilliSecond")
	if ok == false {
		kubeApiServerHealthCheckTimeoutInMilliSecond = KubeApiServerHealthCheckTimeoutInMilliSecond
	}

	kubeApiServerToken, ok := LocalConfiguration.GetString("kubeApiServerToken")
	if ok == false {
		log.Error("Fail to get configuration kubeApiServerToken")
		return "", "", errors.New("Fail to get configuration kubeApiServerToken")
	}

	token := "Bearer " + kubeApiServerToken
	headerMap := make(map[string]string)
	headerMap["Authorization"] = token

	for _, kubeApiServerEndPoint := range kubeApiServerEndPointSlice {
		result, err := restclient.HealthCheck(
			kubeApiServerEndPoint,
			headerMap,
			time.Duration(kubeApiServerHealthCheckTimeoutInMilliSecond)*time.Millisecond)

		if result {
			return kubeApiServerEndPoint, token, nil
		} else {
			if err != nil {
				log.Error(err)
			}
		}
	}

	log.Error("No available kube apiserver endpoint")
	return "", "", errors.New("No available kube apiserver endpoint")
}
<commit_msg>Externd timeout<commit_after>// Copyright 2015 CloudAwan LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configuration

import (
	"errors"
	selfLogger "github.com/cloudawan/cloudone/utility/logger"
	"github.com/cloudawan/cloudone_utility/configuration"
	"github.com/cloudawan/cloudone_utility/logger"
	"github.com/cloudawan/cloudone_utility/restclient"
	"io/ioutil"
	"time"
)

var log = selfLogger.GetLogManager().GetLogger("utility")

var configurationContent = `
{
	"certificate": "/etc/cloudone/development_cert.pem",
	"key": "/etc/cloudone/development_key.pem",
	"restapiPort": 8081,
	"etcdEndpoints": ["http://127.0.0.1:4001"],
	"etcdHeaderTimeoutPerRequestInMilliSecond": 10000,
	"etcdBasePath": "/cloudawan/cloudone",
	"storageTypeDefault": 3,
	"cloudoneAnalysisHost": "127.0.0.1",
	"cloudoneAnalysisPort": 8082,
	"kubeApiServerEndPoints": ["https://kubernetes.default.svc.cluster.local:443"],
	"kubeApiServerHealthCheckTimeoutInMilliSecond": 1000,
	"kubeApiServerTokenPath": "/var/run/secrets/kubernetes.io/serviceaccount/token"
}
`

var LocalConfiguration *configuration.Configuration

const (
	KubeApiServerHealthCheckTimeoutInMilliSecond = 1000
)

func init() {
	err := Reload()
	if err != nil {
		log.Critical(err)
		panic(err)
	}
}

func Reload() error {
	localConfiguration, err := configuration.CreateConfiguration("cloudone", configurationContent)
	if err == nil {
		LocalConfiguration = localConfiguration
		if err := reloadFile(); err != nil {
			return err
		}
	}

	return err
}

const (
	StorageTypeDefault   = 0
	StorageTypeDummy     = 1
	StorageTypeCassandra = 2
	StorageTypeEtcd      = 3
)

func GetStorageTypeDefault() (int, error) {
	value, ok := LocalConfiguration.GetInt("storageTypeDefault")
	if ok == false {
		log.Critical("Can't load storageTypeDefault")
		return 0, errors.New("Can't load storageTypeDefault")
	}
	return value, nil
}

func reloadFile() error {
	// Token
	kubeApiServerTokenPath, ok := LocalConfiguration.GetString("kubeApiServerTokenPath")
	if ok == false {
		log.Error("Fail to get configuration kubeApiServerTokenPath")
		return errors.New("Fail to get configuration kubeApiServerTokenPath")
	}

	fileContent, err := ioutil.ReadFile(kubeApiServerTokenPath)
	if err != nil {
		log.Error("Fail to get the file content of kubeApiServerTokenPath %s", kubeApiServerTokenPath)
		return errors.New("Fail to get the file content of kubeApiServerTokenPath " + kubeApiServerTokenPath)
	}
	LocalConfiguration.SetNative("kubeApiServerToken", string(fileContent))

	return nil
}

func GetAvailablekubeApiServerEndPoint() (returnedEndPoint string, returnedToken string, returnedError error) {
	defer func() {
		if err := recover(); err != nil {
			returnedEndPoint = ""
			returnedToken = ""
			returnedError = err.(error)
			log.Error("GetAvailablekubeApiServerEndPoint Error: %s", err)
			log.Error(logger.GetStackTrace(4096, false))
		}
	}()

	kubeApiServerEndPointSlice, ok := LocalConfiguration.GetStringSlice("kubeApiServerEndPoints")
	if ok == false {
		log.Error("Fail to get configuration kubeApiServerEndPoints")
		return "", "", errors.New("Fail to get configuration kubeApiServerEndPoints")
	}

	kubeApiServerHealthCheckTimeoutInMilliSecond, ok := LocalConfiguration.GetInt("kubeApiServerHealthCheckTimeoutInMilliSecond")
	if ok == false {
		kubeApiServerHealthCheckTimeoutInMilliSecond = KubeApiServerHealthCheckTimeoutInMilliSecond
	}

	kubeApiServerToken, ok := LocalConfiguration.GetString("kubeApiServerToken")
	if ok == false {
		log.Error("Fail to get configuration kubeApiServerToken")
		return "", "", errors.New("Fail to get configuration kubeApiServerToken")
	}

	token := "Bearer " + kubeApiServerToken
	headerMap := make(map[string]string)
	headerMap["Authorization"] = token

	for _, kubeApiServerEndPoint := range kubeApiServerEndPointSlice {
		result, err := restclient.HealthCheck(
			kubeApiServerEndPoint,
			headerMap,
			time.Duration(kubeApiServerHealthCheckTimeoutInMilliSecond)*time.Millisecond)

		if result {
			return kubeApiServerEndPoint, token, nil
		} else {
			if err != nil {
				log.Error(err)
			}
		}
	}

	log.Error("No available kube apiserver endpoint")
	return "", "", errors.New("No available kube apiserver endpoint")
}
<|endoftext|>


<commit_before>// Package ca provides easy to use certificate authority related functions.
// This is a lightweight wrapper around "crypto/x509" package for
// creating CA certs, client certs, signing requests, and more.
//
// Any "cert, key []byte" type of function parameters and return types are
// always PEM encoded X.509 certificate and private key pairs.
// You can store the certificate/key pair with standard naming as
// "cert.pem" and "key.pem" in the file system.
//
// This package is mostly based on the example code provided at:
// http://golang.org/src/crypto/tls/generate_cert.go
package ca

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"math/big"
	"net"
	"strings"
	"time"
)

// todo modify CreateCACert to accept certs (for intermediate CAs) or null for self-signed root CA
// add CraeteCertChain to follow recommended flow and return both byte arrays and parsed tls server/client certs
// update example to use new CraeteCertChain function

// CreateCACert creates a self-signed CA certificate.
// The created certificate can be used for signing intermediate CA certificates and CRLs.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateCACert(subject pkix.Name, validFor time.Duration, keyLength int) (cert, key []byte, err error) {
	c, p, err := createBaseCert(subject, validFor, keyLength)
	c.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
	c.IsCA = true

	cert, key, err = signAndEncodeCert(c, p, c, p)
	return
}

// CreateIntermediateCACert creates an intermediate CA certificate for signing server or client certificates and CRLs.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateIntermediateCACert(subject pkix.Name, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	var (
		sc, c *x509.Certificate
		sk, k *rsa.PrivateKey
	)

	if sc, sk, err = parseCertAndKey(signingCert, signingKey); err != nil {
		return
	}

	if c, k, err = createBaseCert(subject, validFor, keyLength); err != nil {
		return
	}

	c.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
	c.IsCA = true

	cert, key, err = signAndEncodeCert(sc, sk, c, k)
	return
}

// CreateServerCert creates a hosting certificate for servers using TLS.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateServerCert(subject pkix.Name, host string, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	var (
		sc, c *x509.Certificate
		sk, k *rsa.PrivateKey
	)

	if sc, sk, err = parseCertAndKey(signingCert, signingKey); err != nil {
		return
	}

	if c, k, err = createBaseCert(subject, validFor, keyLength); err != nil {
		return
	}

	c.KeyUsage = x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature
	c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
	c.IsCA = false
	setHosts(host, c)

	cert, key, err = signAndEncodeCert(sc, sk, c, k)
	return
}

// CreateClientCert creates a client certificate.
// Created certificate will have its extended key usage set to 'client authentication' and will be ready for use in TLS client authentication.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateClientCert(subject pkix.Name, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	var (
		sc, c *x509.Certificate
		sk, k *rsa.PrivateKey
	)

	if sc, sk, err = parseCertAndKey(signingCert, signingKey); err != nil {
		return
	}

	if c, k, err = createBaseCert(subject, validFor, keyLength); err != nil {
		return
	}

	c.KeyUsage = x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature
	c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
	c.IsCA = false

	cert, key, err = signAndEncodeCert(sc, sk, c, k)
	return
}

// createBaseCert creates and returns x509.Certificate (unsigned) and rsa.PrivateKey objects with basic paramters set.
func createBaseCert(subject pkix.Name, validFor time.Duration, keyLength int) (*x509.Certificate, *rsa.PrivateKey, error) {
	privKey, err := rsa.GenerateKey(rand.Reader, keyLength)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate certificate private key using RSA: %v", err)
	}

	notBefore := time.Now()
	notAfter := notBefore.Add(validFor)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate the certificate serial number: %v", err)
	}

	cert := x509.Certificate{
		SerialNumber:          serialNumber,
		Subject:               subject,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		BasicConstraintsValid: true,
	}

	return &cert, privKey, nil
}

// setHosts parses the comma separated host name / IP list and adds them to Subject Alternate Name list of a server/hosting certificate.
func setHosts(host string, cert *x509.Certificate) {
	hosts := strings.Split(host, ",")
	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			cert.IPAddresses = append(cert.IPAddresses, ip)
		} else {
			cert.DNSNames = append(cert.DNSNames, h)
		}
	}
}

// Parses PEM encoded X.509 certificate and private key pair into x509.Certificate and rsa.PrivateKey objects.
func parseCertAndKey(cert, key []byte) (c *x509.Certificate, k *rsa.PrivateKey, err error) {
	pc, _ := pem.Decode(cert)
	if c, err = x509.ParseCertificate(pc.Bytes); err != nil {
		err = fmt.Errorf("Failed to parse private key with error: %v", err)
		return
	}

	pk, _ := pem.Decode(key)
	if k, err = x509.ParsePKCS1PrivateKey(pk.Bytes); err != nil {
		err = fmt.Errorf("Failed to parse certificate with error: %v", err)
		return
	}

	return
}

// signAndEncodeCert signs a given certificate with given signing cert/key pair and encodes resulting signed cert and private key in PEM format and returns.
func signAndEncodeCert(signingCert *x509.Certificate, signingKey *rsa.PrivateKey, c *x509.Certificate, k *rsa.PrivateKey) (cert, key []byte, err error) {
	certDerBytes, err := x509.CreateCertificate(rand.Reader, c, signingCert, &k.PublicKey, signingKey)
	if err != nil {
		return
	}

	cert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDerBytes})
	key = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)})
	return
}
<commit_msg>add CraeteCertChain stub<commit_after>// Package ca provides easy to use certificate authority related functions.
// This is a lightweight wrapper around "crypto/x509" package for
// creating CA certs, client certs, signing requests, and more.
//
// Any "cert, key []byte" type of function parameters and return types are
// always PEM encoded X.509 certificate and private key pairs.
// You can store the certificate/key pair with standard naming as
// "cert.pem" and "key.pem" in the file system.
//
// This package is mostly based on the example code provided at:
// http://golang.org/src/crypto/tls/generate_cert.go
package ca

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"fmt"
	"math/big"
	"net"
	"strings"
	"time"
)

// todo modify CreateCACert to accept certs (for intermediate CAs) or null for self-signed root CA
// add CraeteCertChain to follow recommended flow and return both byte arrays and parsed tls server/client certs
// update example to use new CraeteCertChain function

// CraeteCertChain generates an entire certificate chain with the following hierarchy:
// Root CA -> Intermediate CA -> Server Certificate & Client Certificate
func CraeteCertChain() {

}

// GenCACert generates a CA certificate.
// If signingCert and signingKey are not provided, the certificate is created as a self-signed root CA.
// If signingCert and signingKey are provided, the certificate is created as an intermediate CA, signed with provided certificate.
// The generated certificate can only be used for signing other certificates and CRLs.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func GenCACert(subject pkix.Name, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	c, p, err := createBaseCert(subject, validFor, keyLength)
	c.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
	c.IsCA = true

	cert, key, err = signAndEncodeCert(c, p, c, p)
	return
}

// CreateIntermediateCACert creates an intermediate CA certificate for signing server or client certificates and CRLs.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateIntermediateCACert(subject pkix.Name, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	var (
		sc, c *x509.Certificate
		sk, k *rsa.PrivateKey
	)

	if sc, sk, err = parseCertAndKey(signingCert, signingKey); err != nil {
		return
	}

	if c, k, err = createBaseCert(subject, validFor, keyLength); err != nil {
		return
	}

	c.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
	c.IsCA = true

	cert, key, err = signAndEncodeCert(sc, sk, c, k)
	return
}

// CreateServerCert creates a hosting certificate for servers using TLS.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateServerCert(subject pkix.Name, host string, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	var (
		sc, c *x509.Certificate
		sk, k *rsa.PrivateKey
	)

	if sc, sk, err = parseCertAndKey(signingCert, signingKey); err != nil {
		return
	}

	if c, k, err = createBaseCert(subject, validFor, keyLength); err != nil {
		return
	}

	c.KeyUsage = x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature
	c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
	c.IsCA = false
	setHosts(host, c)

	cert, key, err = signAndEncodeCert(sc, sk, c, k)
	return
}

// CreateClientCert creates a client certificate.
// Created certificate will have its extended key usage set to 'client authentication' and will be ready for use in TLS client authentication.
// The returned slices are the PEM encoded X.509 certificate and private key pair.
func CreateClientCert(subject pkix.Name, validFor time.Duration, keyLength int, signingCert, signingKey []byte) (cert, key []byte, err error) {
	var (
		sc, c *x509.Certificate
		sk, k *rsa.PrivateKey
	)

	if sc, sk, err = parseCertAndKey(signingCert, signingKey); err != nil {
		return
	}

	if c, k, err = createBaseCert(subject, validFor, keyLength); err != nil {
		return
	}

	c.KeyUsage = x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature
	c.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
	c.IsCA = false

	cert, key, err = signAndEncodeCert(sc, sk, c, k)
	return
}

// createBaseCert creates and returns x509.Certificate (unsigned) and rsa.PrivateKey objects with basic paramters set.
func createBaseCert(subject pkix.Name, validFor time.Duration, keyLength int) (*x509.Certificate, *rsa.PrivateKey, error) {
	privKey, err := rsa.GenerateKey(rand.Reader, keyLength)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate certificate private key using RSA: %v", err)
	}

	notBefore := time.Now()
	notAfter := notBefore.Add(validFor)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to generate the certificate serial number: %v", err)
	}

	cert := x509.Certificate{
		SerialNumber:          serialNumber,
		Subject:               subject,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		BasicConstraintsValid: true,
	}

	return &cert, privKey, nil
}

// setHosts parses the comma separated host name / IP list and adds them to Subject Alternate Name list of a server/hosting certificate.
func setHosts(host string, cert *x509.Certificate) {
	hosts := strings.Split(host, ",")
	for _, h := range hosts {
		if ip := net.ParseIP(h); ip != nil {
			cert.IPAddresses = append(cert.IPAddresses, ip)
		} else {
			cert.DNSNames = append(cert.DNSNames, h)
		}
	}
}

// Parses PEM encoded X.509 certificate and private key pair into x509.Certificate and rsa.PrivateKey objects.
func parseCertAndKey(cert, key []byte) (c *x509.Certificate, k *rsa.PrivateKey, err error) {
	pc, _ := pem.Decode(cert)
	if c, err = x509.ParseCertificate(pc.Bytes); err != nil {
		err = fmt.Errorf("Failed to parse private key with error: %v", err)
		return
	}

	pk, _ := pem.Decode(key)
	if k, err = x509.ParsePKCS1PrivateKey(pk.Bytes); err != nil {
		err = fmt.Errorf("Failed to parse certificate with error: %v", err)
		return
	}

	return
}

// signAndEncodeCert signs a given certificate with given signing cert/key pair and encodes resulting signed cert and private key in PEM format and returns.
func signAndEncodeCert(signingCert *x509.Certificate, signingKey *rsa.PrivateKey, c *x509.Certificate, k *rsa.PrivateKey) (cert, key []byte, err error) {
	certDerBytes, err := x509.CreateCertificate(rand.Reader, c, signingCert, &k.PublicKey, signingKey)
	if err != nil {
		return
	}

	cert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDerBytes})
	key = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)})
	return
}
<|endoftext|>


<commit_before>package cc

// Solve populates solutions with valid boards of the dimensions columns*rows
// with valid configurations for pieces.
func Solve(columns, rows int, pieces []Piece, solutions *[]Board) {
	np := len(pieces)

	b := NewBoard(columns, rows)
	place(b, pieces, solutions, 0)

	sm := make(map[string]Board)
	for _, s := range *solutions {
		sm[s.Notation()] = s
	}

	ret := make([]Board, 0)
	for _, v := range sm {
		ret = append(ret, v)
	}

	ret2 := make([]Board, 0)
	// Cull invalid
	for _, s := range ret {
		sum := 0
		for _, c := range s.cells {
			if c.piece != nil {
				sum++
			}
		}
		if sum == np {
			ret2 = append(ret2, s)
		}
	}

	//return ret
	*solutions = ret2
}

// place tries to place the next piece on the board and recurse down
// the search tree. Appends the board and returns when a valid configuration
// is found.
func place(board Board, pieces []Piece, solutions *[]Board, run int) {
	if len(pieces) == 0 {
		*solutions = append(*solutions, board)
		return
	}

	// Shift the pieces to get the first
	p, pieces := pieces[0], pieces[1:]

	for _, c := range board.cells {
		if c.dead || c.piece != nil {
			continue
		}
		canPlace := true
		tr := p.Threatening(&board, c.x, c.y)
		for _, t := range tr {
			tc := board.cells[board.columns*t.y+t.x]
			if tc.piece != nil {
				canPlace = false
			}
		}
		if canPlace {
			b2 := NewBoard(board.columns, board.rows)
			copy(b2.cells, board.cells)
			b2.cells[b2.columns*c.y+c.x].piece = p
			for _, t := range tr {
				b2.cells[b2.columns*t.y+t.x].dead = true
			}
			place(b2, pieces, solutions, run+1)
		}
	}
}
<commit_msg>Exit earlier if no possible solutions can exist<commit_after>package cc

// Solve populates solutions with valid boards of the dimensions columns*rows
// with valid configurations for pieces.
func Solve(columns, rows int, pieces []Piece, solutions *[]Board) {
	np := len(pieces)

	// No possible solutions
	if np != 1 && np >= columns*rows {
		*solutions = []Board{}
		return
	}

	b := NewBoard(columns, rows)
	place(b, pieces, solutions, 0)

	sm := make(map[string]Board)
	for _, s := range *solutions {
		sm[s.Notation()] = s
	}

	ret := make([]Board, 0)
	for _, v := range sm {
		ret = append(ret, v)
	}

	ret2 := make([]Board, 0)
	// Cull invalid
	for _, s := range ret {
		sum := 0
		for _, c := range s.cells {
			if c.piece != nil {
				sum++
			}
		}
		if sum == np {
			ret2 = append(ret2, s)
		}
	}

	//return ret
	*solutions = ret2
}

// place tries to place the next piece on the board and recurse down
// the search tree. Appends the board and returns when a valid configuration
// is found.
func place(board Board, pieces []Piece, solutions *[]Board, run int) {
	if len(pieces) == 0 {
		*solutions = append(*solutions, board)
		return
	}

	// Shift the pieces to get the first
	p, pieces := pieces[0], pieces[1:]

	for _, c := range board.cells {
		if c.dead || c.piece != nil {
			continue
		}
		canPlace := true
		tr := p.Threatening(&board, c.x, c.y)
		for _, t := range tr {
			tc := board.cells[board.columns*t.y+t.x]
			if tc.piece != nil {
				canPlace = false
			}
		}
		if canPlace {
			b2 := NewBoard(board.columns, board.rows)
			copy(b2.cells, board.cells)
			b2.cells[b2.columns*c.y+c.x].piece = p
			for _, t := range tr {
				b2.cells[b2.columns*t.y+t.x].dead = true
			}
			place(b2, pieces, solutions, run+1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/boltdb/bolt"
	"log"
)

// Database is the persisting class
type Database struct {
	DB            *bolt.DB
	Server        *Server
	Tournaments   []*Tournament
	People        []*Person
	tournamentRef map[string]*Tournament
}

var (
	// TournamentKey is the byte string identifying the tournament buckets
	TournamentKey = []byte("tournaments")
	// PeopleKey is the byte string identifying the bucket of people and their data
	PeopleKey = []byte("people")
)

// NewDatabase returns a new database object
func NewDatabase(fn string) (*Database, error) {
	bolt, err := bolt.Open(fn, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	db := &Database{DB: bolt}
	db.tournamentRef = make(map[string]*Tournament)

	return db, nil
}

// LoadTournaments loads the tournaments from the database and into memory
func (d *Database) LoadTournaments() error {
	loaded := 0
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)
		if b == nil {
			// If there is no bucket, bail silently.
			// This only really happens in tests.
			// TODO: Fix pls
			return nil
		}

		err := b.ForEach(func(k []byte, v []byte) error {
			t, err := LoadTournament(v, d)
			if err != nil {
				return err
			}

			d.Tournaments = append(d.Tournaments, t)
			d.tournamentRef[t.ID] = t
			loaded++
			return nil
		})
		return err
	})

	return err
}

// SaveTournament stores the current state of the tournaments into the db
func (d *Database) SaveTournament(t *Tournament) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(TournamentKey)
		if err != nil {
			return err
		}

		json, _ := t.JSON()
		err = b.Put([]byte(t.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	return ret
}

// SavePerson stores a person into the DB
func (d *Database) SavePerson(p *Person) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(PeopleKey)
		if err != nil {
			return err
		}

		json, _ := p.JSON()
		err = b.Put([]byte(p.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	return ret
}

// GetPerson gets a Person{} from the DB
func (d *Database) GetPerson(id string) *Person {
	tx, err := d.DB.Begin(false)
	if err != nil {
		log.Fatal(err)
		return nil
	}
	defer tx.Rollback()

	b := tx.Bucket(PeopleKey)
	out := b.Get([]byte(id))
	p := &Person{}
	_ = json.Unmarshal(out, p)
	return p
}

// LoadPeople loads the people from the database and into memory
func (d *Database) LoadPeople() error {
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(PeopleKey)

		err := b.ForEach(func(k []byte, v []byte) error {
			p, err := LoadPerson(v)
			if err != nil {
				return err
			}

			d.People = append(d.People, p)
			return nil
		})
		return err
	})

	return err
}

// Close closes the database
func (d *Database) Close() error {
	return d.DB.Close()
}
<commit_msg>Reset the db.People array whenever reloading it<commit_after>package main

import (
	"encoding/json"
	"github.com/boltdb/bolt"
	"log"
)

// Database is the persisting class
type Database struct {
	DB            *bolt.DB
	Server        *Server
	Tournaments   []*Tournament
	People        []*Person
	tournamentRef map[string]*Tournament
}

var (
	// TournamentKey is the byte string identifying the tournament buckets
	TournamentKey = []byte("tournaments")
	// PeopleKey is the byte string identifying the bucket of people and their data
	PeopleKey = []byte("people")
)

// NewDatabase returns a new database object
func NewDatabase(fn string) (*Database, error) {
	bolt, err := bolt.Open(fn, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	db := &Database{DB: bolt}
	db.tournamentRef = make(map[string]*Tournament)

	return db, nil
}

// LoadTournaments loads the tournaments from the database and into memory
func (d *Database) LoadTournaments() error {
	loaded := 0
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)
		if b == nil {
			// If there is no bucket, bail silently.
			// This only really happens in tests.
			// TODO: Fix pls
			return nil
		}

		err := b.ForEach(func(k []byte, v []byte) error {
			t, err := LoadTournament(v, d)
			if err != nil {
				return err
			}

			d.Tournaments = append(d.Tournaments, t)
			d.tournamentRef[t.ID] = t
			loaded++
			return nil
		})
		return err
	})

	return err
}

// SaveTournament stores the current state of the tournaments into the db
func (d *Database) SaveTournament(t *Tournament) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(TournamentKey)
		if err != nil {
			return err
		}

		json, _ := t.JSON()
		err = b.Put([]byte(t.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	return ret
}

// SavePerson stores a person into the DB
func (d *Database) SavePerson(p *Person) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(PeopleKey)
		if err != nil {
			return err
		}

		json, _ := p.JSON()
		err = b.Put([]byte(p.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	return ret
}

// GetPerson gets a Person{} from the DB
func (d *Database) GetPerson(id string) *Person {
	tx, err := d.DB.Begin(false)
	if err != nil {
		log.Fatal(err)
		return nil
	}
	defer tx.Rollback()

	b := tx.Bucket(PeopleKey)
	out := b.Get([]byte(id))
	p := &Person{}
	_ = json.Unmarshal(out, p)
	return p
}

// LoadPeople loads the people from the database and into memory
func (d *Database) LoadPeople() error {
	d.People = make([]*Person, 0)
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(PeopleKey)

		err := b.ForEach(func(k []byte, v []byte) error {
			p, err := LoadPerson(v)
			if err != nil {
				return err
			}

			d.People = append(d.People, p)
			return nil
		})
		return err
	})

	return err
}

// Close closes the database
func (d *Database) Close() error {
	return d.DB.Close()
}
<|endoftext|>


<commit_before>package jet

import (
	"database/sql"
)

func Open(driverName, dataSourceName string) (Db, error) {
	godb, err := sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return &db{godb: godb}, nil
}

type db struct {
	godb  *sql.DB
	query string
	args  []interface{}
}

func (j *db) Begin() Tx {
	return &tx{godb: j.godb}
}

func (j *db) Query(query string, args ...interface{}) Queryable {
	j.query = query
	j.args = args
	return j
}

func (j *db) Run() error {
	return j.Rows(nil)
}

func (j *db) Rows(v interface{}, maxRows ...int64) error {
	// Determine max rows
	var max int64 = -1
	if len(maxRows) > 0 {
		max = maxRows[0]
	}
	// Query
	rows, err := j.godb.Query(j.query, j.args...)
	if err != nil {
		return err
	}
	cols, err := rows.Columns()
	if err != nil {
		return err
	}
	var i int64 = 0
	for rows.Next() {
		// Check if max rows has been reached
		if max >= 0 && i >= max {
			break
		}
		// Scan values into containers
		containers := make([]interface{}, 0, len(cols))
		for i := 0; i < cap(containers); i++ {
			var cv interface{}
			containers = append(containers, &cv)
		}
		err := rows.Scan(containers...)
		if err != nil {
			return err
		}

		// Map values
		m := make(map[string]interface{}, len(cols))
		for i, col := range cols {
			m[col] = containers[i]
		}
		err = mapper{m}.unpack(v)
		if err != nil {
			return err
		}
		i++
	}
	return nil
}
<commit_msg>limit rows optimization<commit_after>package jet

import (
	"database/sql"
)

func Open(driverName, dataSourceName string) (Db, error) {
	godb, err := sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return &db{godb: godb}, nil
}

type db struct {
	godb  *sql.DB
	query string
	args  []interface{}
}

func (j *db) Begin() Tx {
	return &tx{godb: j.godb}
}

func (j *db) Query(query string, args ...interface{}) Queryable {
	j.query = query
	j.args = args
	return j
}

func (j *db) Run() error {
	return j.Rows(nil)
}

func (j *db) Rows(v interface{}, maxRows ...int64) error {
	// Determine max rows
	var max int64 = -1
	if len(maxRows) > 0 {
		max = maxRows[0]
	}
	// Query
	rows, err := j.godb.Query(j.query, j.args...)
	if err != nil {
		return err
	}
	cols, err := rows.Columns()
	if err != nil {
		return err
	}
	var i int64 = 0
	for {
		// Check if max rows has been reached
		if max >= 0 && i >= max {
			break
		}
		// Break if no more rows
		if !rows.Next() {
			break
		}
		// Scan values into containers
		containers := make([]interface{}, 0, len(cols))
		for i := 0; i < cap(containers); i++ {
			var cv interface{}
			containers = append(containers, &cv)
		}
		err := rows.Scan(containers...)
		if err != nil {
			return err
		}

		// Map values
		m := make(map[string]interface{}, len(cols))
		for i, col := range cols {
			m[col] = containers[i]
		}
		err = mapper{m}.unpack(v)
		if err != nil {
			return err
		}
		i++
	}
	return nil
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"errors"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"strconv"
	"strings"
)

type Host struct {
	db     *sql.DB
	dbname string
}

// Common for each of the db datastructures used here
type dbDatastructure struct {
	host  *Host
	table string
}

type (
	List     dbDatastructure
	Set      dbDatastructure
	HashMap  dbDatastructure
	KeyValue dbDatastructure
)

const (
	// Version number. Stable API within major version numbers.
	Version = 1.0
	// The default "username:password@host:port/database" that the database is running at
	defaultDatabaseServer = ""     // "username:password@server:port/"
	defaultDatabaseName   = "test" // "main"
	defaultStringLength   = 42     // using VARCHAR, so this will be expanded up to 65535 characters as needed, unless mysql strict mode is enabled
)

/* --- Helper functions --- */

// Test if the local database server is up and running
func TestConnection() (err error) {
	return TestConnectionHost(defaultDatabaseServer)
}

// Test if a given database server at host:port is up and running.
// Also pings.
func TestConnectionHost(hostColonPort string) (err error) {
	// Connect to the given host:port
	db, err := sql.Open("mysql", hostColonPort)
	defer db.Close()
	return db.Ping()
}

// Split a string into two parts, given a delimiter.
// Returns the two parts and true if it works out.
func twoFields(s, delim string) (string, string, bool) {
	if strings.Count(s, delim) != 1 {
		return s, "", false
	}
	fields := strings.Split(s, delim)
	return fields[0], fields[1], true
}

/* --- Host functions --- */

// Create a new database connection.
// connectionString may be on the form "username:password@host:port/database".
func New(connectionString string) *Host {
	// TODO: Find better variable names for these
	dbname := defaultDatabaseName
	hostColonPort := connectionString
	// Extract the database name, if given
	if first, second, ok := twoFields(hostColonPort, "/"); ok {
		if strings.TrimSpace(second) != "" {
			dbname = second
		}
		hostColonPort = first + "/"
	} else if !strings.HasSuffix(hostColonPort, "/") {
		// Add a trailing slash, if missing
		hostColonPort += "/"
	}
	if strings.TrimSpace(hostColonPort) == "/" {
		log.Println("Connecting to local database instance")
	} else {
		log.Println("Connecting to host: " + hostColonPort)
	}
	db, err := sql.Open("mysql", hostColonPort)
	if err != nil {
		panic("Could not connect to " + defaultDatabaseServer + "!")
	}
	host := &Host{db, dbname}
	if err := db.Ping(); err != nil {
		panic("Database does not reply to ping: " + err.Error())
	}
	if err := host.createDatabase(); err != nil {
		panic("Could not create database " + host.dbname + ": " + err.Error())
	}
	if err := host.useDatabase(); err != nil {
		panic("Could not use database " + host.dbname + ": " + err.Error())
	}
	return host
}

// The default database connection
func NewLocalHost() *Host {
	connectionString := defaultDatabaseServer + defaultDatabaseName
	if !strings.HasSuffix(defaultDatabaseServer, "/") {
		connectionString = defaultDatabaseServer + "/" + defaultDatabaseName
	}
	return New(connectionString)
}

// Select a different database. Create the database if needed.
func (host *Host) SelectDatabase(dbname string) error {
	host.dbname = dbname
	if err := host.createDatabase(); err != nil {
		return err
	}
	if err := host.useDatabase(); err != nil {
		return err
	}
	return nil
}

// Will create the database if it does not already exist.
func (host *Host) createDatabase() error {
	if _, err := host.db.Exec("CREATE DATABASE IF NOT EXISTS " + host.dbname + " CHARACTER SET = utf8"); err != nil {
		return err
	}
	log.Println("Created database " + host.dbname)
	return nil
}

// Use the host.dbname database.
func (host *Host) useDatabase() error {
	if _, err := host.db.Exec("USE " + host.dbname); err != nil {
		return err
	}
	log.Println("Using database " + host.dbname)
	return nil
}

/* --- List functions --- */

// Create a new list. Lists are ordered.
func NewList(host *Host, table string) *List {
	l := &List{host, table}
	// list is the name of the column
	if _, err := l.host.db.Exec("CREATE TABLE IF NOT EXISTS " + table + " (id INT PRIMARY KEY AUTO_INCREMENT, list VARCHAR(" + strconv.Itoa(defaultStringLength) + "))"); err != nil {
		// This is more likely to happen at the start of the program,
		// hence the panic.
		panic("Could not create table " + table + ": " + err.Error())
	}
	log.Println("Created table " + table + " in database " + host.dbname)
	return l
}

// Add an element to the list
func (rl *List) Add(value string) error {
	// list is the name of the column
	_, err := rl.host.db.Exec("INSERT INTO "+rl.table+" (list) VALUES (?)", value)
	return err
}

// Get all elements of a list
func (rl *List) GetAll() ([]string, error) {
	rows, err := rl.host.db.Query("SELECT list FROM " + rl.table + " ORDER BY id")
	if err != nil {
		panic(err.Error())
	}
	defer rows.Close()
	var (
		values []string
		value  string
	)
	for rows.Next() {
		err = rows.Scan(&value)
		values = append(values, value)
		if err != nil {
			panic(err.Error())
		}
	}
	if err := rows.Err(); err != nil {
		panic(err.Error())
	}
	return values, nil
}

// Get the last element of a list
func (rl *List) GetLast() (string, error) {
	// Fetches the item with the largest id.
	// Faster than "ORDER BY id DESC limit 1" for large tables.
	rows, err := rl.host.db.Query("SELECT list FROM " + rl.table + " WHERE id = (SELECT MAX(id) FROM " + rl.table + ")")
	if err != nil {
		panic(err.Error())
	}
	defer rows.Close()
	var value string
	// Get the value. Will only loop once.
	for rows.Next() {
		log.Println(".")
		err = rows.Scan(&value)
		if err != nil {
			panic(err.Error())
		}
	}
	if err := rows.Err(); err != nil {
		panic(err.Error())
	}
	return value, nil
}

// Get the last N elements of a list
func (rl *List) GetLastN(n int) ([]string, error) {
	values, err := rl.GetAll()
	if err != nil {
		return []string{}, err
	}
	if len(values) < n {
		return []string{}, errors.New("Too few elements in table at GetLastN")
	}
	return values[len(values)-n:], nil
}

// Remove this list
func (rl *List) Remove() error {
	// Remove the table
	_, err := rl.host.db.Exec("DROP TABLE " + rl.table)
	return err
}

// Clear the list contents
func (rl *List) Clear() error {
	// Clear the table
	_, err := rl.host.db.Exec("TRUNCATE TABLE " + rl.table)
	return err
}

/* --- Set functions --- */

//// Create a new set
//func NewSet(host *sql.DB, table string) *Set {
//	return &Set{host, table, defaultDatabaseName}
//}
//
//// Select a different database
//func (rs *Set) SelectDatabase(dbname string) {
//	rs.dbname = dbname
//}
//
//// Add an element to the set
//func (rs *Set) Add(value string) error {
//	db := rs.host.Get(rs.dbname)
//	_, err := db.Do("SADD", rs.table, value)
//	return err
//}
//
//// Check if a given value is in the set
//func (rs *Set) Has(value string) (bool, error) {
//	db := rs.host.Get(rs.dbname)
//	retval, err := db.Do("SISMEMBER", rs.table, value)
//	if err != nil {
//		panic(err)
//	}
//	return db.Bool(retval, err)
//}
//
//// Get all elements of the set
//func (rs *Set) GetAll() ([]string, error) {
//	db := rs.host.Get(rs.dbname)
//	result, err := db.Values(db.Do("SMEMBERS", rs.table))
//	strs := make([]string, len(result))
//	for i := 0; i < len(result); i++ {
//		strs[i] = getString(result, i)
//	}
//	return strs, err
//}
//
//// Remove an element from the set
//func (rs *Set) Del(value string) error {
//	db := rs.host.Get(rs.dbname)
//	_, err := db.Do("SREM", rs.table, value)
//	return err
//}
//
//// Remove this set
//func (rs *Set) Remove() error {
//	db := rs.host.Get(rs.dbname)
//	_, err := db.Do("DEL", rs.table)
//	return err
//}
//
///* --- HashMap functions --- */
//
//// Create a new hashmap
//func NewHashMap(host *sql.DB, table string) *HashMap {
//	return &HashMap{host, table, defaultDatabaseName}
//}
//
//// Select a different database
//func (rh *HashMap) SelectDatabase(dbname string) {
//	rh.dbname = dbname
//}
//
//// Set a value in a hashmap given the element id (for instance a user id) and the key (for instance "password")
//func (rh *HashMap) Set(elementid, key, value string) error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("HSET", rh.table+":"+elementid, key, value)
//	return err
//}
//
//// Get a value from a hashmap given the element id (for instance a user id) and the key (for instance "password")
//func (rh *HashMap) Get(elementid, key string) (string, error) {
//	db := rh.host.Get(rh.dbname)
//	result, err := db.String(db.Do("HGET", rh.table+":"+elementid, key))
//	if err != nil {
//		return "", err
//	}
//	return result, nil
//}
//
//// Check if a given elementid + key is in the hash map
//func (rh *HashMap) Has(elementid, key string) (bool, error) {
//	db := rh.host.Get(rh.dbname)
//	retval, err := db.Do("HEXISTS", rh.table+":"+elementid, key)
//	if err != nil {
//		panic(err)
//	}
//	return db.Bool(retval, err)
//}
//
//// Check if a given elementid exists as a hash map at all
//func (rh *HashMap) Exists(elementid string) (bool, error) {
//	// TODO: key is not meant to be a wildcard, check for "*"
//	return hasKey(rh.host, rh.table+":"+elementid, rh.dbname)
//}
//
//// Get all elementid's for all hash elements
//func (rh *HashMap) GetAll() ([]string, error) {
//	db := rh.host.Get(rh.dbname)
//	result, err := db.Values(db.Do("KEYS", rh.table+":*"))
//	strs := make([]string, len(result))
//	idlen := len(rh.table)
//	for i := 0; i < len(result); i++ {
//		strs[i] = getString(result, i)[idlen+1:]
//	}
//	return strs, err
//}
//
//// Remove a key for an entry in a hashmap (for instance the email field for a user)
//func (rh *HashMap) DelKey(elementid, key string) error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("HDEL", rh.table+":"+elementid, key)
//	return err
//}
//
//// Remove an element (for instance a user)
//func (rh *HashMap) Del(elementid string) error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("DEL", rh.table+":"+elementid)
//	return err
//}
//
//// Remove this hashmap
//func (rh *HashMap) Remove() error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("DEL", rh.table)
//	return err
//}
//
///* --- KeyValue functions --- */
//
//// Create a new key/value
//func NewKeyValue(host *sql.DB, table string) *KeyValue {
//	return &KeyValue{host, table, defaultDatabaseName}
//}
//
//// Select a different database
//func (rkv *KeyValue) SelectDatabase(dbname string) {
//	rkv.dbname = dbname
//}
//
//// Set a key and value
//func (rkv *KeyValue) Set(key, value string) error {
//	db := rkv.host.Get(rkv.dbname)
//	_, err := db.Do("SET", rkv.table+":"+key, value)
//	return err
//}
//
//// Get a value given a key
//func (rkv *KeyValue) Get(key string) (string, error) {
//	db := rkv.host.Get(rkv.dbname)
//	result, err := db.String(db.Do("GET", rkv.table+":"+key))
//	if err != nil {
//		return "", err
//	}
//	return result, nil
//}
//
//// Remove a key
//func (rkv *KeyValue) Del(key string) error {
//	db := rkv.host.Get(rkv.dbname)
//	_, err := db.Do("DEL", rkv.table+":"+key)
//	return err
//}
//
//// Remove this key/value
//func (rkv *KeyValue) Remove() error {
//	db := rkv.host.Get(rkv.dbname)
//	_, err := db.Do("DEL", rkv.table)
//	return err
//}
//
//// --- Generic db functions ---
//
//// Check if a key exists. The key can be a wildcard (ie. "user*").
//func hasKey(host *sql.DB, wildcard string, dbname string) (bool, error) {
//	db := host.Get(dbname)
//	result, err := db.Values(db.Do("KEYS", wildcard))
//	if err != nil {
//		return false, err
//	}
//	return len(result) > 0, nil
//}
<commit_msg>Added a TODO item<commit_after>package db

import (
	"database/sql"
	"errors"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"strconv"
	"strings"
)

type Host struct {
	db     *sql.DB
	dbname string
}

// Common for each of the db datastructures used here
type dbDatastructure struct {
	host  *Host
	table string
}

type (
	List     dbDatastructure
	Set      dbDatastructure
	HashMap  dbDatastructure
	KeyValue dbDatastructure
)

const (
	// Version number. Stable API within major version numbers.
	Version = 1.0
	// The default "username:password@host:port/database" that the database is running at
	defaultDatabaseServer = ""     // "username:password@server:port/"
	defaultDatabaseName   = "test" // "main"
	defaultStringLength   = 42     // using VARCHAR, so this will be expanded up to 65535 characters as needed, unless mysql strict mode is enabled
)

/* --- Helper functions --- */

// Test if the local database server is up and running
func TestConnection() (err error) {
	return TestConnectionHost(defaultDatabaseServer)
}

// Test if a given database server at host:port is up and running.
// Also pings.
func TestConnectionHost(hostColonPort string) (err error) {
	// Connect to the given host:port
	db, err := sql.Open("mysql", hostColonPort)
	defer db.Close()
	return db.Ping()
}

// Split a string into two parts, given a delimiter.
// Returns the two parts and true if it works out.
func twoFields(s, delim string) (string, string, bool) {
	if strings.Count(s, delim) != 1 {
		return s, "", false
	}
	fields := strings.Split(s, delim)
	return fields[0], fields[1], true
}

/* --- Host functions --- */

// Create a new database connection.
// connectionString may be on the form "username:password@host:port/database".
func New(connectionString string) *Host {
	// TODO: Find better variable names for these
	dbname := defaultDatabaseName
	hostColonPort := connectionString
	// Extract the database name, if given
	if first, second, ok := twoFields(hostColonPort, "/"); ok {
		if strings.TrimSpace(second) != "" {
			dbname = second
		}
		hostColonPort = first + "/"
	} else if !strings.HasSuffix(hostColonPort, "/") {
		// Add a trailing slash, if missing
		hostColonPort += "/"
	}
	if strings.TrimSpace(hostColonPort) == "/" {
		log.Println("Connecting to local database instance")
	} else {
		log.Println("Connecting to host: " + hostColonPort)
	}
	db, err := sql.Open("mysql", hostColonPort)
	if err != nil {
		panic("Could not connect to " + defaultDatabaseServer + "!")
	}
	host := &Host{db, dbname}
	if err := db.Ping(); err != nil {
		panic("Database does not reply to ping: " + err.Error())
	}
	if err := host.createDatabase(); err != nil {
		panic("Could not create database " + host.dbname + ": " + err.Error())
	}
	if err := host.useDatabase(); err != nil {
		panic("Could not use database " + host.dbname + ": " + err.Error())
	}
	return host
}

// The default database connection
func NewLocalHost() *Host {
	connectionString := defaultDatabaseServer + defaultDatabaseName
	if !strings.HasSuffix(defaultDatabaseServer, "/") {
		connectionString = defaultDatabaseServer + "/" + defaultDatabaseName
	}
	return New(connectionString)
}

// Select a different database. Create the database if needed.
func (host *Host) SelectDatabase(dbname string) error {
	host.dbname = dbname
	if err := host.createDatabase(); err != nil {
		return err
	}
	if err := host.useDatabase(); err != nil {
		return err
	}
	return nil
}

// Will create the database if it does not already exist.
func (host *Host) createDatabase() error {
	if _, err := host.db.Exec("CREATE DATABASE IF NOT EXISTS " + host.dbname + " CHARACTER SET = utf8"); err != nil {
		return err
	}
	log.Println("Created database " + host.dbname)
	return nil
}

// Use the host.dbname database.
func (host *Host) useDatabase() error {
	if _, err := host.db.Exec("USE " + host.dbname); err != nil {
		return err
	}
	log.Println("Using database " + host.dbname)
	return nil
}

/* --- List functions --- */

// Create a new list. Lists are ordered.
func NewList(host *Host, table string) *List {
	l := &List{host, table}
	// list is the name of the column
	if _, err := l.host.db.Exec("CREATE TABLE IF NOT EXISTS " + table + " (id INT PRIMARY KEY AUTO_INCREMENT, list VARCHAR(" + strconv.Itoa(defaultStringLength) + "))"); err != nil {
		// This is more likely to happen at the start of the program,
		// hence the panic.
		panic("Could not create table " + table + ": " + err.Error())
	}
	log.Println("Created table " + table + " in database " + host.dbname)
	return l
}

// Add an element to the list
func (rl *List) Add(value string) error {
	// list is the name of the column
	_, err := rl.host.db.Exec("INSERT INTO "+rl.table+" (list) VALUES (?)", value)
	return err
}

// Get all elements of a list
func (rl *List) GetAll() ([]string, error) {
	rows, err := rl.host.db.Query("SELECT list FROM " + rl.table + " ORDER BY id")
	if err != nil {
		panic(err.Error())
	}
	defer rows.Close()
	var (
		values []string
		value  string
	)
	for rows.Next() {
		err = rows.Scan(&value)
		values = append(values, value)
		if err != nil {
			panic(err.Error())
		}
	}
	if err := rows.Err(); err != nil {
		panic(err.Error())
	}
	return values, nil
}

// Get the last element of a list
func (rl *List) GetLast() (string, error) {
	// Fetches the item with the largest id.
	// Faster than "ORDER BY id DESC limit 1" for large tables.
	rows, err := rl.host.db.Query("SELECT list FROM " + rl.table + " WHERE id = (SELECT MAX(id) FROM " + rl.table + ")")
	if err != nil {
		panic(err.Error())
	}
	defer rows.Close()
	var value string
	// Get the value. Will only loop once.
	for rows.Next() {
		err = rows.Scan(&value)
		if err != nil {
			panic(err.Error())
		}
	}
	if err := rows.Err(); err != nil {
		panic(err.Error())
	}
	return value, nil
}

// Get the last N elements of a list
func (rl *List) GetLastN(n int) ([]string, error) {
	// TODO: http://stackoverflow.com/a/574148/131264 instead of GetAll()
	values, err := rl.GetAll()
	if err != nil {
		return []string{}, err
	}
	if len(values) < n {
		return []string{}, errors.New("Too few elements in table at GetLastN")
	}
	return values[len(values)-n:], nil
}

// Remove this list
func (rl *List) Remove() error {
	// Remove the table
	_, err := rl.host.db.Exec("DROP TABLE " + rl.table)
	return err
}

// Clear the list contents
func (rl *List) Clear() error {
	// Clear the table
	_, err := rl.host.db.Exec("TRUNCATE TABLE " + rl.table)
	return err
}

/* --- Set functions --- */

//// Create a new set
//func NewSet(host *sql.DB, table string) *Set {
//	return &Set{host, table, defaultDatabaseName}
//}
//
//// Select a different database
//func (rs *Set) SelectDatabase(dbname string) {
//	rs.dbname = dbname
//}
//
//// Add an element to the set
//func (rs *Set) Add(value string) error {
//	db := rs.host.Get(rs.dbname)
//	_, err := db.Do("SADD", rs.table, value)
//	return err
//}
//
//// Check if a given value is in the set
//func (rs *Set) Has(value string) (bool, error) {
//	db := rs.host.Get(rs.dbname)
//	retval, err := db.Do("SISMEMBER", rs.table, value)
//	if err != nil {
//		panic(err)
//	}
//	return db.Bool(retval, err)
//}
//
//// Get all elements of the set
//func (rs *Set) GetAll() ([]string, error) {
//	db := rs.host.Get(rs.dbname)
//	result, err := db.Values(db.Do("SMEMBERS", rs.table))
//	strs := make([]string, len(result))
//	for i := 0; i < len(result); i++ {
//		strs[i] = getString(result, i)
//	}
//	return strs, err
//}
//
//// Remove an element from the set
//func (rs *Set) Del(value string) error {
//	db := rs.host.Get(rs.dbname)
//	_, err := db.Do("SREM", rs.table, value)
//	return err
//}
//
//// Remove this set
//func (rs *Set) Remove() error {
//	db := rs.host.Get(rs.dbname)
//	_, err := db.Do("DEL", rs.table)
//	return err
//}
//
///* --- HashMap functions --- */
//
//// Create a new hashmap
//func NewHashMap(host *sql.DB, table string) *HashMap {
//	return &HashMap{host, table, defaultDatabaseName}
//}
//
//// Select a different database
//func (rh *HashMap) SelectDatabase(dbname string) {
//	rh.dbname = dbname
//}
//
//// Set a value in a hashmap given the element id (for instance a user id) and the key (for instance "password")
//func (rh *HashMap) Set(elementid, key, value string) error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("HSET", rh.table+":"+elementid, key, value)
//	return err
//}
//
//// Get a value from a hashmap given the element id (for instance a user id) and the key (for instance "password")
//func (rh *HashMap) Get(elementid, key string) (string, error) {
//	db := rh.host.Get(rh.dbname)
//	result, err := db.String(db.Do("HGET", rh.table+":"+elementid, key))
//	if err != nil {
//		return "", err
//	}
//	return result, nil
//}
//
//// Check if a given elementid + key is in the hash map
//func (rh *HashMap) Has(elementid, key string) (bool, error) {
//	db := rh.host.Get(rh.dbname)
//	retval, err := db.Do("HEXISTS", rh.table+":"+elementid, key)
//	if err != nil {
//		panic(err)
//	}
//	return db.Bool(retval, err)
//}
//
//// Check if a given elementid exists as a hash map at all
//func (rh *HashMap) Exists(elementid string) (bool, error) {
//	// TODO: key is not meant to be a wildcard, check for "*"
//	return hasKey(rh.host, rh.table+":"+elementid, rh.dbname)
//}
//
//// Get all elementid's for all hash elements
//func (rh *HashMap) GetAll() ([]string, error) {
//	db := rh.host.Get(rh.dbname)
//	result, err := db.Values(db.Do("KEYS", rh.table+":*"))
//	strs := make([]string, len(result))
//	idlen := len(rh.table)
//	for i := 0; i < len(result); i++ {
//		strs[i] = getString(result, i)[idlen+1:]
//	}
//	return strs, err
//}
//
//// Remove a key for an entry in a hashmap (for instance the email field for a user)
//func (rh *HashMap) DelKey(elementid, key string) error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("HDEL", rh.table+":"+elementid, key)
//	return err
//}
//
//// Remove an element (for instance a user)
//func (rh *HashMap) Del(elementid string) error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("DEL", rh.table+":"+elementid)
//	return err
//}
//
//// Remove this hashmap
//func (rh *HashMap) Remove() error {
//	db := rh.host.Get(rh.dbname)
//	_, err := db.Do("DEL", rh.table)
//	return err
//}
//
///* --- KeyValue functions --- */
//
//// Create a new key/value
//func NewKeyValue(host *sql.DB, table string) *KeyValue {
//	return &KeyValue{host, table, defaultDatabaseName}
//}
//
//// Select a different database
//func (rkv *KeyValue) SelectDatabase(dbname string) {
//	rkv.dbname = dbname
//}
//
//// Set a key and value
//func (rkv *KeyValue) Set(key, value string) error {
//	db := rkv.host.Get(rkv.dbname)
//	_, err := db.Do("SET", rkv.table+":"+key, value)
//	return err
//}
//
//// Get a value given a key
//func (rkv *KeyValue) Get(key string) (string, error) {
//	db := rkv.host.Get(rkv.dbname)
//	result, err := db.String(db.Do("GET", rkv.table+":"+key))
//	if err != nil {
//		return "", err
//	}
//	return result, nil
//}
//
//// Remove a key
//func (rkv *KeyValue) Del(key string) error {
//	db := rkv.host.Get(rkv.dbname)
//	_, err := db.Do("DEL", rkv.table+":"+key)
//	return err
//}
//
//// Remove this key/value
//func (rkv *KeyValue) Remove() error {
//	db := rkv.host.Get(rkv.dbname)
//	_, err := db.Do("DEL", rkv.table)
//	return err
//}
//
//// --- Generic db functions ---
//
//// Check if a key exists. The key can be a wildcard (ie. "user*").
//func hasKey(host *sql.DB, wildcard string, dbname string) (bool, error) {
//	db := host.Get(dbname)
//	result, err := db.Values(db.Do("KEYS", wildcard))
//	if err != nil {
//		return false, err
//	}
//	return len(result) > 0, nil
//}
<|endoftext|>