<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail

import (
	"context"
	"expvar"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/exporter"
	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/tailer"
	"github.com/google/mtail/internal/vm"
	"github.com/google/mtail/internal/waker"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/version"
	"go.opencensus.io/zpages"
)

// Server contains the state of the main mtail program.
type Server struct {
	ctx   context.Context
	store *metrics.Store // Metrics storage
	wg    sync.WaitGroup // wait for main processes to shutdown

	t *tailer.Tailer     // t manages log patterns and log streams, which sends lines to the VMs
	l *vm.Loader         // l loads programs and manages the VM lifecycle
	e *exporter.Exporter // e manages the export of metrics from the store

	lines chan *logline.LogLine // primary communication channel, owned by Tailer.

	reg *prometheus.Registry

	listener net.Listener // Configured with bind address.

	buildInfo          BuildInfo // go build information
	programPath        string    // path to programs to load
	logPathPatterns    []string  // list of patterns to watch for log files to tail
	ignoreRegexPattern string

	oneShot      bool // if set, mtail reads log files from the beginning, once, then exits
	compileOnly  bool // if set, mtail compiles programs then exits
	dumpAst      bool // if set, mtail prints the program syntax tree after parse
	dumpAstTypes bool // if set, mtail prints the program syntax tree after type checking
	dumpBytecode bool // if set, mtail prints the program bytecode after code generation

	overrideLocation     *time.Location // Timezone location to use when parsing timestamps
	staleLogGcWaker      waker.Waker    // Wake to run stale log gc
	logPatternPollWaker  waker.Waker    // Wake to poll for log patterns
	logstreamPollWaker   waker.Waker    // Wake idle logstreams to poll sfor new data
	metricPushInterval   time.Duration  // Interval between metric pushes
	syslogUseCurrentYear bool           // if set, use the current year for timestamps that have no year information
	omitMetricSource     bool           // if set, do not link the source program to a metric
	omitProgLabel        bool           // if set, do not put the program name in the metric labels
	emitMetricTimestamp  bool           // if set, emit the metric's recorded timestamp
	omitDumpMetricsStore bool           // if set, do not print the metric store; useful in test
}

// initLoader constructs a new program loader and performs the initial load of program files in the program directory.
func (m *Server) initLoader() error {
	opts := []vm.Option{
		vm.PrometheusRegisterer(m.reg),
	}
	if m.compileOnly {
		opts = append(opts, vm.CompileOnly())
	}
	if m.oneShot {
		opts = append(opts, vm.ErrorsAbort())
	}
	if m.dumpAst {
		opts = append(opts, vm.DumpAst())
	}
	if m.dumpAstTypes {
		opts = append(opts, vm.DumpAstTypes())
	}
	if m.dumpBytecode {
		opts = append(opts, vm.DumpBytecode())
	}
	if m.syslogUseCurrentYear {
		opts = append(opts, vm.SyslogUseCurrentYear())
	}
	if m.omitMetricSource {
		opts = append(opts, vm.OmitMetricSource())
	}
	if m.overrideLocation != nil {
		opts = append(opts, vm.OverrideLocation(m.overrideLocation))
	}
	var err error
	m.l, err = vm.NewLoader(m.lines, &m.wg, m.programPath, m.store, opts...)
	if err != nil {
		return err
	}
	return nil
}

// initExporter sets up an Exporter for this Server.
func (m *Server) initExporter() (err error) {
	opts := []exporter.Option{}
	if m.omitProgLabel {
		opts = append(opts, exporter.OmitProgLabel())
	}
	if m.emitMetricTimestamp {
		opts = append(opts, exporter.EmitTimestamp())
	}
	if m.metricPushInterval > 0 {
		opts = append(opts, exporter.PushInterval(m.metricPushInterval))
	}
	m.e, err = exporter.New(m.ctx, &m.wg, m.store, opts...)
	if err != nil {
		return err
	}
	m.reg.MustRegister(m.e)

	// Create mtail_build_info metric.
	version.Branch = m.buildInfo.Branch
	version.Version = m.buildInfo.Version
	version.Revision = m.buildInfo.Revision
	m.reg.MustRegister(version.NewCollector("mtail"))
	return nil
}

// initTailer sets up and starts a Tailer for this Server.
func (m *Server) initTailer() (err error) {
	opts := []tailer.Option{
		tailer.IgnoreRegex(m.ignoreRegexPattern),
		tailer.LogPatterns(m.logPathPatterns),
		tailer.LogPatternPollWaker(m.logPatternPollWaker),
		tailer.StaleLogGcWaker(m.staleLogGcWaker),
		tailer.LogstreamPollWaker(m.logstreamPollWaker),
	}
	if m.oneShot {
		opts = append(opts, tailer.OneShot)
	}
	m.t, err = tailer.New(m.ctx, &m.wg, m.lines, opts...)
	return
}

// initHttpServer begins the http server.
func (m *Server) initHttpServer() error {
	initDone := make(chan struct{})
	defer close(initDone)

	if m.listener == nil {
		glog.Info("no listen address configured, not starting http server")
		return nil
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/favicon.ico", FaviconHandler)
	mux.Handle("/", m)
	mux.Handle("/progz", http.HandlerFunc(m.l.ProgzHandler))
	mux.HandleFunc("/json", http.HandlerFunc(m.e.HandleJSON))
	mux.Handle("/metrics", promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{}))
	mux.HandleFunc("/varz", http.HandlerFunc(m.e.HandleVarz))
	mux.Handle("/debug/vars", expvar.Handler())
	mux.HandleFunc("/debug/pprof/", pprof.Index)
	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
	zpages.Handle(mux, "/")

	srv := &http.Server{
		Handler: mux,
	}

	var wg sync.WaitGroup
	errc := make(chan error, 1)

	// This goroutine runs the http server.
	wg.Add(1)
	go func() {
		defer wg.Done()
		<-initDone
		glog.Infof("Listening on %s", m.listener.Addr())
		if err := srv.Serve(m.listener); err != nil && err != http.ErrServerClosed {
			errc <- err
		}
	}()

	// This goroutine manages http server shutdown.
	go func() {
		<-initDone
		select {
		case err := <-errc:
			glog.Info(err)
		case <-m.ctx.Done():
			glog.Info("Shutdown requested.")
			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			srv.SetKeepAlivesEnabled(false)
			if err := srv.Shutdown(ctx); err != nil {
				glog.Info(err)
			}
		}
		// Wait for the Serve routine to exit.
		wg.Wait()
	}()

	return nil
}

// New creates a Server from the supplied Options.  The Server is started by
// the time New returns, it watches the LogPatterns for files, starts tailing
// their changes and sends any new lines found to the virtual machines loaded
// from ProgramPath. If OneShot mode is enabled, it will exit after reading
// each log file from start to finish.
// TODO(jaq): this doesn't need to be a constructor anymore, it could start and
// block until quit, once TestServer.PollWatched is addressed.
func New(ctx context.Context, store *metrics.Store, options ...Option) (*Server, error) {
	m := &Server{
		ctx:   ctx,
		store: store,
		lines: make(chan *logline.LogLine),
		// Using a non-pedantic registry means we can be looser with metrics that
		// are not fully specified at startup.
		reg: prometheus.NewRegistry(),
	}

	// TODO(jaq): Should these move to initExporter?
	expvarDescs := map[string]*prometheus.Desc{
		// internal/tailer/file.go
		"log_errors_total":    prometheus.NewDesc("log_errors_total", "number of IO errors encountered per log file", []string{"logfile"}, nil),
		"log_rotations_total": prometheus.NewDesc("log_rotations_total", "number of log rotation events per log file", []string{"logfile"}, nil),
		"log_truncates_total": prometheus.NewDesc("log_truncates_total", "number of log truncation events log file", []string{"logfile"}, nil),
		"log_lines_total":     prometheus.NewDesc("log_lines_total", "number of lines read per log file", []string{"logfile"}, nil),
		// internal/vm/loader.go
		"lines_total":               prometheus.NewDesc("lines_total", "number of lines received by the program loader", nil, nil),
		"prog_loads_total":          prometheus.NewDesc("prog_loads_total", "number of program load events by program source filename", []string{"prog"}, nil),
		"prog_load_errors_total":    prometheus.NewDesc("prog_load_errors_total", "number of errors encountered when loading per program source filename", []string{"prog"}, nil),
		"prog_runtime_errors_total": prometheus.NewDesc("prog_runtime_errors_total", "number of errors encountered when executing programs per source filename", []string{"prog"}, nil),
	}
	m.reg.MustRegister(
		prometheus.NewGoCollector(),
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
	// Prefix all expvar metrics with 'mtail_'
	prometheus.WrapRegistererWithPrefix("mtail_", m.reg).MustRegister(
		prometheus.NewExpvarCollector(expvarDescs))
	if err := m.SetOption(options...); err != nil {
		return nil, err
	}
	if err := m.initExporter(); err != nil {
		return nil, err
	}
	if err := m.initLoader(); err != nil {
		return nil, err
	}
	if err := m.initTailer(); err != nil {
		return nil, err
	}
	if err := m.initHttpServer(); err != nil {
		return nil, err
	}
	return m, nil
}

// SetOption takes one or more option functions and applies them in order to MtailServer.
func (m *Server) SetOption(options ...Option) error {
	for _, option := range options {
		if err := option.apply(m); err != nil {
			return err
		}
	}
	return nil
}

// Run awaits mtail's shutdown.
// TODO(jaq): remove this once the test server is able to trigger polls on the components.
func (m *Server) Run() error {
	m.wg.Wait()
	if m.compileOnly {
		glog.Info("compile-only is set, exiting")
		return nil
	}
	if m.oneShot {
		if m.omitDumpMetricsStore {
			glog.Info("Store dump disabled, exiting")
			return nil
		}
		if err := m.store.WriteMetrics(os.Stdout); err != nil {
			return err
		}
		return nil
	}
	return nil
}
<commit_msg>Avoid a data race in the new test by having no exporter in oneshot.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package mtail

import (
	"context"
	"expvar"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/exporter"
	"github.com/google/mtail/internal/logline"
	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/tailer"
	"github.com/google/mtail/internal/vm"
	"github.com/google/mtail/internal/waker"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/version"
	"go.opencensus.io/zpages"
)

// Server contains the state of the main mtail program.
type Server struct {
	ctx   context.Context
	store *metrics.Store // Metrics storage
	wg    sync.WaitGroup // wait for main processes to shutdown

	t *tailer.Tailer     // t manages log patterns and log streams, which sends lines to the VMs
	l *vm.Loader         // l loads programs and manages the VM lifecycle
	e *exporter.Exporter // e manages the export of metrics from the store

	lines chan *logline.LogLine // primary communication channel, owned by Tailer.

	reg *prometheus.Registry

	listener net.Listener // Configured with bind address.

	buildInfo          BuildInfo // go build information
	programPath        string    // path to programs to load
	logPathPatterns    []string  // list of patterns to watch for log files to tail
	ignoreRegexPattern string

	oneShot      bool // if set, mtail reads log files from the beginning, once, then exits
	compileOnly  bool // if set, mtail compiles programs then exits
	dumpAst      bool // if set, mtail prints the program syntax tree after parse
	dumpAstTypes bool // if set, mtail prints the program syntax tree after type checking
	dumpBytecode bool // if set, mtail prints the program bytecode after code generation

	overrideLocation     *time.Location // Timezone location to use when parsing timestamps
	staleLogGcWaker      waker.Waker    // Wake to run stale log gc
	logPatternPollWaker  waker.Waker    // Wake to poll for log patterns
	logstreamPollWaker   waker.Waker    // Wake idle logstreams to poll sfor new data
	metricPushInterval   time.Duration  // Interval between metric pushes
	syslogUseCurrentYear bool           // if set, use the current year for timestamps that have no year information
	omitMetricSource     bool           // if set, do not link the source program to a metric
	omitProgLabel        bool           // if set, do not put the program name in the metric labels
	emitMetricTimestamp  bool           // if set, emit the metric's recorded timestamp
	omitDumpMetricsStore bool           // if set, do not print the metric store; useful in test
}

// initLoader constructs a new program loader and performs the initial load of program files in the program directory.
func (m *Server) initLoader() error {
	opts := []vm.Option{
		vm.PrometheusRegisterer(m.reg),
	}
	if m.compileOnly {
		opts = append(opts, vm.CompileOnly())
	}
	if m.oneShot {
		opts = append(opts, vm.ErrorsAbort())
	}
	if m.dumpAst {
		opts = append(opts, vm.DumpAst())
	}
	if m.dumpAstTypes {
		opts = append(opts, vm.DumpAstTypes())
	}
	if m.dumpBytecode {
		opts = append(opts, vm.DumpBytecode())
	}
	if m.syslogUseCurrentYear {
		opts = append(opts, vm.SyslogUseCurrentYear())
	}
	if m.omitMetricSource {
		opts = append(opts, vm.OmitMetricSource())
	}
	if m.overrideLocation != nil {
		opts = append(opts, vm.OverrideLocation(m.overrideLocation))
	}
	var err error
	m.l, err = vm.NewLoader(m.lines, &m.wg, m.programPath, m.store, opts...)
	if err != nil {
		return err
	}
	return nil
}

// initExporter sets up an Exporter for this Server.
func (m *Server) initExporter() (err error) {
	if m.oneShot {
		// This is a hack to avoid a race in test, but assume that in oneshot
		// mode we don't want to export anything.
		return nil
	}
	opts := []exporter.Option{}
	if m.omitProgLabel {
		opts = append(opts, exporter.OmitProgLabel())
	}
	if m.emitMetricTimestamp {
		opts = append(opts, exporter.EmitTimestamp())
	}
	if m.metricPushInterval > 0 {
		opts = append(opts, exporter.PushInterval(m.metricPushInterval))
	}
	m.e, err = exporter.New(m.ctx, &m.wg, m.store, opts...)
	if err != nil {
		return err
	}
	m.reg.MustRegister(m.e)

	// Create mtail_build_info metric.
	version.Branch = m.buildInfo.Branch
	version.Version = m.buildInfo.Version
	version.Revision = m.buildInfo.Revision
	m.reg.MustRegister(version.NewCollector("mtail"))
	return nil
}

// initTailer sets up and starts a Tailer for this Server.
func (m *Server) initTailer() (err error) {
	opts := []tailer.Option{
		tailer.IgnoreRegex(m.ignoreRegexPattern),
		tailer.LogPatterns(m.logPathPatterns),
		tailer.LogPatternPollWaker(m.logPatternPollWaker),
		tailer.StaleLogGcWaker(m.staleLogGcWaker),
		tailer.LogstreamPollWaker(m.logstreamPollWaker),
	}
	if m.oneShot {
		opts = append(opts, tailer.OneShot)
	}
	m.t, err = tailer.New(m.ctx, &m.wg, m.lines, opts...)
	return
}

// initHttpServer begins the http server.
func (m *Server) initHttpServer() error {
	initDone := make(chan struct{})
	defer close(initDone)

	if m.listener == nil {
		glog.Info("no listen address configured, not starting http server")
		return nil
	}

	mux := http.NewServeMux()
	mux.HandleFunc("/favicon.ico", FaviconHandler)
	mux.Handle("/", m)
	mux.Handle("/progz", http.HandlerFunc(m.l.ProgzHandler))
	mux.HandleFunc("/json", http.HandlerFunc(m.e.HandleJSON))
	mux.Handle("/metrics", promhttp.HandlerFor(m.reg, promhttp.HandlerOpts{}))
	mux.HandleFunc("/varz", http.HandlerFunc(m.e.HandleVarz))
	mux.Handle("/debug/vars", expvar.Handler())
	mux.HandleFunc("/debug/pprof/", pprof.Index)
	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
	zpages.Handle(mux, "/")

	srv := &http.Server{
		Handler: mux,
	}

	var wg sync.WaitGroup
	errc := make(chan error, 1)

	// This goroutine runs the http server.
	wg.Add(1)
	go func() {
		defer wg.Done()
		<-initDone
		glog.Infof("Listening on %s", m.listener.Addr())
		if err := srv.Serve(m.listener); err != nil && err != http.ErrServerClosed {
			errc <- err
		}
	}()

	// This goroutine manages http server shutdown.
	go func() {
		<-initDone
		select {
		case err := <-errc:
			glog.Info(err)
		case <-m.ctx.Done():
			glog.Info("Shutdown requested.")
			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			srv.SetKeepAlivesEnabled(false)
			if err := srv.Shutdown(ctx); err != nil {
				glog.Info(err)
			}
		}
		// Wait for the Serve routine to exit.
		wg.Wait()
	}()

	return nil
}

// New creates a Server from the supplied Options.  The Server is started by
// the time New returns, it watches the LogPatterns for files, starts tailing
// their changes and sends any new lines found to the virtual machines loaded
// from ProgramPath. If OneShot mode is enabled, it will exit after reading
// each log file from start to finish.
// TODO(jaq): this doesn't need to be a constructor anymore, it could start and
// block until quit, once TestServer.PollWatched is addressed.
func New(ctx context.Context, store *metrics.Store, options ...Option) (*Server, error) {
	m := &Server{
		ctx:   ctx,
		store: store,
		lines: make(chan *logline.LogLine),
		// Using a non-pedantic registry means we can be looser with metrics that
		// are not fully specified at startup.
		reg: prometheus.NewRegistry(),
	}

	// TODO(jaq): Should these move to initExporter?
	expvarDescs := map[string]*prometheus.Desc{
		// internal/tailer/file.go
		"log_errors_total":    prometheus.NewDesc("log_errors_total", "number of IO errors encountered per log file", []string{"logfile"}, nil),
		"log_rotations_total": prometheus.NewDesc("log_rotations_total", "number of log rotation events per log file", []string{"logfile"}, nil),
		"log_truncates_total": prometheus.NewDesc("log_truncates_total", "number of log truncation events log file", []string{"logfile"}, nil),
		"log_lines_total":     prometheus.NewDesc("log_lines_total", "number of lines read per log file", []string{"logfile"}, nil),
		// internal/vm/loader.go
		"lines_total":               prometheus.NewDesc("lines_total", "number of lines received by the program loader", nil, nil),
		"prog_loads_total":          prometheus.NewDesc("prog_loads_total", "number of program load events by program source filename", []string{"prog"}, nil),
		"prog_load_errors_total":    prometheus.NewDesc("prog_load_errors_total", "number of errors encountered when loading per program source filename", []string{"prog"}, nil),
		"prog_runtime_errors_total": prometheus.NewDesc("prog_runtime_errors_total", "number of errors encountered when executing programs per source filename", []string{"prog"}, nil),
	}
	m.reg.MustRegister(
		prometheus.NewGoCollector(),
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
	// Prefix all expvar metrics with 'mtail_'
	prometheus.WrapRegistererWithPrefix("mtail_", m.reg).MustRegister(
		prometheus.NewExpvarCollector(expvarDescs))
	if err := m.SetOption(options...); err != nil {
		return nil, err
	}
	if err := m.initExporter(); err != nil {
		return nil, err
	}
	if err := m.initLoader(); err != nil {
		return nil, err
	}
	if err := m.initTailer(); err != nil {
		return nil, err
	}
	if err := m.initHttpServer(); err != nil {
		return nil, err
	}
	return m, nil
}

// SetOption takes one or more option functions and applies them in order to MtailServer.
func (m *Server) SetOption(options ...Option) error {
	for _, option := range options {
		if err := option.apply(m); err != nil {
			return err
		}
	}
	return nil
}

// Run awaits mtail's shutdown.
// TODO(jaq): remove this once the test server is able to trigger polls on the components.
func (m *Server) Run() error {
	m.wg.Wait()
	if m.compileOnly {
		glog.Info("compile-only is set, exiting")
		return nil
	}
	if m.oneShot {
		if m.omitDumpMetricsStore {
			glog.Info("Store dump disabled, exiting")
			return nil
		}
		if err := m.store.WriteMetrics(os.Stdout); err != nil {
			return err
		}
		return nil
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"crypto/rand"

	klog "k8s.io/klog/v2"
)

const (
	mapperFilePrefix     = "luks-rbd-"
	mapperFilePathPrefix = "/dev/mapper"

	// Encryption passphrase location in K8s secrets
	encryptionPassphraseKey = "encryptionPassphrase"
	kmsTypeKey              = "encryptionKMSType"

	// Default KMS type
	defaultKMSType = "default"

	// kmsConfigPath is the location of the vault config file
	kmsConfigPath = "/etc/ceph-csi-encryption-kms-config/config.json"

	// Passphrase size - 20 bytes is 160 bits to satisfy:
	// https://tools.ietf.org/html/rfc6749#section-10.10
	encryptionPassphraseSize = 20
)

// EncryptionKMS provides external Key Management System for encryption
// passphrases storage.
type EncryptionKMS interface {
	GetPassphrase(key string) (string, error)
	SavePassphrase(key, value string) error
	DeletePassphrase(key string) error
	GetID() string
}

// MissingPassphrase is an error instructing to generate new passphrase.
type MissingPassphrase struct {
	error
}

// SecretsKMS is default KMS implementation that means no KMS is in use.
type SecretsKMS struct {
	passphrase string
}

func initSecretsKMS(secrets map[string]string) (EncryptionKMS, error) {
	passphraseValue, ok := secrets[encryptionPassphraseKey]
	if !ok {
		return nil, errors.New("missing encryption passphrase in secrets")
	}
	return SecretsKMS{passphrase: passphraseValue}, nil
}

// GetPassphrase returns passphrase from Kubernetes secrets.
func (kms SecretsKMS) GetPassphrase(key string) (string, error) {
	return kms.passphrase, nil
}

// SavePassphrase is not implemented.
func (kms SecretsKMS) SavePassphrase(key, value string) error {
	return fmt.Errorf("save new passphrase is not implemented for Kubernetes secrets")
}

// DeletePassphrase is doing nothing as no new passphrases are saved with
// SecretsKMS.
func (kms SecretsKMS) DeletePassphrase(key string) error {
	return nil
}

// GetID is returning ID representing default KMS `default`.
func (kms SecretsKMS) GetID() string {
	return defaultKMSType
}

// GetKMS returns an instance of Key Management System.
func GetKMS(kmsID string, secrets map[string]string) (EncryptionKMS, error) {
	if kmsID == "" || kmsID == defaultKMSType {
		return initSecretsKMS(secrets)
	}

	// #nosec
	content, err := ioutil.ReadFile(kmsConfigPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read kms configuration from %s: %s",
			kmsConfigPath, err)
	}

	var config map[string]interface{}
	err = json.Unmarshal(content, &config)
	if err != nil {
		return nil, fmt.Errorf("failed to parse kms configuration: %s", err)
	}

	kmsConfigData, ok := config[kmsID].(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("missing encryption KMS configuration with %s", kmsID)
	}
	kmsConfig := make(map[string]string)
	for key, value := range kmsConfigData {
		kmsConfig[key], ok = value.(string)
		if !ok {
			return nil, fmt.Errorf("broken KMS config: '%s' for '%s' is not a string",
				value, key)
		}
	}

	kmsType, ok := kmsConfig[kmsTypeKey]
	if !ok {
		return nil, fmt.Errorf("encryption KMS configuration for %s is missing KMS type", kmsID)
	}

	if kmsType == "vault" {
		return InitVaultKMS(kmsID, kmsConfig, secrets)
	}
	return nil, fmt.Errorf("unknown encryption KMS type %s", kmsType)
}

// GetCryptoPassphrase Retrieves passphrase to encrypt volume.
func GetCryptoPassphrase(ctx context.Context, volumeID string, kms EncryptionKMS) (string, error) {
	passphrase, err := kms.GetPassphrase(volumeID)
	if err == nil {
		return passphrase, nil
	}
	if _, ok := err.(MissingPassphrase); ok {
		DebugLog(ctx, "Encryption passphrase is missing for %s. Generating a new one",
			volumeID)
		passphrase, err = generateNewEncryptionPassphrase()
		if err != nil {
			return "", fmt.Errorf("failed to generate passphrase for %s: %s", volumeID, err)
		}
		err = kms.SavePassphrase(volumeID, passphrase)
		if err != nil {
			return "", fmt.Errorf("failed to save the passphrase for %s: %s", volumeID, err)
		}
		return passphrase, nil
	}
	klog.Errorf(Log(ctx, "failed to get encryption passphrase for %s: %s"), volumeID, err)
	return "", err
}

// generateNewEncryptionPassphrase generates a random passphrase for encryption.
func generateNewEncryptionPassphrase() (string, error) {
	bytesPassphrase := make([]byte, encryptionPassphraseSize)
	_, err := rand.Read(bytesPassphrase)
	if err != nil {
		return "", err
	}
	return base64.URLEncoding.EncodeToString(bytesPassphrase), nil
}

// VolumeMapper returns file name and it's path to where encrypted device should be open.
func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) {
	mapperFile = mapperFilePrefix + volumeID
	mapperFilePath = path.Join(mapperFilePathPrefix, mapperFile)
	return mapperFile, mapperFilePath
}

// EncryptVolume encrypts provided device with LUKS.
func EncryptVolume(ctx context.Context, devicePath, passphrase string) error {
	DebugLog(ctx, "Encrypting device %s with LUKS", devicePath)
	if _, _, err := LuksFormat(devicePath, passphrase); err != nil {
		return fmt.Errorf("failed to encrypt device %s with LUKS: %w", devicePath, err)
	}
	return nil
}

// OpenEncryptedVolume opens volume so that it can be used by the client.
func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase string) error {
	DebugLog(ctx, "Opening device %s with LUKS on %s", devicePath, mapperFile)
	_, _, err := LuksOpen(devicePath, mapperFile, passphrase)
	return err
}

// CloseEncryptedVolume closes encrypted volume so it can be detached.
func CloseEncryptedVolume(ctx context.Context, mapperFile string) error {
	DebugLog(ctx, "Closing LUKS device %s", mapperFile)
	_, _, err := LuksClose(mapperFile)
	return err
}

// IsDeviceOpen determines if encrypted device is already open.
func IsDeviceOpen(ctx context.Context, device string) (bool, error) {
	_, mappedFile, err := DeviceEncryptionStatus(ctx, device)
	return (mappedFile != ""), err
}

// DeviceEncryptionStatus looks to identify if the passed device is a LUKS mapping
// and if so what the device is and the mapper name as used by LUKS.
// If not, just returns the original device and an empty string.
func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevice, mapper string, err error) {
	if !strings.HasPrefix(devicePath, mapperFilePathPrefix) {
		return devicePath, "", nil
	}
	mapPath := strings.TrimPrefix(devicePath, mapperFilePathPrefix+"/")
	stdout, _, err := LuksStatus(mapPath)
	if err != nil {
		DebugLog(ctx, "device %s is not an active LUKS device: %v", devicePath, err)
		return devicePath, "", nil
	}
	lines := strings.Split(string(stdout), "\n")
	if len(lines) < 1 {
		return "", "", fmt.Errorf("device encryption status returned no stdout for %s", devicePath)
	}
	if !strings.HasSuffix(lines[0], " is active.") {
		// Implies this is not a LUKS device
		return devicePath, "", nil
	}
	for i := 1; i < len(lines); i++ {
		kv := strings.SplitN(strings.TrimSpace(lines[i]), ":", 2)
		if len(kv) < 1 {
			return "", "", fmt.Errorf("device encryption status output for %s is badly formatted: %s",
				devicePath, lines[i])
		}
		if strings.Compare(kv[0], "device") == 0 {
			return strings.TrimSpace(kv[1]), mapPath, nil
		}
	}
	// Identified as LUKS, but failed to identify a mapped device
	return "", "", fmt.Errorf("mapped device not found in path %s", devicePath)
}
<commit_msg>util: use local ErrorLog() for GetCryptoPassphrase() instead of klog<commit_after>/*
Copyright 2019 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"crypto/rand"
)

const (
	mapperFilePrefix     = "luks-rbd-"
	mapperFilePathPrefix = "/dev/mapper"

	// Encryption passphrase location in K8s secrets
	encryptionPassphraseKey = "encryptionPassphrase"
	kmsTypeKey              = "encryptionKMSType"

	// Default KMS type
	defaultKMSType = "default"

	// kmsConfigPath is the location of the vault config file
	kmsConfigPath = "/etc/ceph-csi-encryption-kms-config/config.json"

	// Passphrase size - 20 bytes is 160 bits to satisfy:
	// https://tools.ietf.org/html/rfc6749#section-10.10
	encryptionPassphraseSize = 20
)

// EncryptionKMS provides external Key Management System for encryption
// passphrases storage.
type EncryptionKMS interface {
	GetPassphrase(key string) (string, error)
	SavePassphrase(key, value string) error
	DeletePassphrase(key string) error
	GetID() string
}

// MissingPassphrase is an error instructing to generate new passphrase.
type MissingPassphrase struct {
	error
}

// SecretsKMS is default KMS implementation that means no KMS is in use.
type SecretsKMS struct {
	passphrase string
}

func initSecretsKMS(secrets map[string]string) (EncryptionKMS, error) {
	passphraseValue, ok := secrets[encryptionPassphraseKey]
	if !ok {
		return nil, errors.New("missing encryption passphrase in secrets")
	}
	return SecretsKMS{passphrase: passphraseValue}, nil
}

// GetPassphrase returns passphrase from Kubernetes secrets.
func (kms SecretsKMS) GetPassphrase(key string) (string, error) {
	return kms.passphrase, nil
}

// SavePassphrase is not implemented.
func (kms SecretsKMS) SavePassphrase(key, value string) error {
	return fmt.Errorf("save new passphrase is not implemented for Kubernetes secrets")
}

// DeletePassphrase is doing nothing as no new passphrases are saved with
// SecretsKMS.
func (kms SecretsKMS) DeletePassphrase(key string) error {
	return nil
}

// GetID is returning ID representing default KMS `default`.
func (kms SecretsKMS) GetID() string {
	return defaultKMSType
}

// GetKMS returns an instance of Key Management System.
func GetKMS(kmsID string, secrets map[string]string) (EncryptionKMS, error) {
	if kmsID == "" || kmsID == defaultKMSType {
		return initSecretsKMS(secrets)
	}

	// #nosec
	content, err := ioutil.ReadFile(kmsConfigPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read kms configuration from %s: %s",
			kmsConfigPath, err)
	}

	var config map[string]interface{}
	err = json.Unmarshal(content, &config)
	if err != nil {
		return nil, fmt.Errorf("failed to parse kms configuration: %s", err)
	}

	kmsConfigData, ok := config[kmsID].(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("missing encryption KMS configuration with %s", kmsID)
	}
	kmsConfig := make(map[string]string)
	for key, value := range kmsConfigData {
		kmsConfig[key], ok = value.(string)
		if !ok {
			return nil, fmt.Errorf("broken KMS config: '%s' for '%s' is not a string",
				value, key)
		}
	}

	kmsType, ok := kmsConfig[kmsTypeKey]
	if !ok {
		return nil, fmt.Errorf("encryption KMS configuration for %s is missing KMS type", kmsID)
	}

	if kmsType == "vault" {
		return InitVaultKMS(kmsID, kmsConfig, secrets)
	}
	return nil, fmt.Errorf("unknown encryption KMS type %s", kmsType)
}

// GetCryptoPassphrase Retrieves passphrase to encrypt volume.
func GetCryptoPassphrase(ctx context.Context, volumeID string, kms EncryptionKMS) (string, error) {
	passphrase, err := kms.GetPassphrase(volumeID)
	if err == nil {
		return passphrase, nil
	}
	if _, ok := err.(MissingPassphrase); ok {
		DebugLog(ctx, "Encryption passphrase is missing for %s. Generating a new one",
			volumeID)
		passphrase, err = generateNewEncryptionPassphrase()
		if err != nil {
			return "", fmt.Errorf("failed to generate passphrase for %s: %s", volumeID, err)
		}
		err = kms.SavePassphrase(volumeID, passphrase)
		if err != nil {
			return "", fmt.Errorf("failed to save the passphrase for %s: %s", volumeID, err)
		}
		return passphrase, nil
	}
	ErrorLog(Log(ctx, "failed to get encryption passphrase for %s: %s"), volumeID, err)
	return "", err
}

// generateNewEncryptionPassphrase generates a random passphrase for encryption.
func generateNewEncryptionPassphrase() (string, error) {
	bytesPassphrase := make([]byte, encryptionPassphraseSize)
	_, err := rand.Read(bytesPassphrase)
	if err != nil {
		return "", err
	}
	return base64.URLEncoding.EncodeToString(bytesPassphrase), nil
}

// VolumeMapper returns file name and it's path to where encrypted device should be open.
func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) {
	mapperFile = mapperFilePrefix + volumeID
	mapperFilePath = path.Join(mapperFilePathPrefix, mapperFile)
	return mapperFile, mapperFilePath
}

// EncryptVolume encrypts provided device with LUKS.
func EncryptVolume(ctx context.Context, devicePath, passphrase string) error {
	DebugLog(ctx, "Encrypting device %s with LUKS", devicePath)
	if _, _, err := LuksFormat(devicePath, passphrase); err != nil {
		return fmt.Errorf("failed to encrypt device %s with LUKS: %w", devicePath, err)
	}
	return nil
}

// OpenEncryptedVolume opens volume so that it can be used by the client.
func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase string) error {
	DebugLog(ctx, "Opening device %s with LUKS on %s", devicePath, mapperFile)
	_, _, err := LuksOpen(devicePath, mapperFile, passphrase)
	return err
}

// CloseEncryptedVolume closes encrypted volume so it can be detached.
func CloseEncryptedVolume(ctx context.Context, mapperFile string) error {
	DebugLog(ctx, "Closing LUKS device %s", mapperFile)
	_, _, err := LuksClose(mapperFile)
	return err
}

// IsDeviceOpen determines if encrypted device is already open.
func IsDeviceOpen(ctx context.Context, device string) (bool, error) {
	_, mappedFile, err := DeviceEncryptionStatus(ctx, device)
	return (mappedFile != ""), err
}

// DeviceEncryptionStatus looks to identify if the passed device is a LUKS mapping
// and if so what the device is and the mapper name as used by LUKS.
// If not, just returns the original device and an empty string.
func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevice, mapper string, err error) {
	if !strings.HasPrefix(devicePath, mapperFilePathPrefix) {
		return devicePath, "", nil
	}
	mapPath := strings.TrimPrefix(devicePath, mapperFilePathPrefix+"/")
	stdout, _, err := LuksStatus(mapPath)
	if err != nil {
		DebugLog(ctx, "device %s is not an active LUKS device: %v", devicePath, err)
		return devicePath, "", nil
	}
	lines := strings.Split(string(stdout), "\n")
	if len(lines) < 1 {
		return "", "", fmt.Errorf("device encryption status returned no stdout for %s", devicePath)
	}
	if !strings.HasSuffix(lines[0], " is active.") {
		// Implies this is not a LUKS device
		return devicePath, "", nil
	}
	for i := 1; i < len(lines); i++ {
		kv := strings.SplitN(strings.TrimSpace(lines[i]), ":", 2)
		if len(kv) < 1 {
			return "", "", fmt.Errorf("device encryption status output for %s is badly formatted: %s",
				devicePath, lines[i])
		}
		if strings.Compare(kv[0], "device") == 0 {
			return strings.TrimSpace(kv[1]), mapPath, nil
		}
	}
	// Identified as LUKS, but failed to identify a mapped device
	return "", "", fmt.Errorf("mapped device not found in path %s", devicePath)
}
<|endoftext|>


<commit_before>package turnservicecli

// CredentialsResponse defines a REST response containing TURN data.
type CredentialsResponse struct {
	Success bool             `json:"success"`
	Nonce   string           `json:"nonce"`
	Turn    *CredentialsData `json:"turn"`
	Session string           `json:"session,omitempty"`
}

// CredentialsData defines TURN credentials with servers.
type CredentialsData struct {
	TTL      int64         `json:"ttl"`
	Username string        `json:"username"`
	Password string        `json:"password"`
	Servers  []*URNsWithID `json:"servers,omitempty"`
	GeoURI   string        `json:"geo_uri,omitempty"`
}

// URNsWithID defines TURN servers groups with ID.
type URNsWithID struct {
	ID   string   `json:"id"`
	URNs []string `json:"urns"`
	Prio int      `json:"prio"`
}

// GeoResponse defines a REST response containing TURN geo
type GeoResponse struct {
	Success bool     `json:"success"`
	Nonce   string   `json:"nonce"`
	Geo     *GeoData `json:"geo,omitempty"`
}

// GeoData defines ordered TURN IDs
type GeoData struct {
	Prefer []string `json:"prefer"`
}
<commit_msg>Fix punctuation in comments<commit_after>package turnservicecli

// CredentialsResponse defines a REST response containing TURN data.
type CredentialsResponse struct {
	Success bool             `json:"success"`
	Nonce   string           `json:"nonce"`
	Turn    *CredentialsData `json:"turn"`
	Session string           `json:"session,omitempty"`
}

// CredentialsData defines TURN credentials with servers.
type CredentialsData struct {
	TTL      int64         `json:"ttl"`
	Username string        `json:"username"`
	Password string        `json:"password"`
	Servers  []*URNsWithID `json:"servers,omitempty"`
	GeoURI   string        `json:"geo_uri,omitempty"`
}

// URNsWithID defines TURN servers groups with ID.
type URNsWithID struct {
	ID   string   `json:"id"`
	URNs []string `json:"urns"`
	Prio int      `json:"prio"`
}

// GeoResponse defines a REST response containing TURN geo.
type GeoResponse struct {
	Success bool     `json:"success"`
	Nonce   string   `json:"nonce"`
	Geo     *GeoData `json:"geo,omitempty"`
}

// GeoData defines ordered TURN IDs.
type GeoData struct {
	Prefer []string `json:"prefer"`
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unsharded

import (
	"context"
	"flag"
	"fmt"
	"os"
	"testing"

	"vitess.io/vitess/go/test/utils"

	"vitess.io/vitess/go/vt/log"

	"github.com/google/go-cmp/cmp"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/test/endtoend/cluster"
)

var (
	clusterInstance *cluster.LocalProcessCluster
	cell            = "zone1"
	hostname        = "localhost"
	KeyspaceName    = "customer"
	SchemaSQL       = `
CREATE TABLE t1 (
    c1 BIGINT NOT NULL,
    c2 BIGINT NOT NULL,
    c3 BIGINT,
    c4 varchar(100),
    PRIMARY KEY (c1),
    UNIQUE KEY (c2),
    UNIQUE KEY (c3),
    UNIQUE KEY (c4)
) ENGINE=Innodb;

CREATE TABLE allDefaults (
  id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
  name VARCHAR(255)
) ENGINE=Innodb;`
	VSchema = `
{
    "sharded": false,
    "tables": {
        "t1": {
            "columns": [
                {
                    "name": "c1",
                    "type": "INT64"
                },
                {
                    "name": "c2",
                    "type": "INT64"
                },
                {
                    "name": "c3",
                    "type": "INT64"
                },
                {
                    "name": "c4",
                    "type": "VARCHAR"
                }
            ]
        },
        "allDefaults": {
            "columns": [
                {
                    "name": "id",
                    "type": "INT64"
                },
                {
                    "name": "name",
                    "type": "VARCHAR"
                }
            ]
        }
    }
}
`

	createProcSQL = `use vt_customer;
CREATE PROCEDURE GetAllT1Twice()
BEGIN
	SELECT * FROM t1;
	SELECT * FROM t1;
END;
CREATE PROCEDURE sp_insert()
BEGIN
	insert into allDefaults () values ();
END;
CREATE PROCEDURE sp_delete()
BEGIN
	delete from t1;
END;
CREATE PROCEDURE sp_random()
BEGIN
	insert into allDefaults () values ();
    select * from allDefaults;
	delete from allDefaults;
    set autocommit = 0;
END;
`
)

func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitCode := func() int {
		clusterInstance = cluster.NewCluster(cell, hostname)
		defer clusterInstance.Teardown()

		// Start topo server
		if err := clusterInstance.StartTopo(); err != nil {
			return 1
		}

		// Start keyspace
		Keyspace := &cluster.Keyspace{
			Name:      KeyspaceName,
			SchemaSQL: SchemaSQL,
			VSchema:   VSchema,
		}
		if err := clusterInstance.StartUnshardedKeyspace(*Keyspace, 0, false); err != nil {
			log.Fatal(err.Error())
			return 1
		}

		// Start vtgate
		if err := clusterInstance.StartVtgate(); err != nil {
			log.Fatal(err.Error())
			return 1
		}

		masterProcess := clusterInstance.Keyspaces[0].Shards[0].MasterTablet().VttabletProcess
		if _, err := masterProcess.QueryTablet(createProcSQL, KeyspaceName, false); err != nil {
			log.Fatal(err.Error())
			return 1
		}

		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestSelectIntoAndLoadFrom(t *testing.T) {
	// Test is skipped because it requires secure-file-priv variable to be set to not NULL or empty.
	t.Skip()
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.Nil(t, err)
	defer conn.Close()

	defer exec(t, conn, `delete from t1`)
	exec(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'abc')`)
	res := exec(t, conn, `select @@secure_file_priv;`)
	directory := res.Rows[0][0].ToString()
	query := `select * from t1 into outfile '` + directory + `x.txt'`
	exec(t, conn, query)
	defer os.Remove(directory + `x.txt`)
	query = `load data infile '` + directory + `x.txt' into table t1`
	execAssertError(t, conn, query, "Duplicate entry '300' for key 'PRIMARY'")
	exec(t, conn, `delete from t1`)
	exec(t, conn, query)
	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)]]`)
	query = `select * from t1 into dumpfile '` + directory + `x1.txt'`
	exec(t, conn, query)
	defer os.Remove(directory + `x1.txt`)
	query = `select * from t1 into outfile '` + directory + `x2.txt' Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`
	exec(t, conn, query)
	defer os.Remove(directory + `x2.txt`)
	query = `load data infile '` + directory + `x2.txt' replace into table t1 Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`
	exec(t, conn, query)
	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)]]`)
}

func TestCallProcedure(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host:  "localhost",
		Port:  clusterInstance.VtgateMySQLPort,
		Flags: mysql.CapabilityClientMultiResults,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.NoError(t, err)
	defer conn.Close()
	defer exec(t, conn, `delete from t1`)
	exec(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'foo'),(301,101,301,'bar')`)
	results := execMulti(t, conn, `CALL GetAllT1Twice()`)
	require.Equal(t, 2, len(results), "didn't get the two expected result sets")
	utils.MustMatch(t, `[[INT64(300) INT64(100) INT64(300) VARCHAR("foo")] [INT64(301) INT64(101) INT64(301) VARCHAR("bar")]]`, fmt.Sprintf("%v", results[0].Rows))
	utils.MustMatch(t, `[[INT64(300) INT64(100) INT64(300) VARCHAR("foo")] [INT64(301) INT64(101) INT64(301) VARCHAR("bar")]]`, fmt.Sprintf("%v", results[1].Rows))
}

func TestEmptyStatement(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.Nil(t, err)
	defer conn.Close()
	defer exec(t, conn, `delete from t1`)
	execAssertError(t, conn, " \t;", "Query was empty")
	execMulti(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'abc'); ;; insert into t1(c1, c2, c3, c4) values (301,101,301,'abcd');;`)
	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)] [INT64(301) INT64(101) INT64(301)]]`)
}

func TestInsertAllDefaults(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.NoError(t, err)
	defer conn.Close()

	exec(t, conn, `insert into allDefaults () values ()`)
	assertMatches(t, conn, `select * from allDefaults`, "[[INT64(1) NULL]]")
}

func TestDDLUnsharded(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.NoError(t, err)
	defer conn.Close()

	exec(t, conn, `create table tempt1(c1 BIGINT NOT NULL,c2 BIGINT NOT NULL,c3 BIGINT,c4 varchar(100),PRIMARY KEY (c1), UNIQUE KEY (c2),UNIQUE KEY (c3), UNIQUE KEY (c4))`)
	// Test that create view works and the output is as expected
	exec(t, conn, `create view v1 as select * from tempt1`)
	exec(t, conn, `insert into tempt1(c1, c2, c3, c4) values (300,100,300,'abc'),(30,10,30,'ac'),(3,0,3,'a')`)
	assertMatches(t, conn, "select * from v1", `[[INT64(3) INT64(0) INT64(3) VARCHAR("a")] [INT64(30) INT64(10) INT64(30) VARCHAR("ac")] [INT64(300) INT64(100) INT64(300) VARCHAR("abc")]]`)
	exec(t, conn, `drop view v1`)
	exec(t, conn, `drop table tempt1`)
	assertMatches(t, conn, "show tables", `[[VARCHAR("allDefaults")] [VARCHAR("t1")]]`)
}

func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result {
	t.Helper()
	var res []*sqltypes.Result
	qr, more, err := conn.ExecuteFetchMulti(query, 1000, true)
	res = append(res, qr)
	require.NoError(t, err)
	for more == true {
		qr, more, _, err = conn.ReadQueryResult(1000, true)
		require.NoError(t, err)
		res = append(res, qr)
	}
	return res
}

func execAssertError(t *testing.T, conn *mysql.Conn, query string, errorString string) {
	t.Helper()
	_, err := conn.ExecuteFetch(query, 1000, true)
	require.Error(t, err)
	assert.Contains(t, err.Error(), errorString)
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := exec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s", query, diff)
	}
}
<commit_msg>fix vtgate e2e test<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package unsharded

import (
	"context"
	"flag"
	"fmt"
	"os"
	"testing"

	"vitess.io/vitess/go/vt/log"

	"github.com/google/go-cmp/cmp"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/test/endtoend/cluster"
)

var (
	clusterInstance *cluster.LocalProcessCluster
	cell            = "zone1"
	hostname        = "localhost"
	KeyspaceName    = "customer"
	SchemaSQL       = `
CREATE TABLE t1 (
    c1 BIGINT NOT NULL,
    c2 BIGINT NOT NULL,
    c3 BIGINT,
    c4 varchar(100),
    PRIMARY KEY (c1),
    UNIQUE KEY (c2),
    UNIQUE KEY (c3),
    UNIQUE KEY (c4)
) ENGINE=Innodb;

CREATE TABLE allDefaults (
  id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
  name VARCHAR(255)
) ENGINE=Innodb;`
	VSchema = `
{
    "sharded": false,
    "tables": {
        "t1": {
            "columns": [
                {
                    "name": "c1",
                    "type": "INT64"
                },
                {
                    "name": "c2",
                    "type": "INT64"
                },
                {
                    "name": "c3",
                    "type": "INT64"
                },
                {
                    "name": "c4",
                    "type": "VARCHAR"
                }
            ]
        },
        "allDefaults": {
            "columns": [
                {
                    "name": "id",
                    "type": "INT64"
                },
                {
                    "name": "name",
                    "type": "VARCHAR"
                }
            ]
        }
    }
}
`

	createProcSQL = `use vt_customer;
CREATE PROCEDURE sp_insert()
BEGIN
	insert into allDefaults () values ();
END;

CREATE PROCEDURE sp_delete()
BEGIN
	delete from allDefaults;
END;

CREATE PROCEDURE sp_multi_dml()
BEGIN
	insert into allDefaults () values ();
	delete from allDefaults;
END;

CREATE PROCEDURE sp_variable()
BEGIN
	insert into allDefaults () values ();
	SELECT min(id) INTO @myvar FROM allDefaults;
	DELETE FROM allDefaults WHERE id = @myvar;
END;

CREATE PROCEDURE sp_select()
BEGIN
	SELECT * FROM allDefaults;
END;

CREATE PROCEDURE sp_all()
BEGIN
	insert into allDefaults () values ();
    select * from allDefaults;
	delete from allDefaults;
    set autocommit = 0;
END;
`
)

func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitCode := func() int {
		clusterInstance = cluster.NewCluster(cell, hostname)
		defer clusterInstance.Teardown()

		// Start topo server
		if err := clusterInstance.StartTopo(); err != nil {
			return 1
		}

		// Start keyspace
		Keyspace := &cluster.Keyspace{
			Name:      KeyspaceName,
			SchemaSQL: SchemaSQL,
			VSchema:   VSchema,
		}
		if err := clusterInstance.StartUnshardedKeyspace(*Keyspace, 0, false); err != nil {
			log.Fatal(err.Error())
			return 1
		}

		// Start vtgate
		if err := clusterInstance.StartVtgate(); err != nil {
			log.Fatal(err.Error())
			return 1
		}

		masterProcess := clusterInstance.Keyspaces[0].Shards[0].MasterTablet().VttabletProcess
		if _, err := masterProcess.QueryTablet(createProcSQL, KeyspaceName, false); err != nil {
			log.Fatal(err.Error())
			return 1
		}

		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestSelectIntoAndLoadFrom(t *testing.T) {
	// Test is skipped because it requires secure-file-priv variable to be set to not NULL or empty.
	t.Skip()
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.Nil(t, err)
	defer conn.Close()

	defer exec(t, conn, `delete from t1`)
	exec(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'abc')`)
	res := exec(t, conn, `select @@secure_file_priv;`)
	directory := res.Rows[0][0].ToString()
	query := `select * from t1 into outfile '` + directory + `x.txt'`
	exec(t, conn, query)
	defer os.Remove(directory + `x.txt`)
	query = `load data infile '` + directory + `x.txt' into table t1`
	execAssertError(t, conn, query, "Duplicate entry '300' for key 'PRIMARY'")
	exec(t, conn, `delete from t1`)
	exec(t, conn, query)
	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)]]`)
	query = `select * from t1 into dumpfile '` + directory + `x1.txt'`
	exec(t, conn, query)
	defer os.Remove(directory + `x1.txt`)
	query = `select * from t1 into outfile '` + directory + `x2.txt' Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`
	exec(t, conn, query)
	defer os.Remove(directory + `x2.txt`)
	query = `load data infile '` + directory + `x2.txt' replace into table t1 Fields terminated by ';' optionally enclosed by '"' escaped by '\t' lines terminated by '\n'`
	exec(t, conn, query)
	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)]]`)
}

func TestEmptyStatement(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.Nil(t, err)
	defer conn.Close()
	defer exec(t, conn, `delete from t1`)
	execAssertError(t, conn, " \t;", "Query was empty")
	execMulti(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'abc'); ;; insert into t1(c1, c2, c3, c4) values (301,101,301,'abcd');;`)
	assertMatches(t, conn, `select c1,c2,c3 from t1`, `[[INT64(300) INT64(100) INT64(300)] [INT64(301) INT64(101) INT64(301)]]`)
}

func TestInsertAllDefaults(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.NoError(t, err)
	defer conn.Close()

	exec(t, conn, `insert into allDefaults () values ()`)
	assertMatches(t, conn, `select * from allDefaults`, "[[INT64(1) NULL]]")
}

func TestDDLUnsharded(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host: "localhost",
		Port: clusterInstance.VtgateMySQLPort,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.NoError(t, err)
	defer conn.Close()

	exec(t, conn, `create table tempt1(c1 BIGINT NOT NULL,c2 BIGINT NOT NULL,c3 BIGINT,c4 varchar(100),PRIMARY KEY (c1), UNIQUE KEY (c2),UNIQUE KEY (c3), UNIQUE KEY (c4))`)
	// Test that create view works and the output is as expected
	exec(t, conn, `create view v1 as select * from tempt1`)
	exec(t, conn, `insert into tempt1(c1, c2, c3, c4) values (300,100,300,'abc'),(30,10,30,'ac'),(3,0,3,'a')`)
	assertMatches(t, conn, "select * from v1", `[[INT64(3) INT64(0) INT64(3) VARCHAR("a")] [INT64(30) INT64(10) INT64(30) VARCHAR("ac")] [INT64(300) INT64(100) INT64(300) VARCHAR("abc")]]`)
	exec(t, conn, `drop view v1`)
	exec(t, conn, `drop table tempt1`)
	assertMatches(t, conn, "show tables", `[[VARCHAR("allDefaults")] [VARCHAR("t1")]]`)
}

func TestCallProcedure(t *testing.T) {
	defer cluster.PanicHandler(t)
	ctx := context.Background()
	vtParams := mysql.ConnParams{
		Host:  "localhost",
		Port:  clusterInstance.VtgateMySQLPort,
		Flags: mysql.CapabilityClientMultiResults,
	}
	conn, err := mysql.Connect(ctx, &vtParams)
	require.NoError(t, err)
	defer conn.Close()
	qr := exec(t, conn, `CALL sp_insert()`)
	require.EqualValues(t, 1, qr.RowsAffected)

	//_, err = conn.ExecuteFetch(`CALL sp_select()`, 1000, true)
	//require.Error(t, err)
	//require.Contains(t, err.Error(), "Multi-Resultset not supported in stored procedure")
	//
	//_, err = conn.ExecuteFetch(`CALL sp_all()`, 1000, true)
	//require.Error(t, err)
	//require.Contains(t, err.Error(), "Multi-Resultset not supported in stored procedure")

	qr = exec(t, conn, `CALL sp_delete()`)
	require.GreaterOrEqual(t, 1, int(qr.RowsAffected))

	qr = exec(t, conn, `CALL sp_multi_dml()`)
	require.EqualValues(t, 1, qr.RowsAffected)

	qr = exec(t, conn, `CALL sp_variable()`)
	require.EqualValues(t, 1, qr.RowsAffected)
}

func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result {
	t.Helper()
	var res []*sqltypes.Result
	qr, more, err := conn.ExecuteFetchMulti(query, 1000, true)
	res = append(res, qr)
	require.NoError(t, err)
	for more == true {
		qr, more, _, err = conn.ReadQueryResult(1000, true)
		require.NoError(t, err)
		res = append(res, qr)
	}
	return res
}

func execAssertError(t *testing.T, conn *mysql.Conn, query string, errorString string) {
	t.Helper()
	_, err := conn.ExecuteFetch(query, 1000, true)
	require.Error(t, err)
	assert.Contains(t, err.Error(), errorString)
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := exec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s", query, diff)
	}
}
<|endoftext|>


<commit_before>package ginstance

import (
	bosherr "github.com/cloudfoundry/bosh-agent/errors"

	"github.com/frodenas/bosh-google-cpi/api"
	"github.com/frodenas/bosh-google-cpi/google/util"
	"google.golang.org/api/compute/v1"
)

func (i GoogleInstanceService) AttachDisk(id string, diskLink string) (string, error) {
	// Find the instance
	instance, found, err := i.Find(id, "")
	if err != nil {
		return "", err
	}
	if !found {
		return "", api.NewVMNotFoundError(id)
	}

	disk := &compute.AttachedDisk{
		Mode:   "READ_WRITE",
		Source: diskLink,
		Type:   "PERSISTENT",
	}

	// Attach the disk
	i.logger.Debug(googleInstanceServiceLogTag, "Attaching Google Disk '%s' to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	operation, err := i.computeService.Instances.AttachDisk(i.project, gutil.ResourceSplitter(instance.Zone), id, disk).Do()
	if err != nil {
		return "", bosherr.WrapErrorf(err, "Failed to attach Google Disk '%s' to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	}

	if _, err = i.operationService.Waiter(operation, instance.Zone, ""); err != nil {
		return "", bosherr.WrapErrorf(err, "Failed to attach Google Disk '%s' to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	}

	// Find the instance again, as we need to get the new attached disks infor
	instance, found, err = i.Find(id, "")
	if err != nil {
		return "", err
	}
	if !found {
		return "", api.NewVMNotFoundError(id)
	}

	// Look up for the device name
	var deviceName string
	for _, attachedDisk := range instance.Disks {
		if attachedDisk.Source == diskLink {
			deviceName = attachedDisk.DeviceName
		}
	}
	if deviceName == "" {
		return "", bosherr.WrapErrorf(err, "Google Disk '%s' has not been successfully attached to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	}

	return deviceName, nil
}
<commit_msg>Set attached disk device name<commit_after>package ginstance

import (
	bosherr "github.com/cloudfoundry/bosh-agent/errors"

	"github.com/frodenas/bosh-google-cpi/api"
	"github.com/frodenas/bosh-google-cpi/google/util"
	"google.golang.org/api/compute/v1"
)

func (i GoogleInstanceService) AttachDisk(id string, diskLink string) (string, error) {
	// Find the instance
	instance, found, err := i.Find(id, "")
	if err != nil {
		return "", err
	}
	if !found {
		return "", api.NewVMNotFoundError(id)
	}

	deviceName := gutil.ResourceSplitter(diskLink)
	disk := &compute.AttachedDisk{
		DeviceName: deviceName,
		Mode:       "READ_WRITE",
		Source:     diskLink,
		Type:       "PERSISTENT",
	}

	// Attach the disk
	i.logger.Debug(googleInstanceServiceLogTag, "Attaching Google Disk '%s' to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	operation, err := i.computeService.Instances.AttachDisk(i.project, gutil.ResourceSplitter(instance.Zone), id, disk).Do()
	if err != nil {
		return "", bosherr.WrapErrorf(err, "Failed to attach Google Disk '%s' to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	}

	if _, err = i.operationService.Waiter(operation, instance.Zone, ""); err != nil {
		return "", bosherr.WrapErrorf(err, "Failed to attach Google Disk '%s' to Google Instance '%s'", gutil.ResourceSplitter(diskLink), id)
	}

	return deviceName, nil
}
<|endoftext|>


<commit_before>package interpreter

import (
	"errors"
	"os"
)

type Redirecter struct {
	path     string
	isAppend bool
	no       int
	dupFrom  int
}

func NewRedirecter(no int) *Redirecter {
	return &Redirecter{
		path:     "",
		isAppend: false,
		no:       no,
		dupFrom:  -1}
}

func (this *Redirecter) FileNo() int {
	return this.no
}

func (this *Redirecter) DupFrom(fileno int) {
	this.dupFrom = fileno
}

func (this *Redirecter) SetPath(path string) {
	this.path = path
}

func (this *Redirecter) SetAppend() {
	this.isAppend = true
}

func (this *Redirecter) open() (*os.File, error) {
	if this.path == "" {
		return nil, errors.New("Redirecter.open(): path=\"\"")
	}
	if this.no == 0 {
		return os.Open(this.path)
	} else if this.isAppend {
		return os.OpenFile(this.path, os.O_APPEND, 0666)
	} else {
		return os.Create(this.path)
	}
}

func (this *Redirecter) OpenOn(cmd *Interpreter) (*os.File, error) {
	var fd *os.File
	var err error

	switch this.dupFrom {
	case 0, 1, 2:
		fd = cmd.Stdio[this.dupFrom]
	default:
		fd, err = this.open()
		if err != nil {
			return nil, err
		}
	}
	switch this.FileNo() {
	case 0:
		cmd.SetStdin(fd)
	case 1:
		cmd.SetStdout(fd)
	case 2:
		cmd.SetStderr(fd)
	default:
		panic("Assertion failed: Redirecter.OpenAs: this.no not in (0,1,2)")
	}
	return fd, nil
}
<commit_msg>Fix #121 `>> bar` fails when `bar` does not exist<commit_after>package interpreter

import (
	"errors"
	"os"
)

type Redirecter struct {
	path     string
	isAppend bool
	no       int
	dupFrom  int
}

func NewRedirecter(no int) *Redirecter {
	return &Redirecter{
		path:     "",
		isAppend: false,
		no:       no,
		dupFrom:  -1}
}

func (this *Redirecter) FileNo() int {
	return this.no
}

func (this *Redirecter) DupFrom(fileno int) {
	this.dupFrom = fileno
}

func (this *Redirecter) SetPath(path string) {
	this.path = path
}

func (this *Redirecter) SetAppend() {
	this.isAppend = true
}

func (this *Redirecter) open() (*os.File, error) {
	if this.path == "" {
		return nil, errors.New("Redirecter.open(): path=\"\"")
	}
	if this.no == 0 {
		return os.Open(this.path)
	} else if this.isAppend {
		f, err := os.OpenFile(this.path, os.O_APPEND, 0666)
		if err != nil && os.IsNotExist(err) {
			f, err = os.Create(this.path)
		}
		return f, err
	} else {
		return os.Create(this.path)
	}
}

func (this *Redirecter) OpenOn(cmd *Interpreter) (*os.File, error) {
	var fd *os.File
	var err error

	switch this.dupFrom {
	case 0, 1, 2:
		fd = cmd.Stdio[this.dupFrom]
	default:
		fd, err = this.open()
		if err != nil {
			return nil, err
		}
	}
	switch this.FileNo() {
	case 0:
		cmd.SetStdin(fd)
	case 1:
		cmd.SetStdout(fd)
	case 2:
		cmd.SetStderr(fd)
	default:
		panic("Assertion failed: Redirecter.OpenAs: this.no not in (0,1,2)")
	}
	return fd, nil
}
<|endoftext|>


<commit_before>package twitlistserver

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"

	"github.com/eric-fouillet/anaconda"
)

type listGet struct {
	ID      int64
	Members []anaconda.User
}

// listHandler handles GET and POST to /lists/list/{id}
func listHandler(w http.ResponseWriter, r *http.Request, tc TwitterClient) error {
	log.Println("Entered listHandler", r.Method)
	switch r.Method {
	case "GET":
		return listHandlerGet(w, r, tc)
	case "POST":
		return listHandlerPost(w, r, tc)
	}
	return errors.New(fmt.Sprintln("Unsupported method", r.Method))
}

// listHandler handles GET requests to /lists/list/{id}
func listHandlerGet(w http.ResponseWriter, r *http.Request, tc TwitterClient) error {
	id, err := getListID(r)
	if err != nil {
		return fmt.Errorf("Id has an incorrect format ", id)
	}
	users, err := tc.GetListMembers(id)
	if err != nil {
		return err
	}
	render := listGet{id, users}
	return json.NewEncoder(w).Encode(render)
}

type memberIds []struct{ ID int64 }

// listHanlderPut handles POST requests to /lists/list/{id}
func listHandlerPost(w http.ResponseWriter, r *http.Request, tc TwitterClient) error {
	id, err := getListID(r)
	if err != nil {
		return fmt.Errorf("Id has an incorrect format ", id)
	}
	var members memberIds
	if err := json.NewDecoder(r.Body).Decode(&members); err != nil {
		return err
	}
	fmt.Fprintln(w, id, members)
	return nil
}

func getListID(r *http.Request) (int64, error) {
	rawPath := r.URL.Path
	last := strings.LastIndex(rawPath, "/")
	if last == -1 {
		return -1, errors.New("No list ID specified")
	}
	idStr := rawPath[last+1:]
	id, err := strconv.ParseInt(idStr, 10, 64)
	if err != nil {
		return -1, errors.New("Id has an incorrect format " + idStr)
	}
	return id, nil
}
<commit_msg>Small update after go vet<commit_after>package twitlistserver

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"

	"github.com/eric-fouillet/anaconda"
)

type listGet struct {
	ID      int64
	Members []anaconda.User
}

// listHandler handles GET and POST to /lists/list/{id}
func listHandler(w http.ResponseWriter, r *http.Request, tc TwitterClient) error {
	log.Println("Entered listHandler", r.Method)
	switch r.Method {
	case "GET":
		return listHandlerGet(w, r, tc)
	case "POST":
		return listHandlerPost(w, r, tc)
	}
	return errors.New(fmt.Sprintln("Unsupported method", r.Method))
}

// listHandler handles GET requests to /lists/list/{id}
func listHandlerGet(w http.ResponseWriter, r *http.Request, tc TwitterClient) error {
	id, err := getListID(r)
	if err != nil {
		return fmt.Errorf("Id has an incorrect format %v", id)
	}
	users, err := tc.GetListMembers(id)
	if err != nil {
		return err
	}
	render := listGet{id, users}
	return json.NewEncoder(w).Encode(render)
}

type memberIds []struct{ ID int64 }

// listHanlderPut handles POST requests to /lists/list/{id}
func listHandlerPost(w http.ResponseWriter, r *http.Request, tc TwitterClient) error {
	id, err := getListID(r)
	if err != nil {
		return fmt.Errorf("Id has an incorrect format %v", id)
	}
	var members memberIds
	if err := json.NewDecoder(r.Body).Decode(&members); err != nil {
		return err
	}
	fmt.Fprintln(w, id, members)
	return nil
}

func getListID(r *http.Request) (int64, error) {
	rawPath := r.URL.Path
	last := strings.LastIndex(rawPath, "/")
	if last == -1 {
		return -1, errors.New("No list ID specified")
	}
	idStr := rawPath[last+1:]
	id, err := strconv.ParseInt(idStr, 10, 64)
	if err != nil {
		return -1, errors.New("Id has an incorrect format " + idStr)
	}
	return id, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"net"
	"net/url"
	"net/http"
	"os"
	"path/filepath"

	"github.com/zserge/webview"
)

var indexHTML = `
<!doctype html>

		<meta http-equiv="X-UA-Compatible" content="IE=edge">

Hello, world

`

func runLocalHTTP() {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		defer ln.Close()
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte(indexHTML))
		})
		log.Fatal(http.Serve(ln, nil))
	}()
	url := "http://" + ln.Addr().String()
	w := webview.New(webview.Settings{
		Title: "Loaded: Local HTTP Server",
		URL:   url,
	})
	defer w.Exit()
	w.Run()
}

func runLocalFile() {
	dir, err := ioutil.TempDir("", "webview")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(dir)
	tmpfn := filepath.Join(dir, "index.html")
	if err := ioutil.WriteFile(tmpfn, []byte(indexHTML), 0666); err != nil {
		log.Fatal(err)
	}
	abs, err := filepath.Abs(tmpfn)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("local tmp file: ", abs)
	w := webview.New(webview.Settings{
		Title: "Loaded: Local file URL",
		URL:   "file://" + abs,
	})
	defer w.Exit()
	w.Run()
}

func runDataURL() {
	w := webview.New(webview.Settings{
		Title: "Loaded: Data URL",
		URL:   "data:text/html," + url.PathEscape(indexHTML),
	})
	defer w.Exit()
	w.Run()
}

func runInjectJS() {
	w := webview.New(webview.Settings{
		Title: "Loaded: Injected via JavaScript",
		URL:   `data:text/html,<html><script type="text/javascript"></script></html>`,
	})
	defer w.Exit()
	w.Dispatch(func() {
		w.Eval(`document.body.innerHTML = "<h1>Hello, world</h1>";`)
	})
	w.Run()
}

func main() {
	runLocalHTTP()
	//runLocalFile()
	//runDataURL()
	//runInjectJS()
}
<commit_msg>example: fix formatting<commit_after>package main

import (
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"

	"github.com/zserge/webview"
)

var indexHTML = `
<!doctype html>

		<meta http-equiv="X-UA-Compatible" content="IE=edge">

Hello, world

`

func runLocalHTTP() {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		defer ln.Close()
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte(indexHTML))
		})
		log.Fatal(http.Serve(ln, nil))
	}()
	url := "http://" + ln.Addr().String()
	w := webview.New(webview.Settings{
		Title: "Loaded: Local HTTP Server",
		URL:   url,
	})
	defer w.Exit()
	w.Run()
}

func runLocalFile() {
	dir, err := ioutil.TempDir("", "webview")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(dir)
	tmpfn := filepath.Join(dir, "index.html")
	if err := ioutil.WriteFile(tmpfn, []byte(indexHTML), 0666); err != nil {
		log.Fatal(err)
	}
	abs, err := filepath.Abs(tmpfn)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("local tmp file: ", abs)
	w := webview.New(webview.Settings{
		Title: "Loaded: Local file URL",
		URL:   "file://" + abs,
	})
	defer w.Exit()
	w.Run()
}

func runDataURL() {
	w := webview.New(webview.Settings{
		Title: "Loaded: Data URL",
		URL:   "data:text/html," + url.PathEscape(indexHTML),
	})
	defer w.Exit()
	w.Run()
}

func runInjectJS() {
	w := webview.New(webview.Settings{
		Title: "Loaded: Injected via JavaScript",
		URL:   `data:text/html,<html><script type="text/javascript"></script></html>`,
	})
	defer w.Exit()
	w.Dispatch(func() {
		w.Eval(`document.body.innerHTML = "<h1>Hello, world</h1>";`)
	})
	w.Run()
}

func main() {
	runLocalHTTP()
	//runLocalFile()
	//runDataURL()
	//runInjectJS()
}
<|endoftext|>


<commit_before>package backend_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"sort"
	"testing"

	crand "crypto/rand"

	"github.com/restic/restic/backend"
	. "github.com/restic/restic/test"
)

func testBackendConfig(b backend.Backend, t *testing.T) {
	// create config and read it back
	_, err := b.Get(backend.Config, "")
	Assert(t, err != nil, "did not get expected error for non-existing config")

	blob, err := b.Create()
	OK(t, err)

	_, err = blob.Write([]byte("Config"))
	OK(t, err)
	OK(t, blob.Finalize(backend.Config, ""))

	// try accessing the config with different names, should all return the
	// same config
	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
		rd, err := b.Get(backend.Config, name)
		Assert(t, err == nil, "unable to read config")

		buf, err := ioutil.ReadAll(rd)
		OK(t, err)
		OK(t, rd.Close())
		Assert(t, string(buf) == "Config", "wrong data returned for config")
	}
}

func testGetReader(b backend.Backend, t testing.TB) {
	length := rand.Intn(1<<23) + 2000

	data := make([]byte, length)
	_, err := io.ReadFull(crand.Reader, data)
	OK(t, err)

	blob, err := b.Create()
	OK(t, err)

	id := backend.Hash(data)

	_, err = blob.Write([]byte(data))
	OK(t, err)
	OK(t, blob.Finalize(backend.Data, id.String()))

	for i := 0; i < 500; i++ {
		l := rand.Intn(length + 2000)
		o := rand.Intn(length + 2000)

		d := data
		if o < len(d) {
			d = d[o:]
		} else {
			o = len(d)
			d = d[:0]
		}

		if l > 0 && l < len(d) {
			d = d[:l]
		}

		rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l))
		OK(t, err)
		buf, err := ioutil.ReadAll(rd)
		OK(t, err)

		if !bytes.Equal(buf, d) {
			t.Fatalf("data not equal")
		}
	}

	OK(t, b.Remove(backend.Data, id.String()))
}

func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) {
	id := backend.Hash(data)

	blob, err := b.Create()
	OK(t, err)

	_, err = blob.Write([]byte(data))
	OK(t, err)
	OK(t, blob.Finalize(tpe, id.String()))
}

func read(t testing.TB, rd io.Reader, expectedData []byte) {
	buf, err := ioutil.ReadAll(rd)
	OK(t, err)
	if expectedData != nil {
		Equals(t, expectedData, buf)
	}
}

func testBackend(b backend.Backend, t *testing.T) {
	testBackendConfig(b, t)

	for _, tpe := range []backend.Type{
		backend.Data, backend.Key, backend.Lock,
		backend.Snapshot, backend.Index,
	} {
		// detect non-existing files
		for _, test := range TestStrings {
			id, err := backend.ParseID(test.id)
			OK(t, err)

			// test if blob is already in repository
			ret, err := b.Test(tpe, id.String())
			OK(t, err)
			Assert(t, !ret, "blob was found to exist before creating")

			// try to open not existing blob
			_, err = b.Get(tpe, id.String())
			Assert(t, err != nil, "blob data could be extracted before creation")

			// try to read not existing blob
			_, err = b.GetReader(tpe, id.String(), 0, 1)
			Assert(t, err != nil, "blob reader could be obtained before creation")

			// try to get string out, should fail
			ret, err = b.Test(tpe, id.String())
			OK(t, err)
			Assert(t, !ret, "id %q was found (but should not have)", test.id)
		}

		// add files
		for _, test := range TestStrings {
			store(t, b, tpe, []byte(test.data))

			// test Get()
			rd, err := b.Get(tpe, test.id)
			OK(t, err)
			Assert(t, rd != nil, "Get() returned nil")

			read(t, rd, []byte(test.data))
			OK(t, rd.Close())

			// test GetReader()
			rd, err = b.GetReader(tpe, test.id, 0, uint(len(test.data)))
			OK(t, err)
			Assert(t, rd != nil, "GetReader() returned nil")

			read(t, rd, []byte(test.data))
			OK(t, rd.Close())

			// try to read it out with an offset and a length
			start := 1
			end := len(test.data) - 2
			length := end - start
			rd, err = b.GetReader(tpe, test.id, uint(start), uint(length))
			OK(t, err)
			Assert(t, rd != nil, "GetReader() returned nil")

			read(t, rd, []byte(test.data[start:end]))
			OK(t, rd.Close())
		}

		// test adding the first file again
		test := TestStrings[0]

		// create blob
		blob, err := b.Create()
		OK(t, err)

		_, err = blob.Write([]byte(test.data))
		OK(t, err)
		err = blob.Finalize(tpe, test.id)
		Assert(t, err != nil, "expected error, got %v", err)

		// remove and recreate
		err = b.Remove(tpe, test.id)
		OK(t, err)

		// test that the blob is gone
		ok, err := b.Test(tpe, test.id)
		OK(t, err)
		Assert(t, ok == false, "removed blob still present")

		// create blob
		blob, err = b.Create()
		OK(t, err)

		_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
		OK(t, err)
		OK(t, blob.Finalize(tpe, test.id))

		// list items
		IDs := backend.IDs{}

		for _, test := range TestStrings {
			id, err := backend.ParseID(test.id)
			OK(t, err)
			IDs = append(IDs, id)
		}

		sort.Sort(IDs)

		i := 0
		for s := range b.List(tpe, nil) {
			Equals(t, IDs[i].String(), s)
			i++
		}

		// remove content if requested
		if TestCleanup {
			for _, test := range TestStrings {
				id, err := backend.ParseID(test.id)
				OK(t, err)

				found, err := b.Test(tpe, id.String())
				OK(t, err)

				OK(t, b.Remove(tpe, id.String()))

				found, err = b.Test(tpe, id.String())
				OK(t, err)
				Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
			}
		}
	}

	testGetReader(b, t)
}
<commit_msg>Add new test with multiple writes for backends<commit_after>package backend_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"sort"
	"testing"

	crand "crypto/rand"

	"github.com/restic/restic/backend"
	. "github.com/restic/restic/test"
)

func testBackendConfig(b backend.Backend, t *testing.T) {
	// create config and read it back
	_, err := b.Get(backend.Config, "")
	Assert(t, err != nil, "did not get expected error for non-existing config")

	blob, err := b.Create()
	OK(t, err)

	_, err = blob.Write([]byte("Config"))
	OK(t, err)
	OK(t, blob.Finalize(backend.Config, ""))

	// try accessing the config with different names, should all return the
	// same config
	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
		rd, err := b.Get(backend.Config, name)
		Assert(t, err == nil, "unable to read config")

		buf, err := ioutil.ReadAll(rd)
		OK(t, err)
		OK(t, rd.Close())
		Assert(t, string(buf) == "Config", "wrong data returned for config")
	}
}

func testGetReader(b backend.Backend, t testing.TB) {
	length := rand.Intn(1<<24) + 2000

	data := make([]byte, length)
	_, err := io.ReadFull(crand.Reader, data)
	OK(t, err)

	blob, err := b.Create()
	OK(t, err)

	id := backend.Hash(data)

	_, err = blob.Write([]byte(data))
	OK(t, err)
	OK(t, blob.Finalize(backend.Data, id.String()))

	for i := 0; i < 500; i++ {
		l := rand.Intn(length + 2000)
		o := rand.Intn(length + 2000)

		d := data
		if o < len(d) {
			d = d[o:]
		} else {
			o = len(d)
			d = d[:0]
		}

		if l > 0 && l < len(d) {
			d = d[:l]
		}

		rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l))
		OK(t, err)
		buf, err := ioutil.ReadAll(rd)
		OK(t, err)

		if !bytes.Equal(buf, d) {
			t.Fatalf("data not equal")
		}
	}

	OK(t, b.Remove(backend.Data, id.String()))
}

func testWrite(b backend.Backend, t testing.TB) {
	length := rand.Intn(1<<23) + 2000

	data := make([]byte, length)
	_, err := io.ReadFull(crand.Reader, data)
	OK(t, err)
	id := backend.Hash(data)

	for i := 0; i < 10; i++ {
		blob, err := b.Create()
		OK(t, err)

		o := 0
		for o < len(data) {
			l := rand.Intn(len(data) - o)
			if len(data)-o < 20 {
				l = len(data) - o
			}

			n, err := blob.Write(data[o : o+l])
			OK(t, err)
			if n != l {
				t.Fatalf("wrong number of bytes written, want %v, got %v", l, n)
			}

			o += l
		}

		name := fmt.Sprintf("%s-%d", id, i)
		OK(t, blob.Finalize(backend.Data, name))

		rd, err := b.Get(backend.Data, name)
		OK(t, err)

		buf, err := ioutil.ReadAll(rd)
		OK(t, err)

		if len(buf) != len(data) {
			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
		}

		if !bytes.Equal(buf, data) {
			t.Fatalf("data not equal")
		}
	}
}

func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) {
	id := backend.Hash(data)

	blob, err := b.Create()
	OK(t, err)

	_, err = blob.Write([]byte(data))
	OK(t, err)
	OK(t, blob.Finalize(tpe, id.String()))
}

func read(t testing.TB, rd io.Reader, expectedData []byte) {
	buf, err := ioutil.ReadAll(rd)
	OK(t, err)
	if expectedData != nil {
		Equals(t, expectedData, buf)
	}
}

func testBackend(b backend.Backend, t *testing.T) {
	testBackendConfig(b, t)

	for _, tpe := range []backend.Type{
		backend.Data, backend.Key, backend.Lock,
		backend.Snapshot, backend.Index,
	} {
		// detect non-existing files
		for _, test := range TestStrings {
			id, err := backend.ParseID(test.id)
			OK(t, err)

			// test if blob is already in repository
			ret, err := b.Test(tpe, id.String())
			OK(t, err)
			Assert(t, !ret, "blob was found to exist before creating")

			// try to open not existing blob
			_, err = b.Get(tpe, id.String())
			Assert(t, err != nil, "blob data could be extracted before creation")

			// try to read not existing blob
			_, err = b.GetReader(tpe, id.String(), 0, 1)
			Assert(t, err != nil, "blob reader could be obtained before creation")

			// try to get string out, should fail
			ret, err = b.Test(tpe, id.String())
			OK(t, err)
			Assert(t, !ret, "id %q was found (but should not have)", test.id)
		}

		// add files
		for _, test := range TestStrings {
			store(t, b, tpe, []byte(test.data))

			// test Get()
			rd, err := b.Get(tpe, test.id)
			OK(t, err)
			Assert(t, rd != nil, "Get() returned nil")

			read(t, rd, []byte(test.data))
			OK(t, rd.Close())

			// test GetReader()
			rd, err = b.GetReader(tpe, test.id, 0, uint(len(test.data)))
			OK(t, err)
			Assert(t, rd != nil, "GetReader() returned nil")

			read(t, rd, []byte(test.data))
			OK(t, rd.Close())

			// try to read it out with an offset and a length
			start := 1
			end := len(test.data) - 2
			length := end - start
			rd, err = b.GetReader(tpe, test.id, uint(start), uint(length))
			OK(t, err)
			Assert(t, rd != nil, "GetReader() returned nil")

			read(t, rd, []byte(test.data[start:end]))
			OK(t, rd.Close())
		}

		// test adding the first file again
		test := TestStrings[0]

		// create blob
		blob, err := b.Create()
		OK(t, err)

		_, err = blob.Write([]byte(test.data))
		OK(t, err)
		err = blob.Finalize(tpe, test.id)
		Assert(t, err != nil, "expected error, got %v", err)

		// remove and recreate
		err = b.Remove(tpe, test.id)
		OK(t, err)

		// test that the blob is gone
		ok, err := b.Test(tpe, test.id)
		OK(t, err)
		Assert(t, ok == false, "removed blob still present")

		// create blob
		blob, err = b.Create()
		OK(t, err)

		_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
		OK(t, err)
		OK(t, blob.Finalize(tpe, test.id))

		// list items
		IDs := backend.IDs{}

		for _, test := range TestStrings {
			id, err := backend.ParseID(test.id)
			OK(t, err)
			IDs = append(IDs, id)
		}

		sort.Sort(IDs)

		i := 0
		for s := range b.List(tpe, nil) {
			Equals(t, IDs[i].String(), s)
			i++
		}

		// remove content if requested
		if TestCleanup {
			for _, test := range TestStrings {
				id, err := backend.ParseID(test.id)
				OK(t, err)

				found, err := b.Test(tpe, id.String())
				OK(t, err)

				OK(t, b.Remove(tpe, id.String()))

				found, err = b.Test(tpe, id.String())
				OK(t, err)
				Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
			}
		}
	}

	testGetReader(b, t)
	testWrite(b, t)
}
<|endoftext|>