<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The ct_server binary runs the CT personality.
package main

import (
	"context"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/coreos/etcd/clientv3"
	etcdnaming "github.com/coreos/etcd/clientv3/naming"
	"github.com/golang/glog"
	"github.com/google/certificate-transparency-go/trillian/ctfe"
	"github.com/google/certificate-transparency-go/trillian/ctfe/configpb"
	"github.com/google/certificate-transparency-go/trillian/util"
	"github.com/google/trillian"
	"github.com/google/trillian/monitoring/opencensus"
	"github.com/google/trillian/monitoring/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/rs/cors"
	"github.com/tomasen/realip"
	"google.golang.org/grpc"
	"google.golang.org/grpc/balancer/roundrobin"
	"google.golang.org/grpc/naming"

	// Register PEMKeyFile, PrivateKey and PKCS11Config ProtoHandlers
	_ "github.com/google/trillian/crypto/keys/der/proto"
	_ "github.com/google/trillian/crypto/keys/pem/proto"
	_ "github.com/google/trillian/crypto/keys/pkcs11/proto"
)

// Global flags that affect all log instances.
var (
	httpEndpoint       = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port)")
	metricsEndpoint    = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint")
	rpcBackend         = flag.String("log_rpc_server", "", "Backend specification; comma-separated list or etcd service name (if --etcd_servers specified). If unset backends are specified in config (as a LogMultiConfig proto)")
	rpcDeadline        = flag.Duration("rpc_deadline", time.Second*10, "Deadline for backend RPC requests")
	getSTHInterval     = flag.Duration("get_sth_interval", time.Second*180, "Interval between internal get-sth operations (0 to disable)")
	logConfig          = flag.String("log_config", "", "File holding log config in text proto format")
	maxGetEntries      = flag.Int64("max_get_entries", 0, "Max number of entries we allow in a get-entries request (0=>use default 1000)")
	etcdServers        = flag.String("etcd_servers", "", "A comma-separated list of etcd servers")
	etcdHTTPService    = flag.String("etcd_http_service", "trillian-ctfe-http", "Service name to announce our HTTP endpoint under")
	etcdMetricsService = flag.String("etcd_metrics_service", "trillian-ctfe-metrics-http", "Service name to announce our HTTP metrics endpoint under")
	maskInternalErrors = flag.Bool("mask_internal_errors", false, "Don't return error strings with Internal Server Error HTTP responses")
	tracing            = flag.Bool("tracing", false, "If true opencensus Stackdriver tracing will be enabled. See https://opencensus.io/.")
	tracingProjectID   = flag.String("tracing_project_id", "", "project ID to pass to stackdriver. Can be empty for GCP, consult docs for other platforms.")
	tracingPercent     = flag.Int("tracing_percent", 0, "Percent of requests to be traced. Zero is a special case to use the DefaultSampler")
	quotaRemote        = flag.Bool("quota_remote", true, "Enable requesting of quota for IP address sending incoming requests")
	quotaIntermediate  = flag.Bool("quota_intermediate", true, "Enable requesting of quota for intermediate certificates in sumbmitted chains")
	handlerPrefix      = flag.String("handler_prefix", "", "If set e.g. to '/logs' will prefix all handlers that don't define a custom prefix")
)

// nolint:staticcheck
func main() {
	flag.Parse()
	ctx := context.Background()

	if *maxGetEntries > 0 {
		ctfe.MaxGetEntriesAllowed = *maxGetEntries
	}

	var cfg *configpb.LogMultiConfig
	var err error
	// Get log config from file before we start. This is a different proto
	// type if we're using a multi backend configuration (no rpcBackend set
	// in flags). The single-backend config is converted to a multi config so
	// they can be treated the same.
	if len(*rpcBackend) > 0 {
		var cfgs []*configpb.LogConfig
		if cfgs, err = ctfe.LogConfigFromFile(*logConfig); err == nil {
			cfg = ctfe.ToMultiLogConfig(cfgs, *rpcBackend)
		}
	} else {
		cfg, err = ctfe.MultiLogConfigFromFile(*logConfig)
	}

	if err != nil {
		glog.Exitf("Failed to read config: %v", err)
	}

	beMap, err := ctfe.ValidateLogMultiConfig(cfg)
	if err != nil {
		glog.Exitf("Invalid config: %v", err)
	}

	glog.CopyStandardLogTo("WARNING")
	glog.Info("**** CT HTTP Server Starting ****")

	metricsAt := *metricsEndpoint
	if metricsAt == "" {
		metricsAt = *httpEndpoint
	}

	dialOpts := []grpc.DialOption{grpc.WithInsecure()}
	if len(*etcdServers) > 0 {
		// Use etcd to provide endpoint resolution.
		cfg := clientv3.Config{Endpoints: strings.Split(*etcdServers, ","), DialTimeout: 5 * time.Second}
		client, err := clientv3.New(cfg)
		if err != nil {
			glog.Exitf("Failed to connect to etcd at %v: %v", *etcdServers, err)
		}
		etcdRes := &etcdnaming.GRPCResolver{Client: client}
		dialOpts = append(dialOpts, grpc.WithBalancer(grpc.RoundRobin(etcdRes)))

		// Also announce ourselves.
		updateHTTP := naming.Update{Op: naming.Add, Addr: *httpEndpoint}
		updateMetrics := naming.Update{Op: naming.Add, Addr: metricsAt}
		glog.Infof("Announcing our presence in %v with %+v", *etcdHTTPService, updateHTTP)
		etcdRes.Update(ctx, *etcdHTTPService, updateHTTP)
		glog.Infof("Announcing our presence in %v with %+v", *etcdMetricsService, updateMetrics)
		etcdRes.Update(ctx, *etcdMetricsService, updateMetrics)

		byeHTTP := naming.Update{Op: naming.Delete, Addr: *httpEndpoint}
		byeMetrics := naming.Update{Op: naming.Delete, Addr: metricsAt}
		defer func() {
			glog.Infof("Removing our presence in %v with %+v", *etcdHTTPService, byeHTTP)
			etcdRes.Update(ctx, *etcdHTTPService, byeHTTP)
			glog.Infof("Removing our presence in %v with %+v", *etcdMetricsService, byeMetrics)
			etcdRes.Update(ctx, *etcdMetricsService, byeMetrics)
		}()
	} else if strings.Contains(*rpcBackend, ",") {
		glog.Infof("Using FixedBackendResolver")
		// Use a fixed endpoint resolution that just returns the addresses configured on the command line.
		res := util.FixedBackendResolver{}
		dialOpts = append(dialOpts, grpc.WithBalancer(grpc.RoundRobin(res)))
	} else {
		glog.Infof("Using regular DNS resolver")
		dialOpts = append(dialOpts, grpc.WithBalancerName(roundrobin.Name))
	}

	// Dial all our log backends.
	clientMap := make(map[string]trillian.TrillianLogClient)
	for _, be := range beMap {
		glog.Infof("Dialling backend: %v", be)
		if len(beMap) == 1 {
			// If there's only one of them we use the blocking option as we can't
			// serve anything until connected.
			dialOpts = append(dialOpts, grpc.WithBlock())
		}
		conn, err := grpc.Dial(be.BackendSpec, dialOpts...)
		if err != nil {
			glog.Exitf("Could not dial RPC server: %v: %v", be, err)
		}
		defer conn.Close()
		clientMap[be.Name] = trillian.NewTrillianLogClient(conn)
	}

	// Allow cross-origin requests to all handlers registered on corsMux.
	// This is safe for CT log handlers because the log is public and
	// unauthenticated so cross-site scripting attacks are not a concern.
	corsMux := http.NewServeMux()
	corsHandler := cors.AllowAll().Handler(corsMux)
	http.Handle("/", corsHandler)

	// Register handlers for all the configured logs using the correct RPC
	// client.
	for _, c := range cfg.LogConfigs.Config {
		inst, err := setupAndRegister(ctx, clientMap[c.LogBackendName], *rpcDeadline, c, corsMux, *handlerPrefix, *maskInternalErrors)
		if err != nil {
			glog.Exitf("Failed to set up log instance for %+v: %v", cfg, err)
		}
		if *getSTHInterval > 0 {
			go inst.RunUpdateSTH(ctx, *getSTHInterval)
		}
	}

	// Return a 200 on the root, for GCE default health checking :/
	corsMux.HandleFunc("/", func(resp http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/" {
			resp.WriteHeader(http.StatusOK)
		} else {
			resp.WriteHeader(http.StatusNotFound)
		}
	})

	// Export a healthz target.
	corsMux.HandleFunc("/healthz", func(resp http.ResponseWriter, req *http.Request) {
		// TODO(al): Wire this up to tell the truth.
		resp.Write([]byte("ok"))
	})

	if metricsAt != *httpEndpoint {
		// Run a separate handler for metrics.
		go func() {
			mux := http.NewServeMux()
			mux.Handle("/metrics", promhttp.Handler())
			metricsServer := http.Server{Addr: metricsAt, Handler: mux}
			err := metricsServer.ListenAndServe()
			glog.Warningf("Metrics server exited: %v", err)
		}()
	} else {
		// Handle metrics on the DefaultServeMux.
		http.Handle("/metrics", promhttp.Handler())
	}

	// If we're enabling tracing we need to use an instrumented http.Handler.
	var handler http.Handler
	if *tracing {
		handler, err = opencensus.EnableHTTPServerTracing(*tracingProjectID, *tracingPercent)
		if err != nil {
			glog.Exitf("Failed to initialize stackdriver / opencensus tracing: %v", err)
		}
	}

	// Bring up the HTTP server and serve until we get a signal not to.
	srv := http.Server{Addr: *httpEndpoint, Handler: handler}
	shutdownWG := new(sync.WaitGroup)
	go awaitSignal(func() {
		shutdownWG.Add(1)
		defer shutdownWG.Done()
		// Allow 60s for any pending requests to finish then terminate any stragglers
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
		defer cancel()
		glog.Info("Shutting down HTTP server...")
		srv.Shutdown(ctx)
		glog.Info("HTTP server shutdown")
	})

	err = srv.ListenAndServe()
	if err != http.ErrServerClosed {
		glog.Warningf("Server exited: %v", err)
	}
	// Wait will only block if the function passed to awaitSignal was called,
	// in which case it'll block until the HTTP server has gracefully shutdown
	shutdownWG.Wait()
	glog.Flush()
}

// awaitSignal waits for standard termination signals, then runs the given
// function; it should be run as a separate goroutine.
func awaitSignal(doneFn func()) {
	// Arrange notification for the standard set of signals used to terminate a server
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	// Now block main and wait for a signal
	sig := <-sigs
	glog.Warningf("Signal received: %v", sig)
	glog.Flush()

	doneFn()
}

func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, deadline time.Duration, cfg *configpb.LogConfig, mux *http.ServeMux, globalHandlerPrefix string, maskInternalErrors bool) (*ctfe.Instance, error) {
	vCfg, err := ctfe.ValidateLogConfig(cfg)
	if err != nil {
		return nil, err
	}

	opts := ctfe.InstanceOptions{
		Validated:          vCfg,
		Client:             client,
		Deadline:           deadline,
		MetricFactory:      prometheus.MetricFactory{},
		RequestLog:         new(ctfe.DefaultRequestLog),
		MaskInternalErrors: maskInternalErrors,
	}
	if *quotaRemote {
		glog.Info("Enabling quota for requesting IP")
		opts.RemoteQuotaUser = realip.FromRequest
	}
	if *quotaIntermediate {
		glog.Info("Enabling quota for intermediate certificates")
		opts.CertificateQuotaUser = ctfe.QuotaUserForCert
	}
	// Full handler pattern will be of the form "/logs/yyz/ct/v1/add-chain", where "/logs" is the
	// HandlerPrefix and "yyz" is the c.Prefix for this particular log. Use the default
	// HandlerPrefix unless the log config overrides it. The custom prefix in
	// the log configuration intended for use in migration scenarios where logs
	// have an existing URL path that differs from the global one. For example
	// if all new logs are served on "/logs/log/..." and a previously existing
	// log is at "/log/..." this is now supported.
	lhp := globalHandlerPrefix
	if ohPrefix := cfg.OverrideHandlerPrefix; len(ohPrefix) > 0 {
		glog.Infof("Log with prefix: %s is using a custom HandlerPrefix: %s", cfg.Prefix, ohPrefix)
		lhp = "/" + strings.Trim(ohPrefix, "/")
	}
	inst, err := ctfe.SetUpInstance(ctx, opts)
	if err != nil {
		return nil, err
	}
	for path, handler := range inst.Handlers {
		mux.Handle(lhp+path, handler)
	}
	return inst, nil
}
<commit_msg>If realip is determined to be empty string, return clearer sentinel (#624)<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The ct_server binary runs the CT personality.
package main

import (
	"context"
	"flag"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/coreos/etcd/clientv3"
	etcdnaming "github.com/coreos/etcd/clientv3/naming"
	"github.com/golang/glog"
	"github.com/google/certificate-transparency-go/trillian/ctfe"
	"github.com/google/certificate-transparency-go/trillian/ctfe/configpb"
	"github.com/google/certificate-transparency-go/trillian/util"
	"github.com/google/trillian"
	"github.com/google/trillian/monitoring/opencensus"
	"github.com/google/trillian/monitoring/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/rs/cors"
	"github.com/tomasen/realip"
	"google.golang.org/grpc"
	"google.golang.org/grpc/balancer/roundrobin"
	"google.golang.org/grpc/naming"

	// Register PEMKeyFile, PrivateKey and PKCS11Config ProtoHandlers
	_ "github.com/google/trillian/crypto/keys/der/proto"
	_ "github.com/google/trillian/crypto/keys/pem/proto"
	_ "github.com/google/trillian/crypto/keys/pkcs11/proto"
)

// Global flags that affect all log instances.
var (
	httpEndpoint       = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port)")
	metricsEndpoint    = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint")
	rpcBackend         = flag.String("log_rpc_server", "", "Backend specification; comma-separated list or etcd service name (if --etcd_servers specified). If unset backends are specified in config (as a LogMultiConfig proto)")
	rpcDeadline        = flag.Duration("rpc_deadline", time.Second*10, "Deadline for backend RPC requests")
	getSTHInterval     = flag.Duration("get_sth_interval", time.Second*180, "Interval between internal get-sth operations (0 to disable)")
	logConfig          = flag.String("log_config", "", "File holding log config in text proto format")
	maxGetEntries      = flag.Int64("max_get_entries", 0, "Max number of entries we allow in a get-entries request (0=>use default 1000)")
	etcdServers        = flag.String("etcd_servers", "", "A comma-separated list of etcd servers")
	etcdHTTPService    = flag.String("etcd_http_service", "trillian-ctfe-http", "Service name to announce our HTTP endpoint under")
	etcdMetricsService = flag.String("etcd_metrics_service", "trillian-ctfe-metrics-http", "Service name to announce our HTTP metrics endpoint under")
	maskInternalErrors = flag.Bool("mask_internal_errors", false, "Don't return error strings with Internal Server Error HTTP responses")
	tracing            = flag.Bool("tracing", false, "If true opencensus Stackdriver tracing will be enabled. See https://opencensus.io/.")
	tracingProjectID   = flag.String("tracing_project_id", "", "project ID to pass to stackdriver. Can be empty for GCP, consult docs for other platforms.")
	tracingPercent     = flag.Int("tracing_percent", 0, "Percent of requests to be traced. Zero is a special case to use the DefaultSampler")
	quotaRemote        = flag.Bool("quota_remote", true, "Enable requesting of quota for IP address sending incoming requests")
	quotaIntermediate  = flag.Bool("quota_intermediate", true, "Enable requesting of quota for intermediate certificates in sumbmitted chains")
	handlerPrefix      = flag.String("handler_prefix", "", "If set e.g. to '/logs' will prefix all handlers that don't define a custom prefix")
)

const unknownRemoteUser = "UNKNOWN_REMOTE"

// nolint:staticcheck
func main() {
	flag.Parse()
	ctx := context.Background()

	if *maxGetEntries > 0 {
		ctfe.MaxGetEntriesAllowed = *maxGetEntries
	}

	var cfg *configpb.LogMultiConfig
	var err error
	// Get log config from file before we start. This is a different proto
	// type if we're using a multi backend configuration (no rpcBackend set
	// in flags). The single-backend config is converted to a multi config so
	// they can be treated the same.
	if len(*rpcBackend) > 0 {
		var cfgs []*configpb.LogConfig
		if cfgs, err = ctfe.LogConfigFromFile(*logConfig); err == nil {
			cfg = ctfe.ToMultiLogConfig(cfgs, *rpcBackend)
		}
	} else {
		cfg, err = ctfe.MultiLogConfigFromFile(*logConfig)
	}

	if err != nil {
		glog.Exitf("Failed to read config: %v", err)
	}

	beMap, err := ctfe.ValidateLogMultiConfig(cfg)
	if err != nil {
		glog.Exitf("Invalid config: %v", err)
	}

	glog.CopyStandardLogTo("WARNING")
	glog.Info("**** CT HTTP Server Starting ****")

	metricsAt := *metricsEndpoint
	if metricsAt == "" {
		metricsAt = *httpEndpoint
	}

	dialOpts := []grpc.DialOption{grpc.WithInsecure()}
	if len(*etcdServers) > 0 {
		// Use etcd to provide endpoint resolution.
		cfg := clientv3.Config{Endpoints: strings.Split(*etcdServers, ","), DialTimeout: 5 * time.Second}
		client, err := clientv3.New(cfg)
		if err != nil {
			glog.Exitf("Failed to connect to etcd at %v: %v", *etcdServers, err)
		}
		etcdRes := &etcdnaming.GRPCResolver{Client: client}
		dialOpts = append(dialOpts, grpc.WithBalancer(grpc.RoundRobin(etcdRes)))

		// Also announce ourselves.
		updateHTTP := naming.Update{Op: naming.Add, Addr: *httpEndpoint}
		updateMetrics := naming.Update{Op: naming.Add, Addr: metricsAt}
		glog.Infof("Announcing our presence in %v with %+v", *etcdHTTPService, updateHTTP)
		etcdRes.Update(ctx, *etcdHTTPService, updateHTTP)
		glog.Infof("Announcing our presence in %v with %+v", *etcdMetricsService, updateMetrics)
		etcdRes.Update(ctx, *etcdMetricsService, updateMetrics)

		byeHTTP := naming.Update{Op: naming.Delete, Addr: *httpEndpoint}
		byeMetrics := naming.Update{Op: naming.Delete, Addr: metricsAt}
		defer func() {
			glog.Infof("Removing our presence in %v with %+v", *etcdHTTPService, byeHTTP)
			etcdRes.Update(ctx, *etcdHTTPService, byeHTTP)
			glog.Infof("Removing our presence in %v with %+v", *etcdMetricsService, byeMetrics)
			etcdRes.Update(ctx, *etcdMetricsService, byeMetrics)
		}()
	} else if strings.Contains(*rpcBackend, ",") {
		glog.Infof("Using FixedBackendResolver")
		// Use a fixed endpoint resolution that just returns the addresses configured on the command line.
		res := util.FixedBackendResolver{}
		dialOpts = append(dialOpts, grpc.WithBalancer(grpc.RoundRobin(res)))
	} else {
		glog.Infof("Using regular DNS resolver")
		dialOpts = append(dialOpts, grpc.WithBalancerName(roundrobin.Name))
	}

	// Dial all our log backends.
	clientMap := make(map[string]trillian.TrillianLogClient)
	for _, be := range beMap {
		glog.Infof("Dialling backend: %v", be)
		if len(beMap) == 1 {
			// If there's only one of them we use the blocking option as we can't
			// serve anything until connected.
			dialOpts = append(dialOpts, grpc.WithBlock())
		}
		conn, err := grpc.Dial(be.BackendSpec, dialOpts...)
		if err != nil {
			glog.Exitf("Could not dial RPC server: %v: %v", be, err)
		}
		defer conn.Close()
		clientMap[be.Name] = trillian.NewTrillianLogClient(conn)
	}

	// Allow cross-origin requests to all handlers registered on corsMux.
	// This is safe for CT log handlers because the log is public and
	// unauthenticated so cross-site scripting attacks are not a concern.
	corsMux := http.NewServeMux()
	corsHandler := cors.AllowAll().Handler(corsMux)
	http.Handle("/", corsHandler)

	// Register handlers for all the configured logs using the correct RPC
	// client.
	for _, c := range cfg.LogConfigs.Config {
		inst, err := setupAndRegister(ctx, clientMap[c.LogBackendName], *rpcDeadline, c, corsMux, *handlerPrefix, *maskInternalErrors)
		if err != nil {
			glog.Exitf("Failed to set up log instance for %+v: %v", cfg, err)
		}
		if *getSTHInterval > 0 {
			go inst.RunUpdateSTH(ctx, *getSTHInterval)
		}
	}

	// Return a 200 on the root, for GCE default health checking :/
	corsMux.HandleFunc("/", func(resp http.ResponseWriter, req *http.Request) {
		if req.URL.Path == "/" {
			resp.WriteHeader(http.StatusOK)
		} else {
			resp.WriteHeader(http.StatusNotFound)
		}
	})

	// Export a healthz target.
	corsMux.HandleFunc("/healthz", func(resp http.ResponseWriter, req *http.Request) {
		// TODO(al): Wire this up to tell the truth.
		resp.Write([]byte("ok"))
	})

	if metricsAt != *httpEndpoint {
		// Run a separate handler for metrics.
		go func() {
			mux := http.NewServeMux()
			mux.Handle("/metrics", promhttp.Handler())
			metricsServer := http.Server{Addr: metricsAt, Handler: mux}
			err := metricsServer.ListenAndServe()
			glog.Warningf("Metrics server exited: %v", err)
		}()
	} else {
		// Handle metrics on the DefaultServeMux.
		http.Handle("/metrics", promhttp.Handler())
	}

	// If we're enabling tracing we need to use an instrumented http.Handler.
	var handler http.Handler
	if *tracing {
		handler, err = opencensus.EnableHTTPServerTracing(*tracingProjectID, *tracingPercent)
		if err != nil {
			glog.Exitf("Failed to initialize stackdriver / opencensus tracing: %v", err)
		}
	}

	// Bring up the HTTP server and serve until we get a signal not to.
	srv := http.Server{Addr: *httpEndpoint, Handler: handler}
	shutdownWG := new(sync.WaitGroup)
	go awaitSignal(func() {
		shutdownWG.Add(1)
		defer shutdownWG.Done()
		// Allow 60s for any pending requests to finish then terminate any stragglers
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
		defer cancel()
		glog.Info("Shutting down HTTP server...")
		srv.Shutdown(ctx)
		glog.Info("HTTP server shutdown")
	})

	err = srv.ListenAndServe()
	if err != http.ErrServerClosed {
		glog.Warningf("Server exited: %v", err)
	}
	// Wait will only block if the function passed to awaitSignal was called,
	// in which case it'll block until the HTTP server has gracefully shutdown
	shutdownWG.Wait()
	glog.Flush()
}

// awaitSignal waits for standard termination signals, then runs the given
// function; it should be run as a separate goroutine.
func awaitSignal(doneFn func()) {
	// Arrange notification for the standard set of signals used to terminate a server
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

	// Now block main and wait for a signal
	sig := <-sigs
	glog.Warningf("Signal received: %v", sig)
	glog.Flush()

	doneFn()
}

func setupAndRegister(ctx context.Context, client trillian.TrillianLogClient, deadline time.Duration, cfg *configpb.LogConfig, mux *http.ServeMux, globalHandlerPrefix string, maskInternalErrors bool) (*ctfe.Instance, error) {
	vCfg, err := ctfe.ValidateLogConfig(cfg)
	if err != nil {
		return nil, err
	}

	opts := ctfe.InstanceOptions{
		Validated:          vCfg,
		Client:             client,
		Deadline:           deadline,
		MetricFactory:      prometheus.MetricFactory{},
		RequestLog:         new(ctfe.DefaultRequestLog),
		MaskInternalErrors: maskInternalErrors,
	}
	if *quotaRemote {
		glog.Info("Enabling quota for requesting IP")
		opts.RemoteQuotaUser = func(r *http.Request) string {
			var remoteUser = realip.FromRequest(r)
			if len(remoteUser) == 0 {
				return unknownRemoteUser
			}
			return remoteUser
		}
	}
	if *quotaIntermediate {
		glog.Info("Enabling quota for intermediate certificates")
		opts.CertificateQuotaUser = ctfe.QuotaUserForCert
	}
	// Full handler pattern will be of the form "/logs/yyz/ct/v1/add-chain", where "/logs" is the
	// HandlerPrefix and "yyz" is the c.Prefix for this particular log. Use the default
	// HandlerPrefix unless the log config overrides it. The custom prefix in
	// the log configuration intended for use in migration scenarios where logs
	// have an existing URL path that differs from the global one. For example
	// if all new logs are served on "/logs/log/..." and a previously existing
	// log is at "/log/..." this is now supported.
	lhp := globalHandlerPrefix
	if ohPrefix := cfg.OverrideHandlerPrefix; len(ohPrefix) > 0 {
		glog.Infof("Log with prefix: %s is using a custom HandlerPrefix: %s", cfg.Prefix, ohPrefix)
		lhp = "/" + strings.Trim(ohPrefix, "/")
	}
	inst, err := ctfe.SetUpInstance(ctx, opts)
	if err != nil {
		return nil, err
	}
	for path, handler := range inst.Handlers {
		mux.Handle(lhp+path, handler)
	}
	return inst, nil
}
<|endoftext|>


<commit_before>package compare

import (
	"testing"
)

func TestCompare(t *testing.T){
	tt := []struct{
		name string
		actual interface{}
		expected interface{}
		isEqual bool
	}{
		{"a equals a", "a", "a", true},
		{"a not equals b", "a", "b", false},
		{"bool equals bool", true, true, true},
		{"bool not equals bool", true, false, false},
		{"1 equals 1", 1, 1, true},
		{"1 not equals 1", 1, 0, false},
		{"slice equals slice", []string{"one", "two", "three"}, []string{"one", "two", "three"}, true},
		{"slice not equals slice", []string{"one", "two", "three"}, []string{"one", "four", "three"}, false},
		{"map equals map", map[string]int{"one": 1, "two": 2}, map[string]int{"one": 1, "two": 2}, true},
		{"map not equals map", map[string]int{"one": 1, "two": 2}, map[string]int{"one": 1, "two": 3}, false},
	}

	for _, tc := range tt {
		t.Run(tc.name, func(t *testing.T){
			isEqual, _ := IsEqual(tc.actual,tc.expected)
			if isEqual != tc.isEqual{
				t.Fatalf("%v should be %v got %v\n", tc.name, tc.isEqual, isEqual )
			}
		})
	}

}<commit_msg>Added missing Header/License<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compare

import (
	"testing"
)

func TestCompare(t *testing.T){
	tt := []struct{
		name string
		actual interface{}
		expected interface{}
		isEqual bool
	}{
		{"a equals a", "a", "a", true},
		{"a not equals b", "a", "b", false},
		{"bool equals bool", true, true, true},
		{"bool not equals bool", true, false, false},
		{"1 equals 1", 1, 1, true},
		{"1 not equals 1", 1, 0, false},
		{"slice equals slice", []string{"one", "two", "three"}, []string{"one", "two", "three"}, true},
		{"slice not equals slice", []string{"one", "two", "three"}, []string{"one", "four", "three"}, false},
		{"map equals map", map[string]int{"one": 1, "two": 2}, map[string]int{"one": 1, "two": 2}, true},
		{"map not equals map", map[string]int{"one": 1, "two": 2}, map[string]int{"one": 1, "two": 3}, false},
	}

	for _, tc := range tt {
		t.Run(tc.name, func(t *testing.T){
			isEqual, _ := IsEqual(tc.actual,tc.expected)
			if isEqual != tc.isEqual{
				t.Fatalf("%v should be %v got %v\n", tc.name, tc.isEqual, isEqual )
			}
		})
	}

}<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eventpb

import (
	"context"
	"fmt"
	"time"

	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/server/tq"

	"go.chromium.org/luci/cv/internal/common"
)

const (
	// ManageRunTaskClass is the ID of ManageRunTask Class.
	ManageRunTaskClass = "manage-run"
	// ManageRunLongOpTaskClass is the ID of the ManageRunLongOp Class.
	ManageRunLongOpTaskClass = "manage-run-long-op"
	// taskInterval is target frequency of executions of ManageRunTask.
	//
	// See Dispatch() for details.
	taskInterval = time.Second
)

// TasksBinding binds Run Manager tasks to a TQ Dispatcher.
//
// This struct exists to separate task creation and handling,
// which in turns avoids circular dependency.
type TasksBinding struct {
	ManageRun       tq.TaskClassRef
	KickManage      tq.TaskClassRef
	ManageRunLongOp tq.TaskClassRef
	TQDispatcher    *tq.Dispatcher
}

// Register registers tasks with the given TQ Dispatcher.
func Register(tqd *tq.Dispatcher) TasksBinding {
	t := TasksBinding{
		ManageRun: tqd.RegisterTaskClass(tq.TaskClass{
			ID:           ManageRunTaskClass,
			Prototype:    &ManageRunTask{},
			Queue:        "manage-run",
			Kind:         tq.NonTransactional,
			QuietOnError: true,
			Quiet:        true,
		}),
		KickManage: tqd.RegisterTaskClass(tq.TaskClass{
			ID:           fmt.Sprintf("kick-%s", ManageRunTaskClass),
			Prototype:    &KickManageRunTask{},
			Queue:        "kick-manage-run",
			Kind:         tq.Transactional,
			QuietOnError: true,
			Quiet:        true,
		}),
		ManageRunLongOp: tqd.RegisterTaskClass(tq.TaskClass{
			ID:        ManageRunLongOpTaskClass,
			Prototype: &ManageRunLongOpTask{},
			Queue:     "manage-run-long-op",
			Kind:      tq.Transactional,
			// TODO(tandrii): switch to quiet once stable.
			QuietOnError: false,
			Quiet:        false,
		}),
		TQDispatcher: tqd,
	}
	t.KickManage.AttachHandler(func(ctx context.Context, payload proto.Message) error {
		task := payload.(*KickManageRunTask)
		var eta time.Time
		if t := task.GetEta(); t != nil {
			eta = t.AsTime()
		}
		err := t.Dispatch(ctx, task.GetRunId(), eta)
		return common.TQifyError(ctx, err)
	})
	return t
}

// Dispatch ensures invocation of RunManager via ManageRunTask.
//
// RunManager will be invoked at approximately no earlier than both:
//  * eta time (if given)
//  * next possible.
//
// To avoid actually dispatching TQ tasks in tests, use runtest.MockDispatch().
func (tr TasksBinding) Dispatch(ctx context.Context, runID string, eta time.Time) error {
	mock, mocked := ctx.Value(&mockDispatcherContextKey).(func(string, time.Time))

	if datastore.CurrentTransaction(ctx) != nil {
		payload := &KickManageRunTask{RunId: runID}
		if !eta.IsZero() {
			payload.Eta = timestamppb.New(eta)
		}

		if mocked {
			mock(runID, eta)
			return nil
		}
		return tr.TQDispatcher.AddTask(ctx, &tq.Task{
			DeduplicationKey: "", // not allowed in a transaction
			Payload:          payload,
		})
	}

	// If actual local clock is more than `clockDrift` behind, the "next" computed
	// ManageRunTask moment might be already executing, meaning task dedup will
	// ensure no new task will be scheduled AND the already executing run
	// might not have read the Event that was just written.
	// Thus, for safety, this should be large, however, will also leads to higher
	// latency of event processing of non-busy RunManager.
	// TODO(tandrii/yiwzhang): this can be reduced significantly once safety
	// "ping" events are originated from Config import cron tasks.
	const clockDrift = 100 * time.Millisecond
	now := clock.Now(ctx).Add(clockDrift) // Use the worst possible time.
	if eta.IsZero() || eta.Before(now) {
		eta = now
	}
	eta = eta.Truncate(taskInterval).Add(taskInterval)

	if mocked {
		mock(runID, eta)
		return nil
	}
	return tr.TQDispatcher.AddTask(ctx, &tq.Task{
		Title:            runID,
		DeduplicationKey: fmt.Sprintf("%s\n%d", runID, eta.UnixNano()),
		ETA:              eta,
		Payload:          &ManageRunTask{RunId: runID},
	})
}

var mockDispatcherContextKey = "eventpb.mockDispatcher"

// InstallMockDispatcher is used in test to run tests emitting RM events without
// actually dispatching RM tasks.
//
// See runtest.MockDispatch().
func InstallMockDispatcher(ctx context.Context, f func(runID string, eta time.Time)) context.Context {
	return context.WithValue(ctx, &mockDispatcherContextKey, f)
}
<commit_msg>[cv] make TQ handler for long ops quiet.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eventpb

import (
	"context"
	"fmt"
	"time"

	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/server/tq"

	"go.chromium.org/luci/cv/internal/common"
)

const (
	// ManageRunTaskClass is the ID of ManageRunTask Class.
	ManageRunTaskClass = "manage-run"
	// ManageRunLongOpTaskClass is the ID of the ManageRunLongOp Class.
	ManageRunLongOpTaskClass = "manage-run-long-op"
	// taskInterval is target frequency of executions of ManageRunTask.
	//
	// See Dispatch() for details.
	taskInterval = time.Second
)

// TasksBinding binds Run Manager tasks to a TQ Dispatcher.
//
// This struct exists to separate task creation and handling,
// which in turns avoids circular dependency.
type TasksBinding struct {
	ManageRun       tq.TaskClassRef
	KickManage      tq.TaskClassRef
	ManageRunLongOp tq.TaskClassRef
	TQDispatcher    *tq.Dispatcher
}

// Register registers tasks with the given TQ Dispatcher.
func Register(tqd *tq.Dispatcher) TasksBinding {
	t := TasksBinding{
		ManageRun: tqd.RegisterTaskClass(tq.TaskClass{
			ID:           ManageRunTaskClass,
			Prototype:    &ManageRunTask{},
			Queue:        "manage-run",
			Kind:         tq.NonTransactional,
			QuietOnError: true,
			Quiet:        true,
		}),
		KickManage: tqd.RegisterTaskClass(tq.TaskClass{
			ID:           fmt.Sprintf("kick-%s", ManageRunTaskClass),
			Prototype:    &KickManageRunTask{},
			Queue:        "kick-manage-run",
			Kind:         tq.Transactional,
			QuietOnError: true,
			Quiet:        true,
		}),
		ManageRunLongOp: tqd.RegisterTaskClass(tq.TaskClass{
			ID:           ManageRunLongOpTaskClass,
			Prototype:    &ManageRunLongOpTask{},
			Queue:        "manage-run-long-op",
			Kind:         tq.Transactional,
			QuietOnError: true,
			Quiet:        true,
		}),
		TQDispatcher: tqd,
	}
	t.KickManage.AttachHandler(func(ctx context.Context, payload proto.Message) error {
		task := payload.(*KickManageRunTask)
		var eta time.Time
		if t := task.GetEta(); t != nil {
			eta = t.AsTime()
		}
		err := t.Dispatch(ctx, task.GetRunId(), eta)
		return common.TQifyError(ctx, err)
	})
	return t
}

// Dispatch ensures invocation of RunManager via ManageRunTask.
//
// RunManager will be invoked at approximately no earlier than both:
//  * eta time (if given)
//  * next possible.
//
// To avoid actually dispatching TQ tasks in tests, use runtest.MockDispatch().
func (tr TasksBinding) Dispatch(ctx context.Context, runID string, eta time.Time) error {
	mock, mocked := ctx.Value(&mockDispatcherContextKey).(func(string, time.Time))

	if datastore.CurrentTransaction(ctx) != nil {
		payload := &KickManageRunTask{RunId: runID}
		if !eta.IsZero() {
			payload.Eta = timestamppb.New(eta)
		}

		if mocked {
			mock(runID, eta)
			return nil
		}
		return tr.TQDispatcher.AddTask(ctx, &tq.Task{
			DeduplicationKey: "", // not allowed in a transaction
			Payload:          payload,
		})
	}

	// If actual local clock is more than `clockDrift` behind, the "next" computed
	// ManageRunTask moment might be already executing, meaning task dedup will
	// ensure no new task will be scheduled AND the already executing run
	// might not have read the Event that was just written.
	// Thus, for safety, this should be large, however, will also leads to higher
	// latency of event processing of non-busy RunManager.
	// TODO(tandrii/yiwzhang): this can be reduced significantly once safety
	// "ping" events are originated from Config import cron tasks.
	const clockDrift = 100 * time.Millisecond
	now := clock.Now(ctx).Add(clockDrift) // Use the worst possible time.
	if eta.IsZero() || eta.Before(now) {
		eta = now
	}
	eta = eta.Truncate(taskInterval).Add(taskInterval)

	if mocked {
		mock(runID, eta)
		return nil
	}
	return tr.TQDispatcher.AddTask(ctx, &tq.Task{
		Title:            runID,
		DeduplicationKey: fmt.Sprintf("%s\n%d", runID, eta.UnixNano()),
		ETA:              eta,
		Payload:          &ManageRunTask{RunId: runID},
	})
}

var mockDispatcherContextKey = "eventpb.mockDispatcher"

// InstallMockDispatcher is used in test to run tests emitting RM events without
// actually dispatching RM tasks.
//
// See runtest.MockDispatch().
func InstallMockDispatcher(ctx context.Context, f func(runID string, eta time.Time)) context.Context {
	return context.WithValue(ctx, &mockDispatcherContextKey, f)
}
<|endoftext|>


<commit_before>package e2e

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"testing"
	"time"

	"github.com/oinume/lekcije/server/controller"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/util"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var _ = time.UTC
var _ = fmt.Print

func TestOAuthGoogleLogin(t *testing.T) {
	if os.Getenv("CIRCLECI") != "" {
		t.Skipf("Skip because it can't render Google login page.")
	}
	a := assert.New(t)
	require := require.New(t)
	driver := newWebDriver()
	err := driver.Start()
	a.Nil(err)
	defer driver.Stop()

	page, err := driver.NewPage()
	require.Nil(err)
	require.Nil(page.Navigate(server.URL))
	//time.Sleep(10 * time.Second)

	// Check trackingId is set
	cookies, err := page.GetCookies()
	a.Nil(err)
	trackingIDCookie, err := getCookie(cookies, controller.TrackingIDCookieName)
	fmt.Printf("trackingId = %v\n", trackingIDCookie.Value)
	a.Nil(err)
	a.NotEmpty(trackingIDCookie.Value)

	signupButton := page.FindByXPath("//a[contains(@class, 'button-signup')]")
	signupURL, err := signupButton.Attribute("href")
	require.Nil(err)

	require.Nil(page.Navigate(signupURL))
	buttonGoogle := page.FindByXPath("//button[contains(@class, 'button-google')]")
	require.Nil(buttonGoogle.Click())
	//time.Sleep(10 * time.Second)

	googleAccount := os.Getenv("E2E_GOOGLE_ACCOUNT")
	err = page.Find("#Email").Fill(googleAccount)
	require.Nil(err)
	page.Find("#gaia_loginform").Submit()
	require.Nil(err)

	time.Sleep(time.Second * 1)
	page.Find("#Passwd").Fill(os.Getenv("E2E_GOOGLE_PASSWORD"))
	require.Nil(err)
	page.Find("#gaia_loginform").Submit()
	require.Nil(err)

	time.Sleep(time.Second * 3)
	err = page.Find("#submit_approve_access").Click()
	require.Nil(err)
	//time.Sleep(time.Second * 10)
	// TODO: Check HTML content

	cookies, err = page.GetCookies()
	require.Nil(err)
	apiToken := getAPIToken(cookies)
	a.NotEmpty(apiToken)

	user, err := model.NewUserService(db).FindByUserAPIToken(apiToken)
	require.Nil(err)
	a.Equal(googleAccount, user.Email)
}

func TestOAuthGoogleLogout(t *testing.T) {
	if os.Getenv("CIRCLECI") != "" {
		t.Skipf("Skip because PhantomJS can't SetCookie.")
	}

	a := assert.New(t)

	_, apiToken, err := createUserAndLogin("oinume", randomEmail("oinume"), util.RandomString(16))
	a.Nil(err)

	driver := newWebDriver()
	a.Nil(driver.Start())
	defer driver.Stop()

	page, err := driver.NewPage()
	a.Nil(err)
	a.Nil(page.Navigate(server.URL))
	u, err := url.Parse(server.URL)
	a.Nil(err)
	cookie := &http.Cookie{
		Name:     controller.APITokenCookieName,
		Domain:   u.Host,
		Value:    apiToken,
		Path:     "/",
		Expires:  time.Now().Add(model.UserAPITokenExpiration),
		HttpOnly: false,
	}
	a.Nil(page.SetCookie(cookie))
	a.Nil(page.Navigate(server.URL + "/me"))

	a.Nil(page.Navigate(server.URL + "/me/logout"))
	userAPITokenService := model.NewUserAPITokenService(db)
	_, err = userAPITokenService.FindByPK(apiToken)
	a.IsType(&errors.NotFound{}, err)
}

func getAPIToken(cookies []*http.Cookie) string {
	for _, cookie := range cookies {
		if cookie.Name == "apiToken" {
			return cookie.Value
		}
	}
	return ""
}
<commit_msg>Fix TestOAuthGoogleLogin<commit_after>package e2e

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"testing"
	"time"

	"github.com/oinume/lekcije/server/controller"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/util"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var _ = time.UTC
var _ = fmt.Print

func TestOAuthGoogleLogin(t *testing.T) {
	if os.Getenv("CIRCLECI") != "" {
		t.Skipf("Skip because it can't render Google login page.")
	}
	a := assert.New(t)
	require := require.New(t)
	driver := newWebDriver()
	err := driver.Start()
	a.Nil(err)
	defer driver.Stop()

	page, err := driver.NewPage()
	require.Nil(err)
	require.Nil(page.Navigate(server.URL))
	//time.Sleep(15 * time.Second)

	// Check trackingId is set
	cookies, err := page.GetCookies()
	a.Nil(err)
	trackingIDCookie, err := getCookie(cookies, controller.TrackingIDCookieName)
	fmt.Printf("trackingId = %v\n", trackingIDCookie.Value)
	a.Nil(err)
	a.NotEmpty(trackingIDCookie.Value)

	signupButton := page.FindByXPath("//a[contains(@class, 'button-signup')]")
	signupURL, err := signupButton.Attribute("href")
	require.Nil(err)

	require.Nil(page.Navigate(signupURL))
	buttonGoogle := page.FindByXPath("//button[contains(@class, 'button-google')]")
	require.Nil(buttonGoogle.Click())
	//time.Sleep(15 * time.Second)

	time.Sleep(time.Second * 1)
	googleAccount := os.Getenv("E2E_GOOGLE_ACCOUNT")
	err = page.FindByXPath("//input[@name='identifier']").Fill(googleAccount)
	require.Nil(err)
	err = page.FindByXPath("//div[@id='identifierNext']/content/span").Click()
	require.Nil(err)

	time.Sleep(time.Second * 1)
	err = page.FindByXPath("//input[@name='password']").Fill(os.Getenv("E2E_GOOGLE_PASSWORD"))
	require.Nil(err)
	err = page.FindByXPath("//div[@id='passwordNext']/content/span").Click()
	require.Nil(err)

	time.Sleep(time.Second * 3)

	cookies, err = page.GetCookies()
	require.Nil(err)
	apiToken := getAPIToken(cookies)
	a.NotEmpty(apiToken)

	user, err := model.NewUserService(db).FindByUserAPIToken(apiToken)
	require.Nil(err)
	a.Equal(googleAccount, user.Email)

	// TODO: Check HTML content
}

func TestOAuthGoogleLogout(t *testing.T) {
	if os.Getenv("CIRCLECI") != "" {
		t.Skipf("Skip because PhantomJS can't SetCookie.")
	}

	a := assert.New(t)

	_, apiToken, err := createUserAndLogin("oinume", randomEmail("oinume"), util.RandomString(16))
	a.Nil(err)

	driver := newWebDriver()
	a.Nil(driver.Start())
	defer driver.Stop()

	page, err := driver.NewPage()
	a.Nil(err)
	a.Nil(page.Navigate(server.URL))
	u, err := url.Parse(server.URL)
	a.Nil(err)
	cookie := &http.Cookie{
		Name:     controller.APITokenCookieName,
		Domain:   u.Host,
		Value:    apiToken,
		Path:     "/",
		Expires:  time.Now().Add(model.UserAPITokenExpiration),
		HttpOnly: false,
	}
	a.Nil(page.SetCookie(cookie))
	a.Nil(page.Navigate(server.URL + "/me"))

	a.Nil(page.Navigate(server.URL + "/me/logout"))
	userAPITokenService := model.NewUserAPITokenService(db)
	_, err = userAPITokenService.FindByPK(apiToken)
	a.IsType(&errors.NotFound{}, err)
}

func getAPIToken(cookies []*http.Cookie) string {
	for _, cookie := range cookies {
		if cookie.Name == "apiToken" {
			return cookie.Value
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package snowdrift

import (
	"crypto/sha512"
	"encoding/hex"
	"errors"
	"net/http"
	"net/url"

	"bitbucket.org/ww/goautoneg"
	"github.com/codegangsta/martini"
	"github.com/codegangsta/martini-contrib/binding"
	"github.com/codegangsta/martini-contrib/render"
	"github.com/speps/go-hashids"
)

type Config struct {
	Backend      Backend
	HashSalt     string
	URLPrefix    string
	RootRedirect string
	ReportErr    func(err error, req *http.Request)
}

func New(c *Config) *martini.Martini {
	r := martini.NewRouter()
	m := martini.New()
	m.Action(r.Handle)

	ctx := &context{
		Backend:   c.Backend,
		ShortHash: hashids.New(),
		LongHash:  hashids.New(),
		URLPrefix: c.URLPrefix,
		ReportErr: c.ReportErr,
	}
	if c.HashSalt == "" {
		c.HashSalt = "salt"
	}
	ctx.ShortHash.Salt = c.HashSalt
	ctx.LongHash.Salt = c.HashSalt
	ctx.LongHash.MinLength = 12
	m.Map(ctx)
	m.Use(render.Renderer())

	r.Get("/", func(r *http.Request, w http.ResponseWriter) {
		http.Redirect(w, r, c.RootRedirect, http.StatusFound)
	})
	r.Post("/", binding.Bind(link{}), createLink)
	r.Get("/:code", getLink)

	return m
}

var ErrNotFound = errors.New("snowdrift: not found")
var ErrURLExists = errors.New("snowdrift: url already exists")
var ErrCodeExists = errors.New("snowdrift: code already exists")

type Backend interface {
	Add(url, digest, code string) error
	GetCode(digest string) (string, error)
	GetURL(code string) (string, error)
	NextID() (int, error)
}

type link struct {
	LongURL  string `json:"long_url" riak:"long_url"`
	ShortURL string `json:"short_url" riak:"-"`
	Obscure  *bool  `json:"obscure,omitempty" riak:"-"`
}

type context struct {
	Backend
	ShortHash *hashids.HashID
	LongHash  *hashids.HashID
	URLPrefix string
	ReportErr func(err error, req *http.Request)
}

func urlDigest(url string) string {
	digest := sha512.Sum512([]byte(url))
	return hex.EncodeToString(digest[:32])
}

func createLink(c *context, link link, r render.Render, req *http.Request) {
	u, err := url.Parse(link.LongURL)
	if err != nil || u.Scheme != "http" && u.Scheme != "https" {
		r.Error(400)
		return
	}

	digest := urlDigest(link.LongURL)

	code, err := c.GetCode(digest)
	if err == nil {
		link.ShortURL = c.URLPrefix + code
		link.Obscure = nil
		r.JSON(200, link)
		return
	}

	id, err := c.NextID()
	if err != nil {
		r.Error(500)
		if c.ReportErr != nil {
			c.ReportErr(err, req)
		}
		return
	}
	if link.Obscure != nil && *link.Obscure {
		code = c.LongHash.Encrypt([]int{id})
	} else {
		code = c.ShortHash.Encrypt([]int{id})
	}

	if err := c.Add(link.LongURL, digest, code); err != nil {
		r.Error(500)
		if c.ReportErr != nil {
			c.ReportErr(err, req)
		}
		return
	}
	link.Obscure = nil
	link.ShortURL = c.URLPrefix + code
	r.JSON(200, link)
}

func getLink(c *context, params martini.Params, r render.Render, req *http.Request, w http.ResponseWriter) {
	url, err := c.GetURL(params["code"])
	if err == ErrNotFound {
		r.Error(404)
		return
	}
	if err != nil {
		r.Error(500)
		if c.ReportErr != nil {
			c.ReportErr(err, req)
		}
		return
	}
	contentType := goautoneg.Negotiate(req.Header.Get("Accept"), []string{"text/html", "application/json"})
	if contentType == "application/json" {
		r.JSON(200, link{LongURL: url, ShortURL: c.URLPrefix + params["code"]})
		return
	}

	w.Header().Set("Location", url)
	w.WriteHeader(http.StatusMovedPermanently)
}
<commit_msg>Use http helpers for 404 and redirect<commit_after>package snowdrift

import (
	"crypto/sha512"
	"encoding/hex"
	"errors"
	"net/http"
	"net/url"

	"bitbucket.org/ww/goautoneg"
	"github.com/codegangsta/martini"
	"github.com/codegangsta/martini-contrib/binding"
	"github.com/codegangsta/martini-contrib/render"
	"github.com/speps/go-hashids"
)

type Config struct {
	Backend      Backend
	HashSalt     string
	URLPrefix    string
	RootRedirect string
	ReportErr    func(err error, req *http.Request)
}

func New(c *Config) *martini.Martini {
	r := martini.NewRouter()
	m := martini.New()
	m.Action(r.Handle)

	ctx := &context{
		Backend:   c.Backend,
		ShortHash: hashids.New(),
		LongHash:  hashids.New(),
		URLPrefix: c.URLPrefix,
		ReportErr: c.ReportErr,
	}
	if c.HashSalt == "" {
		c.HashSalt = "salt"
	}
	ctx.ShortHash.Salt = c.HashSalt
	ctx.LongHash.Salt = c.HashSalt
	ctx.LongHash.MinLength = 12
	m.Map(ctx)
	m.Use(render.Renderer())

	r.Get("/", func(r *http.Request, w http.ResponseWriter) {
		http.Redirect(w, r, c.RootRedirect, http.StatusFound)
	})
	r.Post("/", binding.Bind(link{}), createLink)
	r.Get("/:code", getLink)

	return m
}

var ErrNotFound = errors.New("snowdrift: not found")
var ErrURLExists = errors.New("snowdrift: url already exists")
var ErrCodeExists = errors.New("snowdrift: code already exists")

type Backend interface {
	Add(url, digest, code string) error
	GetCode(digest string) (string, error)
	GetURL(code string) (string, error)
	NextID() (int, error)
}

type link struct {
	LongURL  string `json:"long_url" riak:"long_url"`
	ShortURL string `json:"short_url" riak:"-"`
	Obscure  *bool  `json:"obscure,omitempty" riak:"-"`
}

type context struct {
	Backend
	ShortHash *hashids.HashID
	LongHash  *hashids.HashID
	URLPrefix string
	ReportErr func(err error, req *http.Request)
}

func urlDigest(url string) string {
	digest := sha512.Sum512([]byte(url))
	return hex.EncodeToString(digest[:32])
}

func createLink(c *context, link link, r render.Render, req *http.Request) {
	u, err := url.Parse(link.LongURL)
	if err != nil || u.Scheme != "http" && u.Scheme != "https" {
		r.Error(400)
		return
	}

	digest := urlDigest(link.LongURL)

	code, err := c.GetCode(digest)
	if err == nil {
		link.ShortURL = c.URLPrefix + code
		link.Obscure = nil
		r.JSON(200, link)
		return
	}

	id, err := c.NextID()
	if err != nil {
		r.Error(500)
		if c.ReportErr != nil {
			c.ReportErr(err, req)
		}
		return
	}
	if link.Obscure != nil && *link.Obscure {
		code = c.LongHash.Encrypt([]int{id})
	} else {
		code = c.ShortHash.Encrypt([]int{id})
	}

	if err := c.Add(link.LongURL, digest, code); err != nil {
		r.Error(500)
		if c.ReportErr != nil {
			c.ReportErr(err, req)
		}
		return
	}
	link.Obscure = nil
	link.ShortURL = c.URLPrefix + code
	r.JSON(200, link)
}

func getLink(c *context, params martini.Params, r render.Render, req *http.Request, w http.ResponseWriter) {
	url, err := c.GetURL(params["code"])
	if err == ErrNotFound {
		http.NotFound(w, req)
		return
	}
	if err != nil {
		r.Error(500)
		if c.ReportErr != nil {
			c.ReportErr(err, req)
		}
		return
	}
	contentType := goautoneg.Negotiate(req.Header.Get("Accept"), []string{"text/html", "application/json"})
	if contentType == "application/json" {
		r.JSON(200, link{LongURL: url, ShortURL: c.URLPrefix + params["code"]})
		return
	}

	http.Redirect(w, req, url, http.StatusMovedPermanently)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/vanng822/uploader"
	"net/http"
	"os"
)

const (
	STORAGE_TYPE_FILE = "file"
)

type StorageConfig struct {
	Type          string
	Configuration string
}

type EndpointConfig struct {
	Endpoint  string
	FileField string
	Storage   *StorageConfig
}

type Config struct {
	Host      string
	Port      int
	Endpoints []*EndpointConfig
}

func LoadConfig(filename string) *Config {
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	conf := Config{}
	err = decoder.Decode(&conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

func main() {

	var (
		config string
		host   string
		port   int
	)

	flag.StringVar(&host, "h", "127.0.0.1", "Host to listen on")
	flag.IntVar(&port, "p", 8080, "Port number to listen on")
	flag.StringVar(&config, "c", "./config/app.json", "Path to configurations")

	conf := LoadConfig(config)

	m := martini.Classic()

	for _, endpoint := range conf.Endpoints {

		go func(endpoint *EndpointConfig) {
			var storage uploader.ImageStorage
			switch endpoint.Storage.Type {
			case STORAGE_TYPE_FILE:
				storage = uploader.NewImageStorageFile(endpoint.Storage.Configuration)
			default:
				panic(fmt.Sprintf("Unsupported storage type %s", endpoint.Storage.Type))
			}
			u := uploader.NewUploader(storage)
			handler := uploader.NewHandler(u)

			m.Group(endpoint.Endpoint, func(r martini.Router) {
				r.Get("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleGet(res, params["filename"])
				})
				r.Post("/", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePost(res, file)
				})
				r.Put("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePut(res, file, params["filename"])
				})
				r.Delete("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleDelete(res, params["filename"])
				})
			})
		}(endpoint)
	}

	http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Host, conf.Port), m)
}
<commit_msg>Use storage factory<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/vanng822/uploader"
	"net/http"
	"os"
)

const (
	STORAGE_TYPE_FILE = "file"
)

type StorageConfig struct {
	Type          string
	Configuration string
}

type EndpointConfig struct {
	Endpoint  string
	FileField string
	Storage   *StorageConfig
}

type Config struct {
	Host      string
	Port      int
	Endpoints []*EndpointConfig
}

func LoadConfig(filename string) *Config {
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	conf := Config{}
	err = decoder.Decode(&conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

func main() {

	var (
		config string
		host   string
		port   int
	)

	flag.StringVar(&host, "h", "", "Host to listen on")
	flag.IntVar(&port, "p", 0, "Port number to listen on")
	flag.StringVar(&config, "c", "./config/app.json", "Path to configurations")
	flag.Parse()

	conf := LoadConfig(config)

	if host != "" {
		conf.Host = host
	}

	if port != 0 {
		conf.Port = port
	}

	m := martini.Classic()
	for _, endpoint := range conf.Endpoints {

		go func(endpoint *EndpointConfig) {
			u := uploader.NewUploader(uploader.GetStorage(endpoint.Storage.Type, endpoint.Storage.Configuration))
			handler := uploader.NewHandler(u)

			m.Group(endpoint.Endpoint, func(r martini.Router) {
				r.Get("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleGet(res, params["filename"])
				})
				r.Post("/", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePost(res, file)
				})
				r.Put("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePut(res, file, params["filename"])
				})
				r.Delete("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleDelete(res, params["filename"])
				})
			})
		}(endpoint)
	}

	http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Host, conf.Port), m)
}
<|endoftext|>


<commit_before>package lazycache

import (
	"fmt"
	kitlog "github.com/go-kit/kit/log"
	"io"
	"net/http"
	"os"
	"path"
	"sync"
)

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string
	logger    kitlog.Logger
	cache     map[string]int

	mutex 		sync.Mutex

	Stats struct {
		cacheRequests int
		cacheMisses   int
	}
}

func (store *LocalImageStore) Has(key string) bool {
	filename := store.LocalRoot + key

	store.mutex.Lock()
	defer store.mutex.Unlock()

	_, has := store.cache[filename]
	if has {
		store.logger.Log("level", "debug", "msg", fmt.Sprintf("Image exists in cache: %s", filename))
		store.cache[filename]++
		return true
	}

	store.logger.Log("level", "debug", "msg", fmt.Sprintf("Checking local image store for \"%s\"", filename))
	_, err := os.Stat(filename)
	if err == nil {
		store.cache[filename] = 1
		return true
	}

	return false
}

func (store *LocalImageStore) Url(key string) (string, bool) {

	store.Stats.cacheRequests++

	if store.Has(key) {
		return store.UrlRoot + key, true
	} else {
		store.Stats.cacheMisses++
		return "", false
	}
}

func RecursiveMkdir(dir string) {
	_, err := os.Stat(dir)
	if err != nil {
		RecursiveMkdir(path.Dir(dir))

		os.Mkdir(dir, 0755)
	}
}

func (store *LocalImageStore) Store(key string, data io.Reader) {
	filename := store.LocalRoot + key
	RecursiveMkdir(path.Dir(filename))

	f, err := os.Create(filename)
	if err != nil {
		store.logger.Log("msg", err.Error(), "type", "error")
	}

	store.mutex.Lock()
	defer store.mutex.Unlock()

	store.cache[filename] = 1
	io.Copy(f, data)
}

func (store LocalImageStore) Retrieve(key string) (io.Reader, error) {

	f, err := os.Open(store.LocalRoot + key)

	return f, err
}

func (store LocalImageStore) Statistics() interface{} {
	return struct {
		Type          string
		CacheRequests int `json: "cache_requests"`
		CacheMisses   int `json: "cache_misses"`
	}{
		Type:          "local_storage",
		CacheRequests: store.Stats.cacheRequests,
		CacheMisses:   store.Stats.cacheMisses,
	}
}

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	localPath := path.Join(store.LocalRoot, r.URL.Path)

	if _, err := os.Stat(localPath); err != nil {
		http.Error(w, fmt.Sprintf("Could not find \"%s\"", localPath), 404)
	} else {
		http.ServeFile(w, r, localPath)
	}
}

func CreateLocalStore(localRoot string, addr string) *LocalImageStore {

	// port := 7080
	// addr := fmt.Sprintf("%s:%d", host, port)

	store := &LocalImageStore{
		LocalRoot: localRoot,
		UrlRoot:   addr,
		logger:    kitlog.With(DefaultLogger, "module", "LocalImageStore"),
		cache:     make(map[string]int),
	}

	DefaultLogger.Log("msg",
		fmt.Sprintf("Creating local image store at \"%s\", exposed at \"%s\"\n", store.LocalRoot, store.UrlRoot))

	s := &http.Server{
		Addr:    addr,
		Handler: store,
	}

	go s.ListenAndServe()

	return store
}
<commit_msg>gofmt on local_image_store.go<commit_after>package lazycache

import (
	"fmt"
	kitlog "github.com/go-kit/kit/log"
	"io"
	"net/http"
	"os"
	"path"
	"sync"
)

type LocalImageStore struct {
	LocalRoot string
	UrlRoot   string
	logger    kitlog.Logger
	cache     map[string]int

	mutex sync.Mutex

	Stats struct {
		cacheRequests int
		cacheMisses   int
	}
}

func (store *LocalImageStore) Has(key string) bool {
	filename := store.LocalRoot + key

	store.mutex.Lock()
	defer store.mutex.Unlock()

	_, has := store.cache[filename]
	if has {
		store.logger.Log("level", "debug", "msg", fmt.Sprintf("Image exists in cache: %s", filename))
		store.cache[filename]++
		return true
	}

	store.logger.Log("level", "debug", "msg", fmt.Sprintf("Checking local image store for \"%s\"", filename))
	_, err := os.Stat(filename)
	if err == nil {
		store.cache[filename] = 1
		return true
	}

	return false
}

func (store *LocalImageStore) Url(key string) (string, bool) {

	store.Stats.cacheRequests++

	if store.Has(key) {
		return store.UrlRoot + key, true
	} else {
		store.Stats.cacheMisses++
		return "", false
	}
}

func RecursiveMkdir(dir string) {
	_, err := os.Stat(dir)
	if err != nil {
		RecursiveMkdir(path.Dir(dir))

		os.Mkdir(dir, 0755)
	}
}

func (store *LocalImageStore) Store(key string, data io.Reader) {
	filename := store.LocalRoot + key
	RecursiveMkdir(path.Dir(filename))

	f, err := os.Create(filename)
	if err != nil {
		store.logger.Log("msg", err.Error(), "type", "error")
	}

	store.mutex.Lock()
	defer store.mutex.Unlock()

	store.cache[filename] = 1
	io.Copy(f, data)
}

func (store LocalImageStore) Retrieve(key string) (io.Reader, error) {

	f, err := os.Open(store.LocalRoot + key)

	return f, err
}

func (store LocalImageStore) Statistics() interface{} {
	return struct {
		Type          string
		CacheRequests int `json: "cache_requests"`
		CacheMisses   int `json: "cache_misses"`
	}{
		Type:          "local_storage",
		CacheRequests: store.Stats.cacheRequests,
		CacheMisses:   store.Stats.cacheMisses,
	}
}

func (store LocalImageStore) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	localPath := path.Join(store.LocalRoot, r.URL.Path)

	if _, err := os.Stat(localPath); err != nil {
		http.Error(w, fmt.Sprintf("Could not find \"%s\"", localPath), 404)
	} else {
		http.ServeFile(w, r, localPath)
	}
}

func CreateLocalStore(localRoot string, addr string) *LocalImageStore {

	// port := 7080
	// addr := fmt.Sprintf("%s:%d", host, port)

	store := &LocalImageStore{
		LocalRoot: localRoot,
		UrlRoot:   addr,
		logger:    kitlog.With(DefaultLogger, "module", "LocalImageStore"),
		cache:     make(map[string]int),
	}

	DefaultLogger.Log("msg",
		fmt.Sprintf("Creating local image store at \"%s\", exposed at \"%s\"\n", store.LocalRoot, store.UrlRoot))

	s := &http.Server{
		Addr:    addr,
		Handler: store,
	}

	go s.ListenAndServe()

	return store
}
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

// The data package provides the mlab-ns2 datastore sturcture.
package data

import (
	"fmt"
	"time"
)

//TODO: Data interface and Get,Put,Cache,Rm functions?
//TODO: only index the columns that are needed
//TODO: add json tags

type SliverTool struct {
	ToolID                 string    `datastore:"tool_id"`
	SliceID                string    `datastore:"slice_id"`
	SiteID                 string    `datastore:"site_id"`
	ServerID               string    `datastore:"server_id"`
	ServerPort             string    `datastore:"server_port"`
	HTTPPort               string    `datastore:"http_port"`                // For web-based tools, this is used to build the URL the client is redirected to: http://fqdn[ipv4|ipv6]:http_port
	FQDN                   string    `datastore:"fqdn"`                     // Unannotated fqdn. v4 and v6 versions can be built if necessary.
	SliverIPv4             string    `datastore:"sliver_ipv4"`              // IP addresses. Can be 'off'
	SliverIPv6             string    `datastore:"sliver_ipv6"`              // IP addresses. Can be 'off'
	StatusIPv4             string    `datastore:"status_ipv4"`              // These can have the following values: online and offline.
	StatusIPv6             string    `datastore:"status_ipv6"`              // These can have the following values: online and offline.
	UpdateRequestTimestamp int64     `datastore:"update_request_timestamp"` // To avoid an additional lookup in the datastore
	Latitude               float64   `datastore:"latitude"`                 // To avoid an additional lookup in the datastore
	Longitude              float64   `datastore:"longitude"`                // To avoid an additional lookup in the datastore
	City                   string    `datastore:"city"`                     // To avoid an additional lookup in the datastore
	Country                string    `datastore:"country"`                  // To avoid an additional lookup in the datastore
	When                   time.Time `datastore:"when"`                     // Date representing the last modification time of this entity.
}

type Site struct {
	SiteID                string    `datastore:"site_id"`
	City                  string    `datastore:"city"`
	Country               string    `datastore:"country"`
	Latitude              float64   `datastore:"latitude"`               // Latitude of the airport that uniquely identifies an M-Lab site.
	Longitude             float64   `datastore:"longitude"`              // Longitude of the airport that uniquely identifies an M-Lab site.
	Metro                 []string  `datastore:"metro"`                  // List of sites and metros, e.g., [ath, ath01].
	RegistrationTimestamp int64     `datastore:"registration_timestamp"` // Date representing the registration time (the first time a new site is added to mlab-ns).
	When                  time.Time `datastore:"when"`                   // Date representing the last modification time of this entity.
}

// MMLocation is a format that comes from pre-processed geolocation data.  It is
// intended to be used with the iptrie location map.
//
// The utility takes maximind data as input, pre-processes the data so that it
// has just the amount of detail that we need and produces a binary file that is
// uploaded to the GAE instance.  On upload the handler will copy it into the
// blobstore and update the datastore with this format.
//
//TODO: reference to the location map compression utility.
//TODO: IPv6 addresses are truncated to a /64 before inclusion?
type MMLocation struct {
	RangeStart int64 // first IP address in the block
	RangeEnd   int64 // last IP address in the block
	Latitude   int   // latitude rounded to the nearest integer
	Longitude  int   // longitude rounded to the nearest integer
}

//XXX deprecated
type MaxmindCityLocation struct {
	LocationID string    `datastore:"location_id"`
	Country    string    `datastore:"country"`
	Region     string    `datastore:"region"`
	City       string    `datastore:"city"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlock struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	LocationID string    `datastore:"location_id"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlockv6 struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	Country    string    `datastore:"country"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type CountryCode struct {
	Name        string    `datastore:"name"`
	Alpha2Code  string    `datastore:"alpha2_code"`
	Alpha3Code  string    `datastore:"alpha3_code"`
	NumericCode int64     `datastore:"numeric_code"`
	Latitude    float64   `datastore:"latitude"`
	Longitude   float64   `datastore:"longitude"`
	When        time.Time `datastore:"when"`
}

type EncryptionKey struct {
	KeyID         string `datastore:"key_id"`         // Name of the key (by default is 'admin').
	EncryptionKey string `datastore:"encryption_key"` // 16 bytes encryption key (AES).
}

type Slice struct {
	SliceID string `datastore:"slice_id"`
	ToolID  string `datastore:"tool_id"`
}

type Tool struct {
	SliceID  string `datastore:"slice_id"`
	ToolID   string `datastore:"tool_id"`
	HTTPPort string `datastore:"http_port"`
}

//TODO(gavaletz): generalize this to credentials?
type Nagios struct {
	KeyID    string `datastore:"key_id"`
	Username string `datastore:"username"`
	Password string `datastore:"password"`
	URL      string `datastore:"url"`
}

type Ping struct {
	Latitude      float64 `datastore:"latitude"`
	Longitude     float64 `datastore:"longitude"`
	ToolID        string  `datastore:"tool_id"`
	AddressFamily string  `datastore:"address_family"`
	Time          float64 `datastore:"time"`
}

func GetSliverToolID(toolID, sliceID, serverID, siteID string) string {
	return fmt.Sprintf("%s-%s-%s-%s", toolID, sliceID, serverID, siteID)
}
<commit_msg>data: Added json tags to Site struct<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

// The data package provides the mlab-ns2 datastore sturcture.
package data

import (
	"fmt"
	"time"
)

//TODO: Data interface and Get,Put,Cache,Rm functions?
//TODO: only index the columns that are needed
//TODO: add json tags

type SliverTool struct {
	ToolID                 string    `datastore:"tool_id"`
	SliceID                string    `datastore:"slice_id"`
	SiteID                 string    `datastore:"site_id"`
	ServerID               string    `datastore:"server_id"`
	ServerPort             string    `datastore:"server_port"`
	HTTPPort               string    `datastore:"http_port"`                // For web-based tools, this is used to build the URL the client is redirected to: http://fqdn[ipv4|ipv6]:http_port
	FQDN                   string    `datastore:"fqdn"`                     // Unannotated fqdn. v4 and v6 versions can be built if necessary.
	SliverIPv4             string    `datastore:"sliver_ipv4"`              // IP addresses. Can be 'off'
	SliverIPv6             string    `datastore:"sliver_ipv6"`              // IP addresses. Can be 'off'
	StatusIPv4             string    `datastore:"status_ipv4"`              // These can have the following values: online and offline.
	StatusIPv6             string    `datastore:"status_ipv6"`              // These can have the following values: online and offline.
	UpdateRequestTimestamp int64     `datastore:"update_request_timestamp"` // To avoid an additional lookup in the datastore
	Latitude               float64   `datastore:"latitude"`                 // To avoid an additional lookup in the datastore
	Longitude              float64   `datastore:"longitude"`                // To avoid an additional lookup in the datastore
	City                   string    `datastore:"city"`                     // To avoid an additional lookup in the datastore
	Country                string    `datastore:"country"`                  // To avoid an additional lookup in the datastore
	When                   time.Time `datastore:"when"`                     // Date representing the last modification time of this entity.
}

type Site struct {
	SiteID                string    `datastore:"site_id" json:"site"`
	City                  string    `datastore:"city" json:"city"`
	Country               string    `datastore:"country" json:"country"`
	Latitude              float64   `datastore:"latitude" json:"latitude"`              // Latitude of the airport that uniquely identifies an M-Lab site.
	Longitude             float64   `datastore:"longitude" json:"longitude"`            // Longitude of the airport that uniquely identifies an M-Lab site.
	Metro                 []string  `datastore:"metro" json:"metro"`                    // List of sites and metros, e.g., [ath, ath01].
	RegistrationTimestamp int64     `datastore:"registration_timestamp" json:"created"` // Date representing the registration time (the first time a new site is added to mlab-ns).
	When                  time.Time `datastore:"when" json:"-"`                         // Date representing the last modification time of this entity.
}

// MMLocation is a format that comes from pre-processed geolocation data.  It is
// intended to be used with the iptrie location map.
//
// The utility takes maximind data as input, pre-processes the data so that it
// has just the amount of detail that we need and produces a binary file that is
// uploaded to the GAE instance.  On upload the handler will copy it into the
// blobstore and update the datastore with this format.
//
//TODO: reference to the location map compression utility.
//TODO: IPv6 addresses are truncated to a /64 before inclusion?
type MMLocation struct {
	RangeStart int64 // first IP address in the block
	RangeEnd   int64 // last IP address in the block
	Latitude   int   // latitude rounded to the nearest integer
	Longitude  int   // longitude rounded to the nearest integer
}

//XXX deprecated
type MaxmindCityLocation struct {
	LocationID string    `datastore:"location_id"`
	Country    string    `datastore:"country"`
	Region     string    `datastore:"region"`
	City       string    `datastore:"city"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlock struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	LocationID string    `datastore:"location_id"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type MaxmindCityBlockv6 struct {
	StartIPNum int64     `datastore:"start_ip_num"`
	EndIPNum   int64     `datastore:"end_ip_num"`
	Country    string    `datastore:"country"`
	Latitude   float64   `datastore:"latitude"`
	Longitude  float64   `datastore:"longitude"`
	When       time.Time `datastore:"when"`
}

//XXX deprecated
type CountryCode struct {
	Name        string    `datastore:"name"`
	Alpha2Code  string    `datastore:"alpha2_code"`
	Alpha3Code  string    `datastore:"alpha3_code"`
	NumericCode int64     `datastore:"numeric_code"`
	Latitude    float64   `datastore:"latitude"`
	Longitude   float64   `datastore:"longitude"`
	When        time.Time `datastore:"when"`
}

type EncryptionKey struct {
	KeyID         string `datastore:"key_id"`         // Name of the key (by default is 'admin').
	EncryptionKey string `datastore:"encryption_key"` // 16 bytes encryption key (AES).
}

type Slice struct {
	SliceID string `datastore:"slice_id"`
	ToolID  string `datastore:"tool_id"`
}

type Tool struct {
	SliceID  string `datastore:"slice_id"`
	ToolID   string `datastore:"tool_id"`
	HTTPPort string `datastore:"http_port"`
}

//TODO(gavaletz): generalize this to credentials?
type Nagios struct {
	KeyID    string `datastore:"key_id"`
	Username string `datastore:"username"`
	Password string `datastore:"password"`
	URL      string `datastore:"url"`
}

type Ping struct {
	Latitude      float64 `datastore:"latitude"`
	Longitude     float64 `datastore:"longitude"`
	ToolID        string  `datastore:"tool_id"`
	AddressFamily string  `datastore:"address_family"`
	Time          float64 `datastore:"time"`
}

func GetSliverToolID(toolID, sliceID, serverID, siteID string) string {
	return fmt.Sprintf("%s-%s-%s-%s", toolID, sliceID, serverID, siteID)
}
<|endoftext|>


<commit_before>package locket_test

import (
	"code.cloudfoundry.org/consuladapter/consulrunner"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"

	"testing"
)

var (
	consulStartingPort int
	consulRunner       *consulrunner.ClusterRunner
)

const (
	defaultScheme = "http"
)

func TestLocket(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Locket Suite")
}

var _ = BeforeSuite(func() {
	consulStartingPort = 5001 + config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength
	consulRunner = consulrunner.NewClusterRunner(consulStartingPort, 1, defaultScheme)

	consulRunner.Start()
	consulRunner.WaitUntilReady()
})

var _ = BeforeEach(func() {
	consulRunner.Reset()
})

var _ = AfterSuite(func() {
	consulRunner.Stop()
})
<commit_msg>pass ClusterRunnerConfig to NewClusterRunner<commit_after>package locket_test

import (
	"code.cloudfoundry.org/consuladapter/consulrunner"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"

	"testing"
)

var (
	consulStartingPort int
	consulRunner       *consulrunner.ClusterRunner
)

const (
	defaultScheme = "http"
)

func TestLocket(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Locket Suite")
}

var _ = BeforeSuite(func() {
	consulStartingPort = 5001 + config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength
	consulRunner = consulrunner.NewClusterRunner(
		consulrunner.ClusterRunnerConfig{
			StartingPort: consulStartingPort,
			NumNodes:     1,
			Scheme:       defaultScheme,
		},
	)

	consulRunner.Start()
	consulRunner.WaitUntilReady()
})

var _ = BeforeEach(func() {
	consulRunner.Reset()
})

var _ = AfterSuite(func() {
	consulRunner.Stop()
})
<|endoftext|>


<commit_before>package revert

// Reverter is a helper type to manage revert functions.
type Reverter struct {
	revertFuncs []func()
}

// New returns a new Reverter.
func New() *Reverter {
	return &Reverter{}
}

// Add adds a revert function to the list to be run when Revert() is called.
func (r *Reverter) Add(f func()) {
	r.revertFuncs = append(r.revertFuncs, f)
}

// Fail runs any revert functions in the reverse order they were added.
// Should be used with defer or when a task has encountered an error and needs to be reverted.
func (r *Reverter) Fail() {
	funcCount := len(r.revertFuncs)
	for k := range r.revertFuncs {
		// Run the revert functions in reverse order.
		k = funcCount - 1 - k
		r.revertFuncs[k]()
	}
}

// Success clears the revert functions previously added.
// Should be called on successful completion of a task to prevent revert functions from being run.
func (r *Reverter) Success() {
	r.revertFuncs = nil
}

// Clone returns a copy of the reverter with the current set of revert functions added.
// This can be used if you want to return a reverting function to an external caller but do not want to actually
// execute the previously deferred reverter.Fail() function.
func (r *Reverter) Clone() *Reverter {
	rNew := New()
	rNew.revertFuncs = append(make([]func(), 0, len(r.revertFuncs)), r.revertFuncs...)

	return rNew
}
<commit_msg>lxd/revert/revert: Add Hook function type<commit_after>package revert

// Hook is a function that can be added to the revert via the Add() function.
// These will be run in the reverse order that they were added if the reverter's Fail() function is called.
type Hook func()

// Reverter is a helper type to manage revert functions.
type Reverter struct {
	revertFuncs []Hook
}

// New returns a new Reverter.
func New() *Reverter {
	return &Reverter{}
}

// Add adds a revert function to the list to be run when Revert() is called.
func (r *Reverter) Add(f Hook) {
	r.revertFuncs = append(r.revertFuncs, f)
}

// Fail runs any revert functions in the reverse order they were added.
// Should be used with defer or when a task has encountered an error and needs to be reverted.
func (r *Reverter) Fail() {
	funcCount := len(r.revertFuncs)
	for k := range r.revertFuncs {
		// Run the revert functions in reverse order.
		k = funcCount - 1 - k
		r.revertFuncs[k]()
	}
}

// Success clears the revert functions previously added.
// Should be called on successful completion of a task to prevent revert functions from being run.
func (r *Reverter) Success() {
	r.revertFuncs = nil
}

// Clone returns a copy of the reverter with the current set of revert functions added.
// This can be used if you want to return a reverting function to an external caller but do not want to actually
// execute the previously deferred reverter.Fail() function.
func (r *Reverter) Clone() *Reverter {
	rNew := New()
	rNew.revertFuncs = append(make([]Hook, 0, len(r.revertFuncs)), r.revertFuncs...)

	return rNew
}
<|endoftext|>


<commit_before>// Package sliceDiff provides functions to determine the difference
// between two slices. There are tests and limited benchmarks for this package.
//
// THIS IS INTENDED ONLY FOR "SMALL" SLICES.
//
// Using this package on slices > 30,000 entries can get slow.. I would
// suggest benchmarking the number / type of slices you intend to use
// in sliceDiff_test.go *before* using on slices with a large number
// of entries.
//
// Use of this source code is governed by a BSD-style license
// which can be found in the LICENSE file
package sliceDiff

// Int64SliceDiff returns the int64 values that are not in
// both source slices
func Int64SliceDiff(sliceOne []int64, sliceTwo []int64) []int64 {
	var diff []int64
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// Uint64SliceDiff returns the uint64 values that are not in
// both source slices
func Uint64SliceDiff(sliceOne []uint64, sliceTwo []uint64) []uint64 {
    var diff []uint64
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Int32SliceDiff returns the int32 values that are not in
// both soruce slices
func Int32SliceDiff(sliceOne []int32, sliceTwo []int32) []int32 {
	var diff []int32
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// Uint32SliceDiff returns the uint32 values that are not in
// both soruce slices
func Uint32SliceDiff(sliceOne []uint32, sliceTwo []uint32) []uint32 {
    var diff []uint32
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Int16SliceDiff returns the int16 values that are not in
// both soruce slices
func Int16SliceDiff(sliceOne []int16, sliceTwo []int16) []int16 {
    var diff []int16
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Uint16SliceDiff returns the uint16 values that are not in
// both soruce slices
func Uint16SliceDiff(sliceOne []uint16, sliceTwo []uint16) []uint16 {
    var diff []uint16
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Int8SliceDiff returns the int16 values that are not in
// both soruce slices
func Int8SliceDiff(sliceOne []int8, sliceTwo []int8) []int8 {
    var diff []int8
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Uint8SliceDiff returns the uint16 values that are not in
// both soruce slices
func Uint8SliceDiff(sliceOne []uint8, sliceTwo []uint8) []uint8 {
    var diff []uint8
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// StringSliceDiff returns the string values that are not in
// both source slices
func StringSliceDiff(sliceOne []string, sliceTwo []string) []string {
	var diff []string
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}
<commit_msg>Fixed documentation error<commit_after>// Package sliceDiff provides functions to determine the difference
// between two slices. There are tests and limited benchmarks for this package.
//
// THIS IS INTENDED ONLY FOR "SMALL" SLICES.
//
// Using this package on slices > 30,000 entries can get slow.. I would
// suggest benchmarking the number / type of slices you intend to use
// in sliceDiff_test.go *before* using on slices with a large number
// of entries.
//
// Use of this source code is governed by a BSD-style license
// which can be found in the LICENSE file
package sliceDiff

// Int64SliceDiff returns the int64 values that are not in
// both source slices
func Int64SliceDiff(sliceOne []int64, sliceTwo []int64) []int64 {
	var diff []int64
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// Uint64SliceDiff returns the uint64 values that are not in
// both source slices
func Uint64SliceDiff(sliceOne []uint64, sliceTwo []uint64) []uint64 {
    var diff []uint64
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Int32SliceDiff returns the int32 values that are not in
// both soruce slices
func Int32SliceDiff(sliceOne []int32, sliceTwo []int32) []int32 {
	var diff []int32
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}

// Uint32SliceDiff returns the uint32 values that are not in
// both soruce slices
func Uint32SliceDiff(sliceOne []uint32, sliceTwo []uint32) []uint32 {
    var diff []uint32
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Int16SliceDiff returns the int16 values that are not in
// both soruce slices
func Int16SliceDiff(sliceOne []int16, sliceTwo []int16) []int16 {
    var diff []int16
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Uint16SliceDiff returns the uint16 values that are not in
// both soruce slices
func Uint16SliceDiff(sliceOne []uint16, sliceTwo []uint16) []uint16 {
    var diff []uint16
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Int8SliceDiff returns the int8 values that are not in
// both soruce slices
func Int8SliceDiff(sliceOne []int8, sliceTwo []int8) []int8 {
    var diff []int8
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// Uint8SliceDiff returns the uint8 values that are not in
// both soruce slices
func Uint8SliceDiff(sliceOne []uint8, sliceTwo []uint8) []uint8 {
    var diff []uint8
    for i := 0; i < 2; i++ {
        for _, s1 := range sliceOne {
            found := false
            for _, s2 := range sliceTwo {
                if s1 == s2 {
                    found = true
                    break
                }
            }
            if !found {
                diff = append(diff,s1)
            }
        }
        if i == 0 {
            sliceOne, sliceTwo = sliceTwo, sliceOne
        }
    }
    return diff
}

// StringSliceDiff returns the string values that are not in
// both source slices
func StringSliceDiff(sliceOne []string, sliceTwo []string) []string {
	var diff []string
	for i := 0; i < 2; i++ {
		for _, s1 := range sliceOne {
			found := false
			for _, s2 := range sliceTwo {
				if s1 == s2 {
					found = true
					break
				}
			}
			if !found {
				diff = append(diff,s1)
			}
		}
		if i == 0 {
			sliceOne, sliceTwo = sliceTwo, sliceOne
		}
	}
	return diff
}
<|endoftext|>