<commit_before>package google

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/url"
	"strings"

	"google.golang.org/api/googleapi"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

type RetryErrorPredicateFunc func(error) (bool, string)

/** ADD GLOBAL ERROR RETRY PREDICATES HERE **/
// Retry predicates that shoud apply to all requests should be added here.
var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
	// Common network errors (usually wrapped by URL error)
	isNetworkTemporaryError,
	isNetworkTimeoutError,
	isIoEOFError,
	isConnectionResetNetworkError,

	// Common GCP error codes
	isCommonRetryableErrorCode,

	//While this might apply only to Cloud SQL, historically,
	// we had this in our global default error retries.
	// Keeping it as a default for now.
	is409OperationInProgressError,
}

/** END GLOBAL ERROR RETRY PREDICATES HERE **/

func isNetworkTemporaryError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Temporary() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Temporary() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isNetworkTimeoutError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Timeout() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Timeout() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isIoEOFError(err error) (bool, string) {
	if err == io.ErrUnexpectedEOF {
		return true, "Got unexpected EOF"
	}

	if urlerr, urlok := err.(*url.Error); urlok {
		wrappedErr := urlerr.Unwrap()
		if wrappedErr == io.ErrUnexpectedEOF {
			return true, "Got unexpected EOF"
		}
	}
	return false, ""
}

const connectionResetByPeerErr = ": connection reset by peer"

func isConnectionResetNetworkError(err error) (bool, string) {
	if strings.HasSuffix(err.Error(), connectionResetByPeerErr) {
		return true, fmt.Sprintf("reset connection error: %v", err)
	}
	return false, ""
}

// Retry 409s because some APIs like Cloud SQL throw a 409 if concurrent calls
// are being made.
//
//The only way right now to determine it is a retryable 409 due to
// concurrent calls is to look at the contents of the error message.
// See https://github.com/hashicorp/terraform-provider-google/issues/3279
func is409OperationInProgressError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 409 && strings.Contains(gerr.Body, "operationInProgress") {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code 409 and error reason 'operationInProgress': %s", err)
		return true, "Operation still in progress"
	}
	return false, ""
}

// Retry on comon googleapi error codes for retryable errors.
// TODO(#5609): This may not need to be applied globally - figure out
// what retryable error codes apply to which API.
func isCommonRetryableErrorCode(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503 {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code: %s", err)
		return true, fmt.Sprintf("Retryable error code %d", gerr.Code)
	}
	return false, ""
}

// We've encountered a few common fingerprint-related strings; if this is one of
// them, we're confident this is an error due to fingerprints.
var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}

// Retry the operation if it looks like a fingerprint mismatch.
func isFingerprintError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code != 412 {
		return false, ""
	}

	for _, msg := range FINGERPRINT_FAIL_ERRORS {
		if strings.Contains(err.Error(), msg) {
			return true, "fingerprint mismatch"
		}
	}

	return false, ""
}

// If a permission necessary to provision a resource is created in the same config
// as the resource itself, the permission may not have propagated by the time terraform
// attempts to create the resource. This allows those errors to be retried until the timeout expires
func iamMemberMissing(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "permission") {
			return true, "Waiting for IAM member permissions to propagate."
		}
	}
	return false, ""
}

// Cloud PubSub returns a 400 error if a topic's parent project was recently created and an
// organization policy has not propagated.
// See https://github.com/hashicorp/terraform-provider-google/issues/4349
func pubsubTopicProjectNotReady(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "retry this operation") {
			log.Printf("[DEBUG] Dismissed error as a retryable operation: %s", err)
			return true, "Waiting for Pubsub topic's project to properly initialize with organiation policy"
		}
	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlInternalError(err error) (bool, string) {
	if gerr, ok := err.(*SqlAdminOperationError); ok {
		// SqlAdminOperationError is a non-interface type so we need to cast it through
		// a layer of interface{}.  :)
		var ierr interface{}
		ierr = gerr
		if serr, ok := ierr.(*sqladmin.OperationErrors); ok && serr.Errors[0].Code == "INTERNAL_ERROR" {
			return true, "Received an internal error, which is sometimes retryable for some SQL resources.  Optimistically retrying."
		}

	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlOperationInProgressError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 {
		if strings.Contains(gerr.Body, "you cannot reuse the name of the deleted instance until one week from the deletion date.") {
			return false, ""
		}

		return true, "Waiting for other concurrent Cloud SQL operations to finish"
	}
	return false, ""
}

// Retry if Monitoring operation returns a 429 with a specific message for
// concurrent operations.
func isMonitoringConcurrentEditError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "too many concurrent edits") {
			return true, "Waiting for other Monitoring changes to finish"
		}
	}
	return false, ""
}

// Retry if App Engine operation returns a 429 with a specific message for
// concurrent operations.
func isAppEngineRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "operation is already in progress") {
			return true, "Waiting for other concurrent App Engine changes to finish"
		}
	}
	return false, ""
}

// Retry if KMS CryptoKeyVersions returns a 400 for PENDING_GENERATION
func isCryptoKeyVersionsPendingGeneration(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "PENDING_GENERATION") {
			return true, "Waiting for pending key generation"
		}
	}
	return false, ""
}

// Retry if getting a resource/operation returns a 404 for specific operations.
// opType should describe the operation for which 404 can be retryable.
func isNotFoundRetryableError(opType string) RetryErrorPredicateFunc {
	return func(err error) (bool, string) {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
			return true, fmt.Sprintf("Retry 404s for %s", opType)
		}
		return false, ""
	}
}

func isStoragePreconditionError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 412 {
		return true, fmt.Sprintf("Retry on storage precondition not met")
	}
	return false, ""
}

func isDataflowJobUpdateRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 404 && strings.Contains(gerr.Body, "in RUNNING OR DRAINING state") {
			return true, "Waiting for job to be in a valid state"
		}
	}
	return false, ""
}

func isPeeringOperationInProgress(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "There is a peering operation in progress") {
			return true, "Waiting peering operation to complete"
		}
	}
	return false, ""
}

func isCloudFunctionsSourceCodeError(err error) (bool, string) {
	if operr, ok := err.(*CommonOpError); ok {
		if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" {
			return true, fmt.Sprintf("Retry on Function failing to pull code from GCS")
		}
	}
	return false, ""
}

func datastoreIndex409Contention(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(gerr.Body, "too much contention") {
			return true, "too much contention - waiting for less activity"
		}
	}
	return false, ""
}
<commit_msg>Add retries for modified services to google_project_service (#3960) (#519)<commit_after>package google

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/url"
	"strings"

	"google.golang.org/api/googleapi"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

type RetryErrorPredicateFunc func(error) (bool, string)

/** ADD GLOBAL ERROR RETRY PREDICATES HERE **/
// Retry predicates that shoud apply to all requests should be added here.
var defaultErrorRetryPredicates = []RetryErrorPredicateFunc{
	// Common network errors (usually wrapped by URL error)
	isNetworkTemporaryError,
	isNetworkTimeoutError,
	isIoEOFError,
	isConnectionResetNetworkError,

	// Common GCP error codes
	isCommonRetryableErrorCode,

	//While this might apply only to Cloud SQL, historically,
	// we had this in our global default error retries.
	// Keeping it as a default for now.
	is409OperationInProgressError,
}

/** END GLOBAL ERROR RETRY PREDICATES HERE **/

func isNetworkTemporaryError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Temporary() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Temporary() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isNetworkTimeoutError(err error) (bool, string) {
	if netErr, ok := err.(*net.OpError); ok && netErr.Timeout() {
		return true, "marked as timeout"
	}
	if urlerr, ok := err.(*url.Error); ok && urlerr.Timeout() {
		return true, "marked as timeout"
	}
	return false, ""
}

func isIoEOFError(err error) (bool, string) {
	if err == io.ErrUnexpectedEOF {
		return true, "Got unexpected EOF"
	}

	if urlerr, urlok := err.(*url.Error); urlok {
		wrappedErr := urlerr.Unwrap()
		if wrappedErr == io.ErrUnexpectedEOF {
			return true, "Got unexpected EOF"
		}
	}
	return false, ""
}

const connectionResetByPeerErr = ": connection reset by peer"

func isConnectionResetNetworkError(err error) (bool, string) {
	if strings.HasSuffix(err.Error(), connectionResetByPeerErr) {
		return true, fmt.Sprintf("reset connection error: %v", err)
	}
	return false, ""
}

// Retry 409s because some APIs like Cloud SQL throw a 409 if concurrent calls
// are being made.
//
//The only way right now to determine it is a retryable 409 due to
// concurrent calls is to look at the contents of the error message.
// See https://github.com/hashicorp/terraform-provider-google/issues/3279
func is409OperationInProgressError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 409 && strings.Contains(gerr.Body, "operationInProgress") {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code 409 and error reason 'operationInProgress': %s", err)
		return true, "Operation still in progress"
	}
	return false, ""
}

// Retry on comon googleapi error codes for retryable errors.
// TODO(#5609): This may not need to be applied globally - figure out
// what retryable error codes apply to which API.
func isCommonRetryableErrorCode(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code == 429 || gerr.Code == 500 || gerr.Code == 502 || gerr.Code == 503 {
		log.Printf("[DEBUG] Dismissed an error as retryable based on error code: %s", err)
		return true, fmt.Sprintf("Retryable error code %d", gerr.Code)
	}
	return false, ""
}

// We've encountered a few common fingerprint-related strings; if this is one of
// them, we're confident this is an error due to fingerprints.
var FINGERPRINT_FAIL_ERRORS = []string{"Invalid fingerprint.", "Supplied fingerprint does not match current metadata fingerprint."}

// Retry the operation if it looks like a fingerprint mismatch.
func isFingerprintError(err error) (bool, string) {
	gerr, ok := err.(*googleapi.Error)
	if !ok {
		return false, ""
	}

	if gerr.Code != 412 {
		return false, ""
	}

	for _, msg := range FINGERPRINT_FAIL_ERRORS {
		if strings.Contains(err.Error(), msg) {
			return true, "fingerprint mismatch"
		}
	}

	return false, ""
}

// If a permission necessary to provision a resource is created in the same config
// as the resource itself, the permission may not have propagated by the time terraform
// attempts to create the resource. This allows those errors to be retried until the timeout expires
func iamMemberMissing(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "permission") {
			return true, "Waiting for IAM member permissions to propagate."
		}
	}
	return false, ""
}

// Cloud PubSub returns a 400 error if a topic's parent project was recently created and an
// organization policy has not propagated.
// See https://github.com/hashicorp/terraform-provider-google/issues/4349
func pubsubTopicProjectNotReady(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "retry this operation") {
			log.Printf("[DEBUG] Dismissed error as a retryable operation: %s", err)
			return true, "Waiting for Pubsub topic's project to properly initialize with organiation policy"
		}
	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlInternalError(err error) (bool, string) {
	if gerr, ok := err.(*SqlAdminOperationError); ok {
		// SqlAdminOperationError is a non-interface type so we need to cast it through
		// a layer of interface{}.  :)
		var ierr interface{}
		ierr = gerr
		if serr, ok := ierr.(*sqladmin.OperationErrors); ok && serr.Errors[0].Code == "INTERNAL_ERROR" {
			return true, "Received an internal error, which is sometimes retryable for some SQL resources.  Optimistically retrying."
		}

	}
	return false, ""
}

// Retry if Cloud SQL operation returns a 429 with a specific message for
// concurrent operations.
func isSqlOperationInProgressError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 409 {
		if strings.Contains(gerr.Body, "you cannot reuse the name of the deleted instance until one week from the deletion date.") {
			return false, ""
		}

		return true, "Waiting for other concurrent Cloud SQL operations to finish"
	}
	return false, ""
}

// Retry if service usage decides you're activating the same service multiple
// times. This can happen if a service and a dependent service aren't batched
// together- eg container.googleapis.com in one request followed by compute.g.c
// in the next (container relies on compute and implicitly activates it)
func serviceUsageServiceBeingActivated(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "Another activation or deactivation is in progress") {
			return false, ""
		}

		return true, "Waiting for same service activation/deactivation to finish"
	}
	return false, ""
}

// Retry if Monitoring operation returns a 429 with a specific message for
// concurrent operations.
func isMonitoringConcurrentEditError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "too many concurrent edits") {
			return true, "Waiting for other Monitoring changes to finish"
		}
	}
	return false, ""
}

// Retry if App Engine operation returns a 429 with a specific message for
// concurrent operations.
func isAppEngineRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(strings.ToLower(gerr.Body), "operation is already in progress") {
			return true, "Waiting for other concurrent App Engine changes to finish"
		}
	}
	return false, ""
}

// Retry if KMS CryptoKeyVersions returns a 400 for PENDING_GENERATION
func isCryptoKeyVersionsPendingGeneration(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
		if strings.Contains(gerr.Body, "PENDING_GENERATION") {
			return true, "Waiting for pending key generation"
		}
	}
	return false, ""
}

// Retry if getting a resource/operation returns a 404 for specific operations.
// opType should describe the operation for which 404 can be retryable.
func isNotFoundRetryableError(opType string) RetryErrorPredicateFunc {
	return func(err error) (bool, string) {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
			return true, fmt.Sprintf("Retry 404s for %s", opType)
		}
		return false, ""
	}
}

func isStoragePreconditionError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 412 {
		return true, fmt.Sprintf("Retry on storage precondition not met")
	}
	return false, ""
}

func isDataflowJobUpdateRetryableError(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 404 && strings.Contains(gerr.Body, "in RUNNING OR DRAINING state") {
			return true, "Waiting for job to be in a valid state"
		}
	}
	return false, ""
}

func isPeeringOperationInProgress(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 400 && strings.Contains(gerr.Body, "There is a peering operation in progress") {
			return true, "Waiting peering operation to complete"
		}
	}
	return false, ""
}

func isCloudFunctionsSourceCodeError(err error) (bool, string) {
	if operr, ok := err.(*CommonOpError); ok {
		if operr.Code == 3 && operr.Message == "Failed to retrieve function source code" {
			return true, fmt.Sprintf("Retry on Function failing to pull code from GCS")
		}
	}
	return false, ""
}

func datastoreIndex409Contention(err error) (bool, string) {
	if gerr, ok := err.(*googleapi.Error); ok {
		if gerr.Code == 409 && strings.Contains(gerr.Body, "too much contention") {
			return true, "too much contention - waiting for less activity"
		}
	}
	return false, ""
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"errors"
	"os/user"
)

func checkRoot() error {
	u, _ := user.Current()
	if u.Name != "root" {
		return errors.New("must be run as root")
	}
	return nil
}
<commit_msg>Check uid instead of username. (#513)<commit_after>/*
Copyright 2017 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"errors"
	"os"
)

func checkRoot() error {
	uid := os.Geteuid()
	if uid != 0 {
		return errors.New("must be run as root")
	}
	return nil
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"io"
)

func init() {
	Commands = append(Commands, &commandCollectionDelete{})
}

type commandCollectionDelete struct {
}

func (c *commandCollectionDelete) Name() string {
	return "collection.delete"
}

func (c *commandCollectionDelete) Help() string {
	return `delete specified collection

	collection.delete <collection_name>

`
}

func (c *commandCollectionDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if len(args) == 0 {
		return nil
	}

	collectionName := args[0]

	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		_, err = client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
			Name: collectionName,
		})
		return err
	})
	if err != nil {
		return
	}

	fmt.Fprintf(writer, "collection %s is deleted.\n", collectionName)

	return nil
}
<commit_msg>shell: adjust command option for collection.delete<commit_after>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"io"
)

func init() {
	Commands = append(Commands, &commandCollectionDelete{})
}

type commandCollectionDelete struct {
}

func (c *commandCollectionDelete) Name() string {
	return "collection.delete"
}

func (c *commandCollectionDelete) Help() string {
	return `delete specified collection

	collection.delete -collectin <collection_name> -force

`
}

func (c *commandCollectionDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	colDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	collectionName := colDeleteCommand.String("collection", "", "collection to delete")
	applyBalancing := colDeleteCommand.Bool("force", false, "apply the collection")
	if err = colDeleteCommand.Parse(args); err != nil {
		return nil
	}

	if !*applyBalancing {
		fmt.Fprintf(writer, "collection %s will be deleted. Use -force to apply the change.\n", collectionName)
		return nil
	}

	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		_, err = client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
			Name: *collectionName,
		})
		return err
	})
	if err != nil {
		return
	}

	fmt.Fprintf(writer, "collection %s is deleted.\n", collectionName)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009-2015 The freegeoip authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package apiserver

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	// embed pprof server.
	_ "net/http/pprof"

	"github.com/fiorix/freegeoip"
	"github.com/fiorix/go-redis/redis"
	gorilla "github.com/gorilla/handlers"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/http2"
)

// Version tag.
var Version = "3.0.7"

var maxmindDB = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

var (
	flAPIPrefix      = flag.String("api-prefix", "/", "Prefix for API endpoints")
	flCORSOrigin     = flag.String("cors-origin", "*", "CORS origin API endpoints")
	flHTTPAddr       = flag.String("http", ":8080", "Address in form of ip:port to listen on for HTTP")
	flHTTPSAddr      = flag.String("https", "", "Address in form of ip:port to listen on for HTTPS")
	flCertFile       = flag.String("cert", "cert.pem", "X.509 certificate file")
	flKeyFile        = flag.String("key", "key.pem", "X.509 key file")
	flReadTimeout    = flag.Duration("read-timeout", 30*time.Second, "Read timeout for HTTP and HTTPS client conns")
	flWriteTimeout   = flag.Duration("write-timeout", 15*time.Second, "Write timeout for HTTP and HTTPS client conns")
	flPublicDir      = flag.String("public", "", "Public directory to serve at the {prefix}/ endpoint")
	flDB             = flag.String("db", maxmindDB, "IP database file or URL")
	flUpdateIntvl    = flag.Duration("update", 24*time.Hour, "Database update check interval")
	flRetryIntvl     = flag.Duration("retry", time.Hour, "Max time to wait before retrying to download database")
	flUseXFF         = flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available (e.g. when running behind proxies)")
	flSilent         = flag.Bool("silent", false, "Do not log HTTP or HTTPS requests to stderr")
	flLogToStdout    = flag.Bool("logtostdout", false, "Log to stdout instead of stderr")
	flRedisAddr      = flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port[,ip:port] for quota")
	flRedisTimeout   = flag.Duration("redis-timeout", 500*time.Millisecond, "Redis read/write timeout")
	flQuotaMax       = flag.Int("quota-max", 0, "Max requests per source IP per interval; set 0 to turn off")
	flQuotaIntvl     = flag.Duration("quota-interval", time.Hour, "Quota expiration interval per source IP querying the API")
	flVersion        = flag.Bool("version", false, "Show version and exit")
	flInternalServer = flag.String("internal-server", "", "Address in form of ip:port to listen on for /metrics and /debug/pprof")
)

// Run is the entrypoint for the freegeoip daemon tool.
func Run() error {
	flag.Parse()

	if *flVersion {
		fmt.Printf("freegeoip v%s\n", Version)
		return nil
	}

	if *flLogToStdout {
		log.SetOutput(os.Stdout)
	}

	log.SetPrefix("[freegeoip] ")

	addrs := strings.Split(*flRedisAddr, ",")
	rc, err := redis.Dial(addrs...)
	if err != nil {
		return err
	}
	rc.Timeout = *flRedisTimeout

	db, err := openDB(*flDB, *flUpdateIntvl, *flRetryIntvl)
	if err != nil {
		return err
	}
	go watchEvents(db)

	ah := NewHandler(&HandlerConfig{
		Prefix:    *flAPIPrefix,
		Origin:    *flCORSOrigin,
		PublicDir: *flPublicDir,
		DB:        db,
		RateLimiter: RateLimiter{
			Redis:    rc,
			Max:      *flQuotaMax,
			Interval: *flQuotaIntvl,
		},
	})

	if !*flSilent {
		ah = gorilla.CombinedLoggingHandler(os.Stderr, ah)
	}

	if *flUseXFF {
		ah = freegeoip.ProxyHandler(ah)
	}

	if len(*flInternalServer) > 0 {
		http.Handle("/metrics", prometheus.Handler())
		log.Println("freegeoip internal server starting on", *flInternalServer)
		go func() { log.Fatal(http.ListenAndServe(*flInternalServer, nil)) }()
	}

	if *flHTTPAddr != "" {
		log.Println("freegeoip http server starting on", *flHTTPAddr)
		srv := &http.Server{
			Addr:         *flHTTPAddr,
			Handler:      ah,
			ReadTimeout:  *flReadTimeout,
			WriteTimeout: *flWriteTimeout,
			ConnState:    ConnStateMetrics(httpConnsGauge),
		}
		go func() { log.Fatal(srv.ListenAndServe()) }()
	}

	if *flHTTPSAddr != "" {
		log.Println("freegeoip https server starting on", *flHTTPSAddr)
		srv := &http.Server{
			Addr:         *flHTTPSAddr,
			Handler:      ah,
			ReadTimeout:  *flReadTimeout,
			WriteTimeout: *flWriteTimeout,
			ConnState:    ConnStateMetrics(httpsConnsGauge),
		}
		http2.ConfigureServer(srv, nil)
		go func() { log.Fatal(srv.ListenAndServeTLS(*flCertFile, *flKeyFile)) }()
	}

	select {}
}
<commit_msg>Update to support new redis client API<commit_after>// Copyright 2009-2015 The freegeoip authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package apiserver

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	// embed pprof server.
	_ "net/http/pprof"

	"github.com/fiorix/freegeoip"
	"github.com/fiorix/go-redis/redis"
	gorilla "github.com/gorilla/handlers"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/http2"
)

// Version tag.
var Version = "3.0.7"

var maxmindDB = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"

var (
	flAPIPrefix      = flag.String("api-prefix", "/", "Prefix for API endpoints")
	flCORSOrigin     = flag.String("cors-origin", "*", "CORS origin API endpoints")
	flHTTPAddr       = flag.String("http", ":8080", "Address in form of ip:port to listen on for HTTP")
	flHTTPSAddr      = flag.String("https", "", "Address in form of ip:port to listen on for HTTPS")
	flCertFile       = flag.String("cert", "cert.pem", "X.509 certificate file")
	flKeyFile        = flag.String("key", "key.pem", "X.509 key file")
	flReadTimeout    = flag.Duration("read-timeout", 30*time.Second, "Read timeout for HTTP and HTTPS client conns")
	flWriteTimeout   = flag.Duration("write-timeout", 15*time.Second, "Write timeout for HTTP and HTTPS client conns")
	flPublicDir      = flag.String("public", "", "Public directory to serve at the {prefix}/ endpoint")
	flDB             = flag.String("db", maxmindDB, "IP database file or URL")
	flUpdateIntvl    = flag.Duration("update", 24*time.Hour, "Database update check interval")
	flRetryIntvl     = flag.Duration("retry", time.Hour, "Max time to wait before retrying to download database")
	flUseXFF         = flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available (e.g. when running behind proxies)")
	flSilent         = flag.Bool("silent", false, "Do not log HTTP or HTTPS requests to stderr")
	flLogToStdout    = flag.Bool("logtostdout", false, "Log to stdout instead of stderr")
	flRedisAddr      = flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port[,ip:port] for quota")
	flRedisTimeout   = flag.Duration("redis-timeout", time.Second, "Redis read/write timeout")
	flQuotaMax       = flag.Int("quota-max", 0, "Max requests per source IP per interval; set 0 to turn off")
	flQuotaIntvl     = flag.Duration("quota-interval", time.Hour, "Quota expiration interval per source IP querying the API")
	flVersion        = flag.Bool("version", false, "Show version and exit")
	flInternalServer = flag.String("internal-server", "", "Address in form of ip:port to listen on for /metrics and /debug/pprof")
)

// Run is the entrypoint for the freegeoip daemon tool.
func Run() error {
	flag.Parse()

	if *flVersion {
		fmt.Printf("freegeoip v%s\n", Version)
		return nil
	}

	if *flLogToStdout {
		log.SetOutput(os.Stdout)
	}

	log.SetPrefix("[freegeoip] ")

	addrs := strings.Split(*flRedisAddr, ",")
	rc, err := redis.NewClient(addrs...)
	if err != nil {
		return err
	}
	rc.SetTimeout(*flRedisTimeout)

	db, err := openDB(*flDB, *flUpdateIntvl, *flRetryIntvl)
	if err != nil {
		return err
	}
	go watchEvents(db)

	ah := NewHandler(&HandlerConfig{
		Prefix:    *flAPIPrefix,
		Origin:    *flCORSOrigin,
		PublicDir: *flPublicDir,
		DB:        db,
		RateLimiter: RateLimiter{
			Redis:    rc,
			Max:      *flQuotaMax,
			Interval: *flQuotaIntvl,
		},
	})

	if !*flSilent {
		ah = gorilla.CombinedLoggingHandler(os.Stderr, ah)
	}

	if *flUseXFF {
		ah = freegeoip.ProxyHandler(ah)
	}

	if len(*flInternalServer) > 0 {
		http.Handle("/metrics", prometheus.Handler())
		log.Println("freegeoip internal server starting on", *flInternalServer)
		go func() { log.Fatal(http.ListenAndServe(*flInternalServer, nil)) }()
	}

	if *flHTTPAddr != "" {
		log.Println("freegeoip http server starting on", *flHTTPAddr)
		srv := &http.Server{
			Addr:         *flHTTPAddr,
			Handler:      ah,
			ReadTimeout:  *flReadTimeout,
			WriteTimeout: *flWriteTimeout,
			ConnState:    ConnStateMetrics(httpConnsGauge),
		}
		go func() { log.Fatal(srv.ListenAndServe()) }()
	}

	if *flHTTPSAddr != "" {
		log.Println("freegeoip https server starting on", *flHTTPSAddr)
		srv := &http.Server{
			Addr:         *flHTTPSAddr,
			Handler:      ah,
			ReadTimeout:  *flReadTimeout,
			WriteTimeout: *flWriteTimeout,
			ConnState:    ConnStateMetrics(httpsConnsGauge),
		}
		http2.ConfigureServer(srv, nil)
		go func() { log.Fatal(srv.ListenAndServeTLS(*flCertFile, *flKeyFile)) }()
	}

	select {}
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"golang.org/x/oauth2"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/clock/testclock"

	. "github.com/smartystreets/goconvey/convey"
)

func TestDiskTokenCache(t *testing.T) {
	t.Parallel()

	tmp, err := ioutil.TempDir("", "disk_token_cache")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmp)

	ctx := context.Background()
	ctx, tc := testclock.UseTime(ctx, testclock.TestRecentTimeLocal)

	Convey("DiskTokenCache works", t, func() {
		// testCacheSemantics is in proc_cache_test.go.
		testCacheSemantics(ctx, &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		})
	})

	Convey("Retains unknown cacheFileEntry fields", t, func() {
		cacheFile := filepath.Join(tmp, CacheFilename)
		testData := `
		{
			"cache": [
				{
					"key": {"key": "a"},
					"token": {
						"access_token": "abc"
					},
					"email": "a@example.com",
					"random_stuff": {"abc": {"def": "zzz"}},
					"abc": "def"
				}
			],
			"last_update": "2021-02-08T23:18:00.463912Z"
		}
		`
		So(ioutil.WriteFile(cacheFile, []byte(testData), 0600), ShouldBeNil)

		cache := &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		}

		tok, err := cache.GetToken(&CacheKey{Key: "a"})
		So(err, ShouldBeNil)
		So(tok, ShouldResemble, &Token{
			Token: oauth2.Token{AccessToken: "abc"},
			Email: "a@example.com",
		})
		So(cache.PutToken(&CacheKey{Key: "a"}, &Token{
			Token: oauth2.Token{
				AccessToken: "def",
				Expiry:      clock.Now(ctx).Add(time.Hour),
			},
			Email: "a@example.com",
		}), ShouldBeNil)

		// Check "random_stuff" and "abc" were preserved by the update.
		blob, err := ioutil.ReadFile(cacheFile)
		So(err, ShouldBeNil)
		So(string(blob), ShouldEqual, `{
  "cache": [
    {
      "key": {
        "key": "a"
      },
      "token": {
        "access_token": "def",
        "expiry": "2016-02-03T13:05:06.000000007Z"
      },
      "email": "a@example.com",
      "last_update": "2016-02-03T12:05:06.000000007Z",
      "abc": "def",
      "random_stuff": {
        "abc": {
          "def": "zzz"
        }
      }
    }
  ],
  "last_update": "2016-02-03T12:05:06.000000007Z"
}`)
	})

	// TODO(vadimsh): This test is flaky on Windows, there's non zero probability
	// that all 15 attempts (see testCacheInParallel) will hit "Access is denied"
	// error. This can be "fixed" by increasing number of attempts or sleeping
	// more between attempts. Both increase test runtime.
	SkipConvey("DiskTokenCache works (parallel)", t, func() {
		// testCacheInParallel is in proc_cache_test.go.
		//
		// Use real clock here to test real-world interaction when retrying disk
		// writes.
		ctx := context.Background()
		testCacheInParallel(ctx, &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		})
	})

	Convey("Cleans up old tokens", t, func() {
		cache := &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		}

		cache.PutToken(&CacheKey{Key: "a"}, &Token{
			Token: oauth2.Token{
				AccessToken: "abc",
				Expiry:      clock.Now(ctx),
			},
		})
		cache.PutToken(&CacheKey{Key: "b"}, &Token{
			Token: oauth2.Token{
				AccessToken:  "abc",
				RefreshToken: "def",
				Expiry:       clock.Now(ctx),
			},
		})

		// GCAccessTokenMaxAge later, "a" is gone while the cache is updated.
		tc.Add(GCAccessTokenMaxAge)
		unused := &Token{
			Token: oauth2.Token{
				AccessToken: "zzz",
				Expiry:      clock.Now(ctx).Add(365 * 24 * time.Hour),
			},
		}
		cache.PutToken(&CacheKey{Key: "unused"}, unused)

		t, err := cache.GetToken(&CacheKey{Key: "a"})
		So(err, ShouldBeNil)
		So(t, ShouldBeNil)

		// "b" is still there.
		t, err = cache.GetToken(&CacheKey{Key: "b"})
		So(err, ShouldBeNil)
		So(t.RefreshToken, ShouldEqual, "def")

		// Some time later "b" is also removed.
		tc.Add(GCRefreshTokenMaxAge)
		cache.PutToken(&CacheKey{Key: "unused"}, unused)

		t, err = cache.GetToken(&CacheKey{Key: "b"})
		So(err, ShouldBeNil)
		So(t, ShouldBeNil)
	})
}
<commit_msg>[auth] fix test to work on all timezones.<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"context"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"golang.org/x/oauth2"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/clock/testclock"

	. "github.com/smartystreets/goconvey/convey"
)

func TestDiskTokenCache(t *testing.T) {
	t.Parallel()

	tmp, err := ioutil.TempDir("", "disk_token_cache")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmp)

	ctx := context.Background()
	ctx, tc := testclock.UseTime(ctx, testclock.TestRecentTimeUTC.Local())

	Convey("DiskTokenCache works", t, func() {
		// testCacheSemantics is in proc_cache_test.go.
		testCacheSemantics(ctx, &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		})
	})

	Convey("Retains unknown cacheFileEntry fields", t, func() {
		cacheFile := filepath.Join(tmp, CacheFilename)
		testData := `
		{
			"cache": [
				{
					"key": {"key": "a"},
					"token": {
						"access_token": "abc"
					},
					"email": "a@example.com",
					"random_stuff": {"abc": {"def": "zzz"}},
					"abc": "def"
				}
			],
			"last_update": "2021-02-08T23:18:00.463912Z"
		}
		`
		So(ioutil.WriteFile(cacheFile, []byte(testData), 0600), ShouldBeNil)

		cache := &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		}

		tok, err := cache.GetToken(&CacheKey{Key: "a"})
		So(err, ShouldBeNil)
		So(tok, ShouldResemble, &Token{
			Token: oauth2.Token{AccessToken: "abc"},
			Email: "a@example.com",
		})
		So(cache.PutToken(&CacheKey{Key: "a"}, &Token{
			Token: oauth2.Token{
				AccessToken: "def",
				Expiry:      clock.Now(ctx).Add(time.Hour).UTC(),
			},
			Email: "a@example.com",
		}), ShouldBeNil)

		// Check "random_stuff" and "abc" were preserved by the update.
		blob, err := ioutil.ReadFile(cacheFile)
		So(err, ShouldBeNil)
		So(string(blob), ShouldEqual, `{
  "cache": [
    {
      "key": {
        "key": "a"
      },
      "token": {
        "access_token": "def",
        "expiry": "2016-02-03T05:05:06.000000007Z"
      },
      "email": "a@example.com",
      "last_update": "2016-02-03T04:05:06.000000007Z",
      "abc": "def",
      "random_stuff": {
        "abc": {
          "def": "zzz"
        }
      }
    }
  ],
  "last_update": "2016-02-03T04:05:06.000000007Z"
}`)
	})

	// TODO(vadimsh): This test is flaky on Windows, there's non zero probability
	// that all 15 attempts (see testCacheInParallel) will hit "Access is denied"
	// error. This can be "fixed" by increasing number of attempts or sleeping
	// more between attempts. Both increase test runtime.
	SkipConvey("DiskTokenCache works (parallel)", t, func() {
		// testCacheInParallel is in proc_cache_test.go.
		//
		// Use real clock here to test real-world interaction when retrying disk
		// writes.
		ctx := context.Background()
		testCacheInParallel(ctx, &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		})
	})

	Convey("Cleans up old tokens", t, func() {
		cache := &DiskTokenCache{
			Context:    ctx,
			SecretsDir: tmp,
		}

		cache.PutToken(&CacheKey{Key: "a"}, &Token{
			Token: oauth2.Token{
				AccessToken: "abc",
				Expiry:      clock.Now(ctx),
			},
		})
		cache.PutToken(&CacheKey{Key: "b"}, &Token{
			Token: oauth2.Token{
				AccessToken:  "abc",
				RefreshToken: "def",
				Expiry:       clock.Now(ctx),
			},
		})

		// GCAccessTokenMaxAge later, "a" is gone while the cache is updated.
		tc.Add(GCAccessTokenMaxAge)
		unused := &Token{
			Token: oauth2.Token{
				AccessToken: "zzz",
				Expiry:      clock.Now(ctx).Add(365 * 24 * time.Hour),
			},
		}
		cache.PutToken(&CacheKey{Key: "unused"}, unused)

		t, err := cache.GetToken(&CacheKey{Key: "a"})
		So(err, ShouldBeNil)
		So(t, ShouldBeNil)

		// "b" is still there.
		t, err = cache.GetToken(&CacheKey{Key: "b"})
		So(err, ShouldBeNil)
		So(t.RefreshToken, ShouldEqual, "def")

		// Some time later "b" is also removed.
		tc.Add(GCRefreshTokenMaxAge)
		cache.PutToken(&CacheKey{Key: "unused"}, unused)

		t, err = cache.GetToken(&CacheKey{Key: "b"})
		So(err, ShouldBeNil)
		So(t, ShouldBeNil)
	})
}
<|endoftext|>


<commit_before>package migrate

import (
	"context"
	"fmt"
	"io"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"

	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

type Migrator struct {
	// Client used for API interactions
	Client client.Client

	// If true, skip checking the 'status.storedVersion' before running the migration.
	// By default, migration will only be run if the CRD contains storedVersions other
	// than the desired target version.
	SkipStoredVersionCheck bool

	// Writers to write informational & error messages to
	Out, ErrOut io.Writer
}

// NewMigrator creates a new migrator with the given API client.
// If either of out or errOut are nil, log messages will be discarded.
func NewMigrator(client client.Client, skipStoredVersionCheck bool, out, errOut io.Writer) *Migrator {
	if out == nil {
		out = io.Discard
	}
	if errOut == nil {
		errOut = io.Discard
	}

	return &Migrator{
		Client:                 client,
		SkipStoredVersionCheck: skipStoredVersionCheck,
		Out:                    out,
		ErrOut:                 errOut,
	}
}

// Run begins the migration of all the named CRDs.
// It will attempt to migrate all resources defined as part of these CRDs to the
// given 'targetVersion', and after completion will update the `status.storedVersions`
// field on the corresponding CRD version to only contain the given targetVersion.
func (m *Migrator) Run(ctx context.Context, targetVersion string, names []string) error {
	fmt.Fprintf(m.Out, "Checking all CustomResourceDefinitions have storage version set to '%s'\n", targetVersion)
	allV1, allCRDs, err := m.ensureCRDStorageVersionEquals(ctx, targetVersion, names)
	if err != nil {
		return err
	}
	if !allV1 {
		fmt.Fprintln(m.ErrOut, "It looks like you are running a pre-1.0 version of cert-manager. Please upgrade cert-manager to v1.6 before upgrading to v1.7.")
		return fmt.Errorf("preflight checks failed")
	}
	fmt.Fprintf(m.Out, "All CustomResourceDefinitions have %q configured as the storage version.\n", targetVersion)

	crdsRequiringMigration := allCRDs
	if !m.SkipStoredVersionCheck {
		fmt.Fprintf(m.Out, "Looking for CRDs that contain resources that require migrating to %q...\n", targetVersion)
		crdsRequiringMigration, err = m.discoverCRDsRequiringMigration(ctx, targetVersion, names)
		if err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to determine resource types that require migration: %v\n", err)
			return err
		}
		if len(crdsRequiringMigration) == 0 {
			fmt.Fprintln(m.Out, "Nothing to do. cert-manager CRDs do not have 'status.storedVersions' containing old API versions. You may proceed to upgrade to cert-manager v1.7.")
			return nil
		}
	} else {
		fmt.Fprintln(m.Out, "Forcing migration of all CRD resources as --skip-stored-version-check=true")
	}

	fmt.Fprintf(m.Out, "Found %d resource types that require migration:\n", len(crdsRequiringMigration))
	for _, crd := range crdsRequiringMigration {
		fmt.Fprintf(m.Out, " - %s\n", crd.Name)
	}

	for _, crd := range crdsRequiringMigration {
		if err := m.migrateResourcesForCRD(ctx, crd); err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to migrate resource: %v\n", err)
			return err
		}
	}

	fmt.Fprintf(m.Out, "Patching CRD resources to set 'status.storedVersions' to %q...\n", targetVersion)
	if err := m.patchCRDStoredVersions(ctx, crdsRequiringMigration); err != nil {
		fmt.Fprintf(m.ErrOut, "Failed to patch 'status.storedVersions' field: %v\n", err)
		return err
	}

	fmt.Fprintln(m.Out, "Successfully migrated all cert-manager resource types. It is now safe to proceed with upgrading to cert-manager v1.7.")
	return nil
}

func (m *Migrator) ensureCRDStorageVersionEquals(ctx context.Context, vers string, names []string) (bool, []*apiext.CustomResourceDefinition, error) {
	var crds []*apiext.CustomResourceDefinition
	for _, crdName := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
			return false, nil, err
		}

		// Discover the storage version
		storageVersion := storageVersionForCRD(crd)

		if storageVersion != vers {
			fmt.Fprintf(m.Out, "CustomResourceDefinition object %q has storage version set to %q. You MUST upgrade to cert-manager v1.0-v1.6 before migrating resources for v1.7.\n", crdName, storageVersion)
			return false, nil, nil
		}

		crds = append(crds, crd)
	}

	return true, crds, nil
}

func (m *Migrator) discoverCRDsRequiringMigration(ctx context.Context, desiredStorageVersion string, names []string) ([]*apiext.CustomResourceDefinition, error) {
	var requireMigration []*apiext.CustomResourceDefinition
	for _, name := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, crd); err != nil {
			return nil, err
		}
		// If no versions are stored, there's nothing to migrate.
		if len(crd.Status.StoredVersions) == 0 {
			continue
		}
		// If more than one entry exists in `storedVersions` OR if the only element in there is not
		// the desired version, perform a migration.
		if len(crd.Status.StoredVersions) > 1 || crd.Status.StoredVersions[0] != desiredStorageVersion {
			requireMigration = append(requireMigration, crd)
		}
	}
	return requireMigration, nil
}

func (m *Migrator) migrateResourcesForCRD(ctx context.Context, crd *apiext.CustomResourceDefinition) error {
	startTime := time.Now()
	fmt.Fprintf(m.Out, "Migrating %q objects in group %q - this may take a while (started at %s)...\n", crd.Spec.Names.Kind, crd.Spec.Group, startTime.Format(time.Stamp))
	list := &unstructured.UnstructuredList{}
	list.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   crd.Spec.Group,
		Version: "v1",
		Kind:    crd.Spec.Names.ListKind,
	})
	if err := m.Client.List(ctx, list); err != nil {
		return err
	}
	fmt.Fprintf(m.Out, " %d resources to migrate\n", len(list.Items))
	for _, obj := range list.Items {
		// retry on any kind of error to handle cases where e.g. the network connection to the apiserver fails
		if err := retry.OnError(wait.Backoff{
			Duration: time.Second, // wait 1s between attempts
			Steps:    3,           // allow up to 3 attempts per object
		}, func(err error) bool {
			// Retry on any errors that are not otherwise skipped/ignored
			return handleUpdateErr(err) != nil
		}, func() error { return m.Client.Update(ctx, &obj) }); handleUpdateErr(err) != nil {
			return err
		}
	}
	fmt.Fprintf(m.Out, " Successfully migrated %d %s objects in %s\n", len(list.Items), crd.Spec.Names.Kind, time.Now().Sub(startTime).Round(time.Second))
	return nil
}

// patchCRDStoredVersions will patch the `status.storedVersions` field of all passed in CRDs to be
// set to an array containing JUST the current storage version.
// This is only safe to run after a successful migration (i.e. a read/write of all resources of the given CRD type).
func (m *Migrator) patchCRDStoredVersions(ctx context.Context, crds []*apiext.CustomResourceDefinition) error {
	for _, crd := range crds {
		// fetch a fresh copy of the CRD to avoid any conflict errors
		freshCRD := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crd.Name}, freshCRD); err != nil {
			return err
		}

		// Check the latest copy of the CRD to ensure that:
		//   1) the storage version is the same as it was at the start of the migration
		//   2) the status.storedVersion field has not changed, and if it has, it has only added the new/desired storage version
		// This helps to avoid cases where the storage version was changed by a third-party midway through the migration,
		// which could lead to corrupted apiservers when we patch the status.storedVersions field below.
		expectedStorageVersion := storageVersionForCRD(crd)
		if storageVersionForCRD(freshCRD) != expectedStorageVersion {
			return newUnexpectedChangeError(crd)
		}
		newlyAddedVersions := storedVersionsAdded(crd, freshCRD)
		if newlyAddedVersions.Len() != 0 && !newlyAddedVersions.Equal(sets.NewString(expectedStorageVersion)) {
			return newUnexpectedChangeError(crd)
		}

		// Set the `status.storedVersions` field to 'v1'
		freshCRD.Status.StoredVersions = []string{"v1"}

		if err := m.Client.Status().Update(ctx, freshCRD); err != nil {
			return err
		}
	}

	return nil
}

// storageVersionForCRD discovers the storage version for a given CRD.
func storageVersionForCRD(crd *apiext.CustomResourceDefinition) string {
	storageVersion := ""
	for _, v := range crd.Spec.Versions {
		if v.Storage {
			storageVersion = v.Name
			break
		}
	}
	return storageVersion
}

// storedVersionsAdded returns a list of any versions added to the `status.storedVersions` field on
// a CRD resource.
func storedVersionsAdded(old, new *apiext.CustomResourceDefinition) sets.String {
	oldStoredVersions := sets.NewString(old.Status.StoredVersions...)
	newStoredVersions := sets.NewString(new.Status.StoredVersions...)
	return newStoredVersions.Difference(oldStoredVersions)
}

// newUnexpectedChangeError creates a new 'error' that informs users that a change to the CRDs
// was detected during the migration process and so the migration must be re-run.
func newUnexpectedChangeError(crd *apiext.CustomResourceDefinition) error {
	errorFmt := "" +
		"The CRD %q unexpectedly changed during the migration. " +
		"This means that either an object was persisted in a non-storage version during the migration, " +
		"or the storage version was changed by someone else (or some automated deployment tooling) whilst the migration " +
		"was in progress.\n\n" +
		"All automated deployment tooling should be in a 'stable state' (i.e. no upgrades to cert-manager CRDs should be" +
		"in progress whilst the migration is running).\n\n" +
		"Please ensure no changes to the CRDs are made during the migration process and re-run the migration until you" +
		"no longer see this message."
	return fmt.Errorf(errorFmt, crd.Name)
}

// handleUpdateErr will absorb certain types of errors that we know can be skipped/passed on
// during a migration of a particular object.
func handleUpdateErr(err error) error {
	if err == nil {
		return nil
	}
	// If the resource no longer exists, don't return the error as the object no longer
	// needs updating to the new API version.
	if apierrors.IsNotFound(err) {
		return nil
	}
	// If there was a conflict, another client must have written the object already which
	// means we don't need to force an update.
	if apierrors.IsConflict(err) {
		return nil
	}
	return err
}
<commit_msg>Indicate whether a migration was actually performed in Run<commit_after>package migrate

import (
	"context"
	"fmt"
	"io"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"

	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

type Migrator struct {
	// Client used for API interactions
	Client client.Client

	// If true, skip checking the 'status.storedVersion' before running the migration.
	// By default, migration will only be run if the CRD contains storedVersions other
	// than the desired target version.
	SkipStoredVersionCheck bool

	// Writers to write informational & error messages to
	Out, ErrOut io.Writer
}

// NewMigrator creates a new migrator with the given API client.
// If either of out or errOut are nil, log messages will be discarded.
func NewMigrator(client client.Client, skipStoredVersionCheck bool, out, errOut io.Writer) *Migrator {
	if out == nil {
		out = io.Discard
	}
	if errOut == nil {
		errOut = io.Discard
	}

	return &Migrator{
		Client:                 client,
		SkipStoredVersionCheck: skipStoredVersionCheck,
		Out:                    out,
		ErrOut:                 errOut,
	}
}

// Run begins the migration of all the named CRDs.
// It will attempt to migrate all resources defined as part of these CRDs to the
// given 'targetVersion', and after completion will update the `status.storedVersions`
// field on the corresponding CRD version to only contain the given targetVersion.
// Returns 'true' if a migration was actually performed, and false if migration was not required.
func (m *Migrator) Run(ctx context.Context, targetVersion string, names []string) (bool, error) {
	fmt.Fprintf(m.Out, "Checking all CustomResourceDefinitions have storage version set to '%s'\n", targetVersion)
	allTargetVersion, allCRDs, err := m.ensureCRDStorageVersionEquals(ctx, targetVersion, names)
	if err != nil {
		return false, err
	}
	if !allTargetVersion {
		fmt.Fprintf(m.ErrOut, "It looks like you are running a version of cert-manager that does not set the storage version of CRDs to %q. Please upgrade cert-manager to v1.6 before upgrading to v1.7.\n", targetVersion)
		return false, fmt.Errorf("preflight checks failed")
	}
	fmt.Fprintf(m.Out, "All CustomResourceDefinitions have %q configured as the storage version.\n", targetVersion)

	crdsRequiringMigration := allCRDs
	if !m.SkipStoredVersionCheck {
		fmt.Fprintf(m.Out, "Looking for CRDs that contain resources that require migrating to %q...\n", targetVersion)
		crdsRequiringMigration, err = m.discoverCRDsRequiringMigration(ctx, targetVersion, names)
		if err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to determine resource types that require migration: %v\n", err)
			return false, err
		}
		if len(crdsRequiringMigration) == 0 {
			fmt.Fprintln(m.Out, "Nothing to do. cert-manager CRDs do not have 'status.storedVersions' containing old API versions. You may proceed to upgrade to cert-manager v1.7.")
			return false, nil
		}
	} else {
		fmt.Fprintln(m.Out, "Forcing migration of all CRD resources as --skip-stored-version-check=true")
	}

	fmt.Fprintf(m.Out, "Found %d resource types that require migration:\n", len(crdsRequiringMigration))
	for _, crd := range crdsRequiringMigration {
		fmt.Fprintf(m.Out, " - %s\n", crd.Name)
	}

	for _, crd := range crdsRequiringMigration {
		if err := m.migrateResourcesForCRD(ctx, crd); err != nil {
			fmt.Fprintf(m.ErrOut, "Failed to migrate resource: %v\n", err)
			return false, err
		}
	}

	fmt.Fprintf(m.Out, "Patching CRD resources to set 'status.storedVersions' to %q...\n", targetVersion)
	if err := m.patchCRDStoredVersions(ctx, crdsRequiringMigration); err != nil {
		fmt.Fprintf(m.ErrOut, "Failed to patch 'status.storedVersions' field: %v\n", err)
		return false, err
	}

	fmt.Fprintln(m.Out, "Successfully migrated all cert-manager resource types. It is now safe to proceed with upgrading to cert-manager v1.7.")
	return true, nil
}

func (m *Migrator) ensureCRDStorageVersionEquals(ctx context.Context, vers string, names []string) (bool, []*apiext.CustomResourceDefinition, error) {
	var crds []*apiext.CustomResourceDefinition
	for _, crdName := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
			return false, nil, err
		}

		// Discover the storage version
		storageVersion := storageVersionForCRD(crd)

		if storageVersion != vers {
			fmt.Fprintf(m.Out, "CustomResourceDefinition object %q has storage version set to %q. You MUST upgrade to cert-manager v1.0-v1.6 before migrating resources for v1.7.\n", crdName, storageVersion)
			return false, nil, nil
		}

		crds = append(crds, crd)
	}

	return true, crds, nil
}

func (m *Migrator) discoverCRDsRequiringMigration(ctx context.Context, desiredStorageVersion string, names []string) ([]*apiext.CustomResourceDefinition, error) {
	var requireMigration []*apiext.CustomResourceDefinition
	for _, name := range names {
		crd := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: name}, crd); err != nil {
			return nil, err
		}
		// If no versions are stored, there's nothing to migrate.
		if len(crd.Status.StoredVersions) == 0 {
			continue
		}
		// If more than one entry exists in `storedVersions` OR if the only element in there is not
		// the desired version, perform a migration.
		if len(crd.Status.StoredVersions) > 1 || crd.Status.StoredVersions[0] != desiredStorageVersion {
			requireMigration = append(requireMigration, crd)
		}
	}
	return requireMigration, nil
}

func (m *Migrator) migrateResourcesForCRD(ctx context.Context, crd *apiext.CustomResourceDefinition) error {
	startTime := time.Now()
	fmt.Fprintf(m.Out, "Migrating %q objects in group %q - this may take a while (started at %s)...\n", crd.Spec.Names.Kind, crd.Spec.Group, startTime.Format(time.Stamp))
	list := &unstructured.UnstructuredList{}
	list.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   crd.Spec.Group,
		Version: "v1",
		Kind:    crd.Spec.Names.ListKind,
	})
	if err := m.Client.List(ctx, list); err != nil {
		return err
	}
	fmt.Fprintf(m.Out, " %d resources to migrate\n", len(list.Items))
	for _, obj := range list.Items {
		// retry on any kind of error to handle cases where e.g. the network connection to the apiserver fails
		if err := retry.OnError(wait.Backoff{
			Duration: time.Second, // wait 1s between attempts
			Steps:    3,           // allow up to 3 attempts per object
		}, func(err error) bool {
			// Retry on any errors that are not otherwise skipped/ignored
			return handleUpdateErr(err) != nil
		}, func() error { return m.Client.Update(ctx, &obj) }); handleUpdateErr(err) != nil {
			return err
		}
	}
	fmt.Fprintf(m.Out, " Successfully migrated %d %s objects in %s\n", len(list.Items), crd.Spec.Names.Kind, time.Now().Sub(startTime).Round(time.Second))
	return nil
}

// patchCRDStoredVersions will patch the `status.storedVersions` field of all passed in CRDs to be
// set to an array containing JUST the current storage version.
// This is only safe to run after a successful migration (i.e. a read/write of all resources of the given CRD type).
func (m *Migrator) patchCRDStoredVersions(ctx context.Context, crds []*apiext.CustomResourceDefinition) error {
	for _, crd := range crds {
		// fetch a fresh copy of the CRD to avoid any conflict errors
		freshCRD := &apiext.CustomResourceDefinition{}
		if err := m.Client.Get(ctx, client.ObjectKey{Name: crd.Name}, freshCRD); err != nil {
			return err
		}

		// Check the latest copy of the CRD to ensure that:
		//   1) the storage version is the same as it was at the start of the migration
		//   2) the status.storedVersion field has not changed, and if it has, it has only added the new/desired storage version
		// This helps to avoid cases where the storage version was changed by a third-party midway through the migration,
		// which could lead to corrupted apiservers when we patch the status.storedVersions field below.
		expectedStorageVersion := storageVersionForCRD(crd)
		if storageVersionForCRD(freshCRD) != expectedStorageVersion {
			return newUnexpectedChangeError(crd)
		}
		newlyAddedVersions := storedVersionsAdded(crd, freshCRD)
		if newlyAddedVersions.Len() != 0 && !newlyAddedVersions.Equal(sets.NewString(expectedStorageVersion)) {
			return newUnexpectedChangeError(crd)
		}

		// Set the `status.storedVersions` field to 'v1'
		freshCRD.Status.StoredVersions = []string{"v1"}

		if err := m.Client.Status().Update(ctx, freshCRD); err != nil {
			return err
		}
	}

	return nil
}

// storageVersionForCRD discovers the storage version for a given CRD.
func storageVersionForCRD(crd *apiext.CustomResourceDefinition) string {
	storageVersion := ""
	for _, v := range crd.Spec.Versions {
		if v.Storage {
			storageVersion = v.Name
			break
		}
	}
	return storageVersion
}

// storedVersionsAdded returns a list of any versions added to the `status.storedVersions` field on
// a CRD resource.
func storedVersionsAdded(old, new *apiext.CustomResourceDefinition) sets.String {
	oldStoredVersions := sets.NewString(old.Status.StoredVersions...)
	newStoredVersions := sets.NewString(new.Status.StoredVersions...)
	return newStoredVersions.Difference(oldStoredVersions)
}

// newUnexpectedChangeError creates a new 'error' that informs users that a change to the CRDs
// was detected during the migration process and so the migration must be re-run.
func newUnexpectedChangeError(crd *apiext.CustomResourceDefinition) error {
	errorFmt := "" +
		"The CRD %q unexpectedly changed during the migration. " +
		"This means that either an object was persisted in a non-storage version during the migration, " +
		"or the storage version was changed by someone else (or some automated deployment tooling) whilst the migration " +
		"was in progress.\n\n" +
		"All automated deployment tooling should be in a 'stable state' (i.e. no upgrades to cert-manager CRDs should be" +
		"in progress whilst the migration is running).\n\n" +
		"Please ensure no changes to the CRDs are made during the migration process and re-run the migration until you" +
		"no longer see this message."
	return fmt.Errorf(errorFmt, crd.Name)
}

// handleUpdateErr will absorb certain types of errors that we know can be skipped/passed on
// during a migration of a particular object.
func handleUpdateErr(err error) error {
	if err == nil {
		return nil
	}
	// If the resource no longer exists, don't return the error as the object no longer
	// needs updating to the new API version.
	if apierrors.IsNotFound(err) {
		return nil
	}
	// If there was a conflict, another client must have written the object already which
	// means we don't need to force an update.
	if apierrors.IsConflict(err) {
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"encoding/binary"
	"fmt"

	"vitess.io/vitess/go/vt/vtgate/evalengine"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
)

var (
	_ MultiColumn = (*RegionExperimental)(nil)
)

func init() {
	Register("region_experimental", NewRegionExperimental)
}

// RegionExperimental is a multi-column unique vindex. The first column is prefixed
// to the hash of the second column to produce the keyspace id.
// RegionExperimental can be used for geo-partitioning because the first column can denote a region,
// and its value will dictate the shard for that region.
type RegionExperimental struct {
	name        string
	regionBytes int
}

// NewRegionExperimental creates a RegionExperimental vindex.
// The supplied map requires all the fields of "consistent_lookup_unique".
// Additionally, it requires a region_bytes argument whose value can be "1", or "2".
func NewRegionExperimental(name string, m map[string]string) (Vindex, error) {
	rbs, ok := m["region_bytes"]
	if !ok {
		return nil, fmt.Errorf("region_experimental missing region_bytes param")
	}
	var rb int
	switch rbs {
	case "1":
		rb = 1
	case "2":
		rb = 2
	default:
		return nil, fmt.Errorf("region_bits must be 1 or 2: %v", rbs)
	}
	return &RegionExperimental{
		name:        name,
		regionBytes: rb,
	}, nil
}

// String returns the name of the vindex.
func (ge *RegionExperimental) String() string {
	return ge.name
}

// Cost returns the cost of this index as 1.
func (ge *RegionExperimental) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (ge *RegionExperimental) IsUnique() bool {
	return true
}

// NeedsVCursor satisfies the Vindex interface.
func (ge *RegionExperimental) NeedsVCursor() bool {
	return false
}

// Map satisfies MultiColumn.
func (ge *RegionExperimental) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) {
	destinations := make([]key.Destination, 0, len(rowsColValues))
	for _, row := range rowsColValues {
		if len(row) != 2 {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		// Compute region prefix.
		rn, err := evalengine.ToUint64(row[0])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		r := make([]byte, 2, 2+8)
		binary.BigEndian.PutUint16(r, uint16(rn))

		// Compute hash.
		hn, err := evalengine.ToUint64(row[1])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		h := vhash(hn)

		// Concatenate and add to destinations.
		if ge.regionBytes == 1 {
			r = r[1:]
		}
		dest := append(r, h...)
		destinations = append(destinations, key.DestinationKeyspaceID(dest))
	}
	return destinations, nil
}

// Verify satisfies MultiColumn.
func (ge *RegionExperimental) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) {
	result := make([]bool, len(rowsColValues))
	destinations, _ := ge.Map(vcursor, rowsColValues)
	for i, dest := range destinations {
		destksid, ok := dest.(key.DestinationKeyspaceID)
		if !ok {
			continue
		}
		result[i] = bytes.Equal([]byte(destksid), ksids[i])
	}
	return result, nil
}

func (ge *RegionExperimental) PartialVindex() bool {
	return true
}
<commit_msg>feat: support partial column in region experimental vindex<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"encoding/binary"
	"fmt"

	"vitess.io/vitess/go/vt/vtgate/evalengine"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
)

var (
	_ MultiColumn = (*RegionExperimental)(nil)
)

func init() {
	Register("region_experimental", NewRegionExperimental)
}

// RegionExperimental is a multi-column unique vindex. The first column is prefixed
// to the hash of the second column to produce the keyspace id.
// RegionExperimental can be used for geo-partitioning because the first column can denote a region,
// and its value will dictate the shard for that region.
type RegionExperimental struct {
	name        string
	regionBytes int
}

// NewRegionExperimental creates a RegionExperimental vindex.
// The supplied map requires all the fields of "consistent_lookup_unique".
// Additionally, it requires a region_bytes argument whose value can be "1", or "2".
func NewRegionExperimental(name string, m map[string]string) (Vindex, error) {
	rbs, ok := m["region_bytes"]
	if !ok {
		return nil, fmt.Errorf("region_experimental missing region_bytes param")
	}
	var rb int
	switch rbs {
	case "1":
		rb = 1
	case "2":
		rb = 2
	default:
		return nil, fmt.Errorf("region_bits must be 1 or 2: %v", rbs)
	}
	return &RegionExperimental{
		name:        name,
		regionBytes: rb,
	}, nil
}

// String returns the name of the vindex.
func (ge *RegionExperimental) String() string {
	return ge.name
}

// Cost returns the cost of this index as 1.
func (ge *RegionExperimental) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (ge *RegionExperimental) IsUnique() bool {
	return true
}

// NeedsVCursor satisfies the Vindex interface.
func (ge *RegionExperimental) NeedsVCursor() bool {
	return false
}

// Map satisfies MultiColumn.
func (ge *RegionExperimental) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) {
	destinations := make([]key.Destination, 0, len(rowsColValues))
	for _, row := range rowsColValues {
		if len(row) > 2 {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		// Compute region prefix.
		rn, err := evalengine.ToUint64(row[0])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		r := make([]byte, 2, 2+8)
		binary.BigEndian.PutUint16(r, uint16(rn))

		// Concatenate and add to destinations.
		if ge.regionBytes == 1 {
			r = r[1:]
		}

		dest := r
		if len(row) == 2 {
			// Compute hash.
			hn, err := evalengine.ToUint64(row[1])
			if err != nil {
				destinations = append(destinations, key.DestinationNone{})
				continue
			}
			h := vhash(hn)
			dest = append(dest, h...)
			destinations = append(destinations, key.DestinationKeyspaceID(dest))
		} else {
			destinations = append(destinations, NewKeyRangeFromPrefix(dest))
		}
	}
	return destinations, nil
}

// Verify satisfies MultiColumn.
func (ge *RegionExperimental) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) {
	result := make([]bool, len(rowsColValues))
	destinations, _ := ge.Map(vcursor, rowsColValues)
	for i, dest := range destinations {
		destksid, ok := dest.(key.DestinationKeyspaceID)
		if !ok {
			continue
		}
		result[i] = bytes.Equal([]byte(destksid), ksids[i])
	}
	return result, nil
}

func (ge *RegionExperimental) PartialVindex() bool {
	return true
}
<|endoftext|>


<commit_before>package stconverter

import (
	"strings"
)

//verilogTemplate templates make the following assumptions:
//1) All variables are VHDL "Variables", not "signals"
//2) All variables are integer types
//3) Everything completes in a single cycle
//4) loops aren't yet supported
const verilogTemplate = `
{{define "expression"}}{{$value := .HasValue}}{{$operator := .HasOperator}}{{/*
	*/}}{{if $value}}{{/*
		*/}}{{if isKnownVar $value}}{{$value}}{{else}}{{$value}}{{end}}{{/*
	*/}}{{else if $operator}}{{/* //then we need to determine how to print this operator
		*/}}{{if $operator.LeftAssociative}}{{/* //print first argument, operator string, then remaining arguments
			*/}}{{$args := reverseArgs .GetArguments}}{{$a := index $args 0}}{{$b := index $args 1}}{{$curPrec := $operator.GetPrecedence}}{{$aop := $a.HasOperator}}{{$bop := $b.HasOperator}}{{/*
			*/}}{{if $aop}}{{if gte $aop.GetPrecedence $curPrec}}({{end}}{{end}}{{template "expression" $a}}{{if $aop}}{{if gte $aop.GetPrecedence $curPrec}}){{end}}{{end}} {{translateOperatorToken $operator.GetToken}} {{if $bop}}{{if gte $bop.GetPrecedence $curPrec}}({{end}}{{end}}{{template "expression" $b}}{{if $bop}}{{if gte $bop.GetPrecedence $curPrec}}){{end}}{{end}}{{/*
		*/}}{{else}}{{/* //print name, opening bracket, then arguments separated by commas
			*/}}{{translateOperatorToken $operator.GetToken}}{{/*
			*/}}{{range $ind, $arg := reverseArgs .GetArguments}}{{if $ind}}, {{end}}{{template "expression" $arg}}{{end}}{{if tokenIsFunctionCall $operator.GetToken}}){{end}}{{/*
		*/}}{{end}}{{/*
	*/}}{{end}}{{/*}}
*/}}{{end}}

{{define "ifelsifelse"}}{{/*
	*/}}{{range $i, $ifThen := .IfThens}}{{/* //cycle through all the ifThens
		*/}}{{if $i}} else {{end}}if ({{template "expression" $ifThen.IfExpression}}) begin
			{{compileSequence $ifThen.ThenSequence}}
		end
		{{end}}{{/*
	*/}}{{if .ElseSequence}} else begin
		{{compileSequence .ElseSequence}}
	end{{end}}{{/*
*/}}{{end}}

{{define "switchcase"}}{{/*
	*/}}case({{template "expression" .SwitchOn}}) {{range $ci, $case := .Cases}}
	{{range $cii, $casev := $case.CaseValues}}{{if $cii}}, {{end}}{{$casev}}{{end}}: begin
		{{compileSequence $case.Sequence}}
	end{{end}}
	{{if .ElseSequence}}default: begin
		{{compileSequence .ElseSequence}}
	end{{end}}
	endcase {{/*
*/}}{{end}}

{{define "forloop"}} --for loops not supported in VHDL
{{end}}

{{define "whileloop"}} --while loops not supported in VHDL
{{end}}

{{define "repeatloop"}} --repeat loops not supported in VHDL
{{end}}`

func verilogTokenIsFunctionCall(token string) bool {
	first := strings.Index(token, "<")
	if len(token) > 2 && token[len(token)-1:] == ">" && first != -1 {
		return true
	}
	return false
}

func verilogTranslateOperatorToken(token string) string {
	//is it our function<n> syntax?
	first := strings.Index(token, "<")
	if len(token) > 2 && token[len(token)-1:] == ">" && first != -1 {
		//we on't need the following... we don't need the number of arguments??
		// ops := token[first+1 : len(token)-1]
		// opsInt, err := strconv.Atoi(ops)
		// if err != nil {
		// 	return token[:first-1] + "("
		// }
		return token[:first] + "("
	}
	//ok, not a function, so it's one of the st Operators
	switch token {
	case stExit:
		return "break"
	case stReturn:
		return "return"
	case stNot:
		return "~"
	case stNegative:
		return "-"
	case stExponentiation:
		return "**"
	case stMultiply:
		return "*"
	case stDivide:
		return "/"
	case stModulo:
		return "%"
	case stAdd:
		return "+"
	case stSubtract:
		return "-"
	case stLessThan:
		return "<"
	case stGreaterThan:
		return ">"
	case stLessThanEqualTo:
		return "<="
	case stGreaterThanEqualTo:
		return ">="
	case stEqual:
		return "=="
	case stInequal:
		return "!="
	case stAnd:
		return "&&"
	case stExlusiveOr:
		return "^"
	case stOr:
		return "||"
	case stAssignment:
		return "="
	}
	//still here? panic
	panic("unsupported token " + token)
	return ""
}
<commit_msg>stconverter: temporary bugfix for Verilog NOT operator<commit_after>package stconverter

import (
	"strings"
)

//verilogTemplate templates make the following assumptions:
//1) All variables are VHDL "Variables", not "signals"
//2) All variables are integer types
//3) Everything completes in a single cycle
//4) loops aren't yet supported
const verilogTemplate = `
{{define "expression"}}{{$value := .HasValue}}{{$operator := .HasOperator}}{{/*
	*/}}{{if $value}}{{/*
		*/}}{{if isKnownVar $value}}{{$value}}{{else}}{{$value}}{{end}}{{/*
	*/}}{{else if $operator}}{{/* //then we need to determine how to print this operator
		*/}}{{if $operator.LeftAssociative}}{{/* //print first argument, operator string, then remaining arguments
			*/}}{{$args := reverseArgs .GetArguments}}{{$a := index $args 0}}{{$b := index $args 1}}{{$curPrec := $operator.GetPrecedence}}{{$aop := $a.HasOperator}}{{$bop := $b.HasOperator}}{{/*
			*/}}{{if $aop}}{{if gte $aop.GetPrecedence $curPrec}}({{end}}{{end}}{{template "expression" $a}}{{if $aop}}{{if gte $aop.GetPrecedence $curPrec}}){{end}}{{end}} {{translateOperatorToken $operator.GetToken}} {{if $bop}}{{if gte $bop.GetPrecedence $curPrec}}({{end}}{{end}}{{template "expression" $b}}{{if $bop}}{{if gte $bop.GetPrecedence $curPrec}}){{end}}{{end}}{{/*
		*/}}{{else}}{{/* //print name, opening bracket, then arguments separated by commas
			*/}}{{translateOperatorToken $operator.GetToken}}{{/*
			*/}}{{range $ind, $arg := reverseArgs .GetArguments}}{{if $ind}}, {{end}}{{template "expression" $arg}}{{end}}{{if tokenIsFunctionCall $operator.GetToken}}){{end}}{{/*
		*/}}{{end}}{{/*
	*/}}{{end}}{{/*}}
*/}}{{end}}

{{define "ifelsifelse"}}{{/*
	*/}}{{range $i, $ifThen := .IfThens}}{{/* //cycle through all the ifThens
		*/}}{{if $i}} else {{end}}if ({{template "expression" $ifThen.IfExpression}}) begin
			{{compileSequence $ifThen.ThenSequence}}
		end
		{{end}}{{/*
	*/}}{{if .ElseSequence}} else begin
		{{compileSequence .ElseSequence}}
	end{{end}}{{/*
*/}}{{end}}

{{define "switchcase"}}{{/*
	*/}}case({{template "expression" .SwitchOn}}) {{range $ci, $case := .Cases}}
	{{range $cii, $casev := $case.CaseValues}}{{if $cii}}, {{end}}{{$casev}}{{end}}: begin
		{{compileSequence $case.Sequence}}
	end{{end}}
	{{if .ElseSequence}}default: begin
		{{compileSequence .ElseSequence}}
	end{{end}}
	endcase {{/*
*/}}{{end}}

{{define "forloop"}} --for loops not supported in VHDL
{{end}}

{{define "whileloop"}} --while loops not supported in VHDL
{{end}}

{{define "repeatloop"}} --repeat loops not supported in VHDL
{{end}}`

func verilogTokenIsFunctionCall(token string) bool {
	if token == "not" {
		return true
	}
	first := strings.Index(token, "<")
	if len(token) > 2 && token[len(token)-1:] == ">" && first != -1 {
		return true
	}
	return false
}

func verilogTranslateOperatorToken(token string) string {
	//is it our function<n> syntax?
	first := strings.Index(token, "<")
	if len(token) > 2 && token[len(token)-1:] == ">" && first != -1 {
		//we on't need the following... we don't need the number of arguments??
		// ops := token[first+1 : len(token)-1]
		// opsInt, err := strconv.Atoi(ops)
		// if err != nil {
		// 	return token[:first-1] + "("
		// }
		return token[:first] + "("
	}
	//TODO: nots should be dealt with a better way
	if token == "not" {
		return "~("
	}
	//ok, not a function, so it's one of the st Operators
	switch token {
	case stExit:
		return "break"
	case stReturn:
		return "return"
	case stNot:
		return "~"
	case stNegative:
		return "-"
	case stExponentiation:
		return "**"
	case stMultiply:
		return "*"
	case stDivide:
		return "/"
	case stModulo:
		return "%"
	case stAdd:
		return "+"
	case stSubtract:
		return "-"
	case stLessThan:
		return "<"
	case stGreaterThan:
		return ">"
	case stLessThanEqualTo:
		return "<="
	case stGreaterThanEqualTo:
		return ">="
	case stEqual:
		return "=="
	case stInequal:
		return "!="
	case stAnd:
		return "&&"
	case stExlusiveOr:
		return "^"
	case stOr:
		return "||"
	case stAssignment:
		return "="
	}
	//still here? panic
	panic("unsupported token " + token)
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2012 The go-gl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package glh

import (
	"image"
	"image/draw"
	"image/png"
	"io"

	"github.com/go-gl/gl"
)

// A 2D Texture which implements Context, so can be used as:
//   `With(texture, func() { .. textured primitives .. })`
// which sets gl.ENABLE_BIT
type Texture struct {
	gl.Texture
	W, H int
}

// Create a new texture, initialize it to have a `gl.LINEAR` filter and use
// `gl.CLAMP_TO_EDGE`.
func NewTexture(w, h int) *Texture {
	texture := &Texture{gl.GenTexture(), w, h}
	With(texture, func() {
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE)
	})
	return texture
}

// Initialize texture storage. _REQUIRED_ before using it as a framebuffer target.
func (t *Texture) Init() {
	With(t, func() {
		n_levels := []int32{0}

		gl.GetTexParameteriv(gl.TEXTURE_2D, gl.TEXTURE_MAX_LEVEL, n_levels)

		for i := 0; i < int(n_levels[0]); i++ {
			gl.TexImage2D(gl.TEXTURE_2D, i, gl.RGBA, t.W, t.H, 0, gl.RGBA,
				gl.UNSIGNED_BYTE, nil)
		}
	})
}

func (b Texture) Enter() {
	gl.PushAttrib(gl.ENABLE_BIT)
	gl.Enable(gl.TEXTURE_2D)
	b.Bind(gl.TEXTURE_2D)
}
func (b Texture) Exit() {
	b.Unbind(gl.TEXTURE_2D)
	gl.PopAttrib()
}

// Return the OpenGL texture as a golang `image.RGBA`
func (t *Texture) AsImage() *image.RGBA {
	rgba := image.NewRGBA(image.Rect(0, 0, t.W, t.H))
	With(t, func() {
		// TODO: check internal format (with GetIntegerv?)
		gl.GetTexImage(gl.TEXTURE_2D, 0, gl.RGBA, gl.UNSIGNED_BYTE, rgba.Pix)
	})
	return rgba
}

func (t *Texture) FromImageRGBA(rgba *image.RGBA, level int) {
	With(t, func() {
		gl.TexImage2D(gl.TEXTURE_2D, level, gl.RGBA,
			rgba.Bounds().Dx(), rgba.Bounds().Dy(),
			0, gl.RGBA, gl.UNSIGNED_BYTE, rgba.Pix)
	})
}

// Initialize this texture with image data from `im`. Note: copies the texture
// if it isn't in RGBA format. This can happen if you load from png files.
func (t *Texture) FromImage(im image.Image, level int) {
	switch trueim := im.(type) {
	case *image.RGBA:
		t.FromImageRGBA(trueim, level)

	default:
		copy := image.NewRGBA(trueim.Bounds())
		draw.Draw(copy, trueim.Bounds(), trueim, image.Pt(0, 0), draw.Src)
		t.FromImageRGBA(copy, level)
	}
}

func (t *Texture) FromPngReader(in io.Reader, level int) error {
	im, err := png.Decode(in)
	if err != nil {
		return err
	}
	t.FromImage(im, level)
	return nil
}
<commit_msg>Texture init now only creates required number of mipmap levels.<commit_after>// Copyright 2012 The go-gl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package glh

import (
	"github.com/go-gl/gl"
	"image"
	"image/draw"
	"image/png"
	"io"
)

// A 2D Texture which implements Context, so can be used as:
//   `With(texture, func() { .. textured primitives .. })`
// which sets gl.ENABLE_BIT
type Texture struct {
	gl.Texture
	W, H int
}

// Create a new texture, initialize it to have a `gl.LINEAR` filter and use
// `gl.CLAMP_TO_EDGE`.
func NewTexture(w, h int) *Texture {
	texture := &Texture{gl.GenTexture(), w, h}
	With(texture, func() {
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
		gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE)
	})
	return texture
}

// Initialize texture storage. _REQUIRED_ before using it as a framebuffer target.
func (t *Texture) Init() {
	With(t, func() {
		// generate base level storage
		gl.TexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, t.W, t.H, 0, gl.RGBA, gl.UNSIGNED_BYTE, nil)
		// generate required number of mipmaps given texture dimensions
		gl.GenerateMipmap(gl.TEXTURE_2D)
	})
}

func (b Texture) Enter() {
	gl.PushAttrib(gl.ENABLE_BIT)
	gl.Enable(gl.TEXTURE_2D)
	b.Bind(gl.TEXTURE_2D)
}
func (b Texture) Exit() {
	b.Unbind(gl.TEXTURE_2D)
	gl.PopAttrib()
}

// Return the OpenGL texture as a golang `image.RGBA`
func (t *Texture) AsImage() *image.RGBA {
	rgba := image.NewRGBA(image.Rect(0, 0, t.W, t.H))
	With(t, func() {
		// TODO: check internal format (with GetIntegerv?)
		gl.GetTexImage(gl.TEXTURE_2D, 0, gl.RGBA, gl.UNSIGNED_BYTE, rgba.Pix)
	})
	return rgba
}

func (t *Texture) FromImageRGBA(rgba *image.RGBA, level int) {
	With(t, func() {
		gl.TexImage2D(gl.TEXTURE_2D, level, gl.RGBA,
			rgba.Bounds().Dx(), rgba.Bounds().Dy(),
			0, gl.RGBA, gl.UNSIGNED_BYTE, rgba.Pix)
	})
}

// Initialize this texture with image data from `im`. Note: copies the texture
// if it isn't in RGBA format. This can happen if you load from png files.
func (t *Texture) FromImage(im image.Image, level int) {
	switch trueim := im.(type) {
	case *image.RGBA:
		t.FromImageRGBA(trueim, level)

	default:
		copy := image.NewRGBA(trueim.Bounds())
		draw.Draw(copy, trueim.Bounds(), trueim, image.Pt(0, 0), draw.Src)
		t.FromImageRGBA(copy, level)
	}
}

func (t *Texture) FromPngReader(in io.Reader, level int) error {
	im, err := png.Decode(in)
	if err != nil {
		return err
	}
	t.FromImage(im, level)
	return nil
}
<|endoftext|>


<commit_before>package typhon

import (
	"context"
	"strconv"
	"time"

	"github.com/monzo/terrors"
)

func TimeoutFilter(defaultTimeout time.Duration) Filter {
	return func(req Request, svc Service) Response {
		timeout := defaultTimeout
		if t, err := strconv.Atoi(req.Header.Get("Timeout")); err == nil {
			timeout = time.Duration(t) * time.Millisecond
		}

		req.Context, _ = context.WithTimeout(req.Context, timeout)
		rspChan := make(chan Response, 1)
		go func() {
			rspChan <- svc(req)
		}()

		select {
		case rsp := <-rspChan:
			return rsp
		case <-req.Context.Done():
			return Response{
				Error: terrors.Timeout("", "Request timed out", nil)}
		}
	}
}
<commit_msg>Fix a potential goroutine leak in TimeoutFilter<commit_after>package typhon

import (
	"context"
	"strconv"
	"time"

	"github.com/monzo/terrors"
)

// TimeoutFilter returns a Filter which will cancel a Request after the given timeout
func TimeoutFilter(defaultTimeout time.Duration) Filter {
	return func(req Request, svc Service) Response {
		timeout := defaultTimeout
		if t, err := strconv.Atoi(req.Header.Get("Timeout")); err == nil {
			timeout = time.Duration(t) * time.Millisecond
		}

		ctx, cancel := context.WithTimeout(req.Context, timeout)
		req.Context = ctx
		defer cancel()
		rspChan := make(chan Response, 1)
		go func() {
			rspChan <- svc(req)
		}()

		select {
		case rsp := <-rspChan:
			return rsp
		case <-req.Context.Done():
			return Response{
				Error: terrors.Timeout("", "Request timed out", nil)}
		}
	}
}
<|endoftext|>


<commit_before>// This file is part of *xliffer*
//
// Copyright (C) 2015, Travelping GmbH <copyright@travelping.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"
)

type toJson struct {
	inFile string
	pretty bool
}

func init() {
	registeredConverters["to-json"] = new(toJson)
}

func (tj *toJson) Description() string {
	return "Converts XLIFF to JSON (key,value)"
}

func (tj *toJson) ParseArgs(base string, args []string) error {
	var fs = flag.NewFlagSet(base+" to-json", flag.ExitOnError)
	fs.StringVar(&tj.inFile, "in", "", "infile")
	fs.BoolVar(&tj.pretty, "pretty", tj.pretty, "pretty print the resulting json")
	return fs.Parse(args)
}

func (tj *toJson) Convert() error {

	var doc, err = xliffFromFile(tj.inFile)
	if err != nil {
		return err
	}

	var mappings = map[string]string{}
	for _, file := range doc.File {

		// note: no support for "groups" yet.
		for _, unit := range file.Body.TransUnit {

			if _, exist := mappings[unit.ID]; exist {
				log.Printf("warning: double entry for key %q", unit.ID)
			}

			mappings[unit.ID] = unit.Target.Inner
		}
	}

	var out []byte
	if tj.pretty {
		out, err = json.MarshalIndent(&mappings, "", "\t")
	} else {
		out, err = json.Marshal(&mappings)
	}

	if err == nil {
		os.Stdout.Write(out)
	}

	return err
}
<commit_msg>add -key-match, -key-to flags to the to-json converter<commit_after>// This file is part of *xliffer*
//
// Copyright (C) 2015, Travelping GmbH <copyright@travelping.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"encoding/json"
	"flag"
	"log"
	"os"
	"regexp"
)

// toJSON converts the target translation to a simple key:value
// structered JSON file
type toJSON struct {
	inFile   string
	pretty   bool
	keyMatch string
	keyTo    string
}

func init() {
	registeredConverters["to-json"] = new(toJSON)
}

func (tj *toJSON) Description() string {
	return "Converts XLIFF to JSON (key,value)"
}

func (tj *toJSON) ParseArgs(base string, args []string) error {
	var fs = flag.NewFlagSet(base+" to-json", flag.ExitOnError)
	fs.StringVar(&tj.inFile, "in", "", "infile")
	fs.StringVar(&tj.keyMatch, "key-match", "", "translate chars in key (regexp)")
	fs.StringVar(&tj.keyTo, "key-to", "", "chars of key gets translated to (string)")
	fs.BoolVar(&tj.pretty, "pretty", tj.pretty, "pretty print the resulting json")
	return fs.Parse(args)
}

func (tj *toJSON) Convert() error {

	var doc, err = xliffFromFile(tj.inFile)
	if err != nil {
		return err
	}

	var keyTrans = func(in string) string { return in }
	if tj.keyMatch != "" {
		rx, err := regexp.CompilePOSIX(tj.keyMatch)
		if err != nil {
			return err
		}
		keyTrans = func(in string) string {
			return rx.ReplaceAllString(in, tj.keyTo)
		}
	}

	var mappings = map[string]string{}
	for _, file := range doc.File {

		// note: no support for "groups" yet.
		for _, unit := range file.Body.TransUnit {

			unitID := keyTrans(unit.ID)

			if _, exist := mappings[unitID]; exist {
				log.Printf("warning: double entry for key %q", unitID)
			}

			mappings[unitID] = unit.Target.Inner
		}
	}

	var out []byte
	if tj.pretty {
		out, err = json.MarshalIndent(&mappings, "", "\t")
	} else {
		out, err = json.Marshal(&mappings)
	}

	if err == nil {
		os.Stdout.Write(out)
	}

	return err
}
<|endoftext|>