<commit_before>package reposerver

import (
	"crypto/tls"
	"fmt"
	"os"
	"path/filepath"

	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"

	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/health"
	"google.golang.org/grpc/health/grpc_health_v1"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/reflection"

	"github.com/argoproj/argo-cd/v2/common"
	versionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/version"
	"github.com/argoproj/argo-cd/v2/reposerver/apiclient"
	reposervercache "github.com/argoproj/argo-cd/v2/reposerver/cache"
	"github.com/argoproj/argo-cd/v2/reposerver/metrics"
	"github.com/argoproj/argo-cd/v2/reposerver/repository"
	"github.com/argoproj/argo-cd/v2/server/version"
	"github.com/argoproj/argo-cd/v2/util/argo"
	"github.com/argoproj/argo-cd/v2/util/env"
	"github.com/argoproj/argo-cd/v2/util/git"
	grpc_util "github.com/argoproj/argo-cd/v2/util/grpc"
	tlsutil "github.com/argoproj/argo-cd/v2/util/tls"
)

// ArgoCDRepoServer is the repo server implementation
type ArgoCDRepoServer struct {
	log           *log.Entry
	repoService   *repository.Service
	metricsServer *metrics.MetricsServer
	gitCredsStore git.CredsStore
	cache         *reposervercache.Cache
	opts          []grpc.ServerOption
	initConstants repository.RepoServerInitConstants
}

// The hostnames to generate self-signed issues with
var tlsHostList []string = []string{"localhost", "reposerver"}

// NewServer returns a new instance of the Argo CD Repo server
func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cache, tlsConfCustomizer tlsutil.ConfigCustomizer, initConstants repository.RepoServerInitConstants, gitCredsStore git.CredsStore) (*ArgoCDRepoServer, error) {
	var tlsConfig *tls.Config

	// Generate or load TLS server certificates to use with this instance of
	// repository server.
	if tlsConfCustomizer != nil {
		var err error
		certPath := fmt.Sprintf("%s/reposerver/tls/tls.crt", env.StringFromEnv(common.EnvAppConfigPath, common.DefaultAppConfigPath))
		keyPath := fmt.Sprintf("%s/reposerver/tls/tls.key", env.StringFromEnv(common.EnvAppConfigPath, common.DefaultAppConfigPath))
		tlsConfig, err = tlsutil.CreateServerTLSConfig(certPath, keyPath, tlsHostList)
		if err != nil {
			return nil, err
		}
		tlsConfCustomizer(tlsConfig)
	}

	if os.Getenv(common.EnvEnableGRPCTimeHistogramEnv) == "true" {
		grpc_prometheus.EnableHandlingTimeHistogram()
	}

	serverLog := log.NewEntry(log.StandardLogger())
	streamInterceptors := []grpc.StreamServerInterceptor{
		otelgrpc.StreamServerInterceptor(),
		grpc_logrus.StreamServerInterceptor(serverLog),
		grpc_prometheus.StreamServerInterceptor,
		grpc_util.PanicLoggerStreamServerInterceptor(serverLog),
	}
	unaryInterceptors := []grpc.UnaryServerInterceptor{
		otelgrpc.UnaryServerInterceptor(),
		grpc_logrus.UnaryServerInterceptor(serverLog),
		grpc_prometheus.UnaryServerInterceptor,
		grpc_util.PanicLoggerUnaryServerInterceptor(serverLog),
		grpc_util.ErrorSanitizerUnaryServerInterceptor(),
	}

	serverOpts := []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)),
		grpc.MaxRecvMsgSize(apiclient.MaxGRPCMessageSize),
		grpc.MaxSendMsgSize(apiclient.MaxGRPCMessageSize),
		grpc.KeepaliveEnforcementPolicy(
			keepalive.EnforcementPolicy{
				MinTime: common.GRPCKeepAliveEnforcementMinimum,
			},
		),
	}

	// We do allow for non-TLS servers to be created, in case of mTLS will be
	// implemented by e.g. a sidecar container.
	if tlsConfig != nil {
		serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsConfig)))
	}
	repoService := repository.NewService(metricsServer, cache, initConstants, argo.NewResourceTracking(), gitCredsStore, filepath.Join(os.TempDir(), "_argocd-repo"))
	if err := repoService.Init(); err != nil {
		return nil, err
	}

	return &ArgoCDRepoServer{
		log:           serverLog,
		metricsServer: metricsServer,
		cache:         cache,
		initConstants: initConstants,
		opts:          serverOpts,
		gitCredsStore: gitCredsStore,
		repoService:   repoService,
	}, nil
}

// CreateGRPC creates new configured grpc server
func (a *ArgoCDRepoServer) CreateGRPC() *grpc.Server {
	server := grpc.NewServer(a.opts...)
	versionpkg.RegisterVersionServiceServer(server, version.NewServer(nil, func() (bool, error) {
		return true, nil
	}))
	apiclient.RegisterRepoServerServiceServer(server, a.repoService)

	healthService := health.NewServer()
	grpc_health_v1.RegisterHealthServer(server, healthService)

	// Register reflection service on gRPC server.
	reflection.Register(server)

	return server
}
<commit_msg>chore: updated error message to include context (#10592) (#10960)<commit_after>package reposerver

import (
	"crypto/tls"
	"fmt"
	"os"
	"path/filepath"

	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"

	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/health"
	"google.golang.org/grpc/health/grpc_health_v1"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/reflection"

	"github.com/argoproj/argo-cd/v2/common"
	versionpkg "github.com/argoproj/argo-cd/v2/pkg/apiclient/version"
	"github.com/argoproj/argo-cd/v2/reposerver/apiclient"
	reposervercache "github.com/argoproj/argo-cd/v2/reposerver/cache"
	"github.com/argoproj/argo-cd/v2/reposerver/metrics"
	"github.com/argoproj/argo-cd/v2/reposerver/repository"
	"github.com/argoproj/argo-cd/v2/server/version"
	"github.com/argoproj/argo-cd/v2/util/argo"
	"github.com/argoproj/argo-cd/v2/util/env"
	"github.com/argoproj/argo-cd/v2/util/git"
	grpc_util "github.com/argoproj/argo-cd/v2/util/grpc"
	tlsutil "github.com/argoproj/argo-cd/v2/util/tls"
)

// ArgoCDRepoServer is the repo server implementation
type ArgoCDRepoServer struct {
	log           *log.Entry
	repoService   *repository.Service
	metricsServer *metrics.MetricsServer
	gitCredsStore git.CredsStore
	cache         *reposervercache.Cache
	opts          []grpc.ServerOption
	initConstants repository.RepoServerInitConstants
}

// The hostnames to generate self-signed issues with
var tlsHostList []string = []string{"localhost", "reposerver"}

// NewServer returns a new instance of the Argo CD Repo server
func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cache, tlsConfCustomizer tlsutil.ConfigCustomizer, initConstants repository.RepoServerInitConstants, gitCredsStore git.CredsStore) (*ArgoCDRepoServer, error) {
	var tlsConfig *tls.Config

	// Generate or load TLS server certificates to use with this instance of
	// repository server.
	if tlsConfCustomizer != nil {
		var err error
		certPath := fmt.Sprintf("%s/reposerver/tls/tls.crt", env.StringFromEnv(common.EnvAppConfigPath, common.DefaultAppConfigPath))
		keyPath := fmt.Sprintf("%s/reposerver/tls/tls.key", env.StringFromEnv(common.EnvAppConfigPath, common.DefaultAppConfigPath))
		tlsConfig, err = tlsutil.CreateServerTLSConfig(certPath, keyPath, tlsHostList)
		if err != nil {
			return nil, fmt.Errorf("error creating server TLS config: %w", err)
		}
		tlsConfCustomizer(tlsConfig)
	}

	if os.Getenv(common.EnvEnableGRPCTimeHistogramEnv) == "true" {
		grpc_prometheus.EnableHandlingTimeHistogram()
	}

	serverLog := log.NewEntry(log.StandardLogger())
	streamInterceptors := []grpc.StreamServerInterceptor{
		otelgrpc.StreamServerInterceptor(),
		grpc_logrus.StreamServerInterceptor(serverLog),
		grpc_prometheus.StreamServerInterceptor,
		grpc_util.PanicLoggerStreamServerInterceptor(serverLog),
	}
	unaryInterceptors := []grpc.UnaryServerInterceptor{
		otelgrpc.UnaryServerInterceptor(),
		grpc_logrus.UnaryServerInterceptor(serverLog),
		grpc_prometheus.UnaryServerInterceptor,
		grpc_util.PanicLoggerUnaryServerInterceptor(serverLog),
		grpc_util.ErrorSanitizerUnaryServerInterceptor(),
	}

	serverOpts := []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)),
		grpc.MaxRecvMsgSize(apiclient.MaxGRPCMessageSize),
		grpc.MaxSendMsgSize(apiclient.MaxGRPCMessageSize),
		grpc.KeepaliveEnforcementPolicy(
			keepalive.EnforcementPolicy{
				MinTime: common.GRPCKeepAliveEnforcementMinimum,
			},
		),
	}

	// We do allow for non-TLS servers to be created, in case of mTLS will be
	// implemented by e.g. a sidecar container.
	if tlsConfig != nil {
		serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsConfig)))
	}
	repoService := repository.NewService(metricsServer, cache, initConstants, argo.NewResourceTracking(), gitCredsStore, filepath.Join(os.TempDir(), "_argocd-repo"))
	if err := repoService.Init(); err != nil {
		return nil, err
	}

	return &ArgoCDRepoServer{
		log:           serverLog,
		metricsServer: metricsServer,
		cache:         cache,
		initConstants: initConstants,
		opts:          serverOpts,
		gitCredsStore: gitCredsStore,
		repoService:   repoService,
	}, nil
}

// CreateGRPC creates new configured grpc server
func (a *ArgoCDRepoServer) CreateGRPC() *grpc.Server {
	server := grpc.NewServer(a.opts...)
	versionpkg.RegisterVersionServiceServer(server, version.NewServer(nil, func() (bool, error) {
		return true, nil
	}))
	apiclient.RegisterRepoServerServiceServer(server, a.repoService)

	healthService := health.NewServer()
	grpc_health_v1.RegisterHealthServer(server, healthService)

	// Register reflection service on gRPC server.
	reflection.Register(server)

	return server
}
<|endoftext|>


<commit_before>package octokat

import (
	"encoding/json"
	"github.com/bmizerany/assert"
	"net/http"
	"reflect"
	"testing"
)

func TestCreateRepository(t *testing.T) {
	setup()
	defer tearDown()

	params := RepositoryParams{}
	params.Name = "Hello-World"
	params.Description = "This is your first repo"
	params.Homepage = "https://github.com"
	params.Private = false
	params.HasIssues = true
	params.HasWiki = true
	params.HasDownloads = true

	mux.HandleFunc("/user/repos", func(w http.ResponseWriter, r *http.Request) {
		var repoParams RepositoryParams
		json.NewDecoder(r.Body).Decode(&repoParams)
		assert.T(t, reflect.DeepEqual(repoParams, params))

		testMethod(t, r, "POST")
		respondWith(w, loadFixture("create_repository.json"))
	})

	options := Options{Params: params}
	repo, _ := client.CreateRepository("", &options)

	assert.Equal(t, 1296269, repo.ID)
	assert.Equal(t, "Hello-World", repo.Name)
	assert.Equal(t, "octocat/Hello-World", repo.FullName)
	assert.Equal(t, "This is your first repo", repo.Description)
	assert.T(t, !repo.Private)
	assert.T(t, repo.Fork)
	assert.Equal(t, "https://api.github.com/repos/octocat/Hello-World", repo.URL)
	assert.Equal(t, "https://github.com/octocat/Hello-World", repo.HTMLURL)
	assert.Equal(t, "https://github.com/octocat/Hello-World.git", repo.CloneURL)
	assert.Equal(t, "git://github.com/octocat/Hello-World.git", repo.GitURL)
	assert.Equal(t, "git@github.com:octocat/Hello-World.git", repo.SSHURL)
	assert.Equal(t, "master", repo.MasterBranch)
}

func TestFork(t *testing.T) {
	//c := NewClient().WithToken(os.Getenv("GITHUB_TOKEN"))
	//repo := Repo{"octokat", "jingweno"}

	//repository, err := c.Fork(repo, nil)
	//assert.Equal(t, nil, err)
	//assert.Equal(t, "octokat", repository.Name)
}
<commit_msg>Remove commented out test<commit_after>package octokat

import (
	"encoding/json"
	"github.com/bmizerany/assert"
	"net/http"
	"reflect"
	"testing"
)

func TestCreateRepository(t *testing.T) {
	setup()
	defer tearDown()

	params := RepositoryParams{}
	params.Name = "Hello-World"
	params.Description = "This is your first repo"
	params.Homepage = "https://github.com"
	params.Private = false
	params.HasIssues = true
	params.HasWiki = true
	params.HasDownloads = true

	mux.HandleFunc("/user/repos", func(w http.ResponseWriter, r *http.Request) {
		var repoParams RepositoryParams
		json.NewDecoder(r.Body).Decode(&repoParams)
		assert.T(t, reflect.DeepEqual(repoParams, params))

		testMethod(t, r, "POST")
		respondWith(w, loadFixture("create_repository.json"))
	})

	options := Options{Params: params}
	repo, _ := client.CreateRepository("", &options)

	assert.Equal(t, 1296269, repo.ID)
	assert.Equal(t, "Hello-World", repo.Name)
	assert.Equal(t, "octocat/Hello-World", repo.FullName)
	assert.Equal(t, "This is your first repo", repo.Description)
	assert.T(t, !repo.Private)
	assert.T(t, repo.Fork)
	assert.Equal(t, "https://api.github.com/repos/octocat/Hello-World", repo.URL)
	assert.Equal(t, "https://github.com/octocat/Hello-World", repo.HTMLURL)
	assert.Equal(t, "https://github.com/octocat/Hello-World.git", repo.CloneURL)
	assert.Equal(t, "git://github.com/octocat/Hello-World.git", repo.GitURL)
	assert.Equal(t, "git@github.com:octocat/Hello-World.git", repo.SSHURL)
	assert.Equal(t, "master", repo.MasterBranch)
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
	formatConfigFile  = "format.json"
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage            StorageAPI
	listObjectMap      map[listParams][]*treeWalker
	listObjectMapMutex *sync.Mutex
}

// isValidFormat - validates input arguments with backend 'format.json'
func isValidFormat(storage StorageAPI, exportPaths ...string) bool {
	// Load saved XL format.json and validate.
	xl, err := loadFormatXL(storage)
	if err != nil {
		log.Errorf("loadFormatXL failed with %s", err)
		return false
	}
	if xl.Version != "1" {
		log.Errorf("Unsupported XL backend format found [%s]", xl.Version)
		return false
	}
	if len(exportPaths) != len(xl.Disks) {
		log.Errorf("Number of disks %d passed at the command-line did not match the backend format %d", len(exportPaths), len(xl.Disks))
		return false
	}
	for index, disk := range xl.Disks {
		if exportPaths[index] != disk {
			log.Errorf("Invalid order of disks detected %s. Required order is %s.", exportPaths, xl.Disks)
			return false
		}
	}
	return true
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		log.Errorf("newXL failed with %s", err)
		return nil, err
	}

	// Initialize object layer - like creating minioMetaBucket,
	// cleaning up tmp files etc.
	initObjectLayer(storage)

	err = checkFormat(storage)
	if err != nil {
		if err == errFileNotFound {
			// Save new XL format.
			errSave := saveFormatXL(storage, &xlFormat{
				Version: "1",
				Disks:   exportPaths,
			})
			if errSave != nil {
				log.Errorf("saveFormatXL failed with %s", errSave)
				return nil, errSave
			}
		} else {
			log.Errorf("Unable to check backend format %s", err)
			if err == errReadQuorum {
				errMsg := fmt.Sprintf("Not all disks %s are available, did not meet read quroum.", exportPaths)
				err = errors.New(errMsg)
			} else if err == errDiskNotFound {
				errMsg := fmt.Sprintf("Disks %s not found.", exportPaths)
				err = errors.New(errMsg)
			} else if err == errVolumeAccessDenied {
				errMsg := fmt.Sprintf("Disks %s access permission denied.", exportPaths)
				err = errors.New(errMsg)
			}
			return nil, err
		}
	}

	// Validate if format exists and input arguments are validated
	// with backend format.
	if !isValidFormat(storage, exportPaths...) {
		return nil, fmt.Errorf("Command-line arguments %s is not valid.", exportPaths)
	}

	// Return successfully initialized object layer.
	return xlObjects{
		storage:            storage,
		listObjectMap:      make(map[listParams][]*treeWalker),
		listObjectMapMutex: &sync.Mutex{},
	}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return nil, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info.Parts); partIndex++ {
			part := info.Parts[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			// Reset offset to 0 as it would be non-0 only for the first loop if startOffset is non-0.
			offset = 0
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	return info, nil
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Check whether the bucket exists.
	if !isBucketExist(xl.storage, bucket) {
		return ObjectInfo{}, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		if err != errFileNotFound {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		var info MultipartObjectInfo
		info, err = getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.Size
		fi.ModTime = info.ModTime
		fi.MD5Sum = info.MD5Sum
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      fi.MD5Sum,
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return BucketNotFound{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err = xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
		return nil
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	var wg = &sync.WaitGroup{}
	var errs = make([]error, len(info.Parts))
	for index, part := range info.Parts {
		wg.Add(1)
		// Start deleting parts in routine.
		go func(index int, part MultipartPartInfo) {
			defer wg.Done()
			partFileName := partNumToPartFileName(part.PartNumber)
			errs[index] = xl.storage.DeleteFile(bucket, pathJoin(object, partFileName))
		}(index, part)
	}
	// Wait for all the deletes to finish.
	wg.Wait()
	// Loop through and validate if any errors, return back the first
	// error occurred.
	for index, err := range errs {
		if err != nil {
			partFileName := partNumToPartFileName(info.Parts[index].PartNumber)
			return toObjectErr(err, bucket, pathJoin(object, partFileName))
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	return listObjectsCommon(xl, bucket, prefix, marker, delimiter, maxKeys)
}
<commit_msg>Closed readCloser for each multipart in xl.GetObject (#1629)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
	formatConfigFile  = "format.json"
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage            StorageAPI
	listObjectMap      map[listParams][]*treeWalker
	listObjectMapMutex *sync.Mutex
}

// isValidFormat - validates input arguments with backend 'format.json'
func isValidFormat(storage StorageAPI, exportPaths ...string) bool {
	// Load saved XL format.json and validate.
	xl, err := loadFormatXL(storage)
	if err != nil {
		log.Errorf("loadFormatXL failed with %s", err)
		return false
	}
	if xl.Version != "1" {
		log.Errorf("Unsupported XL backend format found [%s]", xl.Version)
		return false
	}
	if len(exportPaths) != len(xl.Disks) {
		log.Errorf("Number of disks %d passed at the command-line did not match the backend format %d", len(exportPaths), len(xl.Disks))
		return false
	}
	for index, disk := range xl.Disks {
		if exportPaths[index] != disk {
			log.Errorf("Invalid order of disks detected %s. Required order is %s.", exportPaths, xl.Disks)
			return false
		}
	}
	return true
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		log.Errorf("newXL failed with %s", err)
		return nil, err
	}

	// Initialize object layer - like creating minioMetaBucket,
	// cleaning up tmp files etc.
	initObjectLayer(storage)

	err = checkFormat(storage)
	if err != nil {
		if err == errFileNotFound {
			// Save new XL format.
			errSave := saveFormatXL(storage, &xlFormat{
				Version: "1",
				Disks:   exportPaths,
			})
			if errSave != nil {
				log.Errorf("saveFormatXL failed with %s", errSave)
				return nil, errSave
			}
		} else {
			log.Errorf("Unable to check backend format %s", err)
			if err == errReadQuorum {
				errMsg := fmt.Sprintf("Not all disks %s are available, did not meet read quroum.", exportPaths)
				err = errors.New(errMsg)
			} else if err == errDiskNotFound {
				errMsg := fmt.Sprintf("Disks %s not found.", exportPaths)
				err = errors.New(errMsg)
			} else if err == errVolumeAccessDenied {
				errMsg := fmt.Sprintf("Disks %s access permission denied.", exportPaths)
				err = errors.New(errMsg)
			}
			return nil, err
		}
	}

	// Validate if format exists and input arguments are validated
	// with backend format.
	if !isValidFormat(storage, exportPaths...) {
		return nil, fmt.Errorf("Command-line arguments %s is not valid.", exportPaths)
	}

	// Return successfully initialized object layer.
	return xlObjects{
		storage:            storage,
		listObjectMap:      make(map[listParams][]*treeWalker),
		listObjectMapMutex: &sync.Mutex{},
	}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return nil, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info.Parts); partIndex++ {
			part := info.Parts[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			// Reset offset to 0 as it would be non-0 only for the first loop if startOffset is non-0.
			offset = 0
			if _, err = io.Copy(fileWriter, r); err != nil {
				switch reader := r.(type) {
				case *io.PipeReader:
					reader.CloseWithError(err)
				case io.ReadCloser:
					reader.Close()
				}
				fileWriter.CloseWithError(err)
				return
			}
			// Close the readerCloser that reads multiparts of an object from the xl storage layer.
			// Not closing leaks underlying file descriptors.
			r.Close()
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	return info, nil
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Check whether the bucket exists.
	if !isBucketExist(xl.storage, bucket) {
		return ObjectInfo{}, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		if err != errFileNotFound {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		var info MultipartObjectInfo
		info, err = getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.Size
		fi.ModTime = info.ModTime
		fi.MD5Sum = info.MD5Sum
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      fi.MD5Sum,
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return BucketNotFound{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err = xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
		return nil
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	var wg = &sync.WaitGroup{}
	var errs = make([]error, len(info.Parts))
	for index, part := range info.Parts {
		wg.Add(1)
		// Start deleting parts in routine.
		go func(index int, part MultipartPartInfo) {
			defer wg.Done()
			partFileName := partNumToPartFileName(part.PartNumber)
			errs[index] = xl.storage.DeleteFile(bucket, pathJoin(object, partFileName))
		}(index, part)
	}
	// Wait for all the deletes to finish.
	wg.Wait()
	// Loop through and validate if any errors, return back the first
	// error occurred.
	for index, err := range errs {
		if err != nil {
			partFileName := partNumToPartFileName(info.Parts[index].PartNumber)
			return toObjectErr(err, bucket, pathJoin(object, partFileName))
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	return listObjectsCommon(xl, bucket, prefix, marker, delimiter, maxKeys)
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"context"
	"log"
	"net"
	"os"
	"testing"

	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/genproto/googleapis/monitoring/v3"
	"google.golang.org/genproto/googleapis/rpc/status"
	"google.golang.org/genproto/googleapis/api/monitoredres"
	"google.golang.org/genproto/googleapis/api/metric"
	"google.golang.org/grpc"
	"google.golang.org/grpc/test/bufconn"
	"google.golang.org/protobuf/proto"
)

const bufSize = 1024 * 1024

var (
	client monitoring.MetricServiceClient
	conn   *grpc.ClientConn
	ctx    context.Context
	lis    *bufconn.Listener
)

func setup() {
	// Setup the in-memory server.
	lis = bufconn.Listen(bufSize)
	grpcServer := grpc.NewServer()
	monitoring.RegisterMetricServiceServer(grpcServer, &MockMetricServer{})
	go func() {
		if err := grpcServer.Serve(lis); err != nil {
			log.Fatalf("server exited with error: %v", err)
		}
	}()

	// Setup the connection and client.
	ctx = context.Background()
	var err error
	conn, err = grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure())
	if err != nil {
		log.Fatalf("failed to dial bufnet: %v", err)
	}
	client = monitoring.NewMetricServiceClient(conn)
}

func tearDown() {
	conn.Close()
}

func TestMain(m *testing.M) {
	setup()
	retCode := m.Run()
	tearDown()
	os.Exit(retCode)
}

func bufDialer(context.Context, string) (net.Conn, error) {
	return lis.Dial()
}

func TestMockMetricServer_CreateTimeSeries(t *testing.T) {
	cases := []struct {
		in *monitoring.CreateTimeSeriesRequest
		want *empty.Empty
	}{
		{
			&monitoring.CreateTimeSeriesRequest{
				Name: "test create time series request",
			},
			&empty.Empty{},
		},
	}

	for _, c := range cases {
		response, err := client.CreateTimeSeries(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call CreateTimeSeries %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("CreateTimeSeries(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_ListTimeSeries(t *testing.T) {
	cases := []struct {
		in *monitoring.ListTimeSeriesRequest
		want *monitoring.ListTimeSeriesResponse
	}{
		{
			&monitoring.ListTimeSeriesRequest{
				Name: "test list time series request",
			},
			&monitoring.ListTimeSeriesResponse{
				TimeSeries:      []*monitoring.TimeSeries{},
				NextPageToken:   "",
				ExecutionErrors: []*status.Status{},
			},
		},
	}

	for _, c := range cases {
		response, err := client.ListTimeSeries(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call ListTimeSeries %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("ListTimeSeries(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_GetMonitoredResourceDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.GetMonitoredResourceDescriptorRequest
		want *monitoredres.MonitoredResourceDescriptor
	}{
		{
			&monitoring.GetMonitoredResourceDescriptorRequest{
				Name: "test get metric monitored resource descriptor",
			},
			&monitoredres.MonitoredResourceDescriptor{},
		},
	}

	for _, c := range cases {
		response, err := client.GetMonitoredResourceDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call GetMonitoredResourceDescriptor %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("GetMonitoredResourceDescriptor(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_ListMonitoredResourceDescriptors(t *testing.T) {
	cases := []struct {
		in *monitoring.ListMonitoredResourceDescriptorsRequest
		want *monitoring.ListMonitoredResourceDescriptorsResponse
	}{
		{
			&monitoring.ListMonitoredResourceDescriptorsRequest{
				Name: "test list monitored resource descriptors",
			},
			&monitoring.ListMonitoredResourceDescriptorsResponse{
				ResourceDescriptors: []*monitoredres.MonitoredResourceDescriptor{},
				NextPageToken:       "",
			},
		},
	}

	for _, c := range cases {
		response, err := client.ListMonitoredResourceDescriptors(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call ListMonitoredResourceDescriptors %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("ListMonitoredResourceDescriptors(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_GetMetricDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.GetMetricDescriptorRequest
		want *metric.MetricDescriptor
	}{
		{
			&monitoring.GetMetricDescriptorRequest{
				Name: "test get metric descriptor",
			},
			&metric.MetricDescriptor{},
		},
	}

	for _, c := range cases {
		response, err := client.GetMetricDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call GetMetricDescriptor %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("GetMetricDescriptor(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_CreateMetricDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.CreateMetricDescriptorRequest
		want *metric.MetricDescriptor
	}{
		{
			&monitoring.CreateMetricDescriptorRequest{
				Name: "test create metric descriptor",
				MetricDescriptor: &metric.MetricDescriptor{},
			},
			&metric.MetricDescriptor{},
		},
	}

	for _, c := range cases {
		response, err := client.CreateMetricDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call CreateMetricDescriptorRequest: %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("CreateMetricDescriptorRequest(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_DeleteMetricDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.DeleteMetricDescriptorRequest
		want *empty.Empty
	}{
		{
			&monitoring.DeleteMetricDescriptorRequest{
				Name: "test create metric descriptor",
			},
			&empty.Empty{},
		},
	}

	for _,c := range cases {
		response, err := client.DeleteMetricDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call DeleteMetricDescriptorRequest: %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("DeleteMetricDescriptorRequest(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_ListMetricDescriptors(t *testing.T) {
	cases := []struct {
		in *monitoring.ListMetricDescriptorsRequest
		want *monitoring.ListMetricDescriptorsResponse
	}{
		{
			&monitoring.ListMetricDescriptorsRequest{
				Name: "test list metric decriptors request",
			},
			&monitoring.ListMetricDescriptorsResponse{
				MetricDescriptors: []*metric.MetricDescriptor{},
				NextPageToken:     "",
			},
		},
	}

	for _, c := range cases {
		response, err := client.ListMetricDescriptors(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call ListMetricDescriptors %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("ListMetricDescriptors(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}<commit_msg>address nit default string value/add stop to grpc server<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metric

import (
	"context"
	"log"
	"net"
	"os"
	"testing"

	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/genproto/googleapis/monitoring/v3"
	"google.golang.org/genproto/googleapis/rpc/status"
	"google.golang.org/genproto/googleapis/api/monitoredres"
	"google.golang.org/genproto/googleapis/api/metric"
	"google.golang.org/grpc"
	"google.golang.org/grpc/test/bufconn"
	"google.golang.org/protobuf/proto"
)

const bufSize = 1024 * 1024

var (
	client monitoring.MetricServiceClient
	conn   *grpc.ClientConn
	ctx    context.Context
	grpcServer *grpc.Server
	lis    *bufconn.Listener
)

func setup() {
	// Setup the in-memory server.
	lis = bufconn.Listen(bufSize)
	grpcServer = grpc.NewServer()
	monitoring.RegisterMetricServiceServer(grpcServer, &MockMetricServer{})
	go func() {
		if err := grpcServer.Serve(lis); err != nil {
			log.Fatalf("server exited with error: %v", err)
		}
	}()

	// Setup the connection and client.
	ctx = context.Background()
	var err error
	conn, err = grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(bufDialer), grpc.WithInsecure())
	if err != nil {
		log.Fatalf("failed to dial bufnet: %v", err)
	}
	client = monitoring.NewMetricServiceClient(conn)
}

func tearDown() {
	conn.Close()
	grpcServer.GracefulStop()
}

func TestMain(m *testing.M) {
	setup()
	retCode := m.Run()
	tearDown()
	os.Exit(retCode)
}

func bufDialer(context.Context, string) (net.Conn, error) {
	return lis.Dial()
}

func TestMockMetricServer_CreateTimeSeries(t *testing.T) {
	cases := []struct {
		in *monitoring.CreateTimeSeriesRequest
		want *empty.Empty
	}{
		{
			&monitoring.CreateTimeSeriesRequest{
				Name: "test create time series request",
			},
			&empty.Empty{},
		},
	}

	for _, c := range cases {
		response, err := client.CreateTimeSeries(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call CreateTimeSeries %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("CreateTimeSeries(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_ListTimeSeries(t *testing.T) {
	cases := []struct {
		in *monitoring.ListTimeSeriesRequest
		want *monitoring.ListTimeSeriesResponse
	}{
		{
			&monitoring.ListTimeSeriesRequest{
				Name: "test list time series request",
			},
			&monitoring.ListTimeSeriesResponse{
				TimeSeries:      []*monitoring.TimeSeries{},
				NextPageToken:   "",
				ExecutionErrors: []*status.Status{},
			},
		},
	}

	for _, c := range cases {
		response, err := client.ListTimeSeries(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call ListTimeSeries %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("ListTimeSeries(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_GetMonitoredResourceDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.GetMonitoredResourceDescriptorRequest
		want *monitoredres.MonitoredResourceDescriptor
	}{
		{
			&monitoring.GetMonitoredResourceDescriptorRequest{
				Name: "test get metric monitored resource descriptor",
			},
			&monitoredres.MonitoredResourceDescriptor{},
		},
	}

	for _, c := range cases {
		response, err := client.GetMonitoredResourceDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call GetMonitoredResourceDescriptor %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("GetMonitoredResourceDescriptor(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_ListMonitoredResourceDescriptors(t *testing.T) {
	cases := []struct {
		in *monitoring.ListMonitoredResourceDescriptorsRequest
		want *monitoring.ListMonitoredResourceDescriptorsResponse
	}{
		{
			&monitoring.ListMonitoredResourceDescriptorsRequest{
				Name: "test list monitored resource descriptors",
			},
			&monitoring.ListMonitoredResourceDescriptorsResponse{
				ResourceDescriptors: []*monitoredres.MonitoredResourceDescriptor{},
			},
		},
	}

	for _, c := range cases {
		response, err := client.ListMonitoredResourceDescriptors(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call ListMonitoredResourceDescriptors %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("ListMonitoredResourceDescriptors(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_GetMetricDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.GetMetricDescriptorRequest
		want *metric.MetricDescriptor
	}{
		{
			&monitoring.GetMetricDescriptorRequest{
				Name: "test get metric descriptor",
			},
			&metric.MetricDescriptor{},
		},
	}

	for _, c := range cases {
		response, err := client.GetMetricDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call GetMetricDescriptor %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("GetMetricDescriptor(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_CreateMetricDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.CreateMetricDescriptorRequest
		want *metric.MetricDescriptor
	}{
		{
			&monitoring.CreateMetricDescriptorRequest{
				Name: "test create metric descriptor",
				MetricDescriptor: &metric.MetricDescriptor{},
			},
			&metric.MetricDescriptor{},
		},
	}

	for _, c := range cases {
		response, err := client.CreateMetricDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call CreateMetricDescriptorRequest: %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("CreateMetricDescriptorRequest(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_DeleteMetricDescriptor(t *testing.T) {
	cases := []struct {
		in *monitoring.DeleteMetricDescriptorRequest
		want *empty.Empty
	}{
		{
			&monitoring.DeleteMetricDescriptorRequest{
				Name: "test create metric descriptor",
			},
			&empty.Empty{},
		},
	}

	for _, c := range cases {
		response, err := client.DeleteMetricDescriptor(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call DeleteMetricDescriptorRequest: %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("DeleteMetricDescriptorRequest(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}

func TestMockMetricServer_ListMetricDescriptors(t *testing.T) {
	cases := []struct {
		in *monitoring.ListMetricDescriptorsRequest
		want *monitoring.ListMetricDescriptorsResponse
	}{
		{
			&monitoring.ListMetricDescriptorsRequest{
				Name: "test list metric decriptors request",
			},
			&monitoring.ListMetricDescriptorsResponse{
				MetricDescriptors: []*metric.MetricDescriptor{},
				NextPageToken:     "",
			},
		},
	}

	for _, c := range cases {
		response, err := client.ListMetricDescriptors(ctx, c.in)
		if err != nil {
			t.Fatalf("failed to call ListMetricDescriptors %v", err)
		}

		if !proto.Equal(response, c.want) {
			t.Errorf("ListMetricDescriptors(%q) == %q, want %q", c.in, response, c.want)
		}
	}
}<|endoftext|>


<commit_before>// Copyright 2014, Truveris Inc. All Rights Reserved.
// Use of this source code is governed by the ISC license in the LICENSE file.

package main

import (
	"log"
	"os"

	"github.com/truveris/ygor"
)

var (
	Aliases *ygor.AliasFile
	Minions *ygor.MinionsFile
)

func main() {
	ParseCommandLine()

	err := ParseConfigFile()
	if err != nil {
		log.Fatal("config error: ", err.Error())
	}

	// We have global alias and minions files available to everyone. The
	// alias module and irc io adapter use aliases and everything uses minions.
	Aliases, err = ygor.OpenAliasFile(cfg.AliasFilePath)
	if err != nil {
		log.Fatal("alias file error: ", err.Error())
	}

	Minions, err = ygor.OpenMinionsFile(cfg.MinionsFilePath)
	if err != nil {
		log.Fatal("minions file error: ", err.Error())
	}

	log.Printf("registering modules")
	RegisterModule(&AliasModule{})
	RegisterModule(&ImageModule{})
	RegisterModule(&RebootModule{})
	RegisterModule(&MinionsModule{})
	RegisterModule(&PingModule{})
	RegisterModule(&SayModule{})
	RegisterModule(&ShutUpModule{})
	RegisterModule(&SoundBoardModule{})
	RegisterModule(&XombreroModule{})

	log.Printf("starting i/o adapters")
	ircerrch, minionerrch, err := StartAdapters()
	if err != nil {
		log.Fatal("failed to start adapters: ", err.Error())
	}

	log.Printf("ready")
	for {
		select {
		case err := <-ircerrch:
			log.Printf("irc handler error: %s", err.Error())
		case err := <-minionerrch:
			log.Printf("minion handler error: %s", err.Error())
		case msg := <-InputQueue:
			switch msg.Type {
			case ygor.MsgTypeIRCChannel:
				go IRCMessageHandler(msg)
			case ygor.MsgTypeIRCPrivate:
				go IRCMessageHandler(msg)
			case ygor.MsgTypeMinion:
				go MinionMessageHandler(msg)
			case ygor.MsgTypeExit:
				log.Printf("terminating: %s", msg.Body)
				os.Exit(0)
			case ygor.MsgTypeFatal:
				log.Fatal("fatal error: " + msg.Body)
			default:
				log.Printf("msg handler error: un-handled type"+
					" '%d'", msg.Type)
			}
		}
	}
}
<commit_msg>Dump a trace of all the goroutines to stderr on SIGUSR1.<commit_after>// Copyright 2014, Truveris Inc. All Rights Reserved.
// Use of this source code is governed by the ISC license in the LICENSE file.

package main

import (
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/truveris/ygor"
)

var (
	Aliases *ygor.AliasFile
	Minions *ygor.MinionsFile
)

func WaitForTraceRequest() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGUSR1)

	for _ = range ch {
		log.Printf("Received USR1 signal, printing stack trace:")
		buf := make([]byte, 4096)
		runtime.Stack(buf, true)
		log.Printf("%s", buf)
	}
}

func main() {
	ParseCommandLine()

	err := ParseConfigFile()
	if err != nil {
		log.Fatal("config error: ", err.Error())
	}

	// We have global alias and minions files available to everyone. The
	// alias module and irc io adapter use aliases and everything uses minions.
	Aliases, err = ygor.OpenAliasFile(cfg.AliasFilePath)
	if err != nil {
		log.Fatal("alias file error: ", err.Error())
	}

	Minions, err = ygor.OpenMinionsFile(cfg.MinionsFilePath)
	if err != nil {
		log.Fatal("minions file error: ", err.Error())
	}

	log.Printf("registering modules")
	RegisterModule(&AliasModule{})
	RegisterModule(&ImageModule{})
	RegisterModule(&RebootModule{})
	RegisterModule(&MinionsModule{})
	RegisterModule(&PingModule{})
	RegisterModule(&SayModule{})
	RegisterModule(&ShutUpModule{})
	RegisterModule(&SoundBoardModule{})
	RegisterModule(&XombreroModule{})

	log.Printf("starting i/o adapters")
	ircerrch, minionerrch, err := StartAdapters()
	if err != nil {
		log.Fatal("failed to start adapters: ", err.Error())
	}

	go WaitForTraceRequest()

	log.Printf("ready")
	for {
		select {
		case err := <-ircerrch:
			log.Printf("irc handler error: %s", err.Error())
		case err := <-minionerrch:
			log.Printf("minion handler error: %s", err.Error())
		case msg := <-InputQueue:
			switch msg.Type {
			case ygor.MsgTypeIRCChannel:
				go IRCMessageHandler(msg)
			case ygor.MsgTypeIRCPrivate:
				go IRCMessageHandler(msg)
			case ygor.MsgTypeMinion:
				go MinionMessageHandler(msg)
			case ygor.MsgTypeExit:
				log.Printf("terminating: %s", msg.Body)
				os.Exit(0)
			case ygor.MsgTypeFatal:
				log.Fatal("fatal error: " + msg.Body)
			default:
				log.Printf("msg handler error: un-handled type"+
					" '%d'", msg.Type)
			}
		}
	}
}
<|endoftext|>


<commit_before>// +build slow

package tollbooth

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/didip/tollbooth/limiter"
)

// See: https://github.com/didip/tollbooth/issues/48
// How to run: go test -tags=slow
func Test_Issue48_RequestTerminatedEvenOnLowVolume(t *testing.T) {
	lmt := limiter.New(nil).SetMax(20).SetTTL(time.Second)
	lmt.SetMethods([]string{"GET"})

	limitReachedCounter := 0
	lmt.SetOnLimitReached(func(w http.ResponseWriter, r *http.Request) { limitReachedCounter++ })

	handler := LimitHandler(lmt, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`hello world`))
	}))

	req, err := http.NewRequest("GET", "/doesntmatter", nil)
	if err != nil {
		t.Fatal(err)
	}

	// Why 11 times?
	// Because 11 * 2 = 22, and our limit is 20.
	// If the bug report is as what I understood, then this test is expected to break.
	tries := 11
	for i := 0; i < tries; i++ {
		// Twice per second should not be limited
		for j := 0; j < 2; j++ {
			rr := httptest.NewRecorder()
			handler.ServeHTTP(rr, req)

			//Should not be limited
			if status := rr.Code; status != http.StatusOK {
				t.Fatalf("Should be able to handle 20 reqs/second. HTTP status: %v. Expected HTTP status: %v", status, http.StatusOK)
			}
		}

		time.Sleep(time.Second)
	}

	if limitReachedCounter > 0 {
		t.Fatalf("We should never reached the limit, the counter should be 0. limitReachedCounter: %v", limitReachedCounter)
	}
}
<commit_msg>update test for issue 48 to use different ip addresses<commit_after>// +build slow
// How to run: go test -tags=slow

package tollbooth

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/didip/tollbooth/limiter"
)

// See: https://github.com/didip/tollbooth/issues/48
func Test_Issue48_RequestTerminatedEvenOnLowVolumeOnSameIP(t *testing.T) {
	lmt := limiter.New(nil).SetMax(20).SetTTL(time.Second)
	lmt.SetMethods([]string{"GET"})

	limitReachedCounter := 0
	lmt.SetOnLimitReached(func(w http.ResponseWriter, r *http.Request) { limitReachedCounter++ })

	handler := LimitHandler(lmt, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`hello world`))
	}))

	// Why 11 times?
	// Because 11 * 2 = 22, and our limit is 20.
	// If the bug report is as what I understood, then this test is expected to break.
	tries := 11
	for i := 0; i < tries; i++ {
		// Twice per second should not be limited
		for j := 0; j < 2; j++ {
			req, _ := http.NewRequest("GET", "/doesntmatter", nil)
			req.RemoteAddr = "127.0.0.1"

			rr := httptest.NewRecorder()
			handler.ServeHTTP(rr, req)

			//Should not be limited
			if status := rr.Code; status != http.StatusOK {
				t.Fatalf("Should be able to handle 20 reqs/second. HTTP status: %v. Expected HTTP status: %v", status, http.StatusOK)
			}
		}

		time.Sleep(time.Second)
	}

	if limitReachedCounter > 0 {
		t.Fatalf("We should never reached the limit, the counter should be 0. limitReachedCounter: %v", limitReachedCounter)
	}
}

// See: https://github.com/didip/tollbooth/issues/48
func Test_Issue48_RequestTerminatedEvenOnLowVolumeOnDifferentIPs(t *testing.T) {
	lmt := limiter.New(nil).SetMax(20).SetTTL(time.Second)
	lmt.SetMethods([]string{"GET"})

	limitReachedCounter := 0
	lmt.SetOnLimitReached(func(w http.ResponseWriter, r *http.Request) { limitReachedCounter++ })

	handler := LimitHandler(lmt, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`hello world`))
	}))

	// Why 11 times?
	// Because 11 * 2 = 22, and our limit is 20.
	// If the bug report is as what I understood, then this test is expected to break.
	tries := 11
	for i := 0; i < tries; i++ {
		// Twice per second should not be limited
		for j := 0; j < 2; j++ {
			req, _ := http.NewRequest("GET", "/doesntmatter", nil)
			req.RemoteAddr = fmt.Sprintf("127.0.0.%v", i)

			rr := httptest.NewRecorder()
			handler.ServeHTTP(rr, req)

			//Should not be limited
			if status := rr.Code; status != http.StatusOK {
				t.Fatalf("Should be able to handle 20 reqs/second. HTTP status: %v. Expected HTTP status: %v", status, http.StatusOK)
			}
		}

		time.Sleep(time.Second)
	}

	if limitReachedCounter > 0 {
		t.Fatalf("We should never reached the limit, the counter should be 0. limitReachedCounter: %v", limitReachedCounter)
	}
}
<|endoftext|>


<commit_before>package parse

import (
	"fmt"
	"strings"
	"unicode"
)

// tokenType defines a unique type of token
type tokenType int

const (
	tokenEOF tokenType = iota
	tokenText
	tokenComment
	tokenName
	tokenNumber
	tokenTagOpen
	tokenTagName
	tokenTagClose
	tokenPrintOpen
	tokenPrintClose
	tokenParensOpen
	tokenParensClose
	tokenArrayOpen
	tokenArrayClose
	tokenHashOpen
	tokenHashClose
	tokenStringOpen
	tokenStringClose
	tokenInterpolateOpen
	tokenInterpolateClose
	tokenPunctuation
	tokenOperator
	tokenWhitespace
	tokenError
)

var names = map[tokenType]string{
	tokenText:             "TEXT",
	tokenComment:          "COMMENT",
	tokenName:             "NAME",
	tokenNumber:           "NUMBER",
	tokenTagOpen:          "TAG_OPEN",
	tokenTagName:          "TAG_NAME",
	tokenTagClose:         "TAG_CLOSE",
	tokenPrintOpen:        "PRINT_OPEN",
	tokenPrintClose:       "PRINT_CLOSE",
	tokenParensOpen:       "PARENS_OPEN",
	tokenParensClose:      "PARENS_CLOSE",
	tokenArrayOpen:        "ARRAY_OPEN",
	tokenArrayClose:       "ARRAY_CLOSE",
	tokenHashOpen:         "HASH_OPEN",
	tokenHashClose:        "HASH_CLOSE",
	tokenStringOpen:       "STRING_OPEN",
	tokenStringClose:      "STRING_CLOSE",
	tokenInterpolateOpen:  "INTERPOLATE_OPEN",
	tokenInterpolateClose: "INTERPOLATE_CLOSE",
	tokenPunctuation:      "PUNCTUATION",
	tokenOperator:         "OPERATOR",
	tokenWhitespace:       "WHITESPACE",
	tokenError:            "ERROR",
	tokenEOF:              "EOF",
}

func (typ tokenType) String() string {
	return names[typ]
}

const (
	delimEOF              = ""
	delimOpenTag          = "{%"
	delimCloseTag         = "%}"
	delimOpenPrint        = "{{"
	delimClosePrint       = "}}"
	delimOpenComment      = "{#"
	delimCloseComment     = "#}"
	delimOpenInterpolate  = "#{"
	delimCloseInterpolate = "}"
)

type token struct {
	value     string
	tokenType tokenType
	pos
}

func (tok token) String() string {
	return fmt.Sprintf("{%s '%s' %s}", tok.tokenType, tok.value, tok.Pos())
}

// stateFn may emit zero or more tokens.
type stateFn func(*lexer) stateFn

// mode defines the lexer's current operating mode.
type mode int

const (
	modeClosed mode = iota
	modeNormal
	modeInterpolate
)

// lexer contains the current state of a lexer.
type lexer struct {
	start  int // The position of the last emission
	pos    int // The position of the cursor
	line   int // The current line number
	offset int // The current character offset on the current line
	input  string
	tokens chan token
	state  stateFn
	mode   mode
	last   token // The last emitted token
	parens int   // Number of open parenthesis
}

// nextToken returns the next token emitted by the lexer.
func (l *lexer) nextToken() token {
	for v, ok := <-l.tokens; ok; {
		l.last = v
		return v
	}

	return l.last
}

// tokenize kicks things off.
func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}

// newLexer creates a lexer, ready to begin tokenizing.
func newLexer(input string) *lexer {
	return &lexer{0, 0, 1, 0, input, make(chan token), nil, modeNormal, token{}, 0}
}

func (l *lexer) next() (val string) {
	if l.pos >= len(l.input) {
		val = delimEOF

	} else {
		val = l.input[l.pos : l.pos+1]

		l.pos++
	}

	return
}

func (l *lexer) backup() {
	l.pos--
}

func (l *lexer) peek() string {
	val := l.next()
	l.backup()
	return val
}

// emit will create a token with a value starting from the last emission
// until the current cursor position.
func (l *lexer) emit(t tokenType) {
	val := ""
	if l.pos <= len(l.input) {
		val = l.input[l.start:l.pos]
	}

	tok := token{val, t, newPos(l.line, l.offset)}

	if c := strings.Count(val, "\n"); c > 0 {
		l.line += c
		lpos := strings.LastIndex(val, "\n")
		l.offset = len(val[lpos+1:])
	} else {
		l.offset += len(val)
	}

	l.tokens <- tok
	l.start = l.pos
	if tok.tokenType == tokenEOF {
		close(l.tokens)
		l.mode = modeClosed
	}
}

func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	tok := token{fmt.Sprintf(format, args...), tokenError, newPos(l.line, l.offset)}
	l.tokens <- tok

	return nil
}

func lexData(l *lexer) stateFn {
	for {
		switch {
		case strings.HasPrefix(l.input[l.pos:], delimOpenComment):
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return lexCommentOpen

		case strings.HasPrefix(l.input[l.pos:], delimOpenTag):
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return lexTagOpen

		case strings.HasPrefix(l.input[l.pos:], delimOpenPrint):
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return lexPrintOpen
		}

		if l.next() == delimEOF {
			break
		}
	}

	if l.pos > l.start {
		l.emit(tokenText)
	}

	l.emit(tokenEOF)

	return nil
}

func lexExpression(l *lexer) stateFn {
	if l.tryLexOperator() {
		// Special handling for operators is necessary because of the alphabetical operators like "not" and "is"
		return lexExpression
	}
	switch str := l.peek(); {
	case str == delimEOF:
		return lexData

	case strings.HasPrefix(l.input[l.pos:], delimCloseTag):
		if l.pos > l.start {
			return l.errorf("Incompete token?")
		}
		return lexTagClose

	case strings.HasPrefix(l.input[l.pos:], delimClosePrint):
		if l.pos > l.start {
			return l.errorf("Incompete token?")
		}
		return lexPrintClose

	case isPunctuation(str):
		return lexPunctuation

	case strings.ContainsAny(str, "([{"):
		return lexOpenParens

	case strings.ContainsAny(str, "}])"):
		return lexCloseParens

	case strings.ContainsAny(str, "\"'"):
		return lexString

	case isNumeric(str):
		return lexNumber

	case isName(str):
		return lexName

	case isSpace(str):
		return lexSpace

	default:
		return l.errorf("Unknown expression")
	}
}

// tryLexOperator attempts to match the next sequence of characters to a list of operators.
// This is implemented this way because Twig supports many alphabetical operators like "in",
// which require more than just a check of the next character.
func (l *lexer) tryLexOperator() bool {
	op := operatorMatcher.FindString(l.input[l.pos:])
	if op == "" {
		return false
	} else if op == "%" {
		// Ensure this is not a tag close token "%}".
		// Go's regexp engine does not support negative lookahead.
		if l.input[l.pos+1:l.pos+2] == "}" {
			return false
		}
	} else if op == "in" {
		// TODO: Workaround to avoid matching "include" tags.
		if l.input[l.pos+2:l.pos+3] != " " {
			return false
		}
	}
	l.pos += len(op)
	l.emit(tokenOperator)

	return true
}

func lexSpace(l *lexer) stateFn {
	for {
		str := l.next()
		if !isSpace(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenWhitespace)

	return lexExpression
}

func lexNumber(l *lexer) stateFn {
	for {
		str := l.next()
		if !isNumeric(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenNumber)

	return lexExpression
}

func lexPunctuation(l *lexer) stateFn {
	for {
		str := l.next()
		if !isPunctuation(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenPunctuation)

	return lexExpression
}

func lexString(l *lexer) stateFn {
	open := l.next()
	l.emit(tokenStringOpen)
	closePos := strings.Index(l.input[l.pos:], open)
	if closePos < 0 {
		return l.errorf("unclosed string")
	}

	if open == `"` && strings.Contains(l.input[l.pos:closePos], delimOpenInterpolate) {
		input := l.input
		l.input = input[0 : l.pos+closePos]
		for {
			p := strings.Index(l.input[l.pos:], delimOpenInterpolate)
			if p < 0 {
				break
			}
			l.pos += p
			l.emit(tokenText)
			l.pos += len(delimOpenInterpolate)
			l.emit(tokenInterpolateOpen)
			l.mode = modeInterpolate
			for ins := lexExpression; ins != nil; {
				ins = ins(l)
			}
			if l.mode == modeClosed {
				return nil
			}
			l.mode = modeNormal
			l.emit(tokenInterpolateClose)
		}
		if l.pos < len(l.input) {
			l.pos = len(l.input)
			l.emit(tokenText)
		}
		l.input = input
	} else {
		l.pos += closePos
		l.emit(tokenText)
	}

	l.next()
	l.emit(tokenStringClose)

	return lexExpression
}

func lexOpenParens(l *lexer) stateFn {
	switch str := l.next(); {
	case str == "(":
		l.emit(tokenParensOpen)

	case str == "[":
		l.emit(tokenArrayOpen)

	case str == "{":
		l.emit(tokenHashOpen)

	default:
		return l.errorf("Unknown parenthesis")
	}
	l.parens++
	return lexExpression
}

func lexCloseParens(l *lexer) stateFn {
	switch str := l.next(); {
	case str == ")":
		l.emit(tokenParensClose)

	case str == "]":
		l.emit(tokenArrayClose)

	case str == "}":
		if l.parens == 0 && l.mode == modeInterpolate {
			return nil
		}
		l.emit(tokenHashClose)

	default:
		return l.errorf("Unknown parenthesis")
	}
	l.parens--
	return lexExpression
}

func lexName(l *lexer) stateFn {
	for {
		str := l.next()
		if str == delimEOF {
			break
		}
		if !isName(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenName)

	return lexExpression
}

func lexCommentOpen(l *lexer) stateFn {
	l.pos += len(delimOpenComment)
	l.start = l.pos
	til := strings.Index(l.input[l.pos:], delimCloseComment)
	if til < 0 {
		til = len(l.input[l.start:])
	}
	l.pos += til
	l.emit(tokenComment)
	if !strings.HasPrefix(l.input[l.pos:], delimCloseComment) {
		return l.errorf("Unclosed comment")
	}
	l.pos += len(delimCloseComment)
	l.start = l.pos

	return lexData
}

func lexTagOpen(l *lexer) stateFn {
	l.pos += len(delimOpenTag)
	l.emit(tokenTagOpen)

	return lexExpression
}

func lexTagClose(l *lexer) stateFn {
	l.pos += len(delimCloseTag)
	l.emit(tokenTagClose)

	return lexData
}

func lexPrintOpen(l *lexer) stateFn {
	l.pos += len(delimOpenPrint)
	l.emit(tokenPrintOpen)

	return lexExpression
}

func lexPrintClose(l *lexer) stateFn {
	l.pos += len(delimClosePrint)
	l.emit(tokenPrintClose)

	return lexData
}

func isSpace(str string) bool {
	return str == " " || str == "\t" || str == "\n"
}

func isName(str string) bool {
	for _, s := range str {
		if string(s) != "_" && !unicode.IsLetter(s) && !unicode.IsDigit(s) {
			return false
		}
	}

	return true
}

func isNumeric(str string) bool {
	for _, s := range str {
		if !unicode.IsDigit(s) {
			return false
		}
	}

	return true
}

func isPunctuation(str string) bool {
	for _, s := range str {
		if !strings.ContainsAny(string(s), ",|?:.") {
			return false
		}
	}

	return true
}
<commit_msg>Fixed erroneous slice bounds<commit_after>package parse

import (
	"fmt"
	"strings"
	"unicode"
)

// tokenType defines a unique type of token
type tokenType int

const (
	tokenEOF tokenType = iota
	tokenText
	tokenComment
	tokenName
	tokenNumber
	tokenTagOpen
	tokenTagName
	tokenTagClose
	tokenPrintOpen
	tokenPrintClose
	tokenParensOpen
	tokenParensClose
	tokenArrayOpen
	tokenArrayClose
	tokenHashOpen
	tokenHashClose
	tokenStringOpen
	tokenStringClose
	tokenInterpolateOpen
	tokenInterpolateClose
	tokenPunctuation
	tokenOperator
	tokenWhitespace
	tokenError
)

var names = map[tokenType]string{
	tokenText:             "TEXT",
	tokenComment:          "COMMENT",
	tokenName:             "NAME",
	tokenNumber:           "NUMBER",
	tokenTagOpen:          "TAG_OPEN",
	tokenTagName:          "TAG_NAME",
	tokenTagClose:         "TAG_CLOSE",
	tokenPrintOpen:        "PRINT_OPEN",
	tokenPrintClose:       "PRINT_CLOSE",
	tokenParensOpen:       "PARENS_OPEN",
	tokenParensClose:      "PARENS_CLOSE",
	tokenArrayOpen:        "ARRAY_OPEN",
	tokenArrayClose:       "ARRAY_CLOSE",
	tokenHashOpen:         "HASH_OPEN",
	tokenHashClose:        "HASH_CLOSE",
	tokenStringOpen:       "STRING_OPEN",
	tokenStringClose:      "STRING_CLOSE",
	tokenInterpolateOpen:  "INTERPOLATE_OPEN",
	tokenInterpolateClose: "INTERPOLATE_CLOSE",
	tokenPunctuation:      "PUNCTUATION",
	tokenOperator:         "OPERATOR",
	tokenWhitespace:       "WHITESPACE",
	tokenError:            "ERROR",
	tokenEOF:              "EOF",
}

func (typ tokenType) String() string {
	return names[typ]
}

const (
	delimEOF              = ""
	delimOpenTag          = "{%"
	delimCloseTag         = "%}"
	delimOpenPrint        = "{{"
	delimClosePrint       = "}}"
	delimOpenComment      = "{#"
	delimCloseComment     = "#}"
	delimOpenInterpolate  = "#{"
	delimCloseInterpolate = "}"
)

type token struct {
	value     string
	tokenType tokenType
	pos
}

func (tok token) String() string {
	return fmt.Sprintf("{%s '%s' %s}", tok.tokenType, tok.value, tok.Pos())
}

// stateFn may emit zero or more tokens.
type stateFn func(*lexer) stateFn

// mode defines the lexer's current operating mode.
type mode int

const (
	modeClosed mode = iota
	modeNormal
	modeInterpolate
)

// lexer contains the current state of a lexer.
type lexer struct {
	start  int // The position of the last emission
	pos    int // The position of the cursor
	line   int // The current line number
	offset int // The current character offset on the current line
	input  string
	tokens chan token
	state  stateFn
	mode   mode
	last   token // The last emitted token
	parens int   // Number of open parenthesis
}

// nextToken returns the next token emitted by the lexer.
func (l *lexer) nextToken() token {
	for v, ok := <-l.tokens; ok; {
		l.last = v
		return v
	}

	return l.last
}

// tokenize kicks things off.
func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}

// newLexer creates a lexer, ready to begin tokenizing.
func newLexer(input string) *lexer {
	return &lexer{0, 0, 1, 0, input, make(chan token), nil, modeNormal, token{}, 0}
}

func (l *lexer) next() (val string) {
	if l.pos >= len(l.input) {
		val = delimEOF

	} else {
		val = l.input[l.pos : l.pos+1]

		l.pos++
	}

	return
}

func (l *lexer) backup() {
	l.pos--
}

func (l *lexer) peek() string {
	val := l.next()
	l.backup()
	return val
}

// emit will create a token with a value starting from the last emission
// until the current cursor position.
func (l *lexer) emit(t tokenType) {
	val := ""
	if l.pos <= len(l.input) {
		val = l.input[l.start:l.pos]
	}

	tok := token{val, t, newPos(l.line, l.offset)}

	if c := strings.Count(val, "\n"); c > 0 {
		l.line += c
		lpos := strings.LastIndex(val, "\n")
		l.offset = len(val[lpos+1:])
	} else {
		l.offset += len(val)
	}

	l.tokens <- tok
	l.start = l.pos
	if tok.tokenType == tokenEOF {
		close(l.tokens)
		l.mode = modeClosed
	}
}

func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	tok := token{fmt.Sprintf(format, args...), tokenError, newPos(l.line, l.offset)}
	l.tokens <- tok

	return nil
}

func lexData(l *lexer) stateFn {
	for {
		switch {
		case strings.HasPrefix(l.input[l.pos:], delimOpenComment):
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return lexCommentOpen

		case strings.HasPrefix(l.input[l.pos:], delimOpenTag):
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return lexTagOpen

		case strings.HasPrefix(l.input[l.pos:], delimOpenPrint):
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return lexPrintOpen
		}

		if l.next() == delimEOF {
			break
		}
	}

	if l.pos > l.start {
		l.emit(tokenText)
	}

	l.emit(tokenEOF)

	return nil
}

func lexExpression(l *lexer) stateFn {
	if l.tryLexOperator() {
		// Special handling for operators is necessary because of the alphabetical operators like "not" and "is"
		return lexExpression
	}
	switch str := l.peek(); {
	case str == delimEOF:
		return lexData

	case strings.HasPrefix(l.input[l.pos:], delimCloseTag):
		if l.pos > l.start {
			return l.errorf("Incompete token?")
		}
		return lexTagClose

	case strings.HasPrefix(l.input[l.pos:], delimClosePrint):
		if l.pos > l.start {
			return l.errorf("Incompete token?")
		}
		return lexPrintClose

	case isPunctuation(str):
		return lexPunctuation

	case strings.ContainsAny(str, "([{"):
		return lexOpenParens

	case strings.ContainsAny(str, "}])"):
		return lexCloseParens

	case strings.ContainsAny(str, "\"'"):
		return lexString

	case isNumeric(str):
		return lexNumber

	case isName(str):
		return lexName

	case isSpace(str):
		return lexSpace

	default:
		return l.errorf("Unknown expression")
	}
}

// tryLexOperator attempts to match the next sequence of characters to a list of operators.
// This is implemented this way because Twig supports many alphabetical operators like "in",
// which require more than just a check of the next character.
func (l *lexer) tryLexOperator() bool {
	op := operatorMatcher.FindString(l.input[l.pos:])
	if op == "" {
		return false
	} else if op == "%" {
		// Ensure this is not a tag close token "%}".
		// Go's regexp engine does not support negative lookahead.
		if l.input[l.pos+1:l.pos+2] == "}" {
			return false
		}
	} else if op == "in" {
		// TODO: Workaround to avoid matching "include" tags.
		if l.input[l.pos+2:l.pos+3] != " " {
			return false
		}
	}
	l.pos += len(op)
	l.emit(tokenOperator)

	return true
}

func lexSpace(l *lexer) stateFn {
	for {
		str := l.next()
		if !isSpace(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenWhitespace)

	return lexExpression
}

func lexNumber(l *lexer) stateFn {
	for {
		str := l.next()
		if !isNumeric(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenNumber)

	return lexExpression
}

func lexPunctuation(l *lexer) stateFn {
	for {
		str := l.next()
		if !isPunctuation(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenPunctuation)

	return lexExpression
}

func lexString(l *lexer) stateFn {
	open := l.next()
	l.emit(tokenStringOpen)
	closePos := strings.Index(l.input[l.pos:], open)
	if closePos < 0 {
		return l.errorf("unclosed string")
	}

	if open == `"` && strings.Contains(l.input[l.pos:l.pos+closePos], delimOpenInterpolate) {
		input := l.input
		l.input = input[0 : l.pos+closePos]
		for {
			p := strings.Index(l.input[l.pos:], delimOpenInterpolate)
			if p < 0 {
				break
			}
			l.pos += p
			l.emit(tokenText)
			l.pos += len(delimOpenInterpolate)
			l.emit(tokenInterpolateOpen)
			l.mode = modeInterpolate
			for ins := lexExpression; ins != nil; {
				ins = ins(l)
			}
			if l.mode == modeClosed {
				return nil
			}
			l.mode = modeNormal
			l.emit(tokenInterpolateClose)
		}
		if l.pos < len(l.input) {
			l.pos = len(l.input)
			l.emit(tokenText)
		}
		l.input = input
	} else {
		l.pos += closePos
		l.emit(tokenText)
	}

	l.next()
	l.emit(tokenStringClose)

	return lexExpression
}

func lexOpenParens(l *lexer) stateFn {
	switch str := l.next(); {
	case str == "(":
		l.emit(tokenParensOpen)

	case str == "[":
		l.emit(tokenArrayOpen)

	case str == "{":
		l.emit(tokenHashOpen)

	default:
		return l.errorf("Unknown parenthesis")
	}
	l.parens++
	return lexExpression
}

func lexCloseParens(l *lexer) stateFn {
	switch str := l.next(); {
	case str == ")":
		l.emit(tokenParensClose)

	case str == "]":
		l.emit(tokenArrayClose)

	case str == "}":
		if l.parens == 0 && l.mode == modeInterpolate {
			return nil
		}
		l.emit(tokenHashClose)

	default:
		return l.errorf("Unknown parenthesis")
	}
	l.parens--
	return lexExpression
}

func lexName(l *lexer) stateFn {
	for {
		str := l.next()
		if str == delimEOF {
			break
		}
		if !isName(str) {
			l.backup()
			break
		}
	}

	l.emit(tokenName)

	return lexExpression
}

func lexCommentOpen(l *lexer) stateFn {
	l.pos += len(delimOpenComment)
	l.start = l.pos
	til := strings.Index(l.input[l.pos:], delimCloseComment)
	if til < 0 {
		til = len(l.input[l.start:])
	}
	l.pos += til
	l.emit(tokenComment)
	if !strings.HasPrefix(l.input[l.pos:], delimCloseComment) {
		return l.errorf("Unclosed comment")
	}
	l.pos += len(delimCloseComment)
	l.start = l.pos

	return lexData
}

func lexTagOpen(l *lexer) stateFn {
	l.pos += len(delimOpenTag)
	l.emit(tokenTagOpen)

	return lexExpression
}

func lexTagClose(l *lexer) stateFn {
	l.pos += len(delimCloseTag)
	l.emit(tokenTagClose)

	return lexData
}

func lexPrintOpen(l *lexer) stateFn {
	l.pos += len(delimOpenPrint)
	l.emit(tokenPrintOpen)

	return lexExpression
}

func lexPrintClose(l *lexer) stateFn {
	l.pos += len(delimClosePrint)
	l.emit(tokenPrintClose)

	return lexData
}

func isSpace(str string) bool {
	return str == " " || str == "\t" || str == "\n"
}

func isName(str string) bool {
	for _, s := range str {
		if string(s) != "_" && !unicode.IsLetter(s) && !unicode.IsDigit(s) {
			return false
		}
	}

	return true
}

func isNumeric(str string) bool {
	for _, s := range str {
		if !unicode.IsDigit(s) {
			return false
		}
	}

	return true
}

func isPunctuation(str string) bool {
	for _, s := range str {
		if !strings.ContainsAny(string(s), ",|?:.") {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"

	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/cache/metercacher"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	dependentsCacheSize = 1024
	jobsCacheSize       = 2048
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	cachingEnabled bool
	jobsCache      cache.Cacher
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*state, error) {
	jobsCacheMetricsNamespace := fmt.Sprintf("%s_jobs_cache", metricsNamespace)
	jobsCache, err := metercacher.New(jobsCacheMetricsNamespace, metricsRegisterer, &cache.LRU{Size: jobsCacheSize})
	if err != nil {
		return nil, fmt.Errorf("couldn't create metered cache: %s", err)
	}
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		cachingEnabled:  true,
		jobsCache:       jobsCache,
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: dependentsCacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}, nil
}

// AddRunnableJob adds [jobID] to the runnable queue
func (s *state) AddRunnableJob(jobID ids.ID) error {
	return s.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (s *state) HasRunnableJob() (bool, error) {
	isEmpty, err := s.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (s *state) RemoveRunnableJob() (Job, error) {
	jobIDBytes, err := s.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := s.runnableJobIDs.Delete(jobIDBytes); err != nil {
		return nil, err
	}

	jobID, err := ids.ToID(jobIDBytes)
	if err != nil {
		return nil, fmt.Errorf("couldn't convert job ID bytes to job ID: %s", err)
	}
	job, err := s.GetJob(jobID)
	if err != nil {
		return nil, err
	}
	return job, s.jobs.Delete((jobIDBytes))
}

// PutJob adds the job to the queue
func (s *state) PutJob(job Job) error {
	id := job.ID()
	if s.cachingEnabled {
		s.jobsCache.Put(id, job)
	}
	return s.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (s *state) HasJob(id ids.ID) (bool, error) {
	if s.cachingEnabled {
		if _, exists := s.jobsCache.Get(id); exists {
			return true, nil
		}
	}
	return s.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (s *state) GetJob(id ids.ID) (Job, error) {
	if s.cachingEnabled {
		if job, exists := s.jobsCache.Get(id); exists {
			return job.(Job), nil
		}
	}
	jobBytes, err := s.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	job, err := s.parser.Parse(jobBytes)
	if err == nil && s.cachingEnabled {
		s.jobsCache.Put(id, job)
	}
	return job, err
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (s *state) AddDependency(dependency, dependent ids.ID) error {
	dependentsDB := s.getDependentsDB(dependency)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (s *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependentsDB := s.getDependentsDB(dependency)
	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (s *state) DisableCaching() {
	s.dependentsCache.Flush()
	s.jobsCache.Flush()
	s.cachingEnabled = false
}

func (s *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := s.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (s *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := s.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (s *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := s.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}

func (s *state) getDependentsDB(dependency ids.ID) linkeddb.LinkedDB {
	if s.cachingEnabled {
		if dependentsDBIntf, ok := s.dependentsCache.Get(dependency); ok {
			return dependentsDBIntf.(linkeddb.LinkedDB)
		}
	}
	dependencyDB := prefixdb.New(dependency[:], s.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	if s.cachingEnabled {
		s.dependentsCache.Put(dependency, dependentsDB)
	}
	return dependentsDB
}
<commit_msg>Remove double parantheses<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"

	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/cache/metercacher"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	dependentsCacheSize = 1024
	jobsCacheSize       = 2048
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	cachingEnabled bool
	jobsCache      cache.Cacher
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*state, error) {
	jobsCacheMetricsNamespace := fmt.Sprintf("%s_jobs_cache", metricsNamespace)
	jobsCache, err := metercacher.New(jobsCacheMetricsNamespace, metricsRegisterer, &cache.LRU{Size: jobsCacheSize})
	if err != nil {
		return nil, fmt.Errorf("couldn't create metered cache: %s", err)
	}
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		cachingEnabled:  true,
		jobsCache:       jobsCache,
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: dependentsCacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}, nil
}

// AddRunnableJob adds [jobID] to the runnable queue
func (s *state) AddRunnableJob(jobID ids.ID) error {
	return s.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (s *state) HasRunnableJob() (bool, error) {
	isEmpty, err := s.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (s *state) RemoveRunnableJob() (Job, error) {
	jobIDBytes, err := s.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := s.runnableJobIDs.Delete(jobIDBytes); err != nil {
		return nil, err
	}

	jobID, err := ids.ToID(jobIDBytes)
	if err != nil {
		return nil, fmt.Errorf("couldn't convert job ID bytes to job ID: %s", err)
	}
	job, err := s.GetJob(jobID)
	if err != nil {
		return nil, err
	}
	return job, s.jobs.Delete(jobIDBytes)
}

// PutJob adds the job to the queue
func (s *state) PutJob(job Job) error {
	id := job.ID()
	if s.cachingEnabled {
		s.jobsCache.Put(id, job)
	}
	return s.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (s *state) HasJob(id ids.ID) (bool, error) {
	if s.cachingEnabled {
		if _, exists := s.jobsCache.Get(id); exists {
			return true, nil
		}
	}
	return s.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (s *state) GetJob(id ids.ID) (Job, error) {
	if s.cachingEnabled {
		if job, exists := s.jobsCache.Get(id); exists {
			return job.(Job), nil
		}
	}
	jobBytes, err := s.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	job, err := s.parser.Parse(jobBytes)
	if err == nil && s.cachingEnabled {
		s.jobsCache.Put(id, job)
	}
	return job, err
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (s *state) AddDependency(dependency, dependent ids.ID) error {
	dependentsDB := s.getDependentsDB(dependency)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (s *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependentsDB := s.getDependentsDB(dependency)
	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (s *state) DisableCaching() {
	s.dependentsCache.Flush()
	s.jobsCache.Flush()
	s.cachingEnabled = false
}

func (s *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := s.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (s *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := s.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (s *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := s.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}

func (s *state) getDependentsDB(dependency ids.ID) linkeddb.LinkedDB {
	if s.cachingEnabled {
		if dependentsDBIntf, ok := s.dependentsCache.Get(dependency); ok {
			return dependentsDBIntf.(linkeddb.LinkedDB)
		}
	}
	dependencyDB := prefixdb.New(dependency[:], s.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	if s.cachingEnabled {
		s.dependentsCache.Put(dependency, dependentsDB)
	}
	return dependentsDB
}
<|endoftext|>