<commit_before>package integration

import (
	"fmt"
	"os"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
)

var _ = Describe("Upgrade", func() {
	Describe("Upgrading a cluster using offline mode", func() {
		Describe("From KET version v1.2.2", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.2.2")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using Ubuntu 16.04", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using RedHat 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(RedHat7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
			})

			// This spec will be used for testing non-destructive kismatic features on
			// an upgraded cluster.
			// This spec is open to modification when new assertions have to be made.
			Context("Using a skunkworks cluster", func() {
				ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
					WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 2, Storage: 2}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
						// reserve one of the workers for the add-worker test
						allWorkers := nodes.worker
						nodes.worker = allWorkers[0 : len(nodes.worker)-1]

						// Standup cluster with previous version
						opts := installOptions{allowPackageInstallation: true}
						err := installKismatic(nodes, opts, sshKey)
						FailIfError(err)

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Perform upgrade
						upgradeCluster()

						sub := SubDescribe("Using an upgraded cluster")
						defer sub.Check()

						sub.It("should allow adding a new storage volume", func() error {
							planFile, err := os.Open("kismatic-testing.yaml")
							if err != nil {
								return err
							}
							return createVolume(planFile, "test-vol", 1, 1, "")
						})

						sub.It("should allow adding a worker node", func() error {
							newWorker := allWorkers[len(allWorkers)-1]
							return addWorkerToCluster(newWorker)
						})

						sub.It("should have an accessible dashboard", func() error {
							return nil
						})

						sub.It("should be able to deploy a workload with ingress", func() error {
							return verifyIngressNodes(nodes.master[0], nodes.ingress, sshKey)
						})

						sub.It("should have an accessible dashboard", func() error {
							return canAccessDashboard()
						})

						sub.It("should not have kube-apiserver systemd service", func() error {
							return nil
						})

						// This test should always be last
						sub.It("should still be a highly available cluster after upgrade", func() error {
							By("Removing a Kubernetes master node")
							if err = aws.TerminateNode(nodes.master[0]); err != nil {
								return fmt.Errorf("could not remove node: %v", err)
							}
							By("Re-running Kuberang")
							if err = runViaSSH([]string{"sudo kuberang"}, []NodeDeets{nodes.master[1]}, sshKey, 5*time.Minute); err != nil {
								return err
							}
							return nil
						})
					})
				})
			})

			Context("Using a cluster that has no internet access [slow] [upgrade]", func() {
				ItOnAWS("should result in an upgraded cluster", func(aws infrastructureProvisioner) {
					distro := CentOS7
					WithInfrastructure(NodeCount{Etcd: 3, Master: 1, Worker: 1}, distro, aws, func(nodes provisionedNodes, sshKey string) {
						// Standup cluster with previous version
						// Need to allowPackageInstallation=true to install old versions of packages
						opts := installOptions{
							allowPackageInstallation:    true,
							disconnectedInstallation:    true,
							modifyHostsFiles:            true,
							autoConfigureDockerRegistry: true,
						}
						err := installKismatic(nodes, opts, sshKey)
						FailIfError(err)

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Remove old packages
						By("Removing old packages")
						RemoveKismaticPackages()

						// Cleanup old cluster file and create a new one
						By("Recreating kismatic-testing.yaml file")
						err = os.Remove("kismatic-testing.yaml")
						FailIfError(err)
						opts = installOptions{
							allowPackageInstallation:    false,
							disconnectedInstallation:    true,
							modifyHostsFiles:            true,
							autoConfigureDockerRegistry: true,
						}
						writePlanFile(buildPlan(nodes, opts, sshKey))

						// Manually install the new packages
						InstallKismaticPackages(nodes, distro, sshKey, true)

						// Lock down internet access
						err = disableInternetAccess(nodes.allNodes(), sshKey)
						FailIfError(err)

						// Confirm there is not internet
						if err := verifyNoInternetAccess(nodes.allNodes(), sshKey); err == nil {
							Fail("was able to ping google with outgoing connections blocked")
						}

						// Perform upgrade
						upgradeCluster()
					})
				})
			})
		})

		Describe("From KET version v1.1.1", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.1.1")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using CentOS 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using RedHat 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(RedHat7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using a larger cluster layout with Ubuntu 16.04", func() {
					ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 0, Storage: 0}, Ubuntu1604LTS, aws, func(nodes provisionedNodes, sshKey string) {
							installAndUpgrade(nodes, sshKey)
						})
					})
				})
			})
		})

		Describe("From KET version v1.0.3", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.0.3")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using CentOS 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using Ubuntu 16.04", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
				Context("Using a larger cluster layout with RedHat 7", func() {
					ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 0, Storage: 0}, RedHat7, aws, func(nodes provisionedNodes, sshKey string) {
							installAndUpgrade(nodes, sshKey)
						})
					})
				})
			})
		})
	})
})

func installAndUpgradeMinikube(node NodeDeets, sshKey string) {
	// Install previous version cluster
	err := installKismaticMini(node, sshKey)
	FailIfError(err)
	extractCurrentKismaticInstaller()
	upgradeCluster()
}

func installAndUpgrade(nodes provisionedNodes, sshKey string) {
	// Standup cluster with previous version
	opts := installOptions{allowPackageInstallation: true}
	err := installKismatic(nodes, opts, sshKey)
	FailIfError(err)

	// Extract current version of kismatic
	extractCurrentKismaticInstaller()

	// Perform upgrade
	upgradeCluster()
}

func extractCurrentKismaticInstaller() {
	// Extract current version of kismatic
	pwd, err := os.Getwd()
	FailIfError(err)
	err = extractCurrentKismatic(pwd)
	FailIfError(err)
}
func upgradeCluster() {
	// Perform upgrade
	cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	FailIfError(err)

	assertClusterVersionIsCurrent()
}
<commit_msg>Remove unnecessary stub tests<commit_after>package integration

import (
	"fmt"
	"os"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
)

var _ = Describe("Upgrade", func() {
	Describe("Upgrading a cluster using offline mode", func() {
		Describe("From KET version v1.2.2", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.2.2")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using Ubuntu 16.04", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using RedHat 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(RedHat7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
			})

			// This spec will be used for testing non-destructive kismatic features on
			// an upgraded cluster.
			// This spec is open to modification when new assertions have to be made.
			Context("Using a skunkworks cluster", func() {
				ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
					WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 2, Storage: 2}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) {
						// reserve one of the workers for the add-worker test
						allWorkers := nodes.worker
						nodes.worker = allWorkers[0 : len(nodes.worker)-1]

						// Standup cluster with previous version
						opts := installOptions{allowPackageInstallation: true}
						err := installKismatic(nodes, opts, sshKey)
						FailIfError(err)

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Perform upgrade
						upgradeCluster()

						sub := SubDescribe("Using an upgraded cluster")
						defer sub.Check()

						sub.It("should allow adding a new storage volume", func() error {
							planFile, err := os.Open("kismatic-testing.yaml")
							if err != nil {
								return err
							}
							return createVolume(planFile, "test-vol", 1, 1, "")
						})

						sub.It("should allow adding a worker node", func() error {
							newWorker := allWorkers[len(allWorkers)-1]
							return addWorkerToCluster(newWorker)
						})

						sub.It("should be able to deploy a workload with ingress", func() error {
							return verifyIngressNodes(nodes.master[0], nodes.ingress, sshKey)
						})

						sub.It("should have an accessible dashboard", func() error {
							return canAccessDashboard()
						})

						// This test should always be last
						sub.It("should still be a highly available cluster after upgrade", func() error {
							By("Removing a Kubernetes master node")
							if err = aws.TerminateNode(nodes.master[0]); err != nil {
								return fmt.Errorf("could not remove node: %v", err)
							}
							By("Re-running Kuberang")
							if err = runViaSSH([]string{"sudo kuberang"}, []NodeDeets{nodes.master[1]}, sshKey, 5*time.Minute); err != nil {
								return err
							}
							return nil
						})
					})
				})
			})

			Context("Using a cluster that has no internet access [slow] [upgrade]", func() {
				ItOnAWS("should result in an upgraded cluster", func(aws infrastructureProvisioner) {
					distro := CentOS7
					WithInfrastructure(NodeCount{Etcd: 3, Master: 1, Worker: 1}, distro, aws, func(nodes provisionedNodes, sshKey string) {
						// Standup cluster with previous version
						// Need to allowPackageInstallation=true to install old versions of packages
						opts := installOptions{
							allowPackageInstallation:    true,
							disconnectedInstallation:    true,
							modifyHostsFiles:            true,
							autoConfigureDockerRegistry: true,
						}
						err := installKismatic(nodes, opts, sshKey)
						FailIfError(err)

						// Extract current version of kismatic
						extractCurrentKismaticInstaller()

						// Remove old packages
						By("Removing old packages")
						RemoveKismaticPackages()

						// Cleanup old cluster file and create a new one
						By("Recreating kismatic-testing.yaml file")
						err = os.Remove("kismatic-testing.yaml")
						FailIfError(err)
						opts = installOptions{
							allowPackageInstallation:    false,
							disconnectedInstallation:    true,
							modifyHostsFiles:            true,
							autoConfigureDockerRegistry: true,
						}
						writePlanFile(buildPlan(nodes, opts, sshKey))

						// Manually install the new packages
						InstallKismaticPackages(nodes, distro, sshKey, true)

						// Lock down internet access
						err = disableInternetAccess(nodes.allNodes(), sshKey)
						FailIfError(err)

						// Confirm there is not internet
						if err := verifyNoInternetAccess(nodes.allNodes(), sshKey); err == nil {
							Fail("was able to ping google with outgoing connections blocked")
						}

						// Perform upgrade
						upgradeCluster()
					})
				})
			})
		})

		Describe("From KET version v1.1.1", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.1.1")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using CentOS 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using RedHat 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(RedHat7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using a larger cluster layout with Ubuntu 16.04", func() {
					ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 0, Storage: 0}, Ubuntu1604LTS, aws, func(nodes provisionedNodes, sshKey string) {
							installAndUpgrade(nodes, sshKey)
						})
					})
				})
			})
		})

		Describe("From KET version v1.0.3", func() {
			BeforeEach(func() {
				dir := setupTestWorkingDirWithVersion("v1.0.3")
				os.Chdir(dir)
			})
			Context("Using a minikube layout", func() {
				Context("Using CentOS 7", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(CentOS7, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})

				Context("Using Ubuntu 16.04", func() {
					ItOnAWS("should be upgraded [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithMiniInfrastructure(Ubuntu1604LTS, aws, func(node NodeDeets, sshKey string) {
							installAndUpgradeMinikube(node, sshKey)
						})
					})
				})
				Context("Using a larger cluster layout with RedHat 7", func() {
					ItOnAWS("should result in an upgraded cluster [slow] [upgrade]", func(aws infrastructureProvisioner) {
						WithInfrastructureAndDNS(NodeCount{Etcd: 3, Master: 2, Worker: 3, Ingress: 0, Storage: 0}, RedHat7, aws, func(nodes provisionedNodes, sshKey string) {
							installAndUpgrade(nodes, sshKey)
						})
					})
				})
			})
		})
	})
})

func installAndUpgradeMinikube(node NodeDeets, sshKey string) {
	// Install previous version cluster
	err := installKismaticMini(node, sshKey)
	FailIfError(err)
	extractCurrentKismaticInstaller()
	upgradeCluster()
}

func installAndUpgrade(nodes provisionedNodes, sshKey string) {
	// Standup cluster with previous version
	opts := installOptions{allowPackageInstallation: true}
	err := installKismatic(nodes, opts, sshKey)
	FailIfError(err)

	// Extract current version of kismatic
	extractCurrentKismaticInstaller()

	// Perform upgrade
	upgradeCluster()
}

func extractCurrentKismaticInstaller() {
	// Extract current version of kismatic
	pwd, err := os.Getwd()
	FailIfError(err)
	err = extractCurrentKismatic(pwd)
	FailIfError(err)
}
func upgradeCluster() {
	// Perform upgrade
	cmd := exec.Command("./kismatic", "upgrade", "offline", "-f", "kismatic-testing.yaml")
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	err := cmd.Run()
	FailIfError(err)

	assertClusterVersionIsCurrent()
}
<|endoftext|>


<commit_before>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package db

import (
	"fmt"

	"github.com/lxc/lxd/shared/api"
)

// Code generation directives.
//
//go:generate -command mapper lxd-generate db mapper -t certificates.mapper.go
//go:generate mapper reset
//
//go:generate mapper stmt -p db -e certificate objects
//go:generate mapper stmt -p db -e certificate objects-by-Fingerprint
//go:generate mapper stmt -p db -e certificate projects-ref
//go:generate mapper stmt -p db -e certificate projects-ref-by-Fingerprint
//go:generate mapper stmt -p db -e certificate id
//go:generate mapper stmt -p db -e certificate create struct=Certificate
//go:generate mapper stmt -p db -e certificate create-projects-ref
//go:generate mapper stmt -p db -e certificate delete
//go:generate mapper stmt -p db -e certificate update struct=Certificate
//
//go:generate mapper method -p db -e certificate List
//go:generate mapper method -p db -e certificate Get
//go:generate mapper method -p db -e certificate ID struct=Certificate
//go:generate mapper method -p db -e certificate Exists struct=Certificate
//go:generate mapper method -p db -e certificate Create struct=Certificate
//go:generate mapper method -p db -e certificate ProjectsRef
//go:generate mapper method -p db -e certificate Delete
//go:generate mapper method -p db -e certificate Update struct=Certificate

// CertificateTypeClient indicates a client certificate type.
const CertificateTypeClient = 1

// CertificateTypeServer indicates a server certificate type.
const CertificateTypeServer = 2

// CertificateAPITypeToDBType converts an API type to the equivalent DB type.
func CertificateAPITypeToDBType(apiType string) (int, error) {
	switch apiType {
	case api.CertificateTypeClient:
		return CertificateTypeClient, nil
	case api.CertificateTypeServer:
		return CertificateTypeServer, nil
	}

	return -1, fmt.Errorf("Invalid certificate type")
}

// Certificate is here to pass the certificates content from the database around.
type Certificate struct {
	ID          int
	Fingerprint string `db:"primary=yes&comparison=like"`
	Type        int
	Name        string
	Certificate string
	Restricted  bool
	Projects    []string
}

// ToAPIType returns the API equivalent type.
func (cert *Certificate) ToAPIType() string {
	switch cert.Type {
	case CertificateTypeClient:
		return api.CertificateTypeClient
	case CertificateTypeServer:
		return api.CertificateTypeServer
	}

	return api.CertificateTypeUnknown
}

// ToAPI converts the database Certificate struct to an api.Certificate entry.
func (cert *Certificate) ToAPI() api.Certificate {
	resp := api.Certificate{}
	resp.Fingerprint = cert.Fingerprint
	resp.Certificate = cert.Certificate
	resp.Name = cert.Name
	resp.Restricted = cert.Restricted
	resp.Projects = cert.Projects
	resp.Type = cert.ToAPIType()

	return resp
}

// UpdateCertificateProjects updates the list of projects on a certificate.
func (c *ClusterTx) UpdateCertificateProjects(id int, projects []string) error {
	// Clear all projects from the restrictions.
	q := "DELETE FROM certificates_projects WHERE certificate_id=?"
	_, err := c.tx.Exec(q, id)
	if err != nil {
		return err
	}

	// Add the new restrictions.
	for _, name := range projects {
		projID, err := c.GetProjectID(name)
		if err != nil {
			return err
		}

		q := "INSERT INTO certificates_projects (certificate_id, project_id) VALUES (?, ?)"
		_, err = c.tx.Exec(q, id, projID)
		if err != nil {
			return err
		}
	}

	return nil
}

// CertificateFilter can be used to filter results yielded by GetCertInfos
type CertificateFilter struct {
	Fingerprint string // Matched with LIKE
}

// GetCertificate gets an CertBaseInfo object from the database.
// The argument fingerprint will be queried with a LIKE query, means you can
// pass a shortform and will get the full fingerprint.
// There can never be more than one certificate with a given fingerprint, as it is
// enforced by a UNIQUE constraint in the schema.
func (c *Cluster) GetCertificate(fingerprint string) (*Certificate, error) {
	var err error
	var cert *Certificate
	err = c.Transaction(func(tx *ClusterTx) error {
		cert, err = tx.GetCertificate(fingerprint + "%")
		if err != nil {
			return err
		}

		cert, err = tx.GetCertificate(cert.Fingerprint)
		return err
	})
	if err != nil {
		return nil, err
	}

	return cert, nil
}

// CreateCertificate stores a CertInfo object in the db, it will ignore the ID
// field from the CertInfo.
func (c *Cluster) CreateCertificate(cert Certificate) (int64, error) {
	var id int64
	var err error
	err = c.Transaction(func(tx *ClusterTx) error {
		id, err = tx.CreateCertificate(cert)
		return err
	})
	return id, err
}

// DeleteCertificate deletes a certificate from the db.
func (c *Cluster) DeleteCertificate(fingerprint string) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.DeleteCertificate(fingerprint)
	})
	return err
}

// UpdateCertificate updates a certificate in the db.
func (c *Cluster) UpdateCertificate(fingerprint string, cert Certificate) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.UpdateCertificate(fingerprint, cert)
	})
	return err
}

// UpdateCertificateProjects updates the list of projects on a certificate.
func (c *Cluster) UpdateCertificateProjects(id int, projects []string) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.UpdateCertificateProjects(id, projects)
	})
	return err
}
<commit_msg>lxd/db/certificates: Adds GetCertificates function<commit_after>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package db

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/shared/api"
)

// Code generation directives.
//
//go:generate -command mapper lxd-generate db mapper -t certificates.mapper.go
//go:generate mapper reset
//
//go:generate mapper stmt -p db -e certificate objects
//go:generate mapper stmt -p db -e certificate objects-by-Fingerprint
//go:generate mapper stmt -p db -e certificate projects-ref
//go:generate mapper stmt -p db -e certificate projects-ref-by-Fingerprint
//go:generate mapper stmt -p db -e certificate id
//go:generate mapper stmt -p db -e certificate create struct=Certificate
//go:generate mapper stmt -p db -e certificate create-projects-ref
//go:generate mapper stmt -p db -e certificate delete
//go:generate mapper stmt -p db -e certificate update struct=Certificate
//
//go:generate mapper method -p db -e certificate List
//go:generate mapper method -p db -e certificate Get
//go:generate mapper method -p db -e certificate ID struct=Certificate
//go:generate mapper method -p db -e certificate Exists struct=Certificate
//go:generate mapper method -p db -e certificate Create struct=Certificate
//go:generate mapper method -p db -e certificate ProjectsRef
//go:generate mapper method -p db -e certificate Delete
//go:generate mapper method -p db -e certificate Update struct=Certificate

// CertificateTypeClient indicates a client certificate type.
const CertificateTypeClient = 1

// CertificateTypeServer indicates a server certificate type.
const CertificateTypeServer = 2

// CertificateAPITypeToDBType converts an API type to the equivalent DB type.
func CertificateAPITypeToDBType(apiType string) (int, error) {
	switch apiType {
	case api.CertificateTypeClient:
		return CertificateTypeClient, nil
	case api.CertificateTypeServer:
		return CertificateTypeServer, nil
	}

	return -1, fmt.Errorf("Invalid certificate type")
}

// Certificate is here to pass the certificates content from the database around.
type Certificate struct {
	ID          int
	Fingerprint string `db:"primary=yes&comparison=like"`
	Type        int
	Name        string
	Certificate string
	Restricted  bool
	Projects    []string
}

// ToAPIType returns the API equivalent type.
func (cert *Certificate) ToAPIType() string {
	switch cert.Type {
	case CertificateTypeClient:
		return api.CertificateTypeClient
	case CertificateTypeServer:
		return api.CertificateTypeServer
	}

	return api.CertificateTypeUnknown
}

// ToAPI converts the database Certificate struct to an api.Certificate entry.
func (cert *Certificate) ToAPI() api.Certificate {
	resp := api.Certificate{}
	resp.Fingerprint = cert.Fingerprint
	resp.Certificate = cert.Certificate
	resp.Name = cert.Name
	resp.Restricted = cert.Restricted
	resp.Projects = cert.Projects
	resp.Type = cert.ToAPIType()

	return resp
}

// UpdateCertificateProjects updates the list of projects on a certificate.
func (c *ClusterTx) UpdateCertificateProjects(id int, projects []string) error {
	// Clear all projects from the restrictions.
	q := "DELETE FROM certificates_projects WHERE certificate_id=?"
	_, err := c.tx.Exec(q, id)
	if err != nil {
		return err
	}

	// Add the new restrictions.
	for _, name := range projects {
		projID, err := c.GetProjectID(name)
		if err != nil {
			return err
		}

		q := "INSERT INTO certificates_projects (certificate_id, project_id) VALUES (?, ?)"
		_, err = c.tx.Exec(q, id, projID)
		if err != nil {
			return err
		}
	}

	return nil
}

// CertificateFilter can be used to filter results yielded by GetCertInfos
type CertificateFilter struct {
	Fingerprint string // Matched with LIKE
}

// GetCertificate gets an CertBaseInfo object from the database.
// The argument fingerprint will be queried with a LIKE query, means you can
// pass a shortform and will get the full fingerprint.
// There can never be more than one certificate with a given fingerprint, as it is
// enforced by a UNIQUE constraint in the schema.
func (c *Cluster) GetCertificate(fingerprint string) (*Certificate, error) {
	var err error
	var cert *Certificate
	err = c.Transaction(func(tx *ClusterTx) error {
		cert, err = tx.GetCertificate(fingerprint + "%")
		if err != nil {
			return err
		}

		cert, err = tx.GetCertificate(cert.Fingerprint)
		return err
	})
	if err != nil {
		return nil, err
	}

	return cert, nil
}

// CreateCertificate stores a CertInfo object in the db, it will ignore the ID
// field from the CertInfo.
func (c *Cluster) CreateCertificate(cert Certificate) (int64, error) {
	var id int64
	var err error
	err = c.Transaction(func(tx *ClusterTx) error {
		id, err = tx.CreateCertificate(cert)
		return err
	})
	return id, err
}

// DeleteCertificate deletes a certificate from the db.
func (c *Cluster) DeleteCertificate(fingerprint string) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.DeleteCertificate(fingerprint)
	})
	return err
}

// UpdateCertificate updates a certificate in the db.
func (c *Cluster) UpdateCertificate(fingerprint string, cert Certificate) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.UpdateCertificate(fingerprint, cert)
	})
	return err
}

// UpdateCertificateProjects updates the list of projects on a certificate.
func (c *Cluster) UpdateCertificateProjects(id int, projects []string) error {
	err := c.Transaction(func(tx *ClusterTx) error {
		return tx.UpdateCertificateProjects(id, projects)
	})
	return err
}

// GetCertificates returns all available local certificates.
func (n *NodeTx) GetCertificates() ([]Certificate, error) {
	dbCerts := []struct {
		fingerprint string
		certType    int
		name        string
		certificate string
	}{}
	dest := func(i int) []interface{} {
		dbCerts = append(dbCerts, struct {
			fingerprint string
			certType    int
			name        string
			certificate string
		}{})
		return []interface{}{&dbCerts[i].fingerprint, &dbCerts[i].certType, &dbCerts[i].name, &dbCerts[i].certificate}
	}

	stmt, err := n.tx.Prepare("SELECT fingerprint, type, name, certificate FROM certificates")
	if err != nil {
		return nil, err
	}
	defer stmt.Close()

	err = query.SelectObjects(stmt, dest)
	if err != nil {
		return nil, err
	}

	certs := make([]Certificate, 0, len(dbCerts))
	for _, dbCert := range dbCerts {
		certs = append(certs, Certificate{
			Fingerprint: dbCert.fingerprint,
			Type:        dbCert.certType,
			Name:        dbCert.name,
			Certificate: dbCert.certificate,
		})
	}

	return certs, nil
}
<|endoftext|>


<commit_before>package dnsmasq

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/storage/filesystem"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/subprocess"
	"github.com/lxc/lxd/shared/version"
)

const staticAllocationDeviceSeparator = "."

// DHCPAllocation represents an IP allocation from dnsmasq.
type DHCPAllocation struct {
	IP     net.IP
	Name   string
	MAC    net.HardwareAddr
	Static bool
}

// ConfigMutex used to coordinate access to the dnsmasq config files.
var ConfigMutex sync.Mutex

// UpdateStaticEntry writes a single dhcp-host line for a network/instance combination.
func UpdateStaticEntry(network string, projectName string, instanceName string, deviceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error {
	hwaddr = strings.ToLower(hwaddr)
	line := hwaddr

	// Generate the dhcp-host line
	if ipv4Address != "" {
		line += fmt.Sprintf(",%s", ipv4Address)
	}

	if ipv6Address != "" {
		line += fmt.Sprintf(",[%s]", ipv6Address)
	}

	if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" {
		line += fmt.Sprintf(",%s", project.DNS(projectName, instanceName))
	}

	if line == hwaddr {
		return nil
	}

	entryFile := dnsMasqEntryFileName(projectName, instanceName, deviceName)
	err := ioutil.WriteFile(shared.VarPath("networks", network, "dnsmasq.hosts", entryFile), []byte(line+"\n"), 0644)
	if err != nil {
		return err
	}

	return nil
}

// RemoveStaticEntry removes a single dhcp-host line for a network/instance combination.
func RemoveStaticEntry(network string, projectName string, instanceName string, deviceName string) error {
	entryFile := dnsMasqEntryFileName(projectName, instanceName, deviceName)
	err := os.Remove(shared.VarPath("networks", network, "dnsmasq.hosts", entryFile))
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Kill kills dnsmasq for a particular network (or optionally reloads it).
func Kill(name string, reload bool) error {
	pidPath := shared.VarPath("networks", name, "dnsmasq.pid")

	// If the pid file doesn't exist, there is no process to kill.
	if !shared.PathExists(pidPath) {
		return nil
	}

	// Import saved subprocess details
	p, err := subprocess.ImportProcess(pidPath)
	if err != nil {
		return fmt.Errorf("Could not read pid file: %s", err)
	}

	if reload {
		err = p.Reload()
		if err != nil && err != subprocess.ErrNotRunning {
			return fmt.Errorf("Could not reload dnsmasq: %s", err)
		}

		return nil
	}

	err = p.Stop()
	if err != nil && err != subprocess.ErrNotRunning {
		return fmt.Errorf("Unable to kill dnsmasq: %s", err)
	}

	time.Sleep(100 * time.Millisecond) // Give OS time to release sockets.

	return nil
}

// GetVersion returns the version of dnsmasq.
func GetVersion() (*version.DottedVersion, error) {
	output, err := shared.RunCommandCLocale("dnsmasq", "--version")
	if err != nil {
		return nil, fmt.Errorf("Failed to check dnsmasq version: %v", err)
	}

	lines := strings.Split(string(output), " ")
	return version.Parse(lines[2])
}

// DHCPStaticAllocation retrieves the dnsmasq statically allocated MAC and IPs for an instance.
// Returns MAC, IPv4 and IPv6 DHCPAllocation structs respectively.
// The projectName, instanceName, and deviceName are used to generate the entryFileName. If the entryFileName
// argument is provided, then projectName, instanceName, and deviceName are ignored.
func DHCPStaticAllocation(network, projectName, instanceName, deviceName, entryFileName string) (net.HardwareAddr, DHCPAllocation, DHCPAllocation, error) {
	var IPv4, IPv6 DHCPAllocation
	var mac net.HardwareAddr

	if entryFileName == "" {
		entryFileName = dnsMasqEntryFileName(projectName, instanceName, deviceName)
	}

	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.hosts", entryFileName))
	if err != nil {
		return nil, IPv4, IPv6, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.SplitN(scanner.Text(), ",", -1)
		for _, field := range fields {
			// Check if field is IPv4 or IPv6 address.
			if strings.Count(field, ".") == 3 {
				IP := net.ParseIP(field)
				if IP.To4() == nil {
					return nil, IPv4, IPv6, fmt.Errorf("Error parsing IP address %q", field)
				}
				IPv4 = DHCPAllocation{Name: instanceName, Static: true, IP: IP.To4(), MAC: mac}

			} else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") {
				IP := net.ParseIP(field[1 : len(field)-1])
				if IP == nil {
					return nil, IPv4, IPv6, fmt.Errorf("Error parsing IP address %q", field)
				}
				IPv6 = DHCPAllocation{Name: instanceName, Static: true, IP: IP, MAC: mac}
			} else if strings.Count(field, ":") == 5 {
				// This field is expected to come first, so that mac variable can be used with
				// populating the DHCPAllocation structs too.
				mac, err = net.ParseMAC(field)
				if err != nil {
					return nil, IPv4, IPv6, fmt.Errorf("Error parsing MAC address %q", field)
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, IPv4, IPv6, err
	}

	return mac, IPv4, IPv6, nil
}

// DHCPAllAllocations returns a map of IPs currently allocated (statically and dynamically)
// in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing
// the net.IP format. The value of each map item is a DHCPAllocation struct containing at least
// whether the allocation was static or dynamic and optionally instance name or MAC address.
// MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable).
// Static allocations are not overridden by dynamic allocations, allowing for instance name to be
// included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be
// reliably linked to instances using either name or MAC because dnsmasq does not record the MAC
// address for these records, and the recorded host name can be set by the instance if the dns.mode
// for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return
// any identifying info.
func DHCPAllAllocations(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) {
	IPv4s := make(map[[4]byte]DHCPAllocation)
	IPv6s := make(map[[16]byte]DHCPAllocation)

	// First read all statically allocated IPs.
	files, err := ioutil.ReadDir(shared.VarPath("networks", network, "dnsmasq.hosts"))
	if err != nil && os.IsNotExist(err) {
		return nil, nil, err
	}

	for _, entry := range files {
		_, IPv4, IPv6, err := DHCPStaticAllocation(network, "", "", "", entry.Name())
		if err != nil {
			return nil, nil, err
		}

		if IPv4.IP != nil {
			var IPKey [4]byte
			copy(IPKey[:], IPv4.IP.To4())
			IPv4s[IPKey] = IPv4
		}

		if IPv6.IP != nil {
			var IPKey [16]byte
			copy(IPKey[:], IPv6.IP.To16())
			IPv6s[IPKey] = IPv6
		}
	}

	// Next read all dynamic allocated IPs.
	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.leases"))
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 5 {
			IP := net.ParseIP(fields[2])
			if IP == nil {
				return nil, nil, fmt.Errorf("Error parsing IP address: %v", fields[2])
			}

			// Handle IPv6 addresses.
			if IP.To4() == nil {
				var IPKey [16]byte
				copy(IPKey[:], IP.To16())

				// Don't replace IPs from static config as more reliable.
				if IPv6s[IPKey].Name != "" {
					continue
				}

				IPv6s[IPKey] = DHCPAllocation{
					Static: false,
					IP:     IP.To16(),
				}
			} else {
				// MAC only available in IPv4 leases.
				MAC, err := net.ParseMAC(fields[1])
				if err != nil {
					return nil, nil, err
				}

				var IPKey [4]byte
				copy(IPKey[:], IP.To4())

				// Don't replace IPs from static config as more reliable.
				if IPv4s[IPKey].Name != "" {
					continue
				}

				IPv4s[IPKey] = DHCPAllocation{
					MAC:    MAC,
					Static: false,
					IP:     IP.To4(),
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, nil, err
	}

	return IPv4s, IPv6s, nil
}

// StaticAllocationFileName returns the file name to use for a dnsmasq instance device static allocation.
func StaticAllocationFileName(projectName string, instanceName string, deviceName string) string {
	escapedDeviceName := filesystem.PathNameEncode(deviceName)

	return strings.Join([]string{project.Instance(projectName, instanceName), escapedDeviceName}, staticAllocationDeviceSeparator)
}
<commit_msg>lxd/dnsmasq: StaticAllocationFileName usage<commit_after>package dnsmasq

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/storage/filesystem"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/subprocess"
	"github.com/lxc/lxd/shared/version"
)

const staticAllocationDeviceSeparator = "."

// DHCPAllocation represents an IP allocation from dnsmasq.
type DHCPAllocation struct {
	IP     net.IP
	Name   string
	MAC    net.HardwareAddr
	Static bool
}

// ConfigMutex used to coordinate access to the dnsmasq config files.
var ConfigMutex sync.Mutex

// UpdateStaticEntry writes a single dhcp-host line for a network/instance combination.
func UpdateStaticEntry(network string, projectName string, instanceName string, deviceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error {
	hwaddr = strings.ToLower(hwaddr)
	line := hwaddr

	// Generate the dhcp-host line
	if ipv4Address != "" {
		line += fmt.Sprintf(",%s", ipv4Address)
	}

	if ipv6Address != "" {
		line += fmt.Sprintf(",[%s]", ipv6Address)
	}

	if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" {
		line += fmt.Sprintf(",%s", project.DNS(projectName, instanceName))
	}

	if line == hwaddr {
		return nil
	}

	deviceStaticFileName := StaticAllocationFileName(projectName, instanceName, deviceName)
	err := ioutil.WriteFile(shared.VarPath("networks", network, "dnsmasq.hosts", deviceStaticFileName), []byte(line+"\n"), 0644)
	if err != nil {
		return err
	}

	return nil
}

// RemoveStaticEntry removes a single dhcp-host line for a network/instance combination.
func RemoveStaticEntry(network string, projectName string, instanceName string, deviceName string) error {
	deviceStaticFileName := StaticAllocationFileName(projectName, instanceName, deviceName)
	err := os.Remove(shared.VarPath("networks", network, "dnsmasq.hosts", deviceStaticFileName))
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Kill kills dnsmasq for a particular network (or optionally reloads it).
func Kill(name string, reload bool) error {
	pidPath := shared.VarPath("networks", name, "dnsmasq.pid")

	// If the pid file doesn't exist, there is no process to kill.
	if !shared.PathExists(pidPath) {
		return nil
	}

	// Import saved subprocess details
	p, err := subprocess.ImportProcess(pidPath)
	if err != nil {
		return fmt.Errorf("Could not read pid file: %s", err)
	}

	if reload {
		err = p.Reload()
		if err != nil && err != subprocess.ErrNotRunning {
			return fmt.Errorf("Could not reload dnsmasq: %s", err)
		}

		return nil
	}

	err = p.Stop()
	if err != nil && err != subprocess.ErrNotRunning {
		return fmt.Errorf("Unable to kill dnsmasq: %s", err)
	}

	time.Sleep(100 * time.Millisecond) // Give OS time to release sockets.

	return nil
}

// GetVersion returns the version of dnsmasq.
func GetVersion() (*version.DottedVersion, error) {
	output, err := shared.RunCommandCLocale("dnsmasq", "--version")
	if err != nil {
		return nil, fmt.Errorf("Failed to check dnsmasq version: %v", err)
	}

	lines := strings.Split(string(output), " ")
	return version.Parse(lines[2])
}

// DHCPStaticAllocation retrieves the dnsmasq statically allocated MAC and IPs for an instance.
// Returns MAC, IPv4 and IPv6 DHCPAllocation structs respectively.
// The projectName, instanceName, and deviceName are used to generate the entryFileName. If the entryFileName
// argument is provided, then projectName, instanceName, and deviceName are ignored.
func DHCPStaticAllocation(network, projectName, instanceName, deviceName, entryFileName string) (net.HardwareAddr, DHCPAllocation, DHCPAllocation, error) {
	var IPv4, IPv6 DHCPAllocation
	var mac net.HardwareAddr

	if entryFileName == "" {
		entryFileName = StaticAllocationFileName(projectName, instanceName, deviceName)
	}

	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.hosts", entryFileName))
	if err != nil {
		return nil, IPv4, IPv6, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.SplitN(scanner.Text(), ",", -1)
		for _, field := range fields {
			// Check if field is IPv4 or IPv6 address.
			if strings.Count(field, ".") == 3 {
				IP := net.ParseIP(field)
				if IP.To4() == nil {
					return nil, IPv4, IPv6, fmt.Errorf("Error parsing IP address %q", field)
				}
				IPv4 = DHCPAllocation{Name: instanceName, Static: true, IP: IP.To4(), MAC: mac}

			} else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") {
				IP := net.ParseIP(field[1 : len(field)-1])
				if IP == nil {
					return nil, IPv4, IPv6, fmt.Errorf("Error parsing IP address %q", field)
				}
				IPv6 = DHCPAllocation{Name: instanceName, Static: true, IP: IP, MAC: mac}
			} else if strings.Count(field, ":") == 5 {
				// This field is expected to come first, so that mac variable can be used with
				// populating the DHCPAllocation structs too.
				mac, err = net.ParseMAC(field)
				if err != nil {
					return nil, IPv4, IPv6, fmt.Errorf("Error parsing MAC address %q", field)
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, IPv4, IPv6, err
	}

	return mac, IPv4, IPv6, nil
}

// DHCPAllAllocations returns a map of IPs currently allocated (statically and dynamically)
// in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing
// the net.IP format. The value of each map item is a DHCPAllocation struct containing at least
// whether the allocation was static or dynamic and optionally instance name or MAC address.
// MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable).
// Static allocations are not overridden by dynamic allocations, allowing for instance name to be
// included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be
// reliably linked to instances using either name or MAC because dnsmasq does not record the MAC
// address for these records, and the recorded host name can be set by the instance if the dns.mode
// for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return
// any identifying info.
func DHCPAllAllocations(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) {
	IPv4s := make(map[[4]byte]DHCPAllocation)
	IPv6s := make(map[[16]byte]DHCPAllocation)

	// First read all statically allocated IPs.
	files, err := ioutil.ReadDir(shared.VarPath("networks", network, "dnsmasq.hosts"))
	if err != nil && os.IsNotExist(err) {
		return nil, nil, err
	}

	for _, entry := range files {
		_, IPv4, IPv6, err := DHCPStaticAllocation(network, "", "", "", entry.Name())
		if err != nil {
			return nil, nil, err
		}

		if IPv4.IP != nil {
			var IPKey [4]byte
			copy(IPKey[:], IPv4.IP.To4())
			IPv4s[IPKey] = IPv4
		}

		if IPv6.IP != nil {
			var IPKey [16]byte
			copy(IPKey[:], IPv6.IP.To16())
			IPv6s[IPKey] = IPv6
		}
	}

	// Next read all dynamic allocated IPs.
	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.leases"))
	if err != nil {
		return nil, nil, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 5 {
			IP := net.ParseIP(fields[2])
			if IP == nil {
				return nil, nil, fmt.Errorf("Error parsing IP address: %v", fields[2])
			}

			// Handle IPv6 addresses.
			if IP.To4() == nil {
				var IPKey [16]byte
				copy(IPKey[:], IP.To16())

				// Don't replace IPs from static config as more reliable.
				if IPv6s[IPKey].Name != "" {
					continue
				}

				IPv6s[IPKey] = DHCPAllocation{
					Static: false,
					IP:     IP.To16(),
				}
			} else {
				// MAC only available in IPv4 leases.
				MAC, err := net.ParseMAC(fields[1])
				if err != nil {
					return nil, nil, err
				}

				var IPKey [4]byte
				copy(IPKey[:], IP.To4())

				// Don't replace IPs from static config as more reliable.
				if IPv4s[IPKey].Name != "" {
					continue
				}

				IPv4s[IPKey] = DHCPAllocation{
					MAC:    MAC,
					Static: false,
					IP:     IP.To4(),
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, nil, err
	}

	return IPv4s, IPv6s, nil
}

// StaticAllocationFileName returns the file name to use for a dnsmasq instance device static allocation.
func StaticAllocationFileName(projectName string, instanceName string, deviceName string) string {
	escapedDeviceName := filesystem.PathNameEncode(deviceName)

	return strings.Join([]string{project.Instance(projectName, instanceName), escapedDeviceName}, staticAllocationDeviceSeparator)
}
<|endoftext|>


<commit_before>package endpoint

import (
	"cloud.google.com/go/pubsub"
	"context"
	"errors"
	"fmt"
	"github.com/streadway/amqp"
	"google.golang.org/api/option"
	"os"
	"sync"
	"time"
)

const pubsubExpiresAfter = time.Second * 30

var errMissingGoogleCredentials = errors.New("Could not find GCP credentials - no credential path was submitted and envar GOOGLE_APPLICATION_CREDENTIALS is not set")

// SQSConn is an endpoint connection
type PubSubConn struct {
	mu      sync.Mutex
	ep      Endpoint
	svc     *pubsub.Client
	topic   *pubsub.Topic
	channel *amqp.Channel
	ex      bool
	t       time.Time
}

func (conn *PubSubConn) generatePubSubURL() string {
	if conn.ep.SQS.PlainURL != "" {
		return conn.ep.SQS.PlainURL
	}
	return "projects/" + conn.ep.PubSub.Project + "/topics/" + conn.ep.PubSub.Topic
}

func (conn *PubSubConn) close() {
	if conn.svc != nil {
		conn.svc.Close()
		conn.svc = nil
	}
}

// Send sends a message
func (conn *PubSubConn) Send(msg string) error {
	conn.mu.Lock()
	defer conn.mu.Unlock()

	if conn.ex {
		return errExpired
	}

	ctx := context.Background()

	conn.t = time.Now()

	if conn.svc == nil {
		var creds option.ClientOption
		credPath := conn.ep.PubSub.CredPath
		if credPath != "" {

			creds = option.WithCredentialsFile(credPath)
		} else {

			envCredPath := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
			creds = option.WithCredentialsFile(credPath)

			if envCredPath == "" {
				fmt.Println(errMissingGoogleCredentials)
				return errMissingGoogleCredentials
			}
		}
		svc, err := pubsub.NewClient(ctx, conn.ep.PubSub.Project, creds)
		if err != nil {
			fmt.Println(err)
			return err
		}

		topic := svc.Topic(conn.ep.PubSub.Topic)

		conn.svc = svc
		conn.topic = topic
	}

	// Send message
	res := conn.topic.Publish(ctx, &pubsub.Message{
		Data: []byte(msg),
	})
	_, err := res.Get(ctx)
	if err != nil {
		fmt.Println(err)
		return err
	}
	return nil
}

func (conn *PubSubConn) Expired() bool {
	conn.mu.Lock()
	defer conn.mu.Unlock()
	if !conn.ex {
		if time.Now().Sub(conn.t) > pubsubExpiresAfter {
			conn.ex = true
			conn.close()
		}
	}
	return conn.ex
}

func newPubSubConn(ep Endpoint) *PubSubConn {
	return &PubSubConn{
		ep: ep,
		t:  time.Now(),
	}
}
<commit_msg>fixed default credentials<commit_after>package endpoint

import (
	"cloud.google.com/go/pubsub"
	"context"
	"errors"
	"fmt"
	"github.com/streadway/amqp"
	"google.golang.org/api/option"
	"sync"
	"time"
)

const pubsubExpiresAfter = time.Second * 30

var errMissingGoogleCredentials = errors.New("Could not find GCP credentials - no credential path was submitted and envar GOOGLE_APPLICATION_CREDENTIALS is not set")

// SQSConn is an endpoint connection
type PubSubConn struct {
	mu      sync.Mutex
	ep      Endpoint
	svc     *pubsub.Client
	topic   *pubsub.Topic
	channel *amqp.Channel
	ex      bool
	t       time.Time
}

func (conn *PubSubConn) generatePubSubURL() string {
	if conn.ep.SQS.PlainURL != "" {
		return conn.ep.SQS.PlainURL
	}
	return "projects/" + conn.ep.PubSub.Project + "/topics/" + conn.ep.PubSub.Topic
}

func (conn *PubSubConn) close() {
	if conn.svc != nil {
		conn.svc.Close()
		conn.svc = nil
	}
}

// Send sends a message
func (conn *PubSubConn) Send(msg string) error {
	conn.mu.Lock()
	defer conn.mu.Unlock()

	if conn.ex {
		return errExpired
	}

	ctx := context.Background()

	conn.t = time.Now()

	if conn.svc == nil {
		var creds option.ClientOption
		var svc *pubsub.Client
		var err error
		credPath := conn.ep.PubSub.CredPath

		if credPath != "" {
			creds = option.WithCredentialsFile(credPath)
			svc, err = pubsub.NewClient(ctx, conn.ep.PubSub.Project, creds)
		} else {
			svc, err = pubsub.NewClient(ctx, conn.ep.PubSub.Project)
		}

		if err != nil {
			fmt.Println(err)
			return err
		}

		topic := svc.Topic(conn.ep.PubSub.Topic)

		conn.svc = svc
		conn.topic = topic
	}

	// Send message
	res := conn.topic.Publish(ctx, &pubsub.Message{
		Data: []byte(msg),
	})
	_, err := res.Get(ctx)
	if err != nil {
		fmt.Println(err)
		return err
	}
	return nil
}

func (conn *PubSubConn) Expired() bool {
	conn.mu.Lock()
	defer conn.mu.Unlock()
	if !conn.ex {
		if time.Now().Sub(conn.t) > pubsubExpiresAfter {
			conn.ex = true
			conn.close()
		}
	}
	return conn.ex
}

func newPubSubConn(ep Endpoint) *PubSubConn {
	return &PubSubConn{
		ep: ep,
		t:  time.Now(),
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lattice

import (
	"fmt"
	"io"
	"sync"
	"unicode"
	"unicode/utf8"

	"github.com/ikawaha/kagome/internal/dic"
)

const (
	maximumCost              = 1<<31 - 1
	maximumUnknownWordLength = 1024
	searchModeKanjiLength    = 2
	searchModeKanjiPenalty   = 3000
	searchModeOtherLength    = 7
	searchModeOtherPenalty   = 1700
)

// TokenizeMode represents how to tokenize sentencse.
type TokenizeMode int

const (
	//Normal Mode
	Normal TokenizeMode = iota + 1
	// Search Mode
	Search
	// Extended Mode
	Extended
)

var latticePool = sync.Pool{
	New: func() interface{} {
		return new(Lattice)
	},
}

// Lattice represents a grid of morph nodes.
type Lattice struct {
	Input  string
	Output []*node
	list   [][]*node
	dic    *dic.Dic
	udic   *dic.UserDic
}

// New returns a new lattice.
func New(d *dic.Dic, u *dic.UserDic) *Lattice {
	la := latticePool.Get().(*Lattice)
	la.dic = d
	la.udic = u
	return la
}

// Free releases a memory of a lattice.
func (la *Lattice) Free() {
	la.Input = ""
	la.Output = la.Output[:0]
	for i := range la.list {
		for j := range la.list[i] {
			nodePool.Put(la.list[i][j])
		}
		la.list[i] = la.list[i][:0]
	}
	la.list = la.list[:0]
	la.udic = nil
	latticePool.Put(la)
}

func (la *Lattice) addNode(pos, id, start int, class NodeClass, surface string) {
	var m dic.Morph
	switch class {
	case DUMMY:
		//use default cost
	case KNOWN:
		m = la.dic.Morphs[id]
	case UNKNOWN:
		m = la.dic.UnkMorphs[id]
	case USER:
		// use default cost
	}
	n := newNode()
	n.ID = id
	n.Start = start
	n.Class = class
	n.Left, n.Right, n.Weight = int32(m.LeftID), int32(m.RightID), int32(m.Weight)
	n.Surface = surface
	n.Prev = nil
	p := pos + utf8.RuneCountInString(surface)
	la.list[p] = append(la.list[p], n)
}

// Build builds a lattice from the inputs.
func (la *Lattice) Build(inp string) {
	rc := utf8.RuneCountInString(inp)
	la.Input = inp
	if cap(la.list) < rc+2 {
		const expandRatio = 2
		la.list = make([][]*node, 0, (rc+2)*expandRatio)
	}
	la.list = la.list[0 : rc+2]

	la.addNode(0, BosEosID, 0, DUMMY, inp[0:0])
	la.addNode(rc+1, BosEosID, rc, DUMMY, inp[rc:rc])

	runePos := -1
	for pos, ch := range inp {
		runePos++
		anyMatches := false

		// (1) USER DIC
		if la.udic != nil {
			lens, outputs := la.udic.Index.CommonPrefixSearch(inp[pos:])
			for i, ids := range outputs {
				for j := range ids {
					la.addNode(runePos, ids[j], runePos,
						USER, inp[pos:pos+lens[i]])
				}
			}
			anyMatches = (len(lens) > 0)
		}
		if anyMatches {
			continue
		}

		// (2) KNOWN DIC
		if lens, outputs := la.dic.Index.CommonPrefixSearch(inp[pos:]); len(lens) > 0 {
			anyMatches = true
			for i, ids := range outputs {
				for j := range ids {
					la.addNode(runePos, ids[j], runePos,
						KNOWN, inp[pos:pos+lens[i]])
				}
			}
		}
		// (3) UNKNOWN DIC
		class := la.dic.CharacterCategory(ch)
		if !anyMatches || la.dic.InvokeList[int(class)] {
			endPos := pos + utf8.RuneLen(ch)
			unkWordLen := 1
			if la.dic.GroupList[int(class)] {
				for i, w, size := endPos, 1, len(inp); i < size; i += w {
					var c rune
					c, w = utf8.DecodeRuneInString(inp[i:])
					if la.dic.CharacterCategory(c) != class {
						break
					}
					endPos += w
					unkWordLen++
					if unkWordLen >= maximumUnknownWordLength {
						break
					}
				}
			}
			id := la.dic.UnkIndex[int32(class)]
			for i, w := pos, 0; i < endPos; i += w {
				_, w = utf8.DecodeRuneInString(inp[i:])
				end := i + w
				dup, _ := la.dic.UnkIndexDup[int32(class)]
				for x := 0; x < int(dup)+1; x++ {
					la.addNode(runePos, int(id)+x, runePos,
						UNKNOWN, inp[pos:end])
				}
			}
		}
	}
	return
}

// String returns a debug string of a lattice.
func (la *Lattice) String() string {
	str := ""
	for i, nodes := range la.list {
		str += fmt.Sprintf("[%v] :\n", i)
		for _, node := range nodes {
			str += fmt.Sprintf("%v\n", node)
		}
		str += "\n"
	}
	return str
}

func kanjiOnly(s string) bool {
	for _, r := range s {
		if !unicode.In(r, unicode.Ideographic) {
			return false
		}
	}
	return s != ""
}

func additionalCost(n *node) int {
	l := utf8.RuneCountInString(n.Surface)
	if l > searchModeKanjiLength && kanjiOnly(n.Surface) {
		return (l - searchModeKanjiLength) * searchModeKanjiPenalty
	}
	if l > searchModeOtherLength {
		return (l - searchModeOtherLength) * searchModeOtherPenalty
	}
	return 0
}

// Forward runs forward algorithm of the Viterbi.
func (la *Lattice) Forward(m TokenizeMode) {
	for i, size := 1, len(la.list); i < size; i++ {
		currentList := la.list[i]
		for index, target := range currentList {
			prevList := la.list[target.Start]
			if len(prevList) == 0 {
				la.list[i][index].Cost = maximumCost
				continue
			}
			for j, n := range prevList {
				var c int16
				if n.Class != USER && target.Class != USER {
					c = la.dic.Connection.At(int(n.Right), int(target.Left))
				}
				totalCost := int64(c) + int64(target.Weight) + int64(n.Cost)
				if m != Normal {
					totalCost += int64(additionalCost(n))
				}
				if totalCost > maximumCost {
					totalCost = maximumCost
				}
				if j == 0 || int32(totalCost) < la.list[i][index].Cost {
					la.list[i][index].Cost = int32(totalCost)
					la.list[i][index].Prev = la.list[target.Start][j]
				}
			}
		}
	}
	return
}

// Backward runs backward algorithm of the Viterbi.
func (la *Lattice) Backward(m TokenizeMode) {
	const bufferExpandRatio = 2
	size := len(la.list)
	if size == 0 {
		return
	}
	if cap(la.Output) < size {
		la.Output = make([]*node, 0, size*bufferExpandRatio)
	} else {
		la.Output = la.Output[:0]
	}
	for p := la.list[size-1][0]; p != nil; p = p.Prev {
		if m != Extended || p.Class != UNKNOWN {
			la.Output = append(la.Output, p)
			continue
		}
		runeLen := utf8.RuneCountInString(p.Surface)
		stack := make([]*node, 0, runeLen)
		i := 0
		for _, r := range p.Surface {
			n := nodePool.Get().(*node)
			n.ID = p.ID
			n.Start = p.Start + i
			n.Class = DUMMY
			n.Surface = string(r)
			stack = append(stack, n)
			i++
		}
		for j, end := 0, len(stack); j < end; j++ {
			la.Output = append(la.Output, stack[runeLen-1-j])
		}
	}
}

// Dot outputs the lattice in the graphviz dot format.
func (la *Lattice) Dot(w io.Writer) {
	bests := make(map[*node]struct{})
	for _, n := range la.Output {
		bests[n] = struct{}{}
	}
	type edge struct {
		from *node
		to   *node
	}
	edges := make([]edge, 0, 1024)
	for i, size := 1, len(la.list); i < size; i++ {
		currents := la.list[i]
		for _, to := range currents {
			if to.Class == UNKNOWN {
				if _, ok := bests[to]; !ok {
					continue
				}
			}
			prevs := la.list[to.Start]
			if len(prevs) == 0 {
				continue
			}
			for _, from := range prevs {
				if from.Class == UNKNOWN {
					continue
				}
				edges = append(edges, edge{from, to})
			}
		}
	}
	fmt.Fprintln(w, "graph lattice {")
	fmt.Fprintln(w, "dpi=48;")
	fmt.Fprintln(w, "graph [style=filled, splines=true, overlap=false, fontsize=30, rankdir=LR]")
	fmt.Fprintln(w, "edge [fontname=Helvetica, fontcolor=red, color=\"#606060\"]")
	fmt.Fprintln(w, "node [shape=box, style=filled, fillcolor=\"#e8e8f0\", fontname=Helvetica]")
	for i, list := range la.list {
		for _, n := range list {
			if n.Class == UNKNOWN {
				if _, ok := bests[n]; ok {
					continue
				}
			}
			surf := n.Surface
			if n.ID == BosEosID {
				if i == 0 {
					surf = "BOS"
				} else {
					surf = "EOS"
				}
			}
			if _, ok := bests[n]; ok {
				fmt.Fprintf(w, "\t\"%p\" [label=\"%s\\n%d\",shape=ellipse, peripheries=2];\n", n, surf, n.Weight)
			} else if n.Class != UNKNOWN {
				fmt.Fprintf(w, "\t\"%p\" [label=\"%s\\n%d\"];\n", n, surf, n.Weight)
			}
		}
	}
	for _, e := range edges {
		var c int16
		if e.from.Class != USER && e.to.Class != USER {
			c = la.dic.Connection.At(int(e.from.Right), int(e.to.Left))
		}
		_, l := bests[e.from]
		_, r := bests[e.to]
		if l && r {
			fmt.Fprintf(w, "\t\"%p\" -- \"%p\" [label=\"%d\", style=bold, color=blue, fontcolor=blue];\n",
				e.from, e.to, c)
		} else {
			fmt.Fprintf(w, "\t\"%p\" -- \"%p\" [label=\"%d\"];\n",
				e.from, e.to, c)
		}
	}

	fmt.Fprintln(w, "}")
}
<commit_msg>Fix a bug eliminating UNK nodes in bests<commit_after>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lattice

import (
	"fmt"
	"io"
	"sync"
	"unicode"
	"unicode/utf8"

	"github.com/ikawaha/kagome/internal/dic"
)

const (
	maximumCost              = 1<<31 - 1
	maximumUnknownWordLength = 1024
	searchModeKanjiLength    = 2
	searchModeKanjiPenalty   = 3000
	searchModeOtherLength    = 7
	searchModeOtherPenalty   = 1700
)

// TokenizeMode represents how to tokenize sentencse.
type TokenizeMode int

const (
	//Normal Mode
	Normal TokenizeMode = iota + 1
	// Search Mode
	Search
	// Extended Mode
	Extended
)

var latticePool = sync.Pool{
	New: func() interface{} {
		return new(Lattice)
	},
}

// Lattice represents a grid of morph nodes.
type Lattice struct {
	Input  string
	Output []*node
	list   [][]*node
	dic    *dic.Dic
	udic   *dic.UserDic
}

// New returns a new lattice.
func New(d *dic.Dic, u *dic.UserDic) *Lattice {
	la := latticePool.Get().(*Lattice)
	la.dic = d
	la.udic = u
	return la
}

// Free releases a memory of a lattice.
func (la *Lattice) Free() {
	la.Input = ""
	la.Output = la.Output[:0]
	for i := range la.list {
		for j := range la.list[i] {
			nodePool.Put(la.list[i][j])
		}
		la.list[i] = la.list[i][:0]
	}
	la.list = la.list[:0]
	la.udic = nil
	latticePool.Put(la)
}

func (la *Lattice) addNode(pos, id, start int, class NodeClass, surface string) {
	var m dic.Morph
	switch class {
	case DUMMY:
		//use default cost
	case KNOWN:
		m = la.dic.Morphs[id]
	case UNKNOWN:
		m = la.dic.UnkMorphs[id]
	case USER:
		// use default cost
	}
	n := newNode()
	n.ID = id
	n.Start = start
	n.Class = class
	n.Left, n.Right, n.Weight = int32(m.LeftID), int32(m.RightID), int32(m.Weight)
	n.Surface = surface
	n.Prev = nil
	p := pos + utf8.RuneCountInString(surface)
	la.list[p] = append(la.list[p], n)
}

// Build builds a lattice from the inputs.
func (la *Lattice) Build(inp string) {
	rc := utf8.RuneCountInString(inp)
	la.Input = inp
	if cap(la.list) < rc+2 {
		const expandRatio = 2
		la.list = make([][]*node, 0, (rc+2)*expandRatio)
	}
	la.list = la.list[0 : rc+2]

	la.addNode(0, BosEosID, 0, DUMMY, inp[0:0])
	la.addNode(rc+1, BosEosID, rc, DUMMY, inp[rc:rc])

	runePos := -1
	for pos, ch := range inp {
		runePos++
		anyMatches := false

		// (1) USER DIC
		if la.udic != nil {
			lens, outputs := la.udic.Index.CommonPrefixSearch(inp[pos:])
			for i, ids := range outputs {
				for j := range ids {
					la.addNode(runePos, ids[j], runePos,
						USER, inp[pos:pos+lens[i]])
				}
			}
			anyMatches = (len(lens) > 0)
		}
		if anyMatches {
			continue
		}

		// (2) KNOWN DIC
		if lens, outputs := la.dic.Index.CommonPrefixSearch(inp[pos:]); len(lens) > 0 {
			anyMatches = true
			for i, ids := range outputs {
				for j := range ids {
					la.addNode(runePos, ids[j], runePos,
						KNOWN, inp[pos:pos+lens[i]])
				}
			}
		}
		// (3) UNKNOWN DIC
		class := la.dic.CharacterCategory(ch)
		if !anyMatches || la.dic.InvokeList[int(class)] {
			endPos := pos + utf8.RuneLen(ch)
			unkWordLen := 1
			if la.dic.GroupList[int(class)] {
				for i, w, size := endPos, 1, len(inp); i < size; i += w {
					var c rune
					c, w = utf8.DecodeRuneInString(inp[i:])
					if la.dic.CharacterCategory(c) != class {
						break
					}
					endPos += w
					unkWordLen++
					if unkWordLen >= maximumUnknownWordLength {
						break
					}
				}
			}
			id := la.dic.UnkIndex[int32(class)]
			for i, w := pos, 0; i < endPos; i += w {
				_, w = utf8.DecodeRuneInString(inp[i:])
				end := i + w
				dup, _ := la.dic.UnkIndexDup[int32(class)]
				for x := 0; x < int(dup)+1; x++ {
					la.addNode(runePos, int(id)+x, runePos,
						UNKNOWN, inp[pos:end])
				}
			}
		}
	}
	return
}

// String returns a debug string of a lattice.
func (la *Lattice) String() string {
	str := ""
	for i, nodes := range la.list {
		str += fmt.Sprintf("[%v] :\n", i)
		for _, node := range nodes {
			str += fmt.Sprintf("%v\n", node)
		}
		str += "\n"
	}
	return str
}

func kanjiOnly(s string) bool {
	for _, r := range s {
		if !unicode.In(r, unicode.Ideographic) {
			return false
		}
	}
	return s != ""
}

func additionalCost(n *node) int {
	l := utf8.RuneCountInString(n.Surface)
	if l > searchModeKanjiLength && kanjiOnly(n.Surface) {
		return (l - searchModeKanjiLength) * searchModeKanjiPenalty
	}
	if l > searchModeOtherLength {
		return (l - searchModeOtherLength) * searchModeOtherPenalty
	}
	return 0
}

// Forward runs forward algorithm of the Viterbi.
func (la *Lattice) Forward(m TokenizeMode) {
	for i, size := 1, len(la.list); i < size; i++ {
		currentList := la.list[i]
		for index, target := range currentList {
			prevList := la.list[target.Start]
			if len(prevList) == 0 {
				la.list[i][index].Cost = maximumCost
				continue
			}
			for j, n := range prevList {
				var c int16
				if n.Class != USER && target.Class != USER {
					c = la.dic.Connection.At(int(n.Right), int(target.Left))
				}
				totalCost := int64(c) + int64(target.Weight) + int64(n.Cost)
				if m != Normal {
					totalCost += int64(additionalCost(n))
				}
				if totalCost > maximumCost {
					totalCost = maximumCost
				}
				if j == 0 || int32(totalCost) < la.list[i][index].Cost {
					la.list[i][index].Cost = int32(totalCost)
					la.list[i][index].Prev = la.list[target.Start][j]
				}
			}
		}
	}
	return
}

// Backward runs backward algorithm of the Viterbi.
func (la *Lattice) Backward(m TokenizeMode) {
	const bufferExpandRatio = 2
	size := len(la.list)
	if size == 0 {
		return
	}
	if cap(la.Output) < size {
		la.Output = make([]*node, 0, size*bufferExpandRatio)
	} else {
		la.Output = la.Output[:0]
	}
	for p := la.list[size-1][0]; p != nil; p = p.Prev {
		if m != Extended || p.Class != UNKNOWN {
			la.Output = append(la.Output, p)
			continue
		}
		runeLen := utf8.RuneCountInString(p.Surface)
		stack := make([]*node, 0, runeLen)
		i := 0
		for _, r := range p.Surface {
			n := nodePool.Get().(*node)
			n.ID = p.ID
			n.Start = p.Start + i
			n.Class = DUMMY
			n.Surface = string(r)
			stack = append(stack, n)
			i++
		}
		for j, end := 0, len(stack); j < end; j++ {
			la.Output = append(la.Output, stack[runeLen-1-j])
		}
	}
}

// Dot outputs the lattice in the graphviz dot format.
func (la *Lattice) Dot(w io.Writer) {
	bests := make(map[*node]struct{})
	for _, n := range la.Output {
		bests[n] = struct{}{}
	}
	type edge struct {
		from *node
		to   *node
	}
	edges := make([]edge, 0, 1024)
	for i, size := 1, len(la.list); i < size; i++ {
		currents := la.list[i]
		for _, to := range currents {
			if to.Class == UNKNOWN {
				if _, ok := bests[to]; !ok {
					continue
				}
			}
			prevs := la.list[to.Start]
			if len(prevs) == 0 {
				continue
			}
			for _, from := range prevs {
				if from.Class == UNKNOWN {
					if _, ok := bests[from]; !ok {
						continue
					}
				}
				edges = append(edges, edge{from, to})
			}
		}
	}
	fmt.Fprintln(w, "graph lattice {")
	fmt.Fprintln(w, "dpi=48;")
	fmt.Fprintln(w, "graph [style=filled, splines=true, overlap=false, fontsize=30, rankdir=LR]")
	fmt.Fprintln(w, "edge [fontname=Helvetica, fontcolor=red, color=\"#606060\"]")
	fmt.Fprintln(w, "node [shape=box, style=filled, fillcolor=\"#e8e8f0\", fontname=Helvetica]")
	for i, list := range la.list {
		for _, n := range list {
			if n.Class == UNKNOWN {
				if _, ok := bests[n]; !ok {
					continue
				}
			}
			surf := n.Surface
			if n.ID == BosEosID {
				if i == 0 {
					surf = "BOS"
				} else {
					surf = "EOS"
				}
			}
			if _, ok := bests[n]; ok {
				fmt.Fprintf(w, "\t\"%p\" [label=\"%s\\n%d\",shape=ellipse, peripheries=2];\n", n, surf, n.Weight)
			} else if n.Class != UNKNOWN {
				fmt.Fprintf(w, "\t\"%p\" [label=\"%s\\n%d\"];\n", n, surf, n.Weight)
			}
		}
	}
	for _, e := range edges {
		var c int16
		if e.from.Class != USER && e.to.Class != USER {
			c = la.dic.Connection.At(int(e.from.Right), int(e.to.Left))
		}
		_, l := bests[e.from]
		_, r := bests[e.to]
		if l && r {
			fmt.Fprintf(w, "\t\"%p\" -- \"%p\" [label=\"%d\", style=bold, color=blue, fontcolor=blue];\n",
				e.from, e.to, c)
		} else {
			fmt.Fprintf(w, "\t\"%p\" -- \"%p\" [label=\"%d\"];\n",
				e.from, e.to, c)
		}
	}

	fmt.Fprintln(w, "}")
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"crypto/sha1"
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"html/template"
	"io/ioutil"
	"os"
	"reflect"
	"sort"
	"strconv"
	"sync/atomic"
	"time"

	"golang.org/x/tools/internal/event"
	"golang.org/x/tools/internal/lsp/debug/tag"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/span"
	errors "golang.org/x/xerrors"
)

func New(ctx context.Context, options func(*source.Options)) *Cache {
	index := atomic.AddInt64(&cacheIndex, 1)
	c := &Cache{
		id:      strconv.FormatInt(index, 10),
		fset:    token.NewFileSet(),
		options: options,
	}
	return c
}

type Cache struct {
	id      string
	fset    *token.FileSet
	options func(*source.Options)

	store memoize.Store
}

type fileKey struct {
	uri     span.URI
	modTime time.Time
}

type fileHandle struct {
	uri span.URI
	memoize.NoCopy
	bytes []byte
	hash  string
	err   error
}

func (c *Cache) getFile(ctx context.Context, uri span.URI) (*fileHandle, error) {
	var modTime time.Time
	if fi, err := os.Stat(uri.Filename()); err == nil {
		modTime = fi.ModTime()
	}

	key := fileKey{
		uri:     uri,
		modTime: modTime,
	}
	h := c.store.Bind(key, func(ctx context.Context) interface{} {
		return readFile(ctx, uri, modTime)
	})
	v, err := h.Get(ctx)
	if err != nil {
		return nil, err
	}
	return v.(*fileHandle), nil
}

// ioLimit limits the number of parallel file reads per process.
var ioLimit = make(chan struct{}, 128)

func readFile(ctx context.Context, uri span.URI, origTime time.Time) *fileHandle {
	ctx, done := event.Start(ctx, "cache.getFile", tag.File.Of(uri.Filename()))
	_ = ctx
	defer done()

	ioLimit <- struct{}{}
	defer func() { <-ioLimit }()

	var modTime time.Time
	if fi, err := os.Stat(uri.Filename()); err == nil {
		modTime = fi.ModTime()
	}

	if modTime != origTime {
		return &fileHandle{err: errors.Errorf("%s: file has been modified", uri.Filename())}
	}
	data, err := ioutil.ReadFile(uri.Filename())
	if err != nil {
		return &fileHandle{err: err}
	}
	return &fileHandle{
		uri:   uri,
		bytes: data,
		hash:  hashContents(data),
	}
}

func (c *Cache) NewSession(ctx context.Context) *Session {
	index := atomic.AddInt64(&sessionIndex, 1)
	s := &Session{
		cache:    c,
		id:       strconv.FormatInt(index, 10),
		options:  source.DefaultOptions(),
		overlays: make(map[span.URI]*overlay),
	}
	event.Log(ctx, "New session", KeyCreateSession.Of(s))
	return s
}

func (c *Cache) FileSet() *token.FileSet {
	return c.fset
}

func (h *fileHandle) URI() span.URI {
	return h.uri
}

func (h *fileHandle) Kind() source.FileKind {
	return source.DetectLanguage("", h.uri.Filename())
}

func (h *fileHandle) Version() float64 {
	return 0
}

func (h *fileHandle) Identity() source.FileIdentity {
	return source.FileIdentity{
		URI:        h.uri,
		Identifier: h.hash,
		Kind:       h.Kind(),
	}
}

func (h *fileHandle) Read() ([]byte, error) {
	return h.bytes, h.err
}

func hashContents(contents []byte) string {
	// TODO: consider whether sha1 is the best choice here
	// This hash is used for internal identity detection only
	return fmt.Sprintf("%x", sha1.Sum(contents))
}

var cacheIndex, sessionIndex, viewIndex int64

func (c *Cache) ID() string                     { return c.id }
func (c *Cache) MemStats() map[reflect.Type]int { return c.store.Stats() }

type packageStat struct {
	id        packageID
	mode      source.ParseMode
	file      int64
	ast       int64
	types     int64
	typesInfo int64
	total     int64
}

func (c *Cache) PackageStats(withNames bool) template.HTML {
	var packageStats []packageStat
	c.store.DebugOnlyIterate(func(k, v interface{}) {
		switch k.(type) {
		case packageHandleKey:
			v := v.(*packageData)
			stat := packageStat{
				id:        v.pkg.id,
				mode:      v.pkg.mode,
				types:     typesCost(v.pkg.types.Scope()),
				typesInfo: typesInfoCost(v.pkg.typesInfo),
			}
			for _, f := range v.pkg.compiledGoFiles {
				fvi := f.handle.Cached()
				if fvi == nil {
					continue
				}
				fv := fvi.(*parseGoData)
				stat.file += int64(len(fv.src))
				stat.ast += astCost(fv.ast)
			}
			stat.total = stat.file + stat.ast + stat.types + stat.typesInfo
			packageStats = append(packageStats, stat)
		}
	})
	var totalCost int64
	for _, stat := range packageStats {
		totalCost += stat.total
	}
	sort.Slice(packageStats, func(i, j int) bool {
		return packageStats[i].total > packageStats[j].total
	})
	html := "

Name	total = file + ast + types + types info
\n"
	human := func(n int64) string {
		return fmt.Sprintf("%.2f", float64(n)/(1024*1024))
	}
	var printedCost int64
	for _, stat := range packageStats {
		name := stat.id
		if !withNames {
			name = "-"
		}
		html += fmt.Sprintf("
%v (%v)	%v = %v + %v + %v + %v
\n", name, stat.mode,
			human(stat.total), human(stat.file), human(stat.ast), human(stat.types), human(stat.typesInfo))
		printedCost += stat.total
		if float64(printedCost) > float64(totalCost)*.9 {
			break
		}
	}
	html += "
\n"
	return template.HTML(html)
}

func astCost(f *ast.File) int64 {
	var count int64
	ast.Inspect(f, func(n ast.Node) bool {
		count += 32 // nodes are pretty small.
		return true
	})
	return count
}

func typesCost(scope *types.Scope) int64 {
	cost := 64 + int64(scope.Len())*128 // types.object looks pretty big
	for i := 0; i < scope.NumChildren(); i++ {
		cost += typesCost(scope.Child(i))
	}
	return cost
}

func typesInfoCost(info *types.Info) int64 {
	// Most of these refer to existing objects, with the exception of InitOrder, Selections, and Types.
	cost := 24*len(info.Defs) +
		32*len(info.Implicits) +
		256*len(info.InitOrder) + // these are big, but there aren't many of them.
		32*len(info.Scopes) +
		128*len(info.Selections) + // wild guess
		128*len(info.Types) + // wild guess
		32*len(info.Uses)
	return int64(cost)
}
<commit_msg>internal/lsp/cache: handle a few possible panics in PackageStats<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"crypto/sha1"
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"html/template"
	"io/ioutil"
	"os"
	"reflect"
	"sort"
	"strconv"
	"sync/atomic"
	"time"

	"golang.org/x/tools/internal/event"
	"golang.org/x/tools/internal/lsp/debug/tag"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/span"
	errors "golang.org/x/xerrors"
)

func New(ctx context.Context, options func(*source.Options)) *Cache {
	index := atomic.AddInt64(&cacheIndex, 1)
	c := &Cache{
		id:      strconv.FormatInt(index, 10),
		fset:    token.NewFileSet(),
		options: options,
	}
	return c
}

type Cache struct {
	id      string
	fset    *token.FileSet
	options func(*source.Options)

	store memoize.Store
}

type fileKey struct {
	uri     span.URI
	modTime time.Time
}

type fileHandle struct {
	uri span.URI
	memoize.NoCopy
	bytes []byte
	hash  string
	err   error
}

func (c *Cache) getFile(ctx context.Context, uri span.URI) (*fileHandle, error) {
	var modTime time.Time
	if fi, err := os.Stat(uri.Filename()); err == nil {
		modTime = fi.ModTime()
	}

	key := fileKey{
		uri:     uri,
		modTime: modTime,
	}
	h := c.store.Bind(key, func(ctx context.Context) interface{} {
		return readFile(ctx, uri, modTime)
	})
	v, err := h.Get(ctx)
	if err != nil {
		return nil, err
	}
	return v.(*fileHandle), nil
}

// ioLimit limits the number of parallel file reads per process.
var ioLimit = make(chan struct{}, 128)

func readFile(ctx context.Context, uri span.URI, origTime time.Time) *fileHandle {
	ctx, done := event.Start(ctx, "cache.getFile", tag.File.Of(uri.Filename()))
	_ = ctx
	defer done()

	ioLimit <- struct{}{}
	defer func() { <-ioLimit }()

	var modTime time.Time
	if fi, err := os.Stat(uri.Filename()); err == nil {
		modTime = fi.ModTime()
	}

	if modTime != origTime {
		return &fileHandle{err: errors.Errorf("%s: file has been modified", uri.Filename())}
	}
	data, err := ioutil.ReadFile(uri.Filename())
	if err != nil {
		return &fileHandle{err: err}
	}
	return &fileHandle{
		uri:   uri,
		bytes: data,
		hash:  hashContents(data),
	}
}

func (c *Cache) NewSession(ctx context.Context) *Session {
	index := atomic.AddInt64(&sessionIndex, 1)
	s := &Session{
		cache:    c,
		id:       strconv.FormatInt(index, 10),
		options:  source.DefaultOptions(),
		overlays: make(map[span.URI]*overlay),
	}
	event.Log(ctx, "New session", KeyCreateSession.Of(s))
	return s
}

func (c *Cache) FileSet() *token.FileSet {
	return c.fset
}

func (h *fileHandle) URI() span.URI {
	return h.uri
}

func (h *fileHandle) Kind() source.FileKind {
	return source.DetectLanguage("", h.uri.Filename())
}

func (h *fileHandle) Version() float64 {
	return 0
}

func (h *fileHandle) Identity() source.FileIdentity {
	return source.FileIdentity{
		URI:        h.uri,
		Identifier: h.hash,
		Kind:       h.Kind(),
	}
}

func (h *fileHandle) Read() ([]byte, error) {
	return h.bytes, h.err
}

func hashContents(contents []byte) string {
	// TODO: consider whether sha1 is the best choice here
	// This hash is used for internal identity detection only
	return fmt.Sprintf("%x", sha1.Sum(contents))
}

var cacheIndex, sessionIndex, viewIndex int64

func (c *Cache) ID() string                     { return c.id }
func (c *Cache) MemStats() map[reflect.Type]int { return c.store.Stats() }

type packageStat struct {
	id        packageID
	mode      source.ParseMode
	file      int64
	ast       int64
	types     int64
	typesInfo int64
	total     int64
}

func (c *Cache) PackageStats(withNames bool) template.HTML {
	var packageStats []packageStat
	c.store.DebugOnlyIterate(func(k, v interface{}) {
		switch k.(type) {
		case packageHandleKey:
			v := v.(*packageData)
			if v.pkg == nil {
				break
			}
			var typsCost, typInfoCost int64
			if v.pkg.types != nil {
				typsCost = typesCost(v.pkg.types.Scope())
			}
			if v.pkg.typesInfo != nil {
				typInfoCost = typesInfoCost(v.pkg.typesInfo)
			}
			stat := packageStat{
				id:        v.pkg.id,
				mode:      v.pkg.mode,
				types:     typsCost,
				typesInfo: typInfoCost,
			}
			for _, f := range v.pkg.compiledGoFiles {
				fvi := f.handle.Cached()
				if fvi == nil {
					continue
				}
				fv := fvi.(*parseGoData)
				stat.file += int64(len(fv.src))
				stat.ast += astCost(fv.ast)
			}
			stat.total = stat.file + stat.ast + stat.types + stat.typesInfo
			packageStats = append(packageStats, stat)
		}
	})
	var totalCost int64
	for _, stat := range packageStats {
		totalCost += stat.total
	}
	sort.Slice(packageStats, func(i, j int) bool {
		return packageStats[i].total > packageStats[j].total
	})
	html := "

Name	total = file + ast + types + types info
\n"
	human := func(n int64) string {
		return fmt.Sprintf("%.2f", float64(n)/(1024*1024))
	}
	var printedCost int64
	for _, stat := range packageStats {
		name := stat.id
		if !withNames {
			name = "-"
		}
		html += fmt.Sprintf("
%v (%v)	%v = %v + %v + %v + %v
\n", name, stat.mode,
			human(stat.total), human(stat.file), human(stat.ast), human(stat.types), human(stat.typesInfo))
		printedCost += stat.total
		if float64(printedCost) > float64(totalCost)*.9 {
			break
		}
	}
	html += "
\n"
	return template.HTML(html)
}

func astCost(f *ast.File) int64 {
	var count int64
	ast.Inspect(f, func(n ast.Node) bool {
		count += 32 // nodes are pretty small.
		return true
	})
	return count
}

func typesCost(scope *types.Scope) int64 {
	cost := 64 + int64(scope.Len())*128 // types.object looks pretty big
	for i := 0; i < scope.NumChildren(); i++ {
		cost += typesCost(scope.Child(i))
	}
	return cost
}

func typesInfoCost(info *types.Info) int64 {
	// Most of these refer to existing objects, with the exception of InitOrder, Selections, and Types.
	cost := 24*len(info.Defs) +
		32*len(info.Implicits) +
		256*len(info.InitOrder) + // these are big, but there aren't many of them.
		32*len(info.Scopes) +
		128*len(info.Selections) + // wild guess
		128*len(info.Types) + // wild guess
		32*len(info.Uses)
	return int64(cost)
}
<|endoftext|>


<commit_before>package protocol

import "time"

// DesiredReceiveBufferSize is the kernel UDP receive buffer size that we'd like to use.
const DesiredReceiveBufferSize = (1 << 20) * 2 // 2 MB

// InitialPacketSizeIPv4 is the maximum packet size that we use for sending IPv4 packets.
const InitialPacketSizeIPv4 = 1252

// InitialPacketSizeIPv6 is the maximum packet size that we use for sending IPv6 packets.
const InitialPacketSizeIPv6 = 1232

// MaxCongestionWindowPackets is the maximum congestion window in packet.
const MaxCongestionWindowPackets = 10000

// MaxUndecryptablePackets limits the number of undecryptable packets that are queued in the connection.
const MaxUndecryptablePackets = 32

// ConnectionFlowControlMultiplier determines how much larger the connection flow control windows needs to be relative to any stream's flow control window
// This is the value that Chromium is using
const ConnectionFlowControlMultiplier = 1.5

// DefaultInitialMaxStreamData is the default initial stream-level flow control window for receiving data
const DefaultInitialMaxStreamData = (1 << 10) * 512 // 512 kb

// DefaultInitialMaxData is the connection-level flow control window for receiving data
const DefaultInitialMaxData = ConnectionFlowControlMultiplier * DefaultInitialMaxStreamData

// DefaultMaxReceiveStreamFlowControlWindow is the default maximum stream-level flow control window for receiving data
const DefaultMaxReceiveStreamFlowControlWindow = 6 * (1 << 20) // 6 MB

// DefaultMaxReceiveConnectionFlowControlWindow is the default connection-level flow control window for receiving data
const DefaultMaxReceiveConnectionFlowControlWindow = 15 * (1 << 20) // 15 MB

// WindowUpdateThreshold is the fraction of the receive window that has to be consumed before an higher offset is advertised to the client
const WindowUpdateThreshold = 0.25

// DefaultMaxIncomingStreams is the maximum number of streams that a peer may open
const DefaultMaxIncomingStreams = 100

// DefaultMaxIncomingUniStreams is the maximum number of unidirectional streams that a peer may open
const DefaultMaxIncomingUniStreams = 100

// MaxServerUnprocessedPackets is the max number of packets stored in the server that are not yet processed.
const MaxServerUnprocessedPackets = 1024

// MaxConnUnprocessedPackets is the max number of packets stored in each connection that are not yet processed.
const MaxConnUnprocessedPackets = 256

// SkipPacketInitialPeriod is the initial period length used for packet number skipping to prevent an Optimistic ACK attack.
// Every time a packet number is skipped, the period is doubled, up to SkipPacketMaxPeriod.
const SkipPacketInitialPeriod PacketNumber = 256

// SkipPacketMaxPeriod is the maximum period length used for packet number skipping.
const SkipPacketMaxPeriod PacketNumber = 128 * 1024

// MaxAcceptQueueSize is the maximum number of connections that the server queues for accepting.
// If the queue is full, new connection attempts will be rejected.
const MaxAcceptQueueSize = 32

// TokenValidity is the duration that a (non-retry) token is considered valid
const TokenValidity = 24 * time.Hour

// RetryTokenValidity is the duration that a retry token is considered valid
const RetryTokenValidity = 10 * time.Second

// MaxOutstandingSentPackets is maximum number of packets saved for retransmission.
// When reached, it imposes a soft limit on sending new packets:
// Sending ACKs and retransmission is still allowed, but now new regular packets can be sent.
const MaxOutstandingSentPackets = 2 * MaxCongestionWindowPackets

// MaxTrackedSentPackets is maximum number of sent packets saved for retransmission.
// When reached, no more packets will be sent.
// This value *must* be larger than MaxOutstandingSentPackets.
const MaxTrackedSentPackets = MaxOutstandingSentPackets * 5 / 4

// MaxNonAckElicitingAcks is the maximum number of packets containing an ACK,
// but no ack-eliciting frames, that we send in a row
const MaxNonAckElicitingAcks = 19

// MaxStreamFrameSorterGaps is the maximum number of gaps between received StreamFrames
// prevents DoS attacks against the streamFrameSorter
const MaxStreamFrameSorterGaps = 1000

// MinStreamFrameBufferSize is the minimum data length of a received STREAM frame
// that we use the buffer for. This protects against a DoS where an attacker would send us
// very small STREAM frames to consume a lot of memory.
const MinStreamFrameBufferSize = 128

// MinCoalescedPacketSize is the minimum size of a coalesced packet that we pack.
// If a packet has less than this number of bytes, we won't coalesce any more packets onto it.
const MinCoalescedPacketSize = 128

// MaxCryptoStreamOffset is the maximum offset allowed on any of the crypto streams.
// This limits the size of the ClientHello and Certificates that can be received.
const MaxCryptoStreamOffset = 16 * (1 << 10)

// MinRemoteIdleTimeout is the minimum value that we accept for the remote idle timeout
const MinRemoteIdleTimeout = 5 * time.Second

// DefaultIdleTimeout is the default idle timeout
const DefaultIdleTimeout = 30 * time.Second

// DefaultHandshakeIdleTimeout is the default idle timeout used before handshake completion.
const DefaultHandshakeIdleTimeout = 5 * time.Second

// DefaultHandshakeTimeout is the default timeout for a connection until the crypto handshake succeeds.
const DefaultHandshakeTimeout = 10 * time.Second

// MaxKeepAliveInterval is the maximum time until we send a packet to keep a connection alive.
// It should be shorter than the time that NATs clear their mapping.
const MaxKeepAliveInterval = 20 * time.Second

// RetiredConnectionIDDeleteTimeout is the time we keep closed connections around in order to retransmit the CONNECTION_CLOSE.
// after this time all information about the old connection will be deleted
const RetiredConnectionIDDeleteTimeout = 5 * time.Second

// MinStreamFrameSize is the minimum size that has to be left in a packet, so that we add another STREAM frame.
// This avoids splitting up STREAM frames into small pieces, which has 2 advantages:
// 1. it reduces the framing overhead
// 2. it reduces the head-of-line blocking, when a packet is lost
const MinStreamFrameSize ByteCount = 128

// MaxPostHandshakeCryptoFrameSize is the maximum size of CRYPTO frames
// we send after the handshake completes.
const MaxPostHandshakeCryptoFrameSize = 1000

// MaxAckFrameSize is the maximum size for an ACK frame that we write
// Due to the varint encoding, ACK frames can grow (almost) indefinitely large.
// The MaxAckFrameSize should be large enough to encode many ACK range,
// but must ensure that a maximum size ACK frame fits into one packet.
const MaxAckFrameSize ByteCount = 1000

// MaxDatagramFrameSize is the maximum size of a DATAGRAM frame (RFC 9221).
// The size is chosen such that a DATAGRAM frame fits into a QUIC packet.
const MaxDatagramFrameSize ByteCount = 1220

// DatagramRcvQueueLen is the length of the receive queue for DATAGRAM frames (RFC 9221)
const DatagramRcvQueueLen = 128

// MaxNumAckRanges is the maximum number of ACK ranges that we send in an ACK frame.
// It also serves as a limit for the packet history.
// If at any point we keep track of more ranges, old ranges are discarded.
const MaxNumAckRanges = 32

// MinPacingDelay is the minimum duration that is used for packet pacing
// If the packet packing frequency is higher, multiple packets might be sent at once.
// Example: For a packet pacing delay of 200μs, we would send 5 packets at once, wait for 1ms, and so forth.
const MinPacingDelay = time.Millisecond

// DefaultConnectionIDLength is the connection ID length that is used for multiplexed connections
// if no other value is configured.
const DefaultConnectionIDLength = 4

// MaxActiveConnectionIDs is the number of connection IDs that we're storing.
const MaxActiveConnectionIDs = 4

// MaxIssuedConnectionIDs is the maximum number of connection IDs that we're issuing at the same time.
const MaxIssuedConnectionIDs = 6

// PacketsPerConnectionID is the number of packets we send using one connection ID.
// If the peer provices us with enough new connection IDs, we switch to a new connection ID.
const PacketsPerConnectionID = 10000

// AckDelayExponent is the ack delay exponent used when sending ACKs.
const AckDelayExponent = 3

// Estimated timer granularity.
// The loss detection timer will not be set to a value smaller than granularity.
const TimerGranularity = time.Millisecond

// MaxAckDelay is the maximum time by which we delay sending ACKs.
const MaxAckDelay = 25 * time.Millisecond

// MaxAckDelayInclGranularity is the max_ack_delay including the timer granularity.
// This is the value that should be advertised to the peer.
const MaxAckDelayInclGranularity = MaxAckDelay + TimerGranularity

// KeyUpdateInterval is the maximum number of packets we send or receive before initiating a key update.
const KeyUpdateInterval = 100 * 1000

// Max0RTTQueueingDuration is the maximum time that we store 0-RTT packets in order to wait for the corresponding Initial to be received.
const Max0RTTQueueingDuration = 100 * time.Millisecond

// Max0RTTQueues is the maximum number of connections that we buffer 0-RTT packets for.
const Max0RTTQueues = 32

// Max0RTTQueueLen is the maximum number of 0-RTT packets that we buffer for each connection.
// When a new connection is created, all buffered packets are passed to the connection immediately.
// To avoid blocking, this value has to be smaller than MaxConnUnprocessedPackets.
// To avoid packets being dropped as undecryptable by the connection, this value has to be smaller than MaxUndecryptablePackets.
const Max0RTTQueueLen = 31
<commit_msg>reduce max DATAGRAM frame size, so that DATAGRAMs fit in IPv6 packets (#3581)<commit_after>package protocol

import "time"

// DesiredReceiveBufferSize is the kernel UDP receive buffer size that we'd like to use.
const DesiredReceiveBufferSize = (1 << 20) * 2 // 2 MB

// InitialPacketSizeIPv4 is the maximum packet size that we use for sending IPv4 packets.
const InitialPacketSizeIPv4 = 1252

// InitialPacketSizeIPv6 is the maximum packet size that we use for sending IPv6 packets.
const InitialPacketSizeIPv6 = 1232

// MaxCongestionWindowPackets is the maximum congestion window in packet.
const MaxCongestionWindowPackets = 10000

// MaxUndecryptablePackets limits the number of undecryptable packets that are queued in the connection.
const MaxUndecryptablePackets = 32

// ConnectionFlowControlMultiplier determines how much larger the connection flow control windows needs to be relative to any stream's flow control window
// This is the value that Chromium is using
const ConnectionFlowControlMultiplier = 1.5

// DefaultInitialMaxStreamData is the default initial stream-level flow control window for receiving data
const DefaultInitialMaxStreamData = (1 << 10) * 512 // 512 kb

// DefaultInitialMaxData is the connection-level flow control window for receiving data
const DefaultInitialMaxData = ConnectionFlowControlMultiplier * DefaultInitialMaxStreamData

// DefaultMaxReceiveStreamFlowControlWindow is the default maximum stream-level flow control window for receiving data
const DefaultMaxReceiveStreamFlowControlWindow = 6 * (1 << 20) // 6 MB

// DefaultMaxReceiveConnectionFlowControlWindow is the default connection-level flow control window for receiving data
const DefaultMaxReceiveConnectionFlowControlWindow = 15 * (1 << 20) // 15 MB

// WindowUpdateThreshold is the fraction of the receive window that has to be consumed before an higher offset is advertised to the client
const WindowUpdateThreshold = 0.25

// DefaultMaxIncomingStreams is the maximum number of streams that a peer may open
const DefaultMaxIncomingStreams = 100

// DefaultMaxIncomingUniStreams is the maximum number of unidirectional streams that a peer may open
const DefaultMaxIncomingUniStreams = 100

// MaxServerUnprocessedPackets is the max number of packets stored in the server that are not yet processed.
const MaxServerUnprocessedPackets = 1024

// MaxConnUnprocessedPackets is the max number of packets stored in each connection that are not yet processed.
const MaxConnUnprocessedPackets = 256

// SkipPacketInitialPeriod is the initial period length used for packet number skipping to prevent an Optimistic ACK attack.
// Every time a packet number is skipped, the period is doubled, up to SkipPacketMaxPeriod.
const SkipPacketInitialPeriod PacketNumber = 256

// SkipPacketMaxPeriod is the maximum period length used for packet number skipping.
const SkipPacketMaxPeriod PacketNumber = 128 * 1024

// MaxAcceptQueueSize is the maximum number of connections that the server queues for accepting.
// If the queue is full, new connection attempts will be rejected.
const MaxAcceptQueueSize = 32

// TokenValidity is the duration that a (non-retry) token is considered valid
const TokenValidity = 24 * time.Hour

// RetryTokenValidity is the duration that a retry token is considered valid
const RetryTokenValidity = 10 * time.Second

// MaxOutstandingSentPackets is maximum number of packets saved for retransmission.
// When reached, it imposes a soft limit on sending new packets:
// Sending ACKs and retransmission is still allowed, but now new regular packets can be sent.
const MaxOutstandingSentPackets = 2 * MaxCongestionWindowPackets

// MaxTrackedSentPackets is maximum number of sent packets saved for retransmission.
// When reached, no more packets will be sent.
// This value *must* be larger than MaxOutstandingSentPackets.
const MaxTrackedSentPackets = MaxOutstandingSentPackets * 5 / 4

// MaxNonAckElicitingAcks is the maximum number of packets containing an ACK,
// but no ack-eliciting frames, that we send in a row
const MaxNonAckElicitingAcks = 19

// MaxStreamFrameSorterGaps is the maximum number of gaps between received StreamFrames
// prevents DoS attacks against the streamFrameSorter
const MaxStreamFrameSorterGaps = 1000

// MinStreamFrameBufferSize is the minimum data length of a received STREAM frame
// that we use the buffer for. This protects against a DoS where an attacker would send us
// very small STREAM frames to consume a lot of memory.
const MinStreamFrameBufferSize = 128

// MinCoalescedPacketSize is the minimum size of a coalesced packet that we pack.
// If a packet has less than this number of bytes, we won't coalesce any more packets onto it.
const MinCoalescedPacketSize = 128

// MaxCryptoStreamOffset is the maximum offset allowed on any of the crypto streams.
// This limits the size of the ClientHello and Certificates that can be received.
const MaxCryptoStreamOffset = 16 * (1 << 10)

// MinRemoteIdleTimeout is the minimum value that we accept for the remote idle timeout
const MinRemoteIdleTimeout = 5 * time.Second

// DefaultIdleTimeout is the default idle timeout
const DefaultIdleTimeout = 30 * time.Second

// DefaultHandshakeIdleTimeout is the default idle timeout used before handshake completion.
const DefaultHandshakeIdleTimeout = 5 * time.Second

// DefaultHandshakeTimeout is the default timeout for a connection until the crypto handshake succeeds.
const DefaultHandshakeTimeout = 10 * time.Second

// MaxKeepAliveInterval is the maximum time until we send a packet to keep a connection alive.
// It should be shorter than the time that NATs clear their mapping.
const MaxKeepAliveInterval = 20 * time.Second

// RetiredConnectionIDDeleteTimeout is the time we keep closed connections around in order to retransmit the CONNECTION_CLOSE.
// after this time all information about the old connection will be deleted
const RetiredConnectionIDDeleteTimeout = 5 * time.Second

// MinStreamFrameSize is the minimum size that has to be left in a packet, so that we add another STREAM frame.
// This avoids splitting up STREAM frames into small pieces, which has 2 advantages:
// 1. it reduces the framing overhead
// 2. it reduces the head-of-line blocking, when a packet is lost
const MinStreamFrameSize ByteCount = 128

// MaxPostHandshakeCryptoFrameSize is the maximum size of CRYPTO frames
// we send after the handshake completes.
const MaxPostHandshakeCryptoFrameSize = 1000

// MaxAckFrameSize is the maximum size for an ACK frame that we write
// Due to the varint encoding, ACK frames can grow (almost) indefinitely large.
// The MaxAckFrameSize should be large enough to encode many ACK range,
// but must ensure that a maximum size ACK frame fits into one packet.
const MaxAckFrameSize ByteCount = 1000

// MaxDatagramFrameSize is the maximum size of a DATAGRAM frame (RFC 9221).
// The size is chosen such that a DATAGRAM frame fits into a QUIC packet.
const MaxDatagramFrameSize ByteCount = 1200

// DatagramRcvQueueLen is the length of the receive queue for DATAGRAM frames (RFC 9221)
const DatagramRcvQueueLen = 128

// MaxNumAckRanges is the maximum number of ACK ranges that we send in an ACK frame.
// It also serves as a limit for the packet history.
// If at any point we keep track of more ranges, old ranges are discarded.
const MaxNumAckRanges = 32

// MinPacingDelay is the minimum duration that is used for packet pacing
// If the packet packing frequency is higher, multiple packets might be sent at once.
// Example: For a packet pacing delay of 200μs, we would send 5 packets at once, wait for 1ms, and so forth.
const MinPacingDelay = time.Millisecond

// DefaultConnectionIDLength is the connection ID length that is used for multiplexed connections
// if no other value is configured.
const DefaultConnectionIDLength = 4

// MaxActiveConnectionIDs is the number of connection IDs that we're storing.
const MaxActiveConnectionIDs = 4

// MaxIssuedConnectionIDs is the maximum number of connection IDs that we're issuing at the same time.
const MaxIssuedConnectionIDs = 6

// PacketsPerConnectionID is the number of packets we send using one connection ID.
// If the peer provices us with enough new connection IDs, we switch to a new connection ID.
const PacketsPerConnectionID = 10000

// AckDelayExponent is the ack delay exponent used when sending ACKs.
const AckDelayExponent = 3

// Estimated timer granularity.
// The loss detection timer will not be set to a value smaller than granularity.
const TimerGranularity = time.Millisecond

// MaxAckDelay is the maximum time by which we delay sending ACKs.
const MaxAckDelay = 25 * time.Millisecond

// MaxAckDelayInclGranularity is the max_ack_delay including the timer granularity.
// This is the value that should be advertised to the peer.
const MaxAckDelayInclGranularity = MaxAckDelay + TimerGranularity

// KeyUpdateInterval is the maximum number of packets we send or receive before initiating a key update.
const KeyUpdateInterval = 100 * 1000

// Max0RTTQueueingDuration is the maximum time that we store 0-RTT packets in order to wait for the corresponding Initial to be received.
const Max0RTTQueueingDuration = 100 * time.Millisecond

// Max0RTTQueues is the maximum number of connections that we buffer 0-RTT packets for.
const Max0RTTQueues = 32

// Max0RTTQueueLen is the maximum number of 0-RTT packets that we buffer for each connection.
// When a new connection is created, all buffered packets are passed to the connection immediately.
// To avoid blocking, this value has to be smaller than MaxConnUnprocessedPackets.
// To avoid packets being dropped as undecryptable by the connection, this value has to be smaller than MaxUndecryptablePackets.
const Max0RTTQueueLen = 31
<|endoftext|>