<commit_before>package schema1

import (
	"encoding/json"
	"time"

	"github.com/Microsoft/go-winio/pkg/guid"
	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
)

// ProcessConfig is used as both the input of Container.CreateProcess
// and to convert the parameters to JSON for passing onto the HCS
type ProcessConfig struct {
	ApplicationName   string            `json:",omitempty"`
	CommandLine       string            `json:",omitempty"`
	CommandArgs       []string          `json:",omitempty"` // Used by Linux Containers on Windows
	User              string            `json:",omitempty"`
	WorkingDirectory  string            `json:",omitempty"`
	Environment       map[string]string `json:",omitempty"`
	EmulateConsole    bool              `json:",omitempty"`
	CreateStdInPipe   bool              `json:",omitempty"`
	CreateStdOutPipe  bool              `json:",omitempty"`
	CreateStdErrPipe  bool              `json:",omitempty"`
	ConsoleSize       [2]uint           `json:",omitempty"`
	CreateInUtilityVm bool              `json:",omitempty"` // Used by Linux Containers on Windows
	OCISpecification  *json.RawMessage  `json:",omitempty"` // Used by Linux Containers on Windows
}

type Layer struct {
	ID   string
	Path string
}

type MappedDir struct {
	HostPath          string
	ContainerPath     string
	ReadOnly          bool
	BandwidthMaximum  uint64
	IOPSMaximum       uint64
	CreateInUtilityVM bool
	// LinuxMetadata - Support added in 1803/RS4+.
	LinuxMetadata bool `json:",omitempty"`
}

type MappedPipe struct {
	HostPath          string
	ContainerPipeName string
}

type HvRuntime struct {
	ImagePath           string `json:",omitempty"`
	SkipTemplate        bool   `json:",omitempty"`
	LinuxInitrdFile     string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM
	LinuxKernelFile     string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM
	LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode
	BootSource          string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD
	WritableBootSource  bool   `json:",omitempty"` // Linux Utility VM booting from VHD
}

type MappedVirtualDisk struct {
	HostPath          string `json:",omitempty"` // Path to VHD on the host
	ContainerPath     string // Platform-specific mount point path in the container
	CreateInUtilityVM bool   `json:",omitempty"`
	ReadOnly          bool   `json:",omitempty"`
	Cache             string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing"
	AttachOnly        bool   `json:",omitempty:`
}

// AssignedDevice represents a device that has been directly assigned to a container
//
// NOTE: Support added in RS5
type AssignedDevice struct {
	//  InterfaceClassGUID of the device to assign to container.
	InterfaceClassGUID string `json:"InterfaceClassGuid,omitempty"`
}

// ContainerConfig is used as both the input of CreateContainer
// and to convert the parameters to JSON for passing onto the HCS
type ContainerConfig struct {
	SystemType                  string              // HCS requires this to be hard-coded to "Container"
	Name                        string              // Name of the container. We use the docker ID.
	Owner                       string              `json:",omitempty"` // The management platform that created this container
	VolumePath                  string              `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID}
	IgnoreFlushesDuringBoot     bool                `json:",omitempty"` // Optimization hint for container startup in Windows
	LayerFolderPath             string              `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format  %root%\windowsfilter\containerID
	Layers                      []Layer             // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID
	Credentials                 string              `json:",omitempty"` // Credentials information
	ProcessorCount              uint32              `json:",omitempty"` // Number of processors to assign to the container.
	ProcessorWeight             uint64              `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares.
	ProcessorMaximum            int64               `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit.
	StorageIOPSMaximum          uint64              `json:",omitempty"` // Maximum Storage IOPS
	StorageBandwidthMaximum     uint64              `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
	StorageSandboxSize          uint64              `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
	MemoryMaximumInMB           int64               `json:",omitempty"` // Maximum memory available to the container in Megabytes
	HostName                    string              `json:",omitempty"` // Hostname
	MappedDirectories           []MappedDir         `json:",omitempty"` // List of mapped directories (volumes/mounts)
	MappedPipes                 []MappedPipe        `json:",omitempty"` // List of mapped Windows named pipes
	HvPartition                 bool                // True if it a Hyper-V Container
	NetworkSharedContainerName  string              `json:",omitempty"` // Name (ID) of the container that we will share the network stack with.
	EndpointList                []string            `json:",omitempty"` // List of networking endpoints to be attached to container
	HvRuntime                   *HvRuntime          `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
	Servicing                   bool                `json:",omitempty"` // True if this container is for servicing
	AllowUnqualifiedDNSQuery    bool                `json:",omitempty"` // True to allow unqualified DNS name resolution
	DNSSearchList               string              `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
	ContainerType               string              `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise.
	TerminateOnLastHandleClosed bool                `json:",omitempty"` // Should HCS terminate the container once all handles have been closed
	MappedVirtualDisks          []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start
	AssignedDevices             []AssignedDevice    `json:",omitempty"` // Array of devices to assign. NOTE: Support added in RS5
}

type ComputeSystemQuery struct {
	IDs    []string `json:"Ids,omitempty"`
	Types  []string `json:",omitempty"`
	Names  []string `json:",omitempty"`
	Owners []string `json:",omitempty"`
}

type PropertyType string

const (
	PropertyTypeStatistics        PropertyType = "Statistics"        // V1 and V2
	PropertyTypeProcessList                    = "ProcessList"       // V1 and V2
	PropertyTypeMappedVirtualDisk              = "MappedVirtualDisk" // Not supported in V2 schema call
	PropertyTypeGuestConnection                = "GuestConnection"   // V1 and V2. Nil return from HCS before RS5
)

type PropertyQuery struct {
	PropertyTypes []PropertyType `json:",omitempty"`
}

// ContainerProperties holds the properties for a container and the processes running in that container
type ContainerProperties struct {
	ID                           string `json:"Id"`
	State                        string
	Name                         string
	SystemType                   string
	RuntimeOSType                string `json:"RuntimeOsType,omitempty"`
	Owner                        string
	SiloGUID                     string                              `json:"SiloGuid,omitempty"`
	RuntimeID                    guid.GUID                           `json:"RuntimeId,omitempty"`
	IsRuntimeTemplate            bool                                `json:",omitempty"`
	RuntimeImagePath             string                              `json:",omitempty"`
	Stopped                      bool                                `json:",omitempty"`
	ExitType                     string                              `json:",omitempty"`
	AreUpdatesPending            bool                                `json:",omitempty"`
	ObRoot                       string                              `json:",omitempty"`
	Statistics                   Statistics                          `json:",omitempty"`
	ProcessList                  []ProcessListItem                   `json:",omitempty"`
	MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"`
	GuestConnectionInfo          GuestConnectionInfo                 `json:",omitempty"`
}

// MemoryStats holds the memory statistics for a container
type MemoryStats struct {
	UsageCommitBytes            uint64 `json:"MemoryUsageCommitBytes,omitempty"`
	UsageCommitPeakBytes        uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"`
	UsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"`
}

// ProcessorStats holds the processor statistics for a container
type ProcessorStats struct {
	TotalRuntime100ns  uint64 `json:",omitempty"`
	RuntimeUser100ns   uint64 `json:",omitempty"`
	RuntimeKernel100ns uint64 `json:",omitempty"`
}

// StorageStats holds the storage statistics for a container
type StorageStats struct {
	ReadCountNormalized  uint64 `json:",omitempty"`
	ReadSizeBytes        uint64 `json:",omitempty"`
	WriteCountNormalized uint64 `json:",omitempty"`
	WriteSizeBytes       uint64 `json:",omitempty"`
}

// NetworkStats holds the network statistics for a container
type NetworkStats struct {
	BytesReceived          uint64 `json:",omitempty"`
	BytesSent              uint64 `json:",omitempty"`
	PacketsReceived        uint64 `json:",omitempty"`
	PacketsSent            uint64 `json:",omitempty"`
	DroppedPacketsIncoming uint64 `json:",omitempty"`
	DroppedPacketsOutgoing uint64 `json:",omitempty"`
	EndpointId             string `json:",omitempty"`
	InstanceId             string `json:",omitempty"`
}

// Statistics is the structure returned by a statistics call on a container
type Statistics struct {
	Timestamp          time.Time      `json:",omitempty"`
	ContainerStartTime time.Time      `json:",omitempty"`
	Uptime100ns        uint64         `json:",omitempty"`
	Memory             MemoryStats    `json:",omitempty"`
	Processor          ProcessorStats `json:",omitempty"`
	Storage            StorageStats   `json:",omitempty"`
	Network            []NetworkStats `json:",omitempty"`
}

// ProcessList is the structure of an item returned by a ProcessList call on a container
type ProcessListItem struct {
	CreateTimestamp              time.Time `json:",omitempty"`
	ImageName                    string    `json:",omitempty"`
	KernelTime100ns              uint64    `json:",omitempty"`
	MemoryCommitBytes            uint64    `json:",omitempty"`
	MemoryWorkingSetPrivateBytes uint64    `json:",omitempty"`
	MemoryWorkingSetSharedBytes  uint64    `json:",omitempty"`
	ProcessId                    uint32    `json:",omitempty"`
	UserTime100ns                uint64    `json:",omitempty"`
}

// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container
type MappedVirtualDiskController struct {
	MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"`
}

// GuestDefinedCapabilities is part of the GuestConnectionInfo returned by a GuestConnection call on a utility VM
type GuestDefinedCapabilities struct {
	NamespaceAddRequestSupported bool `json:",omitempty"`
	SignalProcessSupported       bool `json:",omitempty"`
}

// GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM
type GuestConnectionInfo struct {
	SupportedSchemaVersions  []hcsschema.Version      `json:",omitempty"`
	ProtocolVersion          uint32                   `json:",omitempty"`
	GuestDefinedCapabilities GuestDefinedCapabilities `json:",omitempty"`
}

// Type of Request Support in ModifySystem
type RequestType string

// Type of Resource Support in ModifySystem
type ResourceType string

// RequestType const
const (
	Add     RequestType  = "Add"
	Remove  RequestType  = "Remove"
	Network ResourceType = "Network"
)

// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system
// Supported resource types are Network and Request Types are Add/Remove
type ResourceModificationRequestResponse struct {
	Resource ResourceType `json:"ResourceType"`
	Data     interface{}  `json:"Settings"`
	Request  RequestType  `json:"RequestType,omitempty"`
}
<commit_msg>Fixes a malformed JSON tag<commit_after>package schema1

import (
	"encoding/json"
	"time"

	"github.com/Microsoft/go-winio/pkg/guid"
	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
)

// ProcessConfig is used as both the input of Container.CreateProcess
// and to convert the parameters to JSON for passing onto the HCS
type ProcessConfig struct {
	ApplicationName   string            `json:",omitempty"`
	CommandLine       string            `json:",omitempty"`
	CommandArgs       []string          `json:",omitempty"` // Used by Linux Containers on Windows
	User              string            `json:",omitempty"`
	WorkingDirectory  string            `json:",omitempty"`
	Environment       map[string]string `json:",omitempty"`
	EmulateConsole    bool              `json:",omitempty"`
	CreateStdInPipe   bool              `json:",omitempty"`
	CreateStdOutPipe  bool              `json:",omitempty"`
	CreateStdErrPipe  bool              `json:",omitempty"`
	ConsoleSize       [2]uint           `json:",omitempty"`
	CreateInUtilityVm bool              `json:",omitempty"` // Used by Linux Containers on Windows
	OCISpecification  *json.RawMessage  `json:",omitempty"` // Used by Linux Containers on Windows
}

type Layer struct {
	ID   string
	Path string
}

type MappedDir struct {
	HostPath          string
	ContainerPath     string
	ReadOnly          bool
	BandwidthMaximum  uint64
	IOPSMaximum       uint64
	CreateInUtilityVM bool
	// LinuxMetadata - Support added in 1803/RS4+.
	LinuxMetadata bool `json:",omitempty"`
}

type MappedPipe struct {
	HostPath          string
	ContainerPipeName string
}

type HvRuntime struct {
	ImagePath           string `json:",omitempty"`
	SkipTemplate        bool   `json:",omitempty"`
	LinuxInitrdFile     string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM
	LinuxKernelFile     string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM
	LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode
	BootSource          string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD
	WritableBootSource  bool   `json:",omitempty"` // Linux Utility VM booting from VHD
}

type MappedVirtualDisk struct {
	HostPath          string `json:",omitempty"` // Path to VHD on the host
	ContainerPath     string // Platform-specific mount point path in the container
	CreateInUtilityVM bool   `json:",omitempty"`
	ReadOnly          bool   `json:",omitempty"`
	Cache             string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing"
	AttachOnly        bool   `json:",omitempty"`
}

// AssignedDevice represents a device that has been directly assigned to a container
//
// NOTE: Support added in RS5
type AssignedDevice struct {
	//  InterfaceClassGUID of the device to assign to container.
	InterfaceClassGUID string `json:"InterfaceClassGuid,omitempty"`
}

// ContainerConfig is used as both the input of CreateContainer
// and to convert the parameters to JSON for passing onto the HCS
type ContainerConfig struct {
	SystemType                  string              // HCS requires this to be hard-coded to "Container"
	Name                        string              // Name of the container. We use the docker ID.
	Owner                       string              `json:",omitempty"` // The management platform that created this container
	VolumePath                  string              `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID}
	IgnoreFlushesDuringBoot     bool                `json:",omitempty"` // Optimization hint for container startup in Windows
	LayerFolderPath             string              `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format  %root%\windowsfilter\containerID
	Layers                      []Layer             // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID
	Credentials                 string              `json:",omitempty"` // Credentials information
	ProcessorCount              uint32              `json:",omitempty"` // Number of processors to assign to the container.
	ProcessorWeight             uint64              `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares.
	ProcessorMaximum            int64               `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit.
	StorageIOPSMaximum          uint64              `json:",omitempty"` // Maximum Storage IOPS
	StorageBandwidthMaximum     uint64              `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
	StorageSandboxSize          uint64              `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
	MemoryMaximumInMB           int64               `json:",omitempty"` // Maximum memory available to the container in Megabytes
	HostName                    string              `json:",omitempty"` // Hostname
	MappedDirectories           []MappedDir         `json:",omitempty"` // List of mapped directories (volumes/mounts)
	MappedPipes                 []MappedPipe        `json:",omitempty"` // List of mapped Windows named pipes
	HvPartition                 bool                // True if it a Hyper-V Container
	NetworkSharedContainerName  string              `json:",omitempty"` // Name (ID) of the container that we will share the network stack with.
	EndpointList                []string            `json:",omitempty"` // List of networking endpoints to be attached to container
	HvRuntime                   *HvRuntime          `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
	Servicing                   bool                `json:",omitempty"` // True if this container is for servicing
	AllowUnqualifiedDNSQuery    bool                `json:",omitempty"` // True to allow unqualified DNS name resolution
	DNSSearchList               string              `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
	ContainerType               string              `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise.
	TerminateOnLastHandleClosed bool                `json:",omitempty"` // Should HCS terminate the container once all handles have been closed
	MappedVirtualDisks          []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start
	AssignedDevices             []AssignedDevice    `json:",omitempty"` // Array of devices to assign. NOTE: Support added in RS5
}

type ComputeSystemQuery struct {
	IDs    []string `json:"Ids,omitempty"`
	Types  []string `json:",omitempty"`
	Names  []string `json:",omitempty"`
	Owners []string `json:",omitempty"`
}

type PropertyType string

const (
	PropertyTypeStatistics        PropertyType = "Statistics"        // V1 and V2
	PropertyTypeProcessList                    = "ProcessList"       // V1 and V2
	PropertyTypeMappedVirtualDisk              = "MappedVirtualDisk" // Not supported in V2 schema call
	PropertyTypeGuestConnection                = "GuestConnection"   // V1 and V2. Nil return from HCS before RS5
)

type PropertyQuery struct {
	PropertyTypes []PropertyType `json:",omitempty"`
}

// ContainerProperties holds the properties for a container and the processes running in that container
type ContainerProperties struct {
	ID                           string `json:"Id"`
	State                        string
	Name                         string
	SystemType                   string
	RuntimeOSType                string `json:"RuntimeOsType,omitempty"`
	Owner                        string
	SiloGUID                     string                              `json:"SiloGuid,omitempty"`
	RuntimeID                    guid.GUID                           `json:"RuntimeId,omitempty"`
	IsRuntimeTemplate            bool                                `json:",omitempty"`
	RuntimeImagePath             string                              `json:",omitempty"`
	Stopped                      bool                                `json:",omitempty"`
	ExitType                     string                              `json:",omitempty"`
	AreUpdatesPending            bool                                `json:",omitempty"`
	ObRoot                       string                              `json:",omitempty"`
	Statistics                   Statistics                          `json:",omitempty"`
	ProcessList                  []ProcessListItem                   `json:",omitempty"`
	MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"`
	GuestConnectionInfo          GuestConnectionInfo                 `json:",omitempty"`
}

// MemoryStats holds the memory statistics for a container
type MemoryStats struct {
	UsageCommitBytes            uint64 `json:"MemoryUsageCommitBytes,omitempty"`
	UsageCommitPeakBytes        uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"`
	UsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"`
}

// ProcessorStats holds the processor statistics for a container
type ProcessorStats struct {
	TotalRuntime100ns  uint64 `json:",omitempty"`
	RuntimeUser100ns   uint64 `json:",omitempty"`
	RuntimeKernel100ns uint64 `json:",omitempty"`
}

// StorageStats holds the storage statistics for a container
type StorageStats struct {
	ReadCountNormalized  uint64 `json:",omitempty"`
	ReadSizeBytes        uint64 `json:",omitempty"`
	WriteCountNormalized uint64 `json:",omitempty"`
	WriteSizeBytes       uint64 `json:",omitempty"`
}

// NetworkStats holds the network statistics for a container
type NetworkStats struct {
	BytesReceived          uint64 `json:",omitempty"`
	BytesSent              uint64 `json:",omitempty"`
	PacketsReceived        uint64 `json:",omitempty"`
	PacketsSent            uint64 `json:",omitempty"`
	DroppedPacketsIncoming uint64 `json:",omitempty"`
	DroppedPacketsOutgoing uint64 `json:",omitempty"`
	EndpointId             string `json:",omitempty"`
	InstanceId             string `json:",omitempty"`
}

// Statistics is the structure returned by a statistics call on a container
type Statistics struct {
	Timestamp          time.Time      `json:",omitempty"`
	ContainerStartTime time.Time      `json:",omitempty"`
	Uptime100ns        uint64         `json:",omitempty"`
	Memory             MemoryStats    `json:",omitempty"`
	Processor          ProcessorStats `json:",omitempty"`
	Storage            StorageStats   `json:",omitempty"`
	Network            []NetworkStats `json:",omitempty"`
}

// ProcessList is the structure of an item returned by a ProcessList call on a container
type ProcessListItem struct {
	CreateTimestamp              time.Time `json:",omitempty"`
	ImageName                    string    `json:",omitempty"`
	KernelTime100ns              uint64    `json:",omitempty"`
	MemoryCommitBytes            uint64    `json:",omitempty"`
	MemoryWorkingSetPrivateBytes uint64    `json:",omitempty"`
	MemoryWorkingSetSharedBytes  uint64    `json:",omitempty"`
	ProcessId                    uint32    `json:",omitempty"`
	UserTime100ns                uint64    `json:",omitempty"`
}

// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container
type MappedVirtualDiskController struct {
	MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"`
}

// GuestDefinedCapabilities is part of the GuestConnectionInfo returned by a GuestConnection call on a utility VM
type GuestDefinedCapabilities struct {
	NamespaceAddRequestSupported bool `json:",omitempty"`
	SignalProcessSupported       bool `json:",omitempty"`
}

// GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM
type GuestConnectionInfo struct {
	SupportedSchemaVersions  []hcsschema.Version      `json:",omitempty"`
	ProtocolVersion          uint32                   `json:",omitempty"`
	GuestDefinedCapabilities GuestDefinedCapabilities `json:",omitempty"`
}

// Type of Request Support in ModifySystem
type RequestType string

// Type of Resource Support in ModifySystem
type ResourceType string

// RequestType const
const (
	Add     RequestType  = "Add"
	Remove  RequestType  = "Remove"
	Network ResourceType = "Network"
)

// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system
// Supported resource types are Network and Request Types are Add/Remove
type ResourceModificationRequestResponse struct {
	Resource ResourceType `json:"ResourceType"`
	Data     interface{}  `json:"Settings"`
	Request  RequestType  `json:"RequestType,omitempty"`
}
<|endoftext|>


<commit_before>// package tlsdialer contains a customized version of crypto/tls.Dial that
// allows control over whether or not to send the ServerName extension in the
// client handshake.
package tlsdialer

import (
	"crypto/x509"
	"fmt"
	"net"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/mtime"
	"github.com/getlantern/netx"
	"github.com/getlantern/ops"

	"github.com/refraction-networking/utls"
)

var (
	log = golog.LoggerFor("tlsdialer")
)

type timeoutError struct{}

func (timeoutError) Error() string   { return "tlsdialer: DialWithDialer timed out" }
func (timeoutError) Timeout() bool   { return true }
func (timeoutError) Temporary() bool { return true }

// Dialer is a configurable dialer that dials using tls
type Dialer struct {
	DoDial         func(net string, addr string, timeout time.Duration) (net.Conn, error)
	Timeout        time.Duration
	Network        string
	SendServerName bool
	ClientHelloID  tls.ClientHelloID
	Config         *tls.Config
}

// A tls.Conn along with timings for key steps in establishing that Conn
type ConnWithTimings struct {
	// Conn: the conn resulting from dialing
	Conn *tls.Conn
	// ResolutionTime: the amount of time it took to resolve the address
	ResolutionTime time.Duration
	// ConnectTime: the amount of time that it took to connect the socket
	ConnectTime time.Duration
	// HandshakeTime: the amount of time that it took to complete the TLS
	// handshake
	HandshakeTime time.Duration
	// ResolvedAddr: the address to which our dns lookup resolved
	ResolvedAddr *net.TCPAddr
	// VerifiedChains: like tls.ConnectionState.VerifiedChains
	VerifiedChains [][]*x509.Certificate
}

// Like crypto/tls.Dial, but with the ability to control whether or not to
// send the ServerName extension in client handshakes through the sendServerName
// flag.
//
// Note - if sendServerName is false, the VerifiedChains field on the
// connection's ConnectionState will never get populated. Use DialForTimings to
// get back a data structure that includes the verified chains.
func Dial(network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	return DialTimeout(netx.DialTimeout, 1*time.Minute, network, addr, sendServerName, config)
}

// Like Dial, but timing out after the given timeout.
func DialTimeout(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.Dial(network, addr)
}

// Like DialWithDialer but returns a data structure including timings and the
// verified chains.
func DialForTimings(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*ConnWithTimings, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.DialForTimings(network, addr)
}

// Dial dials the given network and address.
func (d *Dialer) Dial(network, addr string) (*tls.Conn, error) {
	cwt, err := d.DialForTimings(network, addr)
	if err != nil {
		return nil, err
	}
	return cwt.Conn, nil
}

// DialForTimings dials the given network and address and returns a ConnWithTimings.
func (d *Dialer) DialForTimings(network, addr string) (*ConnWithTimings, error) {
	result := &ConnWithTimings{}

	var errCh chan error

	if d.Timeout != 0 {
		// We want the Timeout and Deadline values to cover the whole process: TCP
		// connection and TLS handshake. This means that we also need to start our own
		// timers now.
		errCh = make(chan error, 10)
		time.AfterFunc(d.Timeout, func() {
			errCh <- timeoutError{}
		})
	}

	log.Tracef("Resolving addr: %s", addr)
	elapsed := mtime.Stopwatch()
	var err error
	if d.Timeout == 0 {
		log.Tracef("Resolving immediately")
		result.ResolvedAddr, err = netx.Resolve("tcp", addr)
	} else {
		log.Tracef("Resolving on goroutine")
		resolvedCh := make(chan *net.TCPAddr, 10)
		ops.Go(func() {
			resolved, resolveErr := netx.Resolve("tcp", addr)
			log.Tracef("Resolution resulted in %s : %s", resolved, resolveErr)
			resolvedCh <- resolved
			errCh <- resolveErr
		})
		err = <-errCh
		if err == nil {
			log.Tracef("No error, looking for resolved")
			result.ResolvedAddr = <-resolvedCh
		}
	}

	if err != nil {
		return result, err
	}
	result.ResolutionTime = elapsed()
	log.Tracef("Resolved addr %s to %s in %s", addr, result.ResolvedAddr, result.ResolutionTime)

	hostname, _, err := net.SplitHostPort(addr)
	if err != nil {
		return result, fmt.Errorf("Unable to split host and port for %v: %v", addr, err)
	}

	log.Tracef("Dialing %s %s (%s)", network, addr, result.ResolvedAddr)
	elapsed = mtime.Stopwatch()
	resolvedAddr := result.ResolvedAddr.String()
	rawConn, err := d.DoDial(network, resolvedAddr, d.Timeout)
	if err != nil {
		return result, err
	}
	result.ConnectTime = elapsed()
	log.Tracef("Dialed in %s", result.ConnectTime)

	config := d.Config
	if config == nil {
		config = &tls.Config{}
	}

	serverName := config.ServerName

	if serverName == "" {
		log.Trace("No ServerName set, inferring from the hostname to which we're connecting")
		serverName = hostname
	}
	log.Tracef("ServerName is: %s", serverName)

	log.Trace("Copying config so that we can tweak it")
	configCopy := new(tls.Config)
	*configCopy = *config

	if d.SendServerName {
		log.Tracef("Setting ServerName to %s and relying on the usual logic in tls.Conn.Handshake() to do verification", serverName)
		configCopy.ServerName = serverName
	} else {
		log.Trace("Clearing ServerName and disabling verification in tls.Conn.Handshake(). We'll verify manually after handshaking.")
		configCopy.ServerName = ""
		configCopy.InsecureSkipVerify = true
	}

	chid := d.ClientHelloID
	if chid.Browser == "" {
		log.Trace("Defaulting to typical Golang client hello")
		chid = tls.HelloGolang
	}
	conn := tls.UClient(rawConn, configCopy, chid)

	elapsed = mtime.Stopwatch()
	if d.Timeout == 0 {
		log.Trace("Handshaking immediately")
		err = conn.Handshake()
	} else {
		log.Trace("Handshaking on goroutine")
		ops.Go(func() {
			errCh <- conn.Handshake()
		})
		err = <-errCh
	}
	if err == nil {
		result.HandshakeTime = elapsed()
	}
	log.Tracef("Finished handshaking in: %s", result.HandshakeTime)

	if err == nil && !config.InsecureSkipVerify {
		if d.SendServerName {
			log.Trace("Depending on certificate verification in tls.Conn.Handshake()")
			result.VerifiedChains = conn.ConnectionState().VerifiedChains
		} else {
			log.Trace("Manually verifying certificates")
			configCopy.ServerName = ""
			result.VerifiedChains, err = verifyServerCerts(conn.Conn, serverName, configCopy)
		}
	}

	if err != nil {
		log.Tracef("Handshake or verification error, closing underlying connection: %v", err)
		if closeErr := rawConn.Close(); closeErr != nil {
			log.Debugf("Unable to close connection: %v", closeErr)
		}
		return result, err
	}

	result.Conn = conn.Conn
	return result, nil
}

func verifyServerCerts(conn *tls.Conn, serverName string, config *tls.Config) ([][]*x509.Certificate, error) {
	certs := conn.ConnectionState().PeerCertificates

	opts := x509.VerifyOptions{
		Roots:         config.RootCAs,
		CurrentTime:   time.Now(),
		DNSName:       serverName,
		Intermediates: x509.NewCertPool(),
	}

	for i, cert := range certs {
		if i == 0 {
			continue
		}
		opts.Intermediates.AddCert(cert)
	}
	return certs[0].Verify(opts)
}
<commit_msg>Logging remote address when tls portion of dialing fails<commit_after>// package tlsdialer contains a customized version of crypto/tls.Dial that
// allows control over whether or not to send the ServerName extension in the
// client handshake.
package tlsdialer

import (
	"crypto/x509"
	"fmt"
	"net"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/mtime"
	"github.com/getlantern/netx"
	"github.com/getlantern/ops"

	"github.com/refraction-networking/utls"
)

var (
	log = golog.LoggerFor("tlsdialer")
)

type timeoutError struct{}

func (timeoutError) Error() string   { return "tlsdialer: DialWithDialer timed out" }
func (timeoutError) Timeout() bool   { return true }
func (timeoutError) Temporary() bool { return true }

// Dialer is a configurable dialer that dials using tls
type Dialer struct {
	DoDial         func(net string, addr string, timeout time.Duration) (net.Conn, error)
	Timeout        time.Duration
	Network        string
	SendServerName bool
	ClientHelloID  tls.ClientHelloID
	Config         *tls.Config
}

// A tls.Conn along with timings for key steps in establishing that Conn
type ConnWithTimings struct {
	// Conn: the conn resulting from dialing
	Conn *tls.Conn
	// ResolutionTime: the amount of time it took to resolve the address
	ResolutionTime time.Duration
	// ConnectTime: the amount of time that it took to connect the socket
	ConnectTime time.Duration
	// HandshakeTime: the amount of time that it took to complete the TLS
	// handshake
	HandshakeTime time.Duration
	// ResolvedAddr: the address to which our dns lookup resolved
	ResolvedAddr *net.TCPAddr
	// VerifiedChains: like tls.ConnectionState.VerifiedChains
	VerifiedChains [][]*x509.Certificate
}

// Like crypto/tls.Dial, but with the ability to control whether or not to
// send the ServerName extension in client handshakes through the sendServerName
// flag.
//
// Note - if sendServerName is false, the VerifiedChains field on the
// connection's ConnectionState will never get populated. Use DialForTimings to
// get back a data structure that includes the verified chains.
func Dial(network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	return DialTimeout(netx.DialTimeout, 1*time.Minute, network, addr, sendServerName, config)
}

// Like Dial, but timing out after the given timeout.
func DialTimeout(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.Dial(network, addr)
}

// Like DialWithDialer but returns a data structure including timings and the
// verified chains.
func DialForTimings(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*ConnWithTimings, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.DialForTimings(network, addr)
}

// Dial dials the given network and address.
func (d *Dialer) Dial(network, addr string) (*tls.Conn, error) {
	cwt, err := d.DialForTimings(network, addr)
	if err != nil {
		return nil, err
	}
	return cwt.Conn, nil
}

// DialForTimings dials the given network and address and returns a ConnWithTimings.
func (d *Dialer) DialForTimings(network, addr string) (*ConnWithTimings, error) {
	result := &ConnWithTimings{}

	var errCh chan error

	if d.Timeout != 0 {
		// We want the Timeout and Deadline values to cover the whole process: TCP
		// connection and TLS handshake. This means that we also need to start our own
		// timers now.
		errCh = make(chan error, 10)
		time.AfterFunc(d.Timeout, func() {
			errCh <- timeoutError{}
		})
	}

	log.Tracef("Resolving addr: %s", addr)
	elapsed := mtime.Stopwatch()
	var err error
	if d.Timeout == 0 {
		log.Tracef("Resolving immediately")
		result.ResolvedAddr, err = netx.Resolve("tcp", addr)
	} else {
		log.Tracef("Resolving on goroutine")
		resolvedCh := make(chan *net.TCPAddr, 10)
		ops.Go(func() {
			resolved, resolveErr := netx.Resolve("tcp", addr)
			log.Tracef("Resolution resulted in %s : %s", resolved, resolveErr)
			resolvedCh <- resolved
			errCh <- resolveErr
		})
		err = <-errCh
		if err == nil {
			log.Tracef("No error, looking for resolved")
			result.ResolvedAddr = <-resolvedCh
		}
	}

	if err != nil {
		return result, err
	}
	result.ResolutionTime = elapsed()
	log.Tracef("Resolved addr %s to %s in %s", addr, result.ResolvedAddr, result.ResolutionTime)

	hostname, _, err := net.SplitHostPort(addr)
	if err != nil {
		return result, fmt.Errorf("Unable to split host and port for %v: %v", addr, err)
	}

	log.Tracef("Dialing %s %s (%s)", network, addr, result.ResolvedAddr)
	elapsed = mtime.Stopwatch()
	resolvedAddr := result.ResolvedAddr.String()
	rawConn, err := d.DoDial(network, resolvedAddr, d.Timeout)
	if err != nil {
		return result, err
	}
	result.ConnectTime = elapsed()
	log.Tracef("Dialed in %s", result.ConnectTime)

	config := d.Config
	if config == nil {
		config = &tls.Config{}
	}

	serverName := config.ServerName

	if serverName == "" {
		log.Trace("No ServerName set, inferring from the hostname to which we're connecting")
		serverName = hostname
	}
	log.Tracef("ServerName is: %s", serverName)

	log.Trace("Copying config so that we can tweak it")
	configCopy := new(tls.Config)
	*configCopy = *config

	if d.SendServerName {
		log.Tracef("Setting ServerName to %s and relying on the usual logic in tls.Conn.Handshake() to do verification", serverName)
		configCopy.ServerName = serverName
	} else {
		log.Trace("Clearing ServerName and disabling verification in tls.Conn.Handshake(). We'll verify manually after handshaking.")
		configCopy.ServerName = ""
		configCopy.InsecureSkipVerify = true
	}

	chid := d.ClientHelloID
	if chid.Browser == "" {
		log.Trace("Defaulting to typical Golang client hello")
		chid = tls.HelloGolang
	}
	conn := tls.UClient(rawConn, configCopy, chid)

	elapsed = mtime.Stopwatch()
	if d.Timeout == 0 {
		log.Trace("Handshaking immediately")
		err = conn.Handshake()
	} else {
		log.Trace("Handshaking on goroutine")
		ops.Go(func() {
			errCh <- conn.Handshake()
		})
		err = <-errCh
	}
	if err == nil {
		result.HandshakeTime = elapsed()
	}
	log.Tracef("Finished handshaking in: %s", result.HandshakeTime)

	if err == nil && !config.InsecureSkipVerify {
		if d.SendServerName {
			log.Trace("Depending on certificate verification in tls.Conn.Handshake()")
			result.VerifiedChains = conn.ConnectionState().VerifiedChains
		} else {
			log.Trace("Manually verifying certificates")
			configCopy.ServerName = ""
			result.VerifiedChains, err = verifyServerCerts(conn.Conn, serverName, configCopy)
		}
	}

	if err != nil {
		log.Tracef("Handshake or verification error, closing underlying connection: %v", err)
		if closeErr := rawConn.Close(); closeErr != nil {
			log.Debugf("Unable to close connection: %v", closeErr)
		}
		log.Errorf("Error establishing TLS connection to %v: %v", rawConn.RemoteAddr(), err)
		return result, err
	}

	result.Conn = conn.Conn
	return result, nil
}

func verifyServerCerts(conn *tls.Conn, serverName string, config *tls.Config) ([][]*x509.Certificate, error) {
	certs := conn.ConnectionState().PeerCertificates

	opts := x509.VerifyOptions{
		Roots:         config.RootCAs,
		CurrentTime:   time.Now(),
		DNSName:       serverName,
		Intermediates: x509.NewCertPool(),
	}

	for i, cert := range certs {
		if i == 0 {
			continue
		}
		opts.Intermediates.AddCert(cert)
	}
	return certs[0].Verify(opts)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"net/http"
	"time"

	"github.com/lomik/carbon-clickhouse/receiver"

	_ "net/http/pprof"
)

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	inCh := make(chan *receiver.Buffer, 128)
	outCh := make(chan *receiver.WriteBuffer, 128)

	workers := 4
	for i := 0; i < workers; i++ {
		go receiver.PlainParser(inCh, outCh)
	}

	// out channel cleaner
	go func() {
		for {
			w := <-outCh
			receiver.WriteBufferPool.Put(w)
		}
	}()

	rcv, err := receiver.New(
		"tcp://127.0.0.1:0",
		receiver.HandleBuffer(func(b *receiver.Buffer) { inCh <- b }),
	)
	defer rcv.Stop()

	if err != nil {
		log.Fatal(err)
	}

	conn, err := net.Dial("tcp", rcv.(*receiver.TCP).Addr().String())
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()

	var buf bytes.Buffer

	chunk := 50
	for i := 0; i < chunk; i++ {
		buf.Write([]byte("carbon.agents.localhost.cache.size 1412351 1479933789\n"))
	}

	printEvery := 1000000 / chunk

	body := buf.Bytes()

	cnt := 0
	t := time.Now()
	for {
		if _, err := conn.Write(body); err != nil {
			log.Fatal(err)
		}

		cnt++
		if cnt%printEvery == 0 {
			fmt.Printf("%.2f p/s\n", 1000000.0/time.Since(t).Seconds())
			t = time.Now()
		}
	}
}
<commit_msg>remove unused test tool<commit_after><|endoftext|>


<commit_before>package consul

import (
	"log"
	"strings"

	"github.com/hashicorp/consul/api"
)

// passingServices filters out health checks for services which have
// passing health checks and where the neither the service instance itself
// nor the node is in maintenance mode.
func passingServices(checks []*api.HealthCheck, status []string) []*api.HealthCheck {
	var p []*api.HealthCheck
	for _, svc := range checks {
		// first filter out non-service checks
		if svc.ServiceID == "" || svc.CheckID == "serfHealth" || svc.CheckID == "_node_maintenance" || strings.HasPrefix("_service_maintenance:", svc.CheckID) {
			continue
		}

		// then make sure the service health check is passing
		if !contains(status, svc.Status) {
			continue
		}

		// then check whether the agent is still alive and both the
		// node and the service are not in maintenance mode.
		for _, c := range checks {
			if c.CheckID == "serfHealth" && c.Node == svc.Node && c.Status == "critical" {
				log.Printf("[DEBUG] consul: Skipping service %q since agent on node %q is down: %s", svc.ServiceID, svc.Node, c.Output)
				goto skip
			}
			if c.CheckID == "_node_maintenance" && c.Node == svc.Node {
				log.Printf("[DEBUG] consul: Skipping service %q since node %q is in maintenance mode: %s", svc.ServiceID, svc.Node, c.Output)
				goto skip
			}
			if c.CheckID == "_service_maintenance:"+svc.ServiceID && c.Node == svc.Node && c.Status == "critical" {
				log.Printf("[DEBUG] consul: Skipping service %q since it is in maintenance mode: %s", svc.ServiceID, c.Output)
				goto skip
			}
		}

		p = append(p, svc)

	skip:
	}

	return p
}

func contains(slice []string, item string) bool {
	for _, a := range slice {
		if a == item {
			return true
		}
	}
	return false
}
<commit_msg>Simplify conditions in passing.go<commit_after>package consul

import (
	"log"
	"strings"

	"github.com/hashicorp/consul/api"
)

// passingServices filters out health checks for services which have
// passing health checks and where the neither the service instance itself
// nor the node is in maintenance mode.
func passingServices(checks []*api.HealthCheck, status []string) []*api.HealthCheck {
	var p []*api.HealthCheck
	for _, svc := range checks {
		// first filter out non-service checks
		if svc.ServiceID == "" || svc.CheckID == "serfHealth" || svc.CheckID == "_node_maintenance" || strings.HasPrefix("_service_maintenance:", svc.CheckID) {
			continue
		}

		// then make sure the service health check is passing
		if !contains(status, svc.Status) {
			continue
		}

		// then check whether the agent is still alive and both the
		// node and the service are not in maintenance mode.
		for _, c := range checks {
			if c.Node != svc.Node {
				continue
			}
			if c.CheckID == "serfHealth" && c.Status == "critical" {
				log.Printf("[DEBUG] consul: Skipping service %q since agent on node %q is down: %s", svc.ServiceID, svc.Node, c.Output)
				goto skip
			}
			if c.CheckID == "_node_maintenance" {
				log.Printf("[DEBUG] consul: Skipping service %q since node %q is in maintenance mode: %s", svc.ServiceID, svc.Node, c.Output)
				goto skip
			}
			if c.CheckID == "_service_maintenance:"+svc.ServiceID && c.Status == "critical" {
				log.Printf("[DEBUG] consul: Skipping service %q since it is in maintenance mode: %s", svc.ServiceID, c.Output)
				goto skip
			}
		}

		p = append(p, svc)

	skip:
	}

	return p
}

func contains(slice []string, item string) bool {
	for _, a := range slice {
		if a == item {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The log_client binary retrieves leaves from a log.
package main

import (
	"context"
	"flag"
	"fmt"
	"time"

	"github.com/google/trillian"
	"google.golang.org/grpc"
)

var (
	treeIDFlag     = flag.Int64("treeid", 3, "The tree id to use")
	serverPortFlag = flag.Int("port", 8090, "Log server port (must be on localhost)")
	startLeafFlag  = flag.Int64("start_leaf", 0, "The first leaf index to fetch")
	numLeavesFlag  = flag.Int64("num_leaves", 1, "The number of leaves to fetch")
)

func buildGetLeavesByIndexRequest(logID int64, startLeaf, numLeaves int64) *trillian.GetLeavesByIndexRequest {
	if startLeaf < 0 || numLeaves <= 0 {
		panic("Start leaf index and num_leaves must be >= 0")
	}

	var leafIndices []int64

	for l := int64(0); l < numLeaves; l++ {
		leafIndices = append(leafIndices, l+startLeaf)
	}

	return &trillian.GetLeavesByIndexRequest{LogId: logID, LeafIndex: leafIndices}
}

// TODO: Move this code out to a better place when we tidy up the initial test main stuff
// It's just a basic skeleton at the moment.
func main() {
	flag.Parse()

	// TODO: Other options apart from insecure connections
	port := *serverPortFlag
	conn, err := grpc.Dial(fmt.Sprintf("localhost:%d", port), grpc.WithInsecure(), grpc.WithTimeout(time.Second*5))

	if err != nil {
		panic(err)
	}

	defer conn.Close()

	client := trillian.NewTrillianLogClient(conn)

	req := buildGetLeavesByIndexRequest(*treeIDFlag, *startLeafFlag, *numLeavesFlag)
	getLeafByIndexResponse, err := client.GetLeavesByIndex(context.Background(), req)

	if err != nil {
		fmt.Printf("Got error in call: %v", err)
	} else {
		fmt.Printf("Got server response: %v", getLeafByIndexResponse)
	}
}
<commit_msg>Use grpc.DialContext() instead of deprecated grpc.WithTimeout()<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The log_client binary retrieves leaves from a log.
package main

import (
	"context"
	"flag"
	"fmt"
	"time"

	"github.com/google/trillian"
	"google.golang.org/grpc"
)

var (
	treeIDFlag     = flag.Int64("treeid", 3, "The tree id to use")
	serverPortFlag = flag.Int("port", 8090, "Log server port (must be on localhost)")
	startLeafFlag  = flag.Int64("start_leaf", 0, "The first leaf index to fetch")
	numLeavesFlag  = flag.Int64("num_leaves", 1, "The number of leaves to fetch")
)

func buildGetLeavesByIndexRequest(logID int64, startLeaf, numLeaves int64) *trillian.GetLeavesByIndexRequest {
	if startLeaf < 0 || numLeaves <= 0 {
		panic("Start leaf index and num_leaves must be >= 0")
	}

	var leafIndices []int64

	for l := int64(0); l < numLeaves; l++ {
		leafIndices = append(leafIndices, l+startLeaf)
	}

	return &trillian.GetLeavesByIndexRequest{LogId: logID, LeafIndex: leafIndices}
}

// TODO: Move this code out to a better place when we tidy up the initial test main stuff
// It's just a basic skeleton at the moment.
func main() {
	flag.Parse()

	ctx := context.Background()

	// TODO: Other options apart from insecure connections
	port := *serverPortFlag
	dialCtx, cancel := context.WithTimeout(ctx, time.Second*5)
	defer cancel()
	conn, err := grpc.DialContext(dialCtx, fmt.Sprintf("localhost:%d", port), grpc.WithInsecure())

	if err != nil {
		panic(err)
	}

	defer conn.Close()

	client := trillian.NewTrillianLogClient(conn)

	req := buildGetLeavesByIndexRequest(*treeIDFlag, *startLeafFlag, *numLeavesFlag)
	getLeafByIndexResponse, err := client.GetLeavesByIndex(ctx, req)

	if err != nil {
		fmt.Printf("Got error in call: %v", err)
	} else {
		fmt.Printf("Got server response: %v", getLeafByIndexResponse)
	}
}
<|endoftext|>


<commit_before>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testlapack

import (
	"archive/zip"
	"fmt"
	"io"
	"log"
	"testing"
)

type Dlaqr5er interface {
	Dlaqr5(wantt, wantz bool, kacc22 int, n, ktop, kbot, nshfts int, sr, si []float64, h []float64, ldh int, iloz, ihiz int, z []float64, ldz int, v []float64, ldv int, u []float64, ldu int, nh int, wh []float64, ldwh int, nv int, wv []float64, ldwv int)
}

func Dlaqr5Test(t *testing.T, impl Dlaqr5er) {
	r, err := zip.OpenReader("../internal/testdata/dlaqr5test/dlaqr5data.zip")
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()

	for _, f := range r.File {
		tc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		wantt, n, nshfts, ktop, kbot, sr, si, hOrig, hwant, zwant := readDlaqr5Case(tc)
		tc.Close()

		v := make([]float64, nshfts/2*3)
		u := make([]float64, (3*nshfts-3)*(3*nshfts-3))
		nh := n
		wh := make([]float64, (3*nshfts-3)*n)
		nv := n
		wv := make([]float64, n*(3*nshfts-3))

		for _, ldh := range []int{n, n + 1, n + 10} {
			h := make([]float64, (n-1)*ldh+n)
			for _, kacc22 := range []int{0, 1, 2} {
				copyMatrix(n, n, h, ldh, hOrig)
				z := eye(n, ldh)

				impl.Dlaqr5(wantt, true, kacc22,
					n, ktop, kbot,
					nshfts, sr, si,
					h, ldh,
					0, n-1, z, ldh,
					v, 3,
					u, 3*nshfts-3,
					nh, wh, nh,
					nv, wv, 3*nshfts-3)

				if !equalApprox(n, n, h, ldh, hwant, 1e-13) {
					t.Errorf("Case %v, kacc22=%v: unexpected matrix H\nh    =%v\nhwant=%v", f.Name, kacc22, h, hwant)
				}
				if !equalApprox(n, n, z, ldh, zwant, 1e-13) {
					t.Errorf("Case %v, kacc22=%v: unexpected matrix Z\nz    =%v\nzwant=%v", f.Name, kacc22, z, zwant)
				}
			}
		}
	}
}

// readDlaqr5Case reads and returns test data written by internal/testdata/dlaqr5test/main.go.
func readDlaqr5Case(r io.Reader) (wantt bool, n, nshfts, ktop, kbot int, sr, si []float64, h, hwant, zwant []float64) {
	_, err := fmt.Fscanln(r, &wantt, &n, &nshfts, &ktop, &kbot)
	if err != nil {
		log.Fatal(err)
	}

	sr = make([]float64, nshfts)
	si = make([]float64, nshfts)
	h = make([]float64, n*n)
	hwant = make([]float64, n*n)
	zwant = make([]float64, n*n)

	for i := range sr {
		_, err = fmt.Fscanln(r, &sr[i])
		if err != nil {
			log.Fatal(err)
		}
	}
	for i := range si {
		_, err = fmt.Fscanln(r, &si[i])
		if err != nil {
			log.Fatal(err)
		}
	}
	for i := 0; i < n; i++ {
		for j := 0; j < n; j++ {
			_, err = fmt.Fscanln(r, &h[i*n+j])
			if err != nil {
				log.Fatal(err)
			}
		}
	}
	for i := 0; i < n; i++ {
		for j := 0; j < n; j++ {
			_, err = fmt.Fscanln(r, &hwant[i*n+j])
			if err != nil {
				log.Fatal(err)
			}
		}
	}
	for i := 0; i < n; i++ {
		for j := 0; j < n; j++ {
			_, err = fmt.Fscanln(r, &zwant[i*n+j])
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	return wantt, n, nshfts, ktop, kbot, sr, si, h, hwant, zwant
}
<commit_msg>testlapack: update test for Dlaqr5 to read testing data in JSON format<commit_after>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testlapack

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"testing"
)

type Dlaqr5er interface {
	Dlaqr5(wantt, wantz bool, kacc22 int, n, ktop, kbot, nshfts int, sr, si []float64, h []float64, ldh int, iloz, ihiz int, z []float64, ldz int, v []float64, ldv int, u []float64, ldu int, nh int, wh []float64, ldwh int, nv int, wv []float64, ldwv int)
}

type Dlaqr5test struct {
	WantT          bool
	N              int
	NShifts        int
	KTop, KBot     int
	ShiftR, ShiftI []float64
	H              []float64

	HWant []float64
	ZWant []float64
}

func Dlaqr5Test(t *testing.T, impl Dlaqr5er) {
	file, err := os.Open(filepath.FromSlash("../testlapack/testdata/dlaqr5data.json.gz"))
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	r, err := gzip.NewReader(file)
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()

	var tests []Dlaqr5test
	json.NewDecoder(r).Decode(&tests)
	for _, test := range tests {
		wantt := test.WantT
		n := test.N
		nshfts := test.NShifts
		ktop := test.KTop
		kbot := test.KBot
		sr := test.ShiftR
		si := test.ShiftI

		v := make([]float64, nshfts/2*3)
		u := make([]float64, (3*nshfts-3)*(3*nshfts-3))
		nh := n
		wh := make([]float64, (3*nshfts-3)*n)
		nv := n
		wv := make([]float64, n*(3*nshfts-3))

		for _, ldh := range []int{n, n + 1, n + 10} {
			h := make([]float64, (n-1)*ldh+n)

			for _, kacc22 := range []int{0, 1, 2} {
				copyMatrix(n, n, h, ldh, test.H)
				z := eye(n, ldh)

				impl.Dlaqr5(wantt, true, kacc22,
					n, ktop, kbot,
					nshfts, sr, si,
					h, ldh,
					0, n-1, z, ldh,
					v, 3,
					u, 3*nshfts-3,
					nh, wh, nh,
					nv, wv, 3*nshfts-3)

				prefix := fmt.Sprintf("wantt=%v, n=%v, nshfts=%v, ktop=%v, kbot=%v, ldh=%v, kacc22=%v",
					wantt, n, nshfts, ktop, kbot, ldh, kacc22)
				if !equalApprox(n, n, h, ldh, test.HWant, 1e-13) {
					t.Errorf("Case %v: unexpected matrix H\nh    =%v\nhwant=%v", prefix, h, test.HWant)
				}
				if !equalApprox(n, n, z, ldh, test.ZWant, 1e-13) {
					t.Errorf("Case %v: unexpected matrix Z\nz    =%v\nzwant=%v", prefix, z, test.ZWant)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	"../dos"
	"../interpreter"
)

func cmd_pwd(cmd *interpreter.Interpreter) (interpreter.NextT, error) {
	wd, _ := dos.Getwd()
	fmt.Fprintln(cmd.Stdout, wd)
	return interpreter.CONTINUE, nil
}
<commit_msg>Implemented: pwd -N (N:digit)<commit_after>package commands

import (
	"fmt"
	"strconv"

	"../dos"
	. "../interpreter"
)

func cmd_pwd(cmd *Interpreter) (NextT, error) {
	if len(cmd.Args) >= 2 {
		if i, err := strconv.ParseInt(cmd.Args[1], 10, 0); err == nil && i < 0 {
			i += int64(len(cd_history))
			if i < 0 {
				return CONTINUE, fmt.Errorf("pwd %s: too old history", cmd.Args[1])
			}
			fmt.Fprintln(cmd.Stdout, cd_history[i])
			return CONTINUE, nil
		}
	} else {
		wd, _ := dos.Getwd()
		fmt.Fprintln(cmd.Stdout, wd)
	}
	return CONTINUE, nil
}
<|endoftext|>


<commit_before>package tls

import (
	"fmt"
	"os"
	"path"
)

const (
	// VolumePath is the path at which the tls cert and private key (if any)
	// will be mounted in the pachd pod
	VolumePath = "/pachd-tls-cert"

	// CertFile is the name of the mounted file containing a TLS certificate
	// that identifies pachd
	CertFile = "tls.crt"

	// KeyFile is the name of the mounted file containing a private key
	// corresponding to the public certificate in TLSCertFile
	KeyFile = "tls.key"
)

func GetCertPaths() (certPath string, keyPath string, err error) {
	certPath = path.Join(VolumePath, CertFile)
	if _, err = os.Stat(certPath); err != nil {
		err = fmt.Errorf("could not stat public cert at %s: %v", certPath, err)
		return
	}

	keyPath = path.Join(VolumePath, KeyFile)
	if _, err = os.Stat(keyPath); err != nil {
		err = fmt.Errorf("could not stat private key at %s: %v", keyPath, err)
		return
	}
	return
}
<commit_msg>Added docstring<commit_after>package tls

import (
	"fmt"
	"os"
	"path"
)

const (
	// VolumePath is the path at which the tls cert and private key (if any)
	// will be mounted in the pachd pod
	VolumePath = "/pachd-tls-cert"

	// CertFile is the name of the mounted file containing a TLS certificate
	// that identifies pachd
	CertFile = "tls.crt"

	// KeyFile is the name of the mounted file containing a private key
	// corresponding to the public certificate in TLSCertFile
	KeyFile = "tls.key"
)

// GetCertPaths gets the paths to the cert and key files within a cluster
func GetCertPaths() (certPath string, keyPath string, err error) {
	certPath = path.Join(VolumePath, CertFile)
	if _, err = os.Stat(certPath); err != nil {
		err = fmt.Errorf("could not stat public cert at %s: %v", certPath, err)
		return
	}

	keyPath = path.Join(VolumePath, KeyFile)
	if _, err = os.Stat(keyPath); err != nil {
		err = fmt.Errorf("could not stat private key at %s: %v", keyPath, err)
		return
	}
	return
}
<|endoftext|>


<commit_before>package gocql

import (
	"fmt"
	"math"
	"strconv"
	"sync/atomic"
)

// scyllaSupported represents Scylla connection options as sent in SUPPORTED
// frame.
type scyllaSupported struct {
	shard     int
	nrShards  int
	msbIgnore uint64
}

func parseSupported(supported map[string][]string) scyllaSupported {
	const (
		scyllaShard             = "SCYLLA_SHARD"
		scyllaNrShards          = "SCYLLA_NR_SHARDS"
		scyllaPartitioner       = "SCYLLA_PARTITIONER"
		scyllaShardingAlgorithm = "SCYLLA_SHARDING_ALGORITHM"
		scyllaShardingIgnoreMSB = "SCYLLA_SHARDING_IGNORE_MSB"
	)

	var (
		si  scyllaSupported
		err error
	)

	if s, ok := supported[scyllaShard]; ok {
		if si.shard, err = strconv.Atoi(s[0]); err != nil {
			if gocqlDebug {
				Logger.Printf("scylla: failed to parse %s value %v: %s", scyllaShard, s, err)
			}
		}
	}
	if s, ok := supported[scyllaNrShards]; ok {
		if si.nrShards, err = strconv.Atoi(s[0]); err != nil {
			if gocqlDebug {
				Logger.Printf("scylla: failed to parse %s value %v: %s", scyllaNrShards, s, err)
			}
		}
	}
	if s, ok := supported[scyllaShardingIgnoreMSB]; ok {
		if si.msbIgnore, err = strconv.ParseUint(s[0], 10, 64); err != nil {
			if gocqlDebug {
				Logger.Printf("scylla: failed to parse %s value %v: %s", scyllaShardingIgnoreMSB, s, err)
			}
		}
	}

	var (
		partitioner string
		algorithm   string
	)
	if s, ok := supported[scyllaPartitioner]; ok {
		partitioner = s[0]
	}
	if s, ok := supported[scyllaShardingAlgorithm]; ok {
		algorithm = s[0]
	}

	if partitioner != "org.apache.cassandra.dht.Murmur3Partitioner" || algorithm != "biased-token-round-robin" || si.nrShards == 0 || si.msbIgnore == 0 {
		if gocqlDebug {
			Logger.Printf("scylla: unsupported sharding configuration")
		}
		return scyllaSupported{}
	}

	return si
}

// isScyllaConn checks if conn is suitable for scyllaConnPicker.
func isScyllaConn(conn *Conn) bool {
	s := parseSupported(conn.supported)
	return s.nrShards != 0
}

// scyllaConnPicker is a specialised ConnPicker that selects connections based
// on token trying to get connection to a shard containing the given token.
type scyllaConnPicker struct {
	conns     []*Conn
	nrConns   int
	nrShards  int
	msbIgnore uint64
	pos       int32
}

func newScyllaConnPicker(conn *Conn) *scyllaConnPicker {
	s := parseSupported(conn.supported)
	if s.nrShards == 0 {
		panic(fmt.Sprintf("scylla: %s not a sharded connection", conn.Address()))
	}

	if gocqlDebug {
		Logger.Printf("scylla: %s sharding options %+v", conn.Address(), s)
	}

	return &scyllaConnPicker{
		nrShards:  s.nrShards,
		msbIgnore: s.msbIgnore,
	}
}

func (p *scyllaConnPicker) Remove(conn *Conn) {
	s := parseSupported(conn.supported)
	if s.nrShards == 0 {
		// It is possible for Remove to be called before the connection is added to the pool.
		// Ignoring these connections here is safe.
		if gocqlDebug {
			Logger.Printf("scylla: %s has unknown sharding state, ignoring it", conn.Address())
		}
		return
	}
	if gocqlDebug {
		Logger.Printf("scylla: %s remove shard %d connection", conn.Address(), s.shard)
	}

	if p.conns[s.shard] != nil {
		p.conns[s.shard] = nil
		p.nrConns--
	}
}

func (p *scyllaConnPicker) Close() {
	conns := p.conns
	p.conns = nil
	for _, conn := range conns {
		if conn != nil {
			conn.Close()
		}
	}
}

func (p *scyllaConnPicker) Size() (int, int) {
	return p.nrConns, p.nrShards - p.nrConns
}

func (p *scyllaConnPicker) Pick(t token) *Conn {
	if len(p.conns) == 0 {
		return nil
	}

	if t == nil {
		idx := int(atomic.AddInt32(&p.pos, 1))
		for i := 0; i < len(p.conns); i++ {
			if conn := p.conns[(idx+i)%len(p.conns)]; conn != nil {
				return conn
			}
		}
		return nil
	}

	mmt, ok := t.(murmur3Token)
	// double check if that's murmur3 token
	if !ok {
		return nil
	}

	idx := p.shardOf(mmt)
	return p.conns[idx]
}

func (p *scyllaConnPicker) shardOf(token murmur3Token) int {
	shards := uint64(p.nrShards)
	z := uint64(token+math.MinInt64) << p.msbIgnore
	lo := z & 0xffffffff
	hi := (z >> 32) & 0xffffffff
	mul1 := lo * shards
	mul2 := hi * shards
	sum := (mul1 >> 32) + mul2
	return int(sum >> 32)
}

func (p *scyllaConnPicker) Put(conn *Conn) {
	s := parseSupported(conn.supported)
	if s.nrShards == 0 {
		panic(fmt.Sprintf("scylla: %s not a sharded connection", conn.Address()))
	}

	if s.nrShards != len(p.conns) {
		if s.nrShards != p.nrShards {
			panic(fmt.Sprintf("scylla: %s invalid number of shards", conn.Address()))
		}
		conns := p.conns
		p.conns = make([]*Conn, s.nrShards, s.nrShards)
		copy(p.conns, conns)
	}
	if c := p.conns[s.shard]; c != nil {
		conn.Close()
		return
	}
	p.conns[s.shard] = conn
	p.nrConns++
	if gocqlDebug {
		Logger.Printf("scylla: %s put shard %d connection total: %d missing: %d", conn.Address(), s.shard, p.nrConns, p.nrShards-p.nrConns)
	}
}
<commit_msg>scylla: keep at least one connection by shard<commit_after>package gocql

import (
	"fmt"
	"math"
	"strconv"
	"sync"
	"sync/atomic"
)

// scyllaSupported represents Scylla connection options as sent in SUPPORTED
// frame.
type scyllaSupported struct {
	shard     int
	nrShards  int
	msbIgnore uint64
}

func parseSupported(supported map[string][]string) scyllaSupported {
	const (
		scyllaShard             = "SCYLLA_SHARD"
		scyllaNrShards          = "SCYLLA_NR_SHARDS"
		scyllaPartitioner       = "SCYLLA_PARTITIONER"
		scyllaShardingAlgorithm = "SCYLLA_SHARDING_ALGORITHM"
		scyllaShardingIgnoreMSB = "SCYLLA_SHARDING_IGNORE_MSB"
	)

	var (
		si  scyllaSupported
		err error
	)

	if s, ok := supported[scyllaShard]; ok {
		if si.shard, err = strconv.Atoi(s[0]); err != nil {
			if gocqlDebug {
				Logger.Printf("scylla: failed to parse %s value %v: %s", scyllaShard, s, err)
			}
		}
	}
	if s, ok := supported[scyllaNrShards]; ok {
		if si.nrShards, err = strconv.Atoi(s[0]); err != nil {
			if gocqlDebug {
				Logger.Printf("scylla: failed to parse %s value %v: %s", scyllaNrShards, s, err)
			}
		}
	}
	if s, ok := supported[scyllaShardingIgnoreMSB]; ok {
		if si.msbIgnore, err = strconv.ParseUint(s[0], 10, 64); err != nil {
			if gocqlDebug {
				Logger.Printf("scylla: failed to parse %s value %v: %s", scyllaShardingIgnoreMSB, s, err)
			}
		}
	}

	var (
		partitioner string
		algorithm   string
	)
	if s, ok := supported[scyllaPartitioner]; ok {
		partitioner = s[0]
	}
	if s, ok := supported[scyllaShardingAlgorithm]; ok {
		algorithm = s[0]
	}

	if partitioner != "org.apache.cassandra.dht.Murmur3Partitioner" || algorithm != "biased-token-round-robin" || si.nrShards == 0 || si.msbIgnore == 0 {
		if gocqlDebug {
			Logger.Printf("scylla: unsupported sharding configuration")
		}
		return scyllaSupported{}
	}

	return si
}

// isScyllaConn checks if conn is suitable for scyllaConnPicker.
func isScyllaConn(conn *Conn) bool {
	s := parseSupported(conn.supported)
	return s.nrShards != 0
}

// scyllaConnPicker is a specialised ConnPicker that selects connections based
// on token trying to get connection to a shard containing the given token.
type scyllaConnPicker struct {
	muConnsPerShard sync.RWMutex
	connsPerShard   [][]*Conn

	nrConns   int
	nrShards  int
	msbIgnore uint64
	pos       int32
}

func newScyllaConnPicker(conn *Conn) *scyllaConnPicker {
	s := parseSupported(conn.supported)
	if s.nrShards == 0 {
		panic(fmt.Sprintf("scylla: %s not a sharded connection", conn.Address()))
	}

	if gocqlDebug {
		Logger.Printf("scylla: %s sharding options %+v", conn.Address(), s)
	}
	v := &scyllaConnPicker{
		nrShards:      s.nrShards,
		msbIgnore:     s.msbIgnore,
		connsPerShard: make([][]*Conn, s.nrShards),
	}
	return v
}

func (p *scyllaConnPicker) Remove(conn *Conn) {
	s := parseSupported(conn.supported)
	if s.nrShards == 0 {
		// It is possible for Remove to be called before the connection is added to the pool.
		// Ignoring these connections here is safe.
		if gocqlDebug {
			Logger.Printf("scylla: %s has unknown sharding state, ignoring it", conn.Address())
		}
		return
	}
	if gocqlDebug {
		Logger.Printf("scylla: %s remove shard %d connection", conn.Address(), s.shard)
	}

	p.muConnsPerShard.Lock()
	if len(p.connsPerShard[s.shard]) > 0 {
		p.nrConns--
	}
	p.connsPerShard[s.shard] = p.connsPerShard[s.shard][:]
	p.muConnsPerShard.Unlock()
}

func (p *scyllaConnPicker) Close() {
	connsPerShard := p.connsPerShard
	p.connsPerShard = nil

	p.muConnsPerShard.RLock()
	for _, conns := range connsPerShard {
		for _, c := range conns {
			c.Close()
		}
	}
	p.muConnsPerShard.RUnlock()
}

func (p *scyllaConnPicker) Size() (int, int) {
	return p.nrConns, p.nrShards - p.nrConns
}

func (p *scyllaConnPicker) Pick(t token) *Conn {
	if p.nrConns == 0 {
		return nil
	}

	if t == nil {
		idx := int(atomic.AddInt32(&p.pos, 1))
		conn := (*Conn)(nil)
		p.muConnsPerShard.RLock()
		for i := range p.connsPerShard {
			conns := p.connsPerShard[(idx+i)%len(p.connsPerShard)]
			if len(conns) > 0 {
				conn = conns[0]
				break
			}
		}
		p.muConnsPerShard.RUnlock()
		return conn
	}

	mmt, ok := t.(murmur3Token)
	// double check if that's murmur3 token
	if !ok {
		return nil
	}

	idx := p.shardOf(mmt)

	conn := (*Conn)(nil)
	p.muConnsPerShard.RLock()
	conns := p.connsPerShard[idx]
	if len(conns) > 0 {
		conn = conns[0]
	}
	p.muConnsPerShard.RUnlock()
	return conn
}

func (p *scyllaConnPicker) shardOf(token murmur3Token) int {
	shards := uint64(p.nrShards)
	z := uint64(token+math.MinInt64) << p.msbIgnore
	lo := z & 0xffffffff
	hi := (z >> 32) & 0xffffffff
	mul1 := lo * shards
	mul2 := hi * shards
	sum := (mul1 >> 32) + mul2
	return int(sum >> 32)
}

func (p *scyllaConnPicker) Put(conn *Conn) {
	s := parseSupported(conn.supported)
	if s.nrShards == 0 {
		panic(fmt.Sprintf("scylla: %s not a sharded connection", conn.Address()))
	}
	if s.nrShards != p.nrShards {
		panic(fmt.Sprintf("scylla: %s invalid number of shards", conn.Address()))
	}

	p.muConnsPerShard.Lock()
	conns := p.connsPerShard[s.shard]
	wasEmpty := len(conns) == 0
	conns = append(conns, conn)
	p.connsPerShard[s.shard] = conns
	if wasEmpty {
		p.nrConns++
	}
	if p.nrConns == p.nrShards {
		for i, conns := range p.connsPerShard {
			for _, c := range conns[1:] {
				c.Close()
			}
			p.connsPerShard[i] = conns[:1] // keep the first connection
		}
	}
	p.muConnsPerShard.Unlock()

	if gocqlDebug {
		Logger.Printf("scylla: %s put shard %d connection total: %d missing: %d", conn.Address(), s.shard, p.nrConns, p.nrShards-p.nrConns)
	}
}
<|endoftext|>


<commit_before>package crud

import (
	"fmt"
	"strconv"
	"strings"
)

//JoinCon join条件
type JoinCon struct {
	TableName string
	Condition string
}

//JoinCons join条件slice
type JoinCons []JoinCon

//HaveTable join条件中是否已经添加了这张表的join
func (jc JoinCons) HaveTable(tableName string) bool {
	for _, v := range jc {
		if v.TableName == tableName {
			return true
		}
	}
	return false
}

//WhereCon where条件
type WhereCon struct {
	Query string
	Args  []interface{}
}

//Search 搜索结构体
type Search struct {
	table             *Table
	fields            []string
	tableName         string
	joinConditions    JoinCons
	whereConditions   []WhereCon
	orderbyConditions []string
	group             string
	with              string
	having            string
	limit             interface{}
	offset            interface{}

	query string
	args  []interface{}
	raw   bool
}

//Clone 克隆一个当前结构体
func (s *Search) Clone() *Search {
	clone := *s
	return &clone
}

//Fields 需要查询的字段
func (s *Search) Fields(args ...string) *Search {
	s.fields = append(s.fields, args...)
	return s
}

//Where where语法
func (s *Search) Where(query string, values ...interface{}) *Search {
	s.whereConditions = append(s.whereConditions, WhereCon{Query: query, Args: values})
	return s
}

//WhereID id = ?
func (s *Search) WhereID(id interface{}) *Search {
	s.whereConditions = append(s.whereConditions, WhereCon{Query: s.tableName + ".id = ?", Args: []interface{}{id}})
	return s
}

//In in语法
func (s *Search) In(field string, args ...interface{}) *Search {
	//In没有参数的话SQL就会报错
	if len(args) == 0 {
		return s
	}
	s.whereConditions = append(s.whereConditions, WhereCon{Query: fmt.Sprintf("%s IN (%s)", field, placeholder(len(args))), Args: args})
	return s
}

//Joins join语法，自动连表。
func (s *Search) Joins(tablename string, condition ...string) *Search {
	if len(condition) == 1 {
		s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: condition[0]})
	} else {
		if s.table.tableColumns[tablename].HaveColumn(s.tableName + "id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", tablename, s.tableName+"id", s.tableName)})
		} else if s.table.tableColumns[tablename].HaveColumn(s.tableName + "_id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", tablename, s.tableName+"_id", s.tableName)})
		} else if s.table.tableColumns[s.tableName].HaveColumn(tablename + "id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", s.tableName, tablename+"id", tablename)})
		} else if s.table.tableColumns[s.tableName].HaveColumn(tablename + "_id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", s.tableName, tablename+"_id", tablename)})
		}
	}
	return s
}

//OrderBy OrderBy
func (s *Search) OrderBy(field string, isDESC ...bool) *Search {
	if len(isDESC) > 0 && isDESC[0] {
		s.orderbyConditions = append(s.orderbyConditions, field+" DESC")
	} else {
		s.orderbyConditions = append(s.orderbyConditions, field+" ASC")
	}
	return s
}

//TableName tableName
func (s *Search) TableName(name string) *Search {
	s.tableName = name
	return s
}

//Limit LIMIT ?
func (s *Search) Limit(limit interface{}) *Search {
	s.limit = limit
	return s
}

//Offset OFFSET ?
func (s *Search) Offset(offset interface{}) *Search {
	s.offset = offset
	return s
}

//Group GROUP BY
func (s *Search) Group(query string) *Search {
	s.group = query
	return s
}

//Parse 将各个条件整合成可以查询的SQL语句和参数
func (s *Search) Parse() (string, []interface{}) {
	if s.raw == true {
		return s.query, s.args
	}
	var (
		fields       string
		joins        string
		paddingwhere string
		wheres       []string
		orderby      string
		limit        string
		offset       string
	)
	if s.table.tableColumns[s.tableName].HaveColumn(IsDeleted) {
		s.Where("is_deleted = ?", 0)
	}
	s.query = ""
	s.args = []interface{}{}
	if len(s.fields) == 0 {
		fields = "*"
	} else {
		for i := 0; i < len(s.fields); i++ {
			var tableName string
			s.fields[i], tableName, _ = s.warpField(s.fields[i])
			if tableName != s.tableName {
				if !s.joinConditions.HaveTable(tableName) {
					s.Joins(tableName)
				}
			}
		}
		fields = strings.Join(s.fields, ",")
	}
	for _, joincon := range s.joinConditions {
		joins += fmt.Sprintf(" LEFT JOIN %s ON %s", joincon.TableName, joincon.Condition)
	}
	for _, wherecon := range s.whereConditions {
		paddingwhere = " WHERE "
		wheres = append(wheres, wherecon.Query)
		s.args = append(s.args, wherecon.Args...)
	}
	if len(s.orderbyConditions) > 0 {
		orderby = " ORDER BY " + strings.Join(s.orderbyConditions, ",")
	}
	if s.limit != nil {
		limit = " LIMIT ?"
		s.args = append(s.args, s.limit)
	}
	if s.offset != nil {
		offset = " OFFSET ?"
		s.args = append(s.args, s.offset)
	}
	s.query = fmt.Sprintf("SELECT %s FROM `%s`%s%s%s%s%s%s", fields, s.tableName, joins, paddingwhere, strings.Join(wheres, " AND "), orderby, limit, offset)
	s.raw = true
	return s.query, s.args
}

//DISTINCT XX
//DISTICT XXX.XXX AS aaa
//XXX.XXX AS aaa
func (s *Search) warpField(field string) (warpStr string, tablename string, fieldname string) {
	if strings.Contains(field, " ") {
		if strings.Contains(field, "AS") {
			sp := strings.Split(field, " ")
			for i := 0; i < len(sp); i++ {
				if sp[i] == "AS" {
					sp[i-1], tablename, fieldname = s.warpFieldSingel(sp[i-1])
					warpStr = strings.Join(sp, " ")
					break
				}
			}
		} else {
			sp := strings.Split(field, " ")
			sp[len(sp)-1], tablename, fieldname = s.warpFieldSingel(sp[len(sp)-1])
			warpStr = strings.Join(sp, " ")
		}
	} else {
		return s.warpFieldSingel(field)
	}
	return
}

func (s *Search) warpFieldSingel(field string) (warpStr string, tablename string, fieldname string) {

	if strings.Contains(field, ".") {
		sp := strings.Split(field, ".")
		tablename = sp[0]
		fieldname = sp[1]
		if strings.Contains(field, "`") {
			warpStr = field
			return
		}
		warpStr = strings.Replace(field, ".", ".`", 1) + "`"
	} else {
		tablename = s.tableName
		fieldname = field
		if strings.Contains(field, "`") {
			warpStr = field
			return
		}
		switch field {
		case "*", "COUNT(*)":
			warpStr = field
		default:
			warpStr = "`" + field + "`"
		}
	}
	return
}

//结果展示

//RawMap RawMap
func (s *Search) RawMap() RowMap {
	return s.RowMap()
}

//RawsMap RawsMap
func (s *Search) RawsMap() RowsMap {
	return s.RowsMap()
}

//RawsMapInterface RawsMapInterface
func (s *Search) RawsMapInterface() RowsMapInterface {
	return s.RowsMapInterface()
}

//RowMap RowMap
func (s *Search) RowMap() RowMap {
	query, args := s.Parse()
	return s.table.Query(query, args...).RowMap()
}

//RowsMap RowsMap
func (s *Search) RowsMap() RowsMap {
	query, args := s.Parse()
	return s.table.Query(query, args...).RowsMap()
}

//RowsMapInterface RowsMapInterface
func (s *Search) RowsMapInterface() RowsMapInterface {
	query, args := s.Parse()
	return s.table.Query(query, args...).RowsMapInterface()
}

//DoubleSlice DoubleSlice
func (s *Search) DoubleSlice() (map[string]int, [][]string) {
	query, args := s.Parse()
	return s.table.Query(query, args...).DoubleSlice()
}

//Int 如果指定字段，则返回指定字段的int值，否则返回第一个字段作为int值返回。
func (s *Search) Int(args ...string) int {
	row := s.RowMap()
	if len(args) == 0 {
		for _, v := range row {
			i, _ := strconv.Atoi(v)
			return i
		}
	} else {
		i, _ := strconv.Atoi(row[args[0]])
		return i
	}
	return 0
}

//String like int
func (s *Search) String(args ...string) string {
	row := s.RowMap()
	if len(args) == 0 {
		for _, v := range row {
			return v
		}
	} else {
		return row[args[0]]
	}
	return ""
}

//Finds 将查询的结构放入到结构体当中
func (s *Search) Finds(v interface{}) {
	query, args := s.Parse()
	s.table.FindAll(v, append([]interface{}{query}, args...)...)
}

//Count 计算这次查询结果的个数
func (s *Search) Count() int {
	var count int
	s.fields = []string{"COUNT(*)"}
	query, args := s.Parse()
	s.table.Query(query, args...).Find(&count)
	return count
}
<commit_msg>export SqlRows<commit_after>package crud

import (
	"fmt"
	"strconv"
	"strings"
)

//JoinCon join条件
type JoinCon struct {
	TableName string
	Condition string
}

//JoinCons join条件slice
type JoinCons []JoinCon

//HaveTable join条件中是否已经添加了这张表的join
func (jc JoinCons) HaveTable(tableName string) bool {
	for _, v := range jc {
		if v.TableName == tableName {
			return true
		}
	}
	return false
}

//WhereCon where条件
type WhereCon struct {
	Query string
	Args  []interface{}
}

//Search 搜索结构体
type Search struct {
	table             *Table
	fields            []string
	tableName         string
	joinConditions    JoinCons
	whereConditions   []WhereCon
	orderbyConditions []string
	group             string
	with              string
	having            string
	limit             interface{}
	offset            interface{}

	query string
	args  []interface{}
	raw   bool
}

//Clone 克隆一个当前结构体
func (s *Search) Clone() *Search {
	clone := *s
	return &clone
}

//Fields 需要查询的字段
func (s *Search) Fields(args ...string) *Search {
	s.fields = append(s.fields, args...)
	return s
}

//Where where语法
func (s *Search) Where(query string, values ...interface{}) *Search {
	s.whereConditions = append(s.whereConditions, WhereCon{Query: query, Args: values})
	return s
}

//WhereID id = ?
func (s *Search) WhereID(id interface{}) *Search {
	s.whereConditions = append(s.whereConditions, WhereCon{Query: s.tableName + ".id = ?", Args: []interface{}{id}})
	return s
}

//In in语法
func (s *Search) In(field string, args ...interface{}) *Search {
	//In没有参数的话SQL就会报错
	if len(args) == 0 {
		return s
	}
	s.whereConditions = append(s.whereConditions, WhereCon{Query: fmt.Sprintf("%s IN (%s)", field, placeholder(len(args))), Args: args})
	return s
}

//Joins join语法，自动连表。
func (s *Search) Joins(tablename string, condition ...string) *Search {
	if len(condition) == 1 {
		s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: condition[0]})
	} else {
		if s.table.tableColumns[tablename].HaveColumn(s.tableName + "id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", tablename, s.tableName+"id", s.tableName)})
		} else if s.table.tableColumns[tablename].HaveColumn(s.tableName + "_id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", tablename, s.tableName+"_id", s.tableName)})
		} else if s.table.tableColumns[s.tableName].HaveColumn(tablename + "id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", s.tableName, tablename+"id", tablename)})
		} else if s.table.tableColumns[s.tableName].HaveColumn(tablename + "_id") {
			s.joinConditions = append(s.joinConditions, JoinCon{TableName: tablename, Condition: fmt.Sprintf("%s.%s = %s.id", s.tableName, tablename+"_id", tablename)})
		}
	}
	return s
}

//OrderBy OrderBy
func (s *Search) OrderBy(field string, isDESC ...bool) *Search {
	if len(isDESC) > 0 && isDESC[0] {
		s.orderbyConditions = append(s.orderbyConditions, field+" DESC")
	} else {
		s.orderbyConditions = append(s.orderbyConditions, field+" ASC")
	}
	return s
}

//TableName tableName
func (s *Search) TableName(name string) *Search {
	s.tableName = name
	return s
}

//Limit LIMIT ?
func (s *Search) Limit(limit interface{}) *Search {
	s.limit = limit
	return s
}

//Offset OFFSET ?
func (s *Search) Offset(offset interface{}) *Search {
	s.offset = offset
	return s
}

//Group GROUP BY
func (s *Search) Group(query string) *Search {
	s.group = query
	return s
}

//Parse 将各个条件整合成可以查询的SQL语句和参数
func (s *Search) Parse() (string, []interface{}) {
	if s.raw == true {
		return s.query, s.args
	}
	var (
		fields       string
		joins        string
		paddingwhere string
		wheres       []string
		orderby      string
		limit        string
		offset       string
	)
	if s.table.tableColumns[s.tableName].HaveColumn(IsDeleted) {
		s.Where("is_deleted = ?", 0)
	}
	s.query = ""
	s.args = []interface{}{}
	if len(s.fields) == 0 {
		fields = "*"
	} else {
		for i := 0; i < len(s.fields); i++ {
			var tableName string
			s.fields[i], tableName, _ = s.warpField(s.fields[i])
			if tableName != s.tableName {
				if !s.joinConditions.HaveTable(tableName) {
					s.Joins(tableName)
				}
			}
		}
		fields = strings.Join(s.fields, ",")
	}
	for _, joincon := range s.joinConditions {
		joins += fmt.Sprintf(" LEFT JOIN %s ON %s", joincon.TableName, joincon.Condition)
	}
	for _, wherecon := range s.whereConditions {
		paddingwhere = " WHERE "
		wheres = append(wheres, wherecon.Query)
		s.args = append(s.args, wherecon.Args...)
	}
	if len(s.orderbyConditions) > 0 {
		orderby = " ORDER BY " + strings.Join(s.orderbyConditions, ",")
	}
	if s.limit != nil {
		limit = " LIMIT ?"
		s.args = append(s.args, s.limit)
	}
	if s.offset != nil {
		offset = " OFFSET ?"
		s.args = append(s.args, s.offset)
	}
	s.query = fmt.Sprintf("SELECT %s FROM `%s`%s%s%s%s%s%s", fields, s.tableName, joins, paddingwhere, strings.Join(wheres, " AND "), orderby, limit, offset)
	s.raw = true
	return s.query, s.args
}

//DISTINCT XX
//DISTICT XXX.XXX AS aaa
//XXX.XXX AS aaa
func (s *Search) warpField(field string) (warpStr string, tablename string, fieldname string) {
	if strings.Contains(field, " ") {
		if strings.Contains(field, "AS") {
			sp := strings.Split(field, " ")
			for i := 0; i < len(sp); i++ {
				if sp[i] == "AS" {
					sp[i-1], tablename, fieldname = s.warpFieldSingel(sp[i-1])
					warpStr = strings.Join(sp, " ")
					break
				}
			}
		} else {
			sp := strings.Split(field, " ")
			sp[len(sp)-1], tablename, fieldname = s.warpFieldSingel(sp[len(sp)-1])
			warpStr = strings.Join(sp, " ")
		}
	} else {
		return s.warpFieldSingel(field)
	}
	return
}

func (s *Search) warpFieldSingel(field string) (warpStr string, tablename string, fieldname string) {

	if strings.Contains(field, ".") {
		sp := strings.Split(field, ".")
		tablename = sp[0]
		fieldname = sp[1]
		if strings.Contains(field, "`") {
			warpStr = field
			return
		}
		warpStr = strings.Replace(field, ".", ".`", 1) + "`"
	} else {
		tablename = s.tableName
		fieldname = field
		if strings.Contains(field, "`") {
			warpStr = field
			return
		}
		switch field {
		case "*", "COUNT(*)":
			warpStr = field
		default:
			warpStr = "`" + field + "`"
		}
	}
	return
}

//结果展示

//RawMap RawMap
func (s *Search) RawMap() RowMap {
	return s.RowMap()
}

//RawsMap RawsMap
func (s *Search) RawsMap() RowsMap {
	return s.RowsMap()
}

//RawsMapInterface RawsMapInterface
func (s *Search) RawsMapInterface() RowsMapInterface {
	return s.RowsMapInterface()
}

//RowMap RowMap
func (s *Search) RowMap() RowMap {
	query, args := s.Parse()
	return s.table.Query(query, args...).RowMap()
}

//SQLRows SQLRows
func (s *Search) SQLRows() *SQLRows {
	query, args := s.Parse()
	return s.table.Query(query, args...)
}

//RowsMap RowsMap
func (s *Search) RowsMap() RowsMap {
	query, args := s.Parse()
	return s.table.Query(query, args...).RowsMap()
}

//RowsMapInterface RowsMapInterface
func (s *Search) RowsMapInterface() RowsMapInterface {
	query, args := s.Parse()
	return s.table.Query(query, args...).RowsMapInterface()
}

//DoubleSlice DoubleSlice
func (s *Search) DoubleSlice() (map[string]int, [][]string) {
	query, args := s.Parse()
	return s.table.Query(query, args...).DoubleSlice()
}

//Int 如果指定字段，则返回指定字段的int值，否则返回第一个字段作为int值返回。
func (s *Search) Int(args ...string) int {
	row := s.RowMap()
	if len(args) == 0 {
		for _, v := range row {
			i, _ := strconv.Atoi(v)
			return i
		}
	} else {
		i, _ := strconv.Atoi(row[args[0]])
		return i
	}
	return 0
}

//String like int
func (s *Search) String(args ...string) string {
	row := s.RowMap()
	if len(args) == 0 {
		for _, v := range row {
			return v
		}
	} else {
		return row[args[0]]
	}
	return ""
}

//Finds 将查询的结构放入到结构体当中
func (s *Search) Finds(v interface{}) {
	query, args := s.Parse()
	s.table.FindAll(v, append([]interface{}{query}, args...)...)
}

//Count 计算这次查询结果的个数
func (s *Search) Count() int {
	var count int
	s.fields = []string{"COUNT(*)"}
	query, args := s.Parse()
	s.table.Query(query, args...).Find(&count)
	return count
}
<|endoftext|>


<commit_before>package main

type candidate struct {
	fp     *fingerprint
	offset int
}

func deduplicateCandidates(candidates []candidate) []candidate {
	m := make(map[candidate]bool, 0)
	for _, c := range candidates {
		m[c] = true
	}

	deduped := make([]candidate, len(m))
	i := 0
	for k, v := range m {
		if v {
			deduped[i] = k
			i++
		}
	}

	return deduped
}

type approximate_search_strategy func(sfp sub_fingerprint) ([]sub_fingerprint, error)

func noopApproximateSearchStrategy() approximate_search_strategy {
	return func(sfp sub_fingerprint) ([]sub_fingerprint, error) {
		return make([]sub_fingerprint, 0), nil
	}
}

func flipAllApproximateSearchStrategy() approximate_search_strategy {
	return func(sfp sub_fingerprint) ([]sub_fingerprint, error) {
		flipped := make([]sub_fingerprint, SubFingerprintSizeBits)

		for i := 0; i < SubFingerprintSizeBits; i++ {
			flippedSfp, err := sfp.flipBit(uint(i))
			if err != nil {
				return make([]sub_fingerprint, 0), err
			}
			flipped[i] = flippedSfp
		}

		return flipped, nil
	}
}

// Given a query fingerprint block, find candidates based on one or more of the
// sub-fingerprints. The candidates are not filtered in any way, for example on
// BER. This will always do an exact match search on the sub-fingerprints in the
// query fingerprint block, however you can optionally pass a strategy for
// approximate sub-fingerprint searching. This is usually a bit-flipping
// algorithm.
func searchByFingerprintBlock(
	queryFpb fingerprint_block,
	approxSearchStrategy approximate_search_strategy,
	idx index) ([]candidate, error) {

	candidates := make([]candidate, 0)

	// find exact matches for sub-fingerprints in the fingerint block
	// BER threshold filtering will happen after generating all candidates (could
	// be optimized later)
	for queryOffset, querySfp := range queryFpb {
		newCandidates := searchBySubFingerprint(
			querySfp,
			queryOffset,
			idx,
		)

		if len(newCandidates) > 0 {
			candidates = append(candidates, newCandidates...)
		}
	}

	// try approximate searching, if a strategy was provided
	// this will do a depth-first search but since we collect all candidates,
	// this should not matter
	if approxSearchStrategy != nil {
		for queryOffset, querySfp := range queryFpb {
			approxQuerySfps, err := approxSearchStrategy(querySfp)
			if err != nil {
				return make([]candidate, 0), err
			}

			for _, approxQuerySfp := range approxQuerySfps {
				newCandidates := searchBySubFingerprint(
					approxQuerySfp,
					queryOffset,
					idx,
				)

				if len(newCandidates) > 0 {
					candidates = append(candidates, newCandidates...)
				}
			}
		}
	}

	return deduplicateCandidates(candidates), nil
}

// Given a sub-fingerprint and the offset of that sub-fingerprint in the query
// fingerprint block, find an exact match of the sub-fingerprint in the index
// and build a fingerprint block for each posting. The candidate fingerprint
// block is created such that the position in the fingerprint block of both the
// query and candidate sub-fingerprint are the same.
func searchBySubFingerprint(
	querySfp sub_fingerprint,
	queryOffset int,
	idx index) []candidate {

	postings, found := idx[querySfp]
	if !found {
		return make([]candidate, 0)
	}

	candidates := make([]candidate, len(postings))
	for i, posting := range postings {
		start := posting.offset - queryOffset
		candidates[i] = candidate{
			posting.fp,
			start,
		}
	}

	return candidates
}
<commit_msg>Moves out to fingerprint<commit_after>package main

type candidate struct {
	fp     *fingerprint
	offset int
}

func deduplicateCandidates(candidates []candidate) []candidate {
	m := make(map[candidate]bool, 0)
	for _, c := range candidates {
		m[c] = true
	}

	deduped := make([]candidate, len(m))
	i := 0
	for k, v := range m {
		if v {
			deduped[i] = k
			i++
		}
	}

	return deduped
}

type approximate_search_strategy func(sfp sub_fingerprint) ([]sub_fingerprint, error)

func noopApproximateSearchStrategy() approximate_search_strategy {
	return func(sfp sub_fingerprint) ([]sub_fingerprint, error) {
		return make([]sub_fingerprint, 0), nil
	}
}

func flipAllApproximateSearchStrategy(n uint) approximate_search_strategy {
	return func(sfp sub_fingerprint) ([]sub_fingerprint, error) {
		return sfp.flipAllBitsUntil(n)
	}
}

// Given a query fingerprint block, find candidates based on one or more of the
// sub-fingerprints. The candidates are not filtered in any way, for example on
// BER. This will always do an exact match search on the sub-fingerprints in the
// query fingerprint block, however you can optionally pass a strategy for
// approximate sub-fingerprint searching. This is usually a bit-flipping
// algorithm.
func searchByFingerprintBlock(
	queryFpb fingerprint_block,
	approxSearchStrategy approximate_search_strategy,
	idx index) ([]candidate, error) {

	candidates := make([]candidate, 0)

	// find exact matches for sub-fingerprints in the fingerint block
	// BER threshold filtering will happen after generating all candidates (could
	// be optimized later)
	for queryOffset, querySfp := range queryFpb {
		newCandidates := searchBySubFingerprint(
			querySfp,
			queryOffset,
			idx,
		)

		if len(newCandidates) > 0 {
			candidates = append(candidates, newCandidates...)
		}
	}

	// try approximate searching, if a strategy was provided
	// this will do a depth-first search but since we collect all candidates,
	// this should not matter
	if approxSearchStrategy != nil {
		for queryOffset, querySfp := range queryFpb {
			approxQuerySfps, err := approxSearchStrategy(querySfp)
			if err != nil {
				return make([]candidate, 0), err
			}

			for _, approxQuerySfp := range approxQuerySfps {
				newCandidates := searchBySubFingerprint(
					approxQuerySfp,
					queryOffset,
					idx,
				)

				if len(newCandidates) > 0 {
					candidates = append(candidates, newCandidates...)
				}
			}
		}
	}

	return deduplicateCandidates(candidates), nil
}

// Given a sub-fingerprint and the offset of that sub-fingerprint in the query
// fingerprint block, find an exact match of the sub-fingerprint in the index
// and build a fingerprint block for each posting. The candidate fingerprint
// block is created such that the position in the fingerprint block of both the
// query and candidate sub-fingerprint are the same.
func searchBySubFingerprint(
	querySfp sub_fingerprint,
	queryOffset int,
	idx index) []candidate {

	postings, found := idx[querySfp]
	if !found {
		return make([]candidate, 0)
	}

	candidates := make([]candidate, len(postings))
	for i, posting := range postings {
		start := posting.offset - queryOffset
		candidates[i] = candidate{
			posting.fp,
			start,
		}
	}

	return candidates
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"

	"restic"
	"restic/errors"
	"restic/repository"
)

var cmdLs = &cobra.Command{
	Use:   "ls [flags] snapshot-ID",
	Short: "list files in a snapshot",
	Long: `
The "ls" command allows listing files and directories in a snapshot.

The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runLs(globalOptions, args)
	},
}

var listLong bool

func init() {
	cmdRoot.AddCommand(cmdLs)

	cmdLs.Flags().BoolVarP(&listLong, "long", "l", false, "use a long listing format showing size and mode")
}

func printNode(prefix string, n *restic.Node) string {
	if !listLong {
		return filepath.Join(prefix, n.Name)
	}

	switch n.Type {
	case "file":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
	case "dir":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
	case "symlink":
		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
	default:
		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
	}
}

func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
	tree, err := repo.LoadTree(id)
	if err != nil {
		return err
	}

	for _, entry := range tree.Nodes {
		Printf(printNode(prefix, entry) + "\n")

		if entry.Type == "dir" && entry.Subtree != nil {
			err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func runLs(gopts GlobalOptions, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errors.Fatalf("no snapshot ID given")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	snapshotIDString := args[0]
	var id restic.ID
	var paths []string

	if snapshotIDString == "latest" {
		id, err = restic.FindLatestSnapshot(repo, paths, "")
		if err != nil {
			return err
		}
	} else {
		id, err = restic.FindSnapshot(repo, snapshotIDString)
		if err != nil {
			return err
		}
	}

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)

	return printTree("", repo, *sn.Tree)
}
<commit_msg>User interface inconsistency fixed<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"

	"restic"
	"restic/errors"
	"restic/repository"
)

var cmdLs = &cobra.Command{
	Use:   "ls [flags] snapshot-ID",
	Short: "list files in a snapshot",
	Long: `
The "ls" command allows listing files and directories in a snapshot.

The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runLs(globalOptions, args)
	},
}

// LsOptions collects all options for the ls command.
type LsOptions struct {
	ListLong bool
	Host     string
	Paths    []string
}

var lsOptions LsOptions

func init() {
	cmdRoot.AddCommand(cmdLs)

	flags := cmdLs.Flags()
	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")

	flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
}

func printNode(prefix string, n *restic.Node) string {
	if !lsOptions.ListLong {
		return filepath.Join(prefix, n.Name)
	}

	switch n.Type {
	case "file":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
	case "dir":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
	case "symlink":
		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
	default:
		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
	}
}

func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
	tree, err := repo.LoadTree(id)
	if err != nil {
		return err
	}

	for _, entry := range tree.Nodes {
		Printf(printNode(prefix, entry) + "\n")

		if entry.Type == "dir" && entry.Subtree != nil {
			err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func runLs(gopts GlobalOptions, args []string) error {
	if len(args) < 1 || len(args) > 2 {
		return errors.Fatalf("no snapshot ID given")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	snapshotIDString := args[0]
	var id restic.ID

	if snapshotIDString == "latest" {
		id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
		if err != nil {
			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
		}
	} else {
		id, err = restic.FindSnapshot(repo, snapshotIDString)
		if err != nil {
			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
		}
	}

	sn, err := restic.LoadSnapshot(repo, id)
	if err != nil {
		return err
	}

	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)

	return printTree("", repo, *sn.Tree)
}
<|endoftext|>