<commit_before>package api

// Resources represents the system resources avaible for LXD
// API extension: resources
type Resources struct {
	CPU    ResourcesCPU    `json:"cpu" yaml:"cpu"`
	Memory ResourcesMemory `json:"memory" yaml:"memory"`

	// API extension: resources_gpu
	GPU ResourcesGPU `json:"gpu" yaml:"gpu"`

	// API extension: resources_v2
	Network ResourcesNetwork `json:"network" yaml:"network"`
	Storage ResourcesStorage `json:"storage" yaml:"storage"`
}

// ResourcesCPU represents the cpu resources available on the system
// API extension: resources
type ResourcesCPU struct {
	// API extension: resources_v2
	Architecture string `json:"architecture" yaml:"architecture"`

	Sockets []ResourcesCPUSocket `json:"sockets" yaml:"sockets"`
	Total   uint64               `json:"total" yaml:"total"`
}

// ResourcesCPUSocket represents a CPU socket on the system
// API extension: resources_v2
type ResourcesCPUSocket struct {
	Name   string `json:"name,omitempty" yaml:"name,omitempty"`
	Vendor string `json:"vendor,omitempty" yaml:"vendor,omitempty"`

	Socket uint64              `json:"socket" yaml:"socket"`
	Cache  []ResourcesCPUCache `json:"cache,omitempty" yaml:"cache,omitempty"`
	Cores  []ResourcesCPUCore  `json:"cores" yaml:"cores"`

	Frequency        uint64 `json:"frequency,omitempty" yaml:"frequency,omitempty"`
	FrequencyMinimum uint64 `json:"frequency_minimum,omitempty" yaml:"frequency_minimum,omitempty"`
	FrequencyTurbo   uint64 `json:"frequency_turbo,omitempty" yaml:"frequency_turbo,omitempty"`
}

// ResourcesCPUCache represents a CPU cache
// API extension: resources_v2
type ResourcesCPUCache struct {
	Level uint64 `json:"level" yaml:"level"`
	Type  string `json:"type" yaml:"type"`
	Size  uint64 `json:"size" yaml:"size"`
}

// ResourcesCPUCore represents a CPU core on the system
// API extension: resources_v2
type ResourcesCPUCore struct {
	Core     uint64 `json:"core" yaml:"core"`
	NUMANode uint64 `json:"numa_node" yaml:"numa_node"`

	Threads []ResourcesCPUThread `json:"threads" yaml:"threads"`

	Frequency uint64 `json:"frequency,omitempty" yaml:"frequency,omitempty"`
}

// ResourcesCPUThread represents a CPU thread on the system
// API extension: resources_v2
type ResourcesCPUThread struct {
	ID     int64  `json:"id" yaml:"id"`
	Thread uint64 `json:"thread" yaml:"thread"`
	Online bool   `json:"online" yaml:"online"`
}

// ResourcesGPU represents the GPU resources available on the system
// API extension: resources_gpu
type ResourcesGPU struct {
	Cards []ResourcesGPUCard `json:"cards" yaml:"cards"`
	Total uint64             `json:"total" yaml:"total"`
}

// ResourcesGPUCard represents a GPU card on the system
// API extension: resources_v2
type ResourcesGPUCard struct {
	Driver        string `json:"driver,omitempty" yaml:"driver,omitempty"`
	DriverVersion string `json:"driver_version,omitempty" yaml:"driver_version,omitempty"`

	DRM    *ResourcesGPUCardDRM    `json:"drm,omitempty" yaml:"drm,omitempty"`
	SRIOV  *ResourcesGPUCardSRIOV  `json:"sriov,omitempty" yaml:"sriov,omitempty"`
	Nvidia *ResourcesGPUCardNvidia `json:"nvidia,omitempty" yaml:"nvidia,omitempty"`

	NUMANode   uint64 `json:"numa_node" yaml:"numa_node"`
	PCIAddress string `json:"pci_address,omitempty" yaml:"pci_address,omitempty"`

	Vendor    string `json:"vendor,omitempty" yaml:"vendor,omitempty"`
	VendorID  string `json:"vendor_id,omitempty" yaml:"vendor_id,omitempty"`
	Product   string `json:"product,omitempty" yaml:"product,omitempty"`
	ProductID string `json:"product_id,omitempty" yaml:"product_id,omitempty"`
}

// ResourcesGPUCardDRM represents the Linux DRM configuration of the GPU
// API extension: resources_v2
type ResourcesGPUCardDRM struct {
	ID uint64 `json:"id" yaml:"id"`

	CardName   string `json:"card_name" yaml:"card_name"`
	CardDevice string `json:"card_device" yaml:"card_device"`

	ControlName   string `json:"control_name,omitempty" yaml:"control_name,omitempty"`
	ControlDevice string `json:"control_device,omitempty" yaml:"control_device,omitempty"`

	RenderName   string `json:"render_name,omitempty" yaml:"render_name,omitempty"`
	RenderDevice string `json:"render_device,omitempty" yaml:"render_device,omitempty"`
}

// ResourcesGPUCardSRIOV represents the SRIOV configuration of the GPU
// API extension: resources_v2
type ResourcesGPUCardSRIOV struct {
	CurrentVFs uint64 `json:"current_vfs" yaml:"current_vfs"`
	MaximumVFs uint64 `json:"maximum_vfs" yaml:"maximum_vfs"`

	VFs []ResourcesGPUCard `json:"vfs" yaml:"vfs"`
}

// ResourcesGPUCardNvidia represents additional information for NVIDIA GPUs
// API extension: resources_gpu
type ResourcesGPUCardNvidia struct {
	CUDAVersion string `json:"cuda_version,omitempty" yaml:"cuda_version,omitempty"`
	NVRMVersion string `json:"nvrm_version,omitempty" yaml:"nvrm_version,omitempty"`

	Brand        string `json:"brand" yaml:"brand"`
	Model        string `json:"model" yaml:"model"`
	UUID         string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
	Architecture string `json:"architecture,omitempty" yaml:"architecture,omitempty"`

	// API extension: resources_v2
	CardName   string `json:"card_name" yaml:"card_name"`
	CardDevice string `json:"card_device" yaml:"card_device"`
}

// ResourcesNetwork represents the network cards available on the system
// API extension: resources_v2
type ResourcesNetwork struct {
	Cards []ResourcesNetworkCard `json:"cards" yaml:"cards"`
	Total uint64                 `json:"total" yaml:"total"`
}

// ResourcesNetworkCard represents a network card on the system
// API extension: resources_v2
type ResourcesNetworkCard struct {
	Driver        string `json:"driver,omitempty" yaml:"driver,omitempty"`
	DriverVersion string `json:"driver_version,omitempty" yaml:"driver_version,omitempty"`

	Ports []ResourcesNetworkCardPort `json:"ports,omitempty" yaml:"ports,omitempty"`
	SRIOV *ResourcesNetworkCardSRIOV `json:"sriov,omitempty" yaml:"sriov,omitempty"`

	NUMANode   uint64 `json:"numa_node" yaml:"numa_node"`
	PCIAddress string `json:"pci_address,omitempty" yaml:"pci_address,omitempty"`

	Vendor    string `json:"vendor,omitempty" yaml:"vendor,omitempty"`
	VendorID  string `json:"vendor_id,omitempty" yaml:"vendor_id,omitempty"`
	Product   string `json:"product,omitempty" yaml:"product,omitempty"`
	ProductID string `json:"product_id,omitempty" yaml:"product_id,omitempty"`
}

// ResourcesNetworkCardPort represents a network port on the system
// API extension: resources_v2
type ResourcesNetworkCardPort struct {
	ID       string `json:"id" yaml:"id"`
	Address  string `json:"address,omitempty" yaml:"address,omitempty"`
	Port     uint64 `json:"port" yaml:"port"`
	Protocol string `json:"protocol" yaml:"protocol"`

	SupportedModes []string `json:"supported_modes,omitempty" yaml:"supported_modes,omitempty"`
	SupportedPorts []string `json:"supported_ports,omitempty" yaml:"supported_ports,omitempty"`

	PortType        string `json:"port_type,omitempty" yaml:"port_type,omitempty"`
	TransceiverType string `json:"transceiver_type,omitempty" yaml:"transceiver_type,omitempty"`

	AutoNegotiation bool   `json:"auto_negotiation" yaml:"auto_negotiation"`
	LinkDetected    bool   `json:"link_detected" yaml:"link_detected"`
	LinkSpeed       uint64 `json:"link_speed,omitempty" yaml:"link_speed,omitempty"`
	LinkDuplex      string `json:"link_duplex,omitempty" yaml:"link_duplex,omitempty"`

	// API extension: resources_infiniband
	Infiniband *ResourcesNetworkCardPortInfiniband `json:"infiniband,omitempty" yaml:"infiniband,omitempty"`
}

// ResourcesNetworkCardPortInfiniband represents the Linux Infiniband configuration for the port
// API extension: resources_infiniband
type ResourcesNetworkCardPortInfiniband struct {
	IsSMName   string `json:"issm_name,omitempty" yaml:"issm_name,omitempty"`
	IsSMDevice string `json:"issm_device,omitempty" yaml:"issm_device,omitempty"`

	MADName   string `json:"mad_name,omitempty" yaml:"mad_name,omitempty"`
	MADDevice string `json:"mad_device,omitempty" yaml:"mad_device,omitempty"`

	VerbName   string `json:"verb_name,omitempty" yaml:"verb_name,omitempty"`
	VerbDevice string `json:"verb_device,omitempty" yaml:"verb_device,omitempty"`
}

// ResourcesNetworkCardSRIOV represents the SRIOV configuration of the network card
// API extension: resources_v2
type ResourcesNetworkCardSRIOV struct {
	CurrentVFs uint64 `json:"current_vfs" yaml:"current_vfs"`
	MaximumVFs uint64 `json:"maximum_vfs" yaml:"maximum_vfs"`

	VFs []ResourcesNetworkCard `json:"vfs" yaml:"vfs"`
}

// ResourcesStorage represents the local storage
// API extension: resources_v2
type ResourcesStorage struct {
	Disks []ResourcesStorageDisk `json:"disks" yaml:"disks"`
	Total uint64                 `json:"total" yaml:"total"`
}

// ResourcesStorageDisk represents a disk
// API extension: resources_v2
type ResourcesStorageDisk struct {
	ID       string `json:"id" yaml:"id"`
	Device   string `json:"device" yaml:"device"`
	Model    string `json:"model,omitempty" yaml:"model,omitempty"`
	Type     string `json:"type,omitempty" yaml:"type,omitempty"`
	ReadOnly bool   `json:"read_only" yaml:"read_only"`
	Size     uint64 `json:"size" yaml:"size"`

	Removable bool   `json:"removable" yaml:"removable"`
	WWN       string `json:"wwn,omitempty" yaml:"wwn,omitempty"`
	NUMANode  uint64 `json:"numa_node" yaml:"numa_node"`

	// API extension: resources_disk_sata
	DevicePath      string `json:"device_path" yaml:"device_path"`
	BlockSize       uint64 `json:"block_size" yaml:"block_size"`
	FirmwareVersion string `json:"firmware_version,omitempty" yaml:"firmware_version,omitempty"`
	RPM             uint64 `json:"rpm" yaml:"rpm"`
	ModelRevision   string `json:"model_revision,omitempty" yaml:"model_revision,omitempty"`
	Serial          string `json:"serial,omitempty" yaml:"serial,omitempty"`

	Partitions []ResourcesStorageDiskPartition `json:"partitions" yaml:"partitions"`
}

// ResourcesStorageDiskPartition represents a partition on a disk
// API extension: resources_v2
type ResourcesStorageDiskPartition struct {
	ID       string `json:"id" yaml:"id"`
	Device   string `json:"device" yaml:"device"`
	ReadOnly bool   `json:"read_only" yaml:"read_only"`
	Size     uint64 `json:"size" yaml:"size"`

	Partition uint64 `json:"partition" yaml:"partition"`
}

// ResourcesMemory represents the memory resources available on the system
// API extension: resources
type ResourcesMemory struct {
	// API extension: resources_v2
	Nodes          []ResourcesMemoryNode `json:"nodes,omitempty" yaml:"nodes,omitempty"`
	HugepagesTotal uint64                `json:"hugepages_total" yaml:"hugepages_total"`
	HugepagesUsed  uint64                `json:"hugepages_used" yaml:"hugepages_used"`
	HugepagesSize  uint64                `json:"hugepages_size" yaml:"hugepages_size"`

	Used  uint64 `json:"used" yaml:"used"`
	Total uint64 `json:"total" yaml:"total"`
}

// ResourcesMemoryNode represents the node-specific memory resources available on the system
// API extension: resources_v2
type ResourcesMemoryNode struct {
	NUMANode       uint64 `json:"numa_node" yaml:"numa_node"`
	HugepagesUsed  uint64 `json:"hugepages_used" yaml:"hugepages_used"`
	HugepagesTotal uint64 `json:"hugepages_total" yaml:"hugepages_total"`

	Used  uint64 `json:"used" yaml:"used"`
	Total uint64 `json:"total" yaml:"total"`
}

// ResourcesStoragePool represents the resources available to a given storage pool
// API extension: resources
type ResourcesStoragePool struct {
	Space  ResourcesStoragePoolSpace  `json:"space,omitempty" yaml:"space,omitempty"`
	Inodes ResourcesStoragePoolInodes `json:"inodes,omitempty" yaml:"inodes,omitempty"`
}

// ResourcesStoragePoolSpace represents the space available to a given storage pool
// API extension: resources
type ResourcesStoragePoolSpace struct {
	Used  uint64 `json:"used,omitempty" yaml:"used,omitempty"`
	Total uint64 `json:"total" yaml:"total"`
}

// ResourcesStoragePoolInodes represents the inodes available to a given storage pool
// API extension: resources
type ResourcesStoragePoolInodes struct {
	Used  uint64 `json:"used" yaml:"used"`
	Total uint64 `json:"total" yaml:"total"`
}
<commit_msg>shared/api: Drop ModelRevision (same as FirmwareVersion)<commit_after>package api

// Resources represents the system resources avaible for LXD
// API extension: resources
type Resources struct {
	CPU    ResourcesCPU    `json:"cpu" yaml:"cpu"`
	Memory ResourcesMemory `json:"memory" yaml:"memory"`

	// API extension: resources_gpu
	GPU ResourcesGPU `json:"gpu" yaml:"gpu"`

	// API extension: resources_v2
	Network ResourcesNetwork `json:"network" yaml:"network"`
	Storage ResourcesStorage `json:"storage" yaml:"storage"`
}

// ResourcesCPU represents the cpu resources available on the system
// API extension: resources
type ResourcesCPU struct {
	// API extension: resources_v2
	Architecture string `json:"architecture" yaml:"architecture"`

	Sockets []ResourcesCPUSocket `json:"sockets" yaml:"sockets"`
	Total   uint64               `json:"total" yaml:"total"`
}

// ResourcesCPUSocket represents a CPU socket on the system
// API extension: resources_v2
type ResourcesCPUSocket struct {
	Name   string `json:"name,omitempty" yaml:"name,omitempty"`
	Vendor string `json:"vendor,omitempty" yaml:"vendor,omitempty"`

	Socket uint64              `json:"socket" yaml:"socket"`
	Cache  []ResourcesCPUCache `json:"cache,omitempty" yaml:"cache,omitempty"`
	Cores  []ResourcesCPUCore  `json:"cores" yaml:"cores"`

	Frequency        uint64 `json:"frequency,omitempty" yaml:"frequency,omitempty"`
	FrequencyMinimum uint64 `json:"frequency_minimum,omitempty" yaml:"frequency_minimum,omitempty"`
	FrequencyTurbo   uint64 `json:"frequency_turbo,omitempty" yaml:"frequency_turbo,omitempty"`
}

// ResourcesCPUCache represents a CPU cache
// API extension: resources_v2
type ResourcesCPUCache struct {
	Level uint64 `json:"level" yaml:"level"`
	Type  string `json:"type" yaml:"type"`
	Size  uint64 `json:"size" yaml:"size"`
}

// ResourcesCPUCore represents a CPU core on the system
// API extension: resources_v2
type ResourcesCPUCore struct {
	Core     uint64 `json:"core" yaml:"core"`
	NUMANode uint64 `json:"numa_node" yaml:"numa_node"`

	Threads []ResourcesCPUThread `json:"threads" yaml:"threads"`

	Frequency uint64 `json:"frequency,omitempty" yaml:"frequency,omitempty"`
}

// ResourcesCPUThread represents a CPU thread on the system
// API extension: resources_v2
type ResourcesCPUThread struct {
	ID     int64  `json:"id" yaml:"id"`
	Thread uint64 `json:"thread" yaml:"thread"`
	Online bool   `json:"online" yaml:"online"`
}

// ResourcesGPU represents the GPU resources available on the system
// API extension: resources_gpu
type ResourcesGPU struct {
	Cards []ResourcesGPUCard `json:"cards" yaml:"cards"`
	Total uint64             `json:"total" yaml:"total"`
}

// ResourcesGPUCard represents a GPU card on the system
// API extension: resources_v2
type ResourcesGPUCard struct {
	Driver        string `json:"driver,omitempty" yaml:"driver,omitempty"`
	DriverVersion string `json:"driver_version,omitempty" yaml:"driver_version,omitempty"`

	DRM    *ResourcesGPUCardDRM    `json:"drm,omitempty" yaml:"drm,omitempty"`
	SRIOV  *ResourcesGPUCardSRIOV  `json:"sriov,omitempty" yaml:"sriov,omitempty"`
	Nvidia *ResourcesGPUCardNvidia `json:"nvidia,omitempty" yaml:"nvidia,omitempty"`

	NUMANode   uint64 `json:"numa_node" yaml:"numa_node"`
	PCIAddress string `json:"pci_address,omitempty" yaml:"pci_address,omitempty"`

	Vendor    string `json:"vendor,omitempty" yaml:"vendor,omitempty"`
	VendorID  string `json:"vendor_id,omitempty" yaml:"vendor_id,omitempty"`
	Product   string `json:"product,omitempty" yaml:"product,omitempty"`
	ProductID string `json:"product_id,omitempty" yaml:"product_id,omitempty"`
}

// ResourcesGPUCardDRM represents the Linux DRM configuration of the GPU
// API extension: resources_v2
type ResourcesGPUCardDRM struct {
	ID uint64 `json:"id" yaml:"id"`

	CardName   string `json:"card_name" yaml:"card_name"`
	CardDevice string `json:"card_device" yaml:"card_device"`

	ControlName   string `json:"control_name,omitempty" yaml:"control_name,omitempty"`
	ControlDevice string `json:"control_device,omitempty" yaml:"control_device,omitempty"`

	RenderName   string `json:"render_name,omitempty" yaml:"render_name,omitempty"`
	RenderDevice string `json:"render_device,omitempty" yaml:"render_device,omitempty"`
}

// ResourcesGPUCardSRIOV represents the SRIOV configuration of the GPU
// API extension: resources_v2
type ResourcesGPUCardSRIOV struct {
	CurrentVFs uint64 `json:"current_vfs" yaml:"current_vfs"`
	MaximumVFs uint64 `json:"maximum_vfs" yaml:"maximum_vfs"`

	VFs []ResourcesGPUCard `json:"vfs" yaml:"vfs"`
}

// ResourcesGPUCardNvidia represents additional information for NVIDIA GPUs
// API extension: resources_gpu
type ResourcesGPUCardNvidia struct {
	CUDAVersion string `json:"cuda_version,omitempty" yaml:"cuda_version,omitempty"`
	NVRMVersion string `json:"nvrm_version,omitempty" yaml:"nvrm_version,omitempty"`

	Brand        string `json:"brand" yaml:"brand"`
	Model        string `json:"model" yaml:"model"`
	UUID         string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
	Architecture string `json:"architecture,omitempty" yaml:"architecture,omitempty"`

	// API extension: resources_v2
	CardName   string `json:"card_name" yaml:"card_name"`
	CardDevice string `json:"card_device" yaml:"card_device"`
}

// ResourcesNetwork represents the network cards available on the system
// API extension: resources_v2
type ResourcesNetwork struct {
	Cards []ResourcesNetworkCard `json:"cards" yaml:"cards"`
	Total uint64                 `json:"total" yaml:"total"`
}

// ResourcesNetworkCard represents a network card on the system
// API extension: resources_v2
type ResourcesNetworkCard struct {
	Driver        string `json:"driver,omitempty" yaml:"driver,omitempty"`
	DriverVersion string `json:"driver_version,omitempty" yaml:"driver_version,omitempty"`

	Ports []ResourcesNetworkCardPort `json:"ports,omitempty" yaml:"ports,omitempty"`
	SRIOV *ResourcesNetworkCardSRIOV `json:"sriov,omitempty" yaml:"sriov,omitempty"`

	NUMANode   uint64 `json:"numa_node" yaml:"numa_node"`
	PCIAddress string `json:"pci_address,omitempty" yaml:"pci_address,omitempty"`

	Vendor    string `json:"vendor,omitempty" yaml:"vendor,omitempty"`
	VendorID  string `json:"vendor_id,omitempty" yaml:"vendor_id,omitempty"`
	Product   string `json:"product,omitempty" yaml:"product,omitempty"`
	ProductID string `json:"product_id,omitempty" yaml:"product_id,omitempty"`
}

// ResourcesNetworkCardPort represents a network port on the system
// API extension: resources_v2
type ResourcesNetworkCardPort struct {
	ID       string `json:"id" yaml:"id"`
	Address  string `json:"address,omitempty" yaml:"address,omitempty"`
	Port     uint64 `json:"port" yaml:"port"`
	Protocol string `json:"protocol" yaml:"protocol"`

	SupportedModes []string `json:"supported_modes,omitempty" yaml:"supported_modes,omitempty"`
	SupportedPorts []string `json:"supported_ports,omitempty" yaml:"supported_ports,omitempty"`

	PortType        string `json:"port_type,omitempty" yaml:"port_type,omitempty"`
	TransceiverType string `json:"transceiver_type,omitempty" yaml:"transceiver_type,omitempty"`

	AutoNegotiation bool   `json:"auto_negotiation" yaml:"auto_negotiation"`
	LinkDetected    bool   `json:"link_detected" yaml:"link_detected"`
	LinkSpeed       uint64 `json:"link_speed,omitempty" yaml:"link_speed,omitempty"`
	LinkDuplex      string `json:"link_duplex,omitempty" yaml:"link_duplex,omitempty"`

	// API extension: resources_infiniband
	Infiniband *ResourcesNetworkCardPortInfiniband `json:"infiniband,omitempty" yaml:"infiniband,omitempty"`
}

// ResourcesNetworkCardPortInfiniband represents the Linux Infiniband configuration for the port
// API extension: resources_infiniband
type ResourcesNetworkCardPortInfiniband struct {
	IsSMName   string `json:"issm_name,omitempty" yaml:"issm_name,omitempty"`
	IsSMDevice string `json:"issm_device,omitempty" yaml:"issm_device,omitempty"`

	MADName   string `json:"mad_name,omitempty" yaml:"mad_name,omitempty"`
	MADDevice string `json:"mad_device,omitempty" yaml:"mad_device,omitempty"`

	VerbName   string `json:"verb_name,omitempty" yaml:"verb_name,omitempty"`
	VerbDevice string `json:"verb_device,omitempty" yaml:"verb_device,omitempty"`
}

// ResourcesNetworkCardSRIOV represents the SRIOV configuration of the network card
// API extension: resources_v2
type ResourcesNetworkCardSRIOV struct {
	CurrentVFs uint64 `json:"current_vfs" yaml:"current_vfs"`
	MaximumVFs uint64 `json:"maximum_vfs" yaml:"maximum_vfs"`

	VFs []ResourcesNetworkCard `json:"vfs" yaml:"vfs"`
}

// ResourcesStorage represents the local storage
// API extension: resources_v2
type ResourcesStorage struct {
	Disks []ResourcesStorageDisk `json:"disks" yaml:"disks"`
	Total uint64                 `json:"total" yaml:"total"`
}

// ResourcesStorageDisk represents a disk
// API extension: resources_v2
type ResourcesStorageDisk struct {
	ID       string `json:"id" yaml:"id"`
	Device   string `json:"device" yaml:"device"`
	Model    string `json:"model,omitempty" yaml:"model,omitempty"`
	Type     string `json:"type,omitempty" yaml:"type,omitempty"`
	ReadOnly bool   `json:"read_only" yaml:"read_only"`
	Size     uint64 `json:"size" yaml:"size"`

	Removable bool   `json:"removable" yaml:"removable"`
	WWN       string `json:"wwn,omitempty" yaml:"wwn,omitempty"`
	NUMANode  uint64 `json:"numa_node" yaml:"numa_node"`

	// API extension: resources_disk_sata
	DevicePath      string `json:"device_path" yaml:"device_path"`
	BlockSize       uint64 `json:"block_size" yaml:"block_size"`
	FirmwareVersion string `json:"firmware_version,omitempty" yaml:"firmware_version,omitempty"`
	RPM             uint64 `json:"rpm" yaml:"rpm"`
	Serial          string `json:"serial,omitempty" yaml:"serial,omitempty"`

	Partitions []ResourcesStorageDiskPartition `json:"partitions" yaml:"partitions"`
}

// ResourcesStorageDiskPartition represents a partition on a disk
// API extension: resources_v2
type ResourcesStorageDiskPartition struct {
	ID       string `json:"id" yaml:"id"`
	Device   string `json:"device" yaml:"device"`
	ReadOnly bool   `json:"read_only" yaml:"read_only"`
	Size     uint64 `json:"size" yaml:"size"`

	Partition uint64 `json:"partition" yaml:"partition"`
}

// ResourcesMemory represents the memory resources available on the system
// API extension: resources
type ResourcesMemory struct {
	// API extension: resources_v2
	Nodes          []ResourcesMemoryNode `json:"nodes,omitempty" yaml:"nodes,omitempty"`
	HugepagesTotal uint64                `json:"hugepages_total" yaml:"hugepages_total"`
	HugepagesUsed  uint64                `json:"hugepages_used" yaml:"hugepages_used"`
	HugepagesSize  uint64                `json:"hugepages_size" yaml:"hugepages_size"`

	Used  uint64 `json:"used" yaml:"used"`
	Total uint64 `json:"total" yaml:"total"`
}

// ResourcesMemoryNode represents the node-specific memory resources available on the system
// API extension: resources_v2
type ResourcesMemoryNode struct {
	NUMANode       uint64 `json:"numa_node" yaml:"numa_node"`
	HugepagesUsed  uint64 `json:"hugepages_used" yaml:"hugepages_used"`
	HugepagesTotal uint64 `json:"hugepages_total" yaml:"hugepages_total"`

	Used  uint64 `json:"used" yaml:"used"`
	Total uint64 `json:"total" yaml:"total"`
}

// ResourcesStoragePool represents the resources available to a given storage pool
// API extension: resources
type ResourcesStoragePool struct {
	Space  ResourcesStoragePoolSpace  `json:"space,omitempty" yaml:"space,omitempty"`
	Inodes ResourcesStoragePoolInodes `json:"inodes,omitempty" yaml:"inodes,omitempty"`
}

// ResourcesStoragePoolSpace represents the space available to a given storage pool
// API extension: resources
type ResourcesStoragePoolSpace struct {
	Used  uint64 `json:"used,omitempty" yaml:"used,omitempty"`
	Total uint64 `json:"total" yaml:"total"`
}

// ResourcesStoragePoolInodes represents the inodes available to a given storage pool
// API extension: resources
type ResourcesStoragePoolInodes struct {
	Used  uint64 `json:"used" yaml:"used"`
	Total uint64 `json:"total" yaml:"total"`
}
<|endoftext|>


<commit_before>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/charset"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/graceful"
	"code.gitea.io/gitea/modules/indexer"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"

	"github.com/ethantkoenig/rupture"
)

// RepoIndexerStatus status of a repo's entry in the repo indexer
// For now, implicitly refers to default branch
type RepoIndexerStatus struct {
	ID        int64  `xorm:"pk autoincr"`
	RepoID    int64  `xorm:"INDEX"`
	CommitSha string `xorm:"VARCHAR(40)"`
}

func (repo *Repository) getIndexerStatus() error {
	if repo.IndexerStatus != nil {
		return nil
	}
	status := &RepoIndexerStatus{RepoID: repo.ID}
	has, err := x.Get(status)
	if err != nil {
		return err
	} else if !has {
		status.CommitSha = ""
	}
	repo.IndexerStatus = status
	return nil
}

func (repo *Repository) updateIndexerStatus(sha string) error {
	if err := repo.getIndexerStatus(); err != nil {
		return err
	}
	if len(repo.IndexerStatus.CommitSha) == 0 {
		repo.IndexerStatus.CommitSha = sha
		_, err := x.Insert(repo.IndexerStatus)
		return err
	}
	repo.IndexerStatus.CommitSha = sha
	_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
		Update(repo.IndexerStatus)
	return err
}

type repoIndexerOperation struct {
	repo     *Repository
	deleted  bool
	watchers []chan<- error
}

var repoIndexerOperationQueue chan repoIndexerOperation

// InitRepoIndexer initialize the repo indexer
func InitRepoIndexer() {
	if !setting.Indexer.RepoIndexerEnabled {
		return
	}
	waitChannel := make(chan time.Duration)
	repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
	go func() {
		start := time.Now()
		log.Info("Initializing Repository Indexer")
		indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
		go processRepoIndexerOperationQueue()
		waitChannel <- time.Since(start)
	}()
	if setting.Indexer.StartupTimeout > 0 {
		go func() {
			timeout := setting.Indexer.StartupTimeout
			if graceful.Manager.IsChild() && setting.GracefulHammerTime > 0 {
				timeout += setting.GracefulHammerTime
			}
			select {
			case duration := <-waitChannel:
				log.Info("Repository Indexer Initialization took %v", duration)
			case <-time.After(timeout):
				log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
			}
		}()

	}
}

// populateRepoIndexerAsynchronously asynchronously populates the repo indexer
// with pre-existing data. This should only be run when the indexer is created
// for the first time.
func populateRepoIndexerAsynchronously() error {
	exist, err := x.Table("repository").Exist()
	if err != nil {
		return err
	} else if !exist {
		return nil
	}

	// if there is any existing repo indexer metadata in the DB, delete it
	// since we are starting afresh. Also, xorm requires deletes to have a
	// condition, and we want to delete everything, thus 1=1.
	if _, err := x.Where("1=1").Delete(new(RepoIndexerStatus)); err != nil {
		return err
	}

	var maxRepoID int64
	if _, err = x.Select("MAX(id)").Table("repository").Get(&maxRepoID); err != nil {
		return err
	}
	go populateRepoIndexer(maxRepoID)
	return nil
}

// populateRepoIndexer populate the repo indexer with pre-existing data. This
// should only be run when the indexer is created for the first time.
func populateRepoIndexer(maxRepoID int64) {
	log.Info("Populating the repo indexer with existing repositories")
	// start with the maximum existing repo ID and work backwards, so that we
	// don't include repos that are created after gitea starts; such repos will
	// already be added to the indexer, and we don't need to add them again.
	for maxRepoID > 0 {
		repos := make([]*Repository, 0, RepositoryListDefaultPageSize)
		err := x.Where("id <= ?", maxRepoID).
			OrderBy("id DESC").
			Limit(RepositoryListDefaultPageSize).
			Find(&repos)
		if err != nil {
			log.Error("populateRepoIndexer: %v", err)
			return
		} else if len(repos) == 0 {
			break
		}
		for _, repo := range repos {
			repoIndexerOperationQueue <- repoIndexerOperation{
				repo:    repo,
				deleted: false,
			}
			maxRepoID = repo.ID - 1
		}
	}
	log.Info("Done populating the repo indexer with existing repositories")
}

func updateRepoIndexer(repo *Repository) error {
	sha, err := getDefaultBranchSha(repo)
	if err != nil {
		return err
	}
	changes, err := getRepoChanges(repo, sha)
	if err != nil {
		return err
	} else if changes == nil {
		return nil
	}

	batch := indexer.RepoIndexerBatch()
	for _, update := range changes.Updates {
		if err := addUpdate(update, repo, batch); err != nil {
			return err
		}
	}
	for _, filename := range changes.RemovedFilenames {
		if err := addDelete(filename, repo, batch); err != nil {
			return err
		}
	}
	if err = batch.Flush(); err != nil {
		return err
	}
	return repo.updateIndexerStatus(sha)
}

// repoChanges changes (file additions/updates/removals) to a repo
type repoChanges struct {
	Updates          []fileUpdate
	RemovedFilenames []string
}

type fileUpdate struct {
	Filename string
	BlobSha  string
}

func getDefaultBranchSha(repo *Repository) (string, error) {
	stdout, err := git.NewCommand("show-ref", "-s", repo.DefaultBranch).RunInDir(repo.RepoPath())
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(stdout), nil
}

// getRepoChanges returns changes to repo since last indexer update
func getRepoChanges(repo *Repository, revision string) (*repoChanges, error) {
	if err := repo.getIndexerStatus(); err != nil {
		return nil, err
	}

	if len(repo.IndexerStatus.CommitSha) == 0 {
		return genesisChanges(repo, revision)
	}
	return nonGenesisChanges(repo, revision)
}

func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch) error {
	stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha).
		RunInDir(repo.RepoPath())
	if err != nil {
		return err
	}
	if size, err := strconv.Atoi(strings.TrimSpace(stdout)); err != nil {
		return fmt.Errorf("Misformatted git cat-file output: %v", err)
	} else if int64(size) > setting.Indexer.MaxIndexerFileSize {
		return addDelete(update.Filename, repo, batch)
	}

	fileContents, err := git.NewCommand("cat-file", "blob", update.BlobSha).
		RunInDirBytes(repo.RepoPath())
	if err != nil {
		return err
	} else if !base.IsTextFile(fileContents) {
		// FIXME: UTF-16 files will probably fail here
		return nil
	}
	indexerUpdate := indexer.RepoIndexerUpdate{
		Filepath: update.Filename,
		Op:       indexer.RepoIndexerOpUpdate,
		Data: &indexer.RepoIndexerData{
			RepoID:  repo.ID,
			Content: string(charset.ToUTF8DropErrors(fileContents)),
		},
	}
	return indexerUpdate.AddToFlushingBatch(batch)
}

func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) error {
	indexerUpdate := indexer.RepoIndexerUpdate{
		Filepath: filename,
		Op:       indexer.RepoIndexerOpDelete,
		Data: &indexer.RepoIndexerData{
			RepoID: repo.ID,
		},
	}
	return indexerUpdate.AddToFlushingBatch(batch)
}

func isIndexable(entry *git.TreeEntry) bool {
	if !entry.IsRegular() && !entry.IsExecutable() {
		return false
	}
	name := strings.ToLower(entry.Name())
	for _, g := range setting.Indexer.ExcludePatterns {
		if g.Match(name) {
			return false
		}
	}
	for _, g := range setting.Indexer.IncludePatterns {
		if g.Match(name) {
			return true
		}
	}
	return len(setting.Indexer.IncludePatterns) == 0
}

// parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command
func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
	entries, err := git.ParseTreeEntries(stdout)
	if err != nil {
		return nil, err
	}
	var idxCount = 0
	updates := make([]fileUpdate, len(entries))
	for _, entry := range entries {
		if isIndexable(entry) {
			updates[idxCount] = fileUpdate{
				Filename: entry.Name(),
				BlobSha:  entry.ID.String(),
			}
			idxCount++
		}
	}
	return updates[:idxCount], nil
}

// genesisChanges get changes to add repo to the indexer for the first time
func genesisChanges(repo *Repository, revision string) (*repoChanges, error) {
	var changes repoChanges
	stdout, err := git.NewCommand("ls-tree", "--full-tree", "-r", revision).
		RunInDirBytes(repo.RepoPath())
	if err != nil {
		return nil, err
	}
	changes.Updates, err = parseGitLsTreeOutput(stdout)
	return &changes, err
}

// nonGenesisChanges get changes since the previous indexer update
func nonGenesisChanges(repo *Repository, revision string) (*repoChanges, error) {
	diffCmd := git.NewCommand("diff", "--name-status",
		repo.IndexerStatus.CommitSha, revision)
	stdout, err := diffCmd.RunInDir(repo.RepoPath())
	if err != nil {
		// previous commit sha may have been removed by a force push, so
		// try rebuilding from scratch
		log.Warn("git diff: %v", err)
		if err = indexer.DeleteRepoFromIndexer(repo.ID); err != nil {
			return nil, err
		}
		return genesisChanges(repo, revision)
	}
	var changes repoChanges
	updatedFilenames := make([]string, 0, 10)
	for _, line := range strings.Split(stdout, "\n") {
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		filename := strings.TrimSpace(line[1:])
		if len(filename) == 0 {
			continue
		} else if filename[0] == '"' {
			filename, err = strconv.Unquote(filename)
			if err != nil {
				return nil, err
			}
		}

		switch status := line[0]; status {
		case 'M', 'A':
			updatedFilenames = append(updatedFilenames, filename)
		case 'D':
			changes.RemovedFilenames = append(changes.RemovedFilenames, filename)
		default:
			log.Warn("Unrecognized status: %c (line=%s)", status, line)
		}
	}

	cmd := git.NewCommand("ls-tree", "--full-tree", revision, "--")
	cmd.AddArguments(updatedFilenames...)
	lsTreeStdout, err := cmd.RunInDirBytes(repo.RepoPath())
	if err != nil {
		return nil, err
	}
	changes.Updates, err = parseGitLsTreeOutput(lsTreeStdout)
	return &changes, err
}

func processRepoIndexerOperationQueue() {
	for {
		op := <-repoIndexerOperationQueue
		var err error
		if op.deleted {
			if err = indexer.DeleteRepoFromIndexer(op.repo.ID); err != nil {
				log.Error("DeleteRepoFromIndexer: %v", err)
			}
		} else {
			if err = updateRepoIndexer(op.repo); err != nil {
				log.Error("updateRepoIndexer: %v", err)
			}
		}
		for _, watcher := range op.watchers {
			watcher <- err
		}
	}
}

// DeleteRepoFromIndexer remove all of a repository's entries from the indexer
func DeleteRepoFromIndexer(repo *Repository, watchers ...chan<- error) {
	addOperationToQueue(repoIndexerOperation{repo: repo, deleted: true, watchers: watchers})
}

// UpdateRepoIndexer update a repository's entries in the indexer
func UpdateRepoIndexer(repo *Repository, watchers ...chan<- error) {
	addOperationToQueue(repoIndexerOperation{repo: repo, deleted: false, watchers: watchers})
}

func addOperationToQueue(op repoIndexerOperation) {
	if !setting.Indexer.RepoIndexerEnabled {
		return
	}
	select {
	case repoIndexerOperationQueue <- op:
		break
	default:
		go func() {
			repoIndexerOperationQueue <- op
		}()
	}
}
<commit_msg>fix race on indexer (#9136)<commit_after>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/charset"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/graceful"
	"code.gitea.io/gitea/modules/indexer"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"

	"github.com/ethantkoenig/rupture"
)

// RepoIndexerStatus status of a repo's entry in the repo indexer
// For now, implicitly refers to default branch
type RepoIndexerStatus struct {
	ID        int64  `xorm:"pk autoincr"`
	RepoID    int64  `xorm:"INDEX"`
	CommitSha string `xorm:"VARCHAR(40)"`
}

func (repo *Repository) getIndexerStatus() error {
	if repo.IndexerStatus != nil {
		return nil
	}
	status := &RepoIndexerStatus{RepoID: repo.ID}
	has, err := x.Get(status)
	if err != nil {
		return err
	} else if !has {
		status.CommitSha = ""
	}
	repo.IndexerStatus = status
	return nil
}

func (repo *Repository) updateIndexerStatus(sha string) error {
	if err := repo.getIndexerStatus(); err != nil {
		return err
	}
	if len(repo.IndexerStatus.CommitSha) == 0 {
		repo.IndexerStatus.CommitSha = sha
		_, err := x.Insert(repo.IndexerStatus)
		return err
	}
	repo.IndexerStatus.CommitSha = sha
	_, err := x.ID(repo.IndexerStatus.ID).Cols("commit_sha").
		Update(repo.IndexerStatus)
	return err
}

type repoIndexerOperation struct {
	repoID   int64
	deleted  bool
	watchers []chan<- error
}

var repoIndexerOperationQueue chan repoIndexerOperation

// InitRepoIndexer initialize the repo indexer
func InitRepoIndexer() {
	if !setting.Indexer.RepoIndexerEnabled {
		return
	}
	waitChannel := make(chan time.Duration)
	repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
	go func() {
		start := time.Now()
		log.Info("Initializing Repository Indexer")
		indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
		go processRepoIndexerOperationQueue()
		waitChannel <- time.Since(start)
	}()
	if setting.Indexer.StartupTimeout > 0 {
		go func() {
			timeout := setting.Indexer.StartupTimeout
			if graceful.Manager.IsChild() && setting.GracefulHammerTime > 0 {
				timeout += setting.GracefulHammerTime
			}
			select {
			case duration := <-waitChannel:
				log.Info("Repository Indexer Initialization took %v", duration)
			case <-time.After(timeout):
				log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
			}
		}()

	}
}

// populateRepoIndexerAsynchronously asynchronously populates the repo indexer
// with pre-existing data. This should only be run when the indexer is created
// for the first time.
func populateRepoIndexerAsynchronously() error {
	exist, err := x.Table("repository").Exist()
	if err != nil {
		return err
	} else if !exist {
		return nil
	}

	// if there is any existing repo indexer metadata in the DB, delete it
	// since we are starting afresh. Also, xorm requires deletes to have a
	// condition, and we want to delete everything, thus 1=1.
	if _, err := x.Where("1=1").Delete(new(RepoIndexerStatus)); err != nil {
		return err
	}

	var maxRepoID int64
	if _, err = x.Select("MAX(id)").Table("repository").Get(&maxRepoID); err != nil {
		return err
	}
	go populateRepoIndexer(maxRepoID)
	return nil
}

// populateRepoIndexer populate the repo indexer with pre-existing data. This
// should only be run when the indexer is created for the first time.
func populateRepoIndexer(maxRepoID int64) {
	log.Info("Populating the repo indexer with existing repositories")
	// start with the maximum existing repo ID and work backwards, so that we
	// don't include repos that are created after gitea starts; such repos will
	// already be added to the indexer, and we don't need to add them again.
	for maxRepoID > 0 {
		repos := make([]*Repository, 0, RepositoryListDefaultPageSize)
		err := x.Where("id <= ?", maxRepoID).
			OrderBy("id DESC").
			Limit(RepositoryListDefaultPageSize).
			Find(&repos)
		if err != nil {
			log.Error("populateRepoIndexer: %v", err)
			return
		} else if len(repos) == 0 {
			break
		}
		for _, repo := range repos {
			repoIndexerOperationQueue <- repoIndexerOperation{
				repoID:  repo.ID,
				deleted: false,
			}
			maxRepoID = repo.ID - 1
		}
	}
	log.Info("Done populating the repo indexer with existing repositories")
}

func updateRepoIndexer(repoID int64) error {
	repo, err := getRepositoryByID(x, repoID)
	if err != nil {
		return err
	}

	sha, err := getDefaultBranchSha(repo)
	if err != nil {
		return err
	}
	changes, err := getRepoChanges(repo, sha)
	if err != nil {
		return err
	} else if changes == nil {
		return nil
	}

	batch := indexer.RepoIndexerBatch()
	for _, update := range changes.Updates {
		if err := addUpdate(update, repo, batch); err != nil {
			return err
		}
	}
	for _, filename := range changes.RemovedFilenames {
		if err := addDelete(filename, repo, batch); err != nil {
			return err
		}
	}
	if err = batch.Flush(); err != nil {
		return err
	}
	return repo.updateIndexerStatus(sha)
}

// repoChanges changes (file additions/updates/removals) to a repo
type repoChanges struct {
	Updates          []fileUpdate
	RemovedFilenames []string
}

type fileUpdate struct {
	Filename string
	BlobSha  string
}

func getDefaultBranchSha(repo *Repository) (string, error) {
	stdout, err := git.NewCommand("show-ref", "-s", repo.DefaultBranch).RunInDir(repo.RepoPath())
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(stdout), nil
}

// getRepoChanges returns changes to repo since last indexer update
func getRepoChanges(repo *Repository, revision string) (*repoChanges, error) {
	if err := repo.getIndexerStatus(); err != nil {
		return nil, err
	}

	if len(repo.IndexerStatus.CommitSha) == 0 {
		return genesisChanges(repo, revision)
	}
	return nonGenesisChanges(repo, revision)
}

func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch) error {
	stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha).
		RunInDir(repo.RepoPath())
	if err != nil {
		return err
	}
	if size, err := strconv.Atoi(strings.TrimSpace(stdout)); err != nil {
		return fmt.Errorf("Misformatted git cat-file output: %v", err)
	} else if int64(size) > setting.Indexer.MaxIndexerFileSize {
		return addDelete(update.Filename, repo, batch)
	}

	fileContents, err := git.NewCommand("cat-file", "blob", update.BlobSha).
		RunInDirBytes(repo.RepoPath())
	if err != nil {
		return err
	} else if !base.IsTextFile(fileContents) {
		// FIXME: UTF-16 files will probably fail here
		return nil
	}
	indexerUpdate := indexer.RepoIndexerUpdate{
		Filepath: update.Filename,
		Op:       indexer.RepoIndexerOpUpdate,
		Data: &indexer.RepoIndexerData{
			RepoID:  repo.ID,
			Content: string(charset.ToUTF8DropErrors(fileContents)),
		},
	}
	return indexerUpdate.AddToFlushingBatch(batch)
}

func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) error {
	indexerUpdate := indexer.RepoIndexerUpdate{
		Filepath: filename,
		Op:       indexer.RepoIndexerOpDelete,
		Data: &indexer.RepoIndexerData{
			RepoID: repo.ID,
		},
	}
	return indexerUpdate.AddToFlushingBatch(batch)
}

func isIndexable(entry *git.TreeEntry) bool {
	if !entry.IsRegular() && !entry.IsExecutable() {
		return false
	}
	name := strings.ToLower(entry.Name())
	for _, g := range setting.Indexer.ExcludePatterns {
		if g.Match(name) {
			return false
		}
	}
	for _, g := range setting.Indexer.IncludePatterns {
		if g.Match(name) {
			return true
		}
	}
	return len(setting.Indexer.IncludePatterns) == 0
}

// parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command
func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
	entries, err := git.ParseTreeEntries(stdout)
	if err != nil {
		return nil, err
	}
	var idxCount = 0
	updates := make([]fileUpdate, len(entries))
	for _, entry := range entries {
		if isIndexable(entry) {
			updates[idxCount] = fileUpdate{
				Filename: entry.Name(),
				BlobSha:  entry.ID.String(),
			}
			idxCount++
		}
	}
	return updates[:idxCount], nil
}

// genesisChanges get changes to add repo to the indexer for the first time
func genesisChanges(repo *Repository, revision string) (*repoChanges, error) {
	var changes repoChanges
	stdout, err := git.NewCommand("ls-tree", "--full-tree", "-r", revision).
		RunInDirBytes(repo.RepoPath())
	if err != nil {
		return nil, err
	}
	changes.Updates, err = parseGitLsTreeOutput(stdout)
	return &changes, err
}

// nonGenesisChanges get changes since the previous indexer update
func nonGenesisChanges(repo *Repository, revision string) (*repoChanges, error) {
	diffCmd := git.NewCommand("diff", "--name-status",
		repo.IndexerStatus.CommitSha, revision)
	stdout, err := diffCmd.RunInDir(repo.RepoPath())
	if err != nil {
		// previous commit sha may have been removed by a force push, so
		// try rebuilding from scratch
		log.Warn("git diff: %v", err)
		if err = indexer.DeleteRepoFromIndexer(repo.ID); err != nil {
			return nil, err
		}
		return genesisChanges(repo, revision)
	}
	var changes repoChanges
	updatedFilenames := make([]string, 0, 10)
	for _, line := range strings.Split(stdout, "\n") {
		line = strings.TrimSpace(line)
		if len(line) == 0 {
			continue
		}
		filename := strings.TrimSpace(line[1:])
		if len(filename) == 0 {
			continue
		} else if filename[0] == '"' {
			filename, err = strconv.Unquote(filename)
			if err != nil {
				return nil, err
			}
		}

		switch status := line[0]; status {
		case 'M', 'A':
			updatedFilenames = append(updatedFilenames, filename)
		case 'D':
			changes.RemovedFilenames = append(changes.RemovedFilenames, filename)
		default:
			log.Warn("Unrecognized status: %c (line=%s)", status, line)
		}
	}

	cmd := git.NewCommand("ls-tree", "--full-tree", revision, "--")
	cmd.AddArguments(updatedFilenames...)
	lsTreeStdout, err := cmd.RunInDirBytes(repo.RepoPath())
	if err != nil {
		return nil, err
	}
	changes.Updates, err = parseGitLsTreeOutput(lsTreeStdout)
	return &changes, err
}

func processRepoIndexerOperationQueue() {
	for {
		op := <-repoIndexerOperationQueue
		var err error
		if op.deleted {
			if err = indexer.DeleteRepoFromIndexer(op.repoID); err != nil {
				log.Error("DeleteRepoFromIndexer: %v", err)
			}
		} else {
			if err = updateRepoIndexer(op.repoID); err != nil {
				log.Error("updateRepoIndexer: %v", err)
			}
		}
		for _, watcher := range op.watchers {
			watcher <- err
		}
	}
}

// DeleteRepoFromIndexer remove all of a repository's entries from the indexer
func DeleteRepoFromIndexer(repo *Repository, watchers ...chan<- error) {
	addOperationToQueue(repoIndexerOperation{repoID: repo.ID, deleted: true, watchers: watchers})
}

// UpdateRepoIndexer update a repository's entries in the indexer
func UpdateRepoIndexer(repo *Repository, watchers ...chan<- error) {
	addOperationToQueue(repoIndexerOperation{repoID: repo.ID, deleted: false, watchers: watchers})
}

func addOperationToQueue(op repoIndexerOperation) {
	if !setting.Indexer.RepoIndexerEnabled {
		return
	}
	select {
	case repoIndexerOperationQueue <- op:
		break
	default:
		go func() {
			repoIndexerOperationQueue <- op
		}()
	}
}
<|endoftext|>


<commit_before>/*
SharedBuffer reader. What to know when creating such a reader:

	1. Timeless access to all present and future buffer data
	2. Close() must be called when it's done
	3. Will return a EOF after the buffer is closed and all data has been read
*/
package sharedbuffer

import (
	"container/heap"
	"errors"
	"io"
)

// reader represents a consumer of a SharedBuffer
type reader struct {
	at  int
	idx int
	sb  *SharedBuffer
}

var ErrClosedReader = errors.New("closed reader")

// Read some data from the buffer. Will block until data is available or an error occurs
func (r *reader) Read(p []byte) (n int, err error) {
	if r.idx < 0 || r.sb == nil {
		return 0, ErrClosedReader
	}

	r.sb.lock.Lock()
	defer r.sb.lock.Unlock()
	n, err = 0, nil

	// Block until available data or error
	for !r.availableData() && !r.sb.closed {
		r.sb.lock.Unlock()
		<-r.sb.newData
		r.sb.lock.Lock()
	}
	if !r.availableData() && r.sb.closed {
		return 0, io.EOF
	}

	// Copy data and move the reader's position in the buffer
	readStart := r.at - r.sb.start
	n = copy(p, r.sb.buf[readStart:])
	r.at += n

	// Tell SharedBuffer to resort its readers
	heap.Fix(&r.sb.readers, r.idx)
	r.sb.flush()

	return
}

// Close the reader. Tells the SharedBuffer to forget about its data guarantees.
func (r *reader) Close() error {
	r.sb.lock.Lock()
	defer r.sb.lock.Unlock()

	// SharedBuffer can now forget about tracking this reader
	heap.Remove(&r.sb.readers, r.idx)

	r.at = 0
	r.idx = -1
	r.sb = nil

	return nil
}

// availableData returns true if the buffer has new data after the reader's
// current position
func (r reader) availableData() bool {
	return r.at < r.sb.start+len(r.sb.buf)
}

/*
readers is a heap enabled collection of reader instances.

A SharedBuffer effectively stores what data its slowest reader
has yet to consume. As the slowest reader can change after any
read, it is good to have an efficient lookup for the slowest
reader.

A heap of readers reduces a naive O(n) lookup time to O(log n),
where n is the number of active readers.
*/
type readers []*reader

func (rs readers) Len() int           { return len(rs) }
func (rs readers) Less(i, j int) bool { return rs[i].at < rs[j].at }

func (rs readers) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
	rs[i].idx, rs[j].idx = rs[j].idx, rs[i].idx
}

func (rs *readers) Push(x interface{}) {
	r := x.(*reader)
	*rs = append(*rs, r)
	r.idx = len(*rs) - 1
}

func (rs *readers) Pop() interface{} {
	h := *rs
	l := len(h)
	p := h[l-1]
	*rs = h[:l-1]

	p.idx = -1
	return p
}
<commit_msg>SharedBuffer reader formatting fixes<commit_after>/*
SharedBuffer reader. What to know when creating such a reader:

 1. Timeless access to all present and future buffer data
 2. Close() must be called when it's done
 3. Will return a EOF after the buffer is closed and all data has been read
*/
package sharedbuffer

import (
	"container/heap"
	"errors"
	"io"
)

// reader represents a consumer of a SharedBuffer
type reader struct {
	at  int
	idx int
	sb  *SharedBuffer
}

var ErrClosedReader = errors.New("closed reader")

// Read some data from the buffer. Will block until data is available or an error occurs
func (r *reader) Read(p []byte) (n int, err error) {
	if r.idx < 0 || r.sb == nil {
		return 0, ErrClosedReader
	}

	r.sb.lock.Lock()
	defer r.sb.lock.Unlock()
	n, err = 0, nil

	// Block until available data or error
	for !r.availableData() && !r.sb.closed {
		r.sb.lock.Unlock()
		<-r.sb.newData
		r.sb.lock.Lock()
	}
	if !r.availableData() && r.sb.closed {
		return 0, io.EOF
	}

	// Copy data and move the reader's position in the buffer
	readStart := r.at - r.sb.start
	r.at += copy(p, r.sb.buf[readStart:])

	// Tell SharedBuffer to resort its readers
	heap.Fix(&r.sb.readers, r.idx)
	r.sb.flush()

	return
}

// Close the reader. Tells the SharedBuffer to forget about its data guarantees.
func (r *reader) Close() error {
	r.sb.lock.Lock()
	defer r.sb.lock.Unlock()

	// SharedBuffer can now forget about tracking this reader
	heap.Remove(&r.sb.readers, r.idx)

	r.at = 0
	r.idx = -1
	r.sb = nil

	return nil
}

// availableData returns true if the buffer has new data after the reader's
// current position
func (r reader) availableData() bool {
	return r.at < r.sb.start+len(r.sb.buf)
}

/*
readers is a heap enabled collection of reader instances.

A SharedBuffer effectively stores what data its slowest reader
has yet to consume. As the slowest reader can change after any
read, it is good to have an efficient lookup for the slowest
reader.

A heap of readers reduces a naive O(n) lookup time to O(log n),
where n is the number of active readers.
*/
type readers []*reader

func (rs readers) Len() int           { return len(rs) }
func (rs readers) Less(i, j int) bool { return rs[i].at < rs[j].at }

func (rs readers) Swap(i, j int) {
	rs[i], rs[j] = rs[j], rs[i]
	rs[i].idx, rs[j].idx = rs[j].idx, rs[i].idx
}

func (rs *readers) Push(x interface{}) {
	r := x.(*reader)
	*rs = append(*rs, r)
	r.idx = len(*rs) - 1
}

func (rs *readers) Pop() interface{} {
	h := *rs
	l := len(h)
	p := h[l-1]
	*rs = h[:l-1]

	p.idx = -1
	return p
}
<|endoftext|>


<commit_before>package signer

import (
	"errors"
	"net"

	"github.com/Sirupsen/logrus"
	pb "github.com/docker/notary/proto"
	"github.com/endophage/gotuf/data"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// NotarySigner implements a RPC based Trust service that calls the Notary-signer Service
type NotarySigner struct {
	kmClient pb.KeyManagementClient
	sClient  pb.SignerClient
}

// NewNotarySigner is a convinience method that returns NotarySigner
func NewNotarySigner(hostname string, port string, tlscafile string) *NotarySigner {
	var opts []grpc.DialOption
	netAddr := net.JoinHostPort(hostname, port)
	creds, err := credentials.NewClientTLSFromFile(tlscafile, hostname)
	if err != nil {
		logrus.Fatal("fail to read: ", err)
	}
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(netAddr, opts...)

	if err != nil {
		logrus.Fatal("fail to dial: ", err)
	}
	kmClient := pb.NewKeyManagementClient(conn)
	sClient := pb.NewSignerClient(conn)
	return &NotarySigner{
		kmClient: kmClient,
		sClient:  sClient,
	}
}

// Sign signs a byte string with a number of KeyIDs
func (trust *NotarySigner) Sign(keyIDs []string, toSign []byte) ([]data.Signature, error) {
	signatures := make([]data.Signature, 0, len(keyIDs))
	for _, ID := range keyIDs {
		keyID := pb.KeyID{ID: ID}
		sr := &pb.SignatureRequest{
			Content: toSign,
			KeyID:   &keyID,
		}
		sig, err := trust.sClient.Sign(context.Background(), sr)
		if err != nil {
			return nil, err
		}
		signatures = append(signatures, data.Signature{
			KeyID:     sig.KeyInfo.KeyID.ID,
			Method:    data.SigAlgorithm(sig.KeyInfo.Algorithm.Algorithm),
			Signature: sig.Content,
		})
	}
	return signatures, nil
}

// Create creates a remote key and returns the PublicKey associated with the remote private key
func (trust *NotarySigner) Create(role string, algorithm data.KeyAlgorithm) (data.PublicKey, error) {
	publicKey, err := trust.kmClient.CreateKey(context.Background(), &pb.Algorithm{Algorithm: algorithm.String()})
	if err != nil {
		return nil, err
	}
	//TODO(mccauley): Update API to return algorithm and/or take it as a param
	public := data.NewPublicKey(data.KeyAlgorithm(publicKey.KeyInfo.Algorithm.Algorithm), publicKey.PublicKey)
	return public, nil
}

// RemoveKey deletes a key
func (trust *NotarySigner) RemoveKey(keyid string) error {
	//TODO(aaronl): Not implemented yet
	return errors.New("DeleteKey not implemented in NotarySigner")
}

// GetKey retrieves a key
func (trust *NotarySigner) GetKey(keyid string) data.PublicKey {
	//TODO(aaronl): Not implemented yet
	return nil
}
<commit_msg>Implement missing functions in NotarySigner to fully satisfy CryptoService interface<commit_after>package signer

import (
	"net"

	"github.com/Sirupsen/logrus"
	pb "github.com/docker/notary/proto"
	"github.com/endophage/gotuf/data"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// NotarySigner implements a RPC based Trust service that calls the Notary-signer Service
type NotarySigner struct {
	kmClient pb.KeyManagementClient
	sClient  pb.SignerClient
}

// NewNotarySigner is a convinience method that returns NotarySigner
func NewNotarySigner(hostname string, port string, tlscafile string) *NotarySigner {
	var opts []grpc.DialOption
	netAddr := net.JoinHostPort(hostname, port)
	creds, err := credentials.NewClientTLSFromFile(tlscafile, hostname)
	if err != nil {
		logrus.Fatal("fail to read: ", err)
	}
	opts = append(opts, grpc.WithTransportCredentials(creds))
	conn, err := grpc.Dial(netAddr, opts...)

	if err != nil {
		logrus.Fatal("fail to dial: ", err)
	}
	kmClient := pb.NewKeyManagementClient(conn)
	sClient := pb.NewSignerClient(conn)
	return &NotarySigner{
		kmClient: kmClient,
		sClient:  sClient,
	}
}

// Sign signs a byte string with a number of KeyIDs
func (trust *NotarySigner) Sign(keyIDs []string, toSign []byte) ([]data.Signature, error) {
	signatures := make([]data.Signature, 0, len(keyIDs))
	for _, ID := range keyIDs {
		keyID := pb.KeyID{ID: ID}
		sr := &pb.SignatureRequest{
			Content: toSign,
			KeyID:   &keyID,
		}
		sig, err := trust.sClient.Sign(context.Background(), sr)
		if err != nil {
			return nil, err
		}
		signatures = append(signatures, data.Signature{
			KeyID:     sig.KeyInfo.KeyID.ID,
			Method:    data.SigAlgorithm(sig.KeyInfo.Algorithm.Algorithm),
			Signature: sig.Content,
		})
	}
	return signatures, nil
}

// Create creates a remote key and returns the PublicKey associated with the remote private key
func (trust *NotarySigner) Create(role string, algorithm data.KeyAlgorithm) (data.PublicKey, error) {
	publicKey, err := trust.kmClient.CreateKey(context.Background(), &pb.Algorithm{Algorithm: algorithm.String()})
	if err != nil {
		return nil, err
	}
	//TODO(mccauley): Update API to return algorithm and/or take it as a param
	public := data.NewPublicKey(data.KeyAlgorithm(publicKey.KeyInfo.Algorithm.Algorithm), publicKey.PublicKey)
	return public, nil
}

// RemoveKey deletes a key
func (trust *NotarySigner) RemoveKey(keyid string) error {
	_, err := trust.kmClient.DeleteKey(context.Background(), &pb.KeyID{ID: keyid})
	return err
}

// GetKey retrieves a key
func (trust *NotarySigner) GetKey(keyid string) data.PublicKey {
	publicKey, err := trust.kmClient.GetKeyInfo(context.Background(), &pb.KeyID{ID: keyid})
	if err != nil {
		return nil
	}
	return data.NewPublicKey(data.KeyAlgorithm(publicKey.KeyInfo.Algorithm.Algorithm), publicKey.PublicKey)
}
<|endoftext|>


<commit_before>package camoproxy

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

var validReqHeaders = map[string]bool{
	"Accept":            true,
	"Accept-Charset":    true,
	"Accept-Encoding":   true,
	"Accept-Language":   true,
	"Cache-Control":     true,
	"If-None-Match":     true,
	"If-Modified-Since": true,
}

func validateURL(path string, key []byte) (surl string, valid bool) {
	pathParts := strings.SplitN(path[1:], "/", 3)
	valid = false
	if len(pathParts) != 2 {
		log.Println("Bad path format", pathParts)
		return
	}
	hexdig, hexurl := pathParts[0], pathParts[1]
	urlBytes, err := hex.DecodeString(hexurl)
	if err != nil {
		log.Println("Bad Hex Decode", hexurl)
		return
	}
	surl = string(urlBytes)
	//log.Println(surl)
	mac := hmac.New(sha1.New, key)
	mac.Write([]byte(surl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	if macSum != hexdig {
		log.Printf("Bad signature: %s != %s", macSum, hexdig)
		return
	}
	valid = true
	return
}

type ProxyHandler struct {
	Transport       *http.Transport
	HMacKey         []byte
	RegexpAllowlist []*regexp.Regexp
	RegexpDenylist  []*regexp.Regexp
	MaxSize         int64
}

func (p *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	//log.Println("Request:", req.URL)

	// do fiddly things
	if req.Method != "GET" {
		log.Println("Something other than GET received", req.Method)
		http.Error(w, "Method Not Implemented", http.StatusNotImplemented)
		return
	}

	surl, ok := validateURL(req.URL.Path, p.HMacKey)
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	//log.Println("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		log.Println(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if Allowlist is set, require match
	matchFound := true
	if len(p.RegexpAllowlist) > 0 {
		matchFound = false
		for _, rgx := range p.RegexpAllowlist {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out Denylist urls based on regexes. Do this second
	// as Denylist takes precedence
	for _, rgx := range p.RegexpDenylist {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		log.Println("Something weird happened")
		http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		return
	}

	// filter headers
	for hdr, val := range req.Header {
		if validReqHeaders[hdr] {
			nreq.Header[hdr] = val
		}
	}
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", "pew pew pew")

	resp, err := p.Transport.RoundTrip(nreq)
	if err != nil {
		log.Println("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.MaxSize {
		log.Println("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusBadRequest)
		return
	}

	switch resp.StatusCode {
	case 300:
		log.Println("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303:
		// check for redirects. we do not follow.
		log.Println("Refusing to follow redirects")
		http.Error(w, "Refusing to follow redirects", http.StatusNotFound)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			log.Println("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	}

	for hdr, val := range resp.Header {
		h := w.Header()
		h[hdr] = val
	}

	h := w.Header()
	h.Add("X-Content-Type-Options", "nosniff")
	if resp.StatusCode == 304 && h.Get("Content-Type") != "" {
		h.Del("Content-Type")
	}

	w.WriteHeader(resp.StatusCode)
	io.Copy(w, resp.Body)
	//log.Println(req, resp.StatusCode)
}

<commit_msg>go fmt<commit_after>package camoproxy

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"io"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

var validReqHeaders = map[string]bool{
	"Accept":            true,
	"Accept-Charset":    true,
	"Accept-Encoding":   true,
	"Accept-Language":   true,
	"Cache-Control":     true,
	"If-None-Match":     true,
	"If-Modified-Since": true,
}

func validateURL(path string, key []byte) (surl string, valid bool) {
	pathParts := strings.SplitN(path[1:], "/", 3)
	valid = false
	if len(pathParts) != 2 {
		log.Println("Bad path format", pathParts)
		return
	}
	hexdig, hexurl := pathParts[0], pathParts[1]
	urlBytes, err := hex.DecodeString(hexurl)
	if err != nil {
		log.Println("Bad Hex Decode", hexurl)
		return
	}
	surl = string(urlBytes)
	//log.Println(surl)
	mac := hmac.New(sha1.New, key)
	mac.Write([]byte(surl))
	macSum := hex.EncodeToString(mac.Sum([]byte{}))
	if macSum != hexdig {
		log.Printf("Bad signature: %s != %s", macSum, hexdig)
		return
	}
	valid = true
	return
}

type ProxyHandler struct {
	Transport       *http.Transport
	HMacKey         []byte
	RegexpAllowlist []*regexp.Regexp
	RegexpDenylist  []*regexp.Regexp
	MaxSize         int64
}

func (p *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	//log.Println("Request:", req.URL)

	// do fiddly things
	if req.Method != "GET" {
		log.Println("Something other than GET received", req.Method)
		http.Error(w, "Method Not Implemented", http.StatusNotImplemented)
		return
	}

	surl, ok := validateURL(req.URL.Path, p.HMacKey)
	if !ok {
		http.Error(w, "Bad Signature", http.StatusForbidden)
		return
	}
	//log.Println("URL:", surl)

	u, err := url.Parse(surl)
	if err != nil {
		log.Println(err)
		http.Error(w, "Bad url", http.StatusBadRequest)
		return
	}

	if u.Host == "" {
		http.Error(w, "Bad url", http.StatusNotFound)
		return
	}

	// if Allowlist is set, require match
	matchFound := true
	if len(p.RegexpAllowlist) > 0 {
		matchFound = false
		for _, rgx := range p.RegexpAllowlist {
			if rgx.MatchString(u.Host) {
				matchFound = true
			}
		}
	}
	if !matchFound {
		http.Error(w, "Allowlist host failure", http.StatusNotFound)
		return
	}

	// filter out Denylist urls based on regexes. Do this second
	// as Denylist takes precedence
	for _, rgx := range p.RegexpDenylist {
		if rgx.MatchString(u.Host) {
			http.Error(w, "Denylist host failure", http.StatusNotFound)
			return
		}
	}

	nreq, err := http.NewRequest("GET", surl, nil)
	if err != nil {
		log.Println("Something weird happened")
		http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		return
	}

	// filter headers
	for hdr, val := range req.Header {
		if validReqHeaders[hdr] {
			nreq.Header[hdr] = val
		}
	}
	nreq.Header.Add("connection", "close")
	nreq.Header.Add("user-agent", "pew pew pew")

	resp, err := p.Transport.RoundTrip(nreq)
	if err != nil {
		log.Println("Could not connect to endpoint", err)
		if strings.Contains(err.Error(), "timeout") {
			http.Error(w, "Error Fetching Resource", http.StatusBadGateway)
		} else {
			http.Error(w, "Error Fetching Resource", http.StatusNotFound)
		}
		return
	}
	defer resp.Body.Close()

	// check for too large a response
	if resp.ContentLength > p.MaxSize {
		log.Println("Content length exceeded", surl)
		http.Error(w, "Content length exceeded", http.StatusBadRequest)
		return
	}

	switch resp.StatusCode {
	case 300:
		log.Println("Multiple choices not supported")
		http.Error(w, "Multiple choices not supported", http.StatusNotFound)
		return
	case 301, 302, 303:
		// check for redirects. we do not follow.
		log.Println("Refusing to follow redirects")
		http.Error(w, "Refusing to follow redirects", http.StatusNotFound)
		return
	case 404:
		http.Error(w, "Not Found", http.StatusNotFound)
		return
	case 200:
		// check content type
		ct, ok := resp.Header[http.CanonicalHeaderKey("content-type")]
		if !ok || ct[0][:6] != "image/" {
			log.Println("Non-Image content-type returned", u)
			http.Error(w, "Non-Image content-type returned",
				http.StatusBadRequest)
			return
		}
	}

	for hdr, val := range resp.Header {
		h := w.Header()
		h[hdr] = val
	}

	h := w.Header()
	h.Add("X-Content-Type-Options", "nosniff")
	if resp.StatusCode == 304 && h.Get("Content-Type") != "" {
		h.Del("Content-Type")
	}

	w.WriteHeader(resp.StatusCode)
	io.Copy(w, resp.Body)
	//log.Println(req, resp.StatusCode)
}
<|endoftext|>


<commit_before>package engineio

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/kaicheng/engineio/parser"
	"github.com/kaicheng/events"
)

type Socket struct {
	events.EventEmitter

	id         string
	server     *Server
	upgraded   bool
	readyState string
	Request    *Request
	Transport  Transport

	writeBuffer []*parser.Packet
	bufferLock  sync.Mutex

	checkIntervalTimer  *ticker
	upgradeTimeoutTimer *time.Timer
	pingTimeoutTimer    *time.Timer

	timerLock      sync.Mutex
	readyStateLock sync.Mutex
}

func newSocket(id string, srv *Server, transport Transport, req *Request) *Socket {
	socket := new(Socket)
	socket.InitEventEmitter()

	socket.id = id
	socket.server = srv
	socket.upgraded = false
	socket.readyState = "opening"
	socket.Request = req
	socket.setTransport(transport)

	// TODO: make capacity configurable
	socket.writeBuffer = make([]*parser.Packet, 10)[0:0]

	socket.onOpen()
	return socket
}

func (socket *Socket) WriteBuffer() []*parser.Packet {
	socket.bufferLock.Lock()
	defer socket.bufferLock.Unlock()
	return socket.writeBuffer
}

func (socket *Socket) SetWriteBuffer(buf []*parser.Packet) {
	socket.bufferLock.Lock()
	defer socket.bufferLock.Unlock()
	socket.writeBuffer = buf
}

func (socket *Socket) onOpen() {
	socket.readyState = "open"
	socket.Transport.setSid(socket.id)
	pingInterval := (int64)(socket.server.pingInterval / time.Millisecond)
	pingTimeout := (int64)(socket.server.pingTimeout / time.Millisecond)
	upgrades, _ := json.Marshal(socket.getAvailableUpgrades())
	socket.sendPacket("open", []byte(fmt.Sprintf("{\"sid\":\"%s\",\"upgrades\":%s,\"pingInterval\":%d, \"pingTimeout\":%d}",
		socket.id, upgrades, pingInterval, pingTimeout)))

	socket.Emit("open")
	socket.setPingTimeout()
}

func (socket *Socket) onClose(reason, desc string) {
	if "closed" != socket.readyState {
		socket.timerLock.Lock()
		if socket.pingTimeoutTimer != nil {
			socket.pingTimeoutTimer.Stop()
		}
		socket.pingTimeoutTimer = nil
		if socket.checkIntervalTimer != nil {
			socket.checkIntervalTimer.stop()
		}
		socket.checkIntervalTimer = nil
		if socket.upgradeTimeoutTimer != nil {
			socket.upgradeTimeoutTimer.Stop()
		}
		socket.upgradeTimeoutTimer = nil
		socket.timerLock.Unlock()
		socket.clearTransport()
		socket.readyState = "closed"
		socket.Emit("close", reason, desc)
		socket.bufferLock.Lock()
		socket.writeBuffer = socket.writeBuffer[0:0]
		socket.bufferLock.Unlock()

	}
}

func (socket *Socket) sendPacket(strType string, data []byte) {
	if "closing" != socket.readyState {
		debug(fmt.Sprintf("sending packet \"%s\" (\"%s\")", strType, string(data)))
		packet := &parser.Packet{Type: strType, Data: data}
		socket.Emit("packetCreate", packet)
		socket.bufferLock.Lock()
		socket.writeBuffer = append(socket.writeBuffer, packet)
		socket.bufferLock.Unlock()
		socket.flush()
	}
}

func (socket *Socket) sendBinPacket(strType string, data []byte) {
	if "closing" != socket.readyState {
		debug(fmt.Sprintf("sending packet \"%s\" (\"%s\")", strType, string(data)))
		packet := &parser.Packet{Type: strType, Data: data, IsBin: true}
		socket.Emit("packetCreate", packet)
		socket.bufferLock.Lock()
		socket.writeBuffer = append(socket.writeBuffer, packet)
		socket.bufferLock.Unlock()
		socket.flush()
	}
}

func (socket *Socket) onPacket(packet *parser.Packet) {
	if "open" == socket.readyState {
		debug("packet ", packet.Type)
		debug("packet.Data", string(packet.Data))
		socket.Emit("packet", packet)

		socket.setPingTimeout()

		switch packet.Type {
		case "ping":
			debug("got ping")
			socket.sendPacket("pong", nil)
			socket.Emit("heartbeat")
		case "error":
			socket.onClose("parse error", "")
		case "message":
			socket.Emit("data", packet.Data)
			socket.Emit("message", packet.Data)
		}
	} else {
		debug("packet received with closed socket")
	}
}

func (socket *Socket) OnError(err string) {
	debug("transport error")
	socket.onClose("transport error", err)
}

func (socket *Socket) setPingTimeout() {
	socket.timerLock.Lock()
	defer socket.timerLock.Unlock()
	if socket.pingTimeoutTimer != nil {
		socket.pingTimeoutTimer.Stop()
	}
	socket.pingTimeoutTimer = time.AfterFunc(socket.server.pingInterval+socket.server.pingTimeout, func() {
		socket.onClose("ping timeout", "")
	})
}

func (socket *Socket) clearTransport() {
	socket.Transport.On("error", func(arg interface{}) {
		debug("error triggered by discarded transport")
	})
	socket.timerLock.Lock()
	defer socket.timerLock.Unlock()
	if socket.pingTimeoutTimer != nil {
		socket.pingTimeoutTimer.Stop()
	}
	socket.pingTimeoutTimer = nil
}

func (socket *Socket) setupSendCallback() {
}

func (socket *Socket) Send(data []byte) {
	socket.sendPacket("message", data)
}

func (socket *Socket) SendBin(data []byte) {
	socket.sendBinPacket("message", data)
}

func (socket *Socket) Write(data []byte) {
	socket.Send(data)
}

func (socket *Socket) flush() {
	socket.bufferLock.Lock()
	if "closed" != socket.readyState && len(socket.writeBuffer) > 0 {
		socket.bufferLock.Unlock()
		socket.Transport.tryWritable(func() {
			debug("flusing buffer to transport")
			socket.bufferLock.Lock()
			buf := socket.writeBuffer
			socket.writeBuffer = make([]*parser.Packet, 10)[0:0]
			socket.bufferLock.Unlock()
			socket.Emit("flush", buf)
			socket.server.Emit("flush", buf)
			socket.Transport.send(buf)
			socket.Emit("drain")
			socket.server.Emit("drain", socket)
		}, nil)
	} else {
		socket.bufferLock.Unlock()
	}
}

func (socket *Socket) getAvailableUpgrades() []string {
	return socket.server.upgrades(socket.Transport.Name())
}

func (socket *Socket) setTransport(transport Transport) {
	socket.Transport = transport
	transport.Once("error", socket.OnError)
	transport.On("packet", socket.onPacket)
	transport.On("drain", socket.flush)
	transport.Once("close", func() {
		debug("transport on close, closing")
		socket.onClose("transport close", "")
	})
	socket.setupSendCallback()
}

type funcBag struct {
	fn func(*parser.Packet)
}

func (socket *Socket) maybeUpgrade(transport Transport) {
	debug(fmt.Sprintf("might upgrade socket transport from \"%s\" to \"%s\"",
		socket.Transport.Name(), transport.Name()))

	socket.timerLock.Lock()
	socket.upgradeTimeoutTimer = time.AfterFunc(socket.server.upgradeTimeout,
		func() {
			debug("client did not complete upgrade - closing tansport")
			if "open" == transport.readyState() {
				transport.close(nil)
			}
		})
	socket.timerLock.Unlock()

	onPacket := new(funcBag)
	onPacket.fn = func(pkt *parser.Packet) {
		debug("onpacket", pkt.Type, socket.readyState)
		if "ping" == pkt.Type && "probe" == string(pkt.Data) {
			transport.send([]*parser.Packet{&parser.Packet{Type: "pong", Data: []byte("probe")}})
			socket.timerLock.Lock()
			if socket.checkIntervalTimer != nil {
				socket.checkIntervalTimer.stop()
			}
			// TODO: set as a parameter
			socket.checkIntervalTimer = newTicker(100 * time.Millisecond)
			go func(c <-chan time.Time, end <-chan bool) {
				for {
					select {
					case <-c:
						if "polling" == socket.Transport.Name() {
							socket.Transport.tryWritable(func() {
								debug("writing a noop packet to polling for fast upgrade")
								socket.Transport.send([]*parser.Packet{&parser.Packet{Type: "noop"}})
							}, nil)
						}
					case <-end:
						return
					}
				}
			}(socket.checkIntervalTimer.c, socket.checkIntervalTimer.end)
			socket.timerLock.Unlock()
		} else if "upgrade" == pkt.Type && socket.readyState == "open" {
			debug("got upgrade packet - upgrading")
			socket.timerLock.Lock()
			socket.upgradeTimeoutTimer.Stop()
			socket.timerLock.Unlock()
			transport.RemoveListener("packet", onPacket.fn)
			socket.upgraded = true
			socket.clearTransport()
			socket.setTransport(transport)
			socket.Emit("upgrade", transport)
			socket.flush()
			socket.timerLock.Lock()
			socket.checkIntervalTimer.stop()
			socket.checkIntervalTimer = nil
			socket.timerLock.Unlock()
			debug(fmt.Sprintf("upgrade to \"%s\" finishes", transport.Name()))
		} else {
			debug("invalid packet during upgrade")
			transport.close(nil)
		}
	}

	transport.On("packet", onPacket.fn)
}

func (socket *Socket) Close() {
	if "open" == socket.readyState {
		socket.readyState = "closing"
		socket.Transport.close(func() {
			socket.onClose("forced close", "")
		})
	}
}
<commit_msg>Maybe resolve race condition on socket.readyState and socket.transport.<commit_after>package engineio

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/kaicheng/engineio/parser"
	"github.com/kaicheng/events"
)

type Socket struct {
	events.EventEmitter

	id         string
	server     *Server
	upgraded   bool
	readyState string
	Request    *Request
	Transport  Transport

	writeBuffer []*parser.Packet

	checkIntervalTimer  *ticker
	upgradeTimeoutTimer *time.Timer
	pingTimeoutTimer    *time.Timer

	bufferLock     sync.Mutex
	timerLock      sync.Mutex
	readyStateLock sync.Mutex
	transportLock  sync.Mutex
}

func newSocket(id string, srv *Server, transport Transport, req *Request) *Socket {
	socket := new(Socket)
	socket.InitEventEmitter()

	socket.id = id
	socket.server = srv
	socket.upgraded = false
	socket.readyState = "opening"
	socket.Request = req
	socket.setTransport(transport)

	// TODO: make capacity configurable
	socket.writeBuffer = make([]*parser.Packet, 10)[0:0]

	socket.onOpen()
	return socket
}

func (socket *Socket) WriteBuffer() []*parser.Packet {
	socket.bufferLock.Lock()
	defer socket.bufferLock.Unlock()
	return socket.writeBuffer
}

func (socket *Socket) SetWriteBuffer(buf []*parser.Packet) {
	socket.bufferLock.Lock()
	defer socket.bufferLock.Unlock()
	socket.writeBuffer = buf
}

func (socket *Socket) getTransport() Transport {
	socket.transportLock.Lock()
	defer socket.transportLock.Unlock()
	return socket.Transport
}

func (socket *Socket) onOpen() {
	socket.readyState = "open"
	socket.getTransport().setSid(socket.id)
	pingInterval := (int64)(socket.server.pingInterval / time.Millisecond)
	pingTimeout := (int64)(socket.server.pingTimeout / time.Millisecond)
	upgrades, _ := json.Marshal(socket.getAvailableUpgrades())
	socket.sendPacket("open", []byte(fmt.Sprintf("{\"sid\":\"%s\",\"upgrades\":%s,\"pingInterval\":%d, \"pingTimeout\":%d}",
		socket.id, upgrades, pingInterval, pingTimeout)))

	socket.Emit("open")
	socket.setPingTimeout()
}

func (socket *Socket) onClose(reason, desc string) {
	socket.readyStateLock.Lock()
	if "closed" != socket.readyState {
		socket.timerLock.Lock()
		if socket.pingTimeoutTimer != nil {
			socket.pingTimeoutTimer.Stop()
		}
		socket.pingTimeoutTimer = nil
		if socket.checkIntervalTimer != nil {
			socket.checkIntervalTimer.stop()
		}
		socket.checkIntervalTimer = nil
		if socket.upgradeTimeoutTimer != nil {
			socket.upgradeTimeoutTimer.Stop()
		}
		socket.upgradeTimeoutTimer = nil
		socket.timerLock.Unlock()
		socket.clearTransport()
		socket.readyState = "closed"
		socket.readyStateLock.Unlock()
		socket.Emit("close", reason, desc)
		socket.bufferLock.Lock()
		socket.writeBuffer = socket.writeBuffer[0:0]
		socket.bufferLock.Unlock()
	} else {
		socket.readyStateLock.Unlock()
	}
}

func (socket *Socket) sendPacket(strType string, data []byte) {
	socket.readyStateLock.Lock()
	state := socket.readyState
	socket.readyStateLock.Unlock()
	if "closing" != state {
		debug(fmt.Sprintf("sending packet \"%s\" (\"%s\")", strType, string(data)))
		packet := &parser.Packet{Type: strType, Data: data}
		socket.Emit("packetCreate", packet)
		socket.bufferLock.Lock()
		socket.writeBuffer = append(socket.writeBuffer, packet)
		socket.bufferLock.Unlock()
		socket.flush()
	}
}

func (socket *Socket) sendBinPacket(strType string, data []byte) {
	socket.readyStateLock.Lock()
	state := socket.readyState
	socket.readyStateLock.Unlock()
	if "closing" != state {
		debug(fmt.Sprintf("sending packet \"%s\" (\"%s\")", strType, string(data)))
		packet := &parser.Packet{Type: strType, Data: data, IsBin: true}
		socket.Emit("packetCreate", packet)
		socket.bufferLock.Lock()
		socket.writeBuffer = append(socket.writeBuffer, packet)
		socket.bufferLock.Unlock()
		socket.flush()
	}
}

func (socket *Socket) onPacket(packet *parser.Packet) {
	socket.readyStateLock.Lock()
	state := socket.readyState
	socket.readyStateLock.Unlock()

	if "open" == state {
		debug("packet ", packet.Type)
		debug("packet.Data", string(packet.Data))
		socket.Emit("packet", packet)

		socket.setPingTimeout()

		switch packet.Type {
		case "ping":
			debug("got ping")
			socket.sendPacket("pong", nil)
			socket.Emit("heartbeat")
		case "error":
			socket.onClose("parse error", "")
		case "message":
			socket.Emit("data", packet.Data)
			socket.Emit("message", packet.Data)
		}
	} else {
		debug("packet received with closed socket")
	}
}

func (socket *Socket) OnError(err string) {
	debug("transport error")
	socket.onClose("transport error", err)
}

func (socket *Socket) setPingTimeout() {
	socket.timerLock.Lock()
	defer socket.timerLock.Unlock()
	if socket.pingTimeoutTimer != nil {
		socket.pingTimeoutTimer.Stop()
	}
	socket.pingTimeoutTimer = time.AfterFunc(socket.server.pingInterval+socket.server.pingTimeout, func() {
		socket.onClose("ping timeout", "")
	})
}

func (socket *Socket) clearTransport() {
	socket.getTransport().On("error", func(arg interface{}) {
		debug("error triggered by discarded transport")
	})
	socket.timerLock.Lock()
	defer socket.timerLock.Unlock()
	if socket.pingTimeoutTimer != nil {
		socket.pingTimeoutTimer.Stop()
	}
	socket.pingTimeoutTimer = nil
}

func (socket *Socket) setupSendCallback() {
}

func (socket *Socket) Send(data []byte) {
	socket.sendPacket("message", data)
}

func (socket *Socket) SendBin(data []byte) {
	socket.sendBinPacket("message", data)
}

func (socket *Socket) Write(data []byte) {
	socket.Send(data)
}

func (socket *Socket) flush() {
	socket.readyStateLock.Lock()
	state := socket.readyState
	socket.readyStateLock.Unlock()
	socket.bufferLock.Lock()
	if "closed" != state && len(socket.writeBuffer) > 0 {
		socket.bufferLock.Unlock()
		trans := socket.getTransport()
		trans.tryWritable(func() {
			debug("flusing buffer to transport")
			socket.bufferLock.Lock()
			buf := socket.writeBuffer
			socket.writeBuffer = make([]*parser.Packet, 10)[0:0]
			socket.bufferLock.Unlock()
			socket.Emit("flush", buf)
			socket.server.Emit("flush", buf)
			trans.send(buf)
			socket.Emit("drain")
			socket.server.Emit("drain", socket)
		}, nil)
	} else {
		socket.bufferLock.Unlock()
	}
}

func (socket *Socket) getAvailableUpgrades() []string {
	return socket.server.upgrades(socket.Transport.Name())
}

func (socket *Socket) setTransport(transport Transport) {
	socket.transportLock.Lock()
	socket.Transport = transport
	socket.transportLock.Unlock()
	transport.Once("error", socket.OnError)
	transport.On("packet", socket.onPacket)
	transport.On("drain", socket.flush)
	transport.Once("close", func() {
		debug("transport on close, closing")
		socket.onClose("transport close", "")
	})
	socket.setupSendCallback()
}

type funcBag struct {
	fn func(*parser.Packet)
}

func (socket *Socket) maybeUpgrade(transport Transport) {
	debug(fmt.Sprintf("might upgrade socket transport from \"%s\" to \"%s\"",
		socket.getTransport().Name(), transport.Name()))

	socket.timerLock.Lock()
	socket.upgradeTimeoutTimer = time.AfterFunc(socket.server.upgradeTimeout,
		func() {
			debug("client did not complete upgrade - closing tansport")
			if "open" == transport.readyState() {
				transport.close(nil)
			}
		})
	socket.timerLock.Unlock()

	onPacket := new(funcBag)
	onPacket.fn = func(pkt *parser.Packet) {
		if "ping" == pkt.Type && "probe" == string(pkt.Data) {
			transport.send([]*parser.Packet{&parser.Packet{Type: "pong", Data: []byte("probe")}})
			socket.timerLock.Lock()
			if socket.checkIntervalTimer != nil {
				socket.checkIntervalTimer.stop()
			}
			// TODO: set as a parameter
			socket.checkIntervalTimer = newTicker(100 * time.Millisecond)
			go func(c <-chan time.Time, end <-chan bool) {
				for {
					select {
					case <-c:
						trans := socket.getTransport()
						if "polling" == trans.Name() {
							trans.tryWritable(func() {
								debug("writing a noop packet to polling for fast upgrade")
								trans.send([]*parser.Packet{&parser.Packet{Type: "noop"}})
							}, nil)
						}
					case <-end:
						return
					}
				}
			}(socket.checkIntervalTimer.c, socket.checkIntervalTimer.end)
			socket.timerLock.Unlock()
		} else if "upgrade" == pkt.Type {
			socket.readyStateLock.Lock()
			state := socket.readyState
			socket.readyStateLock.Unlock()
			if state == "open" {
				debug("got upgrade packet - upgrading")
				socket.timerLock.Lock()
				socket.upgradeTimeoutTimer.Stop()
				socket.timerLock.Unlock()
				transport.RemoveListener("packet", onPacket.fn)
				socket.upgraded = true
				socket.clearTransport()
				socket.setTransport(transport)
				socket.Emit("upgrade", transport)
				socket.flush()
				socket.timerLock.Lock()
				socket.checkIntervalTimer.stop()
				socket.checkIntervalTimer = nil
				socket.timerLock.Unlock()
				debug(fmt.Sprintf("upgrade to \"%s\" finishes", transport.Name()))
			}
		} else {
			debug("invalid packet during upgrade")
			transport.close(nil)
		}
	}

	transport.On("packet", onPacket.fn)
}

func (socket *Socket) Close() {
	socket.readyStateLock.Lock()
	if "open" == socket.readyState {
		socket.readyState = "closing"
		socket.readyStateLock.Unlock()
		socket.getTransport().close(func() {
			socket.onClose("forced close", "")
		})
	} else {
		socket.readyStateLock.Unlock()
	}
}
<|endoftext|>