<commit_before>package siemens

import (
	"bytes"
	"encoding/binary"
	"net"
	"time"
)

func GetS7Banner(logStruct *S7Log, connection net.Conn) (err error) {
	// Attempt connection
	var connPacketBytes, connResponseBytes []byte
	connPacketBytes, err = makeCOTPConnectionPacketBytes(uint16(0x102), uint16(0x100))
	if err != nil {
		return err
	}

	connResponseBytes, err = sendRequestReadResponse(connection, connPacketBytes)
	if connResponseBytes == nil || len(connResponseBytes) == 0 || err != nil {
		address := connection.RemoteAddr().String()
		connection.Close()
		connection, err = makeNewConnection(address)
		if err != nil {
			return err
		}

		connPacketBytes, err = makeCOTPConnectionPacketBytes(uint16(0x200), uint16(0x100))
		if err != nil {
			return err
		}
		connResponseBytes, err = sendRequestReadResponse(connection, connPacketBytes)
		if err != nil {
			return err
		}
	}

	_, err = unmarshalCOTPConnectionResponse(connResponseBytes)
	if err != nil {
		return err
	}

	// Negotiate S7
	requestPacketBytes, err := makeRequestPacketBytes(S7_REQUEST, makeNegotiatePDUParamBytes(), nil)
	if err != nil {
		return err
	}
	_, err = sendRequestReadResponse(connection, requestPacketBytes)
	if err != nil {
		return err
	}

	logStruct.IsS7 = true

	// Make Module Identification request
	moduleIdentificationResponse, err := readRequest(connection, S7_SZL_MODULE_IDENTIFICATION)
	if err != nil {
		return nil // mask errors after detecting IsS7
	}
	parseModuleIdentificatioNRequest(logStruct, &moduleIdentificationResponse)

	// Make Component Identification request
	componentIdentificationResponse, err := readRequest(connection, S7_SZL_COMPONENT_IDENTIFICATION)
	if err != nil {
		return nil // mask errors after detecting IsS7
	}
	parseComponentIdentificationResponse(logStruct, &componentIdentificationResponse)

	return nil
}

func makeNewConnection(address string) (newConnection net.Conn, err error) {
	newConnection, err = net.Dial("tcp", address)

	if err != nil {
		return newConnection, err
	}
	newConnection.SetDeadline(time.Now().Add(time.Second * 3))

	return newConnection, nil
}

func makeCOTPConnectionPacketBytes(dstTsap uint16, srcTsap uint16) ([]byte, error) {
	var cotpConnPacket COTPConnectionPacket
	cotpConnPacket.DestinationRef = uint16(0x00) // nmap uses 0x00
	cotpConnPacket.SourceRef = uint16(0x04)      // nmap uses 0x14
	cotpConnPacket.DestinationTSAP = dstTsap
	cotpConnPacket.SourceTSAP = srcTsap
	cotpConnPacket.TPDUSize = byte(0x0a) // nmap uses 0x0a

	cotpConnPacketBytes, err := cotpConnPacket.Marshal()
	if err != nil {
		return nil, err
	}

	var tpktPacket TPKTPacket
	tpktPacket.Data = cotpConnPacketBytes
	bytes, err := tpktPacket.Marshal()
	if err != nil {
		return nil, err
	}

	return bytes, nil
}

func makeRequestPacketBytes(pduType byte, parameters []byte, data []byte) ([]byte, error) {
	var s7Packet S7Packet
	s7Packet.PDUType = pduType
	s7Packet.RequestId = S7_REQUEST_ID
	s7Packet.Parameters = parameters
	s7Packet.Data = data
	s7PacketBytes, err := s7Packet.Marshal()
	if err != nil {
		return nil, err
	}

	var cotpDataPacket COTPDataPacket
	cotpDataPacket.Data = s7PacketBytes
	cotpDataPacketBytes, err := cotpDataPacket.Marshal()
	if err != nil {
		return nil, err
	}

	var tpktPacket TPKTPacket
	tpktPacket.Data = cotpDataPacketBytes
	bytes, err := tpktPacket.Marshal()
	if err != nil {
		return nil, err
	}
	return bytes, nil
}

// Send a generic packet request and return the response
func sendRequestReadResponse(connection net.Conn, requestBytes []byte) ([]byte, error) {
	connection.Write(requestBytes)
	responseBytes := make([]byte, 2048)
	bytesRead, err := connection.Read(responseBytes)
	if err != nil {
		return nil, err
	}

	return responseBytes[0:bytesRead], nil
}

func unmarshalCOTPConnectionResponse(responseBytes []byte) (cotpConnPacket COTPConnectionPacket, err error) {
	var tpktPacket TPKTPacket
	if err := tpktPacket.Unmarshal(responseBytes); err != nil {
		return cotpConnPacket, err
	}

	if err := cotpConnPacket.Unmarshal(tpktPacket.Data); err != nil {
		return cotpConnPacket, err
	}

	return cotpConnPacket, nil
}

func makeNegotiatePDUParamBytes() (bytes []byte) {
	uint16BytesHolder := make([]byte, 2)
	bytes = make([]byte, 0, 8)        // fixed param length for negotiating PDU params
	bytes = append(bytes, byte(0xf0)) // negotiate PDU function code
	bytes = append(bytes, byte(0))    // ?
	binary.BigEndian.PutUint16(uint16BytesHolder, 0x01)
	bytes = append(bytes, uint16BytesHolder...) // min # of parallel jobs
	binary.BigEndian.PutUint16(uint16BytesHolder, 0x01)
	bytes = append(bytes, uint16BytesHolder...) // max # of parallel jobs

	return bytes
}

func makeReadRequestParamBytes(data []byte) (bytes []byte) {
	bytes = make([]byte, 0, 16)

	bytes = append(bytes, byte(0x00)) // magic parameter
	bytes = append(bytes, byte(0x01)) // magic parameter
	bytes = append(bytes, byte(0x12)) // magic parameter
	bytes = append(bytes, byte(0x04)) // param length
	bytes = append(bytes, byte(0x11)) // ?
	bytes = append(bytes, byte((S7_SZL_REQUEST*0x10)+S7_SZL_FUNCTIONS))
	bytes = append(bytes, byte(S7_SZL_READ))
	bytes = append(bytes, byte(0))

	return bytes
}

func makeReadRequestDataBytes(szlId uint16) []byte {
	bytes := make([]byte, 0, 4)
	bytes = append(bytes, byte(0xff))
	bytes = append(bytes, byte(0x09))
	uint16BytesHolder := make([]byte, 2)
	binary.BigEndian.PutUint16(uint16BytesHolder, uint16(4)) // size of subsequent data
	bytes = append(bytes, uint16BytesHolder...)
	binary.BigEndian.PutUint16(uint16BytesHolder, szlId)
	bytes = append(bytes, uint16BytesHolder...) // szl id
	binary.BigEndian.PutUint16(uint16BytesHolder, 1)
	bytes = append(bytes, uint16BytesHolder...) // szl index

	return bytes
}

func makeReadRequestBytes(szlId uint16) ([]byte, error) {
	readRequestParamBytes := makeReadRequestParamBytes(makeReadRequestDataBytes(szlId))
	readRequestBytes, err := makeRequestPacketBytes(S7_REQUEST_USER_DATA, readRequestParamBytes, makeReadRequestDataBytes(szlId))
	if err != nil {
		return nil, err
	}

	return readRequestBytes, nil
}

func unmarshalReadResponse(bytes []byte) (S7Packet, error) {
	var tpktPacket TPKTPacket
	var cotpDataPacket COTPDataPacket
	var s7Packet S7Packet
	if err := tpktPacket.Unmarshal(bytes); err != nil {
		return s7Packet, err
	}

	if err := cotpDataPacket.Unmarshal(tpktPacket.Data); err != nil {
		return s7Packet, err
	}

	if err := s7Packet.Unmarshal(cotpDataPacket.Data); err != nil {
		return s7Packet, err
	}

	return s7Packet, nil
}

func parseComponentIdentificationResponse(logStruct *S7Log, s7Packet *S7Packet) error {
	if len(s7Packet.Data) < S7_DATA_BYTE_OFFSET {
		return errS7PacketTooShort
	}

	fields := bytes.FieldsFunc(s7Packet.Data[S7_DATA_BYTE_OFFSET:], func(c rune) bool {
		return int(c) == 0
	})

	for i := len(fields) - 1; i >= 0; i-- {
		switch i {
		case 0:
			logStruct.System = string(fields[i][1:]) // exclude index byte
		case 1:
			logStruct.Module = string(fields[i][1:])
		case 2:
			logStruct.PlantId = string(fields[i][1:])
		case 3:
			logStruct.Copyright = string(fields[i][1:])
		case 4:
			logStruct.SerialNumber = string(fields[i][1:])
		case 5:
			logStruct.ModuleType = string(fields[i][1:])
		case 6:
			logStruct.ReservedForOS = string(fields[i][1:])
		case 7:
			logStruct.MemorySerialNumber = string(fields[i][1:])
		case 8:
			logStruct.CpuProfile = string(fields[i][1:])
		case 9:
			logStruct.OEMId = string(fields[i][1:])
		case 10:
			logStruct.Location = string(fields[i][1:])
		}
	}

	return nil
}

func parseModuleIdentificatioNRequest(logStruct *S7Log, s7Packet *S7Packet) error {
	if len(s7Packet.Data) < S7_DATA_BYTE_OFFSET {
		return errS7PacketTooShort
	}

	fields := bytes.FieldsFunc(s7Packet.Data[S7_DATA_BYTE_OFFSET:], func(c rune) bool {
		return int(c) == 0
	})

	for i := len(fields) - 1; i >= 0; i-- {
		switch i {
		case 0:
			logStruct.ModuleId = string(fields[i][1:]) // exclude index byte
		case 5:
			logStruct.Hardware = string(fields[i][1:])
		case 6:
			logStruct.Firmware = string(fields[i][1:])
		}
	}

	return nil
}

func readRequest(connection net.Conn, slzId uint16) (packet S7Packet, err error) {
	readRequestBytes, err := makeReadRequestBytes(slzId)
	if err != nil {
		return packet, err
	}
	readResponse, err := sendRequestReadResponse(connection, readRequestBytes)
	if err != nil {
		return packet, err
	}
	packet, err = unmarshalReadResponse(readResponse)
	if err != nil {
		return packet, err
	}

	return packet, nil
}
<commit_msg>add pdu length field to s7 request<commit_after>package siemens

import (
	"bytes"
	"encoding/binary"
	"net"
	"time"
)

func GetS7Banner(logStruct *S7Log, connection net.Conn) (err error) {
	// Attempt connection
	var connPacketBytes, connResponseBytes []byte
	connPacketBytes, err = makeCOTPConnectionPacketBytes(uint16(0x102), uint16(0x100))
	if err != nil {
		return err
	}

	connResponseBytes, err = sendRequestReadResponse(connection, connPacketBytes)
	if connResponseBytes == nil || len(connResponseBytes) == 0 || err != nil {
		address := connection.RemoteAddr().String()
		connection.Close()
		connection, err = makeNewConnection(address)
		if err != nil {
			return err
		}

		connPacketBytes, err = makeCOTPConnectionPacketBytes(uint16(0x200), uint16(0x100))
		if err != nil {
			return err
		}
		connResponseBytes, err = sendRequestReadResponse(connection, connPacketBytes)
		if err != nil {
			return err
		}
	}

	_, err = unmarshalCOTPConnectionResponse(connResponseBytes)
	if err != nil {
		return err
	}

	// Negotiate S7
	requestPacketBytes, err := makeRequestPacketBytes(S7_REQUEST, makeNegotiatePDUParamBytes(), nil)
	if err != nil {
		return err
	}
	_, err = sendRequestReadResponse(connection, requestPacketBytes)
	if err != nil {
		return err
	}

	logStruct.IsS7 = true

	// Make Module Identification request
	moduleIdentificationResponse, err := readRequest(connection, S7_SZL_MODULE_IDENTIFICATION)
	if err != nil {
		return nil // mask errors after detecting IsS7
	}
	parseModuleIdentificatioNRequest(logStruct, &moduleIdentificationResponse)

	// Make Component Identification request
	componentIdentificationResponse, err := readRequest(connection, S7_SZL_COMPONENT_IDENTIFICATION)
	if err != nil {
		return nil // mask errors after detecting IsS7
	}
	parseComponentIdentificationResponse(logStruct, &componentIdentificationResponse)

	return nil
}

func makeNewConnection(address string) (newConnection net.Conn, err error) {
	newConnection, err = net.Dial("tcp", address)

	if err != nil {
		return newConnection, err
	}
	newConnection.SetDeadline(time.Now().Add(time.Second * 3))

	return newConnection, nil
}

func makeCOTPConnectionPacketBytes(dstTsap uint16, srcTsap uint16) ([]byte, error) {
	var cotpConnPacket COTPConnectionPacket
	cotpConnPacket.DestinationRef = uint16(0x00) // nmap uses 0x00
	cotpConnPacket.SourceRef = uint16(0x04)      // nmap uses 0x14
	cotpConnPacket.DestinationTSAP = dstTsap
	cotpConnPacket.SourceTSAP = srcTsap
	cotpConnPacket.TPDUSize = byte(0x0a) // nmap uses 0x0a

	cotpConnPacketBytes, err := cotpConnPacket.Marshal()
	if err != nil {
		return nil, err
	}

	var tpktPacket TPKTPacket
	tpktPacket.Data = cotpConnPacketBytes
	bytes, err := tpktPacket.Marshal()
	if err != nil {
		return nil, err
	}

	return bytes, nil
}

func makeRequestPacketBytes(pduType byte, parameters []byte, data []byte) ([]byte, error) {
	var s7Packet S7Packet
	s7Packet.PDUType = pduType
	s7Packet.RequestId = S7_REQUEST_ID
	s7Packet.Parameters = parameters
	s7Packet.Data = data
	s7PacketBytes, err := s7Packet.Marshal()
	if err != nil {
		return nil, err
	}

	var cotpDataPacket COTPDataPacket
	cotpDataPacket.Data = s7PacketBytes
	cotpDataPacketBytes, err := cotpDataPacket.Marshal()
	if err != nil {
		return nil, err
	}

	var tpktPacket TPKTPacket
	tpktPacket.Data = cotpDataPacketBytes
	bytes, err := tpktPacket.Marshal()
	if err != nil {
		return nil, err
	}
	return bytes, nil
}

// Send a generic packet request and return the response
func sendRequestReadResponse(connection net.Conn, requestBytes []byte) ([]byte, error) {
	connection.Write(requestBytes)
	responseBytes := make([]byte, 2048)
	bytesRead, err := connection.Read(responseBytes)
	if err != nil {
		return nil, err
	}

	return responseBytes[0:bytesRead], nil
}

func unmarshalCOTPConnectionResponse(responseBytes []byte) (cotpConnPacket COTPConnectionPacket, err error) {
	var tpktPacket TPKTPacket
	if err := tpktPacket.Unmarshal(responseBytes); err != nil {
		return cotpConnPacket, err
	}

	if err := cotpConnPacket.Unmarshal(tpktPacket.Data); err != nil {
		return cotpConnPacket, err
	}

	return cotpConnPacket, nil
}

func makeNegotiatePDUParamBytes() (bytes []byte) {
	uint16BytesHolder := make([]byte, 2)
	bytes = make([]byte, 0, 8)        // fixed param length for negotiating PDU params
	bytes = append(bytes, byte(0xf0)) // negotiate PDU function code
	bytes = append(bytes, byte(0))    // ?
	binary.BigEndian.PutUint16(uint16BytesHolder, 0x01)
	bytes = append(bytes, uint16BytesHolder...) // min # of parallel jobs
	binary.BigEndian.PutUint16(uint16BytesHolder, 0x01)
	bytes = append(bytes, uint16BytesHolder...) // max # of parallel jobs
	binary.BigEndian.PutUint16(uint16BytesHolder, 0x01e0)
	bytes = append(bytes, uint16BytesHolder...) // pdu length
	return bytes
}

func makeReadRequestParamBytes(data []byte) (bytes []byte) {
	bytes = make([]byte, 0, 16)

	bytes = append(bytes, byte(0x00)) // magic parameter
	bytes = append(bytes, byte(0x01)) // magic parameter
	bytes = append(bytes, byte(0x12)) // magic parameter
	bytes = append(bytes, byte(0x04)) // param length
	bytes = append(bytes, byte(0x11)) // ?
	bytes = append(bytes, byte((S7_SZL_REQUEST*0x10)+S7_SZL_FUNCTIONS))
	bytes = append(bytes, byte(S7_SZL_READ))
	bytes = append(bytes, byte(0))

	return bytes
}

func makeReadRequestDataBytes(szlId uint16) []byte {
	bytes := make([]byte, 0, 4)
	bytes = append(bytes, byte(0xff))
	bytes = append(bytes, byte(0x09))
	uint16BytesHolder := make([]byte, 2)
	binary.BigEndian.PutUint16(uint16BytesHolder, uint16(4)) // size of subsequent data
	bytes = append(bytes, uint16BytesHolder...)
	binary.BigEndian.PutUint16(uint16BytesHolder, szlId)
	bytes = append(bytes, uint16BytesHolder...) // szl id
	binary.BigEndian.PutUint16(uint16BytesHolder, 1)
	bytes = append(bytes, uint16BytesHolder...) // szl index

	return bytes
}

func makeReadRequestBytes(szlId uint16) ([]byte, error) {
	readRequestParamBytes := makeReadRequestParamBytes(makeReadRequestDataBytes(szlId))
	readRequestBytes, err := makeRequestPacketBytes(S7_REQUEST_USER_DATA, readRequestParamBytes, makeReadRequestDataBytes(szlId))
	if err != nil {
		return nil, err
	}

	return readRequestBytes, nil
}

func unmarshalReadResponse(bytes []byte) (S7Packet, error) {
	var tpktPacket TPKTPacket
	var cotpDataPacket COTPDataPacket
	var s7Packet S7Packet
	if err := tpktPacket.Unmarshal(bytes); err != nil {
		return s7Packet, err
	}

	if err := cotpDataPacket.Unmarshal(tpktPacket.Data); err != nil {
		return s7Packet, err
	}

	if err := s7Packet.Unmarshal(cotpDataPacket.Data); err != nil {
		return s7Packet, err
	}

	return s7Packet, nil
}

func parseComponentIdentificationResponse(logStruct *S7Log, s7Packet *S7Packet) error {
	if len(s7Packet.Data) < S7_DATA_BYTE_OFFSET {
		return errS7PacketTooShort
	}

	fields := bytes.FieldsFunc(s7Packet.Data[S7_DATA_BYTE_OFFSET:], func(c rune) bool {
		return int(c) == 0
	})

	for i := len(fields) - 1; i >= 0; i-- {
		switch i {
		case 0:
			logStruct.System = string(fields[i][1:]) // exclude index byte
		case 1:
			logStruct.Module = string(fields[i][1:])
		case 2:
			logStruct.PlantId = string(fields[i][1:])
		case 3:
			logStruct.Copyright = string(fields[i][1:])
		case 4:
			logStruct.SerialNumber = string(fields[i][1:])
		case 5:
			logStruct.ModuleType = string(fields[i][1:])
		case 6:
			logStruct.ReservedForOS = string(fields[i][1:])
		case 7:
			logStruct.MemorySerialNumber = string(fields[i][1:])
		case 8:
			logStruct.CpuProfile = string(fields[i][1:])
		case 9:
			logStruct.OEMId = string(fields[i][1:])
		case 10:
			logStruct.Location = string(fields[i][1:])
		}
	}

	return nil
}

func parseModuleIdentificatioNRequest(logStruct *S7Log, s7Packet *S7Packet) error {
	if len(s7Packet.Data) < S7_DATA_BYTE_OFFSET {
		return errS7PacketTooShort
	}

	fields := bytes.FieldsFunc(s7Packet.Data[S7_DATA_BYTE_OFFSET:], func(c rune) bool {
		return int(c) == 0
	})

	for i := len(fields) - 1; i >= 0; i-- {
		switch i {
		case 0:
			logStruct.ModuleId = string(fields[i][1:]) // exclude index byte
		case 5:
			logStruct.Hardware = string(fields[i][1:])
		case 6:
			logStruct.Firmware = string(fields[i][1:])
		}
	}

	return nil
}

func readRequest(connection net.Conn, slzId uint16) (packet S7Packet, err error) {
	readRequestBytes, err := makeReadRequestBytes(slzId)
	if err != nil {
		return packet, err
	}
	readResponse, err := sendRequestReadResponse(connection, readRequestBytes)
	if err != nil {
		return packet, err
	}
	packet, err = unmarshalReadResponse(readResponse)
	if err != nil {
		return packet, err
	}

	return packet, nil
}
<|endoftext|>


<commit_before>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import "gopkg.in/yaml.v2"

const (
	ObjectMetaSchema = `properties:
  name:
    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
    type: string
  namespace:
    description: "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces"
    type: string
  labels:
    additionalProperties:
      type: string
    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
    type: object
  annotations:
    additionalProperties:
      type: string
    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
    type: object
type: object`

	ObjectMetaFullSchema = `properties:
  name:
    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
    type: string
  namespace:
    description: "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces"
    type: string
  labels:
    additionalProperties:
      type: string
    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
    type: object
  annotations:
    additionalProperties:
      type: string
    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
    type: object
  clusterName:
    description: The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.
    type: string
  creationTimestamp:
    description: "CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. \n Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"
    format: date-time
    type: string
  deletionGracePeriodSeconds:
    description: Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.
    format: int64
    type: integer
  deletionTimestamp:
    description: "DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested. \n Populated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"
    format: date-time
    type: string
  finalizers:
    description: Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order.  Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.
    items:
      type: string
    type: array
  generateName:
    description: "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server. \n If this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header). \n Applied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency"
    type: string
  generation:
    description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.
    format: int64
    type: integer
  managedFields:
    description: ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like "ci-cd". The set of fields is always in the version that the workflow used when modifying the object.
    items:
      description: ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.
      properties:
        apiVersion:
          description: APIVersion defines the version of this resource that this field set applies to. The format is "group/version" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.
          type: string
        fieldsType:
          description: 'FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: "FieldsV1"'
          type: string
        fieldsV1:
          description: FieldsV1 holds the first JSON version format as described in the "FieldsV1" type.
          type: object
        manager:
          description: Manager is an identifier of the workflow managing these fields.
          type: string
        operation:
          description: Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.
          type: string
        time:
          description: Time is timestamp of when these fields were set. It should always be empty if Operation is 'Apply'
          format: date-time
          type: string
      type: object
    type: array
  ownerReferences:
    description: List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.
    items:
      description: OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.
      properties:
        apiVersion:
          description: API version of the referent.
          type: string
        blockOwnerDeletion:
          description: If true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs "delete" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.
          type: boolean
        controller:
          description: If true, this reference points to the managing controller.
          type: boolean
        kind:
          description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
          type: string
        name:
          description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
          type: string
        uid:
          description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids'
          type: string
      required:
      - apiVersion
      - kind
      - name
      - uid
      type: object
    type: array
  resourceVersion:
    description: "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources. \n Populated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency"
    type: string
  selfLink:
    description: "SelfLink is a URL representing this object. Populated by the system. Read-only. \n DEPRECATED Kubernetes will stop propagating this field in 1.20 release and the field is planned to be removed in 1.21 release."
    type: string
  uid:
    description: "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations. \n Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids"
    type: string
type: object`
)

func FormatMetadata(data []byte) ([]byte, error) {
	var root yaml.MapSlice
	err := yaml.Unmarshal(data, &root)
	if err != nil {
		return nil, err
	}

	spec := find(root, "spec")
	if spec == nil {
		return data, nil
	}
	validation := find(spec.(yaml.MapSlice), "validation")
	if validation == nil {
		return data, nil
	}
	openAPIV3Schema := find(validation.(yaml.MapSlice), "openAPIV3Schema")
	if openAPIV3Schema == nil {
		return data, nil
	}
	prop1 := find(openAPIV3Schema.(yaml.MapSlice), "properties")
	if prop1 == nil {
		return data, nil
	}
	metadata := find(prop1.(yaml.MapSlice), "metadata")
	if metadata == nil {
		return data, nil
	}
	prop2 := find(metadata.(yaml.MapSlice), "properties")
	if prop2 == nil {
		return data, nil
	}
	p := prop2.(yaml.MapSlice)
	moveToIndex(0, "name", p)
	if _, ok := moveToIndex(1, "namespace", p); ok {
		moveToIndex(2, "labels", p)
		moveToIndex(3, "annotations", p)
	} else {
		moveToIndex(1, "labels", p)
		moveToIndex(2, "annotations", p)
	}

	return yaml.Marshal(root)
}

func find(node yaml.MapSlice, key string) interface{} {
	for i := range node {
		if node[i].Key.(string) == key {
			return node[i].Value
		}
	}
	return nil
}

func moveToIndex(idx int, needle string, haystack yaml.MapSlice) (yaml.MapSlice, bool) {
	if len(haystack) != idx && haystack[0].Key.(string) == needle {
		return haystack, true
	}
	for i, elem := range haystack {
		if elem.Key.(string) == needle {
			haystack[idx], haystack[i] = haystack[i], haystack[idx]
			return haystack, true
		}
	}
	return nil, false
}
<commit_msg>Fix linter error (#97)<commit_after>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import "gopkg.in/yaml.v2"

const (
	ObjectMetaSchema = `properties:
  name:
    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
    type: string
  namespace:
    description: "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces"
    type: string
  labels:
    additionalProperties:
      type: string
    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
    type: object
  annotations:
    additionalProperties:
      type: string
    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
    type: object
type: object`

	ObjectMetaFullSchema = `properties:
  name:
    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
    type: string
  namespace:
    description: "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces"
    type: string
  labels:
    additionalProperties:
      type: string
    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
    type: object
  annotations:
    additionalProperties:
      type: string
    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
    type: object
  clusterName:
    description: The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.
    type: string
  creationTimestamp:
    description: "CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. \n Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"
    format: date-time
    type: string
  deletionGracePeriodSeconds:
    description: Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.
    format: int64
    type: integer
  deletionTimestamp:
    description: "DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested. \n Populated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"
    format: date-time
    type: string
  finalizers:
    description: Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order.  Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.
    items:
      type: string
    type: array
  generateName:
    description: "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server. \n If this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header). \n Applied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency"
    type: string
  generation:
    description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.
    format: int64
    type: integer
  managedFields:
    description: ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like "ci-cd". The set of fields is always in the version that the workflow used when modifying the object.
    items:
      description: ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.
      properties:
        apiVersion:
          description: APIVersion defines the version of this resource that this field set applies to. The format is "group/version" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.
          type: string
        fieldsType:
          description: 'FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: "FieldsV1"'
          type: string
        fieldsV1:
          description: FieldsV1 holds the first JSON version format as described in the "FieldsV1" type.
          type: object
        manager:
          description: Manager is an identifier of the workflow managing these fields.
          type: string
        operation:
          description: Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.
          type: string
        time:
          description: Time is timestamp of when these fields were set. It should always be empty if Operation is 'Apply'
          format: date-time
          type: string
      type: object
    type: array
  ownerReferences:
    description: List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.
    items:
      description: OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.
      properties:
        apiVersion:
          description: API version of the referent.
          type: string
        blockOwnerDeletion:
          description: If true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs "delete" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.
          type: boolean
        controller:
          description: If true, this reference points to the managing controller.
          type: boolean
        kind:
          description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
          type: string
        name:
          description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
          type: string
        uid:
          description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids'
          type: string
      required:
      - apiVersion
      - kind
      - name
      - uid
      type: object
    type: array
  resourceVersion:
    description: "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources. \n Populated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency"
    type: string
  selfLink:
    description: "SelfLink is a URL representing this object. Populated by the system. Read-only. \n DEPRECATED Kubernetes will stop propagating this field in 1.20 release and the field is planned to be removed in 1.21 release."
    type: string
  uid:
    description: "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations. \n Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids"
    type: string
type: object`
)

func FormatMetadata(data []byte) ([]byte, error) {
	var root yaml.MapSlice
	err := yaml.Unmarshal(data, &root)
	if err != nil {
		return nil, err
	}

	spec := find(root, "spec")
	if spec == nil {
		return data, nil
	}
	validation := find(spec.(yaml.MapSlice), "validation")
	if validation == nil {
		return data, nil
	}
	openAPIV3Schema := find(validation.(yaml.MapSlice), "openAPIV3Schema")
	if openAPIV3Schema == nil {
		return data, nil
	}
	prop1 := find(openAPIV3Schema.(yaml.MapSlice), "properties")
	if prop1 == nil {
		return data, nil
	}
	metadata := find(prop1.(yaml.MapSlice), "metadata")
	if metadata == nil {
		return data, nil
	}
	prop2 := find(metadata.(yaml.MapSlice), "properties")
	if prop2 == nil {
		return data, nil
	}
	p := prop2.(yaml.MapSlice)
	moveToIndex(0, "name", p)
	if moveToIndex(1, "namespace", p) {
		moveToIndex(2, "labels", p)
		moveToIndex(3, "annotations", p)
	} else {
		moveToIndex(1, "labels", p)
		moveToIndex(2, "annotations", p)
	}

	return yaml.Marshal(root)
}

func find(node yaml.MapSlice, key string) interface{} {
	for i := range node {
		if node[i].Key.(string) == key {
			return node[i].Value
		}
	}
	return nil
}

func moveToIndex(idx int, needle string, haystack yaml.MapSlice) bool {
	if len(haystack) != idx && haystack[0].Key.(string) == needle {
		return true
	}
	for i, elem := range haystack {
		if elem.Key.(string) == needle {
			haystack[idx], haystack[i] = haystack[i], haystack[idx]
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package memory

import (
	"fmt"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

// PriorityRWMutex is a mutex that allows fair access to high priority read operations while
// blocking low priority operations if a write lock is waiting.
type PriorityRWMutex struct {
	lock        sync.RWMutex
	lowPrioLock sync.RWMutex
}

type BlockContext struct {
	lock         *PriorityRWMutex
	preLockTime  time.Time
	postLockTime time.Time
	postOpTime   time.Time
}

// RLockHigh allows *guaranteed fast* (or very high priority) ops to acquire the read lock as
// quickly as possible. Note: This can block write lock acquisition
func (pm *PriorityRWMutex) RLockHigh() {
	// Don't increment activeReads just grab the lock. If an active write is holding the lock it
	// will block, otherwise it *may* block a write acquisition, but that is opted into.
	pm.lock.RLock()
}

// RUnlockHigh unlocks read lock called via RLockHigh
func (pm *PriorityRWMutex) RUnlockHigh() {
	pm.lock.RUnlock()
}

// Synonym for RLockLow
func (pm *PriorityRWMutex) RLock() BlockContext {
	return pm.RLockLow()
}

// Synonym for RUnlockLow
func (pm *PriorityRWMutex) RUnlock(bc *BlockContext) {
	pm.RUnlockLow(bc)
}

// RLockLow will block until requested writes complete. New writes that come in also get priority.
func (pm *PriorityRWMutex) RLockLow() BlockContext {
	// Wait for any pending writes
	bc := BlockContext{lock: pm, preLockTime: time.Now()}
	pm.lowPrioLock.RLock()
	pm.lock.RLock()
	bc.postLockTime = time.Now()
	return bc
}

// RUnlockLow unlocks read lock called via RLockLow
func (pm *PriorityRWMutex) RUnlockLow(bc *BlockContext) {
	bc.postOpTime = time.Now()
	pm.lock.RUnlock()
	pm.lowPrioLock.RUnlock()
}

// Lock will block new low prio read locks until can acquire the lock. High prio read locks will
// not be blocked or accounted for.
func (pm *PriorityRWMutex) Lock() BlockContext {
	bc := BlockContext{lock: pm, preLockTime: time.Now()}
	pm.lowPrioLock.Lock()
	pm.lock.Lock()
	bc.postLockTime = time.Now()
	return bc
}

// Unlock unlocks mutex acquired via Lock
func (pm *PriorityRWMutex) Unlock(bc *BlockContext) {
	bc.postOpTime = time.Now()
	pm.lowPrioLock.Unlock()
	pm.lock.Unlock()
}

func (bc *BlockContext) RUnlockLow(op string, logCb func() interface{}) {
	bc.lock.RUnlockLow(bc)

	lockWaitTime := bc.postLockTime.Sub(bc.preLockTime)
	lockHoldTime := bc.postOpTime.Sub(bc.postLockTime)

	if lockHoldTime > 5*time.Second {
		bc.logLongOp("Read", op, getDetails(logCb))
	} else if lockWaitTime > time.Second && lockWaitTime > 5*lockHoldTime {
		bc.logLongOp("Blocked Read", op, getDetails(logCb))
	}
}

func (bc *BlockContext) Unlock(op string, logCb func() interface{}) {
	bc.lock.Unlock(bc)

	lockWaitTime := bc.postLockTime.Sub(bc.preLockTime)
	lockHoldTime := bc.postOpTime.Sub(bc.postLockTime)

	if lockHoldTime > 100*time.Millisecond {
		bc.logLongOp("Write", op, getDetails(logCb))
	} else if lockWaitTime > 5*time.Second {
		bc.logLongOp("Blocked Write", op, getDetails(logCb))
	}
}

func (bc *BlockContext) logLongOp(opType, op, details string) {
	waitSecs := bc.postLockTime.Sub(bc.preLockTime).Seconds()
	holdsSecs := bc.postOpTime.Sub(bc.postLockTime).Seconds()

	timeFormat := "15:04:05.000"

	log.Infof("Long %s %s: lockWaitTime = %f, lockHoldTime = %f %s absoluteTimes = (%v -> %v -> %v)",
		opType, op, waitSecs, holdsSecs, details,
		bc.preLockTime.Format(timeFormat), bc.postLockTime.Format(timeFormat), bc.postOpTime.Format(timeFormat))
}

func getDetails(logCb func() interface{}) string {
	if logCb == nil {
		return ""
	}
	return fmt.Sprintf(", details = '%v'", logCb())
}
<commit_msg>Fix unlock order<commit_after>package memory

import (
	"fmt"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

// PriorityRWMutex is a mutex that allows fair access to high priority read operations while
// blocking low priority operations if a write lock is waiting.
type PriorityRWMutex struct {
	lock        sync.RWMutex
	lowPrioLock sync.RWMutex
}

type BlockContext struct {
	lock         *PriorityRWMutex
	preLockTime  time.Time
	postLockTime time.Time
	postOpTime   time.Time
}

// RLockHigh allows *guaranteed fast* (or very high priority) ops to acquire the read lock as
// quickly as possible. Note: This can block write lock acquisition
func (pm *PriorityRWMutex) RLockHigh() {
	// Don't increment activeReads just grab the lock. If an active write is holding the lock it
	// will block, otherwise it *may* block a write acquisition, but that is opted into.
	pm.lock.RLock()
}

// RUnlockHigh unlocks read lock called via RLockHigh
func (pm *PriorityRWMutex) RUnlockHigh() {
	pm.lock.RUnlock()
}

// Synonym for RLockLow
func (pm *PriorityRWMutex) RLock() BlockContext {
	return pm.RLockLow()
}

// Synonym for RUnlockLow
func (pm *PriorityRWMutex) RUnlock(bc *BlockContext) {
	pm.RUnlockLow(bc)
}

// RLockLow will block until requested writes complete. New writes that come in also get priority.
func (pm *PriorityRWMutex) RLockLow() BlockContext {
	// Wait for any pending writes
	bc := BlockContext{lock: pm, preLockTime: time.Now()}
	pm.lowPrioLock.RLock()
	pm.lock.RLock()
	bc.postLockTime = time.Now()
	return bc
}

// RUnlockLow unlocks read lock called via RLockLow
func (pm *PriorityRWMutex) RUnlockLow(bc *BlockContext) {
	bc.postOpTime = time.Now()
	pm.lock.RUnlock()
	pm.lowPrioLock.RUnlock()
}

// Lock will block new low prio read locks until can acquire the lock. High prio read locks will
// not be blocked or accounted for.
func (pm *PriorityRWMutex) Lock() BlockContext {
	bc := BlockContext{lock: pm, preLockTime: time.Now()}
	pm.lowPrioLock.Lock()
	pm.lock.Lock()
	bc.postLockTime = time.Now()
	return bc
}

// Unlock unlocks mutex acquired via Lock
func (pm *PriorityRWMutex) Unlock(bc *BlockContext) {
	bc.postOpTime = time.Now()
	pm.lock.Unlock()
	pm.lowPrioLock.Unlock()
}

func (bc *BlockContext) RUnlockLow(op string, logCb func() interface{}) {
	bc.lock.RUnlockLow(bc)

	lockWaitTime := bc.postLockTime.Sub(bc.preLockTime)
	lockHoldTime := bc.postOpTime.Sub(bc.postLockTime)

	if lockHoldTime > 5*time.Second {
		bc.logLongOp("Read", op, getDetails(logCb))
	} else if lockWaitTime > time.Second && lockWaitTime > 5*lockHoldTime {
		bc.logLongOp("Blocked Read", op, getDetails(logCb))
	}
}

func (bc *BlockContext) Unlock(op string, logCb func() interface{}) {
	bc.lock.Unlock(bc)

	lockWaitTime := bc.postLockTime.Sub(bc.preLockTime)
	lockHoldTime := bc.postOpTime.Sub(bc.postLockTime)

	if lockHoldTime > 100*time.Millisecond {
		bc.logLongOp("Write", op, getDetails(logCb))
	} else if lockWaitTime > 5*time.Second {
		bc.logLongOp("Blocked Write", op, getDetails(logCb))
	}
}

func (bc *BlockContext) logLongOp(opType, op, details string) {
	waitSecs := bc.postLockTime.Sub(bc.preLockTime).Seconds()
	holdsSecs := bc.postOpTime.Sub(bc.postLockTime).Seconds()

	timeFormat := "15:04:05.000"

	log.Infof("Long %s %s: lockWaitTime = %f, lockHoldTime = %f %s absoluteTimes = (%v -> %v -> %v)",
		opType, op, waitSecs, holdsSecs, details,
		bc.preLockTime.Format(timeFormat), bc.postLockTime.Format(timeFormat), bc.postOpTime.Format(timeFormat))
}

func getDetails(logCb func() interface{}) string {
	if logCb == nil {
		return ""
	}
	return fmt.Sprintf(", details = '%v'", logCb())
}
<|endoftext|>


<commit_before>package watch

import (
	"fmt"
	"log"
	"os"
	"reflect"
	"time"

	consulapi "github.com/hashicorp/consul/api"
)

const (
	// retryInterval is the base retry value
	retryInterval = 5 * time.Second

	// maximum back off time, this is to prevent
	// exponential runaway
	maxBackoffTime = 180 * time.Second
)

// Run is used to run a watch plan
func (p *WatchPlan) Run(address string) error {
	// Setup the client
	p.address = address
	conf := consulapi.DefaultConfig()
	conf.Address = address
	conf.Datacenter = p.Datacenter
	conf.Token = p.Token
	client, err := consulapi.NewClient(conf)
	if err != nil {
		return fmt.Errorf("Failed to connect to agent: %v", err)
	}
	p.client = client

	// Create the logger
	output := p.LogOutput
	if output == nil {
		output = os.Stderr
	}
	logger := log.New(output, "", log.LstdFlags)

	// Loop until we are canceled
	failures := 0
OUTER:
	for !p.shouldStop() {
		// Invoke the handler
		index, result, err := p.Func(p)

		// Check if we should terminate since the function
		// could have blocked for a while
		if p.shouldStop() {
			break
		}

		// Handle an error in the watch function
		if err != nil {
			// Perform an exponential backoff
			failures++
			retry := retryInterval * time.Duration(failures*failures)
			if retry > maxBackoffTime {
				retry = maxBackoffTime
			}
			logger.Printf("consul.watch: Watch (type: %s) errored: %v, retry in %v",
				p.Type, err, retry)
			select {
			case <-time.After(retry):
				continue OUTER
			case <-p.stopCh:
				return nil
			}
		}

		// Clear the failures
		failures = 0

		// If the index is unchanged do nothing
		if index == p.lastIndex {
			continue
		}

		// Update the index, look for change
		oldIndex := p.lastIndex
		p.lastIndex = index
		if oldIndex != 0 && reflect.DeepEqual(p.lastResult, result) {
			continue
		}

		// Handle the updated result
		p.lastResult = result
		if p.Handler != nil {
			p.Handler(index, result)
		}
	}
	return nil
}

// Stop is used to stop running the watch plan
func (p *WatchPlan) Stop() {
	p.stopLock.Lock()
	defer p.stopLock.Unlock()
	if p.stop {
		return
	}
	p.stop = true
	close(p.stopCh)
}

func (p *WatchPlan) shouldStop() bool {
	select {
	case <-p.stopCh:
		return true
	default:
		return false
	}
}
<commit_msg>reset watch's lastIndex on error<commit_after>package watch

import (
	"fmt"
	"log"
	"os"
	"reflect"
	"time"

	consulapi "github.com/hashicorp/consul/api"
)

const (
	// retryInterval is the base retry value
	retryInterval = 5 * time.Second

	// maximum back off time, this is to prevent
	// exponential runaway
	maxBackoffTime = 180 * time.Second
)

// Run is used to run a watch plan
func (p *WatchPlan) Run(address string) error {
	// Setup the client
	p.address = address
	conf := consulapi.DefaultConfig()
	conf.Address = address
	conf.Datacenter = p.Datacenter
	conf.Token = p.Token
	client, err := consulapi.NewClient(conf)
	if err != nil {
		return fmt.Errorf("Failed to connect to agent: %v", err)
	}
	p.client = client

	// Create the logger
	output := p.LogOutput
	if output == nil {
		output = os.Stderr
	}
	logger := log.New(output, "", log.LstdFlags)

	// Loop until we are canceled
	failures := 0
OUTER:
	for !p.shouldStop() {
		// Invoke the handler
		index, result, err := p.Func(p)

		// Check if we should terminate since the function
		// could have blocked for a while
		if p.shouldStop() {
			break
		}

		// Handle an error in the watch function
		if err != nil {
			// Perform an exponential backoff
			failures++
			p.lastIndex = 1
			retry := retryInterval * time.Duration(failures*failures)
			if retry > maxBackoffTime {
				retry = maxBackoffTime
			}
			logger.Printf("consul.watch: Watch (type: %s) errored: %v, retry in %v",
				p.Type, err, retry)
			select {
			case <-time.After(retry):
				continue OUTER
			case <-p.stopCh:
				return nil
			}
		}

		// Clear the failures
		failures = 0

		// If the index is unchanged do nothing
		if index == p.lastIndex {
			continue
		}

		// Update the index, look for change
		oldIndex := p.lastIndex
		p.lastIndex = index
		if oldIndex != 0 && reflect.DeepEqual(p.lastResult, result) {
			continue
		}

		// Handle the updated result
		p.lastResult = result
		if p.Handler != nil {
			p.Handler(index, result)
		}
	}
	return nil
}

// Stop is used to stop running the watch plan
func (p *WatchPlan) Stop() {
	p.stopLock.Lock()
	defer p.stopLock.Unlock()
	if p.stop {
		return
	}
	p.stop = true
	close(p.stopCh)
}

func (p *WatchPlan) shouldStop() bool {
	select {
	case <-p.stopCh:
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package watch

import (
	"fmt"
	"log"
	"reflect"
	"time"

	dep "github.com/hashicorp/consul-template/dependency"
	"github.com/hashicorp/consul/api"
)

const (
	// The amount of time to do a blocking query for
	defaultWaitTime = 60 * time.Second

	// The amount of time to wait when Consul returns an error
	defaultRetry = 1 * time.Second
)

// View is a representation of a Dependency and the most recent data it has
// received from Consul.
type View struct {
	// Dependency is the dependency that is associated with this View
	Dependency dep.Dependency

	// config is the configuration for the watcher that created this view and
	// contains important information about how this view should behave when
	// polling including retry functions and handling stale queries.
	config *WatcherConfig

	// Data is the most-recently-received data from Consul for this View
	Data         interface{}
	receivedData bool
	lastIndex    uint64

	// stopCh is used to stop polling on this View
	stopCh chan struct{}
}

// NewView creates a new view object from the given Consul API client and
// Dependency. If an error occurs, it will be returned.
func NewView(config *WatcherConfig, d dep.Dependency) (*View, error) {
	if config == nil {
		return nil, fmt.Errorf("view: missing config")
	}

	if d == nil {
		return nil, fmt.Errorf("view: missing dependency")
	}

	return &View{
		Dependency: d,
		config:     config,
		stopCh:     make(chan struct{}),
	}, nil
}

// poll queries the Consul instance for data using the fetch function, but also
// accounts for interrupts on the interrupt channel. This allows the poll
// function to be fired in a goroutine, but then halted even if the fetch
// function is in the middle of a blocking query.
func (v *View) poll(viewCh chan<- *View, errCh chan<- error) {
	currentRetry := defaultRetry

	for {
		doneCh, fetchErrCh := make(chan struct{}, 1), make(chan error, 1)
		go v.fetch(doneCh, fetchErrCh)

		select {
		case <-doneCh:
			// Reset the retry to avoid exponentially incrementing retries when we
			// have some successful requests
			currentRetry = defaultRetry

			log.Printf("[INFO] (view) %s received data from consul", v.display())
			viewCh <- v

			// If we are operating in once mode, do not loop - we received data at
			// least once which is the API promise here.
			if v.config.Once {
				return
			}
		case err := <-fetchErrCh:
			log.Printf("[ERR] (view) %s %s", v.display(), err)

			// Intentionally do not send the error back up to the watcher and just
			// retry. Eventually, once Consul API implements errwrap and multierror,
			// we can check the "type" of error and conditionally alert back, but for
			// now we will just swallow the error and continue.
			// if err.Contains(Something) {
			//   errCh <- err
			// }

			// Sleep and retry
			if v.config.RetryFunc != nil {
				currentRetry = v.config.RetryFunc(currentRetry)
			}
			time.Sleep(currentRetry)
			continue
		case <-v.stopCh:
			log.Printf("[DEBUG] (view) %s stopping poll (received on view stopCh)", v.display())
			return
		}
	}
}

// fetch queries the Consul instance for the attached dependency. This API
// promises that either data will be written to doneCh or an error will be
// written to errCh. It is designed to be run in a goroutine that selects the
// result of doneCh and errCh. It is assumed that only one instance of fetch
// is running per View and therefore no locking or mutexes are used.
func (v *View) fetch(doneCh chan<- struct{}, errCh chan<- error) {
	log.Printf("[DEBUG] (view) %s starting fetch", v.display())

	var allowStale bool
	if v.config.MaxStale != 0 {
		allowStale = true
	}

	for {
		options := &api.QueryOptions{
			AllowStale: allowStale,
			WaitTime:   defaultWaitTime,
			WaitIndex:  v.lastIndex,
		}
		data, qm, err := v.Dependency.Fetch(v.config.Client, options)
		if err != nil {
			errCh <- err
			return
		}

		if qm == nil {
			errCh <- fmt.Errorf("consul returned nil qm; this should never happen" +
				"and is probably a bug in consul-template or consulapi")
			return
		}

		if allowStale && qm.LastContact > v.config.MaxStale {
			allowStale = false
			log.Printf("[DEBUG] (view) %s stale data (last contact exceeded max_stale)", v.display())
			continue
		}

		if v.config.MaxStale != 0 {
			allowStale = true
		}

		if qm.LastIndex == v.lastIndex {
			log.Printf("[DEBUG] (view) %s no new data (index was the same)", v.display())
			continue
		}

		v.lastIndex = qm.LastIndex

		if v.receivedData && reflect.DeepEqual(data, v.Data) {
			log.Printf("[DEBUG] (view) %s no new data (contents were the same)", v.display())
			continue
		}

		v.Data = data
		v.receivedData = true
		close(doneCh)
		return
	}
}

// display returns a string that represents this view.
func (v *View) display() string {
	return v.Dependency.Display()
}

// stop halts polling of this view.
func (v *View) stop() {
	close(v.stopCh)
}
<commit_msg>Do not block when writing to the channel<commit_after>package watch

import (
	"fmt"
	"log"
	"reflect"
	"time"

	dep "github.com/hashicorp/consul-template/dependency"
	"github.com/hashicorp/consul/api"
)

const (
	// The amount of time to do a blocking query for
	defaultWaitTime = 60 * time.Second

	// The amount of time to wait when Consul returns an error
	defaultRetry = 1 * time.Second
)

// View is a representation of a Dependency and the most recent data it has
// received from Consul.
type View struct {
	// Dependency is the dependency that is associated with this View
	Dependency dep.Dependency

	// config is the configuration for the watcher that created this view and
	// contains important information about how this view should behave when
	// polling including retry functions and handling stale queries.
	config *WatcherConfig

	// Data is the most-recently-received data from Consul for this View
	Data         interface{}
	receivedData bool
	lastIndex    uint64

	// stopCh is used to stop polling on this View
	stopCh chan struct{}
}

// NewView creates a new view object from the given Consul API client and
// Dependency. If an error occurs, it will be returned.
func NewView(config *WatcherConfig, d dep.Dependency) (*View, error) {
	if config == nil {
		return nil, fmt.Errorf("view: missing config")
	}

	if d == nil {
		return nil, fmt.Errorf("view: missing dependency")
	}

	return &View{
		Dependency: d,
		config:     config,
		stopCh:     make(chan struct{}),
	}, nil
}

// poll queries the Consul instance for data using the fetch function, but also
// accounts for interrupts on the interrupt channel. This allows the poll
// function to be fired in a goroutine, but then halted even if the fetch
// function is in the middle of a blocking query.
func (v *View) poll(viewCh chan<- *View, errCh chan<- error) {
	currentRetry := defaultRetry

	for {
		doneCh, fetchErrCh := make(chan struct{}, 1), make(chan error, 1)
		go v.fetch(doneCh, fetchErrCh)

		select {
		case <-doneCh:
			// Reset the retry to avoid exponentially incrementing retries when we
			// have some successful requests
			currentRetry = defaultRetry

			log.Printf("[INFO] (view) %s received data from consul", v.display())
			select {
			case viewCh <- v:
			case <-v.stopCh:
			}

			// If we are operating in once mode, do not loop - we received data at
			// least once which is the API promise here.
			if v.config.Once {
				return
			}
		case err := <-fetchErrCh:
			log.Printf("[ERR] (view) %s %s", v.display(), err)

			// Intentionally do not send the error back up to the watcher and just
			// retry. Eventually, once Consul API implements errwrap and multierror,
			// we can check the "type" of error and conditionally alert back, but for
			// now we will just swallow the error and continue.
			// if err.Contains(Something) {
			//   errCh <- err
			// }

			// Sleep and retry
			if v.config.RetryFunc != nil {
				currentRetry = v.config.RetryFunc(currentRetry)
			}
			time.Sleep(currentRetry)
			continue
		case <-v.stopCh:
			log.Printf("[DEBUG] (view) %s stopping poll (received on view stopCh)", v.display())
			return
		}
	}
}

// fetch queries the Consul instance for the attached dependency. This API
// promises that either data will be written to doneCh or an error will be
// written to errCh. It is designed to be run in a goroutine that selects the
// result of doneCh and errCh. It is assumed that only one instance of fetch
// is running per View and therefore no locking or mutexes are used.
func (v *View) fetch(doneCh chan<- struct{}, errCh chan<- error) {
	log.Printf("[DEBUG] (view) %s starting fetch", v.display())

	var allowStale bool
	if v.config.MaxStale != 0 {
		allowStale = true
	}

	for {
		options := &api.QueryOptions{
			AllowStale: allowStale,
			WaitTime:   defaultWaitTime,
			WaitIndex:  v.lastIndex,
		}
		data, qm, err := v.Dependency.Fetch(v.config.Client, options)
		if err != nil {
			errCh <- err
			return
		}

		if qm == nil {
			errCh <- fmt.Errorf("consul returned nil qm; this should never happen" +
				"and is probably a bug in consul-template or consulapi")
			return
		}

		if allowStale && qm.LastContact > v.config.MaxStale {
			allowStale = false
			log.Printf("[DEBUG] (view) %s stale data (last contact exceeded max_stale)", v.display())
			continue
		}

		if v.config.MaxStale != 0 {
			allowStale = true
		}

		if qm.LastIndex == v.lastIndex {
			log.Printf("[DEBUG] (view) %s no new data (index was the same)", v.display())
			continue
		}

		v.lastIndex = qm.LastIndex

		if v.receivedData && reflect.DeepEqual(data, v.Data) {
			log.Printf("[DEBUG] (view) %s no new data (contents were the same)", v.display())
			continue
		}

		v.Data = data
		v.receivedData = true
		close(doneCh)
		return
	}
}

// display returns a string that represents this view.
func (v *View) display() string {
	return v.Dependency.Display()
}

// stop halts polling of this view.
func (v *View) stop() {
	close(v.stopCh)
}
<|endoftext|>


<commit_before>package bridgezone

import (
	"context"
	"reflect"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	"k8s.io/client-go/kubernetes"

	clientaws "github.com/giantswarm/aws-operator/client/aws"
	"github.com/giantswarm/aws-operator/service/controller/controllercontext"
	"github.com/giantswarm/aws-operator/service/credential"
)

const (
	name = "bridgezonev31"
)

type Config struct {
	HostAWSConfig clientaws.Config
	K8sClient     kubernetes.Interface
	Logger        micrologger.Logger

	Route53Enabled bool
}

// Resource is bridgezone resource making sure we have fallback delegation in
// old DNS structure. TODO This is only for the migration period. TODO When we
// delete the "intermediate" zone this resource becomes noop and we do not need
// it anymore.
//
// Old structure looks like:
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	└── NS k8s.installation.eu-central-1.aws.gigantic.io (default control plane account)
//
//	k8s.installation.eu-central-1.aws.gigantic.io (default control plane account)
//	├── A api.old_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//	├── A ingress.old_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//	├── A api.old_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.old_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//
// New structure looks like:
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	└── NS new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	└── NS new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//
//	new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//
//	new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//
// For the migration period for new clusters we need also to add delegation to
// k8s.installation.eu-central-1.aws.gigantic.io because of the AWS DNS caching issues.
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	├── NS k8s.installation.eu-central-1.aws.gigantic.io (default tenant account)
//	└── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)

//	k8s.installation.eu-central-1.aws.gigantic.io (default tenant account)
//	├── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.old_cluster.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.old_cluster.k8s.installation.eu-central-1.aws.gigantic.io

//	cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//
// NOTE: In the code below k8s.installation.eu-central-1.aws.gigantic.io zone is called
// "intermediate" and cluster_id.k8s.installation.eu-central-1.aws.gigantic.io zone is
// called "final". This resource *only* ensures we have delegation from the
// intermediate zone to the final zone, but only if the intermediate zone
// exists.
//
// After everything is fully migrated the DNS layout should look like:
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	├── NS k8s.installation.eu-central-1.aws.gigantic.io (default guest account)
//	└── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc guest account)
//
//	k8s.installation.eu-central-1.aws.gigantic.io (default guest account)
//	└── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc guest account)
//
//	cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc guest account)
//	├── A api.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//
// At this point we should be fine with removing
// k8s.installation.eu-central-1.aws.gigantic.io NS record from
// installation.eu-central-1.aws.gigantic.io zone. Then after a couple of days
// when delegation propagates and DNS caches are refreshed we can delete
// k8s.installation.eu-central-1.aws.gigantic.io zone from the default guest
// account.
//
// NOTE: To complete full migration we need to start reconciling "hostpost"
// CloudFormation stack. This stack is responsible for creating
// cluster_id.k8s.installation.eu-central-1.aws.gigantic.io delegation in the
// installation.eu-central-1.aws.gigantic.io. Till this happens this resource
// cannot be deleted.
//
//	See https://github.com/giantswarm/aws-operator/pull/1373.
//
type Resource struct {
	hostAWSConfig clientaws.Config
	k8sClient     kubernetes.Interface
	logger        micrologger.Logger

	route53Enabled bool
}

func New(config Config) (*Resource, error) {
	if reflect.DeepEqual(clientaws.Config{}, config.HostAWSConfig) {
		return nil, microerror.Maskf(invalidConfigError, "%T.HostAWSConfig must not be empty", config)
	}
	if config.K8sClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.K8sClient must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	r := &Resource{
		hostAWSConfig: config.HostAWSConfig,
		k8sClient:     config.K8sClient,
		logger:        config.Logger,

		route53Enabled: config.Route53Enabled,
	}

	return r, nil
}

func (r *Resource) Name() string {
	return name
}

// findHostedZoneID fetches Route53 hosted zone IDs based on a given name. The
// implementation fetches up to 100 matching results to find the right one. The
// bridgezone resource here is only concerned with the hosted zone ID of the
// hosted zone name provided. The desired ID will always be carried in the first
// Route53 response as the one we want to fetch is the most accurate and always
// listed as the first item in the response. This is because of the
// lexicographical order of the response items as the API documentation puts it.
// See also
// https://godoc.org/github.com/aws/aws-sdk-go/service/route53#Route53.ListHostedZonesByName.
//
//     Retrieves a list of your hosted zones in lexicographic order.
//
// Here is an example to make it clearer. Let's consider the following hosted
// zone name.
//
//     9cvgo.k8s.ginger.eu-central-1.aws.gigantic.io
//
// Given this name, findHostedZoneID will receive a response from Route53
// similar to the following example, containing a single hosted zone carrying
// its ID.
//
//     {
//       ...
//       HostedZones: [{
//         ...
//         Id: "/hostedzone/Z1A4QS1NDU6NW6",
//         Name: "9cvgo.k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       }],
//       ...
//     }
//
// The example above was about a very specific domain name, which list result
// could only find a single item in the response. Let's consider a less specific
// domain name as input for findHostedZoneID.
//
//     k8s.ginger.eu-central-1.aws.gigantic.io
//
// The result from Route53 will again list all the childs within the given
// domain name. In the example response below there where only two tenant
// clusters.
//
//     {
//       ...
//       HostedZones: [{
//         ...
//         Id: "/hostedzone/Z1HJGG5VLG8GZH",
//         Name: "k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       },{
//         ...
//         Id: "/hostedzone/Z1KSFLSM1JEQYM",
//         Name: "0tz6i.k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       },{
//         ...
//         Id: "/hostedzone/Z1A4QS1NDU6NW6",
//         Name: "9cvgo.k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       }],
//       ...
//     }
//
func (r *Resource) findHostedZoneID(ctx context.Context, client *route53.Route53, name string) (string, error) {
	in := &route53.ListHostedZonesByNameInput{
		DNSName: aws.String(name),
	}

	out, err := client.ListHostedZonesByName(in)
	if err != nil {
		return "", microerror.Mask(err)
	}

	for _, hostedZone := range out.HostedZones {
		if *hostedZone.Name == name {
			return *hostedZone.Id, nil
		}
	}

	return "", microerror.Maskf(notFoundError, "hosted zone name %#q", name)
}

func (r *Resource) getNameServersAndTTL(ctx context.Context, client *route53.Route53, zoneID, name string) (nameServers []string, ttl int64, err error) {
	one := "1"
	ns := route53.RRTypeNs
	in := &route53.ListResourceRecordSetsInput{
		HostedZoneId:    &zoneID,
		MaxItems:        &one,
		StartRecordName: &name,
		StartRecordType: &ns,
	}
	out, err := client.ListResourceRecordSetsWithContext(ctx, in)
	if err != nil {
		return nil, 0, microerror.Mask(err)
	}

	if len(out.ResourceRecordSets) == 0 {
		return nil, 0, microerror.Maskf(notFoundError, "NS record %q for HostedZone %q not found", name, zoneID)
	}
	if len(out.ResourceRecordSets) != 1 {
		return nil, 0, microerror.Maskf(executionError, "expected single NS record %q for HostedZone %q, found %#v", name, zoneID, out.ResourceRecordSets)
	}

	rs := *out.ResourceRecordSets[0]

	if strings.TrimSuffix(*rs.Name, ".") != name {
		return nil, 0, microerror.Maskf(notFoundError, "NS record %q for HostedZone %q not found", name, zoneID)
	}

	var servers []string
	for _, r := range rs.ResourceRecords {
		servers = append(servers, *r.Value)
	}

	return servers, *rs.TTL, nil
}

func (r *Resource) route53Clients(ctx context.Context) (guest, defaultGuest *route53.Route53, err error) {
	// guest
	{
		cc, err := controllercontext.FromContext(ctx)
		if err != nil {
			return nil, nil, microerror.Mask(err)
		}
		guest = cc.Client.TenantCluster.AWS.Route53
	}

	// defaultGuest
	{
		arn, err := credential.GetDefaultARN(r.k8sClient)
		if err != nil {
			return nil, nil, microerror.Mask(err)
		}

		c := r.hostAWSConfig
		c.RoleARN = arn

		newClients, err := clientaws.NewClients(c)
		if err != nil {
			return nil, nil, microerror.Mask(err)
		}

		defaultGuest = newClients.Route53
	}

	return guest, defaultGuest, nil
}
<commit_msg>resource/bridgezone: fix godoc (#2037)<commit_after>package bridgezone

import (
	"context"
	"reflect"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/micrologger"
	"k8s.io/client-go/kubernetes"

	clientaws "github.com/giantswarm/aws-operator/client/aws"
	"github.com/giantswarm/aws-operator/service/controller/controllercontext"
	"github.com/giantswarm/aws-operator/service/credential"
)

const (
	name = "bridgezonev31"
)

type Config struct {
	HostAWSConfig clientaws.Config
	K8sClient     kubernetes.Interface
	Logger        micrologger.Logger

	Route53Enabled bool
}

// Resource is bridgezone resource making sure we have fallback delegation in
// old DNS structure. TODO This is only for the migration period. TODO When we
// delete the "intermediate" zone this resource becomes noop and we do not need
// it anymore.
//
// Old structure looks like:
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	└── NS k8s.installation.eu-central-1.aws.gigantic.io (default control plane account)
//
//	k8s.installation.eu-central-1.aws.gigantic.io (default control plane account)
//	├── A api.old_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//	├── A ingress.old_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//	├── A api.old_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.old_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//
// New structure looks like:
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	└── NS new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	└── NS new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//
//	new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.new_cluster_a.k8s.installation.eu-central-1.aws.gigantic.io
//
//	new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.new_cluster_b.k8s.installation.eu-central-1.aws.gigantic.io
//
// For the migration period for new clusters we need also to add delegation to
// k8s.installation.eu-central-1.aws.gigantic.io because of the AWS DNS caching issues.
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	├── NS k8s.installation.eu-central-1.aws.gigantic.io (default tenant account)
//	└── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//
//	k8s.installation.eu-central-1.aws.gigantic.io (default tenant account)
//	├── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.old_cluster.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.old_cluster.k8s.installation.eu-central-1.aws.gigantic.io
//
//	cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc tenant account)
//	├── A api.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//
// NOTE: In the code below k8s.installation.eu-central-1.aws.gigantic.io zone is called
// "intermediate" and cluster_id.k8s.installation.eu-central-1.aws.gigantic.io zone is
// called "final". This resource *only* ensures we have delegation from the
// intermediate zone to the final zone, but only if the intermediate zone
// exists.
//
// After everything is fully migrated the DNS layout should look like:
//
//	installation.eu-central-1.aws.gigantic.io (control plane account)
//	├── NS k8s.installation.eu-central-1.aws.gigantic.io (default guest account)
//	└── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc guest account)
//
//	k8s.installation.eu-central-1.aws.gigantic.io (default guest account)
//	└── NS cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc guest account)
//
//	cluster_id.k8s.installation.eu-central-1.aws.gigantic.io (byoc guest account)
//	├── A api.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//	└── A ingress.cluster_id.k8s.installation.eu-central-1.aws.gigantic.io
//
// At this point we should be fine with removing
// k8s.installation.eu-central-1.aws.gigantic.io NS record from
// installation.eu-central-1.aws.gigantic.io zone. Then after a couple of days
// when delegation propagates and DNS caches are refreshed we can delete
// k8s.installation.eu-central-1.aws.gigantic.io zone from the default guest
// account.
//
// NOTE: To complete full migration we need to start reconciling "hostpost"
// CloudFormation stack. This stack is responsible for creating
// cluster_id.k8s.installation.eu-central-1.aws.gigantic.io delegation in the
// installation.eu-central-1.aws.gigantic.io. Till this happens this resource
// cannot be deleted.
//
//	See https://github.com/giantswarm/aws-operator/pull/1373.
//
type Resource struct {
	hostAWSConfig clientaws.Config
	k8sClient     kubernetes.Interface
	logger        micrologger.Logger

	route53Enabled bool
}

func New(config Config) (*Resource, error) {
	if reflect.DeepEqual(clientaws.Config{}, config.HostAWSConfig) {
		return nil, microerror.Maskf(invalidConfigError, "%T.HostAWSConfig must not be empty", config)
	}
	if config.K8sClient == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.K8sClient must not be empty", config)
	}
	if config.Logger == nil {
		return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config)
	}

	r := &Resource{
		hostAWSConfig: config.HostAWSConfig,
		k8sClient:     config.K8sClient,
		logger:        config.Logger,

		route53Enabled: config.Route53Enabled,
	}

	return r, nil
}

func (r *Resource) Name() string {
	return name
}

// findHostedZoneID fetches Route53 hosted zone IDs based on a given name. The
// implementation fetches up to 100 matching results to find the right one. The
// bridgezone resource here is only concerned with the hosted zone ID of the
// hosted zone name provided. The desired ID will always be carried in the first
// Route53 response as the one we want to fetch is the most accurate and always
// listed as the first item in the response. This is because of the
// lexicographical order of the response items as the API documentation puts it.
// See also
// https://godoc.org/github.com/aws/aws-sdk-go/service/route53#Route53.ListHostedZonesByName.
//
//     Retrieves a list of your hosted zones in lexicographic order.
//
// Here is an example to make it clearer. Let's consider the following hosted
// zone name.
//
//     9cvgo.k8s.ginger.eu-central-1.aws.gigantic.io
//
// Given this name, findHostedZoneID will receive a response from Route53
// similar to the following example, containing a single hosted zone carrying
// its ID.
//
//     {
//       ...
//       HostedZones: [{
//         ...
//         Id: "/hostedzone/Z1A4QS1NDU6NW6",
//         Name: "9cvgo.k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       }],
//       ...
//     }
//
// The example above was about a very specific domain name, which list result
// could only find a single item in the response. Let's consider a less specific
// domain name as input for findHostedZoneID.
//
//     k8s.ginger.eu-central-1.aws.gigantic.io
//
// The result from Route53 will again list all the childs within the given
// domain name. In the example response below there where only two tenant
// clusters.
//
//     {
//       ...
//       HostedZones: [{
//         ...
//         Id: "/hostedzone/Z1HJGG5VLG8GZH",
//         Name: "k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       },{
//         ...
//         Id: "/hostedzone/Z1KSFLSM1JEQYM",
//         Name: "0tz6i.k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       },{
//         ...
//         Id: "/hostedzone/Z1A4QS1NDU6NW6",
//         Name: "9cvgo.k8s.ginger.eu-central-1.aws.gigantic.io.",
//         ...
//       }],
//       ...
//     }
//
func (r *Resource) findHostedZoneID(ctx context.Context, client *route53.Route53, name string) (string, error) {
	in := &route53.ListHostedZonesByNameInput{
		DNSName: aws.String(name),
	}

	out, err := client.ListHostedZonesByName(in)
	if err != nil {
		return "", microerror.Mask(err)
	}

	for _, hostedZone := range out.HostedZones {
		if *hostedZone.Name == name {
			return *hostedZone.Id, nil
		}
	}

	return "", microerror.Maskf(notFoundError, "hosted zone name %#q", name)
}

func (r *Resource) getNameServersAndTTL(ctx context.Context, client *route53.Route53, zoneID, name string) (nameServers []string, ttl int64, err error) {
	one := "1"
	ns := route53.RRTypeNs
	in := &route53.ListResourceRecordSetsInput{
		HostedZoneId:    &zoneID,
		MaxItems:        &one,
		StartRecordName: &name,
		StartRecordType: &ns,
	}
	out, err := client.ListResourceRecordSetsWithContext(ctx, in)
	if err != nil {
		return nil, 0, microerror.Mask(err)
	}

	if len(out.ResourceRecordSets) == 0 {
		return nil, 0, microerror.Maskf(notFoundError, "NS record %q for HostedZone %q not found", name, zoneID)
	}
	if len(out.ResourceRecordSets) != 1 {
		return nil, 0, microerror.Maskf(executionError, "expected single NS record %q for HostedZone %q, found %#v", name, zoneID, out.ResourceRecordSets)
	}

	rs := *out.ResourceRecordSets[0]

	if strings.TrimSuffix(*rs.Name, ".") != name {
		return nil, 0, microerror.Maskf(notFoundError, "NS record %q for HostedZone %q not found", name, zoneID)
	}

	var servers []string
	for _, r := range rs.ResourceRecords {
		servers = append(servers, *r.Value)
	}

	return servers, *rs.TTL, nil
}

func (r *Resource) route53Clients(ctx context.Context) (guest, defaultGuest *route53.Route53, err error) {
	// guest
	{
		cc, err := controllercontext.FromContext(ctx)
		if err != nil {
			return nil, nil, microerror.Mask(err)
		}
		guest = cc.Client.TenantCluster.AWS.Route53
	}

	// defaultGuest
	{
		arn, err := credential.GetDefaultARN(r.k8sClient)
		if err != nil {
			return nil, nil, microerror.Mask(err)
		}

		c := r.hostAWSConfig
		c.RoleARN = arn

		newClients, err := clientaws.NewClients(c)
		if err != nil {
			return nil, nil, microerror.Mask(err)
		}

		defaultGuest = newClients.Route53
	}

	return guest, defaultGuest, nil
}
<|endoftext|>


<commit_before>package stringutil

import "bytes"

// String2ByteSlice is for sortint of string-[]byte pairs
type String2ByteSlice struct {
	Key   string
	Value []byte
}

// String2ByteSliceList is list of string2ByteSlice
type String2ByteSliceList []String2ByteSlice

func (list String2ByteSliceList) Len() int { return len(list) }
func (list String2ByteSliceList) Less(i, j int) bool {
	return list[i].Key < list[j].Key
}
func (list String2ByteSliceList) Swap(i, j int) {
	list[i], list[j] = list[j], list[i]
}

// ReversedString2ByteSliceList is reversed String2ByteSliceList
type ReversedString2ByteSliceList struct {
	String2ByteSliceList
}

// Less ...
func (list ReversedString2ByteSliceList) Less(i, j int) bool {
	return list.String2ByteSliceList[i].Key > list.String2ByteSliceList[j].Key
}

// ByValue ...
type ByValue struct {
	String2ByteSliceList
}

// Less ...
func (list ByValue) Less(i, j int) bool {
	c := bytes.Compare(list.String2ByteSliceList[i].Value, list.String2ByteSliceList[j].Value)
	if c == -1 {
		return true
	}
	return false
}

// ReversedByValue ...
type ReversedByValue struct {
	String2ByteSliceList
}

// Less ...
func (list ReversedByValue) Less(i, j int) bool {
	c := bytes.Compare(list.String2ByteSliceList[j].Value, list.String2ByteSliceList[i].Value)
	if c == -1 {
		return true
	}
	return false
}
<commit_msg>add global option NaturalOrder for String2ByteSlice<commit_after>package stringutil

import (
	"bytes"

	"github.com/shenwei356/natsort"
)

// String2ByteSlice is for sortint of string-[]byte pairs
type String2ByteSlice struct {
	Key   string
	Value []byte
}

// String2ByteSliceList is list of string2ByteSlice
type String2ByteSliceList []String2ByteSlice

// NaturalOrder is the global variable for sorting String2ByteSlice
var NaturalOrder = false

func (list String2ByteSliceList) Len() int { return len(list) }
func (list String2ByteSliceList) Less(i, j int) bool {
	if NaturalOrder {
		return natsort.Compare(list[i].Key, list[j].Key)
	}
	return list[i].Key < list[j].Key
}
func (list String2ByteSliceList) Swap(i, j int) {
	list[i], list[j] = list[j], list[i]
}

// ReversedString2ByteSliceList is reversed String2ByteSliceList
type ReversedString2ByteSliceList struct {
	String2ByteSliceList
}

// Less ...
func (list ReversedString2ByteSliceList) Less(i, j int) bool {
	if NaturalOrder {
		return !natsort.Compare(list.String2ByteSliceList[i].Key, list.String2ByteSliceList[j].Key)
	}
	return list.String2ByteSliceList[i].Key > list.String2ByteSliceList[j].Key
}

// ByValue ...
type ByValue struct {
	String2ByteSliceList
}

// Less ...
func (list ByValue) Less(i, j int) bool {
	c := bytes.Compare(list.String2ByteSliceList[i].Value, list.String2ByteSliceList[j].Value)
	if c == -1 {
		return true
	}
	return false
}

// ReversedByValue ...
type ReversedByValue struct {
	String2ByteSliceList
}

// Less ...
func (list ReversedByValue) Less(i, j int) bool {
	c := bytes.Compare(list.String2ByteSliceList[j].Value, list.String2ByteSliceList[i].Value)
	if c == -1 {
		return true
	}
	return false
}
<|endoftext|>