<commit_before>package memcached

import (
	"encoding/binary"
	"fmt"
	"github.com/couchbase/gomemcached"
	"math"
)

type SystemEventType int

const InvalidSysEvent SystemEventType = -1

const (
	CollectionCreate  SystemEventType = 0
	CollectionDrop    SystemEventType = iota
	CollectionFlush   SystemEventType = iota // KV did not implement
	ScopeCreate       SystemEventType = iota
	ScopeDrop         SystemEventType = iota
	CollectionChanged SystemEventType = iota
)

type ScopeCreateEvent interface {
	GetSystemEventName() (string, error)
	GetScopeId() (uint32, error)
	GetManifestId() (uint64, error)
}

type CollectionCreateEvent interface {
	GetSystemEventName() (string, error)
	GetScopeId() (uint32, error)
	GetCollectionId() (uint32, error)
	GetManifestId() (uint64, error)
	GetMaxTTL() (uint32, error)
}

type CollectionDropEvent interface {
	GetScopeId() (uint32, error)
	GetCollectionId() (uint32, error)
	GetManifestId() (uint64, error)
}

type ScopeDropEvent interface {
	GetScopeId() (uint32, error)
	GetManifestId() (uint64, error)
}

type CollectionChangedEvent interface {
	GetCollectionId() (uint32, error)
	GetManifestId() (uint64, error)
	GetMaxTTL() (uint32, error)
}

var ErrorInvalidOp error = fmt.Errorf("Invalid Operation")
var ErrorInvalidVersion error = fmt.Errorf("Invalid version for parsing")
var ErrorValueTooShort error = fmt.Errorf("Value length is too short")
var ErrorNoMaxTTL error = fmt.Errorf("This event has no max TTL")

// UprEvent memcached events for UPR streams.
type UprEvent struct {
	Opcode          gomemcached.CommandCode // Type of event
	Status          gomemcached.Status      // Response status
	VBucket         uint16                  // VBucket this event applies to
	DataType        uint8                   // data type
	Opaque          uint16                  // 16 MSB of opaque
	VBuuid          uint64                  // This field is set by downstream
	Flags           uint32                  // Item flags
	Expiry          uint32                  // Item expiration time
	Key, Value      []byte                  // Item key/value
	OldValue        []byte                  // TODO: TBD: old document value
	Cas             uint64                  // CAS value of the item
	Seqno           uint64                  // sequence number of the mutation
	RevSeqno        uint64                  // rev sequence number : deletions
	LockTime        uint32                  // Lock time
	MetadataSize    uint16                  // Metadata size
	SnapstartSeq    uint64                  // start sequence number of this snapshot
	SnapendSeq      uint64                  // End sequence number of the snapshot
	SnapshotType    uint32                  // 0: disk 1: memory
	FailoverLog     *FailoverLog            // Failover log containing vvuid and sequnce number
	Error           error                   // Error value in case of a failure
	ExtMeta         []byte                  // Extended Metadata
	AckSize         uint32                  // The number of bytes that can be Acked to DCP
	SystemEvent     SystemEventType         // Only valid if IsSystemEvent() is true
	SysEventVersion uint8                   // Based on the version, the way Extra bytes is parsed is different
	ValueLen        int                     // Cache it to avoid len() calls for performance
	CollectionId    uint64                  // Valid if Collection is in use
}

// FailoverLog containing vvuid and sequnce number
type FailoverLog [][2]uint64

// Containing a pair of vbno and the high seqno
type VBSeqnos [][2]uint64

func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFromDCP int) *UprEvent {
	event := &UprEvent{
		Opcode:       rq.Opcode,
		VBucket:      stream.Vbucket,
		VBuuid:       stream.Vbuuid,
		Value:        rq.Body,
		Cas:          rq.Cas,
		ExtMeta:      rq.ExtMeta,
		DataType:     rq.DataType,
		ValueLen:     len(rq.Body),
		SystemEvent:  InvalidSysEvent,
		CollectionId: math.MaxUint64,
	}

	event.PopulateFieldsBasedOnStreamType(rq, stream.StreamType)

	// set AckSize for events that need to be acked to DCP,
	// i.e., events with CommandCodes that need to be buffered in DCP
	if _, ok := gomemcached.BufferedCommandCodeMap[rq.Opcode]; ok {
		event.AckSize = uint32(bytesReceivedFromDCP)
	}

	// 16 LSBits are used by client library to encode vbucket number.
	// 16 MSBits are left for application to multiplex on opaque value.
	event.Opaque = appOpaque(rq.Opaque)

	if len(rq.Extras) >= uprMutationExtraLen &&
		event.Opcode == gomemcached.UPR_MUTATION {

		event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
		event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.Flags = binary.BigEndian.Uint32(rq.Extras[16:20])
		event.Expiry = binary.BigEndian.Uint32(rq.Extras[20:24])
		event.LockTime = binary.BigEndian.Uint32(rq.Extras[24:28])
		event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[28:30])

	} else if len(rq.Extras) >= uprDeletetionWithDeletionTimeExtraLen &&
		event.Opcode == gomemcached.UPR_DELETION {

		event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
		event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.Expiry = binary.BigEndian.Uint32(rq.Extras[16:20])

	} else if len(rq.Extras) >= uprDeletetionExtraLen &&
		event.Opcode == gomemcached.UPR_DELETION ||
		event.Opcode == gomemcached.UPR_EXPIRATION {

		event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
		event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[16:18])

	} else if len(rq.Extras) >= uprSnapshotExtraLen &&
		event.Opcode == gomemcached.UPR_SNAPSHOT {

		event.SnapstartSeq = binary.BigEndian.Uint64(rq.Extras[:8])
		event.SnapendSeq = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
	} else if event.IsSystemEvent() {
		event.PopulateEvent(rq.Extras)
	} else if event.IsSeqnoAdv() {
		event.PopulateSeqnoAdv(rq.Extras)
	}

	return event
}

func (event *UprEvent) PopulateFieldsBasedOnStreamType(rq gomemcached.MCRequest, streamType DcpStreamType) {
	switch streamType {
	case CollectionsNonStreamId:
		switch rq.Opcode {
		// Only these will have CID encoded within the key
		case gomemcached.UPR_MUTATION,
			gomemcached.UPR_DELETION,
			gomemcached.UPR_EXPIRATION:
			uleb128 := Uleb128(rq.Key)
			result, bytesShifted := uleb128.ToUint64(rq.Keylen)
			event.CollectionId = result
			event.Key = rq.Key[bytesShifted:]
		default:
			event.Key = rq.Key
		}
	case CollectionsStreamId:
		// TODO - not implemented
		fallthrough
	case NonCollectionStream:
		// Let default behavior be legacy stream type
		fallthrough
	default:
		event.Key = rq.Key
	}
}

func (event *UprEvent) String() string {
	name := gomemcached.CommandNames[event.Opcode]
	if name == "" {
		name = fmt.Sprintf("#%d", event.Opcode)
	}
	return name
}

func (event *UprEvent) IsSnappyDataType() bool {
	return event.Opcode == gomemcached.UPR_MUTATION && (event.DataType&SnappyDataType > 0)
}

func (event *UprEvent) IsCollectionType() bool {
	return event.IsSystemEvent() || event.CollectionId <= math.MaxUint32
}

func (event *UprEvent) IsSystemEvent() bool {
	return event.Opcode == gomemcached.DCP_SYSTEM_EVENT
}

func (event *UprEvent) IsSeqnoAdv() bool {
	return event.Opcode == gomemcached.DCP_SEQNO_ADV
}

func (event *UprEvent) PopulateEvent(extras []byte) {
	if len(extras) < dcpSystemEventExtraLen {
		// Wrong length, don't parse
		return
	}

	event.Seqno = binary.BigEndian.Uint64(extras[:8])
	event.SystemEvent = SystemEventType(binary.BigEndian.Uint32(extras[8:12]))
	var versionTemp uint16 = binary.BigEndian.Uint16(extras[12:14])
	event.SysEventVersion = uint8(versionTemp >> 8)
}

func (event *UprEvent) PopulateSeqnoAdv(extras []byte) {
	if len(extras) < dcpSeqnoAdvExtraLen {
		// Wrong length, don't parse
		return
	}

	event.Seqno = binary.BigEndian.Uint64(extras[:8])
}

func (event *UprEvent) GetSystemEventName() (string, error) {
	switch event.SystemEvent {
	case CollectionCreate:
		fallthrough
	case ScopeCreate:
		return string(event.Key), nil
	default:
		return "", ErrorInvalidOp
	}
}

func (event *UprEvent) GetManifestId() (uint64, error) {
	switch event.SystemEvent {
	// Version 0 only checks
	case CollectionChanged:
		fallthrough
	case ScopeDrop:
		fallthrough
	case ScopeCreate:
		fallthrough
	case CollectionDrop:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		fallthrough
	case CollectionCreate:
		// CollectionCreate supports version 1
		if event.SysEventVersion > 1 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 8 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint64(event.Value[0:8]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

func (event *UprEvent) GetCollectionId() (uint32, error) {
	switch event.SystemEvent {
	case CollectionDrop:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		fallthrough
	case CollectionCreate:
		if event.SysEventVersion > 1 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 16 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[12:16]), nil
	case CollectionChanged:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 12 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[8:12]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

func (event *UprEvent) GetScopeId() (uint32, error) {
	switch event.SystemEvent {
	// version 0 checks
	case ScopeCreate:
		fallthrough
	case ScopeDrop:
		fallthrough
	case CollectionDrop:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		fallthrough
	case CollectionCreate:
		// CollectionCreate could be either 0 or 1
		if event.SysEventVersion > 1 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 12 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[8:12]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

func (event *UprEvent) GetMaxTTL() (uint32, error) {
	switch event.SystemEvent {
	case CollectionCreate:
		if event.SysEventVersion < 1 {
			return 0, ErrorNoMaxTTL
		}
		if event.ValueLen < 20 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[16:20]), nil
	case CollectionChanged:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 16 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[12:16]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

type Uleb128 []byte

func (u Uleb128) ToUint64(cachedLen int) (result uint64, bytesShifted int) {
	var shift uint = 0

	for curByte := 0; curByte < cachedLen; curByte++ {
		oneByte := u[curByte]
		last7Bits := 0x7f & oneByte
		result |= uint64(last7Bits) << shift
		bytesShifted++
		if oneByte&0x80 == 0 {
			break
		}
		shift += 7
	}

	return
}
<commit_msg>MB-39564 - gomemcached should store collectionID as uint32<commit_after>package memcached

import (
	"encoding/binary"
	"fmt"
	"github.com/couchbase/gomemcached"
	"math"
)

type SystemEventType int

const InvalidSysEvent SystemEventType = -1

const (
	CollectionCreate  SystemEventType = 0
	CollectionDrop    SystemEventType = iota
	CollectionFlush   SystemEventType = iota // KV did not implement
	ScopeCreate       SystemEventType = iota
	ScopeDrop         SystemEventType = iota
	CollectionChanged SystemEventType = iota
)

type ScopeCreateEvent interface {
	GetSystemEventName() (string, error)
	GetScopeId() (uint32, error)
	GetManifestId() (uint64, error)
}

type CollectionCreateEvent interface {
	GetSystemEventName() (string, error)
	GetScopeId() (uint32, error)
	GetCollectionId() (uint32, error)
	GetManifestId() (uint64, error)
	GetMaxTTL() (uint32, error)
}

type CollectionDropEvent interface {
	GetScopeId() (uint32, error)
	GetCollectionId() (uint32, error)
	GetManifestId() (uint64, error)
}

type ScopeDropEvent interface {
	GetScopeId() (uint32, error)
	GetManifestId() (uint64, error)
}

type CollectionChangedEvent interface {
	GetCollectionId() (uint32, error)
	GetManifestId() (uint64, error)
	GetMaxTTL() (uint32, error)
}

var ErrorInvalidOp error = fmt.Errorf("Invalid Operation")
var ErrorInvalidVersion error = fmt.Errorf("Invalid version for parsing")
var ErrorValueTooShort error = fmt.Errorf("Value length is too short")
var ErrorNoMaxTTL error = fmt.Errorf("This event has no max TTL")

// UprEvent memcached events for UPR streams.
type UprEvent struct {
	Opcode          gomemcached.CommandCode // Type of event
	Status          gomemcached.Status      // Response status
	VBucket         uint16                  // VBucket this event applies to
	DataType        uint8                   // data type
	Opaque          uint16                  // 16 MSB of opaque
	VBuuid          uint64                  // This field is set by downstream
	Flags           uint32                  // Item flags
	Expiry          uint32                  // Item expiration time
	Key, Value      []byte                  // Item key/value
	OldValue        []byte                  // TODO: TBD: old document value
	Cas             uint64                  // CAS value of the item
	Seqno           uint64                  // sequence number of the mutation
	RevSeqno        uint64                  // rev sequence number : deletions
	LockTime        uint32                  // Lock time
	MetadataSize    uint16                  // Metadata size
	SnapstartSeq    uint64                  // start sequence number of this snapshot
	SnapendSeq      uint64                  // End sequence number of the snapshot
	SnapshotType    uint32                  // 0: disk 1: memory
	FailoverLog     *FailoverLog            // Failover log containing vvuid and sequnce number
	Error           error                   // Error value in case of a failure
	ExtMeta         []byte                  // Extended Metadata
	AckSize         uint32                  // The number of bytes that can be Acked to DCP
	SystemEvent     SystemEventType         // Only valid if IsSystemEvent() is true
	SysEventVersion uint8                   // Based on the version, the way Extra bytes is parsed is different
	ValueLen        int                     // Cache it to avoid len() calls for performance
	CollectionId    uint32                  // Valid if Collection is in use
}

// FailoverLog containing vvuid and sequnce number
type FailoverLog [][2]uint64

// Containing a pair of vbno and the high seqno
type VBSeqnos [][2]uint64

func makeUprEvent(rq gomemcached.MCRequest, stream *UprStream, bytesReceivedFromDCP int) *UprEvent {
	event := &UprEvent{
		Opcode:       rq.Opcode,
		VBucket:      stream.Vbucket,
		VBuuid:       stream.Vbuuid,
		Value:        rq.Body,
		Cas:          rq.Cas,
		ExtMeta:      rq.ExtMeta,
		DataType:     rq.DataType,
		ValueLen:     len(rq.Body),
		SystemEvent:  InvalidSysEvent,
		CollectionId: math.MaxUint32,
	}

	event.PopulateFieldsBasedOnStreamType(rq, stream.StreamType)

	// set AckSize for events that need to be acked to DCP,
	// i.e., events with CommandCodes that need to be buffered in DCP
	if _, ok := gomemcached.BufferedCommandCodeMap[rq.Opcode]; ok {
		event.AckSize = uint32(bytesReceivedFromDCP)
	}

	// 16 LSBits are used by client library to encode vbucket number.
	// 16 MSBits are left for application to multiplex on opaque value.
	event.Opaque = appOpaque(rq.Opaque)

	if len(rq.Extras) >= uprMutationExtraLen &&
		event.Opcode == gomemcached.UPR_MUTATION {

		event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
		event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.Flags = binary.BigEndian.Uint32(rq.Extras[16:20])
		event.Expiry = binary.BigEndian.Uint32(rq.Extras[20:24])
		event.LockTime = binary.BigEndian.Uint32(rq.Extras[24:28])
		event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[28:30])

	} else if len(rq.Extras) >= uprDeletetionWithDeletionTimeExtraLen &&
		event.Opcode == gomemcached.UPR_DELETION {

		event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
		event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.Expiry = binary.BigEndian.Uint32(rq.Extras[16:20])

	} else if len(rq.Extras) >= uprDeletetionExtraLen &&
		event.Opcode == gomemcached.UPR_DELETION ||
		event.Opcode == gomemcached.UPR_EXPIRATION {

		event.Seqno = binary.BigEndian.Uint64(rq.Extras[:8])
		event.RevSeqno = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.MetadataSize = binary.BigEndian.Uint16(rq.Extras[16:18])

	} else if len(rq.Extras) >= uprSnapshotExtraLen &&
		event.Opcode == gomemcached.UPR_SNAPSHOT {

		event.SnapstartSeq = binary.BigEndian.Uint64(rq.Extras[:8])
		event.SnapendSeq = binary.BigEndian.Uint64(rq.Extras[8:16])
		event.SnapshotType = binary.BigEndian.Uint32(rq.Extras[16:20])
	} else if event.IsSystemEvent() {
		event.PopulateEvent(rq.Extras)
	} else if event.IsSeqnoAdv() {
		event.PopulateSeqnoAdv(rq.Extras)
	}

	return event
}

func (event *UprEvent) PopulateFieldsBasedOnStreamType(rq gomemcached.MCRequest, streamType DcpStreamType) {
	switch streamType {
	case CollectionsNonStreamId:
		switch rq.Opcode {
		// Only these will have CID encoded within the key
		case gomemcached.UPR_MUTATION,
			gomemcached.UPR_DELETION,
			gomemcached.UPR_EXPIRATION:
			uleb128 := Uleb128(rq.Key)
			result, bytesShifted := uleb128.ToUint32(rq.Keylen)
			event.CollectionId = result
			event.Key = rq.Key[bytesShifted:]
		default:
			event.Key = rq.Key
		}
	case CollectionsStreamId:
		// TODO - not implemented
		fallthrough
	case NonCollectionStream:
		// Let default behavior be legacy stream type
		fallthrough
	default:
		event.Key = rq.Key
	}
}

func (event *UprEvent) String() string {
	name := gomemcached.CommandNames[event.Opcode]
	if name == "" {
		name = fmt.Sprintf("#%d", event.Opcode)
	}
	return name
}

func (event *UprEvent) IsSnappyDataType() bool {
	return event.Opcode == gomemcached.UPR_MUTATION && (event.DataType&SnappyDataType > 0)
}

func (event *UprEvent) IsCollectionType() bool {
	return event.IsSystemEvent() || event.CollectionId <= math.MaxUint32
}

func (event *UprEvent) IsSystemEvent() bool {
	return event.Opcode == gomemcached.DCP_SYSTEM_EVENT
}

func (event *UprEvent) IsSeqnoAdv() bool {
	return event.Opcode == gomemcached.DCP_SEQNO_ADV
}

func (event *UprEvent) PopulateEvent(extras []byte) {
	if len(extras) < dcpSystemEventExtraLen {
		// Wrong length, don't parse
		return
	}

	event.Seqno = binary.BigEndian.Uint64(extras[:8])
	event.SystemEvent = SystemEventType(binary.BigEndian.Uint32(extras[8:12]))
	var versionTemp uint16 = binary.BigEndian.Uint16(extras[12:14])
	event.SysEventVersion = uint8(versionTemp >> 8)
}

func (event *UprEvent) PopulateSeqnoAdv(extras []byte) {
	if len(extras) < dcpSeqnoAdvExtraLen {
		// Wrong length, don't parse
		return
	}

	event.Seqno = binary.BigEndian.Uint64(extras[:8])
}

func (event *UprEvent) GetSystemEventName() (string, error) {
	switch event.SystemEvent {
	case CollectionCreate:
		fallthrough
	case ScopeCreate:
		return string(event.Key), nil
	default:
		return "", ErrorInvalidOp
	}
}

func (event *UprEvent) GetManifestId() (uint64, error) {
	switch event.SystemEvent {
	// Version 0 only checks
	case CollectionChanged:
		fallthrough
	case ScopeDrop:
		fallthrough
	case ScopeCreate:
		fallthrough
	case CollectionDrop:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		fallthrough
	case CollectionCreate:
		// CollectionCreate supports version 1
		if event.SysEventVersion > 1 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 8 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint64(event.Value[0:8]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

func (event *UprEvent) GetCollectionId() (uint32, error) {
	switch event.SystemEvent {
	case CollectionDrop:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		fallthrough
	case CollectionCreate:
		if event.SysEventVersion > 1 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 16 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[12:16]), nil
	case CollectionChanged:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 12 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[8:12]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

func (event *UprEvent) GetScopeId() (uint32, error) {
	switch event.SystemEvent {
	// version 0 checks
	case ScopeCreate:
		fallthrough
	case ScopeDrop:
		fallthrough
	case CollectionDrop:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		fallthrough
	case CollectionCreate:
		// CollectionCreate could be either 0 or 1
		if event.SysEventVersion > 1 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 12 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[8:12]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

func (event *UprEvent) GetMaxTTL() (uint32, error) {
	switch event.SystemEvent {
	case CollectionCreate:
		if event.SysEventVersion < 1 {
			return 0, ErrorNoMaxTTL
		}
		if event.ValueLen < 20 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[16:20]), nil
	case CollectionChanged:
		if event.SysEventVersion > 0 {
			return 0, ErrorInvalidVersion
		}
		if event.ValueLen < 16 {
			return 0, ErrorValueTooShort
		}
		return binary.BigEndian.Uint32(event.Value[12:16]), nil
	default:
		return 0, ErrorInvalidOp
	}
}

type Uleb128 []byte

func (u Uleb128) ToUint32(cachedLen int) (result uint32, bytesShifted int) {
	var shift uint = 0

	for curByte := 0; curByte < cachedLen; curByte++ {
		oneByte := u[curByte]
		last7Bits := 0x7f & oneByte
		result |= uint32(last7Bits) << shift
		bytesShifted++
		if oneByte&0x80 == 0 {
			break
		}
		shift += 7
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	// Stdlib
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	// Internal
	"github.com/salsita/salsaflow/asciiart"
	"github.com/salsita/salsaflow/git"
	"github.com/salsita/salsaflow/log"
	"github.com/salsita/salsaflow/uuid"
)

const diffSeparator = "# ------------------------ >8 ------------------------"

const (
	secretFilename = "AreYouWhoIThinkYouAreHuh"
	secretReply    = "IAmSalsaFlowHookYaDoofus!"
)

func main() {
	if len(os.Args) != 2 {
		panic(fmt.Errorf("argv: %#v", os.Args))
	}

	if os.Args[1] == secretFilename {
		fmt.Println(secretReply)
		return
	}

	if err := run(os.Args[1]); err != nil {
		asciiart.PrintGrimReaper("COMMIT ABORTED")
		log.Fatalln("Error:", err)
	}
}

func run(messagePath string) error {
	// Open the commit message file.
	file, err := os.OpenFile(messagePath, os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	defer file.Close()

	// Read and parse the commit message.
	var (
		dropEmptyLines bool = true
		changeIdSeen   bool
		lines          []string
	)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		// Read the next line.
		var (
			line        = scanner.Text()
			trimmedLine = strings.TrimSpace(line)
		)

		// Drop leading empty lines.
		if dropEmptyLines {
			if trimmedLine == "" {
				continue
			}
			dropEmptyLines = false
		}

		// Drop comments.
		if strings.HasPrefix(trimmedLine, "#") {
			continue
		}

		// Drop the diff that can be appended do the commit message when
		// git commit -v is used. Git would drop the diff later anyway.
		if line == diffSeparator {
			break
		}

		// Check for the Change-Id tag.
		if git.ChangeIdTagPattern.MatchString(trimmedLine) {
			if changeIdSeen {
				return errors.New("multiple Change-Id tags detected")
			}
			changeIdSeen = true
		}

		// Finally, append the line to the commit message.
		lines = append(lines, line)
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	// Do nothing in case the file is empty.
	if len(lines) == 0 {
		return nil
	}

	// Do nothing in case Change-Id is already there.
	if changeIdSeen {
		return nil
	}

	// Make sure a single empty line is following the current content.
	// Do not insert an empty line in case the last line is Story-Id.
	for lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	if !strings.HasPrefix(lines[len(lines)-1], "Story-Id") {
		lines = append(lines, "")
	}

	// Append the Change-Id tag.
	changeId, err := uuid.New()
	if err != nil {
		return err
	}
	lines = append(lines, fmt.Sprintf("Change-Id: %v", changeId))

	// Write the content back to the disk (truncate the file first).
	_, err = file.Seek(0, os.SEEK_SET)
	if err != nil {
		return err
	}
	if err := file.Truncate(0); err != nil {
		return err
	}

	_, err = io.Copy(file, strings.NewReader(strings.Join(lines, "\n")))
	return err
}
<commit_msg>hook/commit-msg: Fix code review issues<commit_after>package main

import (
	// Stdlib
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	// Internal
	"github.com/salsita/salsaflow/asciiart"
	"github.com/salsita/salsaflow/git"
	"github.com/salsita/salsaflow/log"
	"github.com/salsita/salsaflow/uuid"
)

const diffSeparator = "# ------------------------ >8 ------------------------"

const (
	secretFilename = "AreYouWhoIThinkYouAreHuh"
	secretReply    = "IAmSalsaFlowHookYaDoofus!"
)

func main() {
	if len(os.Args) != 2 {
		panic(fmt.Errorf("argv: %#v", os.Args))
	}

	if os.Args[1] == secretFilename {
		fmt.Println(secretReply)
		return
	}

	if err := run(os.Args[1]); err != nil {
		asciiart.PrintGrimReaper("COMMIT ABORTED")
		log.Fatalln("Error:", err)
	}
}

func run(messagePath string) error {
	// Open the commit message file.
	file, err := os.OpenFile(messagePath, os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	defer file.Close()

	// Read and parse the commit message.
	var (
		dropEmptyLines bool = true
		changeIdSeen   bool
		lines          []string
	)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		// Read the next line.
		var (
			line        = scanner.Text()
			trimmedLine = strings.TrimSpace(line)
		)

		// Drop leading empty lines.
		if dropEmptyLines {
			if trimmedLine == "" {
				continue
			}
			dropEmptyLines = false
		}

		// Drop comments.
		if strings.HasPrefix(trimmedLine, "#") {
			continue
		}

		// Drop the diff that can be appended do the commit message when
		// git commit -v is used. Git would drop the diff later anyway.
		if line == diffSeparator {
			break
		}

		// Check for the Change-Id tag.
		if git.ChangeIdTagPattern.MatchString(trimmedLine) {
			if changeIdSeen {
				return errors.New("multiple Change-Id tags detected")
			}
			changeIdSeen = true
		}

		// Finally, append the line to the commit message.
		lines = append(lines, line)
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	// Do nothing in case the file is empty.
	if len(lines) == 0 {
		return nil
	}

	// Do nothing in case Change-Id is already there.
	if changeIdSeen {
		return nil
	}

	// Make sure a single empty line is following the current content.
	// Do not insert an empty line in case the last line is Story-Id.
	for lines[len(lines)-1] == "" {
		lines = lines[:len(lines)-1]
	}
	if line := strings.ToLower(lines[len(lines)-1]); !strings.HasPrefix(line, "story-id") {
		lines = append(lines, "")
	}

	// Append the Change-Id tag.
	changeId, err := uuid.New()
	if err != nil {
		return err
	}
	lines = append(lines, fmt.Sprintf("Change-Id: %v", changeId))

	// Write the content back to the disk (truncate the file first).
	_, err = file.Seek(0, os.SEEK_SET)
	if err != nil {
		return err
	}
	if err := file.Truncate(0); err != nil {
		return err
	}

	_, err = io.Copy(file, strings.NewReader(strings.Join(lines, "\n")))
	return err
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package manager

import (
	"github.com/webx-top/db"
	"github.com/webx-top/echo"

	"github.com/admpub/nging/v4/application/handler"
	"github.com/admpub/nging/v4/application/library/role"
	"github.com/admpub/nging/v4/application/library/role/roleutils"
	"github.com/admpub/nging/v4/application/model"
)

func Role(ctx echo.Context) error {
	m := model.NewUserRole(ctx)
	_, err := handler.PagingWithLister(ctx, m)
	ret := handler.Err(ctx, err)
	ctx.Set(`listData`, m.Objects())
	return ctx.Render(`/manager/role`, ret)
}

func RoleAdd(ctx echo.Context) error {
	var err error
	m := model.NewUserRole(ctx)
	if ctx.IsPost() {
		ctx.Begin()
		err = ctx.MustBind(m.NgingUserRole)
		if err == nil {
			_, err = m.Add()
		}
		if err == nil {
			err = roleutils.AddUserRolePermission(ctx, m.Id)
		}
		ctx.End(err == nil)
		if err == nil {
			handler.SendOk(ctx, ctx.T(`操作成功`))
			return ctx.Redirect(handler.URLFor(`/manager/role`))
		}
	} else {
		id := ctx.Formx(`copyId`).Uint()
		if id > 0 {
			err = m.Get(nil, `id`, id)
			if err == nil {
				echo.StructToForm(ctx, m.NgingUserRole, ``, echo.LowerCaseFirstLetter)
				ctx.Request().Form().Set(`id`, `0`)
			}
		}
	}
	ctx.Set(`activeURL`, `/manager/role`)
	ctx.Set(`data`, m)
	permission := role.NewRolePermission()
	ctx.Set(`permission`, permission)
	ctx.Set(`permissionTypes`, role.UserRolePermissionType.Slice())
	roleutils.UserRolePermissionTypeFireRender(ctx)
	return ctx.Render(`/manager/role_edit`, handler.Err(ctx, err))
}

func RoleEdit(ctx echo.Context) error {
	id := ctx.Formx(`id`).Uint()
	m := model.NewUserRole(ctx)
	err := m.Get(nil, `id`, id)
	if err != nil {
		handler.SendFail(ctx, err.Error())
		return ctx.Redirect(handler.URLFor(`/manager/role`))
	}
	if ctx.IsPost() {
		ctx.Begin()
		err = ctx.MustBind(m.NgingUserRole)
		if err == nil {
			err = m.Edit(nil, `id`, id)
		}
		if err == nil {
			err = roleutils.EditUserRolePermission(ctx, m.Id)
		}
		ctx.End(err == nil)
		if err == nil {
			handler.SendOk(ctx, ctx.T(`修改成功`))
			return ctx.Redirect(handler.URLFor(`/manager/role`))
		}
	}

	echo.StructToForm(ctx, m.NgingUserRole, ``, echo.LowerCaseFirstLetter)
	ctx.Set(`activeURL`, `/manager/role`)
	ctx.Set(`data`, m)
	rpM := model.NewUserRolePermission(ctx)
	rpM.ListByOffset(nil, nil, 0, -1, `role_id`, m.Id)
	permissionList := []*role.UserRoleWithPermissions{
		{
			NgingUserRole: m.NgingUserRole,
			Permissions:   rpM.Objects(),
		},
	}
	permission := role.NewRolePermission().Init(permissionList)
	ctx.Set(`permission`, permission)
	ctx.Set(`permissionTypes`, role.UserRolePermissionType.Slice())
	roleutils.UserRolePermissionTypeFireRender(ctx)
	return ctx.Render(`/manager/role_edit`, handler.Err(ctx, err))
}

func RoleDelete(ctx echo.Context) error {
	id := ctx.Formx(`id`).Uint()
	m := model.NewUserRole(ctx)
	if id == 1 {
		handler.SendFail(ctx, ctx.T(`超级管理员角色不可删除`))
		return ctx.Redirect(handler.URLFor(`/manager/role`))
	}
	err := m.Delete(nil, db.Cond{`id`: id})
	if err == nil {
		rpM := model.NewUserRolePermission(ctx)
		rpM.Delete(nil, `role_id`, id)
		handler.SendOk(ctx, ctx.T(`操作成功`))
	} else {
		handler.SendFail(ctx, err.Error())
	}

	return ctx.Redirect(handler.URLFor(`/manager/role`))
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package manager

import (
	"github.com/webx-top/db"
	"github.com/webx-top/echo"

	"github.com/admpub/nging/v4/application/handler"
	"github.com/admpub/nging/v4/application/library/role"
	"github.com/admpub/nging/v4/application/library/role/roleutils"
	"github.com/admpub/nging/v4/application/model"
)

func Role(ctx echo.Context) error {
	m := model.NewUserRole(ctx)
	_, err := handler.PagingWithLister(ctx, m)
	ret := handler.Err(ctx, err)
	ctx.Set(`listData`, m.Objects())
	return ctx.Render(`/manager/role`, ret)
}

func RoleAdd(ctx echo.Context) error {
	var err error
	m := model.NewUserRole(ctx)
	permission := role.NewRolePermission()
	if ctx.IsPost() {
		ctx.Begin()
		err = ctx.MustBind(m.NgingUserRole)
		if err == nil {
			_, err = m.Add()
		}
		if err == nil {
			err = roleutils.AddUserRolePermission(ctx, m.Id)
		}
		ctx.End(err == nil)
		if err == nil {
			handler.SendOk(ctx, ctx.T(`操作成功`))
			return ctx.Redirect(handler.URLFor(`/manager/role`))
		}
	} else {
		id := ctx.Formx(`copyId`).Uint()
		if id > 0 {
			err = m.Get(nil, `id`, id)
			if err == nil {
				echo.StructToForm(ctx, m.NgingUserRole, ``, echo.LowerCaseFirstLetter)
				ctx.Request().Form().Set(`id`, `0`)
				rpM := model.NewUserRolePermission(ctx)
				rpM.ListByOffset(nil, nil, 0, -1, `role_id`, m.Id)
				permissionList := []*role.UserRoleWithPermissions{
					{
						NgingUserRole: m.NgingUserRole,
						Permissions:   rpM.Objects(),
					},
				}
				permission.Init(permissionList)
			}
		}
	}
	ctx.Set(`activeURL`, `/manager/role`)
	ctx.Set(`data`, m)
	ctx.Set(`permission`, permission)
	ctx.Set(`permissionTypes`, role.UserRolePermissionType.Slice())
	roleutils.UserRolePermissionTypeFireRender(ctx)
	return ctx.Render(`/manager/role_edit`, handler.Err(ctx, err))
}

func RoleEdit(ctx echo.Context) error {
	id := ctx.Formx(`id`).Uint()
	m := model.NewUserRole(ctx)
	err := m.Get(nil, `id`, id)
	if err != nil {
		handler.SendFail(ctx, err.Error())
		return ctx.Redirect(handler.URLFor(`/manager/role`))
	}
	if ctx.IsPost() {
		ctx.Begin()
		err = ctx.MustBind(m.NgingUserRole)
		if err == nil {
			err = m.Edit(nil, `id`, id)
		}
		if err == nil {
			err = roleutils.EditUserRolePermission(ctx, m.Id)
		}
		ctx.End(err == nil)
		if err == nil {
			handler.SendOk(ctx, ctx.T(`修改成功`))
			return ctx.Redirect(handler.URLFor(`/manager/role`))
		}
	}

	echo.StructToForm(ctx, m.NgingUserRole, ``, echo.LowerCaseFirstLetter)
	ctx.Set(`activeURL`, `/manager/role`)
	ctx.Set(`data`, m)
	rpM := model.NewUserRolePermission(ctx)
	rpM.ListByOffset(nil, nil, 0, -1, `role_id`, m.Id)
	permissionList := []*role.UserRoleWithPermissions{
		{
			NgingUserRole: m.NgingUserRole,
			Permissions:   rpM.Objects(),
		},
	}
	permission := role.NewRolePermission().Init(permissionList)
	ctx.Set(`permission`, permission)
	ctx.Set(`permissionTypes`, role.UserRolePermissionType.Slice())
	roleutils.UserRolePermissionTypeFireRender(ctx)
	return ctx.Render(`/manager/role_edit`, handler.Err(ctx, err))
}

func RoleDelete(ctx echo.Context) error {
	id := ctx.Formx(`id`).Uint()
	m := model.NewUserRole(ctx)
	if id == 1 {
		handler.SendFail(ctx, ctx.T(`超级管理员角色不可删除`))
		return ctx.Redirect(handler.URLFor(`/manager/role`))
	}
	err := m.Delete(nil, db.Cond{`id`: id})
	if err == nil {
		rpM := model.NewUserRolePermission(ctx)
		rpM.Delete(nil, `role_id`, id)
		handler.SendOk(ctx, ctx.T(`操作成功`))
	} else {
		handler.SendFail(ctx, err.Error())
	}

	return ctx.Redirect(handler.URLFor(`/manager/role`))
}
<|endoftext|>


<commit_before>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// Package api Golang bindings for Baruwa REST API
package api

// AuthServer holds an authentication server
type AuthServer struct {
	ID              int    `json:"id,omitempty"`
	Address         string `json:"address"`
	Protocol        int    `json:"protocol"`
	Port            int    `json:"port"`
	Enabled         bool   `json:"enabled"`
	SplitAddress    bool   `json:"split_address"`
	UserMapTemplate string `json:"user_map_template"`
}

// GetAuthServer returns an authentication server
// https://www.baruwa.com/docs/api/#retrieve-authentication-settings
func (c *Client) GetAuthServer(id int) (server AuthServer, err error) {
	return
}

// CreateAuthServer creates an authentication server
// https://www.baruwa.com/docs/api/#create-authentication-settings
func (c *Client) CreateAuthServer(server *AuthServer) (err error) {
	return
}

// UpdateAuthServer updates an authentication server
// https://www.baruwa.com/docs/api/#update-authentication-settings
func (c *Client) UpdateAuthServer(server *AuthServer) (err error) {
	return
}

// DeleteAuthServer deletes an authentication server
// https://www.baruwa.com/docs/api/#delete-authentication-settings
func (c *Client) DeleteAuthServer(id int) (err error) {
	return
}
<commit_msg>FIX: Use a pointer<commit_after>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// Package api Golang bindings for Baruwa REST API
package api

// AuthServer holds an authentication server
type AuthServer struct {
	ID              int    `json:"id,omitempty"`
	Address         string `json:"address"`
	Protocol        int    `json:"protocol"`
	Port            int    `json:"port"`
	Enabled         bool   `json:"enabled"`
	SplitAddress    bool   `json:"split_address"`
	UserMapTemplate string `json:"user_map_template"`
}

// GetAuthServer returns an authentication server
// https://www.baruwa.com/docs/api/#retrieve-authentication-settings
func (c *Client) GetAuthServer(id int) (server *AuthServer, err error) {
	return
}

// CreateAuthServer creates an authentication server
// https://www.baruwa.com/docs/api/#create-authentication-settings
func (c *Client) CreateAuthServer(server *AuthServer) (err error) {
	return
}

// UpdateAuthServer updates an authentication server
// https://www.baruwa.com/docs/api/#update-authentication-settings
func (c *Client) UpdateAuthServer(server *AuthServer) (err error) {
	return
}

// DeleteAuthServer deletes an authentication server
// https://www.baruwa.com/docs/api/#delete-authentication-settings
func (c *Client) DeleteAuthServer(id int) (err error) {
	return
}
<|endoftext|>


<commit_before>package statusservice

/*
Copyright 2019 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"github.com/crunchydata/postgres-operator/apiserver"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
	"github.com/crunchydata/postgres-operator/config"
	"github.com/crunchydata/postgres-operator/kubeapi"
	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/client-go/kubernetes"
	"sort"
)

func Status(ns string) msgs.StatusResponse {
	var err error

	response := msgs.StatusResponse{}
	response.Status = msgs.Status{Code: msgs.Ok, Msg: ""}

	err = getStatus(&response.Result, ns)
	if err != nil {
		response.Status.Code = msgs.Error
		response.Status.Msg = "error getting status report"
	}

	return response
}

func getStatus(results *msgs.StatusDetail, ns string) error {

	var err error
	results.OperatorStartTime = getOperatorStart(apiserver.PgoNamespace)
	results.NumBackups = getNumBackups(ns)
	results.NumClaims = getNumClaims(ns)
	results.NumDatabases = getNumDatabases(ns)
	results.VolumeCap = getVolumeCap(ns)
	results.DbTags = getDBTags(ns)
	results.NotReady = getNotReady(ns)
	results.Nodes = getNodes()
	results.Labels = getLabels(ns)
	return err
}

func getOperatorStart(ns string) string {
	pods, err := kubeapi.GetPods(apiserver.Clientset, "name="+config.LABEL_OPERATOR, ns)
	if err != nil {
		log.Error(err)
		return "error"
	}

	for _, p := range pods.Items {
		if string(p.Status.Phase) == "Running" {
			return p.Status.StartTime.String()
		}
	}
	log.Error("a running postgres-operator pod is not found")

	return "error"
}

func getNumBackups(ns string) int {
	//count the number of Jobs with pgbackup=true and completionTime not nil
	jobs, err := kubeapi.GetJobs(apiserver.Clientset, config.LABEL_PGBACKUP, ns)
	if err != nil {
		log.Error(err)
		return 0
	}
	return len(jobs.Items)
}

func getNumClaims(ns string) int {
	//count number of PVCs with pgremove=true
	pvcs, err := kubeapi.GetPVCs(apiserver.Clientset, config.LABEL_PGREMOVE, ns)
	if err != nil {
		log.Error(err)
		return 0
	}
	return len(pvcs.Items)
}

func getNumDatabases(ns string) int {
	//count number of Deployments with pg-cluster
	deps, err := kubeapi.GetDeployments(apiserver.Clientset, config.LABEL_PG_CLUSTER, ns)
	if err != nil {
		log.Error(err)
		return 0
	}
	return len(deps.Items)
}

func getVolumeCap(ns string) string {
	//sum all PVCs storage capacity
	pvcs, err := kubeapi.GetPVCs(apiserver.Clientset, config.LABEL_PGREMOVE, ns)
	if err != nil {
		log.Error(err)
		return "error"
	}

	var capTotal int64
	capTotal = 0
	for _, p := range pvcs.Items {
		capTotal = capTotal + getClaimCapacity(apiserver.Clientset, &p)
	}
	q := resource.NewQuantity(capTotal, resource.BinarySI)
	log.Infof("capTotal string is %s\n", q.String())
	return q.String()
}

func getDBTags(ns string) map[string]int {
	results := make(map[string]int)
	//count all pods with pg-cluster, sum by image tag value
	pods, err := kubeapi.GetPods(apiserver.Clientset, config.LABEL_PG_CLUSTER, ns)
	if err != nil {
		log.Error(err)
		return results
	}
	for _, p := range pods.Items {
		for _, c := range p.Spec.Containers {
			results[c.Image]++
		}
	}

	return results
}

func getNotReady(ns string) []string {
	//show all pods with pg-cluster that have status.Phase of not Running
	agg := make([]string, 0)

	pods, err := kubeapi.GetPods(apiserver.Clientset, config.LABEL_PG_CLUSTER, ns)
	if err != nil {
		log.Error(err)
		return agg
	}
	for _, p := range pods.Items {
		for _, stat := range p.Status.ContainerStatuses {
			if !stat.Ready {
				agg = append(agg, p.ObjectMeta.Name)
			}
		}
	}

	return agg
}

func getClaimCapacity(clientset *kubernetes.Clientset, pvc *v1.PersistentVolumeClaim) int64 {
	qty := pvc.Status.Capacity[v1.ResourceStorage]
	diskSize := resource.MustParse(qty.String())
	diskSizeInt64, _ := diskSize.AsInt64()

	return diskSizeInt64

}

func getNodes() []msgs.NodeInfo {
	result := make([]msgs.NodeInfo, 0)

	nodes, err := kubeapi.GetAllNodes(apiserver.Clientset)
	if err != nil {
		log.Error(err)
		return result
	}

	for _, node := range nodes.Items {
		r := msgs.NodeInfo{}
		r.Labels = node.ObjectMeta.Labels
		r.Name = node.ObjectMeta.Name
		clen := len(node.Status.Conditions)
		if clen > 0 {
			r.Status = string(node.Status.Conditions[clen-1].Type)
		}
		result = append(result, r)

	}

	return result
}

func getLabels(ns string) []msgs.KeyValue {
	var ss []msgs.KeyValue
	results := make(map[string]int)
	// GetDeployments gets a list of deployments using a label selector
	deps, err := kubeapi.GetDeployments(apiserver.Clientset, "", ns)
	if err != nil {
		log.Error(err)
		return ss
	}

	for _, dep := range deps.Items {

		for k, v := range dep.ObjectMeta.Labels {
			lv := k + "=" + v
			if results[lv] == 0 {
				results[lv] = 1
			} else {
				results[lv] = results[lv] + 1
			}
		}

	}

	for k, v := range results {
		ss = append(ss, msgs.KeyValue{k, v})
	}

	sort.Slice(ss, func(i, j int) bool {
		return ss[i].Value > ss[j].Value
	})

	return ss

}
<commit_msg>The "Databases Not Ready" field now shows the (#714)<commit_after>package statusservice

/*
Copyright 2019 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"fmt"
	"sort"

	crv1 "github.com/crunchydata/postgres-operator/apis/cr/v1"
	"github.com/crunchydata/postgres-operator/apiserver"
	msgs "github.com/crunchydata/postgres-operator/apiservermsgs"
	"github.com/crunchydata/postgres-operator/config"
	"github.com/crunchydata/postgres-operator/kubeapi"
	log "github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/client-go/kubernetes"
)

func Status(ns string) msgs.StatusResponse {
	var err error

	response := msgs.StatusResponse{}
	response.Status = msgs.Status{Code: msgs.Ok, Msg: ""}

	err = getStatus(&response.Result, ns)
	if err != nil {
		response.Status.Code = msgs.Error
		response.Status.Msg = "error getting status report"
	}

	return response
}

func getStatus(results *msgs.StatusDetail, ns string) error {

	var err error
	results.OperatorStartTime = getOperatorStart(apiserver.PgoNamespace)
	results.NumBackups = getNumBackups(ns)
	results.NumClaims = getNumClaims(ns)
	results.NumDatabases = getNumDatabases(ns)
	results.VolumeCap = getVolumeCap(ns)
	results.DbTags = getDBTags(ns)
	results.NotReady = getNotReady(ns)
	results.Nodes = getNodes()
	results.Labels = getLabels(ns)
	return err
}

func getOperatorStart(ns string) string {
	pods, err := kubeapi.GetPods(apiserver.Clientset, "name="+config.LABEL_OPERATOR, ns)
	if err != nil {
		log.Error(err)
		return "error"
	}

	for _, p := range pods.Items {
		if string(p.Status.Phase) == "Running" {
			return p.Status.StartTime.String()
		}
	}
	log.Error("a running postgres-operator pod is not found")

	return "error"
}

func getNumBackups(ns string) int {
	//count the number of Jobs with pgbackup=true and completionTime not nil
	jobs, err := kubeapi.GetJobs(apiserver.Clientset, config.LABEL_PGBACKUP, ns)
	if err != nil {
		log.Error(err)
		return 0
	}
	return len(jobs.Items)
}

func getNumClaims(ns string) int {
	//count number of PVCs with pgremove=true
	pvcs, err := kubeapi.GetPVCs(apiserver.Clientset, config.LABEL_PGREMOVE, ns)
	if err != nil {
		log.Error(err)
		return 0
	}
	return len(pvcs.Items)
}

func getNumDatabases(ns string) int {
	//count number of Deployments with pg-cluster
	deps, err := kubeapi.GetDeployments(apiserver.Clientset, config.LABEL_PG_CLUSTER, ns)
	if err != nil {
		log.Error(err)
		return 0
	}
	return len(deps.Items)
}

func getVolumeCap(ns string) string {
	//sum all PVCs storage capacity
	pvcs, err := kubeapi.GetPVCs(apiserver.Clientset, config.LABEL_PGREMOVE, ns)
	if err != nil {
		log.Error(err)
		return "error"
	}

	var capTotal int64
	capTotal = 0
	for _, p := range pvcs.Items {
		capTotal = capTotal + getClaimCapacity(apiserver.Clientset, &p)
	}
	q := resource.NewQuantity(capTotal, resource.BinarySI)
	log.Infof("capTotal string is %s\n", q.String())
	return q.String()
}

func getDBTags(ns string) map[string]int {
	results := make(map[string]int)
	//count all pods with pg-cluster, sum by image tag value
	pods, err := kubeapi.GetPods(apiserver.Clientset, config.LABEL_PG_CLUSTER, ns)
	if err != nil {
		log.Error(err)
		return results
	}
	for _, p := range pods.Items {
		for _, c := range p.Spec.Containers {
			results[c.Image]++
		}
	}

	return results
}

func getNotReady(ns string) []string {
	//show all database pods for each pgcluster that are not yet running
	agg := make([]string, 0)
	clusterList := crv1.PgclusterList{}
	kubeapi.Getpgclusters(apiserver.RESTClient, &clusterList, ns)

	for _, cluster := range clusterList.Items {

		selector := fmt.Sprintf("%s=crunchydata,name=%s", config.LABEL_VENDOR, cluster.Spec.ClusterName)
		pods, err := kubeapi.GetPods(apiserver.Clientset, selector, ns)
		if err != nil {
			log.Error(err)
			return agg
		} else if len(pods.Items) > 1 {
			log.Error(fmt.Errorf("Multiple database pods found with the same name using selector while searching for "+
				"databases that are not ready using selector %s", selector))
			return agg
		} else if len(pods.Items) == 0 {
			log.Error(fmt.Errorf("No database pods found while searching for database pods that are not ready using "+
				"selector %s", selector))
			return agg
		}

		pod := pods.Items[0]
		for _, stat := range pod.Status.ContainerStatuses {
			if !stat.Ready {
				agg = append(agg, pod.ObjectMeta.Name)
			}
		}
	}

	return agg
}

func getClaimCapacity(clientset *kubernetes.Clientset, pvc *v1.PersistentVolumeClaim) int64 {
	qty := pvc.Status.Capacity[v1.ResourceStorage]
	diskSize := resource.MustParse(qty.String())
	diskSizeInt64, _ := diskSize.AsInt64()

	return diskSizeInt64

}

func getNodes() []msgs.NodeInfo {
	result := make([]msgs.NodeInfo, 0)

	nodes, err := kubeapi.GetAllNodes(apiserver.Clientset)
	if err != nil {
		log.Error(err)
		return result
	}

	for _, node := range nodes.Items {
		r := msgs.NodeInfo{}
		r.Labels = node.ObjectMeta.Labels
		r.Name = node.ObjectMeta.Name
		clen := len(node.Status.Conditions)
		if clen > 0 {
			r.Status = string(node.Status.Conditions[clen-1].Type)
		}
		result = append(result, r)

	}

	return result
}

func getLabels(ns string) []msgs.KeyValue {
	var ss []msgs.KeyValue
	results := make(map[string]int)
	// GetDeployments gets a list of deployments using a label selector
	deps, err := kubeapi.GetDeployments(apiserver.Clientset, "", ns)
	if err != nil {
		log.Error(err)
		return ss
	}

	for _, dep := range deps.Items {

		for k, v := range dep.ObjectMeta.Labels {
			lv := k + "=" + v
			if results[lv] == 0 {
				results[lv] = 1
			} else {
				results[lv] = results[lv] + 1
			}
		}

	}

	for k, v := range results {
		ss = append(ss, msgs.KeyValue{k, v})
	}

	sort.Slice(ss, func(i, j int) bool {
		return ss[i].Value > ss[j].Value
	})

	return ss

}
<|endoftext|>


<commit_before>// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package crypto

import (
	"bytes"
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"strings"
	"time"

	vault "github.com/hashicorp/vault/api"
	"github.com/minio/minio/cmd/logger"
)

var (
	//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
)

// VaultKey represents vault encryption key-ring.
type VaultKey struct {
	Name    string `json:"name"`    // The name of the encryption key-ring
	Version int    `json:"version"` // The key version
}

// VaultAuth represents vault authentication type.
// Currently the only supported authentication type is AppRole.
type VaultAuth struct {
	Type    string       `json:"type"`    // The authentication type
	AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
}

// VaultAppRole represents vault AppRole authentication credentials
type VaultAppRole struct {
	ID     string `json:"id"`     // The AppRole access ID
	Secret string `json:"secret"` // The AppRole secret
}

// VaultConfig represents vault configuration.
type VaultConfig struct {
	Endpoint  string    `json:"endpoint"` // The vault API endpoint as URL
	CAPath    string    `json:"-"`        // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
	Auth      VaultAuth `json:"auth"`     // The vault authentication configuration
	Key       VaultKey  `json:"key-id"`   // The named key used for key-generation / decryption.
	Namespace string    `json:"-"`        // The vault namespace of enterprise vault instances
}

// vaultService represents a connection to a vault KMS.
type vaultService struct {
	config        *VaultConfig
	client        *vault.Client
	leaseDuration time.Duration
}

var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS

// empty/default vault configuration used to check whether a particular is empty.
var emptyVaultConfig = VaultConfig{}

// IsEmpty returns true if the vault config struct is an
// empty configuration.
func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }

// Verify returns a nil error if the vault configuration
// is valid. A valid configuration is either empty or
// contains valid non-default values.
func (v *VaultConfig) Verify() (err error) {
	if v.IsEmpty() {
		return // an empty configuration is valid
	}
	switch {
	case v.Endpoint == "":
		err = errors.New("crypto: missing hashicorp vault endpoint")
	case strings.ToLower(v.Auth.Type) != "approle":
		err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
	case v.Auth.AppRole.ID == "":
		err = errors.New("crypto: missing hashicorp vault AppRole ID")
	case v.Auth.AppRole.Secret == "":
		err = errors.New("crypto: missing hashicorp vault AppSecret ID")
	case v.Key.Name == "":
		err = errors.New("crypto: missing hashicorp vault key name")
	case v.Key.Version < 0:
		err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
	}
	return
}

// NewVault initializes Hashicorp Vault KMS by authenticating
// to Vault with the credentials in config and gets a client
// token for future api calls.
func NewVault(config VaultConfig) (KMS, error) {
	if config.IsEmpty() {
		return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
	}
	if err := config.Verify(); err != nil {
		return nil, err
	}

	vaultCfg := vault.Config{Address: config.Endpoint}
	if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
		return nil, err
	}
	client, err := vault.NewClient(&vaultCfg)
	if err != nil {
		return nil, err
	}
	if config.Namespace != "" {
		client.SetNamespace(config.Namespace)
	}
	v := &vaultService{client: client, config: &config}

	if err := v.authenticate(); err != nil {
		return nil, err
	}
	return v, nil
}

// renewSecret tries to renew the given secret. It blocks
// until it receives either the new secret or encounters an error.
func (v *vaultService) renewSecret(secret *vault.Secret) (*vault.Secret, error) {
	renewer, err := v.client.NewRenewer(&vault.RenewerInput{
		Secret: secret,
	})
	if err != nil {
		logger.CriticalIf(context.Background(), fmt.Errorf("crypto: failed to create hashicorp vault renewer: %s", err))
	}
	go renewer.Renew()
	defer renewer.Stop()

	for {
		select {
		case err := <-renewer.DoneCh():
			if err != nil {
				return nil, err
			}
		case renew := <-renewer.RenewCh():
			if renew.Secret == nil || renew.Secret.Auth == nil {
				return nil, ErrKMSAuthLogin
			}
			return renew.Secret, nil
		}
	}
}

// login tries to authenticate the minio server to
// the Vault KMS using the approle ID and secret.
func (v *vaultService) login() (*vault.Secret, error) {
	payload := map[string]interface{}{
		"role_id":   v.config.Auth.AppRole.ID,
		"secret_id": v.config.Auth.AppRole.Secret,
	}
	secret, err := v.client.Logical().Write("auth/approle/login", payload)
	if err != nil {
		return nil, err
	}
	if secret == nil || secret.Auth == nil {
		return nil, ErrKMSAuthLogin
	}
	return secret, nil
}

// authenticate tries to authenticate the minio server
// to the Vault KMS and starts a background job to renew
// the login.
func (v *vaultService) authenticate() error {
	secret, err := v.login()
	if err != nil {
		return err
	}
	v.client.SetToken(secret.Auth.ClientToken)
	v.leaseDuration = time.Duration(secret.Auth.LeaseDuration)

	// Start background job trying to renew the token
	// or (if this fails) try to login again with app-ID and app-Secret.
	go func(secret *vault.Secret) {
		for {
			newSecret, err := v.renewSecret(secret) // try to renew the secret (blocking)
			if err != nil {
				// Try to login again with app-ID and app-Secret
				if newSecret, err = v.login(); err != nil { // failed -> try again
					time.Sleep(1 * time.Minute) // retry delay
					continue
				}
			}
			secret = newSecret // Now newSecret contains a valid, non-nil *vault.Secret
			v.client.SetToken(secret.Auth.ClientToken)
			v.leaseDuration = time.Duration(secret.Auth.LeaseDuration)
		}
	}(secret)
	return nil
}

// GenerateKey returns a new plaintext key, generated by the KMS,
// and a sealed version of this plaintext key encrypted using the
// named key referenced by keyID. It also binds the generated key
// cryptographically to the provided context.
func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
	var contextStream bytes.Buffer
	ctx.WriteTo(&contextStream)

	payload := map[string]interface{}{
		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
	}
	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
	if err != nil {
		return key, sealedKey, err
	}
	sealKey := s.Data["ciphertext"].(string)
	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
	if err != nil {
		return key, sealedKey, err
	}
	copy(key[:], []byte(plainKey))
	return key, []byte(sealKey), nil
}

// UnsealKey returns the decrypted sealedKey as plaintext key.
// Therefore it sends the sealedKey to the KMS which decrypts
// it using the named key referenced by keyID and responses with
// the plaintext key.
//
// The context must be same context as the one provided while
// generating the plaintext key / sealedKey.
func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
	var contextStream bytes.Buffer
	ctx.WriteTo(&contextStream)

	payload := map[string]interface{}{
		"ciphertext": string(sealedKey),
		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
	}
	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
	if err != nil {
		return key, err
	}
	base64Key := s.Data["plaintext"].(string)
	plainKey, err := base64.StdEncoding.DecodeString(base64Key)
	if err != nil {
		return key, err
	}
	copy(key[:], []byte(plainKey))
	return key, nil
}
<commit_msg>Revert PR #7241 to fix vault renewal (#7259)<commit_after>// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package crypto

import (
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"strings"
	"time"

	vault "github.com/hashicorp/vault/api"
)

var (
	//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
)

// VaultKey represents vault encryption key-ring.
type VaultKey struct {
	Name    string `json:"name"`    // The name of the encryption key-ring
	Version int    `json:"version"` // The key version
}

// VaultAuth represents vault authentication type.
// Currently the only supported authentication type is AppRole.
type VaultAuth struct {
	Type    string       `json:"type"`    // The authentication type
	AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
}

// VaultAppRole represents vault AppRole authentication credentials
type VaultAppRole struct {
	ID     string `json:"id"`     // The AppRole access ID
	Secret string `json:"secret"` // The AppRole secret
}

// VaultConfig represents vault configuration.
type VaultConfig struct {
	Endpoint  string    `json:"endpoint"` // The vault API endpoint as URL
	CAPath    string    `json:"-"`        // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
	Auth      VaultAuth `json:"auth"`     // The vault authentication configuration
	Key       VaultKey  `json:"key-id"`   // The named key used for key-generation / decryption.
	Namespace string    `json:"-"`        // The vault namespace of enterprise vault instances
}

// vaultService represents a connection to a vault KMS.
type vaultService struct {
	config        *VaultConfig
	client        *vault.Client
	secret        *vault.Secret
	leaseDuration time.Duration
}

var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS

// empty/default vault configuration used to check whether a particular is empty.
var emptyVaultConfig = VaultConfig{}

// IsEmpty returns true if the vault config struct is an
// empty configuration.
func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }

// Verify returns a nil error if the vault configuration
// is valid. A valid configuration is either empty or
// contains valid non-default values.
func (v *VaultConfig) Verify() (err error) {
	if v.IsEmpty() {
		return // an empty configuration is valid
	}
	switch {
	case v.Endpoint == "":
		err = errors.New("crypto: missing hashicorp vault endpoint")
	case strings.ToLower(v.Auth.Type) != "approle":
		err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
	case v.Auth.AppRole.ID == "":
		err = errors.New("crypto: missing hashicorp vault AppRole ID")
	case v.Auth.AppRole.Secret == "":
		err = errors.New("crypto: missing hashicorp vault AppSecret ID")
	case v.Key.Name == "":
		err = errors.New("crypto: missing hashicorp vault key name")
	case v.Key.Version < 0:
		err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
	}
	return
}

// NewVault initializes Hashicorp Vault KMS by authenticating
// to Vault with the credentials in config and gets a client
// token for future api calls.
func NewVault(config VaultConfig) (KMS, error) {
	if config.IsEmpty() {
		return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
	}
	if err := config.Verify(); err != nil {
		return nil, err
	}

	vaultCfg := vault.Config{Address: config.Endpoint}
	if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
		return nil, err
	}
	client, err := vault.NewClient(&vaultCfg)
	if err != nil {
		return nil, err
	}
	if config.Namespace != "" {
		client.SetNamespace(config.Namespace)
	}
	v := &vaultService{client: client, config: &config}
	if err := v.authenticate(); err != nil {
		return nil, err
	}
	v.renewToken()
	return v, nil
}

// renewToken starts a new go-routine which renews
// the vault authentication token periodically and re-authenticates
// if the token renewal fails
func (v *vaultService) renewToken() {
	retryDelay := v.leaseDuration / 2
	go func() {
		for {
			if v.secret == nil {
				if err := v.authenticate(); err != nil {
					time.Sleep(retryDelay)
					continue
				}
			}
			s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
			if err != nil || s == nil {
				v.secret = nil
				time.Sleep(retryDelay)
				continue
			}
			if ok, err := s.TokenIsRenewable(); !ok || err != nil {
				v.secret = nil
				continue
			}
			ttl, err := s.TokenTTL()
			if err != nil {
				v.secret = nil
				continue
			}
			v.secret = s
			retryDelay = ttl / 2
			time.Sleep(retryDelay)
		}
	}()
}

// authenticate logs the app to vault, and starts the auto renewer
// before secret expires
func (v *vaultService) authenticate() (err error) {
	payload := map[string]interface{}{
		"role_id":   v.config.Auth.AppRole.ID,
		"secret_id": v.config.Auth.AppRole.Secret,
	}
	var tokenID string
	var ttl time.Duration
	var secret *vault.Secret
	secret, err = v.client.Logical().Write("auth/approle/login", payload)
	if err != nil {
		return
	}
	if secret == nil {
		err = ErrKMSAuthLogin
		return
	}

	tokenID, err = secret.TokenID()
	if err != nil {
		err = ErrKMSAuthLogin
		return
	}
	ttl, err = secret.TokenTTL()
	if err != nil {
		err = ErrKMSAuthLogin
		return
	}
	v.client.SetToken(tokenID)
	v.secret = secret
	v.leaseDuration = ttl
	return
}

// GenerateKey returns a new plaintext key, generated by the KMS,
// and a sealed version of this plaintext key encrypted using the
// named key referenced by keyID. It also binds the generated key
// cryptographically to the provided context.
func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
	var contextStream bytes.Buffer
	ctx.WriteTo(&contextStream)

	payload := map[string]interface{}{
		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
	}
	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
	if err != nil {
		return key, sealedKey, err
	}
	sealKey := s.Data["ciphertext"].(string)
	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
	if err != nil {
		return key, sealedKey, err
	}
	copy(key[:], []byte(plainKey))
	return key, []byte(sealKey), nil
}

// UnsealKey returns the decrypted sealedKey as plaintext key.
// Therefore it sends the sealedKey to the KMS which decrypts
// it using the named key referenced by keyID and responses with
// the plaintext key.
//
// The context must be same context as the one provided while
// generating the plaintext key / sealedKey.
func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
	var contextStream bytes.Buffer
	ctx.WriteTo(&contextStream)

	payload := map[string]interface{}{
		"ciphertext": string(sealedKey),
		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
	}
	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
	if err != nil {
		return key, err
	}
	base64Key := s.Data["plaintext"].(string)
	plainKey, err := base64.StdEncoding.DecodeString(base64Key)
	if err != nil {
		return key, err
	}
	copy(key[:], []byte(plainKey))
	return key, nil
}
<|endoftext|>


<commit_before>package main

import (
	"expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/thecodearchive/gitarchive/camli"
	"github.com/thecodearchive/gitarchive/github"
	"github.com/thecodearchive/gitarchive/metrics"
	"github.com/thecodearchive/gitarchive/queue"
)

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	var t time.Time
	resume, err := ioutil.ReadFile(MustGetenv("RESUME_PATH"))
	if os.IsNotExist(err) {
		t = time.Now().Truncate(time.Hour).Add(-12 * time.Hour)
		log.Println("[ ] Can't load resume file, starting 12 hours ago")
	} else {
		fatalIfErr(err)
		t, err = time.Parse(github.HourFormat, string(resume))
		fatalIfErr(err)
		log.Printf("[+] Resuming from %s", t.Format(github.HourFormat))
	}

	exp := expvar.NewMap("drinker")
	expEvents := new(expvar.Map).Init()
	expLatest := new(expvar.String)
	exp.Set("latestevent", expLatest)
	exp.Set("events", expEvents)

	err = metrics.StartInfluxExport(MustGetenv("INFLUX_ADDR"), "drinker", exp)
	fatalIfErr(err)

	u, err := camli.NewUploader()
	fatalIfErr(err)

	qDriver := "sqlite3"
	if strings.Index(MustGetenv("QUEUE_ADDR"), "@") != -1 {
		qDriver = "mysql"
	}
	log.Printf("[ ] Opening queue (%s)...", qDriver)
	q, err := queue.Open(qDriver, MustGetenv("QUEUE_ADDR"))
	fatalIfErr(err)
	defer func() {
		log.Println("[ ] Closing queue...")
		fatalIfErr(q.Close())
	}()

	st := github.NewStarTracker(10000000, MustGetenv("GITHUB_TOKEN"))
	exp.Set("github", st.Expvar())
	if f, err := os.Open(MustGetenv("CACHE_PATH")); err != nil {
		log.Println("[ ] Can't load StarTracker cache, starting empty")
	} else {
		log.Println("[+] Loaded StarTracker cache")
		st.LoadCache(f)
		f.Close()
	}
	defer func() {
		f, err := os.Create(MustGetenv("CACHE_PATH"))
		fatalIfErr(err)
		log.Println("[ ] Writing StarTracker cache...")
		st.SaveCache(f)
		fatalIfErr(f.Close())
	}()

	d := &Drinker{
		q: q, st: st, u: u,
		exp: exp, expEvents: expEvents, expLatest: expLatest,
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		log.Println("[ ] Stopping gracefully...")
		d.Stop()
	}()

	startTime := t.Add(time.Hour).Add(2 * time.Minute)
	for {
		if time.Now().Before(startTime) {
			log.Printf("[ ] Waiting for the %s archive until %s...",
				t.Format(github.HourFormat), startTime)
			if !interruptableSleep(startTime.Sub(time.Now())) {
				break
			}
		}
		log.Printf("[ ] Opening %s archive download...", t.Format(github.HourFormat))
		a, err := github.DownloadArchive(t)
		fatalIfErr(err) // TODO: make more graceful
		if a == nil {
			exp.Add("archives404", 1)
			startTime = time.Now().Add(2 * time.Minute)
			continue
		}

		log.Printf("[+] Archive %s found, consuming...", t.Format(github.HourFormat))
		err = d.DrinkArchive(a)
		a.Close()
		if err == StoppedError {
			break
		}
		fatalIfErr(err) // TODO: make more graceful

		exp.Add("archivesfinished", 1)
		t = t.Add(time.Hour)
		fatalIfErr(ioutil.WriteFile(MustGetenv("RESUME_PATH"),
			[]byte(t.Format(github.HourFormat)), 0664))
		startTime = t.Add(time.Hour).Add(2 * time.Minute)
	}

	log.Println("[+] Processed events until", expLatest)
	fmt.Print(exp.String())
}

func fatalIfErr(err error) {
	if err != nil {
		log.Panic(err) // panic to let the defer run
	}
}

func MustGetenv(name string) string {
	val := os.Getenv(name)
	if val == "" {
		log.Panicln("Missing environment variable:", name)
	}
	return val
}
<commit_msg>drinker: check that cache loading is successful<commit_after>package main

import (
	"expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/thecodearchive/gitarchive/camli"
	"github.com/thecodearchive/gitarchive/github"
	"github.com/thecodearchive/gitarchive/metrics"
	"github.com/thecodearchive/gitarchive/queue"
)

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	var t time.Time
	resume, err := ioutil.ReadFile(MustGetenv("RESUME_PATH"))
	if os.IsNotExist(err) {
		t = time.Now().Truncate(time.Hour).Add(-12 * time.Hour)
		log.Println("[ ] Can't load resume file, starting 12 hours ago")
	} else {
		fatalIfErr(err)
		t, err = time.Parse(github.HourFormat, string(resume))
		fatalIfErr(err)
		log.Printf("[+] Resuming from %s", t.Format(github.HourFormat))
	}

	exp := expvar.NewMap("drinker")
	expEvents := new(expvar.Map).Init()
	expLatest := new(expvar.String)
	exp.Set("latestevent", expLatest)
	exp.Set("events", expEvents)

	err = metrics.StartInfluxExport(MustGetenv("INFLUX_ADDR"), "drinker", exp)
	fatalIfErr(err)

	u, err := camli.NewUploader()
	fatalIfErr(err)

	qDriver := "sqlite3"
	if strings.Index(MustGetenv("QUEUE_ADDR"), "@") != -1 {
		qDriver = "mysql"
	}
	log.Printf("[ ] Opening queue (%s)...", qDriver)
	q, err := queue.Open(qDriver, MustGetenv("QUEUE_ADDR"))
	fatalIfErr(err)
	defer func() {
		log.Println("[ ] Closing queue...")
		fatalIfErr(q.Close())
	}()

	st := github.NewStarTracker(10000000, MustGetenv("GITHUB_TOKEN"))
	exp.Set("github", st.Expvar())
	if f, err := os.Open(MustGetenv("CACHE_PATH")); err != nil {
		log.Println("[ ] Can't load StarTracker cache, starting empty")
	} else {
		fatalIfErr(st.LoadCache(f))
		f.Close()
		log.Println("[+] Loaded StarTracker cache")
	}
	defer func() {
		f, err := os.Create(MustGetenv("CACHE_PATH"))
		fatalIfErr(err)
		log.Println("[ ] Writing StarTracker cache...")
		st.SaveCache(f)
		fatalIfErr(f.Close())
	}()

	d := &Drinker{
		q: q, st: st, u: u,
		exp: exp, expEvents: expEvents, expLatest: expLatest,
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		log.Println("[ ] Stopping gracefully...")
		d.Stop()
	}()

	startTime := t.Add(time.Hour).Add(2 * time.Minute)
	for {
		if time.Now().Before(startTime) {
			log.Printf("[ ] Waiting for the %s archive until %s...",
				t.Format(github.HourFormat), startTime)
			if !interruptableSleep(startTime.Sub(time.Now())) {
				break
			}
		}
		log.Printf("[ ] Opening %s archive download...", t.Format(github.HourFormat))
		a, err := github.DownloadArchive(t)
		fatalIfErr(err) // TODO: make more graceful
		if a == nil {
			exp.Add("archives404", 1)
			startTime = time.Now().Add(2 * time.Minute)
			continue
		}

		log.Printf("[+] Archive %s found, consuming...", t.Format(github.HourFormat))
		err = d.DrinkArchive(a)
		a.Close()
		if err == StoppedError {
			break
		}
		fatalIfErr(err) // TODO: make more graceful

		exp.Add("archivesfinished", 1)
		t = t.Add(time.Hour)
		fatalIfErr(ioutil.WriteFile(MustGetenv("RESUME_PATH"),
			[]byte(t.Format(github.HourFormat)), 0664))
		startTime = t.Add(time.Hour).Add(2 * time.Minute)
	}

	log.Println("[+] Processed events until", expLatest)
	fmt.Print(exp.String())
}

func fatalIfErr(err error) {
	if err != nil {
		log.Panic(err) // panic to let the defer run
	}
}

func MustGetenv(name string) string {
	val := os.Getenv(name)
	if val == "" {
		log.Panicln("Missing environment variable:", name)
	}
	return val
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/logutils"

	"github.com/clusterit/orca/cmd"
	"github.com/clusterit/orca/config"
	"github.com/clusterit/orca/etcd"
	"github.com/clusterit/orca/logging"
	"github.com/clusterit/orca/users"

	"github.com/spf13/viper"

	"golang.org/x/crypto/ssh"
)

var (
	fetcher       UserFetcher
	configuration *config.Gateway
	sshConfig     ssh.ServerConfig
	configer      config.Configer
	zone          string
	lock          sync.Mutex
	revision      string
)

func init() {
	viper.SetEnvPrefix(cmd.OrcaPrefix)
	viper.AutomaticEnv()
	viper.SetDefault("bind", ":2222")
	viper.SetDefault("etcd", "http://localhost:4001")
	viper.SetDefault("zone", "intranet")

	zone = viper.GetString("zone")
	etcds := strings.Split(viper.GetString("etcd"), ",")
	cc, err := etcd.Init(etcds)
	if err != nil {
		panic(err)
	}
	fetcher, err = NewHttpFetcher(cc)
	if err != nil {
		panic(err)
	}
	cfger, err := config.New(cc)
	if err != nil {
		panic(err)
	}
	configer = cfger

}

func initWithConfig(gw *config.Gateway) error {
	lock.Lock()
	defer lock.Unlock()

	signer, err := ssh.ParsePrivateKey([]byte(gw.HostKey))
	if err != nil {
		return err
	}
	filter := &logutils.LevelFilter{
		Levels:   logging.Levels,
		MinLevel: logging.ByName(gw.LogLevel),
		Writer:   os.Stderr,
	}
	log.SetOutput(filter)

	sshConfig = ssh.ServerConfig{
		PublicKeyCallback: keyAuth,
	}
	sshConfig.AddHostKey(signer)
	configuration = gw
	return nil
}

func initWithSettings(zone string) error {
	cfg, _, err := cmd.ForceZone(configer, zone, true, true)
	if err != nil {
		return err
	}
	initWithConfig(cfg)

	go func() {
		ngw, stp, err := configer.Gateway(zone)
		if err != nil {
			Log(logging.Error, "cannot create watcher for gateway config: %s")
			return
		}
		for gw := range ngw {
			Log(logging.Debug, "new gateway config: %#v", gw)
			initWithConfig(&gw)
		}
		close(stp)
	}()
	return nil
}

func checkAllowed(u *users.User) error {
	if !configuration.CheckAllow {
		return nil
	}
	if u.Allowance == nil {
		return fmt.Errorf("please activate your account")
	}
	if u.Allowance.Until.Before(time.Now()) {
		return fmt.Errorf("your activation timed out")
	}

	return nil
}

func keyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
	pubk := string(ssh.MarshalAuthorizedKey(key))
	usr, err := fetcher.UserByKey(zone, strings.TrimSpace(pubk))
	if err != nil {
		Log(logging.Debug, "remote: %s: cannot fetch key for user '%s': %s", conn.RemoteAddr().String(), conn.User(), err)
		return nil, err
	}
	if err := checkAllowed(usr); err != nil {
		Log(logging.Debug, "remote: %s: not allowed to login for user '%s': %s", conn.RemoteAddr().String(), conn.User(), err)
		return nil, err
	}
	Log(logging.Info, "remote: %s: login by %+v", conn.RemoteAddr().String(), usr)
	return &ssh.Permissions{Extensions: map[string]string{
		"user_id": usr.Id}}, nil
}

func main() {
	initWithSettings(zone)

	bind := viper.GetString("bind")
	socket, err := net.Listen("tcp", bind)
	if err != nil {
		panic(err)
	}
	Log(logging.Info, "gateway listens on %#v ...", socket.Addr().String())
	for {

		tcpConn, err := socket.Accept()
		if err != nil {
			Log(logging.Error, "failed to accept incoming connection (%s)", err)
			continue
		}

		_, err = NewSession(ssh.NewServerConn(tcpConn, &sshConfig))

		if err != nil {
			Log(logging.Error, "failed to handshake (%s)", err)
			continue
		}

	}
}
<commit_msg>return orca revision in version string<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/logutils"

	"github.com/clusterit/orca/cmd"
	"github.com/clusterit/orca/config"
	"github.com/clusterit/orca/etcd"
	"github.com/clusterit/orca/logging"
	"github.com/clusterit/orca/users"

	"github.com/spf13/viper"

	"golang.org/x/crypto/ssh"
)

var (
	fetcher       UserFetcher
	configuration *config.Gateway
	sshConfig     ssh.ServerConfig
	configer      config.Configer
	zone          string
	lock          sync.Mutex
	revision      = "latest"
)

func init() {
	viper.SetEnvPrefix(cmd.OrcaPrefix)
	viper.AutomaticEnv()
	viper.SetDefault("bind", ":2222")
	viper.SetDefault("etcd", "http://localhost:4001")
	viper.SetDefault("zone", "intranet")

	zone = viper.GetString("zone")
	etcds := strings.Split(viper.GetString("etcd"), ",")
	cc, err := etcd.Init(etcds)
	if err != nil {
		panic(err)
	}
	fetcher, err = NewHttpFetcher(cc)
	if err != nil {
		panic(err)
	}
	cfger, err := config.New(cc)
	if err != nil {
		panic(err)
	}
	configer = cfger

}

func initWithConfig(gw *config.Gateway) error {
	lock.Lock()
	defer lock.Unlock()

	signer, err := ssh.ParsePrivateKey([]byte(gw.HostKey))
	if err != nil {
		return err
	}
	filter := &logutils.LevelFilter{
		Levels:   logging.Levels,
		MinLevel: logging.ByName(gw.LogLevel),
		Writer:   os.Stderr,
	}
	log.SetOutput(filter)

	sshConfig = ssh.ServerConfig{
		PublicKeyCallback: keyAuth,
		ServerVersion:     fmt.Sprintf("SSH-2.0-orca_%s", revision),
	}
	sshConfig.AddHostKey(signer)
	configuration = gw
	return nil
}

func initWithSettings(zone string) error {
	cfg, _, err := cmd.ForceZone(configer, zone, true, true)
	if err != nil {
		return err
	}
	initWithConfig(cfg)

	go func() {
		ngw, stp, err := configer.Gateway(zone)
		if err != nil {
			Log(logging.Error, "cannot create watcher for gateway config: %s")
			return
		}
		for gw := range ngw {
			Log(logging.Debug, "new gateway config: %#v", gw)
			initWithConfig(&gw)
		}
		close(stp)
	}()
	return nil
}

func checkAllowed(u *users.User) error {
	if !configuration.CheckAllow {
		return nil
	}
	if u.Allowance == nil {
		return fmt.Errorf("please activate your account")
	}
	if u.Allowance.Until.Before(time.Now()) {
		return fmt.Errorf("your activation timed out")
	}

	return nil
}

func keyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
	pubk := string(ssh.MarshalAuthorizedKey(key))
	usr, err := fetcher.UserByKey(zone, strings.TrimSpace(pubk))
	if err != nil {
		Log(logging.Debug, "remote: %s: cannot fetch key for user '%s': %s", conn.RemoteAddr().String(), conn.User(), err)
		return nil, err
	}
	if err := checkAllowed(usr); err != nil {
		Log(logging.Debug, "remote: %s: not allowed to login for user '%s': %s", conn.RemoteAddr().String(), conn.User(), err)
		return nil, err
	}
	Log(logging.Info, "remote: %s: login by %+v", conn.RemoteAddr().String(), usr)
	return &ssh.Permissions{Extensions: map[string]string{
		"user_id": usr.Id}}, nil
}

func main() {
	initWithSettings(zone)

	bind := viper.GetString("bind")
	socket, err := net.Listen("tcp", bind)
	if err != nil {
		panic(err)
	}
	Log(logging.Info, "gateway listens on %#v ...", socket.Addr().String())
	for {

		tcpConn, err := socket.Accept()
		if err != nil {
			Log(logging.Error, "failed to accept incoming connection (%s)", err)
			continue
		}

		_, err = NewSession(ssh.NewServerConn(tcpConn, &sshConfig))

		if err != nil {
			Log(logging.Error, "failed to handshake (%s)", err)
			continue
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"path/filepath"

	"gnd.la/log"

	"gopkgs.com/command.v1"
)

var (
	commands = []*command.Cmd{
		{
			Name: "dev",
			Help: "Start the Gondola development server",
			Func: devCommand,
			Options: &devOptions{
				Dir:  ".",
				Port: 8888,
			},
		},
		{
			Name:    "new",
			Help:    "Create a new Gondola project",
			Usage:   "<dir>",
			Func:    newCommand,
			Options: &newOptions{Template: "hello"},
		},
		{
			Name:     "build",
			Help:     "Build packages",
			Usage:    "[package-1] [package-2] ... [package-n]",
			LongHelp: buildHelp,
			Func:     buildCommand,
			Options:  &buildOptions{Go: "go"},
		},
		{
			Name: "clean",
			Help: "Cleans any Gondola packages which use conditional compilation - DO THIS BEFORE BUILDING A BINARY FOR DEPLOYMENT - see golang.org/issue/3172",
			Func: cleanCommand,
		},
		{
			Name:    "profile",
			Help:    "Show profiling information for a remote server running a Gondola app",
			Usage:   "<url>",
			Func:    profileCommand,
			Options: &profileOptions{Method: "GET"},
		},
		{
			Name:    "gen-app",
			Help:    "Generate boilerplate code for a Gondola app from the appfile.yaml file",
			Func:    genAppCommand,
			Options: &genAppOptions{},
		},
		{
			Name:    "bake",
			Help:    "Converts all assets in <dir> into Go code and generates a VFS named with <name>",
			Usage:   "-dir=<dir> -name=<name> ... additional flags",
			Options: &bakeOptions{},
			Func:    bakeCommand,
		},
		{
			Name:    "random-string",
			Help:    "Generates a random string suitable for use as the app secret",
			Func:    randomStringCommand,
			Options: &randomStringOptions{Length: defaultRandomLength},
		},
		{
			Name:  "rm-gen",
			Help:  "Remove Gondola generated files (identified by *.gen.*)",
			Usage: "[dir]",
			Func:  rmGenCommand,
		},
		{
			Name:    "make-messages",
			Help:    "Generate strings files from the current package (including its non-package subdirectories, like templates)",
			Func:    makeMessagesCommand,
			Options: &makeMessagesOptions{Out: filepath.Join("_messages", "messages.pot")},
		},
		{
			Name:    "compile-messages",
			Help:    "Compiles all po files from the current directory and its subdirectories",
			Func:    compileMessagesCommand,
			Options: &compileMessagesOptions{Out: "messages.go"},
		},
		{
			Name:    "gen",
			Help:    "Perform code generation in the current directory according the rules in the config file",
			Options: &genOptions{Genfile: "genfile.yaml"},
		},
		{
			Name:    "gae-dev",
			Help:    "Start the Gondola App Engine development server",
			Func:    gaeDevCommand,
			Options: &gaeDevOptions{Host: "localhost", Port: 8080, AdminPort: 8000},
		},
		{
			Name:    "gae-test",
			Help:    "Start serving your app on localhost and run gnd.la/app/tester tests against it",
			Func:    gaeTestCommand,
			Options: &gaeTestOptions{},
		},
		{
			Name:    "gae-deploy",
			Help:    "Deploy your application to App Engine",
			Func:    gaeDeployCommand,
			Options: &gaeDeployOptions{},
		},
	}
)

type commonOptions struct {
	Quiet bool `name:"q" help:"Disable verbose output"`
}

func main() {
	opts := &command.Options{
		Options: &commonOptions{},
		Func: func(opts *commonOptions) {
			if opts.Quiet {
				log.SetLevel(log.LError)
			} else {
				log.SetLevel(log.LDebug)
			}
		},
	}
	command.Exit(command.RunOpts(nil, opts, commands))
}
<commit_msg>Fix Func for the gondola gen cmd<commit_after>package main

import (
	"path/filepath"

	"gnd.la/log"

	"gopkgs.com/command.v1"
)

var (
	commands = []*command.Cmd{
		{
			Name: "dev",
			Help: "Start the Gondola development server",
			Func: devCommand,
			Options: &devOptions{
				Dir:  ".",
				Port: 8888,
			},
		},
		{
			Name:    "new",
			Help:    "Create a new Gondola project",
			Usage:   "<dir>",
			Func:    newCommand,
			Options: &newOptions{Template: "hello"},
		},
		{
			Name:     "build",
			Help:     "Build packages",
			Usage:    "[package-1] [package-2] ... [package-n]",
			LongHelp: buildHelp,
			Func:     buildCommand,
			Options:  &buildOptions{Go: "go"},
		},
		{
			Name: "clean",
			Help: "Cleans any Gondola packages which use conditional compilation - DO THIS BEFORE BUILDING A BINARY FOR DEPLOYMENT - see golang.org/issue/3172",
			Func: cleanCommand,
		},
		{
			Name:    "profile",
			Help:    "Show profiling information for a remote server running a Gondola app",
			Usage:   "<url>",
			Func:    profileCommand,
			Options: &profileOptions{Method: "GET"},
		},
		{
			Name:    "gen-app",
			Help:    "Generate boilerplate code for a Gondola app from the appfile.yaml file",
			Func:    genAppCommand,
			Options: &genAppOptions{},
		},
		{
			Name:    "bake",
			Help:    "Converts all assets in <dir> into Go code and generates a VFS named with <name>",
			Usage:   "-dir=<dir> -name=<name> ... additional flags",
			Options: &bakeOptions{},
			Func:    bakeCommand,
		},
		{
			Name:    "random-string",
			Help:    "Generates a random string suitable for use as the app secret",
			Func:    randomStringCommand,
			Options: &randomStringOptions{Length: defaultRandomLength},
		},
		{
			Name:  "rm-gen",
			Help:  "Remove Gondola generated files (identified by *.gen.*)",
			Usage: "[dir]",
			Func:  rmGenCommand,
		},
		{
			Name:    "make-messages",
			Help:    "Generate strings files from the current package (including its non-package subdirectories, like templates)",
			Func:    makeMessagesCommand,
			Options: &makeMessagesOptions{Out: filepath.Join("_messages", "messages.pot")},
		},
		{
			Name:    "compile-messages",
			Help:    "Compiles all po files from the current directory and its subdirectories",
			Func:    compileMessagesCommand,
			Options: &compileMessagesOptions{Out: "messages.go"},
		},
		{
			Name:    "gen",
			Help:    "Perform code generation in the current directory according the rules in the config file",
			Func:    genCommand,
			Options: &genOptions{Genfile: "genfile.yaml"},
		},
		{
			Name:    "gae-dev",
			Help:    "Start the Gondola App Engine development server",
			Func:    gaeDevCommand,
			Options: &gaeDevOptions{Host: "localhost", Port: 8080, AdminPort: 8000},
		},
		{
			Name:    "gae-test",
			Help:    "Start serving your app on localhost and run gnd.la/app/tester tests against it",
			Func:    gaeTestCommand,
			Options: &gaeTestOptions{},
		},
		{
			Name:    "gae-deploy",
			Help:    "Deploy your application to App Engine",
			Func:    gaeDeployCommand,
			Options: &gaeDeployOptions{},
		},
	}
)

type commonOptions struct {
	Quiet bool `name:"q" help:"Disable verbose output"`
}

func main() {
	opts := &command.Options{
		Options: &commonOptions{},
		Func: func(opts *commonOptions) {
			if opts.Quiet {
				log.SetLevel(log.LError)
			} else {
				log.SetLevel(log.LDebug)
			}
		},
	}
	command.Exit(command.RunOpts(nil, opts, commands))
}
<|endoftext|>


<commit_before>package main

import (
    "log"
    "github.com/keltia/gondole"
    "github.com/urfave/cli"
    "os"
)

var (
    fVerbose bool
)

func Register(c *cli.Context) error {

    return nil
}

func init() {
    cli.VersionFlag = cli.BoolFlag{Name: "version, V"}

    cli.VersionPrinter = func(c *cli.Context) {
        log.Printf("API wrapper: %s Mastodon CLI: %s\n", c.App.Version, gondole.APIVersion)
    }
}

func main() {
    app := cli.NewApp()
    app.Name = "gondole"
    app.Usage = "Mastodon CLI interface"
    app.Author = "Ollivier Robert <roberto@keltia.net>"
    app.Version = gondole.APIVersion
    //app.HideVersion = true

    app.Before = Register

    app.Flags = []cli.Flag{
        cli.BoolFlag{
            Name:        "verbose,v",
            Usage:       "verbose mode",
            Destination: &fVerbose,
        },
    }
    app.Run(os.Args)
}
<commit_msg>Hook up the application registration + gofmt.<commit_after>package main

import (
	"github.com/keltia/gondole"
	"github.com/urfave/cli"
	"log"
	"os"
)

var (
	fVerbose bool
    instance *gondole.Gondole
	cnf      *gondole.Config
)

func Register(c *cli.Context) (err error) {

    instance, err = gondole.NewApp("gondole-cli", nil, gondole.NoRedirect)
	return err
}

func init() {
	cli.VersionFlag = cli.BoolFlag{Name: "version, V"}

	cli.VersionPrinter = func(c *cli.Context) {
		log.Printf("API wrapper: %s Mastodon CLI: %s\n", c.App.Version, gondole.APIVersion)
	}
}

func main() {

	app := cli.NewApp()
	app.Name = "gondole"
	app.Usage = "Mastodon CLI interface"
	app.Author = "Ollivier Robert <roberto@keltia.net>"
	app.Version = gondole.APIVersion
	//app.HideVersion = true

	app.Before = Register

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "verbose,v",
			Usage:       "verbose mode",
			Destination: &fVerbose,
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/errors"
	"github.com/hellofresh/janus/pkg/loader"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	"github.com/hellofresh/janus/pkg/web"
	"github.com/spf13/cobra"
	mgo "gopkg.in/mgo.v2"
)

var (
	repo             api.Repository
	oAuthServersRepo oauth.Repository
	err              error
	globalConfig     *config.Specification
)

// RunServer is the run command to start Janus
func RunServer(cmd *cobra.Command, args []string) {
	log.Info("Janus starting...")

	globalConfig, err = config.Load(configFile)
	if nil != err {
		log.WithError(err).Panic("Could not parse the environment configurations")
	}

	initLogger(globalConfig.LogLevel)
	initTracing(globalConfig.Tracing)
	storage := initStorage(globalConfig.Storage)
	statsClient := initStatsdClient(globalConfig.Stats)
	defer statsClient.Close()

	dsnURL, err := url.Parse(globalConfig.Database.DSN)
	switch dsnURL.Scheme {
	case "mongodb":
		log.WithField("dsn", globalConfig.Database.DSN).Debug("Trying to connect to DB")
		session, err := mgo.Dial(globalConfig.Database.DSN)
		if err != nil {
			log.Panic(err)
		}

		defer session.Close()

		log.Debug("Connected to mongodb")
		session.SetMode(mgo.Monotonic, true)

		log.Debug("Loading API definitions from Mongo DB")
		repo, err = api.NewMongoAppRepository(session)
		if err != nil {
			log.Panic(err)
		}

		// create the proxy
		log.Debug("Loading OAuth servers definitions from Mongo DB")
		oAuthServersRepo, err = oauth.NewMongoRepository(session)
		if err != nil {
			log.Panic(err)
		}
	case "file":
		var apiPath = dsnURL.Path + "/apis"
		var authPath = dsnURL.Path + "/auth"

		log.WithField("path", apiPath).Debug("Loading API definitions from file system")
		repo, err = api.NewFileSystemRepository(apiPath)
		if err != nil {
			log.Panic(err)
		}

		log.WithField("path", authPath).Debug("Loading OAuth servers definitions from file system")
		oAuthServersRepo, err = oauth.NewFileSystemRepository(authPath)
		if err != nil {
			log.Panic(err)
		}
	default:
		log.WithError(errors.ErrInvalidScheme).Error("No Database selected")
	}

	p := proxy.WithParams(proxy.Params{
		StatsClient:            statsClient,
		FlushInterval:          globalConfig.BackendFlushInterval,
		IdleConnectionsPerHost: globalConfig.MaxIdleConnsPerHost,
		CloseIdleConnsPeriod:   globalConfig.CloseIdleConnsPeriod,
		InsecureSkipVerify:     globalConfig.InsecureSkipVerify,
	})
	defer p.Close()

	// create router with a custom not found handler
	router.DefaultOptions.NotFoundHandler = web.NotFound
	r := router.NewHTTPTreeMuxWithOptions(router.DefaultOptions)
	r.Use(
		middleware.NewStats(statsClient).Handler,
		middleware.NewLogger().Handler,
		middleware.NewRecovery(web.RecoveryHandler).Handler,
		middleware.NewOpenTracing(globalConfig.Web.IsHTTPS()).Handler,
	)

	pluginLoader := plugin.NewLoader()
	pluginLoader.Add(
		plugin.NewRateLimit(storage),
		plugin.NewCORS(),
		plugin.NewOAuth2(oAuthServersRepo, storage),
		plugin.NewCompression(),
	)

	// create proxy register
	register := proxy.NewRegister(r, p)
	var (
		apiSubs   *store.Subscription
		oauthSubs *store.Subscription
	)

	if subscriber, ok := storage.(store.Subscriber); ok {
		apiSubs = subscriber.Subscribe("api_updates")
		oauthSubs = subscriber.Subscribe("oauth_updates")
	}

	apiLoader := loader.NewAPILoader(register, pluginLoader, apiSubs)
	apiLoader.LoadDefinitions(repo)

	oauthLoader := loader.NewOAuthLoader(register, storage, oauthSubs)
	oauthLoader.LoadDefinitions(oAuthServersRepo)

	wp := web.Provider{
		Port:     globalConfig.Web.Port,
		Cred:     globalConfig.Web.Credentials,
		CertFile: globalConfig.Web.CertFile,
		KeyFile:  globalConfig.Web.KeyFile,
		APIRepo:  repo,
		AuthRepo: oAuthServersRepo,
		ReadOnly: globalConfig.Web.ReadOnly,
	}

	if publisher, ok := storage.(store.Publisher); ok {
		wp.Publisher = publisher
	}

	wp.Provide()

	log.Fatal(listenAndServe(r))
}

func listenAndServe(handler http.Handler) error {
	address := fmt.Sprintf(":%v", globalConfig.Port)
	log.WithField("address", address).Info("Listening on")
	log.Info("Janus started")
	if globalConfig.Web.IsHTTPS() {
		return http.ListenAndServeTLS(address, globalConfig.CertFile, globalConfig.KeyFile, handler)
	}

	log.Info("Certificate and certificate key were not found, defaulting to HTTP")
	return http.ListenAndServe(address, handler)
}
<commit_msg>Using the enw router<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/errors"
	"github.com/hellofresh/janus/pkg/loader"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	"github.com/hellofresh/janus/pkg/web"
	"github.com/spf13/cobra"
	mgo "gopkg.in/mgo.v2"
)

var (
	repo             api.Repository
	oAuthServersRepo oauth.Repository
	err              error
	globalConfig     *config.Specification
)

// RunServer is the run command to start Janus
func RunServer(cmd *cobra.Command, args []string) {
	log.Info("Janus starting...")

	globalConfig, err = config.Load(configFile)
	if nil != err {
		log.WithError(err).Panic("Could not parse the environment configurations")
	}

	initLogger(globalConfig.LogLevel)
	initTracing(globalConfig.Tracing)
	storage := initStorage(globalConfig.Storage)
	statsClient := initStatsdClient(globalConfig.Stats)
	defer statsClient.Close()

	dsnURL, err := url.Parse(globalConfig.Database.DSN)
	switch dsnURL.Scheme {
	case "mongodb":
		log.WithField("dsn", globalConfig.Database.DSN).Debug("Trying to connect to DB")
		session, err := mgo.Dial(globalConfig.Database.DSN)
		if err != nil {
			log.Panic(err)
		}

		defer session.Close()

		log.Debug("Connected to mongodb")
		session.SetMode(mgo.Monotonic, true)

		log.Debug("Loading API definitions from Mongo DB")
		repo, err = api.NewMongoAppRepository(session)
		if err != nil {
			log.Panic(err)
		}

		// create the proxy
		log.Debug("Loading OAuth servers definitions from Mongo DB")
		oAuthServersRepo, err = oauth.NewMongoRepository(session)
		if err != nil {
			log.Panic(err)
		}
	case "file":
		var apiPath = dsnURL.Path + "/apis"
		var authPath = dsnURL.Path + "/auth"

		log.WithField("path", apiPath).Debug("Loading API definitions from file system")
		repo, err = api.NewFileSystemRepository(apiPath)
		if err != nil {
			log.Panic(err)
		}

		log.WithField("path", authPath).Debug("Loading OAuth servers definitions from file system")
		oAuthServersRepo, err = oauth.NewFileSystemRepository(authPath)
		if err != nil {
			log.Panic(err)
		}
	default:
		log.WithError(errors.ErrInvalidScheme).Error("No Database selected")
	}

	p := proxy.WithParams(proxy.Params{
		StatsClient:            statsClient,
		FlushInterval:          globalConfig.BackendFlushInterval,
		IdleConnectionsPerHost: globalConfig.MaxIdleConnsPerHost,
		CloseIdleConnsPeriod:   globalConfig.CloseIdleConnsPeriod,
		InsecureSkipVerify:     globalConfig.InsecureSkipVerify,
	})
	defer p.Close()

	// create router with a custom not found handler
	router.DefaultOptions.NotFoundHandler = web.NotFound
	r := router.NewChiRouterWithOptions(router.DefaultOptions)
	r.Use(
		middleware.NewStats(statsClient).Handler,
		middleware.NewLogger().Handler,
		middleware.NewRecovery(web.RecoveryHandler).Handler,
		middleware.NewOpenTracing(globalConfig.Web.IsHTTPS()).Handler,
	)

	pluginLoader := plugin.NewLoader()
	pluginLoader.Add(
		plugin.NewRateLimit(storage),
		plugin.NewCORS(),
		plugin.NewOAuth2(oAuthServersRepo, storage),
		plugin.NewCompression(),
	)

	// create proxy register
	register := proxy.NewRegister(r, p)
	var (
		apiSubs   *store.Subscription
		oauthSubs *store.Subscription
	)

	if subscriber, ok := storage.(store.Subscriber); ok {
		apiSubs = subscriber.Subscribe("api_updates")
		oauthSubs = subscriber.Subscribe("oauth_updates")
	}

	apiLoader := loader.NewAPILoader(register, pluginLoader, apiSubs)
	apiLoader.LoadDefinitions(repo)

	oauthLoader := loader.NewOAuthLoader(register, storage, oauthSubs)
	oauthLoader.LoadDefinitions(oAuthServersRepo)

	wp := web.Provider{
		Port:     globalConfig.Web.Port,
		Cred:     globalConfig.Web.Credentials,
		CertFile: globalConfig.Web.CertFile,
		KeyFile:  globalConfig.Web.KeyFile,
		APIRepo:  repo,
		AuthRepo: oAuthServersRepo,
		ReadOnly: globalConfig.Web.ReadOnly,
	}

	if publisher, ok := storage.(store.Publisher); ok {
		wp.Publisher = publisher
	}

	wp.Provide()

	log.Fatal(listenAndServe(r))
}

func listenAndServe(handler http.Handler) error {
	address := fmt.Sprintf(":%v", globalConfig.Port)
	log.WithField("address", address).Info("Listening on")
	log.Info("Janus started")
	if globalConfig.Web.IsHTTPS() {
		return http.ListenAndServeTLS(address, globalConfig.CertFile, globalConfig.KeyFile, handler)
	}

	log.Info("Certificate and certificate key were not found, defaulting to HTTP")
	return http.ListenAndServe(address, handler)
}
<|endoftext|>