<commit_before>package main

import (
	"github.com/google/gopacket/layers"
	"bytes"
	"time"
	"github.com/google/gopacket"
	"io"
	"sync"
	"strconv"
)

// gopacket provide a tcp connection, however it split one tcp connection into two stream.
// So it is hard to match http request and response. we make our own connection here

const MAX_TCP_SEQ uint32 = 0xFFFFFFFF
const TCP_SEQ_WINDOW = 1 << 10

type TcpAssembler struct {
	connectionDict    map[string]*TcpConnection
	lock              sync.Mutex
	connectionHandler ConnectionHandler
	filterIp          string
	filterPort        uint16
}

func newTcpAssembler(connectionHandler ConnectionHandler) *TcpAssembler {
	return &TcpAssembler{connectionDict:map[string]*TcpConnection{}, connectionHandler:connectionHandler}
}

func (assembler *TcpAssembler) assemble(flow gopacket.Flow, tcp *layers.TCP, timestamp time.Time) {
	src := EndPoint{ip:flow.Src().String(), port:uint16(tcp.SrcPort)}
	dst := EndPoint{ip:flow.Dst().String(), port:uint16(tcp.DstPort)}
	dropped := false
	if assembler.filterIp != "" {
		if src.ip != assembler.filterIp && dst.ip != assembler.filterIp {
			dropped = true
		}
	}
	if assembler.filterPort != 0 {
		if src.port != assembler.filterPort && dst.port != assembler.filterPort {
			dropped = true
		}
	}
	if dropped {
		return
	}

	srcString := src.String()
	dstString := dst.String()
	var key string
	if srcString < dstString {
		key = srcString + "-" + dstString
	} else {
		key = dstString + "-" + srcString
	}
	connection := assembler.retrieveConnection(src, dst, key)

	connection.onReceive(src, dst, tcp, timestamp)

	if connection.closed() {
		assembler.deleteConnection(key)
		connection.finish()
	}

	//TODO: cleanup timeout connections
}

// get connection this packet belong to; create new one if is new connection
func (assembler *TcpAssembler) retrieveConnection(src, dst EndPoint, key string) *TcpConnection {
	assembler.lock.Lock()
	defer assembler.lock.Unlock()
	connection := assembler.connectionDict[key]
	if (connection == nil) {
		connection = newTcpConnection(key)
		assembler.connectionDict[key] = connection
		assembler.connectionHandler.handle(src, dst, connection)
	}
	return connection
}

// remove connection (when is closed or timeout)
func (assembler *TcpAssembler) deleteConnection(key string) {
	assembler.lock.Lock()
	defer assembler.lock.Unlock()
	delete(assembler.connectionDict, key)
}

// flush older packets
func (assembler *TcpAssembler) flushOlderThan(time time.Time) {
	var connections []*TcpConnection
	assembler.lock.Lock()
	for _, connection := range assembler.connectionDict {
		connections = append(connections, connection)
	}
	assembler.lock.Unlock()

	//for _, connection := range connections {
	//	connection.flushOlderThan(time)
	//}
}

func (assembler *TcpAssembler) finishAll() {
	assembler.lock.Lock()
	defer assembler.lock.Unlock()
	for _, connection := range assembler.connectionDict {
		connection.finish()
	}
	assembler.connectionDict = nil
	assembler.connectionHandler.finish()
}

type ConnectionHandler interface {
	handle(src EndPoint, dst EndPoint, connection *TcpConnection)
	finish()
}

// one tcp connection
type  TcpConnection struct {
	upStream      *NetworkStream // stream from client to server
	downStream    *NetworkStream // stream from server to client
	clientId      EndPoint       // the client key(by ip and port)
	lastTimestamp time.Time      // timestamp receive last packet
	isHttp        bool
	key           string
}

type EndPoint struct {
	ip   string
	port uint16
}

func (p EndPoint) equals(p2 EndPoint) bool {
	return p.ip == p2.ip && p.port == p2.port
}

func (p EndPoint) String() string {
	return p.ip + ":" + strconv.Itoa(int(p.port))
}

type ConnectionId struct {
	src EndPoint
	dst EndPoint
}

// create tcp connection, by the first tcp packet. this packet should from client to server
func newTcpConnection(key string) *TcpConnection {
	connection := &TcpConnection{
		upStream:newNetworkStream(),
		downStream:newNetworkStream(),
		key: key,
	}
	return connection
}

// when receive tcp packet
func (connection *TcpConnection) onReceive(src, dst EndPoint, tcp *layers.TCP, timestamp time.Time) {
	connection.lastTimestamp = timestamp
	payload := tcp.Payload

	if !connection.isHttp {
		// skip no-http data
		if !isHttpRequestData(payload) {
			return
		}
		// receive first valid http data packet
		connection.clientId = src
		connection.isHttp = true
	}

	var sendStream, confirmStream *NetworkStream
	//var up bool
	if connection.clientId.equals(src) {
		sendStream = connection.upStream
		confirmStream = connection.downStream
		//up = true
	} else {
		sendStream = connection.downStream
		confirmStream = connection.upStream
		//up = false
	}

	if len(payload) > 0 {
		sendStream.appendPacket(tcp)
	}

	if tcp.SYN {
		// do nothing
	}

	if tcp.ACK {
		// confirm
		confirmStream.confirmPacket(tcp.Ack)
	}

	// terminate connection
	if tcp.FIN || tcp.RST {
		sendStream.closed = true
	}
}

func (connection *TcpConnection) flushOlderThan(time time.Time) {
	// flush all data
	//connection.upStream.window
	//connection.downStream.window
	// remove and close connection
	connection.upStream.closed = true
	connection.downStream.closed = true
	connection.finish()

}

func (connection *TcpConnection) closed() bool {
	return connection.upStream.closed && connection.downStream.closed
}

func (connection *TcpConnection) finish() {
	connection.upStream.finish()
	connection.downStream.finish()
}

// tread one-direction tcp data as stream. impl reader closer
type NetworkStream struct {
	window *ReceiveWindow
	c      chan []byte
	remain []byte
	ignore bool
	closed bool
}

func newNetworkStream() *NetworkStream {
	return &NetworkStream{window:newReceiveWindow(), c:make(chan []byte, 1000)}
}

func (stream *NetworkStream) appendPacket(tcp *layers.TCP) {
	if stream.ignore {
		return
	}
	stream.window.insert(tcp)
}

func (stream *NetworkStream) confirmPacket(ack uint32) {
	if stream.ignore {
		return
	}
	stream.window.confirm(ack, stream.c)
}

func (stream *NetworkStream) finish() {
	close(stream.c)
}

func (stream *NetworkStream) Read(p []byte) (n int, err error) {
	for len(stream.remain) == 0 {
		data, ok := <-stream.c
		if !ok {
			err = io.EOF
			return
		}
		stream.remain = data
	}

	if len(stream.remain) > len(p) {
		n = copy(p, stream.remain[:len(p)])
		stream.remain = stream.remain[len(p):]
	} else {
		n = copy(p, stream.remain)
		stream.remain = nil
	}
	return
}

func (stream *NetworkStream) Close() error {
	stream.ignore = true
	return nil
}

type ReceiveWindow struct {
	size   int
	start  int
	buffer []*layers.TCP
}

func newReceiveWindow() *ReceiveWindow {
	buffer := make([]*layers.TCP, 32)
	return &ReceiveWindow{buffer:buffer}
}

func (window *ReceiveWindow) destroy() {
	window.size = 0
	window.start = 0
	window.buffer = nil
}

func (window *ReceiveWindow) insert(packet *layers.TCP) {
	idx := 0
	for ; idx < window.size; idx ++ {
		index := (idx + window.start) % len(window.buffer)
		current := window.buffer[index]
		result := compareTcpSeq(current.Seq, packet.Seq)
		if result == 0 {
			// duplicated
			return
		}
		if result > 0 {
			// insert at index
			break
		}
	}

	if window.size == len(window.buffer) {
		window.expand()
	}

	if idx == window.size {
		// append at last
		index := (idx + window.start) % len(window.buffer)
		window.buffer[index] = packet
	} else {
		// insert at index
		for i := window.size; i >= idx; i-- {
			next := (i + window.start + 1) % len(window.buffer)
			current := (i + window.start) % len(window.buffer)
			window.buffer[next] = window.buffer[current]
		}
		index := (idx + window.start) % len(window.buffer)
		window.buffer[index] = packet

	}

	window.size++
}

func (window *ReceiveWindow) confirm(ack uint32, c chan []byte) {
	idx := 0
	for ; idx < window.size; idx ++ {
		index := (idx + window.start) % len(window.buffer)
		current := window.buffer[index]
		result := compareTcpSeq(current.Seq, ack)
		if result > 0 {
			break
		}
		window.buffer[index] = nil
		c <- current.Payload
	}
	window.start = (window.start + idx) % len(window.buffer)
	window.size = window.size - idx
}

func (window *ReceiveWindow) expand() {
	buffer := make([]*layers.TCP, len(window.buffer) * 2)
	end := window.start + window.size
	if end < len(window.buffer) {
		copy(buffer, window.buffer[window.start:window.start + window.size])
	} else {
		copy(buffer, window.buffer[window.start:])
		copy(buffer[len(window.buffer) - window.start:], window.buffer[:end - len(window.buffer)])
	}
	window.start = 0
	window.buffer = buffer
}

type Buffer []*layers.TCP
// impl sort.Interface
// Len is the number of elements in the collection.
func (buffer Buffer) Len() int {
	return len(buffer)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (buffer Buffer) Less(i, j int) bool {
	return compareTcpSeq(buffer[i].Seq, buffer[j].Seq) < 0
}
// Swap swaps the elements with indexes i and j.
func (buffer Buffer) Swap(i, j int) {
	buffer[i], buffer[j] = buffer[j], buffer[i]
}

// compare two tcp sequences, if seq1 is earlier, return num < 0, if seq1 == seq2, return 0, else return num > 0
func compareTcpSeq(seq1, seq2 uint32) int {
	if seq1 < TCP_SEQ_WINDOW && seq2 > MAX_TCP_SEQ - TCP_SEQ_WINDOW {
		return int(seq1 + MAX_TCP_SEQ - seq2)
	} else if seq2 < TCP_SEQ_WINDOW && seq1 > MAX_TCP_SEQ - TCP_SEQ_WINDOW {
		return int(seq1 - (MAX_TCP_SEQ + seq2))
	}
	return int(int32(seq1 - seq2))
}

var HTTP_METHODS = map[string]bool{"GET":true, "POST":true, "PUT":true, "DELETE":true, "HEAD":true, "TRACE":true,
	"OPTIONS":true, "PATCH":true}

// if is first http request packet
func isHttpRequestData(body []byte) bool {
	if len(body) < 8 {
		return false
	}
	data := body[0:8]
	idx := bytes.IndexByte(data, byte(' '))
	if (idx < 0) {
		return false
	}

	method := string(data[:idx])
	return HTTP_METHODS[method]
}<commit_msg>remove timeout connections<commit_after>package main

import (
	"github.com/google/gopacket/layers"
	"bytes"
	"time"
	"github.com/google/gopacket"
	"io"
	"sync"
	"strconv"
)

// gopacket provide a tcp connection, however it split one tcp connection into two stream.
// So it is hard to match http request and response. we make our own connection here

const MAX_TCP_SEQ uint32 = 0xFFFFFFFF
const TCP_SEQ_WINDOW = 1 << 10

type TcpAssembler struct {
	connectionDict    map[string]*TcpConnection
	lock              sync.Mutex
	connectionHandler ConnectionHandler
	filterIp          string
	filterPort        uint16
}

func newTcpAssembler(connectionHandler ConnectionHandler) *TcpAssembler {
	return &TcpAssembler{connectionDict:map[string]*TcpConnection{}, connectionHandler:connectionHandler}
}

func (assembler *TcpAssembler) assemble(flow gopacket.Flow, tcp *layers.TCP, timestamp time.Time) {
	src := EndPoint{ip:flow.Src().String(), port:uint16(tcp.SrcPort)}
	dst := EndPoint{ip:flow.Dst().String(), port:uint16(tcp.DstPort)}
	dropped := false
	if assembler.filterIp != "" {
		if src.ip != assembler.filterIp && dst.ip != assembler.filterIp {
			dropped = true
		}
	}
	if assembler.filterPort != 0 {
		if src.port != assembler.filterPort && dst.port != assembler.filterPort {
			dropped = true
		}
	}
	if dropped {
		return
	}

	srcString := src.String()
	dstString := dst.String()
	var key string
	if srcString < dstString {
		key = srcString + "-" + dstString
	} else {
		key = dstString + "-" + srcString
	}
	connection := assembler.retrieveConnection(src, dst, key)

	connection.onReceive(src, dst, tcp, timestamp)

	if connection.closed() {
		assembler.deleteConnection(key)
		connection.finish()
	}

	//TODO: cleanup timeout connections
}

// get connection this packet belong to; create new one if is new connection
func (assembler *TcpAssembler) retrieveConnection(src, dst EndPoint, key string) *TcpConnection {
	assembler.lock.Lock()
	defer assembler.lock.Unlock()
	connection := assembler.connectionDict[key]
	if (connection == nil) {
		connection = newTcpConnection(key)
		assembler.connectionDict[key] = connection
		assembler.connectionHandler.handle(src, dst, connection)
	}
	return connection
}

// remove connection (when is closed or timeout)
func (assembler *TcpAssembler) deleteConnection(key string) {
	assembler.lock.Lock()
	defer assembler.lock.Unlock()
	delete(assembler.connectionDict, key)
}

// flush timeout connections
func (assembler *TcpAssembler) flushOlderThan(time time.Time) {
	var connections []*TcpConnection
	assembler.lock.Lock()
	for _, connection := range assembler.connectionDict {
		if connection.lastTimestamp.Before(time) {
			connections = append(connections, connection)
		}
	}
	for _, connection := range connections {
		delete(assembler.connectionDict, connection.key)
	}
	assembler.lock.Unlock()

	for _, connection := range connections {
		connection.flushOlderThan()
	}
}

func (assembler *TcpAssembler) finishAll() {
	assembler.lock.Lock()
	defer assembler.lock.Unlock()
	for _, connection := range assembler.connectionDict {
		connection.finish()
	}
	assembler.connectionDict = nil
	assembler.connectionHandler.finish()
}

type ConnectionHandler interface {
	handle(src EndPoint, dst EndPoint, connection *TcpConnection)
	finish()
}

// one tcp connection
type  TcpConnection struct {
	upStream      *NetworkStream // stream from client to server
	downStream    *NetworkStream // stream from server to client
	clientId      EndPoint       // the client key(by ip and port)
	lastTimestamp time.Time      // timestamp receive last packet
	isHttp        bool
	key           string
}

type EndPoint struct {
	ip   string
	port uint16
}

func (p EndPoint) equals(p2 EndPoint) bool {
	return p.ip == p2.ip && p.port == p2.port
}

func (p EndPoint) String() string {
	return p.ip + ":" + strconv.Itoa(int(p.port))
}

type ConnectionId struct {
	src EndPoint
	dst EndPoint
}

// create tcp connection, by the first tcp packet. this packet should from client to server
func newTcpConnection(key string) *TcpConnection {
	connection := &TcpConnection{
		upStream:newNetworkStream(),
		downStream:newNetworkStream(),
		key: key,
	}
	return connection
}

// when receive tcp packet
func (connection *TcpConnection) onReceive(src, dst EndPoint, tcp *layers.TCP, timestamp time.Time) {
	connection.lastTimestamp = timestamp
	payload := tcp.Payload

	if !connection.isHttp {
		// skip no-http data
		if !isHttpRequestData(payload) {
			return
		}
		// receive first valid http data packet
		connection.clientId = src
		connection.isHttp = true
	}

	var sendStream, confirmStream *NetworkStream
	//var up bool
	if connection.clientId.equals(src) {
		sendStream = connection.upStream
		confirmStream = connection.downStream
		//up = true
	} else {
		sendStream = connection.downStream
		confirmStream = connection.upStream
		//up = false
	}

	if len(payload) > 0 {
		sendStream.appendPacket(tcp)
	}

	if tcp.SYN {
		// do nothing
	}

	if tcp.ACK {
		// confirm
		confirmStream.confirmPacket(tcp.Ack)
	}

	// terminate connection
	if tcp.FIN || tcp.RST {
		sendStream.closed = true
	}
}

// just close this connection?
func (connection *TcpConnection) flushOlderThan() {
	// flush all data
	//connection.upStream.window
	//connection.downStream.window
	// remove and close connection
	connection.upStream.closed = true
	connection.downStream.closed = true
	connection.finish()

}

func (connection *TcpConnection) closed() bool {
	return connection.upStream.closed && connection.downStream.closed
}

func (connection *TcpConnection) finish() {
	connection.upStream.finish()
	connection.downStream.finish()
}

// tread one-direction tcp data as stream. impl reader closer
type NetworkStream struct {
	window *ReceiveWindow
	c      chan []byte
	remain []byte
	ignore bool
	closed bool
}

func newNetworkStream() *NetworkStream {
	return &NetworkStream{window:newReceiveWindow(), c:make(chan []byte, 1000)}
}

func (stream *NetworkStream) appendPacket(tcp *layers.TCP) {
	if stream.ignore {
		return
	}
	stream.window.insert(tcp)
}

func (stream *NetworkStream) confirmPacket(ack uint32) {
	if stream.ignore {
		return
	}
	stream.window.confirm(ack, stream.c)
}

func (stream *NetworkStream) finish() {
	close(stream.c)
}

func (stream *NetworkStream) Read(p []byte) (n int, err error) {
	for len(stream.remain) == 0 {
		data, ok := <-stream.c
		if !ok {
			err = io.EOF
			return
		}
		stream.remain = data
	}

	if len(stream.remain) > len(p) {
		n = copy(p, stream.remain[:len(p)])
		stream.remain = stream.remain[len(p):]
	} else {
		n = copy(p, stream.remain)
		stream.remain = nil
	}
	return
}

func (stream *NetworkStream) Close() error {
	stream.ignore = true
	return nil
}

type ReceiveWindow struct {
	size   int
	start  int
	buffer []*layers.TCP
}

func newReceiveWindow() *ReceiveWindow {
	buffer := make([]*layers.TCP, 32)
	return &ReceiveWindow{buffer:buffer}
}

func (window *ReceiveWindow) destroy() {
	window.size = 0
	window.start = 0
	window.buffer = nil
}

func (window *ReceiveWindow) insert(packet *layers.TCP) {
	idx := 0
	for ; idx < window.size; idx ++ {
		index := (idx + window.start) % len(window.buffer)
		current := window.buffer[index]
		result := compareTcpSeq(current.Seq, packet.Seq)
		if result == 0 {
			// duplicated
			return
		}
		if result > 0 {
			// insert at index
			break
		}
	}

	if window.size == len(window.buffer) {
		window.expand()
	}

	if idx == window.size {
		// append at last
		index := (idx + window.start) % len(window.buffer)
		window.buffer[index] = packet
	} else {
		// insert at index
		for i := window.size; i >= idx; i-- {
			next := (i + window.start + 1) % len(window.buffer)
			current := (i + window.start) % len(window.buffer)
			window.buffer[next] = window.buffer[current]
		}
		index := (idx + window.start) % len(window.buffer)
		window.buffer[index] = packet

	}

	window.size++
}

func (window *ReceiveWindow) confirm(ack uint32, c chan []byte) {
	idx := 0
	for ; idx < window.size; idx ++ {
		index := (idx + window.start) % len(window.buffer)
		current := window.buffer[index]
		result := compareTcpSeq(current.Seq, ack)
		if result > 0 {
			break
		}
		window.buffer[index] = nil
		c <- current.Payload
	}
	window.start = (window.start + idx) % len(window.buffer)
	window.size = window.size - idx
}

func (window *ReceiveWindow) expand() {
	buffer := make([]*layers.TCP, len(window.buffer) * 2)
	end := window.start + window.size
	if end < len(window.buffer) {
		copy(buffer, window.buffer[window.start:window.start + window.size])
	} else {
		copy(buffer, window.buffer[window.start:])
		copy(buffer[len(window.buffer) - window.start:], window.buffer[:end - len(window.buffer)])
	}
	window.start = 0
	window.buffer = buffer
}

type Buffer []*layers.TCP
// impl sort.Interface
// Len is the number of elements in the collection.
func (buffer Buffer) Len() int {
	return len(buffer)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (buffer Buffer) Less(i, j int) bool {
	return compareTcpSeq(buffer[i].Seq, buffer[j].Seq) < 0
}
// Swap swaps the elements with indexes i and j.
func (buffer Buffer) Swap(i, j int) {
	buffer[i], buffer[j] = buffer[j], buffer[i]
}

// compare two tcp sequences, if seq1 is earlier, return num < 0, if seq1 == seq2, return 0, else return num > 0
func compareTcpSeq(seq1, seq2 uint32) int {
	if seq1 < TCP_SEQ_WINDOW && seq2 > MAX_TCP_SEQ - TCP_SEQ_WINDOW {
		return int(seq1 + MAX_TCP_SEQ - seq2)
	} else if seq2 < TCP_SEQ_WINDOW && seq1 > MAX_TCP_SEQ - TCP_SEQ_WINDOW {
		return int(seq1 - (MAX_TCP_SEQ + seq2))
	}
	return int(int32(seq1 - seq2))
}

var HTTP_METHODS = map[string]bool{"GET":true, "POST":true, "PUT":true, "DELETE":true, "HEAD":true, "TRACE":true,
	"OPTIONS":true, "PATCH":true}

// if is first http request packet
func isHttpRequestData(body []byte) bool {
	if len(body) < 8 {
		return false
	}
	data := body[0:8]
	idx := bytes.IndexByte(data, byte(' '))
	if (idx < 0) {
		return false
	}

	method := string(data[:idx])
	return HTTP_METHODS[method]
}<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package views

import (
	"encoding/json"
	"strings"

	"github.com/ernestio/api-gateway/models"
	"log"

	graph "gopkg.in/r3labs/graph.v2"
)

// ServiceRender : Service representation to be rendered on the frontend
type ServiceRender struct {
	ID              string              `json:"id"`
	DatacenterID    int                 `json:"datacenter_id"`
	Name            string              `json:"name"`
	Version         string              `json:"version"`
	Status          string              `json:"status"`
	UserID          int                 `json:"user_id"`
	UserName        string              `json:"user_name"`
	LastKnownError  string              `json:"last_known_error"`
	Options         string              `json:"options"`
	Definition      string              `json:"definition"`
	Vpcs            []map[string]string `json:"vpcs"`
	Networks        []map[string]string `json:"networks"`
	Instances       []map[string]string `json:"instances"`
	Nats            []map[string]string `json:"nats"`
	SecurityGroups  []map[string]string `json:"security_groups"`
	Elbs            []map[string]string `json:"elbs"`
	RDSClusters     []map[string]string `json:"rds_clusters"`
	RDSInstances    []map[string]string `json:"rds_instances"`
	EBSVolumes      []map[string]string `json:"ebs_volumes"`
	LoadBalancers   []map[string]string `json:"load_balancers"`
	SQLDatabases    []map[string]string `json:"sql_databases"`
	VirtualMachines []map[string]string `json:"virtual_machines"`
}

// Render : Map a Service to a ServiceRender
func (o *ServiceRender) Render(s models.Service) (err error) {
	o.ID = s.ID
	o.DatacenterID = s.DatacenterID
	o.Name = s.Name
	o.Version = s.Version.String()
	o.Status = s.Status
	o.UserID = s.UserID
	o.UserName = s.UserName
	if def, ok := s.Definition.(string); ok == true {
		o.Definition = def
	}

	g, err := s.Mapping()
	if err != nil {
		log.Println(err.Error())
		return err
	}

	o.Vpcs = RenderVpcs(g)
	o.Networks = RenderNetworks(g)
	o.SecurityGroups = RenderSecurityGroups(g)
	o.Nats = RenderNats(g)
	o.Instances = RenderInstances(g)
	o.Elbs = RenderELBs(g)
	o.RDSClusters = RenderRDSClusters(g)
	o.RDSInstances = RenderRDSInstances(g)
	o.EBSVolumes = RenderEBSVolumes(g)
	o.LoadBalancers = RenderLoadBalancers(g)
	o.SQLDatabases = RenderSQLDatabases(g)
	o.VirtualMachines = RenderVirtualMachines(g)

	return err
}

// RenderVpcs : renders a services vpcs
func RenderVpcs(g *graph.Graph) []map[string]string {
	var vpcs []map[string]string

	for _, n := range g.GetComponents().ByType("vpc") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["vpc_aws_id"].(string)
		subnet, _ := (*gc)["vpc_subnet"].(string)
		vpcs = append(vpcs, map[string]string{
			"name":       name,
			"vpc_id":     id,
			"vpc_subnet": subnet,
		})
	}

	return vpcs
}

// RenderNetworks : renders a services networks
func RenderNetworks(g *graph.Graph) []map[string]string {
	var networks []map[string]string

	for _, n := range g.GetComponents().ByType("network") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["network_aws_id"].(string)
		az, _ := (*gc)["availability_zone"].(string)
		networks = append(networks, map[string]string{
			"name":              name,
			"network_aws_id":    id,
			"availability_zone": az,
		})
	}

	return networks
}

// RenderSecurityGroups : renders a services security groups
func RenderSecurityGroups(g *graph.Graph) []map[string]string {
	var sgs []map[string]string

	for _, n := range g.GetComponents().ByType("firewall") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["security_group_aws_id"].(string)
		sgs = append(sgs, map[string]string{
			"name":                  name,
			"security_group_aws_id": id,
		})
	}

	return sgs
}

// RenderNats : renders a services nat gateways
func RenderNats(g *graph.Graph) []map[string]string {
	var nats []map[string]string

	for _, n := range g.GetComponents().ByType("nat") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["nat_gateway_aws_id"].(string)
		pubIP, _ := (*gc)["nat_gateway_allocation_ip"].(string)
		nats = append(nats, map[string]string{
			"name":               name,
			"nat_gateway_aws_id": id,
			"public_ip":          pubIP,
		})
	}

	return nats
}

// RenderELBs : renders a services elbs
func RenderELBs(g *graph.Graph) []map[string]string {
	var elbs []map[string]string

	for _, n := range g.GetComponents().ByType("elb") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		dns, _ := (*gc)["dns_name"].(string)
		elbs = append(elbs, map[string]string{
			"name":     name,
			"dns_name": dns,
		})
	}

	return elbs
}

// RenderInstances : renders a services instances
func RenderInstances(g *graph.Graph) []map[string]string {
	var instances []map[string]string

	for _, n := range g.GetComponents().ByType("instance") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["instance_aws_id"].(string)
		pip, _ := (*gc)["public_ip"].(string)
		ip, _ := (*gc)["ip"].(string)
		instances = append(instances, map[string]string{
			"name":            name,
			"instance_aws_id": id,
			"public_ip":       pip,
			"ip":              ip,
		})
	}

	return instances
}

// RenderRDSClusters : renders a services rds clusters
func RenderRDSClusters(g *graph.Graph) []map[string]string {
	var rdss []map[string]string

	for _, n := range g.GetComponents().ByType("rds_cluster") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		endpoint, _ := (*gc)["endpoint"].(string)
		rdss = append(rdss, map[string]string{
			"name":     name,
			"endpoint": endpoint,
		})
	}

	return rdss
}

// RenderRDSInstances : renders a services rds instances
func RenderRDSInstances(g *graph.Graph) []map[string]string {
	var rdss []map[string]string

	for _, n := range g.GetComponents().ByType("rds_instance") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		endpoint, _ := (*gc)["endpoint"].(string)
		rdss = append(rdss, map[string]string{
			"name":     name,
			"endpoint": endpoint,
		})
	}

	return rdss
}

// RenderEBSVolumes : renders a services ebs volumes
func RenderEBSVolumes(g *graph.Graph) []map[string]string {
	var rdss []map[string]string

	for _, n := range g.GetComponents().ByType("ebs_volume") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["volume_aws_id"].(string)
		rdss = append(rdss, map[string]string{
			"name":          name,
			"volume_aws_id": id,
		})
	}

	return rdss
}

// RenderLoadBalancers : renders load balancers
func RenderLoadBalancers(g *graph.Graph) []map[string]string {
	var lbs []map[string]string
	ips := listIPAddresses(g)

	for _, n := range g.GetComponents().ByType("lb") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["id"].(string)
		configs, _ := (*gc)["frontend_ip_configurations"].([]interface{})
		cfg, _ := configs[0].(map[string]interface{})
		ipID, _ := cfg["public_ip_address_id"].(string)
		ip, _ := ips[ipID]

		lbs = append(lbs, map[string]string{
			"name":      name,
			"id":        id,
			"public_ip": ip,
		})
	}

	return lbs
}

func listIPAddresses(g *graph.Graph) map[string]string {
	existingIPs := make(map[string]string, 0)

	for _, ip := range g.GetComponents().ByType("public_ip") {
		gc := ip.(*graph.GenericComponent)
		id, _ := (*gc)["id"].(string)
		ipAddress, _ := (*gc)["ip_address"].(string)
		existingIPs[id] = ipAddress
	}

	return existingIPs
}

// RenderVirtualMachines : renders virtual machines
func RenderVirtualMachines(g *graph.Graph) []map[string]string {
	var resources []map[string]string
	mappedIPs := make(map[string]interface{}, 0)
	existingIPs := listIPAddresses(g)

	for _, ni := range g.GetComponents().ByType("network_interface") {
		var public []string
		var private []string

		gc := ni.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		ips := make(map[string][]string)

		configs, _ := (*gc)["ip_configuration"].([]interface{})
		for _, cfg := range configs {
			c, _ := cfg.(map[string]interface{})
			pubID, _ := c["public_ip_address_id"].(string)
			pri, _ := c["private_ip_address"].(string)
			if pub, ok := existingIPs[pubID]; ok {
				public = append(public, pub)
			}
			private = append(private, pri)
		}

		ips["public"] = public
		ips["private"] = private
		mappedIPs[name] = make(map[string][]string, 0)
		mappedIPs[name] = ips
	}

	for _, n := range g.GetComponents().ByType("virtual_machine") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["id"].(string)
		networks, _ := (*gc)["network_interfaces"].([]interface{})
		publicIPs := make([]string, 0)
		privateIPs := make([]string, 0)
		for _, ni := range networks {
			netName := ni.(string)
			if val, ok := mappedIPs[netName]; ok {
				ips, _ := val.(map[string][]string)
				publicIPs = append(publicIPs, ips["public"]...)
				privateIPs = append(privateIPs, ips["private"]...)
			}
		}

		resources = append(resources, map[string]string{
			"name":       name,
			"id":         id,
			"public_ip":  strings.Join(publicIPs, ", "),
			"private_ip": strings.Join(privateIPs, ", "),
		})
	}

	return resources
}

// RenderSQLDatabases : renders sql databases
func RenderSQLDatabases(g *graph.Graph) []map[string]string {
	return renderResources(g, "sql_database", func(gc *graph.GenericComponent) map[string]string {
		name, _ := (*gc)["name"].(string)
		server, _ := (*gc)["server_name"].(string)
		id, _ := (*gc)["id"].(string)

		return map[string]string{
			"name":        name,
			"server_name": server,
			"id":          id,
		}
	})
}

type convert func(*graph.GenericComponent) map[string]string

func renderResources(g *graph.Graph, resourceType string, f convert) (resources []map[string]string) {
	for _, n := range g.GetComponents().ByType(resourceType) {
		gc := n.(*graph.GenericComponent)
		resources = append(resources, f(gc))
	}

	return
}

// RenderCollection : Maps a collection of Service on a collection of ServiceRender
func (o *ServiceRender) RenderCollection(services []models.Service) (list []ServiceRender, err error) {
	for _, s := range services {
		var output ServiceRender
		if err := output.Render(s); err == nil {
			list = append(list, output)
		}
	}

	return list, nil
}

// ToJSON : Converts a ServiceRender to json string
func (o *ServiceRender) ToJSON() ([]byte, error) {
	return json.Marshal(o)
}
<commit_msg>display fill server name<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package views

import (
	"encoding/json"
	"strings"

	"github.com/ernestio/api-gateway/models"
	"log"

	graph "gopkg.in/r3labs/graph.v2"
)

// ServiceRender : Service representation to be rendered on the frontend
type ServiceRender struct {
	ID              string              `json:"id"`
	DatacenterID    int                 `json:"datacenter_id"`
	Name            string              `json:"name"`
	Version         string              `json:"version"`
	Status          string              `json:"status"`
	UserID          int                 `json:"user_id"`
	UserName        string              `json:"user_name"`
	LastKnownError  string              `json:"last_known_error"`
	Options         string              `json:"options"`
	Definition      string              `json:"definition"`
	Vpcs            []map[string]string `json:"vpcs"`
	Networks        []map[string]string `json:"networks"`
	Instances       []map[string]string `json:"instances"`
	Nats            []map[string]string `json:"nats"`
	SecurityGroups  []map[string]string `json:"security_groups"`
	Elbs            []map[string]string `json:"elbs"`
	RDSClusters     []map[string]string `json:"rds_clusters"`
	RDSInstances    []map[string]string `json:"rds_instances"`
	EBSVolumes      []map[string]string `json:"ebs_volumes"`
	LoadBalancers   []map[string]string `json:"load_balancers"`
	SQLDatabases    []map[string]string `json:"sql_databases"`
	VirtualMachines []map[string]string `json:"virtual_machines"`
}

// Render : Map a Service to a ServiceRender
func (o *ServiceRender) Render(s models.Service) (err error) {
	o.ID = s.ID
	o.DatacenterID = s.DatacenterID
	o.Name = s.Name
	o.Version = s.Version.String()
	o.Status = s.Status
	o.UserID = s.UserID
	o.UserName = s.UserName
	if def, ok := s.Definition.(string); ok == true {
		o.Definition = def
	}

	g, err := s.Mapping()
	if err != nil {
		log.Println(err.Error())
		return err
	}

	o.Vpcs = RenderVpcs(g)
	o.Networks = RenderNetworks(g)
	o.SecurityGroups = RenderSecurityGroups(g)
	o.Nats = RenderNats(g)
	o.Instances = RenderInstances(g)
	o.Elbs = RenderELBs(g)
	o.RDSClusters = RenderRDSClusters(g)
	o.RDSInstances = RenderRDSInstances(g)
	o.EBSVolumes = RenderEBSVolumes(g)
	o.LoadBalancers = RenderLoadBalancers(g)
	o.SQLDatabases = RenderSQLDatabases(g)
	o.VirtualMachines = RenderVirtualMachines(g)

	return err
}

// RenderVpcs : renders a services vpcs
func RenderVpcs(g *graph.Graph) []map[string]string {
	var vpcs []map[string]string

	for _, n := range g.GetComponents().ByType("vpc") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["vpc_aws_id"].(string)
		subnet, _ := (*gc)["vpc_subnet"].(string)
		vpcs = append(vpcs, map[string]string{
			"name":       name,
			"vpc_id":     id,
			"vpc_subnet": subnet,
		})
	}

	return vpcs
}

// RenderNetworks : renders a services networks
func RenderNetworks(g *graph.Graph) []map[string]string {
	var networks []map[string]string

	for _, n := range g.GetComponents().ByType("network") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["network_aws_id"].(string)
		az, _ := (*gc)["availability_zone"].(string)
		networks = append(networks, map[string]string{
			"name":              name,
			"network_aws_id":    id,
			"availability_zone": az,
		})
	}

	return networks
}

// RenderSecurityGroups : renders a services security groups
func RenderSecurityGroups(g *graph.Graph) []map[string]string {
	var sgs []map[string]string

	for _, n := range g.GetComponents().ByType("firewall") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["security_group_aws_id"].(string)
		sgs = append(sgs, map[string]string{
			"name":                  name,
			"security_group_aws_id": id,
		})
	}

	return sgs
}

// RenderNats : renders a services nat gateways
func RenderNats(g *graph.Graph) []map[string]string {
	var nats []map[string]string

	for _, n := range g.GetComponents().ByType("nat") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["nat_gateway_aws_id"].(string)
		pubIP, _ := (*gc)["nat_gateway_allocation_ip"].(string)
		nats = append(nats, map[string]string{
			"name":               name,
			"nat_gateway_aws_id": id,
			"public_ip":          pubIP,
		})
	}

	return nats
}

// RenderELBs : renders a services elbs
func RenderELBs(g *graph.Graph) []map[string]string {
	var elbs []map[string]string

	for _, n := range g.GetComponents().ByType("elb") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		dns, _ := (*gc)["dns_name"].(string)
		elbs = append(elbs, map[string]string{
			"name":     name,
			"dns_name": dns,
		})
	}

	return elbs
}

// RenderInstances : renders a services instances
func RenderInstances(g *graph.Graph) []map[string]string {
	var instances []map[string]string

	for _, n := range g.GetComponents().ByType("instance") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["instance_aws_id"].(string)
		pip, _ := (*gc)["public_ip"].(string)
		ip, _ := (*gc)["ip"].(string)
		instances = append(instances, map[string]string{
			"name":            name,
			"instance_aws_id": id,
			"public_ip":       pip,
			"ip":              ip,
		})
	}

	return instances
}

// RenderRDSClusters : renders a services rds clusters
func RenderRDSClusters(g *graph.Graph) []map[string]string {
	var rdss []map[string]string

	for _, n := range g.GetComponents().ByType("rds_cluster") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		endpoint, _ := (*gc)["endpoint"].(string)
		rdss = append(rdss, map[string]string{
			"name":     name,
			"endpoint": endpoint,
		})
	}

	return rdss
}

// RenderRDSInstances : renders a services rds instances
func RenderRDSInstances(g *graph.Graph) []map[string]string {
	var rdss []map[string]string

	for _, n := range g.GetComponents().ByType("rds_instance") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		endpoint, _ := (*gc)["endpoint"].(string)
		rdss = append(rdss, map[string]string{
			"name":     name,
			"endpoint": endpoint,
		})
	}

	return rdss
}

// RenderEBSVolumes : renders a services ebs volumes
func RenderEBSVolumes(g *graph.Graph) []map[string]string {
	var rdss []map[string]string

	for _, n := range g.GetComponents().ByType("ebs_volume") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["volume_aws_id"].(string)
		rdss = append(rdss, map[string]string{
			"name":          name,
			"volume_aws_id": id,
		})
	}

	return rdss
}

// RenderLoadBalancers : renders load balancers
func RenderLoadBalancers(g *graph.Graph) []map[string]string {
	var lbs []map[string]string
	ips := listIPAddresses(g)

	for _, n := range g.GetComponents().ByType("lb") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["id"].(string)
		configs, _ := (*gc)["frontend_ip_configurations"].([]interface{})
		cfg, _ := configs[0].(map[string]interface{})
		ipID, _ := cfg["public_ip_address_id"].(string)
		ip, _ := ips[ipID]

		lbs = append(lbs, map[string]string{
			"name":      name,
			"id":        id,
			"public_ip": ip,
		})
	}

	return lbs
}

func listIPAddresses(g *graph.Graph) map[string]string {
	existingIPs := make(map[string]string, 0)

	for _, ip := range g.GetComponents().ByType("public_ip") {
		gc := ip.(*graph.GenericComponent)
		id, _ := (*gc)["id"].(string)
		ipAddress, _ := (*gc)["ip_address"].(string)
		existingIPs[id] = ipAddress
	}

	return existingIPs
}

// RenderVirtualMachines : renders virtual machines
func RenderVirtualMachines(g *graph.Graph) []map[string]string {
	var resources []map[string]string
	mappedIPs := make(map[string]interface{}, 0)
	existingIPs := listIPAddresses(g)

	for _, ni := range g.GetComponents().ByType("network_interface") {
		var public []string
		var private []string

		gc := ni.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		ips := make(map[string][]string)

		configs, _ := (*gc)["ip_configuration"].([]interface{})
		for _, cfg := range configs {
			c, _ := cfg.(map[string]interface{})
			pubID, _ := c["public_ip_address_id"].(string)
			pri, _ := c["private_ip_address"].(string)
			if pub, ok := existingIPs[pubID]; ok {
				public = append(public, pub)
			}
			private = append(private, pri)
		}

		ips["public"] = public
		ips["private"] = private
		mappedIPs[name] = make(map[string][]string, 0)
		mappedIPs[name] = ips
	}

	for _, n := range g.GetComponents().ByType("virtual_machine") {
		gc := n.(*graph.GenericComponent)
		name, _ := (*gc)["name"].(string)
		id, _ := (*gc)["id"].(string)
		networks, _ := (*gc)["network_interfaces"].([]interface{})
		publicIPs := make([]string, 0)
		privateIPs := make([]string, 0)
		for _, ni := range networks {
			netName := ni.(string)
			if val, ok := mappedIPs[netName]; ok {
				ips, _ := val.(map[string][]string)
				publicIPs = append(publicIPs, ips["public"]...)
				privateIPs = append(privateIPs, ips["private"]...)
			}
		}

		resources = append(resources, map[string]string{
			"name":       name,
			"id":         id,
			"public_ip":  strings.Join(publicIPs, ", "),
			"private_ip": strings.Join(privateIPs, ", "),
		})
	}

	return resources
}

// RenderSQLDatabases : renders sql databases
func RenderSQLDatabases(g *graph.Graph) []map[string]string {
	return renderResources(g, "sql_database", func(gc *graph.GenericComponent) map[string]string {
		name, _ := (*gc)["name"].(string)
		server, _ := (*gc)["server_name"].(string)
		id, _ := (*gc)["id"].(string)

		return map[string]string{
			"name":        name,
			"server_name": server + ".database.windows.net",
			"id":          id,
		}
	})
}

type convert func(*graph.GenericComponent) map[string]string

func renderResources(g *graph.Graph, resourceType string, f convert) (resources []map[string]string) {
	for _, n := range g.GetComponents().ByType(resourceType) {
		gc := n.(*graph.GenericComponent)
		resources = append(resources, f(gc))
	}

	return
}

// RenderCollection : Maps a collection of Service on a collection of ServiceRender
func (o *ServiceRender) RenderCollection(services []models.Service) (list []ServiceRender, err error) {
	for _, s := range services {
		var output ServiceRender
		if err := output.Render(s); err == nil {
			list = append(list, output)
		}
	}

	return list, nil
}

// ToJSON : Converts a ServiceRender to json string
func (o *ServiceRender) ToJSON() ([]byte, error) {
	return json.Marshal(o)
}
<|endoftext|>


<commit_before>package vsphere

import (
	"bytes"
	"log"
	"strings"

	"github.com/cblomart/vsphere-graphite/utils"

	"github.com/vmware/govmomi/vim25/types"
)

// Cache will hold some informations in memory
type Cache map[string]interface{}

/*
var (
		//create a map to resolve object names
		morToName map[string]*string

		//create a map to resolve vm to datastore
		vmToDatastore map[string]*[]types.ManagedObjectReference

		//create a map to resolve vm to network
		vmToNetwork map[string]*[]types.ManagedObjectReference

		//create a map to resolve vm to host
		vmToHost map[string]*string

		//create a map to resolve host to parent - in a cluster the parent should be a cluster
		morToParent map[string]*string

		//create a map to resolve mor to vms - object that contains multiple vms as child objects
		morToVms map[string]*[]types.ManagedObjectReference

		//create a map to resolve mor to tags
		morToTags map[string]*[]types.Tag

		//create a map to resolve mor to numcpu
		morToNumCPU map[string]*int32

		//create a map to resolve mor to memorysizemb
		morToMemorySizeMB map[string]*int32

		//create a map to resolve diskinfos
		morToDiskInfos map[string]*[]types.GuestDiskInfo

		//resolve metrics to name
		metricToName map[int32]*string

		//crate a ma to resolve folders
		morToFolderPath map[string]*[]string
	//create a map to resolve object names
	morToName Cache

	//create a map to resolve vm to datastore
	vmToDatastore Cache

	//create a map to resolve vm to network
	vmToNetwork Cache

	//create a map to resolve vm to host
	vmToHost Cache

	//create a map to resolve host to parent - in a cluster the parent should be a cluster
	morToParent Cache

	//create a map to resolve mor to vms - object that contains multiple vms as child objects
	morToVms Cache

	//create a map to resolve mor to tags
	morToTags Cache

	//create a map to resolve mor to numcpu
	morToNumCPU Cache

	//create a map to resolve mor to memorysizemb
	morToMemorySizeMB Cache

	//create a map to resolve diskinfos
	morToDiskInfos Cache

	//resolve metrics to name
	metricToName Cache

	//create a map to resolve folders
	morToFolderPath Cache
)
*/

// Gets an index
func index(vcenter, section, i string) string {
	var buffer bytes.Buffer
	buffer.WriteString(vcenter)
	buffer.WriteString("|")
	buffer.WriteString(section)
	buffer.WriteString("|")
	buffer.WriteString(i)
	return buffer.String()
}

// Add a value to the cache
func (c *Cache) Add(vcenter, section, i string, v interface{}) {
	if len(vcenter) == 0 || len(section) == 0 || len(i) == 0 || v == nil {
		return
	}
	switch typed := v.(type) {
	case *string:
		c.add(vcenter, section, i, v)
	case *[]string:
		if len(*typed) > 0 {
			c.add(vcenter, section, i, v)
		}
	case *int32:
		c.add(vcenter, section, i, v)
	case *types.ManagedObjectReference:
		c.add(vcenter, section, i, &((*typed).Value))
	case *types.ArrayOfManagedObjectReference:
		if len((*typed).ManagedObjectReference) > 0 {
			c.add(vcenter, section, i, &((*typed).ManagedObjectReference))
		}
	case *types.ArrayOfTag:
		if len((*typed).Tag) > 0 {
			c.add(vcenter, section, i, &((*typed).Tag))
		}
	case *types.ArrayOfGuestDiskInfo:
		if len((*typed).GuestDiskInfo) > 0 {
			c.add(vcenter, section, i, &((*typed).GuestDiskInfo))
		}
	default:
		log.Printf("Adding a unhandled type %T to cache for %s section %s and ref %s\n", v, vcenter, section, i)
	}
}

// add to the cache without type check
func (c *Cache) add(vcenter, section, i string, v interface{}) {
	if v != nil {
		(*c)[index(vcenter, section, i)] = v
	}
}

// get a value from the cache
func (c *Cache) get(vcenter, section, i string) interface{} {
	if v, ok := (*c)[index(vcenter, section, i)]; ok {
		return v
	}
	return nil
}

// GetString gets a string from cache
func (c *Cache) GetString(vcenter, section, i string) *string {
	if v, ok := c.get(vcenter, section, i).(*string); ok {
		return v
	}
	return nil
}

// GetStrings gets an array of strings from cache
func (c *Cache) GetStrings(vcenter, section, i string) *[]string {
	if v, ok := c.get(vcenter, section, i).(*[]string); ok {
		return v
	}
	return nil
}

// GetInt32 get an int32 from cache
func (c *Cache) GetInt32(vcenter, section, i string) *int32 {
	if v, ok := c.get(vcenter, section, i).(*int32); ok {
		return v
	}
	return nil
}

// GetMoref gets a managed object reference from cache
func (c *Cache) GetMoref(vcenter, section, i string) *types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetMorefs gets an array of managed references from cache
func (c *Cache) GetMorefs(vcenter, section, i string) *[]types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*[]types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetTags gets an array of vsphere tags from cache
func (c *Cache) GetTags(vcenter, section, i string) *[]types.Tag {
	if v, ok := c.get(vcenter, section, i).(*[]types.Tag); ok {
		return v
	}
	return nil
}

// GetDiskInfos gets an array of diskinfos from cache
func (c *Cache) GetDiskInfos(vcenter, section, i string) *[]types.GuestDiskInfo {
	if v, ok := c.get(vcenter, section, i).(*[]types.GuestDiskInfo); ok {
		return v
	}
	return nil
}

// Clean cache of unknows references
func (c *Cache) Clean(vcenter string, section string, refs []string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter
		if m[0] != vcenter && m[1] != section {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// CleanAll cleans all sections of unknown references
// poolpaths and metrics are ignored as they will be cleaned real time
func (c *Cache) CleanAll(vcenter string, refs []string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] == "metrics" || m[1] == "poolpaths" {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// Purge purges a section of the cache
func (c *Cache) Purge(vcenter, section string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		delete(*c, e)
	}
}

// lookup items in the cache
func (c *Cache) lookup(vcenter, section string) *map[string]interface{} {
	result := make(map[string]interface{})
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		result[m[2]] = (*c)[e]
	}
	return &result
}

// LookupString looks for items in the cache of type string
func (c *Cache) LookupString(vcenter, section string) *map[string]*string {
	result := make(map[string]*string)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*string); ok {
			result[key] = typed
		}
	}
	return &result
}

// LookupMorefs looks for items in the cache of type Morefs
func (c *Cache) LookupMorefs(vcenter, section string) *map[string]*[]types.ManagedObjectReference {
	result := make(map[string]*[]types.ManagedObjectReference)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*[]types.ManagedObjectReference); ok {
			result[key] = typed
		}
	}
	return &result
}

// FindHostAndCluster finds host and cluster of a host or a vm
func (c *Cache) FindHostAndCluster(vcenter, moref string) (string, string) {
	// get host
	if strings.HasPrefix(moref, "vm-") {
		// find host of the vm
		host := c.GetString(vcenter, "hosts", moref)
		if host == nil {
			log.Printf("VM %s has no host.\n", moref)
			return "", ""
		}
		moref = *host
	}
	// find hostname
	hostnameptr := cache.GetString(vcenter, "names", moref)
	hostname := ""
	if hostnameptr != nil {
		hostname = *hostnameptr
	}
	// find cluster
	cluster := cache.GetString(vcenter, "parents", moref)
	if cluster == nil {
		log.Printf("Host %s has no parents.\n", moref)
		return hostname, ""
	}
	if strings.HasPrefix(*cluster, "domain-s") {
		//ignore standalone hosts
		return hostname, ""
	}
	if !strings.HasPrefix(*cluster, "domain-c") {
		log.Printf("Host %s has no suitable parent %s.", moref, *cluster)
		return hostname, ""
	}
	clusternameptr := cache.GetString(vcenter, "names", *cluster)
	if clusternameptr == nil {
		return hostname, ""
	}
	return hostname, *clusternameptr
}

// FindString finds and return a string
func (c *Cache) FindString(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return *ptr
}

// FindName finds an object in cache and resolves its name
func (c *Cache) FindName(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return cache.FindString(vcenter, "names", *ptr)
}

// FindNames finds objects in cache and resolves their names
func (c *Cache) FindNames(vcenter, section, moref string) []string {
	ptr := cache.GetMorefs(vcenter, section, moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	names := make([]string, len(*ptr))
	for _, mor := range *ptr {
		nptr := cache.GetString(vcenter, "names", mor.Value)
		if nptr == nil {
			continue
		}
		if len(*nptr) == 0 {
			continue
		}
		names = append(names, *nptr)
	}
	return names
}

// FindTags finds objects in cachee and create a tag array
func (c *Cache) FindTags(vcenter, moref string) []string {
	ptr := cache.GetTags(vcenter, "tags", moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	tags := make([]string, len(*ptr))
	for _, tag := range *ptr {
		tags = append(tags, tag.Key)
	}
	return tags
}

// FindMetricName find metricname from cache
func (c *Cache) FindMetricName(vcenter string, id int32) string {
	ptr := cache.GetString(vcenter, "metrics", utils.ValToString(id, "", true))
	if ptr == nil {
		return ""
	}
	return *ptr
}
<commit_msg>Error handling in buffer<commit_after>package vsphere

import (
	"bytes"
	"log"
	"strings"

	"github.com/cblomart/vsphere-graphite/utils"

	"github.com/vmware/govmomi/vim25/types"
)

// Cache will hold some informations in memory
type Cache map[string]interface{}

/*
var (
		//create a map to resolve object names
		morToName map[string]*string

		//create a map to resolve vm to datastore
		vmToDatastore map[string]*[]types.ManagedObjectReference

		//create a map to resolve vm to network
		vmToNetwork map[string]*[]types.ManagedObjectReference

		//create a map to resolve vm to host
		vmToHost map[string]*string

		//create a map to resolve host to parent - in a cluster the parent should be a cluster
		morToParent map[string]*string

		//create a map to resolve mor to vms - object that contains multiple vms as child objects
		morToVms map[string]*[]types.ManagedObjectReference

		//create a map to resolve mor to tags
		morToTags map[string]*[]types.Tag

		//create a map to resolve mor to numcpu
		morToNumCPU map[string]*int32

		//create a map to resolve mor to memorysizemb
		morToMemorySizeMB map[string]*int32

		//create a map to resolve diskinfos
		morToDiskInfos map[string]*[]types.GuestDiskInfo

		//resolve metrics to name
		metricToName map[int32]*string

		//crate a ma to resolve folders
		morToFolderPath map[string]*[]string
	//create a map to resolve object names
	morToName Cache

	//create a map to resolve vm to datastore
	vmToDatastore Cache

	//create a map to resolve vm to network
	vmToNetwork Cache

	//create a map to resolve vm to host
	vmToHost Cache

	//create a map to resolve host to parent - in a cluster the parent should be a cluster
	morToParent Cache

	//create a map to resolve mor to vms - object that contains multiple vms as child objects
	morToVms Cache

	//create a map to resolve mor to tags
	morToTags Cache

	//create a map to resolve mor to numcpu
	morToNumCPU Cache

	//create a map to resolve mor to memorysizemb
	morToMemorySizeMB Cache

	//create a map to resolve diskinfos
	morToDiskInfos Cache

	//resolve metrics to name
	metricToName Cache

	//create a map to resolve folders
	morToFolderPath Cache
)
*/

// Gets an index
func index(vcenter, section, i string) string {
	var buffer bytes.Buffer
	err := buffer.WriteString(vcenter)
	err = buffer.WriteString("|")
	err = buffer.WriteString(section)
	err = buffer.WriteString("|")
	err = buffer.WriteString(i)
	if err {
		return ""
	}
	return buffer.String()
}

// Add a value to the cache
func (c *Cache) Add(vcenter, section, i string, v interface{}) {
	if len(vcenter) == 0 || len(section) == 0 || len(i) == 0 || v == nil {
		return
	}
	switch typed := v.(type) {
	case *string:
		c.add(vcenter, section, i, v)
	case *[]string:
		if len(*typed) > 0 {
			c.add(vcenter, section, i, v)
		}
	case *int32:
		c.add(vcenter, section, i, v)
	case *types.ManagedObjectReference:
		c.add(vcenter, section, i, &((*typed).Value))
	case *types.ArrayOfManagedObjectReference:
		if len((*typed).ManagedObjectReference) > 0 {
			c.add(vcenter, section, i, &((*typed).ManagedObjectReference))
		}
	case *types.ArrayOfTag:
		if len((*typed).Tag) > 0 {
			c.add(vcenter, section, i, &((*typed).Tag))
		}
	case *types.ArrayOfGuestDiskInfo:
		if len((*typed).GuestDiskInfo) > 0 {
			c.add(vcenter, section, i, &((*typed).GuestDiskInfo))
		}
	default:
		log.Printf("Adding a unhandled type %T to cache for %s section %s and ref %s\n", v, vcenter, section, i)
	}
}

// add to the cache without type check
func (c *Cache) add(vcenter, section, i string, v interface{}) {
	if v != nil {
		(*c)[index(vcenter, section, i)] = v
	}
}

// get a value from the cache
func (c *Cache) get(vcenter, section, i string) interface{} {
	if v, ok := (*c)[index(vcenter, section, i)]; ok {
		return v
	}
	return nil
}

// GetString gets a string from cache
func (c *Cache) GetString(vcenter, section, i string) *string {
	if v, ok := c.get(vcenter, section, i).(*string); ok {
		return v
	}
	return nil
}

// GetStrings gets an array of strings from cache
func (c *Cache) GetStrings(vcenter, section, i string) *[]string {
	if v, ok := c.get(vcenter, section, i).(*[]string); ok {
		return v
	}
	return nil
}

// GetInt32 get an int32 from cache
func (c *Cache) GetInt32(vcenter, section, i string) *int32 {
	if v, ok := c.get(vcenter, section, i).(*int32); ok {
		return v
	}
	return nil
}

// GetMoref gets a managed object reference from cache
func (c *Cache) GetMoref(vcenter, section, i string) *types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetMorefs gets an array of managed references from cache
func (c *Cache) GetMorefs(vcenter, section, i string) *[]types.ManagedObjectReference {
	if v, ok := c.get(vcenter, section, i).(*[]types.ManagedObjectReference); ok {
		return v
	}
	return nil
}

// GetTags gets an array of vsphere tags from cache
func (c *Cache) GetTags(vcenter, section, i string) *[]types.Tag {
	if v, ok := c.get(vcenter, section, i).(*[]types.Tag); ok {
		return v
	}
	return nil
}

// GetDiskInfos gets an array of diskinfos from cache
func (c *Cache) GetDiskInfos(vcenter, section, i string) *[]types.GuestDiskInfo {
	if v, ok := c.get(vcenter, section, i).(*[]types.GuestDiskInfo); ok {
		return v
	}
	return nil
}

// Clean cache of unknows references
func (c *Cache) Clean(vcenter string, section string, refs []string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter
		if m[0] != vcenter && m[1] != section {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// CleanAll cleans all sections of unknown references
// poolpaths and metrics are ignored as they will be cleaned real time
func (c *Cache) CleanAll(vcenter string, refs []string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts or delete
		if len(m) != 3 {
			delete(*c, e)
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] == "metrics" || m[1] == "poolpaths" {
			continue
		}
		// find the value in the range
		found := false
		for _, ref := range refs {
			if m[2] == ref {
				found = true
				break
			}
		}
		// remove if not found
		if !found {
			log.Printf("removing %s from cache\n", e)
			delete(*c, e)
		}
	}
}

// Purge purges a section of the cache
func (c *Cache) Purge(vcenter, section string) {
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		delete(*c, e)
	}
}

// lookup items in the cache
func (c *Cache) lookup(vcenter, section string) *map[string]interface{} {
	result := make(map[string]interface{})
	for e := range *c {
		// get back index parts
		m := strings.Split(e, "|")
		// check that we have tree parts
		if len(m) != 3 {
			continue
		}
		// check vcenter and ignored sections
		if m[0] != vcenter || m[1] != section {
			continue
		}
		result[m[2]] = (*c)[e]
	}
	return &result
}

// LookupString looks for items in the cache of type string
func (c *Cache) LookupString(vcenter, section string) *map[string]*string {
	result := make(map[string]*string)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*string); ok {
			result[key] = typed
		}
	}
	return &result
}

// LookupMorefs looks for items in the cache of type Morefs
func (c *Cache) LookupMorefs(vcenter, section string) *map[string]*[]types.ManagedObjectReference {
	result := make(map[string]*[]types.ManagedObjectReference)
	for key, val := range *c.lookup(vcenter, section) {
		if typed, ok := val.(*[]types.ManagedObjectReference); ok {
			result[key] = typed
		}
	}
	return &result
}

// FindHostAndCluster finds host and cluster of a host or a vm
func (c *Cache) FindHostAndCluster(vcenter, moref string) (string, string) {
	// get host
	if strings.HasPrefix(moref, "vm-") {
		// find host of the vm
		host := c.GetString(vcenter, "hosts", moref)
		if host == nil {
			log.Printf("VM %s has no host.\n", moref)
			return "", ""
		}
		moref = *host
	}
	// find hostname
	hostnameptr := cache.GetString(vcenter, "names", moref)
	hostname := ""
	if hostnameptr != nil {
		hostname = *hostnameptr
	}
	// find cluster
	cluster := cache.GetString(vcenter, "parents", moref)
	if cluster == nil {
		log.Printf("Host %s has no parents.\n", moref)
		return hostname, ""
	}
	if strings.HasPrefix(*cluster, "domain-s") {
		//ignore standalone hosts
		return hostname, ""
	}
	if !strings.HasPrefix(*cluster, "domain-c") {
		log.Printf("Host %s has no suitable parent %s.", moref, *cluster)
		return hostname, ""
	}
	clusternameptr := cache.GetString(vcenter, "names", *cluster)
	if clusternameptr == nil {
		return hostname, ""
	}
	return hostname, *clusternameptr
}

// FindString finds and return a string
func (c *Cache) FindString(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return *ptr
}

// FindName finds an object in cache and resolves its name
func (c *Cache) FindName(vcenter, section, moref string) string {
	ptr := cache.GetString(vcenter, section, moref)
	if ptr == nil {
		return ""
	}
	return cache.FindString(vcenter, "names", *ptr)
}

// FindNames finds objects in cache and resolves their names
func (c *Cache) FindNames(vcenter, section, moref string) []string {
	ptr := cache.GetMorefs(vcenter, section, moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	names := make([]string, len(*ptr))
	for _, mor := range *ptr {
		nptr := cache.GetString(vcenter, "names", mor.Value)
		if nptr == nil {
			continue
		}
		if len(*nptr) == 0 {
			continue
		}
		names = append(names, *nptr)
	}
	return names
}

// FindTags finds objects in cachee and create a tag array
func (c *Cache) FindTags(vcenter, moref string) []string {
	ptr := cache.GetTags(vcenter, "tags", moref)
	if ptr == nil {
		return []string{}
	}
	if len(*ptr) == 0 {
		return []string{}
	}
	tags := make([]string, len(*ptr))
	for _, tag := range *ptr {
		tags = append(tags, tag.Key)
	}
	return tags
}

// FindMetricName find metricname from cache
func (c *Cache) FindMetricName(vcenter string, id int32) string {
	ptr := cache.GetString(vcenter, "metrics", utils.ValToString(id, "", true))
	if ptr == nil {
		return ""
	}
	return *ptr
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package wallet

import (
	"github.com/roasbeef/btcd/wire"
	"github.com/roasbeef/btcutil"
	"github.com/roasbeef/btcwallet/chain"
	"github.com/roasbeef/btcwallet/waddrmgr"
	"github.com/roasbeef/btcwallet/wtxmgr"
)

// RescanProgressMsg reports the current progress made by a rescan for a
// set of wallet addresses.
type RescanProgressMsg struct {
	Addresses    []btcutil.Address
	Notification *chain.RescanProgress
}

// RescanFinishedMsg reports the addresses that were rescanned when a
// rescanfinished message was received rescanning a batch of addresses.
type RescanFinishedMsg struct {
	Addresses    []btcutil.Address
	Notification *chain.RescanFinished
}

// RescanJob is a job to be processed by the RescanManager.  The job includes
// a set of wallet addresses, a starting height to begin the rescan, and
// outpoints spendable by the addresses thought to be unspent.  After the
// rescan completes, the error result of the rescan RPC is sent on the Err
// channel.
type RescanJob struct {
	InitialSync bool
	Addrs       []btcutil.Address
	OutPoints   []*wire.OutPoint
	BlockStamp  waddrmgr.BlockStamp
	err         chan error
}

// rescanBatch is a collection of one or more RescanJobs that were merged
// together before a rescan is performed.
type rescanBatch struct {
	initialSync bool
	addrs       []btcutil.Address
	outpoints   []*wire.OutPoint
	bs          waddrmgr.BlockStamp
	errChans    []chan error
}

// SubmitRescan submits a RescanJob to the RescanManager.  A channel is
// returned with the final error of the rescan.  The channel is buffered
// and does not need to be read to prevent a deadlock.
func (w *Wallet) SubmitRescan(job *RescanJob) <-chan error {
	errChan := make(chan error, 1)
	job.err = errChan
	w.rescanAddJob <- job
	return errChan
}

// batch creates the rescanBatch for a single rescan job.
func (job *RescanJob) batch() *rescanBatch {
	return &rescanBatch{
		initialSync: job.InitialSync,
		addrs:       job.Addrs,
		outpoints:   job.OutPoints,
		bs:          job.BlockStamp,
		errChans:    []chan error{job.err},
	}
}

// merge merges the work from k into j, setting the starting height to
// the minimum of the two jobs.  This method does not check for
// duplicate addresses or outpoints.
func (b *rescanBatch) merge(job *RescanJob) {
	if job.InitialSync {
		b.initialSync = true
	}
	b.addrs = append(b.addrs, job.Addrs...)
	b.outpoints = append(b.outpoints, job.OutPoints...)
	if job.BlockStamp.Height < b.bs.Height {
		b.bs = job.BlockStamp
	}
	b.errChans = append(b.errChans, job.err)
}

// done iterates through all error channels, duplicating sending the error
// to inform callers that the rescan finished (or could not complete due
// to an error).
func (b *rescanBatch) done(err error) {
	for _, c := range b.errChans {
		c <- err
	}
}

// rescanBatchHandler handles incoming rescan request, serializing rescan
// submissions, and possibly batching many waiting requests together so they
// can be handled by a single rescan after the current one completes.
func (w *Wallet) rescanBatchHandler() {
	var curBatch, nextBatch *rescanBatch
	quit := w.quitChan()

out:
	for {
		select {
		case job := <-w.rescanAddJob:
			if curBatch == nil {
				// Set current batch as this job and send
				// request.
				curBatch = job.batch()
				w.rescanBatch <- curBatch
			} else {
				// Create next batch if it doesn't exist, or
				// merge the job.
				if nextBatch == nil {
					nextBatch = job.batch()
				} else {
					nextBatch.merge(job)
				}
			}

		case n := <-w.rescanNotifications:
			switch n := n.(type) {
			case *chain.RescanProgress:
				w.rescanProgress <- &RescanProgressMsg{
					Addresses:    curBatch.addrs,
					Notification: n,
				}

			case *chain.RescanFinished:
				if curBatch == nil {
					log.Warnf("Received rescan finished " +
						"notification but no rescan " +
						"currently running")
					continue
				}
				w.rescanFinished <- &RescanFinishedMsg{
					Addresses:    curBatch.addrs,
					Notification: n,
				}

				curBatch, nextBatch = nextBatch, nil

				if curBatch != nil {
					w.rescanBatch <- curBatch
				}

			default:
				// Unexpected message
				panic(n)
			}

		case <-quit:
			break out
		}
	}

	w.wg.Done()
}

// rescanProgressHandler handles notifications for partially and fully completed
// rescans by marking each rescanned address as partially or fully synced.
func (w *Wallet) rescanProgressHandler() {
	quit := w.quitChan()
out:
	for {
		// These can't be processed out of order since both chans are
		// unbuffured and are sent from same context (the batch
		// handler).
		select {
		case msg := <-w.rescanProgress:
			n := msg.Notification
			log.Infof("Rescanned through block %v (height %d)",
				n.Hash, n.Height)

		case msg := <-w.rescanFinished:
			n := msg.Notification
			addrs := msg.Addresses
			noun := pickNoun(len(addrs), "address", "addresses")
			log.Infof("Finished rescan for %d %s (synced to block "+
				"%s, height %d)", len(addrs), noun, n.Hash,
				n.Height)

			go w.resendUnminedTxs()

		case <-quit:
			break out
		}
	}
	w.wg.Done()
}

// rescanRPCHandler reads batch jobs sent by rescanBatchHandler and sends the
// RPC requests to perform a rescan.  New jobs are not read until a rescan
// finishes.
func (w *Wallet) rescanRPCHandler() {
	chainClient, err := w.requireChainClient()
	if err != nil {
		log.Errorf("rescanRPCHandler called without an RPC client")
		w.wg.Done()
		return
	}

	quit := w.quitChan()

out:
	for {
		select {
		case batch := <-w.rescanBatch:
			// Log the newly-started rescan.
			numAddrs := len(batch.addrs)
			noun := pickNoun(numAddrs, "address", "addresses")
			log.Infof("Started rescan from block %v (height %d) for %d %s",
				batch.bs.Hash, batch.bs.Height, numAddrs, noun)

			err := chainClient.Rescan(&batch.bs.Hash, batch.addrs,
				batch.outpoints)
			if err != nil {
				log.Errorf("Rescan for %d %s failed: %v", numAddrs,
					noun, err)
			}
			batch.done(err)
		case <-quit:
			break out
		}
	}

	w.wg.Done()
}

// Rescan begins a rescan for all active addresses and unspent outputs of
// a wallet.  This is intended to be used to sync a wallet back up to the
// current best block in the main chain, and is considered an initial sync
// rescan.
func (w *Wallet) Rescan(addrs []btcutil.Address, unspent []wtxmgr.Credit) error {
	outpoints := make([]*wire.OutPoint, len(unspent))
	for i, output := range unspent {
		outpoints[i] = &output.OutPoint
	}

	job := &RescanJob{
		InitialSync: true,
		Addrs:       addrs,
		OutPoints:   outpoints,
		BlockStamp:  w.Manager.SyncedTo(),
	}

	// Submit merged job and block until rescan completes.
	return <-w.SubmitRescan(job)
}
<commit_msg>wallet: fix nil dereference in rescanprogress ntfn handler<commit_after>// Copyright (c) 2013-2017 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package wallet

import (
	"github.com/roasbeef/btcd/wire"
	"github.com/roasbeef/btcutil"
	"github.com/roasbeef/btcwallet/chain"
	"github.com/roasbeef/btcwallet/waddrmgr"
	"github.com/roasbeef/btcwallet/wtxmgr"
)

// RescanProgressMsg reports the current progress made by a rescan for a
// set of wallet addresses.
type RescanProgressMsg struct {
	Addresses    []btcutil.Address
	Notification *chain.RescanProgress
}

// RescanFinishedMsg reports the addresses that were rescanned when a
// rescanfinished message was received rescanning a batch of addresses.
type RescanFinishedMsg struct {
	Addresses    []btcutil.Address
	Notification *chain.RescanFinished
}

// RescanJob is a job to be processed by the RescanManager.  The job includes
// a set of wallet addresses, a starting height to begin the rescan, and
// outpoints spendable by the addresses thought to be unspent.  After the
// rescan completes, the error result of the rescan RPC is sent on the Err
// channel.
type RescanJob struct {
	InitialSync bool
	Addrs       []btcutil.Address
	OutPoints   []*wire.OutPoint
	BlockStamp  waddrmgr.BlockStamp
	err         chan error
}

// rescanBatch is a collection of one or more RescanJobs that were merged
// together before a rescan is performed.
type rescanBatch struct {
	initialSync bool
	addrs       []btcutil.Address
	outpoints   []*wire.OutPoint
	bs          waddrmgr.BlockStamp
	errChans    []chan error
}

// SubmitRescan submits a RescanJob to the RescanManager.  A channel is
// returned with the final error of the rescan.  The channel is buffered
// and does not need to be read to prevent a deadlock.
func (w *Wallet) SubmitRescan(job *RescanJob) <-chan error {
	errChan := make(chan error, 1)
	job.err = errChan
	w.rescanAddJob <- job
	return errChan
}

// batch creates the rescanBatch for a single rescan job.
func (job *RescanJob) batch() *rescanBatch {
	return &rescanBatch{
		initialSync: job.InitialSync,
		addrs:       job.Addrs,
		outpoints:   job.OutPoints,
		bs:          job.BlockStamp,
		errChans:    []chan error{job.err},
	}
}

// merge merges the work from k into j, setting the starting height to
// the minimum of the two jobs.  This method does not check for
// duplicate addresses or outpoints.
func (b *rescanBatch) merge(job *RescanJob) {
	if job.InitialSync {
		b.initialSync = true
	}
	b.addrs = append(b.addrs, job.Addrs...)
	b.outpoints = append(b.outpoints, job.OutPoints...)
	if job.BlockStamp.Height < b.bs.Height {
		b.bs = job.BlockStamp
	}
	b.errChans = append(b.errChans, job.err)
}

// done iterates through all error channels, duplicating sending the error
// to inform callers that the rescan finished (or could not complete due
// to an error).
func (b *rescanBatch) done(err error) {
	for _, c := range b.errChans {
		c <- err
	}
}

// rescanBatchHandler handles incoming rescan request, serializing rescan
// submissions, and possibly batching many waiting requests together so they
// can be handled by a single rescan after the current one completes.
func (w *Wallet) rescanBatchHandler() {
	var curBatch, nextBatch *rescanBatch
	quit := w.quitChan()

out:
	for {
		select {
		case job := <-w.rescanAddJob:
			if curBatch == nil {
				// Set current batch as this job and send
				// request.
				curBatch = job.batch()
				w.rescanBatch <- curBatch
			} else {
				// Create next batch if it doesn't exist, or
				// merge the job.
				if nextBatch == nil {
					nextBatch = job.batch()
				} else {
					nextBatch.merge(job)
				}
			}

		case n := <-w.rescanNotifications:
			switch n := n.(type) {
			case *chain.RescanProgress:
				if curBatch == nil {
					log.Warnf("Received rescan progress " +
						"notification but no rescan " +
						"currently running")
					continue
				}
				w.rescanProgress <- &RescanProgressMsg{
					Addresses:    curBatch.addrs,
					Notification: n,
				}

			case *chain.RescanFinished:
				if curBatch == nil {
					log.Warnf("Received rescan finished " +
						"notification but no rescan " +
						"currently running")
					continue
				}
				w.rescanFinished <- &RescanFinishedMsg{
					Addresses:    curBatch.addrs,
					Notification: n,
				}

				curBatch, nextBatch = nextBatch, nil

				if curBatch != nil {
					w.rescanBatch <- curBatch
				}

			default:
				// Unexpected message
				panic(n)
			}

		case <-quit:
			break out
		}
	}

	w.wg.Done()
}

// rescanProgressHandler handles notifications for partially and fully completed
// rescans by marking each rescanned address as partially or fully synced.
func (w *Wallet) rescanProgressHandler() {
	quit := w.quitChan()
out:
	for {
		// These can't be processed out of order since both chans are
		// unbuffured and are sent from same context (the batch
		// handler).
		select {
		case msg := <-w.rescanProgress:
			n := msg.Notification
			log.Infof("Rescanned through block %v (height %d)",
				n.Hash, n.Height)

		case msg := <-w.rescanFinished:
			n := msg.Notification
			addrs := msg.Addresses
			noun := pickNoun(len(addrs), "address", "addresses")
			log.Infof("Finished rescan for %d %s (synced to block "+
				"%s, height %d)", len(addrs), noun, n.Hash,
				n.Height)

			go w.resendUnminedTxs()

		case <-quit:
			break out
		}
	}
	w.wg.Done()
}

// rescanRPCHandler reads batch jobs sent by rescanBatchHandler and sends the
// RPC requests to perform a rescan.  New jobs are not read until a rescan
// finishes.
func (w *Wallet) rescanRPCHandler() {
	chainClient, err := w.requireChainClient()
	if err != nil {
		log.Errorf("rescanRPCHandler called without an RPC client")
		w.wg.Done()
		return
	}

	quit := w.quitChan()

out:
	for {
		select {
		case batch := <-w.rescanBatch:
			// Log the newly-started rescan.
			numAddrs := len(batch.addrs)
			noun := pickNoun(numAddrs, "address", "addresses")
			log.Infof("Started rescan from block %v (height %d) for %d %s",
				batch.bs.Hash, batch.bs.Height, numAddrs, noun)

			err := chainClient.Rescan(&batch.bs.Hash, batch.addrs,
				batch.outpoints)
			if err != nil {
				log.Errorf("Rescan for %d %s failed: %v", numAddrs,
					noun, err)
			}
			batch.done(err)
		case <-quit:
			break out
		}
	}

	w.wg.Done()
}

// Rescan begins a rescan for all active addresses and unspent outputs of
// a wallet.  This is intended to be used to sync a wallet back up to the
// current best block in the main chain, and is considered an initial sync
// rescan.
func (w *Wallet) Rescan(addrs []btcutil.Address, unspent []wtxmgr.Credit) error {
	outpoints := make([]*wire.OutPoint, len(unspent))
	for i, output := range unspent {
		outpoints[i] = &output.OutPoint
	}

	job := &RescanJob{
		InitialSync: true,
		Addrs:       addrs,
		OutPoints:   outpoints,
		BlockStamp:  w.Manager.SyncedTo(),
	}

	// Submit merged job and block until rescan completes.
	return <-w.SubmitRescan(job)
}
<|endoftext|>


<commit_before>package image

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
)

func (image *Image) listObjects() []hash.Hash {
	hashes := make([]hash.Hash, 0, image.FileSystem.NumRegularInodes+2)
	for _, inode := range image.FileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				hashes = append(hashes, inode.Hash)
			}
		}
	}
	if image.ReleaseNotes != nil && image.ReleaseNotes.Object != nil {
		hashes = append(hashes, *image.ReleaseNotes.Object)
	}
	if image.BuildLog != nil && image.BuildLog.Object != nil {
		hashes = append(hashes, *image.BuildLog.Object)
	}
	return hashes
}
<commit_msg>Change lib/image.Image.ListObjects() to use forEachObject() iterator.<commit_after>package image

import (
	"github.com/Symantec/Dominator/lib/hash"
)

func (image *Image) listObjects() []hash.Hash {
	hashes := make([]hash.Hash, 0, image.FileSystem.NumRegularInodes+2)
	image.forEachObject(func(hashVal hash.Hash) error {
		hashes = append(hashes, hashVal)
		return nil
	})
	return hashes
}
<|endoftext|>