<commit_before>package main

import (
	"fmt"
	"github.com/mitchellh/copystructure"
	"strings"
)

const version1 = "2015-07-25"
const version2 = "2015-12-19"
const version3 = "2016-07-29"

func GenerateAnswers(delta *MetadataDelta) (Versions, error) {
	// 1. generate interim data
	s := len(delta.Data)
	interim := &Interim{
		UUIDToService:                   make(map[string]map[string]interface{}, s),
		UUIDToContainer:                 make(map[string]map[string]interface{}, s),
		UUIDToStack:                     make(map[string]map[string]interface{}, s),
		UUIDToHost:                      make(map[string]map[string]interface{}, s),
		StackUUIDToServicesUUID:         make(map[string][]string, s),
		ServiceUUIDNameToContainersUUID: make(map[string][]string, s),
		ContainerUUIDToContainerLink:    make(map[string]map[string]interface{}, s),
		ServiceUUIDToServiceLink:        make(map[string]map[string]interface{}, s),
		Networks:                        []interface{}{},
		Default:                         make(map[string]interface{}, s),
	}

	for _, o := range delta.Data {
		processMetadataObject(o, interim)
	}
	// 2. Generate versions from temp data
	versions, err := generateVersions(interim)
	if err != nil {
		return nil, err
	}
	return versions, nil
}

func generateVersions(interim *Interim) (Versions, error) {
	versions := make(map[string]Answers)
	vs := []string{version1, version2, version3}
	for _, v := range vs {
		versionedData, err := applyVersionToData(*interim, v)
		if err != nil {
			return nil, err
		}
		addToVersions(versions, v, versionedData)
	}
	//tag the latest
	versions["latest"] = versions[version3]
	return versions, nil
}

func addToVersions(versions Versions, version string, versionedData *Interim) {
	answers := make(map[string]interface{})
	defaultAnswers := addDefaultToAnswers(answers, versionedData)
	addClientToAnswers(answers, defaultAnswers, versionedData)
	versions[version] = answers
}

func addClientToAnswers(answers Answers, defaultAnswers map[string]interface{}, versionedData *Interim) {
	for _, c := range versionedData.UUIDToContainer {
		if c["primary_ip"] == nil {
			continue
		}
		clientAnswers := make(map[string]interface{})
		self := make(map[string]interface{})
		self["container"] = c
		if c["stack_uuid"] != nil {
			self["stack"] = versionedData.UUIDToStack[c["stack_uuid"].(string)]
			self["service"] = versionedData.UUIDToService[getServiceUUID(c["service_uuid"].(string), c["service_name"].(string))]
		}
		if c["host_uuid"] != nil {
			self["host"] = versionedData.UUIDToHost[c["host_uuid"].(string)]
		}
		clientAnswers["self"] = self
		mergeDefaults(clientAnswers, defaultAnswers)
		answers[c["primary_ip"].(string)] = clientAnswers
	}
}

func mergeDefaults(clientAnswers map[string]interface{}, defaultAnswers map[string]interface{}) {
	for k, v := range defaultAnswers {
		_, exists := clientAnswers[k]
		if !exists {
			clientAnswers[k] = v
		}
	}
}

func addDefaultToAnswers(answers Answers, versionedData *Interim) map[string]interface{} {
	defaultAnswers := make(map[string]interface{})
	var containers []interface{}
	for _, c := range versionedData.UUIDToContainer {
		if _, ok := defaultAnswers["containers"]; ok {
			containers = defaultAnswers["containers"].([]interface{})
		}
		containers = append(containers, c)
	}
	defaultAnswers["containers"] = containers

	var stacks []interface{}
	for _, s := range versionedData.UUIDToStack {
		if _, ok := defaultAnswers["stacks"]; ok {
			stacks = defaultAnswers["stacks"].([]interface{})
		}
		stacks = append(stacks, s)
	}
	defaultAnswers["stacks"] = stacks

	var services []interface{}
	for _, s := range versionedData.UUIDToService {
		if _, ok := defaultAnswers["services"]; ok {
			services = defaultAnswers["services"].([]interface{})
		}
		services = append(services, s)
	}
	defaultAnswers["services"] = services

	var hosts []interface{}
	for _, h := range versionedData.UUIDToHost {
		if _, ok := defaultAnswers["hosts"]; ok {
			hosts = defaultAnswers["hosts"].([]interface{})
		}
		hosts = append(hosts, h)
	}
	defaultAnswers["hosts"] = hosts
	defaultAnswers["networks"] = versionedData.Networks

	if val, ok := versionedData.Default["version"]; ok {
		defaultAnswers["version"] = val
	}

	if selfVal, ok := versionedData.Default["self"]; ok {
		self := selfVal.(map[string]interface{})
		if hostVal, ok := self["host"]; ok {
			host := hostVal.(map[string]interface{})
			self["host"] = versionedData.UUIDToHost[host["uuid"].(string)]
		}
		defaultAnswers["self"] = self
	}

	answers[DEFAULT_KEY] = defaultAnswers
	return defaultAnswers
}

func applyVersionToData(orig Interim, version string) (*Interim, error) {
	copied, err := copystructure.Copy(orig)
	if err != nil {
		return nil, err
	}
	modified := copied.(Interim)
	// 1. Process containers
	for _, c := range modified.UUIDToContainer {
		switch version {
		case version3:
			if c["name"] != nil {
				c["name"] = strings.ToLower(c["name"].(string))
			}
			if c["service_name"] != nil {
				c["service_name"] = strings.ToLower(c["service_name"].(string))
			}
			if c["stack_name"] != nil {
				c["stack_name"] = strings.ToLower(c["stack_name"].(string))
			}
			if _, ok := c["ports"]; ok {
				// set port ip to 0.0.0.0 if not specified
				originalPorts := c["ports"].([]interface{})
				var newPorts []interface{}
				for _, p := range originalPorts {
					port := p.(string)
					splitted := strings.Split(port, ":")
					if len(splitted) == 3 {
						newPorts = append(newPorts, port)
					} else {
						port = fmt.Sprintf("0.0.0.0:%s", port)
						newPorts = append(newPorts, port)
					}
				}
				c["ports"] = newPorts
			}

		default:
			if _, ok := c["ports"]; ok {
				originalPorts := c["ports"].([]interface{})
				// set port ip to host's ip if not specified
				if len(originalPorts) > 0 {
					var newPorts []interface{}
					for _, p := range originalPorts {
						port := p.(string)
						splitted := strings.Split(port, ":")
						if len(splitted) == 3 && splitted[0] != "0.0.0.0" {
							newPorts = append(newPorts, port)
						} else {
							if len(splitted) == 3 {
								port = fmt.Sprintf("%s%s", c["host_ip"], strings.TrimPrefix(port, "0.0.0.0"))
							} else {
								port = fmt.Sprintf("%s:%s", c["host_ip"], port)
							}

							newPorts = append(newPorts, port)
						}
					}
					c["ports"] = newPorts
				}
			}
		}
		c["links"] = modified.ContainerUUIDToContainerLink[c["uuid"].(string)]
		//delete helper field (needed for the port)
		delete(c, "host_ip")
	}

	// 2. Process services
	for _, s := range modified.UUIDToService {
		sUUID := getServiceUUID(s["uuid"].(string), s["name"].(string))

		stackUUID := s["stack_uuid"].(string)
		// add itself to the stack list
		var svcUUIDs []string
		if _, ok := modified.StackUUIDToServicesUUID[stackUUID]; ok {
			svcUUIDs = modified.StackUUIDToServicesUUID[stackUUID]
		}
		svcUUIDs = append(svcUUIDs, getServiceUUID(s["uuid"].(string), s["name"].(string)))
		modified.StackUUIDToServicesUUID[stackUUID] = svcUUIDs
		var cs []interface{}
		var cNames []interface{}
		cUUIDs := modified.ServiceUUIDNameToContainersUUID[sUUID]
		if cUUIDs != nil {
			for _, cUUID := range cUUIDs {
				if c, ok := modified.UUIDToContainer[cUUID]; ok {
					cs = append(cs, c)
					cNames = append(cNames, c["name"].(string))
				}
			}
		}
		switch version {
		case version1:
			s["containers"] = cNames
		case version2:
			s["containers"] = cs
		case version3:
			s["containers"] = cs
			s["name"] = strings.ToLower(s["name"].(string))
			s["stack_name"] = strings.ToLower(s["stack_name"].(string))
		}
		// add service links
		s["links"] = modified.ServiceUUIDToServiceLink[s["uuid"].(string)]
		// populate stack/service info on container
		if cUUIDs != nil {
			for _, cUUID := range cUUIDs {
				if _, ok := modified.UUIDToContainer[cUUID]; ok {
					modified.UUIDToContainer[cUUID]["service_name"] = s["name"]
					modified.UUIDToContainer[cUUID]["service_uuid"] = s["uuid"]
					modified.UUIDToContainer[cUUID]["stack_name"] = s["stack_name"]
					modified.UUIDToContainer[cUUID]["stack_uuid"] = stackUUID
				}
			}
		}
	}

	// 3. Process stacks
	for _, s := range modified.UUIDToStack {
		var svcs []interface{}
		var svcsNames []interface{}
		svcsUUIDs := modified.StackUUIDToServicesUUID[s["uuid"].(string)]
		if svcsUUIDs != nil {
			for _, svcUUID := range svcsUUIDs {
				if svc, ok := modified.UUIDToService[svcUUID]; ok {
					svcs = append(svcs, svc)
					svcsNames = append(svcsNames, svc["name"].(string))
				}
			}
		}
		switch version {
		case version1:
			s["services"] = svcsNames
		case version2:
			s["services"] = svcs
		case version3:
			s["services"] = svcs
			s["name"] = strings.ToLower(s["name"].(string))
		}
	}

	// 4. process hosts
	for _, h := range modified.UUIDToHost {
		switch version {
		case version3:
			delete(h, "hostId")
		}
	}

	return &modified, nil
}

func processMetadataObject(o map[string]interface{}, interim *Interim) {
	if val, ok := o["metadata_kind"]; ok {
		switch val.(string) {
		case "container":
			addContainer(o, interim)
		case "stack":
			addStack(o, interim)
		case "network":
			addNetwork(o, interim)
		case "host":
			addHost(o, interim)
		case "defaultData":
			addDefault(o, interim)
		case "serviceContainerLink":
			addServiceContainerLink(o, interim)
		case "containerLink":
			addContainerLink(o, interim)
		case "serviceLink":
			addServiceLink(o, interim)
		case "service":
			addService(o, interim)
		}
	}
}

func addContainer(container map[string]interface{}, interim *Interim) {
	interim.UUIDToContainer[container["uuid"].(string)] = container
}

func addService(service map[string]interface{}, interim *Interim) {
	interim.UUIDToService[getServiceUUID(service["uuid"].(string), service["name"].(string))] = service
}

func addServiceContainerLink(link map[string]interface{}, interim *Interim) {
	UUID := getServiceUUID(link["service_uuid"].(string), link["service_name"].(string))
	var containerUUIDList []string
	if _, ok := interim.ServiceUUIDNameToContainersUUID[UUID]; ok {
		containerUUIDList = interim.ServiceUUIDNameToContainersUUID[UUID]
	}
	containerUUIDList = append(containerUUIDList, link["container_uuid"].(string))
	interim.ServiceUUIDNameToContainersUUID[UUID] = containerUUIDList
}

func addServiceLink(link map[string]interface{}, interim *Interim) {
	serviceUUID := link["service_uuid"].(string)
	links := make(map[string]interface{})
	if _, ok := interim.ServiceUUIDToServiceLink[serviceUUID]; ok {
		links = interim.ServiceUUIDToServiceLink[serviceUUID]
	}
	linkKey := link["key"].(string)
	links[linkKey] = link["value"].(string)

	interim.ServiceUUIDToServiceLink[serviceUUID] = links
}

func addContainerLink(link map[string]interface{}, interim *Interim) {
	containerUUID := link["container_uuid"].(string)
	links := make(map[string]interface{})
	if _, ok := interim.ContainerUUIDToContainerLink[containerUUID]; ok {
		links = interim.ContainerUUIDToContainerLink[containerUUID]
	}
	linkKey := link["key"].(string)
	links[linkKey] = link["value"].(string)

	interim.ContainerUUIDToContainerLink[containerUUID] = links
}

func getServiceUUID(UUID string, name string) string {
	return fmt.Sprintf("%s_%s", UUID, name)
}

func addStack(stack map[string]interface{}, interim *Interim) {
	interim.UUIDToStack[stack["uuid"].(string)] = stack
}

func addNetwork(network map[string]interface{}, interim *Interim) {
	interim.Networks = append(interim.Networks, network)
}

func addHost(host map[string]interface{}, interim *Interim) {
	interim.UUIDToHost[host["uuid"].(string)] = host
}

func addDefault(def map[string]interface{}, interim *Interim) {
	interim.Default = def
}
<commit_msg>address self/service issue with upppercase<commit_after>package main

import (
	"fmt"
	"github.com/mitchellh/copystructure"
	"strings"
)

const version1 = "2015-07-25"
const version2 = "2015-12-19"
const version3 = "2016-07-29"

func GenerateAnswers(delta *MetadataDelta) (Versions, error) {
	// 1. generate interim data
	s := len(delta.Data)
	interim := &Interim{
		UUIDToService:                   make(map[string]map[string]interface{}, s),
		UUIDToContainer:                 make(map[string]map[string]interface{}, s),
		UUIDToStack:                     make(map[string]map[string]interface{}, s),
		UUIDToHost:                      make(map[string]map[string]interface{}, s),
		StackUUIDToServicesUUID:         make(map[string][]string, s),
		ServiceUUIDNameToContainersUUID: make(map[string][]string, s),
		ContainerUUIDToContainerLink:    make(map[string]map[string]interface{}, s),
		ServiceUUIDToServiceLink:        make(map[string]map[string]interface{}, s),
		Networks:                        []interface{}{},
		Default:                         make(map[string]interface{}, s),
	}

	for _, o := range delta.Data {
		processMetadataObject(o, interim)
	}
	// 2. Generate versions from temp data
	versions, err := generateVersions(interim)
	if err != nil {
		return nil, err
	}
	return versions, nil
}

func generateVersions(interim *Interim) (Versions, error) {
	versions := make(map[string]Answers)
	vs := []string{version1, version2, version3}
	for _, v := range vs {
		versionedData, err := applyVersionToData(*interim, v)
		if err != nil {
			return nil, err
		}
		addToVersions(versions, v, versionedData)
	}
	//tag the latest
	versions["latest"] = versions[version3]
	return versions, nil
}

func addToVersions(versions Versions, version string, versionedData *Interim) {
	answers := make(map[string]interface{})
	defaultAnswers := addDefaultToAnswers(answers, versionedData)
	addClientToAnswers(answers, defaultAnswers, versionedData)
	versions[version] = answers
}

func addClientToAnswers(answers Answers, defaultAnswers map[string]interface{}, versionedData *Interim) {
	for _, c := range versionedData.UUIDToContainer {
		if c["primary_ip"] == nil {
			continue
		}
		clientAnswers := make(map[string]interface{})
		self := make(map[string]interface{})
		self["container"] = c
		if c["stack_uuid"] != nil {
			self["stack"] = versionedData.UUIDToStack[c["stack_uuid"].(string)]
			self["service"] = versionedData.UUIDToService[getServiceUUID(c["service_uuid"].(string), c["service_name"].(string))]
		}
		if c["host_uuid"] != nil {
			self["host"] = versionedData.UUIDToHost[c["host_uuid"].(string)]
		}
		clientAnswers["self"] = self
		mergeDefaults(clientAnswers, defaultAnswers)
		answers[c["primary_ip"].(string)] = clientAnswers
	}
}

func mergeDefaults(clientAnswers map[string]interface{}, defaultAnswers map[string]interface{}) {
	for k, v := range defaultAnswers {
		_, exists := clientAnswers[k]
		if !exists {
			clientAnswers[k] = v
		}
	}
}

func addDefaultToAnswers(answers Answers, versionedData *Interim) map[string]interface{} {
	defaultAnswers := make(map[string]interface{})
	var containers []interface{}
	for _, c := range versionedData.UUIDToContainer {
		if _, ok := defaultAnswers["containers"]; ok {
			containers = defaultAnswers["containers"].([]interface{})
		}
		containers = append(containers, c)
	}
	defaultAnswers["containers"] = containers

	var stacks []interface{}
	for _, s := range versionedData.UUIDToStack {
		if _, ok := defaultAnswers["stacks"]; ok {
			stacks = defaultAnswers["stacks"].([]interface{})
		}
		stacks = append(stacks, s)
	}
	defaultAnswers["stacks"] = stacks

	var services []interface{}
	for _, s := range versionedData.UUIDToService {
		if _, ok := defaultAnswers["services"]; ok {
			services = defaultAnswers["services"].([]interface{})
		}
		services = append(services, s)
	}
	defaultAnswers["services"] = services

	var hosts []interface{}
	for _, h := range versionedData.UUIDToHost {
		if _, ok := defaultAnswers["hosts"]; ok {
			hosts = defaultAnswers["hosts"].([]interface{})
		}
		hosts = append(hosts, h)
	}
	defaultAnswers["hosts"] = hosts
	defaultAnswers["networks"] = versionedData.Networks

	if val, ok := versionedData.Default["version"]; ok {
		defaultAnswers["version"] = val
	}

	if selfVal, ok := versionedData.Default["self"]; ok {
		self := selfVal.(map[string]interface{})
		if hostVal, ok := self["host"]; ok {
			host := hostVal.(map[string]interface{})
			self["host"] = versionedData.UUIDToHost[host["uuid"].(string)]
		}
		defaultAnswers["self"] = self
	}

	answers[DEFAULT_KEY] = defaultAnswers
	return defaultAnswers
}

func applyVersionToData(orig Interim, version string) (*Interim, error) {
	copied, err := copystructure.Copy(orig)
	if err != nil {
		return nil, err
	}
	modified := copied.(Interim)
	// 1. Process containers
	for _, c := range modified.UUIDToContainer {
		switch version {
		case version3:
			if c["name"] != nil {
				c["name"] = strings.ToLower(c["name"].(string))
			}
			if c["service_name"] != nil {
				c["service_name"] = strings.ToLower(c["service_name"].(string))
			}
			if c["stack_name"] != nil {
				c["stack_name"] = strings.ToLower(c["stack_name"].(string))
			}
			if _, ok := c["ports"]; ok {
				// set port ip to 0.0.0.0 if not specified
				originalPorts := c["ports"].([]interface{})
				var newPorts []interface{}
				for _, p := range originalPorts {
					port := p.(string)
					splitted := strings.Split(port, ":")
					if len(splitted) == 3 {
						newPorts = append(newPorts, port)
					} else {
						port = fmt.Sprintf("0.0.0.0:%s", port)
						newPorts = append(newPorts, port)
					}
				}
				c["ports"] = newPorts
			}

		default:
			if _, ok := c["ports"]; ok {
				originalPorts := c["ports"].([]interface{})
				// set port ip to host's ip if not specified
				if len(originalPorts) > 0 {
					var newPorts []interface{}
					for _, p := range originalPorts {
						port := p.(string)
						splitted := strings.Split(port, ":")
						if len(splitted) == 3 && splitted[0] != "0.0.0.0" {
							newPorts = append(newPorts, port)
						} else {
							if len(splitted) == 3 {
								port = fmt.Sprintf("%s%s", c["host_ip"], strings.TrimPrefix(port, "0.0.0.0"))
							} else {
								port = fmt.Sprintf("%s:%s", c["host_ip"], port)
							}

							newPorts = append(newPorts, port)
						}
					}
					c["ports"] = newPorts
				}
			}
		}
		c["links"] = modified.ContainerUUIDToContainerLink[c["uuid"].(string)]
		//delete helper field (needed for the port)
		delete(c, "host_ip")
	}

	// 2. Process services
	for _, s := range modified.UUIDToService {
		sUUID := getServiceUUID(s["uuid"].(string), s["name"].(string))

		stackUUID := s["stack_uuid"].(string)
		// add itself to the stack list
		var svcUUIDs []string
		if _, ok := modified.StackUUIDToServicesUUID[stackUUID]; ok {
			svcUUIDs = modified.StackUUIDToServicesUUID[stackUUID]
		}
		svcUUIDs = append(svcUUIDs, getServiceUUID(s["uuid"].(string), s["name"].(string)))
		modified.StackUUIDToServicesUUID[stackUUID] = svcUUIDs
		var cs []interface{}
		var cNames []interface{}
		cUUIDs := modified.ServiceUUIDNameToContainersUUID[sUUID]
		if cUUIDs != nil {
			for _, cUUID := range cUUIDs {
				if c, ok := modified.UUIDToContainer[cUUID]; ok {
					cs = append(cs, c)
					cNames = append(cNames, c["name"].(string))
				}
			}
		}
		switch version {
		case version1:
			s["containers"] = cNames
		case version2:
			s["containers"] = cs
		case version3:
			s["containers"] = cs
			s["name"] = strings.ToLower(s["name"].(string))
			s["stack_name"] = strings.ToLower(s["stack_name"].(string))
		}
		// add service links
		s["links"] = modified.ServiceUUIDToServiceLink[s["uuid"].(string)]
		// populate stack/service info on container
		if cUUIDs != nil {
			for _, cUUID := range cUUIDs {
				if _, ok := modified.UUIDToContainer[cUUID]; ok {
					modified.UUIDToContainer[cUUID]["service_name"] = s["name"]
					modified.UUIDToContainer[cUUID]["service_uuid"] = s["uuid"]
					modified.UUIDToContainer[cUUID]["stack_name"] = s["stack_name"]
					modified.UUIDToContainer[cUUID]["stack_uuid"] = stackUUID
				}
			}
		}
	}

	// 3. Process stacks
	for _, s := range modified.UUIDToStack {
		var svcs []interface{}
		var svcsNames []interface{}
		svcsUUIDs := modified.StackUUIDToServicesUUID[s["uuid"].(string)]
		if svcsUUIDs != nil {
			for _, svcUUID := range svcsUUIDs {
				if svc, ok := modified.UUIDToService[svcUUID]; ok {
					svcs = append(svcs, svc)
					svcsNames = append(svcsNames, svc["name"].(string))
				}
			}
		}
		switch version {
		case version1:
			s["services"] = svcsNames
		case version2:
			s["services"] = svcs
		case version3:
			s["services"] = svcs
			s["name"] = strings.ToLower(s["name"].(string))
		}
	}

	// 4. process hosts
	for _, h := range modified.UUIDToHost {
		switch version {
		case version3:
			delete(h, "hostId")
		}
	}

	return &modified, nil
}

func processMetadataObject(o map[string]interface{}, interim *Interim) {
	if val, ok := o["metadata_kind"]; ok {
		switch val.(string) {
		case "container":
			addContainer(o, interim)
		case "stack":
			addStack(o, interim)
		case "network":
			addNetwork(o, interim)
		case "host":
			addHost(o, interim)
		case "defaultData":
			addDefault(o, interim)
		case "serviceContainerLink":
			addServiceContainerLink(o, interim)
		case "containerLink":
			addContainerLink(o, interim)
		case "serviceLink":
			addServiceLink(o, interim)
		case "service":
			addService(o, interim)
		}
	}
}

func addContainer(container map[string]interface{}, interim *Interim) {
	interim.UUIDToContainer[container["uuid"].(string)] = container
}

func addService(service map[string]interface{}, interim *Interim) {
	interim.UUIDToService[getServiceUUID(service["uuid"].(string), service["name"].(string))] = service
}

func addServiceContainerLink(link map[string]interface{}, interim *Interim) {
	UUID := getServiceUUID(link["service_uuid"].(string), link["service_name"].(string))
	var containerUUIDList []string
	if _, ok := interim.ServiceUUIDNameToContainersUUID[UUID]; ok {
		containerUUIDList = interim.ServiceUUIDNameToContainersUUID[UUID]
	}
	containerUUIDList = append(containerUUIDList, link["container_uuid"].(string))
	interim.ServiceUUIDNameToContainersUUID[UUID] = containerUUIDList
}

func addServiceLink(link map[string]interface{}, interim *Interim) {
	serviceUUID := link["service_uuid"].(string)
	links := make(map[string]interface{})
	if _, ok := interim.ServiceUUIDToServiceLink[serviceUUID]; ok {
		links = interim.ServiceUUIDToServiceLink[serviceUUID]
	}
	linkKey := link["key"].(string)
	links[linkKey] = link["value"].(string)

	interim.ServiceUUIDToServiceLink[serviceUUID] = links
}

func addContainerLink(link map[string]interface{}, interim *Interim) {
	containerUUID := link["container_uuid"].(string)
	links := make(map[string]interface{})
	if _, ok := interim.ContainerUUIDToContainerLink[containerUUID]; ok {
		links = interim.ContainerUUIDToContainerLink[containerUUID]
	}
	linkKey := link["key"].(string)
	links[linkKey] = link["value"].(string)

	interim.ContainerUUIDToContainerLink[containerUUID] = links
}

func getServiceUUID(UUID string, name string) string {
	return strings.ToLower(fmt.Sprintf("%s_%s", UUID, name))
}

func addStack(stack map[string]interface{}, interim *Interim) {
	interim.UUIDToStack[stack["uuid"].(string)] = stack
}

func addNetwork(network map[string]interface{}, interim *Interim) {
	interim.Networks = append(interim.Networks, network)
}

func addHost(host map[string]interface{}, interim *Interim) {
	interim.UUIDToHost[host["uuid"].(string)] = host
}

func addDefault(def map[string]interface{}, interim *Interim) {
	interim.Default = def
}
<|endoftext|>


<commit_before>package acgen

import (
	"fmt"
	"io"
)

type Flag struct {
	Short       []string
	Long        []string
	Arg         string
	Description string
}

type Command struct {
	Name  string
	Flags []*Flag
}

type Generator func(w io.Writer, c *Command) error

var generators = make(map[string]Generator)

func RegisterGenerator(name string, g Generator) {
	generators[name] = g
}

func LookGenerator(name string) (g Generator, err error) {
	if _, ok := generators[name]; !ok {
		return nil, fmt.Errorf("%s: is not supported", name)
	}
	return generators[name], nil
}
<commit_msg>Use sync.Mutex for acgen.generators<commit_after>package acgen

import (
	"fmt"
	"io"
	"sync"
)

type Flag struct {
	Short       []string
	Long        []string
	Arg         string
	Description string
}

type Command struct {
	Name  string
	Flags []*Flag
}

type Generator func(w io.Writer, c *Command) error

var (
	generatorsMu sync.Mutex
	generators   = make(map[string]Generator)
)

func RegisterGenerator(name string, g Generator) {
	generatorsMu.Lock()
	defer generatorsMu.Unlock()
	generators[name] = g
}

func LookGenerator(name string) (g Generator, err error) {
	generatorsMu.Lock()
	defer generatorsMu.Unlock()
	if _, ok := generators[name]; !ok {
		return nil, fmt.Errorf("%s: is not supported", name)
	}
	return generators[name], nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/intervention-engine/fhir/models"
	"math/rand"
	"net/http"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "get-risky"
	app.Usage = "Upload risk assessments for patients retrieved from a FHIR URL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "fhir, f",
			Usage: "URL for the FHIR server",
		},
	}
	app.Action = func(c *cli.Context) {
		fhirUrl := c.String("fhir")
		if fhirUrl == "" {
			fmt.Println("You must provide a FHIR URL")
		} else {
			patientIds := GetPatientIds(fhirUrl + "/Patient")
			r := rand.New(rand.NewSource(99))
			for _, id := range patientIds {
				for week := 0; week < 24; week++ {
					datetime := time.Now().Add(-time.Hour*168*time.Duration(week))
					UploadRiskAssessment(fhirUrl+"/Observation", fhirUrl+"/Patient", id, datetime, r)
				}
			}
		}
	}

	app.Run(os.Args)
}

func GetPatientIds(patientUrl string) []string {
	resp, err := http.Get(patientUrl)
	defer resp.Body.Close()
	if err != nil {
		panic(err)
	}
	decoder := json.NewDecoder(resp.Body)
	patientBundle := &models.PatientBundle{}
	err = decoder.Decode(patientBundle)
	if err != nil {
		panic(err)
	}
	var patientIds []string
	for _, p := range patientBundle.Entry {
		patientIds = append(patientIds, p.Id)
	}
	return patientIds
}

func UploadRiskAssessment(observationUrl, patientUrl, patientId string, datetime time.Time, r *rand.Rand) {
	observation := &models.Observation{Reliability: "ok", Status: "final"}
	randomRisk := r.Intn(4)
	if randomRisk == 0 {
		randomRisk = 1
	}
	observation.ValueQuantity = &models.Quantity{Value: float64(randomRisk)}
	cc := models.CodeableConcept{}
	cc.Coding = []models.Coding{models.Coding{System: "http://loinc.org", Code: "75492-9"}}
	observation.Name = &cc
	observation.AppliesPeriod = &models.Period{Start: &models.FHIRDateTime{Precision: models.Timestamp, Time: datetime}, End: &models.FHIRDateTime{Precision: models.Timestamp, Time: datetime}}
	observation.Subject = &models.Reference{Reference: patientUrl + "/" + patientId}
	json, _ := json.Marshal(observation)
	body := bytes.NewReader(json)
	_, err := http.Post(observationUrl, "application/json+fhir", body)
	if err != nil {
		panic("HTTP request failed")
	}
}
<commit_msg>Kill get-risky since it is no longer relevant<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/intervention-engine/fhir/models"
	"math/rand"
	"net/http"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "get-risky"
	app.Usage = "Upload risk assessments for patients retrieved from a FHIR URL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "fhir, f",
			Usage: "URL for the FHIR server",
		},
	}
	app.Action = func(c *cli.Context) {
		fhirUrl := c.String("fhir")
		if fhirUrl == "" {
			fmt.Println("You must provide a FHIR URL")
		} else {
			patientIds := GetPatientIds(fhirUrl + "/Patient")
			r := rand.New(rand.NewSource(99))
			for _, id := range patientIds {
				for week := 0; week < 24; week++ {
					datetime := time.Now().Add(-time.Hour*168*time.Duration(week))
					UploadRiskAssessment(fhirUrl+"/Observation", fhirUrl+"/Patient", id, datetime, r)
				}
			}
		}
	}

	app.Run(os.Args)
}

func GetPatientIds(patientUrl string) []string {
	resp, err := http.Get(patientUrl)
	defer resp.Body.Close()
	if err != nil {
		panic(err)
	}
	decoder := json.NewDecoder(resp.Body)
	patientBundle := &models.PatientBundle{}
	err = decoder.Decode(patientBundle)
	if err != nil {
		panic(err)
	}
	var patientIds []string
	for _, p := range patientBundle.Entry {
		patientIds = append(patientIds, p.Id)
	}
	return patientIds
}

func UploadRiskAssessment(observationUrl, patientUrl, patientId string, datetime time.Time, r *rand.Rand) {
	observation := &models.Observation{Reliability: "ok", Status: "final"}
	randomRisk := r.Intn(4)
	if randomRisk == 0 {
		randomRisk = 1
	}
	observation.ValueQuantity = models.Quantity{Value: float64(randomRisk)}
	cc := models.CodeableConcept{}
	cc.Coding = []models.Coding{models.Coding{System: "http://loinc.org", Code: "75492-9"}}
	observation.Name = cc
	observation.AppliesDateTime = models.FHIRDateTime{Precision: models.Timestamp, Time: datetime}
	observation.Subject = models.Reference{Reference: patientUrl + "/" + patientId}
	json, _ := json.Marshal(observation)
	body := bytes.NewReader(json)
	_, err := http.Post(observationUrl, "application/json+fhir", body)
	if err != nil {
		panic("HTTP request failed")
	}
}
<commit_msg>updated get risky to use observation AppliesPeriod instead of AppliesDateTime<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/intervention-engine/fhir/models"
	"math/rand"
	"net/http"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "get-risky"
	app.Usage = "Upload risk assessments for patients retrieved from a FHIR URL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "fhir, f",
			Usage: "URL for the FHIR server",
		},
	}
	app.Action = func(c *cli.Context) {
		fhirUrl := c.String("fhir")
		if fhirUrl == "" {
			fmt.Println("You must provide a FHIR URL")
		} else {
			patientIds := GetPatientIds(fhirUrl + "/Patient")
			r := rand.New(rand.NewSource(99))
			for _, id := range patientIds {
				for week := 0; week < 24; week++ {
					datetime := time.Now().Add(-time.Hour*168*time.Duration(week))
					UploadRiskAssessment(fhirUrl+"/Observation", fhirUrl+"/Patient", id, datetime, r)
				}
			}
		}
	}

	app.Run(os.Args)
}

func GetPatientIds(patientUrl string) []string {
	resp, err := http.Get(patientUrl)
	defer resp.Body.Close()
	if err != nil {
		panic(err)
	}
	decoder := json.NewDecoder(resp.Body)
	patientBundle := &models.PatientBundle{}
	err = decoder.Decode(patientBundle)
	if err != nil {
		panic(err)
	}
	var patientIds []string
	for _, p := range patientBundle.Entry {
		patientIds = append(patientIds, p.Id)
	}
	return patientIds
}

func UploadRiskAssessment(observationUrl, patientUrl, patientId string, datetime time.Time, r *rand.Rand) {
	observation := &models.Observation{Reliability: "ok", Status: "final"}
	randomRisk := r.Intn(4)
	if randomRisk == 0 {
		randomRisk = 1
	}
	observation.ValueQuantity = &models.Quantity{Value: float64(randomRisk)}
	cc := models.CodeableConcept{}
	cc.Coding = []models.Coding{models.Coding{System: "http://loinc.org", Code: "75492-9"}}
	observation.Name = &cc
	observation.AppliesPeriod = &models.Period{Start: &models.FHIRDateTime{Precision: models.Timestamp, Time: datetime}, End: &models.FHIRDateTime{Precision: models.Timestamp, Time: datetime}}
	observation.Subject = &models.Reference{Reference: patientUrl + "/" + patientId}
	json, _ := json.Marshal(observation)
	body := bytes.NewReader(json)
	_, err := http.Post(observationUrl, "application/json+fhir", body)
	if err != nil {
		panic("HTTP request failed")
	}
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Liu Dong <ddliuhb@gmail.com>.
// Licensed under the MIT license.

// Powerful and easy to use http client
package httpclient

import (
    "io"
    "net/http"
)

// The default client for convenience
var defaultClient *HttpClient

var Defaults func(Map) *HttpClient
var Begin func() *HttpClient
var Do func(string, string, map[string]string, io.Reader)(*Response, error)
var Get func(string, map[string]string)(*Response, error)
var Head func(string, map[string]string)(*Response, error)
var Post func(string, map[string]string)(*Response, error)
var PostMultipart func(string, map[string]string)(*Response, error)
var WithOption func(int, interface{}) *HttpClient
var WithOptions func(Map) *HttpClient
var WithHeader func(string, string) *HttpClient
var WithHeaders func(map[string]string) *HttpClient
var WithCookie func(...*http.Cookie) *HttpClient
var Cookies func(string)([]*http.Cookie)
var CookieValues func(string)(map[string]string)
var CookieValue func(string, string)(string)

func init() {
    defaultClient = NewHttpClient()
    Defaults = defaultClient.Defaults
    Begin = defaultClient.Begin
    Do = defaultClient.Do
    Get = defaultClient.Get
    Head = defaultClient.Head
    Post = defaultClient.Post
    PostMultipart = defaultClient.PostMultipart
    WithOption = defaultClient.WithOption
    WithOptions = defaultClient.WithOptions
    WithHeader = defaultClient.WithHeader
    WithHeaders = defaultClient.WithHeaders
    WithCookie = defaultClient.WithCookie
    Cookies = defaultClient.Cookies
    CookieValues = defaultClient.CookieValues
    CookieValue = defaultClient.CookieValue
}<commit_msg>add DELETE function to default_client<commit_after>// Copyright 2014-2015 Liu Dong <ddliuhb@gmail.com>.
// Licensed under the MIT license.

// Powerful and easy to use http client
package httpclient

import (
    "io"
    "net/http"
)

// The default client for convenience
var defaultClient *HttpClient

var Defaults func(Map) *HttpClient
var Begin func() *HttpClient
var Do func(string, string, map[string]string, io.Reader)(*Response, error)
var Get func(string, map[string]string)(*Response, error)
var Delete func(string, map[string]string)(*Response, error)
var Head func(string, map[string]string)(*Response, error)
var Post func(string, map[string]string)(*Response, error)
var PostMultipart func(string, map[string]string)(*Response, error)
var WithOption func(int, interface{}) *HttpClient
var WithOptions func(Map) *HttpClient
var WithHeader func(string, string) *HttpClient
var WithHeaders func(map[string]string) *HttpClient
var WithCookie func(...*http.Cookie) *HttpClient
var Cookies func(string)([]*http.Cookie)
var CookieValues func(string)(map[string]string)
var CookieValue func(string, string)(string)

func init() {
    defaultClient = NewHttpClient()
    Defaults = defaultClient.Defaults
    Begin = defaultClient.Begin
    Do = defaultClient.Do
    Get = defaultClient.Get
    Delete = defaultClient.Delete
    Head = defaultClient.Head
    Post = defaultClient.Post
    PostMultipart = defaultClient.PostMultipart
    WithOption = defaultClient.WithOption
    WithOptions = defaultClient.WithOptions
    WithHeader = defaultClient.WithHeader
    WithHeaders = defaultClient.WithHeaders
    WithCookie = defaultClient.WithCookie
    Cookies = defaultClient.Cookies
    CookieValues = defaultClient.CookieValues
    CookieValue = defaultClient.CookieValue
}
<|endoftext|>


<commit_before>package arel

type DeleteManager struct {
	AstNode
}

func NewDeleteManager() *DeleteManager {
	return new(DeleteManager)
}
<commit_msg>Embed BaseNode in DeleteManager<commit_after>package arel

type DeleteManager struct {
	BaseNode
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/cblomart/vsphere-graphite/backend"
	"github.com/cblomart/vsphere-graphite/config"
	"github.com/cblomart/vsphere-graphite/vsphere"

	"github.com/takama/daemon"

	"code.cloudfoundry.org/bytefmt"
)

const (
	// name of the service
	name        = "vsphere-graphite"
	description = "send vsphere stats to graphite"
)

var dependencies = []string{}

var stdlog, errlog *log.Logger

var commit, tag string

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

func queryVCenter(vcenter vsphere.VCenter, conf config.Configuration, channel *chan backend.Point) {
	vcenter.Query(conf.Interval, conf.Domain, conf.Properties, channel)
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	usage := "Usage: vsphere-graphite install | remove | start | stop | status"

	// if received any kind of command, do it
	if len(os.Args) > 1 {
		command := os.Args[1]
		text := usage
		var err error
		switch command {
		case "install":
			text, err = service.Install()
		case "remove":
			text, err = service.Remove()
		case "start":
			text, err = service.Start()
		case "stop":
			text, err = service.Stop()
		case "status":
			text, err = service.Status()
		}
		return text, err
	}

	stdlog.Println("Starting daemon:", path.Base(os.Args[0]))

	// read the configuration
	file, err := os.Open("/etc/" + path.Base(os.Args[0]) + ".json")
	if err != nil {
		return "Could not open configuration file", err
	}
	jsondec := json.NewDecoder(file)
	conf := config.Configuration{}
	err = jsondec.Decode(&conf)
	if err != nil {
		return "Could not decode configuration file", err
	}

	if conf.FlushSize == 0 {
		conf.FlushSize = 1000
	}

	if conf.CPUProfiling {
		f, err := ioutil.TempFile("/tmp", "vsphere-graphite-cpu.profile") // nolint: vetshadow
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		stdlog.Println("Will write cpu profiling to: ", f.Name())
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	//force backend values to environement varialbles if present
	s := reflect.ValueOf(conf.Backend).Elem()
	numfields := s.NumField()
	for i := 0; i < numfields; i++ {
		f := s.Field(i)
		if f.CanSet() {
			//exported field
			envname := strings.ToUpper(s.Type().Name() + "_" + s.Type().Field(i).Name)
			envval := os.Getenv(envname)
			if len(envval) > 0 {
				//environment variable set with name
				switch ftype := f.Type().Name(); ftype {
				case "string":
					f.SetString(envval)
				case "int":
					val, err := strconv.ParseInt(envval, 10, 64) // nolint: vetshadow
					if err == nil {
						f.SetInt(val)
					}
				}
			}
		}
	}

	for _, vcenter := range conf.VCenters {
		vcenter.Init(conf.Metrics, stdlog, errlog)
	}

	err = conf.Backend.Init(stdlog, errlog)
	if err != nil {
		return "Could not initialize backend", err
	}
	defer conf.Backend.Disconnect()

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM) // nolint: megacheck

	// Set up a channel to receive the metrics
	metrics := make(chan backend.Point, conf.FlushSize)

	// Set up a ticker to collect metrics at givent interval
	ticker := time.NewTicker(time.Second * time.Duration(conf.Interval))
	defer ticker.Stop()

	// Start retriveing and sending metrics
	stdlog.Println("Retrieving metrics")
	for _, vcenter := range conf.VCenters {
		go queryVCenter(*vcenter, conf, &metrics)
	}

	// Memory statisctics
	var memstats runtime.MemStats
	// timer to execute memory collection
	memtimer := time.NewTimer(time.Second * time.Duration(10))
	// Memory profiling
	var mf *os.File
	if conf.MEMProfiling {
		mf, err = ioutil.TempFile("/tmp", "vsphere-graphite-mem.profile")
		if err != nil {
			log.Fatal("could not create MEM profile: ", err)
		}
		defer mf.Close() // nolint: errcheck
	}
	// buffer for points to send
	pointbuffer := make([]*backend.Point, conf.FlushSize)
	bufferindex := 0

	for {
		select {
		case value := <-metrics:
			// reset timer as a point has been revieved
			if !memtimer.Stop() {
				select {
				case <-memtimer.C:
				default:
				}
			}
			memtimer.Reset(time.Second * time.Duration(5))
			pointbuffer[bufferindex] = &value
			bufferindex++
			if bufferindex == len(pointbuffer) {
				conf.Backend.SendMetrics(pointbuffer)
				stdlog.Printf("Sent %d logs to backend", bufferindex)
				ClearBuffer(pointbuffer)
				bufferindex = 0
			}
		case <-ticker.C:
			stdlog.Println("Retrieving metrics")
			for _, vcenter := range conf.VCenters {
				go queryVCenter(*vcenter, conf, &metrics)
			}
		case <-memtimer.C:
			// sent remaining values
			conf.Backend.SendMetrics(pointbuffer)
			stdlog.Printf("Sent %d logs to backend", bufferindex)
			bufferindex = 0
			ClearBuffer(pointbuffer)
			runtime.GC()
			debug.FreeOSMemory()
			runtime.ReadMemStats(&memstats)
			stdlog.Printf("Memory usage : sys=%s alloc=%s\n", bytefmt.ByteSize(memstats.Sys), bytefmt.ByteSize(memstats.Alloc))
			if conf.MEMProfiling {
				stdlog.Println("Writing mem profiling to: ", mf.Name())
				debug.WriteHeapDump(mf.Fd())
			}
		case killSignal := <-interrupt:
			stdlog.Println("Got signal:", killSignal)
			if bufferindex > 0 {
				conf.Backend.SendMetrics(pointbuffer[:bufferindex])
				stdlog.Printf("Sent %d logs to backend", bufferindex)
			}
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// ClearBuffer : set all values in pointer array to nil
func ClearBuffer(buffer []*backend.Point) {
	for i := 0; i < len(buffer); i++ {
		buffer[i] = nil
	}
}

func init() {
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
}

func main() {
	if len(commit) == 0 && len(tag) == 0 {
		stdlog.Println("No version information")
	} else {
		stdlog.Print("Version information")
		if len(commit) > 0 {
			stdlog.Print(" - Commit: ", commit)
		}
		if len(tag) > 0 {
			stdlog.Println(" - Version: ", tag)
		}
		stdlog.Print("\n")
	}
	srv, err := daemon.New(name, description, dependencies...)
	if err != nil {
		errlog.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		errlog.Println(status, "Error: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<commit_msg>get all properties per default<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/cblomart/vsphere-graphite/backend"
	"github.com/cblomart/vsphere-graphite/config"
	"github.com/cblomart/vsphere-graphite/vsphere"

	"github.com/takama/daemon"

	"code.cloudfoundry.org/bytefmt"
)

const (
	// name of the service
	name        = "vsphere-graphite"
	description = "send vsphere stats to graphite"
)

var dependencies = []string{}

var stdlog, errlog *log.Logger

var commit, tag string

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

func queryVCenter(vcenter vsphere.VCenter, conf config.Configuration, channel *chan backend.Point) {
	vcenter.Query(conf.Interval, conf.Domain, conf.Properties, channel)
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	usage := "Usage: vsphere-graphite install | remove | start | stop | status"

	// if received any kind of command, do it
	if len(os.Args) > 1 {
		command := os.Args[1]
		text := usage
		var err error
		switch command {
		case "install":
			text, err = service.Install()
		case "remove":
			text, err = service.Remove()
		case "start":
			text, err = service.Start()
		case "stop":
			text, err = service.Stop()
		case "status":
			text, err = service.Status()
		}
		return text, err
	}

	stdlog.Println("Starting daemon:", path.Base(os.Args[0]))

	// read the configuration
	file, err := os.Open("/etc/" + path.Base(os.Args[0]) + ".json")
	if err != nil {
		return "Could not open configuration file", err
	}
	jsondec := json.NewDecoder(file)
	conf := config.Configuration{}
	err = jsondec.Decode(&conf)
	if err != nil {
		return "Could not decode configuration file", err
	}

	// defaults to all properties
	if conf.Properties == nil {
		conf.Properties = []string{"all"}
	}

	if conf.FlushSize == 0 {
		conf.FlushSize = 1000
	}

	if conf.CPUProfiling {
		f, err := ioutil.TempFile("/tmp", "vsphere-graphite-cpu.profile") // nolint: vetshadow
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		stdlog.Println("Will write cpu profiling to: ", f.Name())
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	//force backend values to environement varialbles if present
	s := reflect.ValueOf(conf.Backend).Elem()
	numfields := s.NumField()
	for i := 0; i < numfields; i++ {
		f := s.Field(i)
		if f.CanSet() {
			//exported field
			envname := strings.ToUpper(s.Type().Name() + "_" + s.Type().Field(i).Name)
			envval := os.Getenv(envname)
			if len(envval) > 0 {
				//environment variable set with name
				switch ftype := f.Type().Name(); ftype {
				case "string":
					f.SetString(envval)
				case "int":
					val, err := strconv.ParseInt(envval, 10, 64) // nolint: vetshadow
					if err == nil {
						f.SetInt(val)
					}
				}
			}
		}
	}

	for _, vcenter := range conf.VCenters {
		vcenter.Init(conf.Metrics, stdlog, errlog)
	}

	err = conf.Backend.Init(stdlog, errlog)
	if err != nil {
		return "Could not initialize backend", err
	}
	defer conf.Backend.Disconnect()

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM) // nolint: megacheck

	// Set up a channel to receive the metrics
	metrics := make(chan backend.Point, conf.FlushSize)

	// Set up a ticker to collect metrics at givent interval
	ticker := time.NewTicker(time.Second * time.Duration(conf.Interval))
	defer ticker.Stop()

	// Start retriveing and sending metrics
	stdlog.Println("Retrieving metrics")
	for _, vcenter := range conf.VCenters {
		go queryVCenter(*vcenter, conf, &metrics)
	}

	// Memory statisctics
	var memstats runtime.MemStats
	// timer to execute memory collection
	memtimer := time.NewTimer(time.Second * time.Duration(10))
	// Memory profiling
	var mf *os.File
	if conf.MEMProfiling {
		mf, err = ioutil.TempFile("/tmp", "vsphere-graphite-mem.profile")
		if err != nil {
			log.Fatal("could not create MEM profile: ", err)
		}
		defer mf.Close() // nolint: errcheck
	}
	// buffer for points to send
	pointbuffer := make([]*backend.Point, conf.FlushSize)
	bufferindex := 0

	for {
		select {
		case value := <-metrics:
			// reset timer as a point has been revieved
			if !memtimer.Stop() {
				select {
				case <-memtimer.C:
				default:
				}
			}
			memtimer.Reset(time.Second * time.Duration(5))
			pointbuffer[bufferindex] = &value
			bufferindex++
			if bufferindex == len(pointbuffer) {
				conf.Backend.SendMetrics(pointbuffer)
				stdlog.Printf("Sent %d logs to backend", bufferindex)
				ClearBuffer(pointbuffer)
				bufferindex = 0
			}
		case <-ticker.C:
			stdlog.Println("Retrieving metrics")
			for _, vcenter := range conf.VCenters {
				go queryVCenter(*vcenter, conf, &metrics)
			}
		case <-memtimer.C:
			// sent remaining values
			conf.Backend.SendMetrics(pointbuffer)
			stdlog.Printf("Sent %d logs to backend", bufferindex)
			bufferindex = 0
			ClearBuffer(pointbuffer)
			runtime.GC()
			debug.FreeOSMemory()
			runtime.ReadMemStats(&memstats)
			stdlog.Printf("Memory usage : sys=%s alloc=%s\n", bytefmt.ByteSize(memstats.Sys), bytefmt.ByteSize(memstats.Alloc))
			if conf.MEMProfiling {
				stdlog.Println("Writing mem profiling to: ", mf.Name())
				debug.WriteHeapDump(mf.Fd())
			}
		case killSignal := <-interrupt:
			stdlog.Println("Got signal:", killSignal)
			if bufferindex > 0 {
				conf.Backend.SendMetrics(pointbuffer[:bufferindex])
				stdlog.Printf("Sent %d logs to backend", bufferindex)
			}
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// ClearBuffer : set all values in pointer array to nil
func ClearBuffer(buffer []*backend.Point) {
	for i := 0; i < len(buffer); i++ {
		buffer[i] = nil
	}
}

func init() {
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
}

func main() {
	if len(commit) == 0 && len(tag) == 0 {
		stdlog.Println("No version information")
	} else {
		stdlog.Print("Version information")
		if len(commit) > 0 {
			stdlog.Print(" - Commit: ", commit)
		}
		if len(tag) > 0 {
			stdlog.Println(" - Version: ", tag)
		}
		stdlog.Print("\n")
	}
	srv, err := daemon.New(name, description, dependencies...)
	if err != nil {
		errlog.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		errlog.Println(status, "Error: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<|endoftext|>


<commit_before>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package compression

import (
	"bytes"
	"compress/gzip"
	"errors"
	"io"
	"sync"

	"github.com/ava-labs/avalanchego/utils"
)

var errTooLarge = errors.New("message too large")

// gzipCompressor implements Compressor
type gzipCompressor struct {
	maxSize int64

	lock sync.Mutex

	writeBuffer *bytes.Buffer
	gzipWriter  *gzip.Writer

	bytesReader *bytes.Reader
	gzipReader  *gzip.Reader
}

// Compress [msg] and returns the compressed bytes.
func (g *gzipCompressor) Compress(msg []byte) ([]byte, error) {
	if int64(len(msg)) > g.maxSize {
		return nil, errTooLarge
	}

	g.lock.Lock()
	defer g.lock.Unlock()

	g.writeBuffer.Reset()
	g.gzipWriter.Reset(g.writeBuffer)
	if _, err := g.gzipWriter.Write(msg); err != nil {
		return nil, err
	}
	if err := g.gzipWriter.Close(); err != nil {
		return nil, err
	}

	compressed := g.writeBuffer.Bytes()
	compressedCopy := utils.CopyBytes(compressed)
	return compressedCopy, nil
}

// Decompress decompresses [msg].
func (g *gzipCompressor) Decompress(msg []byte) ([]byte, error) {
	g.lock.Lock()
	defer g.lock.Unlock()

	g.bytesReader.Reset(msg)
	if err := g.gzipReader.Reset(g.bytesReader); err != nil {
		return nil, err
	}
	limitedReader := io.LimitReader(g.gzipReader, g.maxSize+1)
	decompressed, err := io.ReadAll(limitedReader)
	if err != nil {
		return nil, err
	}
	if int64(len(decompressed)) > g.maxSize {
		return nil, errTooLarge
	}
	return decompressed, g.gzipReader.Close()
}

// NewGzipCompressor returns a new gzip Compressor that compresses
func NewGzipCompressor(maxSize int64) Compressor {
	var buf bytes.Buffer
	return &gzipCompressor{
		maxSize: maxSize,

		writeBuffer: &buf,
		gzipWriter:  gzip.NewWriter(&buf),

		bytesReader: &bytes.Reader{},
		gzipReader:  &gzip.Reader{},
	}
}
<commit_msg>updated nit error<commit_after>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package compression

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"sync"

	"github.com/ava-labs/avalanchego/utils"
)

// gzipCompressor implements Compressor
type gzipCompressor struct {
	maxSize int64

	lock sync.Mutex

	writeBuffer *bytes.Buffer
	gzipWriter  *gzip.Writer

	bytesReader *bytes.Reader
	gzipReader  *gzip.Reader
}

// Compress [msg] and returns the compressed bytes.
func (g *gzipCompressor) Compress(msg []byte) ([]byte, error) {
	if int64(len(msg)) > g.maxSize {
		return nil, fmt.Errorf("msg length (%d) > maximum msg length (%d)", len(msg), g.maxSize)
	}

	g.lock.Lock()
	defer g.lock.Unlock()

	g.writeBuffer.Reset()
	g.gzipWriter.Reset(g.writeBuffer)
	if _, err := g.gzipWriter.Write(msg); err != nil {
		return nil, err
	}
	if err := g.gzipWriter.Close(); err != nil {
		return nil, err
	}

	compressed := g.writeBuffer.Bytes()
	compressedCopy := utils.CopyBytes(compressed)
	return compressedCopy, nil
}

// Decompress decompresses [msg].
func (g *gzipCompressor) Decompress(msg []byte) ([]byte, error) {
	g.lock.Lock()
	defer g.lock.Unlock()

	g.bytesReader.Reset(msg)
	if err := g.gzipReader.Reset(g.bytesReader); err != nil {
		return nil, err
	}
	limitedReader := io.LimitReader(g.gzipReader, g.maxSize+1)
	decompressed, err := io.ReadAll(limitedReader)
	if err != nil {
		return nil, err
	}
	if int64(len(decompressed)) > g.maxSize {
		return nil, fmt.Errorf("msg length > maximum msg length (%d)", g.maxSize)
	}
	return decompressed, g.gzipReader.Close()
}

// NewGzipCompressor returns a new gzip Compressor that compresses
func NewGzipCompressor(maxSize int64) Compressor {
	var buf bytes.Buffer
	return &gzipCompressor{
		maxSize: maxSize,

		writeBuffer: &buf,
		gzipWriter:  gzip.NewWriter(&buf),

		bytesReader: &bytes.Reader{},
		gzipReader:  &gzip.Reader{},
	}
}
<|endoftext|>


<commit_before>package helper

import (
	"bytes"
	"fmt"
	"image/png"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/image/webp"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/gomarkdown/markdown"
	"github.com/gomarkdown/markdown/html"
	"github.com/gomarkdown/markdown/parser"
	"github.com/sirupsen/logrus"
)

// DownloadFile downloads the given non-authenticated URL.
func DownloadFile(url string) (*[]byte, error) {
	return DownloadFileAuth(url, "")
}

// DownloadFileAuth downloads the given URL using the specified authentication token.
func DownloadFileAuth(url string, auth string) (*[]byte, error) {
	var buf bytes.Buffer
	client := &http.Client{
		Timeout: time.Second * 5,
	}
	req, err := http.NewRequest("GET", url, nil)
	if auth != "" {
		req.Header.Add("Authorization", auth)
	}
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	io.Copy(&buf, resp.Body)
	data := buf.Bytes()
	return &data, nil
}

// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
// specified as non-zero GetSubLines will also clip long lines to the maximum
// length and insert a warning marker that the line was clipped.
//
// TODO: The current implementation has the inconvenient that it disregards
// word boundaries when splitting but this is hard to solve without potentially
// breaking formatting and other stylistic effects.
func GetSubLines(message string, maxLineLength int) []string {
	const clippingMessage = " <clipped message>"

	var lines []string
	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
		if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
			lines = append(lines, line)
			continue
		}

		// !!! WARNING !!!
		// Before touching the splitting logic below please ensure that you PROPERLY
		// understand how strings, runes and range loops over strings work in Go.
		// A good place to start is to read https://blog.golang.org/strings. :-)
		var splitStart int
		var startOfPreviousRune int
		for i := range line {
			if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
				lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
				splitStart = startOfPreviousRune
			}
			startOfPreviousRune = i
		}
		// This last append is safe to do without looking at the remaining byte-length
		// as we assume that the byte-length of the last rune will never exceed that of
		// the byte-length of the clipping message.
		lines = append(lines, line[splitStart:])
	}
	return lines
}

// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
	extra := msg.Extra
	rmsg := []config.Message{}
	for _, f := range extra[config.EventFileFailureSize] {
		fi := f.(config.FileInfo)
		text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
		rmsg = append(rmsg, config.Message{
			Text:     text,
			Username: "<system> ",
			Channel:  msg.Channel,
			Account:  msg.Account,
		})
	}
	return rmsg
}

// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
	if sha, ok := av[userid]; ok {
		return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
	}
	return ""
}

// HandleDownloadSize checks a specified filename against the configured download blacklist
// and checks a specified file-size against the configure limit.
func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
	// check blacklist here
	for _, entry := range general.MediaDownloadBlackList {
		if entry != "" {
			re, err := regexp.Compile(entry)
			if err != nil {
				logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
				continue
			}
			if re.MatchString(name) {
				return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name)
			}
		}
	}
	logger.Debugf("Trying to download %#v with size %#v", name, size)
	if int(size) > general.MediaDownloadSize {
		msg.Event = config.EventFileFailureSize
		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
			Name:    name,
			Comment: msg.Text,
			Size:    size,
		})
		return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
	}
	return nil
}

// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
	var avatar bool
	logger.Debugf("Download OK %#v %#v", name, len(*data))
	if msg.Event == config.EventAvatarDownload {
		avatar = true
	}
	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
		Name:    name,
		Data:    data,
		URL:     url,
		Comment: comment,
		Avatar:  avatar,
	})
}

var emptyLineMatcher = regexp.MustCompile("\n+")

// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
// trims any preceding or trailing newline characters as well.
func RemoveEmptyNewLines(msg string) string {
	return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
}

// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
// to the message in case it does so.
func ClipMessage(text string, length int) string {
	const clippingMessage = " <clipped message>"
	if len(text) > length {
		text = text[:length-len(clippingMessage)]
		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
			text = text[:len(text)-size]
		}
		text += clippingMessage
	}
	return text
}

// ParseMarkdown takes in an input string as markdown and parses it to html
func ParseMarkdown(input string) string {
	extensions := parser.HardLineBreak | parser.NoIntraEmphasis
	markdownParser := parser.NewWithExtensions(extensions)
	renderer := html.NewRenderer(html.RendererOptions{
		Flags: 0,
	})
	parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
	res := string(parsedMarkdown)
	res = strings.TrimPrefix(res, "
")
	res = strings.TrimSuffix(res, "
\n")
	return res
}

// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
func ConvertWebPToPNG(data *[]byte) error {
	r := bytes.NewReader(*data)
	m, err := webp.Decode(r)
	if err != nil {
		return err
	}
	var output []byte
	w := bytes.NewBuffer(output)
	if err := png.Encode(w, m); err != nil {
		return err
	}
	*data = w.Bytes()
	return nil
}

// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
func CanConvertTgsToX() error {
	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
	cmd := exec.Command("lottie_convert.py", "--help")
	return cmd.Run()
}

// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
// This relies on an external command, which is ugly, but works.
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
	// lottie can't handle input from a pipe, so write to a temporary file:
	tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
	if err != nil {
		return err
	}
	tmpFileName := tmpFile.Name()
	defer func() {
		if removeErr := os.Remove(tmpFileName); removeErr != nil {
			logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
		}
	}()

	if _, writeErr := tmpFile.Write(*data); writeErr != nil {
		return writeErr
	}
	// Must close before calling lottie to avoid data races:
	if closeErr := tmpFile.Close(); closeErr != nil {
		return closeErr
	}

	// Call lottie to transform:
	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
	cmd.Stderr = nil
	// NB: lottie writes progress into to stderr in all cases.
	stdout, stderr := cmd.Output()
	if stderr != nil {
		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
		return stderr
	}

	*data = stdout
	return nil
}
<commit_msg>Parse fencedcode in ParseMarkdown. Fixes #1127 (#1329)<commit_after>package helper

import (
	"bytes"
	"fmt"
	"image/png"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/image/webp"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/gomarkdown/markdown"
	"github.com/gomarkdown/markdown/html"
	"github.com/gomarkdown/markdown/parser"
	"github.com/sirupsen/logrus"
)

// DownloadFile downloads the given non-authenticated URL.
func DownloadFile(url string) (*[]byte, error) {
	return DownloadFileAuth(url, "")
}

// DownloadFileAuth downloads the given URL using the specified authentication token.
func DownloadFileAuth(url string, auth string) (*[]byte, error) {
	var buf bytes.Buffer
	client := &http.Client{
		Timeout: time.Second * 5,
	}
	req, err := http.NewRequest("GET", url, nil)
	if auth != "" {
		req.Header.Add("Authorization", auth)
	}
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	io.Copy(&buf, resp.Body)
	data := buf.Bytes()
	return &data, nil
}

// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
// specified as non-zero GetSubLines will also clip long lines to the maximum
// length and insert a warning marker that the line was clipped.
//
// TODO: The current implementation has the inconvenient that it disregards
// word boundaries when splitting but this is hard to solve without potentially
// breaking formatting and other stylistic effects.
func GetSubLines(message string, maxLineLength int) []string {
	const clippingMessage = " <clipped message>"

	var lines []string
	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
		if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
			lines = append(lines, line)
			continue
		}

		// !!! WARNING !!!
		// Before touching the splitting logic below please ensure that you PROPERLY
		// understand how strings, runes and range loops over strings work in Go.
		// A good place to start is to read https://blog.golang.org/strings. :-)
		var splitStart int
		var startOfPreviousRune int
		for i := range line {
			if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
				lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
				splitStart = startOfPreviousRune
			}
			startOfPreviousRune = i
		}
		// This last append is safe to do without looking at the remaining byte-length
		// as we assume that the byte-length of the last rune will never exceed that of
		// the byte-length of the clipping message.
		lines = append(lines, line[splitStart:])
	}
	return lines
}

// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
	extra := msg.Extra
	rmsg := []config.Message{}
	for _, f := range extra[config.EventFileFailureSize] {
		fi := f.(config.FileInfo)
		text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
		rmsg = append(rmsg, config.Message{
			Text:     text,
			Username: "<system> ",
			Channel:  msg.Channel,
			Account:  msg.Account,
		})
	}
	return rmsg
}

// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
	if sha, ok := av[userid]; ok {
		return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
	}
	return ""
}

// HandleDownloadSize checks a specified filename against the configured download blacklist
// and checks a specified file-size against the configure limit.
func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
	// check blacklist here
	for _, entry := range general.MediaDownloadBlackList {
		if entry != "" {
			re, err := regexp.Compile(entry)
			if err != nil {
				logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
				continue
			}
			if re.MatchString(name) {
				return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name)
			}
		}
	}
	logger.Debugf("Trying to download %#v with size %#v", name, size)
	if int(size) > general.MediaDownloadSize {
		msg.Event = config.EventFileFailureSize
		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
			Name:    name,
			Comment: msg.Text,
			Size:    size,
		})
		return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
	}
	return nil
}

// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
	var avatar bool
	logger.Debugf("Download OK %#v %#v", name, len(*data))
	if msg.Event == config.EventAvatarDownload {
		avatar = true
	}
	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
		Name:    name,
		Data:    data,
		URL:     url,
		Comment: comment,
		Avatar:  avatar,
	})
}

var emptyLineMatcher = regexp.MustCompile("\n+")

// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
// trims any preceding or trailing newline characters as well.
func RemoveEmptyNewLines(msg string) string {
	return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
}

// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
// to the message in case it does so.
func ClipMessage(text string, length int) string {
	const clippingMessage = " <clipped message>"
	if len(text) > length {
		text = text[:length-len(clippingMessage)]
		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
			text = text[:len(text)-size]
		}
		text += clippingMessage
	}
	return text
}

// ParseMarkdown takes in an input string as markdown and parses it to html
func ParseMarkdown(input string) string {
	extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode
	markdownParser := parser.NewWithExtensions(extensions)
	renderer := html.NewRenderer(html.RendererOptions{
		Flags: 0,
	})
	parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
	res := string(parsedMarkdown)
	res = strings.TrimPrefix(res, "
")
	res = strings.TrimSuffix(res, "
\n")
	return res
}

// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
func ConvertWebPToPNG(data *[]byte) error {
	r := bytes.NewReader(*data)
	m, err := webp.Decode(r)
	if err != nil {
		return err
	}
	var output []byte
	w := bytes.NewBuffer(output)
	if err := png.Encode(w, m); err != nil {
		return err
	}
	*data = w.Bytes()
	return nil
}

// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
func CanConvertTgsToX() error {
	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
	cmd := exec.Command("lottie_convert.py", "--help")
	return cmd.Run()
}

// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
// This relies on an external command, which is ugly, but works.
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
	// lottie can't handle input from a pipe, so write to a temporary file:
	tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
	if err != nil {
		return err
	}
	tmpFileName := tmpFile.Name()
	defer func() {
		if removeErr := os.Remove(tmpFileName); removeErr != nil {
			logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
		}
	}()

	if _, writeErr := tmpFile.Write(*data); writeErr != nil {
		return writeErr
	}
	// Must close before calling lottie to avoid data races:
	if closeErr := tmpFile.Close(); closeErr != nil {
		return closeErr
	}

	// Call lottie to transform:
	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
	cmd.Stderr = nil
	// NB: lottie writes progress into to stderr in all cases.
	stdout, stderr := cmd.Output()
	if stderr != nil {
		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
		return stderr
	}

	*data = stdout
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package palm

import (
	"log"
	"math/rand"
	"os"
	"sync"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/Workiva/go-datastructures/common"
)

func checkTree(t testing.TB, tree *ptree) bool {
	return true

	if tree.root == nil {
		return true
	}

	return checkNode(t, tree.root)
}

func checkNode(t testing.TB, n *node) bool {
	if n.keys.len() == 0 {
		assert.Equal(t, uint64(0), n.nodes.len())
		return false
	}

	if n.isLeaf {
		assert.Equal(t, uint64(0), n.nodes.len())
		return false
	}

	if !assert.Equal(t, n.keys.len()+1, n.nodes.len()) {
		return false
	}

	for i, k := range n.keys.list {
		nd := n.nodes.list[i]
		if !assert.NotNil(t, nd) {
			return false
		}

		if !assert.True(t, k.Compare(nd.key()) >= 0) {
			t.Logf(`N: %+v %p, n.keys[i]: %+v, n.nodes[i]: %+v`, n, n, k, nd)
			return false
		}
	}

	k := n.keys.last()
	nd := n.nodes.byPosition(n.nodes.len() - 1)
	if !assert.True(t, k.Compare(nd.key()) < 0) {
		t.Logf(`m: %+v, %p, n.nodes[len(n.nodes)-1].key(): %+v, n.keys.last(): %+v`, n, n, nd, k)
		return false
	}
	for _, child := range n.nodes.list {
		if !assert.NotNil(t, child) {
			return false
		}
		if !checkNode(t, child) {
			return false
		}
	}

	return true
}

func getConsoleLogger() *log.Logger {
	return log.New(os.Stderr, "", log.LstdFlags)
}

func generateRandomKeys(num int) common.Comparators {
	keys := make(common.Comparators, 0, num)
	for i := 0; i < num; i++ {
		m := rand.Int()
		keys = append(keys, mockKey(m%50))
	}
	return keys
}

func generateKeys(num int) common.Comparators {
	keys := make(common.Comparators, 0, num)
	for i := 0; i < num; i++ {
		keys = append(keys, mockKey(i))
	}

	return keys
}

func TestSimpleInsert(t *testing.T) {
	tree := newTree(16, 16)
	defer tree.Dispose()
	m1 := mockKey(1)

	tree.Insert(m1)
	assert.Equal(t, common.Comparators{m1}, tree.Get(m1))
	assert.Equal(t, uint64(1), tree.Len())
	checkTree(t, tree)
}

func TestSimpleDelete(t *testing.T) {
	tree := newTree(8, 8)
	defer tree.Dispose()
	m1 := mockKey(1)
	tree.Insert(m1)

	tree.Delete(m1)
	assert.Equal(t, uint64(0), tree.Len())
	assert.Equal(t, common.Comparators{nil}, tree.Get(m1))
	checkTree(t, tree)
}

func TestMultipleAdd(t *testing.T) {
	tree := newTree(16, 16)
	defer tree.Dispose()
	m1 := mockKey(1)
	m2 := mockKey(10)

	tree.Insert(m1, m2)
	if !assert.Equal(t, common.Comparators{m1, m2}, tree.Get(m1, m2)) {
		tree.print(getConsoleLogger())
	}
	assert.Equal(t, uint64(2), tree.Len())
	checkTree(t, tree)
}

func TestMultipleDelete(t *testing.T) {
	tree := newTree(16, 16)
	defer tree.Dispose()
	m1 := mockKey(1)
	m2 := mockKey(10)
	tree.Insert(m1, m2)

	tree.Delete(m1, m2)
	assert.Equal(t, uint64(0), tree.Len())
	assert.Equal(t, common.Comparators{nil, nil}, tree.Get(m1, m2))
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitOddAryReverseOrder(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateKeys(100)
	reversed := reverseKeys(keys)

	tree.Insert(reversed...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleDeleteOddAryReverseOrder(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateKeys(100)
	reversed := reverseKeys(keys)
	tree.Insert(reversed...)
	assert.Equal(t, uint64(100), tree.Len())

	tree.Delete(reversed...)
	assert.Equal(t, uint64(0), tree.Len())
	for _, k := range reversed {
		assert.Equal(t, common.Comparators{nil}, tree.Get(k))
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitOddAry(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitOddAryRandomOrder(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateRandomKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleBulkInsertOddAry(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys1 := generateRandomKeys(100)
	keys2 := generateRandomKeys(100)

	tree.Insert(keys1...)

	if !assert.Equal(t, keys1, tree.Get(keys1...)) {
		tree.print(getConsoleLogger())
	}

	tree.Insert(keys2...)

	if !assert.Equal(t, keys2, tree.Get(keys2...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleBulkInsertEvenAry(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys1 := generateRandomKeys(100)
	keys2 := generateRandomKeys(100)

	tree.Insert(keys1...)
	tree.Insert(keys2...)

	if !assert.Equal(t, keys1, tree.Get(keys1...)) {
		tree.print(getConsoleLogger())
	}

	if !assert.Equal(t, keys2, tree.Get(keys2...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitEvenAryReverseOrder(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateKeys(100)
	reversed := reverseKeys(keys)

	tree.Insert(reversed...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitEvenAry(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitEvenAryRandomOrder(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateRandomKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestInsertOverwrite(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateKeys(10)
	duplicate := mockKey(0)
	tree.Insert(keys...)

	tree.Insert(duplicate)
	assert.Equal(t, common.Comparators{duplicate}, tree.Get(duplicate))
	checkTree(t, tree)
}

func TestSimultaneousReadsAndWrites(t *testing.T) {
	numLoops := 3
	keys := make([]common.Comparators, 0, numLoops)
	for i := 0; i < numLoops; i++ {
		keys = append(keys, generateRandomKeys(10))
	}

	tree := newTree(16, 16)
	defer tree.Dispose()
	var wg sync.WaitGroup
	wg.Add(numLoops)
	for i := 0; i < numLoops; i++ {
		go func(i int) {
			tree.Insert(keys[i]...)
			tree.Get(keys[i]...)
			wg.Done()
		}(i)
	}

	wg.Wait()

	for i := 0; i < numLoops; i++ {
		assert.Equal(t, keys[i], tree.Get(keys[i]...))
	}
	checkTree(t, tree)
}

func BenchmarkReadAndWrites(b *testing.B) {
	numItems := 1000
	keys := make([]common.Comparators, 0, b.N)
	for i := 0; i < b.N; i++ {
		keys = append(keys, generateRandomKeys(numItems))
	}

	tree := newTree(16, 8)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Insert(keys[i]...)
		tree.Get(keys[i]...)
	}
}

func BenchmarkSimultaneousReadsAndWrites(b *testing.B) {
	numItems := 1000
	numRoutines := 8
	keys := make([]common.Comparators, 0, numRoutines)
	for i := 0; i < numRoutines; i++ {
		keys = append(keys, generateRandomKeys(numItems))
	}

	tree := newTree(16, 8)
	var wg sync.WaitGroup
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		wg.Add(numRoutines)
		for j := 0; j < numRoutines; j++ {
			go func(j int) {
				tree.Insert(keys[j]...)
				tree.Get(keys[j]...)
				wg.Done()
			}(j)
		}

		wg.Wait()
	}
}

func BenchmarkBulkAdd(b *testing.B) {
	numItems := 10000
	keys := generateRandomKeys(numItems)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree := newTree(8, 8)
		tree.Insert(keys...)
		tree.Dispose()
	}
}

func BenchmarkAdd(b *testing.B) {
	numItems := 500
	keys := generateRandomKeys(numItems)
	tree := newTree(32, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Insert(keys[i%numItems])
	}
}

func BenchmarkBulkAddToExisting(b *testing.B) {
	numItems := 100000
	keySet := make([]common.Comparators, 0, b.N)
	for i := 0; i < b.N; i++ {
		keySet = append(keySet, generateRandomKeys(numItems))
	}

	tree := newTree(8, 8)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Insert(keySet[i]...)
	}
}

func BenchmarkGet(b *testing.B) {
	numItems := 10000
	keys := generateRandomKeys(numItems)
	tree := newTree(32, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Get(keys[i%numItems])
	}
}

func BenchmarkBulkGet(b *testing.B) {
	numItems := 1000
	keys := generateRandomKeys(numItems)
	tree := newTree(16, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Get(keys...)
	}
}
<commit_msg>Added some benchmarks around delete.<commit_after>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package palm

import (
	"log"
	"math/rand"
	"os"
	"sync"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/Workiva/go-datastructures/common"
)

func checkTree(t testing.TB, tree *ptree) bool {
	return true

	if tree.root == nil {
		return true
	}

	return checkNode(t, tree.root)
}

func checkNode(t testing.TB, n *node) bool {
	if n.keys.len() == 0 {
		assert.Equal(t, uint64(0), n.nodes.len())
		return false
	}

	if n.isLeaf {
		assert.Equal(t, uint64(0), n.nodes.len())
		return false
	}

	if !assert.Equal(t, n.keys.len()+1, n.nodes.len()) {
		return false
	}

	for i, k := range n.keys.list {
		nd := n.nodes.list[i]
		if !assert.NotNil(t, nd) {
			return false
		}

		if !assert.True(t, k.Compare(nd.key()) >= 0) {
			t.Logf(`N: %+v %p, n.keys[i]: %+v, n.nodes[i]: %+v`, n, n, k, nd)
			return false
		}
	}

	k := n.keys.last()
	nd := n.nodes.byPosition(n.nodes.len() - 1)
	if !assert.True(t, k.Compare(nd.key()) < 0) {
		t.Logf(`m: %+v, %p, n.nodes[len(n.nodes)-1].key(): %+v, n.keys.last(): %+v`, n, n, nd, k)
		return false
	}
	for _, child := range n.nodes.list {
		if !assert.NotNil(t, child) {
			return false
		}
		if !checkNode(t, child) {
			return false
		}
	}

	return true
}

func getConsoleLogger() *log.Logger {
	return log.New(os.Stderr, "", log.LstdFlags)
}

func generateRandomKeys(num int) common.Comparators {
	keys := make(common.Comparators, 0, num)
	for i := 0; i < num; i++ {
		m := rand.Int()
		keys = append(keys, mockKey(m%50))
	}
	return keys
}

func generateKeys(num int) common.Comparators {
	keys := make(common.Comparators, 0, num)
	for i := 0; i < num; i++ {
		keys = append(keys, mockKey(i))
	}

	return keys
}

func TestSimpleInsert(t *testing.T) {
	tree := newTree(16, 16)
	defer tree.Dispose()
	m1 := mockKey(1)

	tree.Insert(m1)
	assert.Equal(t, common.Comparators{m1}, tree.Get(m1))
	assert.Equal(t, uint64(1), tree.Len())
	checkTree(t, tree)
}

func TestSimpleDelete(t *testing.T) {
	tree := newTree(8, 8)
	defer tree.Dispose()
	m1 := mockKey(1)
	tree.Insert(m1)

	tree.Delete(m1)
	assert.Equal(t, uint64(0), tree.Len())
	assert.Equal(t, common.Comparators{nil}, tree.Get(m1))
	checkTree(t, tree)
}

func TestMultipleAdd(t *testing.T) {
	tree := newTree(16, 16)
	defer tree.Dispose()
	m1 := mockKey(1)
	m2 := mockKey(10)

	tree.Insert(m1, m2)
	if !assert.Equal(t, common.Comparators{m1, m2}, tree.Get(m1, m2)) {
		tree.print(getConsoleLogger())
	}
	assert.Equal(t, uint64(2), tree.Len())
	checkTree(t, tree)
}

func TestMultipleDelete(t *testing.T) {
	tree := newTree(16, 16)
	defer tree.Dispose()
	m1 := mockKey(1)
	m2 := mockKey(10)
	tree.Insert(m1, m2)

	tree.Delete(m1, m2)
	assert.Equal(t, uint64(0), tree.Len())
	assert.Equal(t, common.Comparators{nil, nil}, tree.Get(m1, m2))
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitOddAryReverseOrder(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateKeys(100)
	reversed := reverseKeys(keys)

	tree.Insert(reversed...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleDeleteOddAryReverseOrder(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateKeys(100)
	reversed := reverseKeys(keys)
	tree.Insert(reversed...)
	assert.Equal(t, uint64(100), tree.Len())

	tree.Delete(reversed...)
	assert.Equal(t, uint64(0), tree.Len())
	for _, k := range reversed {
		assert.Equal(t, common.Comparators{nil}, tree.Get(k))
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitOddAry(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitOddAryRandomOrder(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys := generateRandomKeys(10)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleBulkInsertOddAry(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()
	keys1 := generateRandomKeys(100)
	keys2 := generateRandomKeys(100)

	tree.Insert(keys1...)

	if !assert.Equal(t, keys1, tree.Get(keys1...)) {
		tree.print(getConsoleLogger())
	}

	tree.Insert(keys2...)

	if !assert.Equal(t, keys2, tree.Get(keys2...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleBulkInsertEvenAry(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys1 := generateRandomKeys(100)
	keys2 := generateRandomKeys(100)

	tree.Insert(keys1...)
	tree.Insert(keys2...)

	if !assert.Equal(t, keys1, tree.Get(keys1...)) {
		tree.print(getConsoleLogger())
	}

	if !assert.Equal(t, keys2, tree.Get(keys2...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitEvenAryReverseOrder(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateKeys(100)
	reversed := reverseKeys(keys)

	tree.Insert(reversed...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitEvenAry(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestMultipleInsertCausesSplitEvenAryRandomOrder(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateRandomKeys(100)

	tree.Insert(keys...)
	if !assert.Equal(t, keys, tree.Get(keys...)) {
		tree.print(getConsoleLogger())
	}
	checkTree(t, tree)
}

func TestInsertOverwrite(t *testing.T) {
	tree := newTree(4, 4)
	defer tree.Dispose()
	keys := generateKeys(10)
	duplicate := mockKey(0)
	tree.Insert(keys...)

	tree.Insert(duplicate)
	assert.Equal(t, common.Comparators{duplicate}, tree.Get(duplicate))
	checkTree(t, tree)
}

func TestSimultaneousReadsAndWrites(t *testing.T) {
	numLoops := 3
	keys := make([]common.Comparators, 0, numLoops)
	for i := 0; i < numLoops; i++ {
		keys = append(keys, generateRandomKeys(10))
	}

	tree := newTree(16, 16)
	defer tree.Dispose()
	var wg sync.WaitGroup
	wg.Add(numLoops)
	for i := 0; i < numLoops; i++ {
		go func(i int) {
			tree.Insert(keys[i]...)
			tree.Get(keys[i]...)
			wg.Done()
		}(i)
	}

	wg.Wait()

	for i := 0; i < numLoops; i++ {
		assert.Equal(t, keys[i], tree.Get(keys[i]...))
	}
	checkTree(t, tree)
}

func TestInsertAndDelete(t *testing.T) {
	tree := newTree(1024, 1024)
	defer tree.Dispose()

	keys := generateKeys(100)
	keys1 := keys[:50]
	keys2 := keys[50:]
	tree.Insert(keys1...)
	assert.Equal(t, uint64(len(keys1)), tree.Len())
	var wg sync.WaitGroup
	wg.Add(2)

	go func() {
		tree.Insert(keys2...)
		wg.Done()
	}()

	go func() {
		tree.Delete(keys1...)
		wg.Done()
	}()

	wg.Wait()

	assert.Equal(t, uint64(len(keys2)), tree.Len())
	assert.Equal(t, keys2, tree.Get(keys2...))
}

func TestInsertAndDeletesWithSplits(t *testing.T) {
	tree := newTree(3, 3)
	defer tree.Dispose()

	keys := generateKeys(100)
	keys1 := keys[:50]
	keys2 := keys[50:]
	tree.Insert(keys1...)
	assert.Equal(t, uint64(len(keys1)), tree.Len())
	var wg sync.WaitGroup
	wg.Add(2)

	go func() {
		tree.Insert(keys2...)
		wg.Done()
	}()

	go func() {
		tree.Delete(keys1...)
		wg.Done()
	}()

	wg.Wait()

	assert.Equal(t, uint64(len(keys2)), tree.Len())
	assert.Equal(t, keys2, tree.Get(keys2...))
}

func BenchmarkReadAndWrites(b *testing.B) {
	numItems := 1000
	keys := make([]common.Comparators, 0, b.N)
	for i := 0; i < b.N; i++ {
		keys = append(keys, generateRandomKeys(numItems))
	}

	tree := newTree(16, 8)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Insert(keys[i]...)
		tree.Get(keys[i]...)
	}
}

func BenchmarkSimultaneousReadsAndWrites(b *testing.B) {
	numItems := 1000
	numRoutines := 8
	keys := make([]common.Comparators, 0, numRoutines)
	for i := 0; i < numRoutines; i++ {
		keys = append(keys, generateRandomKeys(numItems))
	}

	tree := newTree(16, 8)
	var wg sync.WaitGroup
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		wg.Add(numRoutines)
		for j := 0; j < numRoutines; j++ {
			go func(j int) {
				tree.Insert(keys[j]...)
				tree.Get(keys[j]...)
				wg.Done()
			}(j)
		}

		wg.Wait()
	}
}

func BenchmarkBulkAdd(b *testing.B) {
	numItems := 10000
	keys := generateRandomKeys(numItems)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree := newTree(8, 8)
		tree.Insert(keys...)
	}
}

func BenchmarkAdd(b *testing.B) {
	numItems := 500
	keys := generateRandomKeys(numItems)
	tree := newTree(32, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Insert(keys[i%numItems])
	}
}

func BenchmarkBulkAddToExisting(b *testing.B) {
	numItems := 100000
	keySet := make([]common.Comparators, 0, b.N)
	for i := 0; i < b.N; i++ {
		keySet = append(keySet, generateRandomKeys(numItems))
	}

	tree := newTree(8, 8)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Insert(keySet[i]...)
	}
}

func BenchmarkGet(b *testing.B) {
	numItems := 10000
	keys := generateRandomKeys(numItems)
	tree := newTree(32, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Get(keys[i%numItems])
	}
}

func BenchmarkBulkGet(b *testing.B) {
	numItems := 1000
	keys := generateRandomKeys(numItems)
	tree := newTree(16, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Get(keys...)
	}
}

func BenchmarkDelete(b *testing.B) {
	numItems := b.N
	keys := generateRandomKeys(numItems)
	tree := newTree(8, 8)
	tree.Insert(keys...)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		tree.Delete(keys[i%numItems])
	}
}

func BenchmarkBulkDelete(b *testing.B) {
	numItems := 10000
	keys := generateRandomKeys(numItems)
	trees := make([]*ptree, 0, b.N)
	for i := 0; i < b.N; i++ {
		tree := newTree(8, 8)
		tree.Insert(keys...)
		trees = append(trees, tree)
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		trees[i].Delete(keys...)
	}
}
<|endoftext|>


<commit_before>package CloudForest

import (
	"math"
)

/*
DEntropyTarget wraps a categorical feature for use in entropy driven classification
as in Ross Quinlan's ID3 (Iterative Dichotomizer 3) with a the entropy modified to use
"disutility entropy"

I = - k Sum ri * pi * log(pi)

*/
type DEntropyTarget struct {
	CatFeature
	Costs []float64
}

//NewDEntropyTarget creates a RefretTarget and initializes DEntropyTarget.Costs to the proper length.
func NewDEntropyTarget(f CatFeature) *DEntropyTarget {
	return &DEntropyTarget{f, make([]float64, f.NCats())}
}

/*NewDEntropyTarget.SetCosts puts costs in a map[string]float64 by feature name into the proper
entries in NewDEntropyTarget.Costs.*/
func (target *DEntropyTarget) SetCosts(costmap map[string]float64) {
	for i := 0; i < target.NCats(); i++ {
		c := target.NumToCat(i)
		target.Costs[i] = costmap[c]
	}
}

/*
DEntropyTarget.SplitImpurity is a version of Split Impurity that calls DEntropyTarget.Impurity
*/
func (target *DEntropyTarget) SplitImpurity(l *[]int, r *[]int, m *[]int, allocs *BestSplitAllocs) (impurityDecrease float64) {
	nl := float64(len(*l))
	nr := float64(len(*r))
	nm := 0.0

	impurityDecrease = nl * target.Impurity(l, allocs.LCounter)
	impurityDecrease += nr * target.Impurity(r, allocs.RCounter)
	if m != nil && len(*m) > 0 {
		nm = float64(len(*m))
		impurityDecrease += nm * target.Impurity(m, allocs.Counter)
	}

	impurityDecrease /= nl + nr + nm
	return
}

//UpdateSImpFromAllocs willl be called when splits are being built by moving cases from r to l as in learning from numerical variables.
//Here it just wraps SplitImpurity but it can be implemented to provide further optimization.
func (target *DEntropyTarget) UpdateSImpFromAllocs(l *[]int, r *[]int, m *[]int, allocs *BestSplitAllocs, movedRtoL *[]int) (impurityDecrease float64) {
	target.MoveCountsRtoL(allocs, movedRtoL)
	nl := float64(len(*l))
	nr := float64(len(*r))
	nm := 0.0

	impurityDecrease = nl * target.ImpFromCounts(len(*l), allocs.LCounter)
	impurityDecrease += nr * target.ImpFromCounts(len(*r), allocs.RCounter)
	if m != nil && len(*m) > 0 {
		nm = float64(len(*m))
		impurityDecrease += nm * target.ImpFromCounts(len(*m), allocs.Counter)
	}

	impurityDecrease /= nl + nr + nm
	return
}

func (target *DEntropyTarget) ImpFromCounts(total int, counts *[]int) (e float64) {
	p := 0.0
	for c, i := range *counts {
		if i > 0 {
			p = float64(i) / float64(total)
			e -= target.Costs[c] * p * math.Log(p)
		}
	}
	return

}

//DEntropyTarget.Impurity implements categorical entropy as sum(pj*log2(pj)) where pj
//is the number of cases with the j'th category over the total number of cases.
func (target *DEntropyTarget) Impurity(cases *[]int, counts *[]int) (e float64) {

	total := len(*cases)
	target.CountPerCat(cases, counts)

	p := 0.0
	for _, i := range *counts {
		if i > 0 {
			p = float64(i) / float64(total)
			e -= p * math.Log(p)
		}

	}

	return

}
<commit_msg>probs from dentropy target<commit_after>package CloudForest

import (
	"fmt"
	"math"
)

/*
DEntropyTarget wraps a categorical feature for use in entropy driven classification
as in Ross Quinlan's ID3 (Iterative Dichotomizer 3) with a the entropy modified to use
"disutility entropy"

I = - k Sum ri * pi * log(pi)

*/
type DEntropyTarget struct {
	CatFeature
	Costs []float64
}

//NewDEntropyTarget creates a RefretTarget and initializes DEntropyTarget.Costs to the proper length.
func NewDEntropyTarget(f CatFeature) *DEntropyTarget {
	return &DEntropyTarget{f, make([]float64, f.NCats())}
}

/*NewDEntropyTarget.SetCosts puts costs in a map[string]float64 by feature name into the proper
entries in NewDEntropyTarget.Costs.*/
func (target *DEntropyTarget) SetCosts(costmap map[string]float64) {
	for i := 0; i < target.NCats(); i++ {
		c := target.NumToCat(i)
		target.Costs[i] = costmap[c]
	}
}

/*
DEntropyTarget.SplitImpurity is a version of Split Impurity that calls DEntropyTarget.Impurity
*/
func (target *DEntropyTarget) SplitImpurity(l *[]int, r *[]int, m *[]int, allocs *BestSplitAllocs) (impurityDecrease float64) {
	nl := float64(len(*l))
	nr := float64(len(*r))
	nm := 0.0

	impurityDecrease = nl * target.Impurity(l, allocs.LCounter)
	impurityDecrease += nr * target.Impurity(r, allocs.RCounter)
	if m != nil && len(*m) > 0 {
		nm = float64(len(*m))
		impurityDecrease += nm * target.Impurity(m, allocs.Counter)
	}

	impurityDecrease /= nl + nr + nm
	return
}

//UpdateSImpFromAllocs willl be called when splits are being built by moving cases from r to l as in learning from numerical variables.
//Here it just wraps SplitImpurity but it can be implemented to provide further optimization.
func (target *DEntropyTarget) UpdateSImpFromAllocs(l *[]int, r *[]int, m *[]int, allocs *BestSplitAllocs, movedRtoL *[]int) (impurityDecrease float64) {
	target.MoveCountsRtoL(allocs, movedRtoL)
	nl := float64(len(*l))
	nr := float64(len(*r))
	nm := 0.0

	impurityDecrease = nl * target.ImpFromCounts(len(*l), allocs.LCounter)
	impurityDecrease += nr * target.ImpFromCounts(len(*r), allocs.RCounter)
	if m != nil && len(*m) > 0 {
		nm = float64(len(*m))
		impurityDecrease += nm * target.ImpFromCounts(len(*m), allocs.Counter)
	}

	impurityDecrease /= nl + nr + nm
	return
}

func (target *DEntropyTarget) ImpFromCounts(total int, counts *[]int) (e float64) {
	p := 0.0
	for c, i := range *counts {
		if i > 0 {
			p = float64(i) / float64(total)
			e -= target.Costs[c] * p * math.Log(p)
		}
	}
	return

}

func (target *DEntropyTarget) FindPredicted(cases []int) (pred string) {
	prob_true := 0.0
	t := target.CatToNum("True")
	weightedvoted := true
	if weightedvoted {
		count := 0.0
		total := 0.0
		for _, i := range cases {
			ti := target.Geti(i)
			cost := target.Costs[ti]
			if ti == t {
				count += cost
			}
			total += cost

		}
		prob_true = count / total

	} else {
		count := 0
		for _, i := range cases {
			if target.Geti(i) == t {
				count++
			}

		}
		prob_true = float64(count) / float64(len(cases))
	}
	return fmt.Sprintf("%v", prob_true)
}

//DEntropyTarget.Impurity implements categorical entropy as sum(pj*log2(pj)) where pj
//is the number of cases with the j'th category over the total number of cases.
func (target *DEntropyTarget) Impurity(cases *[]int, counts *[]int) (e float64) {

	total := len(*cases)
	target.CountPerCat(cases, counts)

	p := 0.0
	for _, i := range *counts {
		if i > 0 {
			p = float64(i) / float64(total)
			e -= p * math.Log(p)
		}

	}

	return

}
<|endoftext|>


<commit_before>package validators

import (
	"fmt"

	"github.com/markbates/going/validate"
)

type StringIsPresent struct {
	Name  string
	Field string
}

func (v *StringIsPresent) IsValid(errors *validate.Errors) {
	if v.Field == "" {
		errors.Add(GenerateKey(v.Name), fmt.Sprintf("%s can not be blank.", v.Name))
	}
}
<commit_msg>trim the space on a string validation<commit_after>package validators

import (
	"fmt"
	"strings"

	"github.com/markbates/going/validate"
)

type StringIsPresent struct {
	Name  string
	Field string
}

func (v *StringIsPresent) IsValid(errors *validate.Errors) {
	if strings.TrimSpace(v.Field) == "" {
		errors.Add(GenerateKey(v.Name), fmt.Sprintf("%s can not be blank.", v.Name))
	}
}
<|endoftext|>