<commit_before>package models

import (
	"encoding/json"
	"strconv"

	"github.com/cloudfoundry/cli/cf/formatters"
	. "github.com/cloudfoundry/cli/cf/i18n"
)

type SpaceQuota struct {
	GUID                    string      `json:"guid,omitempty"`
	Name                    string      `json:"name"`
	MemoryLimit             int64       `json:"memory_limit"`          // in Megabytes
	InstanceMemoryLimit     int64       `json:"instance_memory_limit"` // in Megabytes
	RoutesLimit             int         `json:"total_routes"`
	ServicesLimit           int         `json:"total_services"`
	NonBasicServicesAllowed bool        `json:"non_basic_services_allowed"`
	OrgGUID                 string      `json:"organization_guid"`
	AppInstanceLimit        int         `json:"app_instance_limit"`
	ReservedRoutePortsLimit json.Number `json:"total_reserved_route_ports,omitempty"`
}

const UnlimitedDisplay = T("unlimited")

func (q SpaceQuota) FormattedMemoryLimit() string {
	return formatters.ByteSize(q.MemoryLimit * formatters.MEGABYTE)
}

func (q SpaceQuota) FormattedInstanceMemoryLimit() string {
	if q.InstanceMemoryLimit == -1 {
		return UnlimitedDisplay
	}
	return formatters.ByteSize(q.InstanceMemoryLimit * formatters.MEGABYTE)
}

func (q SpaceQuota) FormattedAppInstanceLimit() string {
	appInstanceLimit := UnlimitedDisplay
	if q.AppInstanceLimit != -1 { //TODO - figure out how to use resources.UnlimitedAppInstances
		appInstanceLimit = strconv.Itoa(q.AppInstanceLimit)
	}

	return appInstanceLimit
}

func (q SpaceQuota) FormattedServicesLimit() string {
	servicesLimit := UnlimitedDisplay
	if q.ServicesLimit != -1 {
		servicesLimit = strconv.Itoa(q.ServicesLimit)
	}

	return servicesLimit
}

func (q SpaceQuota) FormattedRoutePortsLimit() string {
	reservedRoutePortsLimit := UnlimitedDisplay
	if q.ReservedRoutePortsLimit != "-1" {
		reservedRoutePortsLimit = string(q.ReservedRoutePortsLimit)
	}

	return reservedRoutePortsLimit
}

type SpaceQuotaResponse struct {
	GUID                    string      `json:"guid,omitempty"`
	Name                    string      `json:"name"`
	MemoryLimit             int64       `json:"memory_limit"`          // in Megabytes
	InstanceMemoryLimit     int64       `json:"instance_memory_limit"` // in Megabytes
	RoutesLimit             int         `json:"total_routes"`
	ServicesLimit           int         `json:"total_services"`
	NonBasicServicesAllowed bool        `json:"non_basic_services_allowed"`
	OrgGUID                 string      `json:"organization_guid"`
	AppInstanceLimit        json.Number `json:"app_instance_limit"`
	ReservedRoutePortsLimit json.Number `json:"total_reserved_route_ports"`
}
<commit_msg>Fix UnlimitedDisplay constant<commit_after>package models

import (
	"encoding/json"
	"strconv"

	"github.com/cloudfoundry/cli/cf/formatters"
	. "github.com/cloudfoundry/cli/cf/i18n"
)

type SpaceQuota struct {
	GUID                    string      `json:"guid,omitempty"`
	Name                    string      `json:"name"`
	MemoryLimit             int64       `json:"memory_limit"`          // in Megabytes
	InstanceMemoryLimit     int64       `json:"instance_memory_limit"` // in Megabytes
	RoutesLimit             int         `json:"total_routes"`
	ServicesLimit           int         `json:"total_services"`
	NonBasicServicesAllowed bool        `json:"non_basic_services_allowed"`
	OrgGUID                 string      `json:"organization_guid"`
	AppInstanceLimit        int         `json:"app_instance_limit"`
	ReservedRoutePortsLimit json.Number `json:"total_reserved_route_ports,omitempty"`
}

const UnlimitedDisplay = "unlimited"

func (q SpaceQuota) FormattedMemoryLimit() string {
	return formatters.ByteSize(q.MemoryLimit * formatters.MEGABYTE)
}

func (q SpaceQuota) FormattedInstanceMemoryLimit() string {
	if q.InstanceMemoryLimit == -1 {
		return T(UnlimitedDisplay)
	}
	return formatters.ByteSize(q.InstanceMemoryLimit * formatters.MEGABYTE)
}

func (q SpaceQuota) FormattedAppInstanceLimit() string {
	appInstanceLimit := T(UnlimitedDisplay)
	if q.AppInstanceLimit != -1 { //TODO - figure out how to use resources.UnlimitedAppInstances
		appInstanceLimit = strconv.Itoa(q.AppInstanceLimit)
	}

	return appInstanceLimit
}

func (q SpaceQuota) FormattedServicesLimit() string {
	servicesLimit := T(UnlimitedDisplay)
	if q.ServicesLimit != -1 {
		servicesLimit = strconv.Itoa(q.ServicesLimit)
	}

	return servicesLimit
}

func (q SpaceQuota) FormattedRoutePortsLimit() string {
	reservedRoutePortsLimit := T(UnlimitedDisplay)
	if q.ReservedRoutePortsLimit != "-1" {
		reservedRoutePortsLimit = string(q.ReservedRoutePortsLimit)
	}

	return reservedRoutePortsLimit
}

type SpaceQuotaResponse struct {
	GUID                    string      `json:"guid,omitempty"`
	Name                    string      `json:"name"`
	MemoryLimit             int64       `json:"memory_limit"`          // in Megabytes
	InstanceMemoryLimit     int64       `json:"instance_memory_limit"` // in Megabytes
	RoutesLimit             int         `json:"total_routes"`
	ServicesLimit           int         `json:"total_services"`
	NonBasicServicesAllowed bool        `json:"non_basic_services_allowed"`
	OrgGUID                 string      `json:"organization_guid"`
	AppInstanceLimit        json.Number `json:"app_instance_limit"`
	ReservedRoutePortsLimit json.Number `json:"total_reserved_route_ports"`
}
<|endoftext|>


<commit_before>package types_test

import (
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/dclupgrade/types"
)

func TestGenesisState_Validate(t *testing.T) {
	for _, tc := range []struct {
		desc     string
		genState *types.GenesisState
		valid    bool
	}{
		{
			desc:     "default is valid",
			genState: types.DefaultGenesis(),
			valid:    true,
		},
		{
			desc: "valid genesis state",
			genState: &types.GenesisState{
				ProposedUpgradeList: []types.ProposedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "1",
						},
					},
				},
				ApprovedUpgradeList: []types.ApprovedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "1",
						},
					},
				},
				// this line is used by starport scaffolding # types/genesis/validField
			},
			valid: true,
		},
		{
			desc: "duplicated proposedUpgrade",
			genState: &types.GenesisState{
				ProposedUpgradeList: []types.ProposedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
				},
			},
			valid: false,
		},
		{
			desc: "duplicated approvedUpgrade",
			genState: &types.GenesisState{
				ApprovedUpgradeList: []types.ApprovedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "1",
						},
					},
				},
			},
			valid: false,
		},
		// this line is used by starport scaffolding # types/genesis/testcase
	} {
		t.Run(tc.desc, func(t *testing.T) {
			err := tc.genState.Validate()
			if tc.valid {
				require.NoError(t, err)
			} else {
				require.Error(t, err)
			}
		})
	}
}
<commit_msg>test fixed<commit_after>package types_test

import (
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/dclupgrade/types"
)

func TestGenesisState_Validate(t *testing.T) {
	for _, tc := range []struct {
		desc     string
		genState *types.GenesisState
		valid    bool
	}{
		{
			desc:     "default is valid",
			genState: types.DefaultGenesis(),
			valid:    true,
		},
		{
			desc: "valid genesis state",
			genState: &types.GenesisState{
				ProposedUpgradeList: []types.ProposedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "1",
						},
					},
				},
				ApprovedUpgradeList: []types.ApprovedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "1",
						},
					},
				},
				// this line is used by starport scaffolding # types/genesis/validField
			},
			valid: true,
		},
		{
			desc: "duplicated proposedUpgrade",
			genState: &types.GenesisState{
				ProposedUpgradeList: []types.ProposedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
				},
			},
			valid: false,
		},
		{
			desc: "duplicated approvedUpgrade",
			genState: &types.GenesisState{
				ApprovedUpgradeList: []types.ApprovedUpgrade{
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
					{
						Plan: types.Plan{
							Name: "0",
						},
					},
				},
			},
			valid: false,
		},
		// this line is used by starport scaffolding # types/genesis/testcase
	} {
		t.Run(tc.desc, func(t *testing.T) {
			err := tc.genState.Validate()
			if tc.valid {
				require.NoError(t, err)
			} else {
				require.Error(t, err)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Bertrand Janin <b@janin.com>
// Use of this source code is governed by the ISC license in the LICENSE file.

package main

import (
	"time"
	"strconv"
	"fmt"
)

type SoundBoardModule struct{}

func formatPlayTuneCommand(filename string, duration uint64) string {
	if duration > 0 {
		return fmt.Sprintf("play %s %d", filename, duration)
	} else {
		return fmt.Sprintf("play %s", filename)
	}
}

func (module SoundBoardModule) PrivMsg(msg *PrivMsg) { }

func Play(where string, params []string) {
	var duration uint64 = 0
	var err error

	if len(params) == 0 {
		privMsg(where, "usage: play sound [duration]")
		return
	}

	if len(params) > 1 {
		duration, err = strconv.ParseUint(params[1], 10, 8)
		if err != nil {
			duration = 0
		}
	}

	SendToMinion(formatPlayTuneCommand(params[0], duration))
}

func PlayAfrica(where string, params []string) {
	params = append([]string{"africa.ogg"}, params...)

	Play(where, params)

	go func() {
		time.Sleep(2 * time.Second)
		privAction(where, "hears the drums echoing tonight,")
		time.Sleep(5 * time.Second)
		privMsg(where, "But she hears only whispers of some quiet conversation")
		time.Sleep(9 * time.Second)
		privMsg(where, "She's coming in the 12:30 flight")
		time.Sleep(3 * time.Second)
		privMsg(where, "The moonlit wings reflect the stars that guide me towards salvation")
	}()
}

func PlayJeopardy(where string, params []string) {
	params = append([]string{"jeopardy.mp3"}, params...)

	privAction(where, "queues some elevator music...")
	Play(where, params)
}

func (module SoundBoardModule) Init() {
	RegisterCommand(NewCommandFromFunction("play", Play))
	RegisterCommand(NewCommandFromFunction("africa", PlayAfrica))
	RegisterCommand(NewCommandFromFunction("jeopardy", PlayJeopardy))
}
<commit_msg>Fix hardcoded sounds...<commit_after>// Copyright (c) 2014 Bertrand Janin <b@janin.com>
// Use of this source code is governed by the ISC license in the LICENSE file.

package main

import (
	"time"
	"strconv"
	"fmt"
)

type SoundBoardModule struct{}

func formatPlayTuneCommand(filename string, duration uint64) string {
	if duration > 0 {
		return fmt.Sprintf("play %s %d", filename, duration)
	} else {
		return fmt.Sprintf("play %s", filename)
	}
}

func (module SoundBoardModule) PrivMsg(msg *PrivMsg) { }

func Play(where string, params []string) {
	var duration uint64 = 0
	var err error

	if len(params) == 0 {
		privMsg(where, "usage: play sound [duration]")
		return
	}

	if len(params) > 1 {
		duration, err = strconv.ParseUint(params[1], 10, 8)
		if err != nil {
			duration = 0
		}
	}

	SendToMinion(formatPlayTuneCommand(params[0], duration))
}

func PlayAfrica(where string, params []string) {
	params = append([]string{"tunes/africa.ogg"}, params...)

	Play(where, params)

	go func() {
		time.Sleep(2 * time.Second)
		privAction(where, "hears the drums echoing tonight,")
		time.Sleep(5 * time.Second)
		privMsg(where, "But she hears only whispers of some quiet conversation")
		time.Sleep(9 * time.Second)
		privMsg(where, "She's coming in the 12:30 flight")
		time.Sleep(3 * time.Second)
		privMsg(where, "The moonlit wings reflect the stars that guide me towards salvation")
	}()
}

func PlayJeopardy(where string, params []string) {
	params = append([]string{"tunes/jeopardy.mp3"}, params...)

	privAction(where, "queues some elevator music...")
	Play(where, params)
}

func (module SoundBoardModule) Init() {
	RegisterCommand(NewCommandFromFunction("play", Play))
	RegisterCommand(NewCommandFromFunction("africa", PlayAfrica))
	RegisterCommand(NewCommandFromFunction("jeopardy", PlayJeopardy))
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Pseudo-versions
//
// Code authors are expected to tag the revisions they want users to use,
// including prereleases. However, not all authors tag versions at all,
// and not all commits a user might want to try will have tags.
// A pseudo-version is a version with a special form that allows us to
// address an untagged commit and order that version with respect to
// other versions we might encounter.
//
// A pseudo-version takes one of the general forms:
//
//	(1) vX.0.0-yyyymmddhhmmss-abcdef123456
//	(2) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456
//	(3) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible
//	(4) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456
//	(5) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible
//
// If there is no recently tagged version with the right major version vX,
// then form (1) is used, creating a space of pseudo-versions at the bottom
// of the vX version range, less than any tagged version, including the unlikely v0.0.0.
//
// If the most recent tagged version before the target commit is vX.Y.Z or vX.Y.Z+incompatible,
// then the pseudo-version uses form (2) or (3), making it a prerelease for the next
// possible semantic version after vX.Y.Z. The leading 0 segment in the prerelease string
// ensures that the pseudo-version compares less than possible future explicit prereleases
// like vX.Y.(Z+1)-rc1 or vX.Y.(Z+1)-1.
//
// If the most recent tagged version before the target commit is vX.Y.Z-pre or vX.Y.Z-pre+incompatible,
// then the pseudo-version uses form (4) or (5), making it a slightly later prerelease.

package modfetch

import (
	"cmd/go/internal/semver"
	"fmt"
	"internal/lazyregexp"
	"strings"
	"time"
)

// PseudoVersion returns a pseudo-version for the given major version ("v1")
// preexisting older tagged version ("" or "v1.2.3" or "v1.2.3-pre"), revision time,
// and revision identifier (usually a 12-byte commit hash prefix).
func PseudoVersion(major, older string, t time.Time, rev string) string {
	if major == "" {
		major = "v0"
	}
	major = strings.TrimSuffix(major, "-unstable") // make gopkg.in/macaroon-bakery.v2-unstable use "v2"
	segment := fmt.Sprintf("%s-%s", t.UTC().Format("20060102150405"), rev)
	build := semver.Build(older)
	older = semver.Canonical(older)
	if older == "" {
		return major + ".0.0-" + segment // form (1)
	}
	if semver.Prerelease(older) != "" {
		return older + ".0." + segment + build // form (4), (5)
	}

	// Form (2), (3).
	// Extract patch from vMAJOR.MINOR.PATCH
	v := older[:]
	i := strings.LastIndex(v, ".") + 1
	v, patch := v[:i], v[i:]

	// Increment PATCH by adding 1 to decimal:
	// scan right to left turning 9s to 0s until you find a digit to increment.
	// (Number might exceed int64, but math/big is overkill.)
	digits := []byte(patch)
	for i = len(digits) - 1; i >= 0 && digits[i] == '9'; i-- {
		digits[i] = '0'
	}
	if i >= 0 {
		digits[i]++
	} else {
		// digits is all zeros
		digits[0] = '1'
		digits = append(digits, '0')
	}
	patch = string(digits)

	// Reassemble.
	return v + patch + "-0." + segment + build
}

var pseudoVersionRE = lazyregexp.New(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+incompatible)?$`)

// IsPseudoVersion reports whether v is a pseudo-version.
func IsPseudoVersion(v string) bool {
	return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v)
}

// PseudoVersionTime returns the time stamp of the pseudo-version v.
// It returns an error if v is not a pseudo-version or if the time stamp
// embedded in the pseudo-version is not a valid time.
func PseudoVersionTime(v string) (time.Time, error) {
	timestamp, _, err := parsePseudoVersion(v)
	if err != nil {
		return time.Time{}, err
	}
	t, err := time.Parse("20060102150405", timestamp)
	if err != nil {
		return time.Time{}, fmt.Errorf("pseudo-version with malformed time %s: %q", timestamp, v)
	}
	return t, nil
}

// PseudoVersionRev returns the revision identifier of the pseudo-version v.
// It returns an error if v is not a pseudo-version.
func PseudoVersionRev(v string) (rev string, err error) {
	_, rev, err = parsePseudoVersion(v)
	return
}

func parsePseudoVersion(v string) (timestamp, rev string, err error) {
	if !IsPseudoVersion(v) {
		return "", "", fmt.Errorf("malformed pseudo-version %q", v)
	}
	v = strings.TrimSuffix(v, "+incompatible")
	j := strings.LastIndex(v, "-")
	v, rev = v[:j], v[j+1:]
	i := strings.LastIndex(v, "-")
	if j := strings.LastIndex(v, "."); j > i {
		timestamp = v[j+1:]
	} else {
		timestamp = v[i+1:]
	}
	return timestamp, rev, nil
}
<commit_msg>cmd/go: simplify some modfetch code<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Pseudo-versions
//
// Code authors are expected to tag the revisions they want users to use,
// including prereleases. However, not all authors tag versions at all,
// and not all commits a user might want to try will have tags.
// A pseudo-version is a version with a special form that allows us to
// address an untagged commit and order that version with respect to
// other versions we might encounter.
//
// A pseudo-version takes one of the general forms:
//
//	(1) vX.0.0-yyyymmddhhmmss-abcdef123456
//	(2) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456
//	(3) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible
//	(4) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456
//	(5) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible
//
// If there is no recently tagged version with the right major version vX,
// then form (1) is used, creating a space of pseudo-versions at the bottom
// of the vX version range, less than any tagged version, including the unlikely v0.0.0.
//
// If the most recent tagged version before the target commit is vX.Y.Z or vX.Y.Z+incompatible,
// then the pseudo-version uses form (2) or (3), making it a prerelease for the next
// possible semantic version after vX.Y.Z. The leading 0 segment in the prerelease string
// ensures that the pseudo-version compares less than possible future explicit prereleases
// like vX.Y.(Z+1)-rc1 or vX.Y.(Z+1)-1.
//
// If the most recent tagged version before the target commit is vX.Y.Z-pre or vX.Y.Z-pre+incompatible,
// then the pseudo-version uses form (4) or (5), making it a slightly later prerelease.

package modfetch

import (
	"cmd/go/internal/semver"
	"fmt"
	"internal/lazyregexp"
	"strings"
	"time"
)

// PseudoVersion returns a pseudo-version for the given major version ("v1")
// preexisting older tagged version ("" or "v1.2.3" or "v1.2.3-pre"), revision time,
// and revision identifier (usually a 12-byte commit hash prefix).
func PseudoVersion(major, older string, t time.Time, rev string) string {
	if major == "" {
		major = "v0"
	}
	major = strings.TrimSuffix(major, "-unstable") // make gopkg.in/macaroon-bakery.v2-unstable use "v2"
	segment := fmt.Sprintf("%s-%s", t.UTC().Format("20060102150405"), rev)
	build := semver.Build(older)
	older = semver.Canonical(older)
	if older == "" {
		return major + ".0.0-" + segment // form (1)
	}
	if semver.Prerelease(older) != "" {
		return older + ".0." + segment + build // form (4), (5)
	}

	// Form (2), (3).
	// Extract patch from vMAJOR.MINOR.PATCH
	i := strings.LastIndex(older, ".") + 1
	v, patch := older[:i], older[i:]

	// Increment PATCH by adding 1 to decimal:
	// scan right to left turning 9s to 0s until you find a digit to increment.
	// (Number might exceed int64, but math/big is overkill.)
	digits := []byte(patch)
	for i = len(digits) - 1; i >= 0 && digits[i] == '9'; i-- {
		digits[i] = '0'
	}
	if i >= 0 {
		digits[i]++
	} else {
		// digits is all zeros
		digits[0] = '1'
		digits = append(digits, '0')
	}
	patch = string(digits)

	// Reassemble.
	return v + patch + "-0." + segment + build
}

var pseudoVersionRE = lazyregexp.New(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+incompatible)?$`)

// IsPseudoVersion reports whether v is a pseudo-version.
func IsPseudoVersion(v string) bool {
	return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v)
}

// PseudoVersionTime returns the time stamp of the pseudo-version v.
// It returns an error if v is not a pseudo-version or if the time stamp
// embedded in the pseudo-version is not a valid time.
func PseudoVersionTime(v string) (time.Time, error) {
	timestamp, _, err := parsePseudoVersion(v)
	if err != nil {
		return time.Time{}, err
	}
	t, err := time.Parse("20060102150405", timestamp)
	if err != nil {
		return time.Time{}, fmt.Errorf("pseudo-version with malformed time %s: %q", timestamp, v)
	}
	return t, nil
}

// PseudoVersionRev returns the revision identifier of the pseudo-version v.
// It returns an error if v is not a pseudo-version.
func PseudoVersionRev(v string) (rev string, err error) {
	_, rev, err = parsePseudoVersion(v)
	return
}

func parsePseudoVersion(v string) (timestamp, rev string, err error) {
	if !IsPseudoVersion(v) {
		return "", "", fmt.Errorf("malformed pseudo-version %q", v)
	}
	v = strings.TrimSuffix(v, "+incompatible")
	j := strings.LastIndex(v, "-")
	v, rev = v[:j], v[j+1:]
	i := strings.LastIndex(v, "-")
	if j := strings.LastIndex(v, "."); j > i {
		timestamp = v[j+1:]
	} else {
		timestamp = v[i+1:]
	}
	return timestamp, rev, nil
}
<|endoftext|>


<commit_before>package topology

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"sync"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
)

/*
This package is created to resolve these replica placement issues:
1. growth factor for each replica level, e.g., add 10 volumes for 1 copy, 20 volumes for 2 copies, 30 volumes for 3 copies
2. in time of tight storage, how to reduce replica level
3. optimizing for hot data on faster disk, cold data on cheaper storage,
4. volume allocation for each bucket
*/

type VolumeGrowRequest struct {
	Option *VolumeGrowOption
	Count  int
	ErrCh  chan error
}

type VolumeGrowOption struct {
	Collection         string                        `json:"collection,omitempty"`
	ReplicaPlacement   *super_block.ReplicaPlacement `json:"replication,omitempty"`
	Ttl                *needle.TTL                   `json:"ttl,omitempty"`
	DiskType           types.DiskType                `json:"disk,omitempty"`
	Preallocate        int64                         `json:"preallocate,omitempty"`
	DataCenter         string                        `json:"dataCenter,omitempty"`
	Rack               string                        `json:"rack,omitempty"`
	DataNode           string                        `json:"dataNode,omitempty"`
	MemoryMapMaxSizeMb uint32                        `json:"memoryMapMaxSizeMb,omitempty"`
}

type VolumeGrowth struct {
	accessLock sync.Mutex
}

func (o *VolumeGrowOption) String() string {
	blob, _ := json.Marshal(o)
	return string(blob)
}

func (o *VolumeGrowOption) Threshold() float64 {
	v := util.GetViper()
	return v.GetFloat64("master.volume_growth.threshold")
}

func NewDefaultVolumeGrowth() *VolumeGrowth {
	return &VolumeGrowth{}
}

// one replication type may need rp.GetCopyCount() actual volumes
// given copyCount, how many logical volumes to create
func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) {
	v := util.GetViper()
	switch copyCount {
	case 1:
		count = v.GetInt("master.volume_growth.copy_1")
	case 2:
		count = v.GetInt("master.volume_growth.copy_2")
	case 3:
		count = v.GetInt("master.volume_growth.copy_3")
	default:
		count = v.GetInt("master.volume_growth.copy_other")
	}
	return
}

func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount int) (count int, err error) {
	if targetCount == 0 {
		targetCount = vg.findVolumeCount(option.ReplicaPlacement.GetCopyCount())
	}
	count, err = vg.GrowByCountAndType(grpcDialOption, targetCount, option, topo)
	if count > 0 && count%option.ReplicaPlacement.GetCopyCount() == 0 {
		return count, nil
	}
	return count, err
}
func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount int, option *VolumeGrowOption, topo *Topology) (counter int, err error) {
	vg.accessLock.Lock()
	defer vg.accessLock.Unlock()

	for i := 0; i < targetCount; i++ {
		if c, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil {
			counter += c
		} else {
			glog.V(0).Infof("create %d volume, created %d: %v", targetCount, counter, e)
			return counter, e
		}
	}
	return
}

func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (int, error) {
	servers, e := vg.findEmptySlotsForOneVolume(topo, option)
	if e != nil {
		return 0, e
	}
	vid, raftErr := topo.NextVolumeId()
	if raftErr != nil {
		return 0, raftErr
	}
	err := vg.grow(grpcDialOption, topo, vid, option, servers...)
	return len(servers), err
}

// 1. find the main data node
// 1.1 collect all data nodes that have 1 slots
// 2.2 collect all racks that have rp.SameRackCount+1
// 2.2 collect all data centers that have DiffRackCount+rp.SameRackCount+1
// 2. find rest data nodes
func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *VolumeGrowOption) (servers []*DataNode, err error) {
	//find main datacenter and other data centers
	rp := option.ReplicaPlacement
	mainDataCenter, otherDataCenters, dc_err := topo.PickNodesByWeight(rp.DiffDataCenterCount+1, option, func(node Node) error {
		if option.DataCenter != "" && node.IsDataCenter() && node.Id() != NodeId(option.DataCenter) {
			return fmt.Errorf("Not matching preferred data center:%s", option.DataCenter)
		}
		if len(node.Children()) < rp.DiffRackCount+1 {
			return fmt.Errorf("Only has %d racks, not enough for %d.", len(node.Children()), rp.DiffRackCount+1)
		}
		if node.AvailableSpaceFor(option) < int64(rp.DiffRackCount+rp.SameRackCount+1) {
			return fmt.Errorf("Free:%d < Expected:%d", node.AvailableSpaceFor(option), rp.DiffRackCount+rp.SameRackCount+1)
		}
		possibleRacksCount := 0
		for _, rack := range node.Children() {
			possibleDataNodesCount := 0
			for _, n := range rack.Children() {
				if n.AvailableSpaceFor(option) >= 1 {
					possibleDataNodesCount++
				}
			}
			if possibleDataNodesCount >= rp.SameRackCount+1 {
				possibleRacksCount++
			}
		}
		if possibleRacksCount < rp.DiffRackCount+1 {
			return fmt.Errorf("Only has %d racks with more than %d free data nodes, not enough for %d.", possibleRacksCount, rp.SameRackCount+1, rp.DiffRackCount+1)
		}
		return nil
	})
	if dc_err != nil {
		return nil, dc_err
	}

	//find main rack and other racks
	mainRack, otherRacks, rackErr := mainDataCenter.(*DataCenter).PickNodesByWeight(rp.DiffRackCount+1, option, func(node Node) error {
		if option.Rack != "" && node.IsRack() && node.Id() != NodeId(option.Rack) {
			return fmt.Errorf("Not matching preferred rack:%s", option.Rack)
		}
		if node.AvailableSpaceFor(option) < int64(rp.SameRackCount+1) {
			return fmt.Errorf("Free:%d < Expected:%d", node.AvailableSpaceFor(option), rp.SameRackCount+1)
		}
		if len(node.Children()) < rp.SameRackCount+1 {
			// a bit faster way to test free racks
			return fmt.Errorf("Only has %d data nodes, not enough for %d.", len(node.Children()), rp.SameRackCount+1)
		}
		possibleDataNodesCount := 0
		for _, n := range node.Children() {
			if n.AvailableSpaceFor(option) >= 1 {
				possibleDataNodesCount++
			}
		}
		if possibleDataNodesCount < rp.SameRackCount+1 {
			return fmt.Errorf("Only has %d data nodes with a slot, not enough for %d.", possibleDataNodesCount, rp.SameRackCount+1)
		}
		return nil
	})
	if rackErr != nil {
		return nil, rackErr
	}

	//find main rack and other racks
	mainServer, otherServers, serverErr := mainRack.(*Rack).PickNodesByWeight(rp.SameRackCount+1, option, func(node Node) error {
		if option.DataNode != "" && node.IsDataNode() && node.Id() != NodeId(option.DataNode) {
			return fmt.Errorf("Not matching preferred data node:%s", option.DataNode)
		}
		if node.AvailableSpaceFor(option) < 1 {
			return fmt.Errorf("Free:%d < Expected:%d", node.AvailableSpaceFor(option), 1)
		}
		return nil
	})
	if serverErr != nil {
		return nil, serverErr
	}

	servers = append(servers, mainServer.(*DataNode))
	for _, server := range otherServers {
		servers = append(servers, server.(*DataNode))
	}
	for _, rack := range otherRacks {
		r := rand.Int63n(rack.AvailableSpaceFor(option))
		if server, e := rack.ReserveOneVolume(r, option); e == nil {
			servers = append(servers, server)
		} else {
			return servers, e
		}
	}
	for _, datacenter := range otherDataCenters {
		r := rand.Int63n(datacenter.AvailableSpaceFor(option))
		if server, e := datacenter.ReserveOneVolume(r, option); e == nil {
			servers = append(servers, server)
		} else {
			return servers, e
		}
	}
	return
}

func (vg *VolumeGrowth) grow(grpcDialOption grpc.DialOption, topo *Topology, vid needle.VolumeId, option *VolumeGrowOption, servers ...*DataNode) error {
	for _, server := range servers {
		if err := AllocateVolume(server, grpcDialOption, vid, option); err == nil {
			vi := storage.VolumeInfo{
				Id:               vid,
				Size:             0,
				Collection:       option.Collection,
				ReplicaPlacement: option.ReplicaPlacement,
				Ttl:              option.Ttl,
				Version:          needle.CurrentVersion,
				DiskType:         string(option.DiskType),
			}
			server.AddOrUpdateVolume(vi)
			topo.RegisterVolumeLayout(vi, server)
			glog.V(0).Infoln("Created Volume", vid, "on", server.NodeImpl.String())
		} else {
			glog.V(0).Infoln("Failed to assign volume", vid, "to", servers, "error", err)
			return fmt.Errorf("Failed to assign %d: %v", vid, err)
		}
	}
	return nil
}
<commit_msg>minor<commit_after>package topology

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"sync"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
)

/*
This package is created to resolve these replica placement issues:
1. growth factor for each replica level, e.g., add 10 volumes for 1 copy, 20 volumes for 2 copies, 30 volumes for 3 copies
2. in time of tight storage, how to reduce replica level
3. optimizing for hot data on faster disk, cold data on cheaper storage,
4. volume allocation for each bucket
*/

type VolumeGrowRequest struct {
	Option *VolumeGrowOption
	Count  int
	ErrCh  chan error
}

type VolumeGrowOption struct {
	Collection         string                        `json:"collection,omitempty"`
	ReplicaPlacement   *super_block.ReplicaPlacement `json:"replication,omitempty"`
	Ttl                *needle.TTL                   `json:"ttl,omitempty"`
	DiskType           types.DiskType                `json:"disk,omitempty"`
	Preallocate        int64                         `json:"preallocate,omitempty"`
	DataCenter         string                        `json:"dataCenter,omitempty"`
	Rack               string                        `json:"rack,omitempty"`
	DataNode           string                        `json:"dataNode,omitempty"`
	MemoryMapMaxSizeMb uint32                        `json:"memoryMapMaxSizeMb,omitempty"`
}

type VolumeGrowth struct {
	accessLock sync.Mutex
}

func (o *VolumeGrowOption) String() string {
	blob, _ := json.Marshal(o)
	return string(blob)
}

func (o *VolumeGrowOption) Threshold() float64 {
	v := util.GetViper()
	return v.GetFloat64("master.volume_growth.threshold")
}

func NewDefaultVolumeGrowth() *VolumeGrowth {
	return &VolumeGrowth{}
}

// one replication type may need rp.GetCopyCount() actual volumes
// given copyCount, how many logical volumes to create
func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) {
	v := util.GetViper()
	switch copyCount {
	case 1:
		count = v.GetInt("master.volume_growth.copy_1")
	case 2:
		count = v.GetInt("master.volume_growth.copy_2")
	case 3:
		count = v.GetInt("master.volume_growth.copy_3")
	default:
		count = v.GetInt("master.volume_growth.copy_other")
	}
	return
}

func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount int) (count int, err error) {
	if targetCount == 0 {
		targetCount = vg.findVolumeCount(option.ReplicaPlacement.GetCopyCount())
	}
	count, err = vg.GrowByCountAndType(grpcDialOption, targetCount, option, topo)
	if count > 0 && count%option.ReplicaPlacement.GetCopyCount() == 0 {
		return count, nil
	}
	return count, err
}
func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount int, option *VolumeGrowOption, topo *Topology) (counter int, err error) {
	vg.accessLock.Lock()
	defer vg.accessLock.Unlock()

	for i := 0; i < targetCount; i++ {
		if c, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil {
			counter += c
		} else {
			glog.V(0).Infof("create %d volume, created %d: %v", targetCount, counter, e)
			return counter, e
		}
	}
	return
}

func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (int, error) {
	servers, e := vg.findEmptySlotsForOneVolume(topo, option)
	if e != nil {
		return 0, e
	}
	vid, raftErr := topo.NextVolumeId()
	if raftErr != nil {
		return 0, raftErr
	}
	err := vg.grow(grpcDialOption, topo, vid, option, servers...)
	return len(servers), err
}

// 1. find the main data node
// 1.1 collect all data nodes that have 1 slots
// 2.2 collect all racks that have rp.SameRackCount+1
// 2.2 collect all data centers that have DiffRackCount+rp.SameRackCount+1
// 2. find rest data nodes
func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *VolumeGrowOption) (servers []*DataNode, err error) {
	//find main datacenter and other data centers
	rp := option.ReplicaPlacement
	mainDataCenter, otherDataCenters, dc_err := topo.PickNodesByWeight(rp.DiffDataCenterCount+1, option, func(node Node) error {
		if option.DataCenter != "" && node.IsDataCenter() && node.Id() != NodeId(option.DataCenter) {
			return fmt.Errorf("Not matching preferred data center:%s", option.DataCenter)
		}
		if len(node.Children()) < rp.DiffRackCount+1 {
			return fmt.Errorf("Only has %d racks, not enough for %d.", len(node.Children()), rp.DiffRackCount+1)
		}
		if node.AvailableSpaceFor(option) < int64(rp.DiffRackCount+rp.SameRackCount+1) {
			return fmt.Errorf("Free:%d < Expected:%d", node.AvailableSpaceFor(option), rp.DiffRackCount+rp.SameRackCount+1)
		}
		possibleRacksCount := 0
		for _, rack := range node.Children() {
			possibleDataNodesCount := 0
			for _, n := range rack.Children() {
				if n.AvailableSpaceFor(option) >= 1 {
					possibleDataNodesCount++
				}
			}
			if possibleDataNodesCount >= rp.SameRackCount+1 {
				possibleRacksCount++
			}
		}
		if possibleRacksCount < rp.DiffRackCount+1 {
			return fmt.Errorf("Only has %d racks with more than %d free data nodes, not enough for %d.", possibleRacksCount, rp.SameRackCount+1, rp.DiffRackCount+1)
		}
		return nil
	})
	if dc_err != nil {
		return nil, dc_err
	}

	//find main rack and other racks
	mainRack, otherRacks, rackErr := mainDataCenter.(*DataCenter).PickNodesByWeight(rp.DiffRackCount+1, option, func(node Node) error {
		if option.Rack != "" && node.IsRack() && node.Id() != NodeId(option.Rack) {
			return fmt.Errorf("Not matching preferred rack:%s", option.Rack)
		}
		if node.AvailableSpaceFor(option) < int64(rp.SameRackCount+1) {
			return fmt.Errorf("Free:%d < Expected:%d", node.AvailableSpaceFor(option), rp.SameRackCount+1)
		}
		if len(node.Children()) < rp.SameRackCount+1 {
			// a bit faster way to test free racks
			return fmt.Errorf("Only has %d data nodes, not enough for %d.", len(node.Children()), rp.SameRackCount+1)
		}
		possibleDataNodesCount := 0
		for _, n := range node.Children() {
			if n.AvailableSpaceFor(option) >= 1 {
				possibleDataNodesCount++
			}
		}
		if possibleDataNodesCount < rp.SameRackCount+1 {
			return fmt.Errorf("Only has %d data nodes with a slot, not enough for %d.", possibleDataNodesCount, rp.SameRackCount+1)
		}
		return nil
	})
	if rackErr != nil {
		return nil, rackErr
	}

	//find main rack and other racks
	mainServer, otherServers, serverErr := mainRack.(*Rack).PickNodesByWeight(rp.SameRackCount+1, option, func(node Node) error {
		if option.DataNode != "" && node.IsDataNode() && node.Id() != NodeId(option.DataNode) {
			return fmt.Errorf("Not matching preferred data node:%s", option.DataNode)
		}
		if node.AvailableSpaceFor(option) < 1 {
			return fmt.Errorf("Free:%d < Expected:%d", node.AvailableSpaceFor(option), 1)
		}
		return nil
	})
	if serverErr != nil {
		return nil, serverErr
	}

	servers = append(servers, mainServer.(*DataNode))
	for _, server := range otherServers {
		servers = append(servers, server.(*DataNode))
	}
	for _, rack := range otherRacks {
		r := rand.Int63n(rack.AvailableSpaceFor(option))
		if server, e := rack.ReserveOneVolume(r, option); e == nil {
			servers = append(servers, server)
		} else {
			return servers, e
		}
	}
	for _, datacenter := range otherDataCenters {
		r := rand.Int63n(datacenter.AvailableSpaceFor(option))
		if server, e := datacenter.ReserveOneVolume(r, option); e == nil {
			servers = append(servers, server)
		} else {
			return servers, e
		}
	}
	return
}

func (vg *VolumeGrowth) grow(grpcDialOption grpc.DialOption, topo *Topology, vid needle.VolumeId, option *VolumeGrowOption, servers ...*DataNode) error {
	for _, server := range servers {
		if err := AllocateVolume(server, grpcDialOption, vid, option); err == nil {
			vi := storage.VolumeInfo{
				Id:               vid,
				Size:             0,
				Collection:       option.Collection,
				ReplicaPlacement: option.ReplicaPlacement,
				Ttl:              option.Ttl,
				Version:          needle.CurrentVersion,
				DiskType:         option.DiskType.String(),
			}
			server.AddOrUpdateVolume(vi)
			topo.RegisterVolumeLayout(vi, server)
			glog.V(0).Infoln("Created Volume", vid, "on", server.NodeImpl.String())
		} else {
			glog.V(0).Infoln("Failed to assign volume", vid, "to", servers, "error", err)
			return fmt.Errorf("Failed to assign %d: %v", vid, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package xml

import (
	"errors"
	"github.com/crabmusket/gosunspec"
	"github.com/crabmusket/gosunspec/impl"
	"github.com/crabmusket/gosunspec/models/model1"
	"github.com/crabmusket/gosunspec/smdx"
	"github.com/crabmusket/gosunspec/spi"
	_ "log"
	"strconv"
)

type blockAnchor struct {
	device   *DeviceElement
	model    *ModelElement
	index    int
	detached bool
}

type pointAnchor struct {
	position int // position of the point in the model, -1 if it is not present
}

type xmlDriver struct {
}

var ErrNoSuchElement = errors.New("no such element")
var ErrEmptyValue = errors.New("empty value")
var ErrTooManyErrors = errors.New("too many errors")

// Open open a new sunspec.Array which is populated from the specified
// DataElement.
func Open(e *DataElement) (sunspec.Array, error) {
	arr := impl.NewArray()
	for x, _ := range e.Devices {
		if dev, err := OpenDevice(e.Devices[x]); err != nil {
			return nil, err
		} else if err := arr.AddDevice(dev.(spi.DeviceSPI)); err != nil {
			return nil, err
		}
	}
	return arr, nil
}

// OpenDevice opens a new sunspec.Device which is populated from the specified
// DeviceElement.
func OpenDevice(dx *DeviceElement) (sunspec.Device, error) {
	d := impl.NewDevice()
	d.SetAnchor(dx)
	xp := &xmlDriver{}

	//
	// Special case for optional model 1 element in XML representation
	//
	// The SunSpec Model Data Exchange specification allows the model 1
	// element to be omitted from device documents in certain circumstances.
	//
	// In these cases several of the common model elements might be
	// exchanged instead as attributes of the device element. To allow
	// a generic API to bs used to read and write these points, it
	// is simplest to create a detached ModelElement for this case.
	//
	// We don't link the model element to the parent device element unless
	// an attempt is made to write a point that is not duplicated by
	// a device element attribute.
	//
	// The intent is to provided a consistent API across XML and non-XML
	// drivers and to prevent model 1 elements magically appearing in the
	// XML device element merely because an XML device element has been
	// opened but also to ensure that if a write is made into the model 1
	// element, that write is preserved and visible to the consumers
	// of the XML model.
	//
	foundModel1 := false
	for _, mx := range dx.Models {
		foundModel1 = (sunspec.ModelId(mx.Id) == model1.ModelID)
		if foundModel1 {
			break
		}
	}

	models := dx.Models
	if !foundModel1 {
		models = make([]*ModelElement, len(dx.Models)+1)
		models[0] = &ModelElement{Id: model1.ModelID}
		copy(models[1:], dx.Models)
	}

	// iterate through the model elements, creating one new model for each
	// then create a block for each index and add points for each point element
	for _, mx := range models {
		smdx := smdx.GetModel(uint16(mx.Id))
		if smdx == nil {
			continue
		}
		max := uint32(0)
		for _, px := range mx.Points {
			if px.Index > max {
				max = px.Index
			}
		}
		repeats := int(max)
		if len(smdx.Blocks) == 1 {
			repeats -= 1
		}
		m := impl.NewModel(smdx, repeats, xp)
		if err := d.AddModel(m); err != nil {
			return nil, err
		} else {
			for pi, px := range mx.Points {
				bi := int(px.Index)
				if bi > 0 && len(smdx.Blocks) == 1 {
					bi = bi - 1
				}
				b := m.MustBlock(bi)
				if p, err := b.Point(px.Id); err != nil {
					return nil, err
				} else {
					p.(spi.PointSPI).SetAnchor(&pointAnchor{position: pi})
				}
			}
			bi := 0
			m.Do(spi.WithBlockSPI(func(b spi.BlockSPI) {
				detached := mx.Id == model1.ModelID && !foundModel1
				b.SetAnchor(&blockAnchor{device: dx, model: mx, index: bi, detached: detached})
				b.Do(spi.WithPointSPI(func(p spi.PointSPI) {
					if p.Anchor() == nil {
						p.SetAnchor(&pointAnchor{position: -1})
					}
				}))
				bi++
			}))
		}
	}

	if !foundModel1 {
		syncDeviceElementToModel1(dx, d)
	}

	return d, nil
}

//
// For a fake model1 element, initialise it to be consistent with the device element
//
func syncDeviceElementToModel1(dx *DeviceElement, d sunspec.Device) {
	b := d.MustModel(model1.ModelID).MustBlock(0)
	b.MustPoint(model1.SN).SetStringValue(dx.Serial)
	b.MustPoint(model1.Md).SetStringValue(dx.Model)
	b.MustPoint(model1.Mn).SetStringValue(dx.Manufacturer)
	if err := b.Write(model1.SN, model1.Md, model1.Mn); err != nil {
		// we are not expecting this
		panic(err)
	}
}

// CopyArray copies an existing SunSpec Array into a new SunSpec Array and an
// XML DataElement. Operations on the returned SunSpec Array edit the
// returned DataElement.
func CopyArray(a sunspec.Array) (sunspec.Array, *DataElement) {
	arr := impl.NewArray()
	x := &DataElement{
		Version: "1",
	}

	devices := []spi.DeviceSPI{}
	a.Do(func(d sunspec.Device) {
		da, dx := CopyDevice(d)
		x.Devices = append(x.Devices, dx)
		devices = append(devices, da.(spi.DeviceSPI))
	})
	for _, c := range devices {
		if err := arr.AddDevice(c.(spi.DeviceSPI)); err != nil {
			// we are not expecting this to happen
			panic(err)
		}
	}
	return arr, x
}

// CopyDevice copies an existing SunSpec Device into a new SunSpec Device and an
// XML DeviceElement. Operations on the returned SunSpec Device edit the
// returned DeviceElement.
func CopyDevice(d sunspec.Device) (sunspec.Device, *DeviceElement) {
	dc := impl.NewDevice()
	dx := &DeviceElement{}
	xp := &xmlDriver{}

	modelElements := map[sunspec.ModelId]*ModelElement{}

	newModelElement := func(id sunspec.ModelId) *ModelElement {

		// we only use labels when there is ambiguity

		var mx *ModelElement
		var ok bool
		if mx, ok = modelElements[id]; ok {
			if mx.Index == 0 {
				mx.Index = 1
			}
			mx = &ModelElement{Id: id, Index: mx.Index + 1}
		} else {
			mx = &ModelElement{Id: id, Index: 0}
		}
		modelElements[id] = mx
		return mx
	}

	d.Do(func(m sunspec.Model) {
		smdx := smdx.GetModel(uint16(m.Id()))

		repeatOnly := m.Blocks() > 1 && len(smdx.Blocks) == 1
		mc := impl.NewModel(smdx, m.Blocks()-1, xp)
		mx := newModelElement(m.Id())

		dx.Models = append(dx.Models, mx)

		for i := 0; i < m.Blocks(); i++ {
			b := m.MustBlock(i).(spi.BlockSPI)
			bc := mc.MustBlock(i).(spi.BlockSPI)

			x := i
			if repeatOnly {
				x = x + 1
			}
			anchor := &blockAnchor{
				device: dx,
				model:  mx,
				index:  x,
			}
			bc.SetAnchor(anchor)
			points, _ := bc.Plan()
			for _, pc := range points {
				pa := &pointAnchor{position: -1}
				pc.SetAnchor(pa)
				p := b.MustPoint(pc.Id())
				if p.Error() == nil {
					pc.SetValue(p.Value())
				}
			}

			// write the copied values into the copied block

			if err := bc.Write(); err != nil {
				// not expecting this to happen
				panic(err)
			}
		}
	})
	return dc, dx
}

func (phys *xmlDriver) Read(b spi.BlockSPI, pointIds ...string) error {
	errCount := 0
	ba := b.Anchor().(*blockAnchor)
	if points, err := b.Plan(pointIds...); err != nil {
		return err
	} else {
		for _, p := range points {
			recordError := func(e error) {
				p.SetError(e)
				errCount++
				// log.Printf("point error: id=%s: %v", p.Id(), e)
			}

			pa := p.Anchor().(*pointAnchor)
			if pa.position < 0 {
				recordError(ErrNoSuchElement)
				continue
			}
			px := ba.model.Points[pa.position]
			if len(px.Value) == 0 {
				recordError(ErrEmptyValue)
				continue
			}
			sfp := p.ScaleFactorPoint()
			v := px.Value
			if sfp != nil {
				vsf := sunspec.ScaleFactor(px.ScaleFactor)
				if sfp.Error() != nil {
					sfp.SetScaleFactor(vsf)
				}
				if vi, err := strconv.Atoi(v); err != nil {
					recordError(err)
					continue
				} else {
					for vsf > sfp.ScaleFactor() {
						vi = vi * 10
						vsf--
					}
					for vsf < sfp.ScaleFactor() {
						vi = vi / 10
						vsf++
					}
					v = strconv.Itoa(vi)
				}
			}
			if err := p.UnmarshalXML(v); err != nil {
				recordError(err)
				continue
			}

			// need to values with scaling factors match the scaling factor or else
			// adjust the value to be consistent with the current scaling factor value
		}
	}
	if errCount > 0 {
		return ErrTooManyErrors
	} else {
		return nil
	}
}

func (phys *xmlDriver) Write(b spi.BlockSPI, pointIds ...string) error {

	write := map[string]bool{}
	if len(pointIds) == 0 {
		b.Do(func(p sunspec.Point) {
			write[p.Id()] = true
		})
	} else {
		for _, id := range pointIds {
			write[id] = true
		}
	}

	ba := b.Anchor().(*blockAnchor)
	b.Do(spi.WithPointSPI(func(p spi.PointSPI) {
		if !write[p.Id()] {
			return
		}
		pa := p.Anchor().(*pointAnchor)
		var px *PointElement
		if pa.position == -1 && p.Error() == nil {
			// add a new point to the model element
			px = &PointElement{Id: p.Id(), Index: uint32(ba.index)}
			pa.position = len(ba.model.Points)
			ba.model.Points = append(ba.model.Points, px)
		} else if pa.position >= 0 && p.Error() != nil {
			// remove a point from the model element
			swap := b.MustPoint(ba.model.Points[len(ba.model.Points)-1].Id).(spi.PointSPI).Anchor().(*pointAnchor)
			ba.model.Points[pa.position] = ba.model.Points[swap.position]
			swap.position = pa.position
			ba.model.Points = ba.model.Points[0 : len(ba.model.Points)-1]
			px = nil
		} else if pa.position >= 0 {
			// update the existing point
			px = ba.model.Points[pa.position]
		}
		if px != nil {
			px.Value = p.MarshalXML()
			if sfp := p.ScaleFactorPoint(); sfp != nil {
				px.ScaleFactor = int16(sfp.ScaleFactor())
			}
			// we could put the description and units (from the SMDX model) in here, but
			// we probably need a configuration option to say whether this is necessary
			// or not
		}
	}))

	// A special case required by a partial redundancy of the XML model.
	//
	// Writes into the model 1 points that also appear in the device element
	// as attributes are mirrored into the device element attributes.
	//
	// In all other cases, we need to link a detached model element to the
	// parent so that these updates are visible to consumers of the XML model.
	//
	// See also: OpenDevice

	if ba.model.Id == model1.ModelID {

		count := 0
		for pid, _ := range write {
			switch pid {
			case model1.SN, model1.Md, model1.Mn:
				// synchronize device element with writes into model
				p := b.MustPoint(pid)
				var v string
				if p.Error() == nil {
					v = p.StringValue()
				} else {
					v = ""
				}
				switch pid {
				case model1.SN:
					ba.device.Serial = v
				case model1.Md:
					ba.device.Model = v
				case model1.Mn:
					ba.device.Manufacturer = v
				}
			default:
				// count writes into other points
				count++
			}
		}

		// If there is a write to a point that isn't encoded in the device
		// element and the model 1 element is currently detached from the device
		// (because it wasn't present on open) then fix that by attaching the
		// model element to the parent device now.

		if count > 0 && ba.detached {
			models := make([]*ModelElement, len(ba.device.Models)+1)
			models[0] = ba.model
			copy(models[1:], ba.device.Models)
			ba.detached = false
			ba.device.Models = models
		}
	}

	return nil
}
<commit_msg>xml: Simplify CopyArray implementation.<commit_after>package xml

import (
	"errors"
	"github.com/crabmusket/gosunspec"
	"github.com/crabmusket/gosunspec/impl"
	"github.com/crabmusket/gosunspec/models/model1"
	"github.com/crabmusket/gosunspec/smdx"
	"github.com/crabmusket/gosunspec/spi"
	_ "log"
	"strconv"
)

type blockAnchor struct {
	device   *DeviceElement
	model    *ModelElement
	index    int
	detached bool
}

type pointAnchor struct {
	position int // position of the point in the model, -1 if it is not present
}

type xmlDriver struct {
}

var ErrNoSuchElement = errors.New("no such element")
var ErrEmptyValue = errors.New("empty value")
var ErrTooManyErrors = errors.New("too many errors")

// Open open a new sunspec.Array which is populated from the specified
// DataElement.
func Open(e *DataElement) (sunspec.Array, error) {
	arr := impl.NewArray()
	for x, _ := range e.Devices {
		if dev, err := OpenDevice(e.Devices[x]); err != nil {
			return nil, err
		} else if err := arr.AddDevice(dev.(spi.DeviceSPI)); err != nil {
			return nil, err
		}
	}
	return arr, nil
}

// OpenDevice opens a new sunspec.Device which is populated from the specified
// DeviceElement.
func OpenDevice(dx *DeviceElement) (sunspec.Device, error) {
	d := impl.NewDevice()
	d.SetAnchor(dx)
	xp := &xmlDriver{}

	//
	// Special case for optional model 1 element in XML representation
	//
	// The SunSpec Model Data Exchange specification allows the model 1
	// element to be omitted from device documents in certain circumstances.
	//
	// In these cases several of the common model elements might be
	// exchanged instead as attributes of the device element. To allow
	// a generic API to bs used to read and write these points, it
	// is simplest to create a detached ModelElement for this case.
	//
	// We don't link the model element to the parent device element unless
	// an attempt is made to write a point that is not duplicated by
	// a device element attribute.
	//
	// The intent is to provided a consistent API across XML and non-XML
	// drivers and to prevent model 1 elements magically appearing in the
	// XML device element merely because an XML device element has been
	// opened but also to ensure that if a write is made into the model 1
	// element, that write is preserved and visible to the consumers
	// of the XML model.
	//
	foundModel1 := false
	for _, mx := range dx.Models {
		foundModel1 = (sunspec.ModelId(mx.Id) == model1.ModelID)
		if foundModel1 {
			break
		}
	}

	models := dx.Models
	if !foundModel1 {
		models = make([]*ModelElement, len(dx.Models)+1)
		models[0] = &ModelElement{Id: model1.ModelID}
		copy(models[1:], dx.Models)
	}

	// iterate through the model elements, creating one new model for each
	// then create a block for each index and add points for each point element
	for _, mx := range models {
		smdx := smdx.GetModel(uint16(mx.Id))
		if smdx == nil {
			continue
		}
		max := uint32(0)
		for _, px := range mx.Points {
			if px.Index > max {
				max = px.Index
			}
		}
		repeats := int(max)
		if len(smdx.Blocks) == 1 {
			repeats -= 1
		}
		m := impl.NewModel(smdx, repeats, xp)
		if err := d.AddModel(m); err != nil {
			return nil, err
		} else {
			for pi, px := range mx.Points {
				bi := int(px.Index)
				if bi > 0 && len(smdx.Blocks) == 1 {
					bi = bi - 1
				}
				b := m.MustBlock(bi)
				if p, err := b.Point(px.Id); err != nil {
					return nil, err
				} else {
					p.(spi.PointSPI).SetAnchor(&pointAnchor{position: pi})
				}
			}
			bi := 0
			m.Do(spi.WithBlockSPI(func(b spi.BlockSPI) {
				detached := mx.Id == model1.ModelID && !foundModel1
				b.SetAnchor(&blockAnchor{device: dx, model: mx, index: bi, detached: detached})
				b.Do(spi.WithPointSPI(func(p spi.PointSPI) {
					if p.Anchor() == nil {
						p.SetAnchor(&pointAnchor{position: -1})
					}
				}))
				bi++
			}))
		}
	}

	if !foundModel1 {
		syncDeviceElementToModel1(dx, d)
	}

	return d, nil
}

//
// For a fake model1 element, initialise it to be consistent with the device element
//
func syncDeviceElementToModel1(dx *DeviceElement, d sunspec.Device) {
	b := d.MustModel(model1.ModelID).MustBlock(0)
	b.MustPoint(model1.SN).SetStringValue(dx.Serial)
	b.MustPoint(model1.Md).SetStringValue(dx.Model)
	b.MustPoint(model1.Mn).SetStringValue(dx.Manufacturer)
	if err := b.Write(model1.SN, model1.Md, model1.Mn); err != nil {
		// we are not expecting this
		panic(err)
	}
}

// CopyArray copies an existing SunSpec Array into a new SunSpec Array and an
// XML DataElement. Operations on the returned SunSpec Array edit the
// returned DataElement.
func CopyArray(a sunspec.Array) (sunspec.Array, *DataElement) {
	arr := impl.NewArray()
	x := &DataElement{
		Version: "1",
	}

	a.Do(func(d sunspec.Device) {
		da, dx := CopyDevice(d)
		x.Devices = append(x.Devices, dx)
		if err := arr.AddDevice(da.(spi.DeviceSPI)); err != nil {
			// we are not expecting this to happen
			panic(err)
		}
	})
	return arr, x
}

// CopyDevice copies an existing SunSpec Device into a new SunSpec Device and an
// XML DeviceElement. Operations on the returned SunSpec Device edit the
// returned DeviceElement.
func CopyDevice(d sunspec.Device) (sunspec.Device, *DeviceElement) {
	dc := impl.NewDevice()
	dx := &DeviceElement{}
	xp := &xmlDriver{}

	modelElements := map[sunspec.ModelId]*ModelElement{}

	newModelElement := func(id sunspec.ModelId) *ModelElement {

		// we only use labels when there is ambiguity

		var mx *ModelElement
		var ok bool
		if mx, ok = modelElements[id]; ok {
			if mx.Index == 0 {
				mx.Index = 1
			}
			mx = &ModelElement{Id: id, Index: mx.Index + 1}
		} else {
			mx = &ModelElement{Id: id, Index: 0}
		}
		modelElements[id] = mx
		return mx
	}

	d.Do(func(m sunspec.Model) {
		smdx := smdx.GetModel(uint16(m.Id()))

		repeatOnly := m.Blocks() > 1 && len(smdx.Blocks) == 1
		mc := impl.NewModel(smdx, m.Blocks()-1, xp)
		mx := newModelElement(m.Id())

		dx.Models = append(dx.Models, mx)

		for i := 0; i < m.Blocks(); i++ {
			b := m.MustBlock(i).(spi.BlockSPI)
			bc := mc.MustBlock(i).(spi.BlockSPI)

			x := i
			if repeatOnly {
				x = x + 1
			}
			anchor := &blockAnchor{
				device: dx,
				model:  mx,
				index:  x,
			}
			bc.SetAnchor(anchor)
			points, _ := bc.Plan()
			for _, pc := range points {
				pa := &pointAnchor{position: -1}
				pc.SetAnchor(pa)
				p := b.MustPoint(pc.Id())
				if p.Error() == nil {
					pc.SetValue(p.Value())
				}
			}

			// write the copied values into the copied block

			if err := bc.Write(); err != nil {
				// not expecting this to happen
				panic(err)
			}
		}
	})
	return dc, dx
}

func (phys *xmlDriver) Read(b spi.BlockSPI, pointIds ...string) error {
	errCount := 0
	ba := b.Anchor().(*blockAnchor)
	if points, err := b.Plan(pointIds...); err != nil {
		return err
	} else {
		for _, p := range points {
			recordError := func(e error) {
				p.SetError(e)
				errCount++
				// log.Printf("point error: id=%s: %v", p.Id(), e)
			}

			pa := p.Anchor().(*pointAnchor)
			if pa.position < 0 {
				recordError(ErrNoSuchElement)
				continue
			}
			px := ba.model.Points[pa.position]
			if len(px.Value) == 0 {
				recordError(ErrEmptyValue)
				continue
			}
			sfp := p.ScaleFactorPoint()
			v := px.Value
			if sfp != nil {
				vsf := sunspec.ScaleFactor(px.ScaleFactor)
				if sfp.Error() != nil {
					sfp.SetScaleFactor(vsf)
				}
				if vi, err := strconv.Atoi(v); err != nil {
					recordError(err)
					continue
				} else {
					for vsf > sfp.ScaleFactor() {
						vi = vi * 10
						vsf--
					}
					for vsf < sfp.ScaleFactor() {
						vi = vi / 10
						vsf++
					}
					v = strconv.Itoa(vi)
				}
			}
			if err := p.UnmarshalXML(v); err != nil {
				recordError(err)
				continue
			}

			// need to values with scaling factors match the scaling factor or else
			// adjust the value to be consistent with the current scaling factor value
		}
	}
	if errCount > 0 {
		return ErrTooManyErrors
	} else {
		return nil
	}
}

func (phys *xmlDriver) Write(b spi.BlockSPI, pointIds ...string) error {

	write := map[string]bool{}
	if len(pointIds) == 0 {
		b.Do(func(p sunspec.Point) {
			write[p.Id()] = true
		})
	} else {
		for _, id := range pointIds {
			write[id] = true
		}
	}

	ba := b.Anchor().(*blockAnchor)
	b.Do(spi.WithPointSPI(func(p spi.PointSPI) {
		if !write[p.Id()] {
			return
		}
		pa := p.Anchor().(*pointAnchor)
		var px *PointElement
		if pa.position == -1 && p.Error() == nil {
			// add a new point to the model element
			px = &PointElement{Id: p.Id(), Index: uint32(ba.index)}
			pa.position = len(ba.model.Points)
			ba.model.Points = append(ba.model.Points, px)
		} else if pa.position >= 0 && p.Error() != nil {
			// remove a point from the model element
			swap := b.MustPoint(ba.model.Points[len(ba.model.Points)-1].Id).(spi.PointSPI).Anchor().(*pointAnchor)
			ba.model.Points[pa.position] = ba.model.Points[swap.position]
			swap.position = pa.position
			ba.model.Points = ba.model.Points[0 : len(ba.model.Points)-1]
			px = nil
		} else if pa.position >= 0 {
			// update the existing point
			px = ba.model.Points[pa.position]
		}
		if px != nil {
			px.Value = p.MarshalXML()
			if sfp := p.ScaleFactorPoint(); sfp != nil {
				px.ScaleFactor = int16(sfp.ScaleFactor())
			}
			// we could put the description and units (from the SMDX model) in here, but
			// we probably need a configuration option to say whether this is necessary
			// or not
		}
	}))

	// A special case required by a partial redundancy of the XML model.
	//
	// Writes into the model 1 points that also appear in the device element
	// as attributes are mirrored into the device element attributes.
	//
	// In all other cases, we need to link a detached model element to the
	// parent so that these updates are visible to consumers of the XML model.
	//
	// See also: OpenDevice

	if ba.model.Id == model1.ModelID {

		count := 0
		for pid, _ := range write {
			switch pid {
			case model1.SN, model1.Md, model1.Mn:
				// synchronize device element with writes into model
				p := b.MustPoint(pid)
				var v string
				if p.Error() == nil {
					v = p.StringValue()
				} else {
					v = ""
				}
				switch pid {
				case model1.SN:
					ba.device.Serial = v
				case model1.Md:
					ba.device.Model = v
				case model1.Mn:
					ba.device.Manufacturer = v
				}
			default:
				// count writes into other points
				count++
			}
		}

		// If there is a write to a point that isn't encoded in the device
		// element and the model 1 element is currently detached from the device
		// (because it wasn't present on open) then fix that by attaching the
		// model element to the parent device now.

		if count > 0 && ba.detached {
			models := make([]*ModelElement, len(ba.device.Models)+1)
			models[0] = ba.model
			copy(models[1:], ba.device.Models)
			ba.detached = false
			ba.device.Models = models
		}
	}

	return nil
}
<|endoftext|>


<commit_before>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    netflow_v9.go
//: details: netflow decoders handler
//: author:  Mehrdad Arshad Rad
//: date:    04/21/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package main

import (
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/VerizonDigital/vflow/producer"
)

// NetflowV9 represents netflow v9 collector
type NetflowV9 struct {
	port    int
	addr    string
	workers int
	stop    bool
	stats   NetflowV9Stats
	pool    chan chan struct{}
}

// NetflowV9UDPMsg represents netflow v9 UDP data
type NetflowV9UDPMsg struct {
	raddr *net.UDPAddr
	body  []byte
}

// NetflowV9Stats represents netflow v9 stats
type NetflowV9Stats struct {
	UDPQueue     int
	MessageQueue int
	UDPCount     uint64
	DecodedCount uint64
	MQErrorCount uint64
	Workers      int32
}

var (
	netflowV9UDPCh = make(chan NetflowV9UDPMsg, 1000)
	netflowV9MQCh  = make(chan []byte, 1000)

	// ipfix udp payload pool
	netflowV9Buffer = &sync.Pool{
		New: func() interface{} {
			return make([]byte, opts.NetflowV9UDPSize)
		},
	}
)

// NewNetflowV9 constructs NetflowV9
func NewNetflowV9() *NetflowV9 {
	return &NetflowV9{
		port:    opts.IPFIXPort,
		workers: opts.IPFIXWorkers,
		pool:    make(chan chan struct{}, maxWorkers),
	}
}

func (i *NetflowV9) run() {
	//TODO
	// exit if the ipfix is disabled
	if !opts.NetflowV9Enabled {
		logger.Println("netflowv9 has been disabled")
		return
	}

	hostPort := net.JoinHostPort(i.addr, strconv.Itoa(i.port))
	udpAddr, _ := net.ResolveUDPAddr("udp", hostPort)

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		logger.Fatal(err)
	}

	atomic.AddInt32(&i.stats.Workers, int32(i.workers))
	for n := 0; n < i.workers; n++ {
		go func() {
			wQuit := make(chan struct{})
			i.pool <- wQuit
			i.netflowV9Worker(wQuit)
		}()
	}

	logger.Printf("netflow v9 is running (workers#: %d)", i.workers)

	go func() {
		p := producer.NewProducer(opts.MQName)

		p.MQConfigFile = opts.MQConfigFile
		p.MQErrorCount = &i.stats.MQErrorCount
		p.Logger = logger
		p.Chan = netflowV9MQCh
		p.Topic = opts.NetflowV9Topic

		if err := p.Run(); err != nil {
			logger.Fatal(err)
		}
	}()

	go func() {
		if !opts.DynWorkers {
			logger.Println("netflow v9 dynamic worker disabled")
			return
		}

		i.dynWorkers()
	}()

	for !i.stop {
		b := netflowV9Buffer.Get().([]byte)
		conn.SetReadDeadline(time.Now().Add(1e9))
		n, raddr, err := conn.ReadFromUDP(b)
		if err != nil {
			continue
		}
		atomic.AddUint64(&i.stats.UDPCount, 1)
		netflowV9UDPCh <- NetflowV9UDPMsg{raddr, b[:n]}
	}

}

func (i *NetflowV9) shutdown() {
	//TODO
}

func (i *NetflowV9) netflowV9Worker(wQuit chan struct{}) {
	// TODO
}

func (i *NetflowV9) status() *NetflowV9Stats {
	//TODO
	return &NetflowV9Stats{}
}

func (i *NetflowV9) dynWorkers() {
	//TODO
}
<commit_msg>add shutdown and stats netflow v9<commit_after>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    netflow_v9.go
//: details: netflow decoders handler
//: author:  Mehrdad Arshad Rad
//: date:    04/21/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package main

import (
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/VerizonDigital/vflow/producer"
)

// NetflowV9 represents netflow v9 collector
type NetflowV9 struct {
	port    int
	addr    string
	workers int
	stop    bool
	stats   NetflowV9Stats
	pool    chan chan struct{}
}

// NetflowV9UDPMsg represents netflow v9 UDP data
type NetflowV9UDPMsg struct {
	raddr *net.UDPAddr
	body  []byte
}

// NetflowV9Stats represents netflow v9 stats
type NetflowV9Stats struct {
	UDPQueue     int
	MessageQueue int
	UDPCount     uint64
	DecodedCount uint64
	MQErrorCount uint64
	Workers      int32
}

var (
	netflowV9UDPCh = make(chan NetflowV9UDPMsg, 1000)
	netflowV9MQCh  = make(chan []byte, 1000)

	// ipfix udp payload pool
	netflowV9Buffer = &sync.Pool{
		New: func() interface{} {
			return make([]byte, opts.NetflowV9UDPSize)
		},
	}
)

// NewNetflowV9 constructs NetflowV9
func NewNetflowV9() *NetflowV9 {
	return &NetflowV9{
		port:    opts.IPFIXPort,
		workers: opts.IPFIXWorkers,
		pool:    make(chan chan struct{}, maxWorkers),
	}
}

func (i *NetflowV9) run() {
	//TODO
	// exit if the ipfix is disabled
	if !opts.NetflowV9Enabled {
		logger.Println("netflowv9 has been disabled")
		return
	}

	hostPort := net.JoinHostPort(i.addr, strconv.Itoa(i.port))
	udpAddr, _ := net.ResolveUDPAddr("udp", hostPort)

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		logger.Fatal(err)
	}

	atomic.AddInt32(&i.stats.Workers, int32(i.workers))
	for n := 0; n < i.workers; n++ {
		go func() {
			wQuit := make(chan struct{})
			i.pool <- wQuit
			i.netflowV9Worker(wQuit)
		}()
	}

	logger.Printf("netflow v9 is running (workers#: %d)", i.workers)

	go func() {
		p := producer.NewProducer(opts.MQName)

		p.MQConfigFile = opts.MQConfigFile
		p.MQErrorCount = &i.stats.MQErrorCount
		p.Logger = logger
		p.Chan = netflowV9MQCh
		p.Topic = opts.NetflowV9Topic

		if err := p.Run(); err != nil {
			logger.Fatal(err)
		}
	}()

	go func() {
		if !opts.DynWorkers {
			logger.Println("netflow v9 dynamic worker disabled")
			return
		}

		i.dynWorkers()
	}()

	for !i.stop {
		b := netflowV9Buffer.Get().([]byte)
		conn.SetReadDeadline(time.Now().Add(1e9))
		n, raddr, err := conn.ReadFromUDP(b)
		if err != nil {
			continue
		}
		atomic.AddUint64(&i.stats.UDPCount, 1)
		netflowV9UDPCh <- NetflowV9UDPMsg{raddr, b[:n]}
	}

}

func (i *NetflowV9) shutdown() {
	// exit if the netflow v9 is disabled
	if !opts.NetflowV9Enabled {
		logger.Println("netflow v9 disabled")
		return
	}

	// stop reading from UDP listener
	i.stop = true
	logger.Println("stopping netflow v9 service gracefully ...")
	time.Sleep(1 * time.Second)

	// dump the templates to storage
	if err := mCache.Dump(opts.NetflowV9TplCacheFile); err != nil {
		logger.Println("couldn't not dump template", err)
	}

	// logging and close UDP channel
	logger.Println("netflow v9 has been shutdown")
	close(netflowV9UDPCh)
}

func (i *NetflowV9) netflowV9Worker(wQuit chan struct{}) {
	// TODO
}

func (i *NetflowV9) status() *NetflowV9Stats {
	return &NetflowV9Stats{
		UDPQueue:     len(netflowV9UDPCh),
		MessageQueue: len(netflowV9MQCh),
		UDPCount:     atomic.LoadUint64(&i.stats.UDPCount),
		DecodedCount: atomic.LoadUint64(&i.stats.DecodedCount),
		MQErrorCount: atomic.LoadUint64(&i.stats.MQErrorCount),
		Workers:      atomic.LoadInt32(&i.stats.Workers),
	}

}

func (i *NetflowV9) dynWorkers() {
	//TODO
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientapi "k8s.io/client-go/tools/clientcmd/api"
	"k8s.io/kubernetes/pkg/api"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
)

func buildConfigFromEnvs(masterURL, kubeconfigPath string) (*restclient.Config, error) {
	if kubeconfigPath == "" && masterURL == "" {
		kubeconfig, err := restclient.InClusterConfig()
		if err != nil {
			return nil, err
		}

		return kubeconfig, nil
	}

	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientapi.Cluster{Server: masterURL}}).ClientConfig()
}

func flattenSubsets(subsets []api.EndpointSubset) []string {
	ips := []string{}
	for _, ss := range subsets {
		for _, addr := range ss.Addresses {
			ips = append(ips, fmt.Sprintf(`"%s"`, addr.IP))
		}
	}
	return ips
}

func main() {
	flag.Parse()

	glog.Info("Kubernetes Elasticsearch logging discovery")

	cc, err := buildConfigFromEnvs(os.Getenv("APISERVER_HOST"), os.Getenv("KUBE_CONFIG_FILE"))
	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	client, err := clientset.NewForConfig(cc)

	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	namespace := metav1.NamespaceSystem
	envNamespace := os.Getenv("NAMESPACE")
	if envNamespace != "" {
		if _, err := client.Core().Namespaces().Get(envNamespace, metav1.GetOptions{}); err != nil {
			glog.Fatalf("%s namespace doesn't exist: %v", envNamespace, err)
		}
		namespace = envNamespace
	}

	var elasticsearch *api.Service
	// Look for endpoints associated with the Elasticsearch loggging service.
	// First wait for the service to become available.
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		elasticsearch, err = client.Core().Services(namespace).Get("elasticsearch-logging", metav1.GetOptions{})
		if err == nil {
			break
		}
	}
	// If we did not find an elasticsearch logging service then log a warning
	// and return without adding any unicast hosts.
	if elasticsearch == nil {
		glog.Warningf("Failed to find the elasticsearch-logging service: %v", err)
		return
	}

	var endpoints *api.Endpoints
	addrs := []string{}
	// Wait for some endpoints.
	count := 0
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		endpoints, err = client.Core().Endpoints(namespace).Get("elasticsearch-logging", metav1.GetOptions{})
		if err != nil {
			continue
		}
		addrs = flattenSubsets(endpoints.Subsets)
		glog.Infof("Found %s", addrs)
		if len(addrs) > 0 && len(addrs) == count {
			break
		}
		count = len(addrs)
	}
	// If there was an error finding endpoints then log a warning and quit.
	if err != nil {
		glog.Warningf("Error finding endpoints: %v", err)
		return
	}

	glog.Infof("Endpoints = %s", addrs)
	fmt.Printf("discovery.zen.ping.unicast.hosts: [%s]\n", strings.Join(addrs, ", "))
}
<commit_msg>extracted elasticsearch-logging service name as environment variable ELASTICSEARCH_SERVICE_NAME with fallback on default<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	clientapi "k8s.io/client-go/tools/clientcmd/api"
	"k8s.io/kubernetes/pkg/api"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
)

func buildConfigFromEnvs(masterURL, kubeconfigPath string) (*restclient.Config, error) {
	if kubeconfigPath == "" && masterURL == "" {
		kubeconfig, err := restclient.InClusterConfig()
		if err != nil {
			return nil, err
		}

		return kubeconfig, nil
	}

	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
		&clientcmd.ConfigOverrides{ClusterInfo: clientapi.Cluster{Server: masterURL}}).ClientConfig()
}

func flattenSubsets(subsets []api.EndpointSubset) []string {
	ips := []string{}
	for _, ss := range subsets {
		for _, addr := range ss.Addresses {
			ips = append(ips, fmt.Sprintf(`"%s"`, addr.IP))
		}
	}
	return ips
}

func main() {
	flag.Parse()

	glog.Info("Kubernetes Elasticsearch logging discovery")

	cc, err := buildConfigFromEnvs(os.Getenv("APISERVER_HOST"), os.Getenv("KUBE_CONFIG_FILE"))
	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	client, err := clientset.NewForConfig(cc)

	if err != nil {
		glog.Fatalf("Failed to make client: %v", err)
	}
	namespace := metav1.NamespaceSystem
	envNamespace := os.Getenv("NAMESPACE")
	if envNamespace != "" {
		if _, err := client.Core().Namespaces().Get(envNamespace, metav1.GetOptions{}); err != nil {
			glog.Fatalf("%s namespace doesn't exist: %v", envNamespace, err)
		}
		namespace = envNamespace
	}

	var elasticsearch *api.Service
	serviceName := os.Getenv("ELASTICSEARCH_SERVICE_NAME")
	if serviceName == "" {
		serviceName = "elasticsearch-logging"
	}

	// Look for endpoints associated with the Elasticsearch loggging service.
	// First wait for the service to become available.
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		elasticsearch, err = client.Core().Services(namespace).Get(serviceName, metav1.GetOptions{})
		if err == nil {
			break
		}
	}
	// If we did not find an elasticsearch logging service then log a warning
	// and return without adding any unicast hosts.
	if elasticsearch == nil {
		glog.Warningf("Failed to find the elasticsearch-logging service: %v", err)
		return
	}

	var endpoints *api.Endpoints
	addrs := []string{}
	// Wait for some endpoints.
	count := 0
	for t := time.Now(); time.Since(t) < 5*time.Minute; time.Sleep(10 * time.Second) {
		endpoints, err = client.Core().Endpoints(namespace).Get(serviceName, metav1.GetOptions{})
		if err != nil {
			continue
		}
		addrs = flattenSubsets(endpoints.Subsets)
		glog.Infof("Found %s", addrs)
		if len(addrs) > 0 && len(addrs) == count {
			break
		}
		count = len(addrs)
	}
	// If there was an error finding endpoints then log a warning and quit.
	if err != nil {
		glog.Warningf("Error finding endpoints: %v", err)
		return
	}

	glog.Infof("Endpoints = %s", addrs)
	fmt.Printf("discovery.zen.ping.unicast.hosts: [%s]\n", strings.Join(addrs, ", "))
}
<|endoftext|>


<commit_before>package setup

import (
	"github.com/mholt/caddy/middleware"
	"github.com/mholt/caddy/middleware/headers"
)

// Headers configures a new Headers middleware instance.
func Headers(c *Controller) (middleware.Middleware, error) {
	rules, err := headersParse(c)
	if err != nil {
		return nil, err
	}

	return func(next middleware.Handler) middleware.Handler {
		return headers.Headers{Next: next, Rules: rules}
	}, nil
}

func headersParse(c *Controller) ([]headers.Rule, error) {
	var rules []headers.Rule

	for c.NextLine() {
		var head headers.Rule
		var isNewPattern bool

		if !c.NextArg() {
			return rules, c.ArgErr()
		}
		pattern := c.Val()

		// See if we already have a definition for this URL pattern...
		for _, h := range rules {
			if h.Url == pattern {
				head = h
				break
			}
		}

		// ...otherwise, this is a new pattern
		if head.Url == "" {
			head.Url = pattern
			isNewPattern = true
		}

		for c.NextBlock() {
			// A block of headers was opened...

			h := headers.Header{Name: c.Val()}

			if c.NextArg() {
				h.Value = c.Val()
			}

			head.Headers = append(head.Headers, h)
		}
		if c.NextArg() {
			// ... or single header was defined as an argument instead.

			h := headers.Header{Name: c.Val()}

			h.Value = c.Val()

			if c.NextArg() {
				h.Value = c.Val()
			}

			head.Headers = append(head.Headers, h)
		}

		if isNewPattern {
			rules = append(rules, head)
		} else {
			for i := 0; i < len(rules); i++ {
				if rules[i].Url == pattern {
					rules[i] = head
					break
				}
			}
		}
	}

	return rules, nil
}
<commit_msg>headers: Replaced usage of Url to Path in setup<commit_after>package setup

import (
	"github.com/mholt/caddy/middleware"
	"github.com/mholt/caddy/middleware/headers"
)

// Headers configures a new Headers middleware instance.
func Headers(c *Controller) (middleware.Middleware, error) {
	rules, err := headersParse(c)
	if err != nil {
		return nil, err
	}

	return func(next middleware.Handler) middleware.Handler {
		return headers.Headers{Next: next, Rules: rules}
	}, nil
}

func headersParse(c *Controller) ([]headers.Rule, error) {
	var rules []headers.Rule

	for c.NextLine() {
		var head headers.Rule
		var isNewPattern bool

		if !c.NextArg() {
			return rules, c.ArgErr()
		}
		pattern := c.Val()

		// See if we already have a definition for this Path pattern...
		for _, h := range rules {
			if h.Path == pattern {
				head = h
				break
			}
		}

		// ...otherwise, this is a new pattern
		if head.Path == "" {
			head.Path = pattern
			isNewPattern = true
		}

		for c.NextBlock() {
			// A block of headers was opened...

			h := headers.Header{Name: c.Val()}

			if c.NextArg() {
				h.Value = c.Val()
			}

			head.Headers = append(head.Headers, h)
		}
		if c.NextArg() {
			// ... or single header was defined as an argument instead.

			h := headers.Header{Name: c.Val()}

			h.Value = c.Val()

			if c.NextArg() {
				h.Value = c.Val()
			}

			head.Headers = append(head.Headers, h)
		}

		if isNewPattern {
			rules = append(rules, head)
		} else {
			for i := 0; i < len(rules); i++ {
				if rules[i].Path == pattern {
					rules[i] = head
					break
				}
			}
		}
	}

	return rules, nil
}
<|endoftext|>


<commit_before>package schema

import (
	"github.com/coopernurse/gorp"
	"log"
	"time"
)

func toString(t time.Time) string { return t.Format(time.UnixDate) }
func fromString(st string) time.Time {
	t, err := time.Parse(time.UnixDate, st)

	if err != nil {
		log.Fatal(err)
	}

	return t
}

// A post to be displayed on the blog
type Post struct {
	Id       int64     `db:"id"`      // The indexed ID of the blog post
	Title    string    `db:"title"`   // The title of the blog post
	Author   string    `db:"author"`  // The blog post's author
	Body     string    `db:"body"`    // The body of the blog post
	SWritten string    `db:"written"` // The time the post was written (in string form)
	Written  time.Time `db:"-"`       // The time the post was written (in time form)
}

// gorp hooks
func (this *Post) PostGet(s gorp.SqlExecutor) {
	this.Written = fromString(this.SWritten)
}

func (this *Post) PreInsert(s gorp.SqlExecutor) {
	this.SWritten = toString(this.Written)
}

func (this *Post) PreUpdate(s gorp.SqlExecutor) {
	this.SWritten = toString(this.Written)
}
<commit_msg>Revert "Added some functions to deal with the abstraction of converting a time.Time to a string and back."<commit_after>package schema

import "time"

// A post to be displayed on the blog
type Post struct {
	Id      int64     `db:"id"`      // The indexed ID of the blog post
	Title   string    `db:"title"`   // The title of the blog post
	Author  string    `db:"author"`  // The blog post's author
	Body    string    `db:"body"`    // The body of the blog post
	Written time.Time `db:"written"` // The time the post was written (in UnixNano)
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package video

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/paths"
	"github.com/andreaskoch/allmark2/model"
	"github.com/andreaskoch/allmark2/services/conversion/markdowntohtml/pattern"
	"github.com/andreaskoch/allmark2/services/conversion/markdowntohtml/util"
	"mime"
	"regexp"
	"strings"
)

var (
	// video: [*description text*](*a link to a youtube video or to a video file*)
	markdownPattern = regexp.MustCompile(`video: \[([^\]]+)\]\(([^)]+)\)`)

	// youtube video link pattern
	youTubeVideoPattern = regexp.MustCompile(`http[s]?://www\.youtube\.com/watch\?v=([^&]+)`)

	// vimeo video link pattern
	vimeoVideoPattern = regexp.MustCompile(`http[s]?://vimeo\.com/([\d]+)`)
)

func New(pathProvider paths.Pather, files []*model.File) *VideoExtension {
	return &VideoExtension{
		pathProvider: pathProvider,
		files:        files,
	}
}

type VideoExtension struct {
	pathProvider paths.Pather
	files        []*model.File
}

func (converter *VideoExtension) Convert(markdown string) (convertedContent string, conversionError error) {

	convertedContent = markdown

	for {

		found, matches := pattern.IsMatch(convertedContent, markdownPattern)
		if !found || (found && len(matches) != 3) {
			break
		}

		// parameters
		originalText := strings.TrimSpace(matches[0])
		title := strings.TrimSpace(matches[1])
		path := strings.TrimSpace(matches[2])

		// get the code
		renderedCode := converter.getVideoCode(title, path)

		// replace markdown
		convertedContent = strings.Replace(convertedContent, originalText, renderedCode, 1)

	}

	return convertedContent, nil
}

func (converter *VideoExtension) getMatchingFile(path string) *model.File {
	for _, file := range converter.files {
		if file.Route().IsMatch(path) && util.IsVideoFile(file) {
			return file
		}
	}

	return nil
}

func (converter *VideoExtension) getVideoCode(title, path string) string {

	fallback := util.GetFallbackLink(title, path)

	// check if the supplied path is a link to a video
	isVideoFile, _ := isVideoFileLink(path)
	if !isVideoFile {
		return fallback
	}

	// internal video file
	if isInternalLink(path) {

		if videoFile := converter.getMatchingFile(path); videoFile != nil {

			fmt.Println(videoFile.Route())

			if mimeType, err := util.GetMimeType(videoFile); err == nil {
				filepath := converter.pathProvider.Path(videoFile.Route().Value())
				return renderVideoFileLink(title, filepath, mimeType)
			}

		}

	} else {

		// external: youtube
		if isYouTube, videoId := isYouTubeLink(path); isYouTube {
			return renderYouTubeVideo(title, videoId)
		}

		// external: vimeo
		if isVimeo, videoId := isVimeoLink(path); isVimeo {
			return renderVimeoVideo(title, videoId)
		}

		// external: html5 video file
		if isVideoFile, mimeType := isVideoFileLink(path); isVideoFile {
			return renderVideoFileLink(title, path, mimeType)
		}

	}

	// return the fallback handler
	return fallback
}

func isYouTubeLink(link string) (isYouTubeLink bool, videoId string) {
	if found, matches := pattern.IsMatch(link, youTubeVideoPattern); found && len(matches) == 2 {
		return true, matches[1]
	}

	return false, ""
}

func renderYouTubeVideo(title, videoId string) string {
	return fmt.Sprintf(`

%s

`, videoId, title, title, videoId)
}

func isVimeoLink(link string) (isVimeoLink bool, videoId string) {
	if found, matches := pattern.IsMatch(link, vimeoVideoPattern); found && len(matches) == 2 {
		return true, matches[1]
	}

	return false, ""
}

func renderVimeoVideo(title, videoId string) string {
	return fmt.Sprintf(`

%s

`, videoId, title, title, videoId)
}

func isVideoFileLink(link string) (isVideoFile bool, mimeType string) {

	// abort if the link does not contain a dot
	if !strings.Contains(link, ".") {
		return false, ""
	}

	normalizedLink := strings.ToLower(link)
	fileExtension := normalizedLink[strings.LastIndex(normalizedLink, "."):]
	mimeType = mime.TypeByExtension(fileExtension)

	switch fileExtension {
	case ".mp4", ".ogg", ".ogv", ".webm", ".3gp":
		return true, mimeType
	default:
		return false, ""
	}

	panic("Unreachable")
}

func isInternalLink(link string) bool {
	return !isExternalLink(link)
}

func isExternalLink(link string) bool {
	lowercase := strings.TrimSpace(strings.ToLower(link))
	return strings.HasPrefix("http://", lowercase) || strings.HasPrefix("htts://", lowercase)
}

func renderVideoFileLink(title, link, mimetype string) string {
	return fmt.Sprintf(`

%s

		<video width="560" height="315" controls>
			<source src="%s" type="%s">
		</video>

`, link, title, title, link, mimetype)
}
<commit_msg>removed debug code<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package video

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/paths"
	"github.com/andreaskoch/allmark2/model"
	"github.com/andreaskoch/allmark2/services/conversion/markdowntohtml/pattern"
	"github.com/andreaskoch/allmark2/services/conversion/markdowntohtml/util"
	"mime"
	"regexp"
	"strings"
)

var (
	// video: [*description text*](*a link to a youtube video or to a video file*)
	markdownPattern = regexp.MustCompile(`video: \[([^\]]+)\]\(([^)]+)\)`)

	// youtube video link pattern
	youTubeVideoPattern = regexp.MustCompile(`http[s]?://www\.youtube\.com/watch\?v=([^&]+)`)

	// vimeo video link pattern
	vimeoVideoPattern = regexp.MustCompile(`http[s]?://vimeo\.com/([\d]+)`)
)

func New(pathProvider paths.Pather, files []*model.File) *VideoExtension {
	return &VideoExtension{
		pathProvider: pathProvider,
		files:        files,
	}
}

type VideoExtension struct {
	pathProvider paths.Pather
	files        []*model.File
}

func (converter *VideoExtension) Convert(markdown string) (convertedContent string, conversionError error) {

	convertedContent = markdown

	for {

		found, matches := pattern.IsMatch(convertedContent, markdownPattern)
		if !found || (found && len(matches) != 3) {
			break
		}

		// parameters
		originalText := strings.TrimSpace(matches[0])
		title := strings.TrimSpace(matches[1])
		path := strings.TrimSpace(matches[2])

		// get the code
		renderedCode := converter.getVideoCode(title, path)

		// replace markdown
		convertedContent = strings.Replace(convertedContent, originalText, renderedCode, 1)

	}

	return convertedContent, nil
}

func (converter *VideoExtension) getMatchingFile(path string) *model.File {
	for _, file := range converter.files {
		if file.Route().IsMatch(path) && util.IsVideoFile(file) {
			return file
		}
	}

	return nil
}

func (converter *VideoExtension) getVideoCode(title, path string) string {

	fallback := util.GetFallbackLink(title, path)

	// check if the supplied path is a link to a video
	isVideoFile, _ := isVideoFileLink(path)
	if !isVideoFile {
		return fallback
	}

	// internal video file
	if isInternalLink(path) {

		if videoFile := converter.getMatchingFile(path); videoFile != nil {

			if mimeType, err := util.GetMimeType(videoFile); err == nil {
				filepath := converter.pathProvider.Path(videoFile.Route().Value())
				return renderVideoFileLink(title, filepath, mimeType)
			}

		}

	} else {

		// external: youtube
		if isYouTube, videoId := isYouTubeLink(path); isYouTube {
			return renderYouTubeVideo(title, videoId)
		}

		// external: vimeo
		if isVimeo, videoId := isVimeoLink(path); isVimeo {
			return renderVimeoVideo(title, videoId)
		}

		// external: html5 video file
		if isVideoFile, mimeType := isVideoFileLink(path); isVideoFile {
			return renderVideoFileLink(title, path, mimeType)
		}

	}

	// return the fallback handler
	return fallback
}

func isYouTubeLink(link string) (isYouTubeLink bool, videoId string) {
	if found, matches := pattern.IsMatch(link, youTubeVideoPattern); found && len(matches) == 2 {
		return true, matches[1]
	}

	return false, ""
}

func renderYouTubeVideo(title, videoId string) string {
	return fmt.Sprintf(`

%s

`, videoId, title, title, videoId)
}

func isVimeoLink(link string) (isVimeoLink bool, videoId string) {
	if found, matches := pattern.IsMatch(link, vimeoVideoPattern); found && len(matches) == 2 {
		return true, matches[1]
	}

	return false, ""
}

func renderVimeoVideo(title, videoId string) string {
	return fmt.Sprintf(`

%s

`, videoId, title, title, videoId)
}

func isVideoFileLink(link string) (isVideoFile bool, mimeType string) {

	// abort if the link does not contain a dot
	if !strings.Contains(link, ".") {
		return false, ""
	}

	normalizedLink := strings.ToLower(link)
	fileExtension := normalizedLink[strings.LastIndex(normalizedLink, "."):]
	mimeType = mime.TypeByExtension(fileExtension)

	switch fileExtension {
	case ".mp4", ".ogg", ".ogv", ".webm", ".3gp":
		return true, mimeType
	default:
		return false, ""
	}

	panic("Unreachable")
}

func isInternalLink(link string) bool {
	return !isExternalLink(link)
}

func isExternalLink(link string) bool {
	lowercase := strings.TrimSpace(strings.ToLower(link))
	return strings.HasPrefix("http://", lowercase) || strings.HasPrefix("htts://", lowercase)
}

func renderVideoFileLink(title, link, mimetype string) string {
	return fmt.Sprintf(`

%s

		<video width="560" height="315" controls>
			<source src="%s" type="%s">
		</video>

`, link, title, title, link, mimetype)
}
<|endoftext|>