<commit_before>package models

import (
	"errors"
	"time"
)

type ActualLRPsByIndex map[int]ActualLRP

type ActualLRPsByProcessGuidAndIndex map[string]ActualLRPsByIndex

func (set ActualLRPsByProcessGuidAndIndex) Add(actual ActualLRP) {
	actuals, found := set[actual.ProcessGuid]
	if !found {
		actuals = ActualLRPsByIndex{}
		set[actual.ProcessGuid] = actuals
	}

	actuals[actual.Index] = actual
}

func (set ActualLRPsByProcessGuidAndIndex) Each(predicate func(actual ActualLRP)) {
	for _, indexSet := range set {
		for _, actual := range indexSet {
			predicate(actual)
		}
	}
}

type ActualLRPState string

const (
	ActualLRPStateUnclaimed ActualLRPState = "UNCLAIMED"
	ActualLRPStateClaimed   ActualLRPState = "CLAIMED"
	ActualLRPStateRunning   ActualLRPState = "RUNNING"
	ActualLRPStateCrashed   ActualLRPState = "CRASHED"
)

var ActualLRPStates = []ActualLRPState{
	ActualLRPStateUnclaimed,
	ActualLRPStateClaimed,
	ActualLRPStateRunning,
	ActualLRPStateCrashed,
}

type ActualLRPKey struct {
	ProcessGuid string `json:"process_guid"`
	Index       int    `json:"index"`
	Domain      string `json:"domain"`
}

func NewActualLRPKey(processGuid string, index int, domain string) ActualLRPKey {
	return ActualLRPKey{
		ProcessGuid: processGuid,
		Index:       index,
		Domain:      domain,
	}
}

func (key ActualLRPKey) Validate() error {
	var validationError ValidationError

	if key.ProcessGuid == "" {
		validationError = validationError.Append(ErrInvalidField{"process_guid"})
	}

	if key.Index < 0 {
		validationError = validationError.Append(ErrInvalidField{"index"})
	}

	if key.Domain == "" {
		validationError = validationError.Append(ErrInvalidField{"domain"})
	}

	if !validationError.Empty() {
		return validationError
	}

	return nil
}

type ActualLRPContainerKey struct {
	InstanceGuid string `json:"instance_guid"`
	CellID       string `json:"cell_id"`
}

var emptyActualLRPContainerKey = ActualLRPContainerKey{}

func (key *ActualLRPContainerKey) Empty() bool {
	return *key == emptyActualLRPContainerKey
}

func NewActualLRPContainerKey(instanceGuid string, cellID string) ActualLRPContainerKey {
	return ActualLRPContainerKey{
		InstanceGuid: instanceGuid,
		CellID:       cellID,
	}
}

func (key ActualLRPContainerKey) Validate() error {
	var validationError ValidationError

	if key.CellID == "" {
		validationError = validationError.Append(ErrInvalidField{"cell_id"})
	}

	if key.InstanceGuid == "" {
		validationError = validationError.Append(ErrInvalidField{"instance_guid"})
	}

	if !validationError.Empty() {
		return validationError
	}

	return nil
}

type ActualLRPNetInfo struct {
	Address string        `json:"address"`
	Ports   []PortMapping `json:"ports"`
}

func NewActualLRPNetInfo(address string, ports []PortMapping) ActualLRPNetInfo {
	return ActualLRPNetInfo{
		Address: address,
		Ports:   ports,
	}
}

func (info *ActualLRPNetInfo) Empty() bool {
	return info.Address == "" && len(info.Ports) == 0
}

func (key ActualLRPNetInfo) Validate() error {
	var validationError ValidationError

	if key.Address == "" {
		return validationError.Append(ErrInvalidField{"address"})
	}

	return nil
}

const MaxCrashBackoff = 16 * time.Minute

type ActualLRPCrashInfo struct {
	CrashCount    int   `json:"crash_count"`
	LastCrashedAt int64 `json:"last_crashed_at"`
}

func NewActualLRPCrashInfo(crashCount int, lastCrashedAt int64) ActualLRPCrashInfo {
	return ActualLRPCrashInfo{
		CrashCount:    crashCount,
		LastCrashedAt: lastCrashedAt,
	}
}

func (crashInfo ActualLRPCrashInfo) ShouldRestartImmediately() bool {
	return crashInfo.CrashCount <= 3
}

func powerOfTwo(pow int) int64 {
	if pow < 0 {
		panic("pow cannot be negative")
	}
	return 1 << uint(pow)
}

type ActualLRP struct {
	ActualLRPKey
	ActualLRPContainerKey
	ActualLRPNetInfo
	ActualLRPCrashInfo
	State ActualLRPState `json:"state"`
	Since int64          `json:"since"`
}

type ActualLRPChange struct {
	Before ActualLRP
	After  ActualLRP
}

const StaleUnclaimedActualLRPDuration = 30 * time.Second
func (actual ActualLRP) ShouldStartUnclaimed(now time.Time) bool {
	if actual.State != ActualLRPStateUnclaimed {
		return false
	}

	if now.Sub(time.Unix(0, actual.Since)) > StaleUnclaimedActualLRPDuration {
		return true
	}

	return false
}

func (actual ActualLRP) CellIsMissing(cellSet CellSet) bool {
	if actual.State == ActualLRPStateUnclaimed ||
		actual.State == ActualLRPStateCrashed {
		return false
	}

	return !cellSet.HasCellID(actual.CellID)
}

func (actual ActualLRP) ShouldRestartCrash(now time.Time) bool {
	if actual.State != ActualLRPStateCrashed {
		return false
	}

	lastCrashedAt := actual.ActualLRPCrashInfo.LastCrashedAt
	count := actual.ActualLRPCrashInfo.CrashCount
	switch {
	case count < 3:
		return true

	case count < 8:
		threshhold := lastCrashedAt + (30*time.Second.Nanoseconds())*powerOfTwo(count-3)
		if threshhold <= now.UnixNano() {
			return true
		}

	case count < 200:
		threshhold := lastCrashedAt + MaxCrashBackoff.Nanoseconds()
		if threshhold <= now.UnixNano() {
			return true
		}
	}

	return false
}

func (before ActualLRP) AllowsTransitionTo(lrpKey ActualLRPKey, containerKey ActualLRPContainerKey, newState ActualLRPState) bool {
	if before.ActualLRPKey != lrpKey {
		return false
	}

	if before.State == ActualLRPStateClaimed && newState == ActualLRPStateRunning {
		return true
	}

	if (before.State == ActualLRPStateClaimed || before.State == ActualLRPStateRunning) &&
		(newState == ActualLRPStateClaimed || newState == ActualLRPStateRunning) &&
		(before.ActualLRPContainerKey != containerKey) {
		return false
	}

	return true
}

func (actual ActualLRP) Validate() error {
	var validationError ValidationError

	err := actual.ActualLRPKey.Validate()
	if err != nil {
		validationError = validationError.Append(err)
	}

	if actual.Since == 0 {
		validationError = validationError.Append(ErrInvalidField{"since"})
	}

	switch actual.State {
	case ActualLRPStateUnclaimed:
		if !actual.ActualLRPContainerKey.Empty() {
			validationError = validationError.Append(errors.New("container key cannot be set when state is unclaimed"))
		}
		if !actual.ActualLRPNetInfo.Empty() {
			validationError = validationError.Append(errors.New("net info cannot be set when state is unclaimed"))
		}

	case ActualLRPStateClaimed:
		if err := actual.ActualLRPContainerKey.Validate(); err != nil {
			validationError = validationError.Append(err)
		}
		if !actual.ActualLRPNetInfo.Empty() {
			validationError = validationError.Append(errors.New("net info cannot be set when state is claimed"))
		}

	case ActualLRPStateRunning:
		if err := actual.ActualLRPContainerKey.Validate(); err != nil {
			validationError = validationError.Append(err)
		}
		if err := actual.ActualLRPNetInfo.Validate(); err != nil {
			validationError = validationError.Append(err)
		}

	case ActualLRPStateCrashed:
		if !actual.ActualLRPContainerKey.Empty() {
			validationError = validationError.Append(errors.New("container key cannot be set when state is crashed"))
		}
		if !actual.ActualLRPNetInfo.Empty() {
			validationError = validationError.Append(errors.New("net info cannot be set when state is crashed"))
		}

	default:
		validationError = validationError.Append(ErrInvalidField{"state"})
	}

	if !validationError.Empty() {
		return validationError
	}

	return nil
}
<commit_msg>dried up crash count restart logic<commit_after>package models

import (
	"errors"
	"time"
)

type ActualLRPsByIndex map[int]ActualLRP

type ActualLRPsByProcessGuidAndIndex map[string]ActualLRPsByIndex

func (set ActualLRPsByProcessGuidAndIndex) Add(actual ActualLRP) {
	actuals, found := set[actual.ProcessGuid]
	if !found {
		actuals = ActualLRPsByIndex{}
		set[actual.ProcessGuid] = actuals
	}

	actuals[actual.Index] = actual
}

func (set ActualLRPsByProcessGuidAndIndex) Each(predicate func(actual ActualLRP)) {
	for _, indexSet := range set {
		for _, actual := range indexSet {
			predicate(actual)
		}
	}
}

type ActualLRPState string

const (
	ActualLRPStateUnclaimed ActualLRPState = "UNCLAIMED"
	ActualLRPStateClaimed   ActualLRPState = "CLAIMED"
	ActualLRPStateRunning   ActualLRPState = "RUNNING"
	ActualLRPStateCrashed   ActualLRPState = "CRASHED"
)

var ActualLRPStates = []ActualLRPState{
	ActualLRPStateUnclaimed,
	ActualLRPStateClaimed,
	ActualLRPStateRunning,
	ActualLRPStateCrashed,
}

type ActualLRPKey struct {
	ProcessGuid string `json:"process_guid"`
	Index       int    `json:"index"`
	Domain      string `json:"domain"`
}

func NewActualLRPKey(processGuid string, index int, domain string) ActualLRPKey {
	return ActualLRPKey{
		ProcessGuid: processGuid,
		Index:       index,
		Domain:      domain,
	}
}

func (key ActualLRPKey) Validate() error {
	var validationError ValidationError

	if key.ProcessGuid == "" {
		validationError = validationError.Append(ErrInvalidField{"process_guid"})
	}

	if key.Index < 0 {
		validationError = validationError.Append(ErrInvalidField{"index"})
	}

	if key.Domain == "" {
		validationError = validationError.Append(ErrInvalidField{"domain"})
	}

	if !validationError.Empty() {
		return validationError
	}

	return nil
}

type ActualLRPContainerKey struct {
	InstanceGuid string `json:"instance_guid"`
	CellID       string `json:"cell_id"`
}

var emptyActualLRPContainerKey = ActualLRPContainerKey{}

func (key *ActualLRPContainerKey) Empty() bool {
	return *key == emptyActualLRPContainerKey
}

func NewActualLRPContainerKey(instanceGuid string, cellID string) ActualLRPContainerKey {
	return ActualLRPContainerKey{
		InstanceGuid: instanceGuid,
		CellID:       cellID,
	}
}

func (key ActualLRPContainerKey) Validate() error {
	var validationError ValidationError

	if key.CellID == "" {
		validationError = validationError.Append(ErrInvalidField{"cell_id"})
	}

	if key.InstanceGuid == "" {
		validationError = validationError.Append(ErrInvalidField{"instance_guid"})
	}

	if !validationError.Empty() {
		return validationError
	}

	return nil
}

type ActualLRPNetInfo struct {
	Address string        `json:"address"`
	Ports   []PortMapping `json:"ports"`
}

func NewActualLRPNetInfo(address string, ports []PortMapping) ActualLRPNetInfo {
	return ActualLRPNetInfo{
		Address: address,
		Ports:   ports,
	}
}

func (info *ActualLRPNetInfo) Empty() bool {
	return info.Address == "" && len(info.Ports) == 0
}

func (key ActualLRPNetInfo) Validate() error {
	var validationError ValidationError

	if key.Address == "" {
		return validationError.Append(ErrInvalidField{"address"})
	}

	return nil
}

const MaxCrashBackoff = 16 * time.Minute

type ActualLRPCrashInfo struct {
	CrashCount    int   `json:"crash_count"`
	LastCrashedAt int64 `json:"last_crashed_at"`
}

func NewActualLRPCrashInfo(crashCount int, lastCrashedAt int64) ActualLRPCrashInfo {
	return ActualLRPCrashInfo{
		CrashCount:    crashCount,
		LastCrashedAt: lastCrashedAt,
	}
}

func (crashInfo ActualLRPCrashInfo) ShouldRestartImmediately() bool {
	return crashInfo.CrashCount < CrashImmediateRestartThreshold
}

type ActualLRP struct {
	ActualLRPKey
	ActualLRPContainerKey
	ActualLRPNetInfo
	ActualLRPCrashInfo
	State ActualLRPState `json:"state"`
	Since int64          `json:"since"`
}

type ActualLRPChange struct {
	Before ActualLRP
	After  ActualLRP
}

const StaleUnclaimedActualLRPDuration = 30 * time.Second

func (actual ActualLRP) ShouldStartUnclaimed(now time.Time) bool {
	if actual.State != ActualLRPStateUnclaimed {
		return false
	}

	if now.Sub(time.Unix(0, actual.Since)) > StaleUnclaimedActualLRPDuration {
		return true
	}

	return false
}

func (actual ActualLRP) CellIsMissing(cellSet CellSet) bool {
	if actual.State == ActualLRPStateUnclaimed ||
		actual.State == ActualLRPStateCrashed {
		return false
	}

	return !cellSet.HasCellID(actual.CellID)
}

func (actual ActualLRP) ShouldRestartCrash(now time.Time) bool {
	if actual.State != ActualLRPStateCrashed {
		return false
	}

	lastCrashedAt := actual.ActualLRPCrashInfo.LastCrashedAt

	switch {
	case actual.ShouldRestartImmediately():
		return true

	case actual.CrashCount < 8:
		nextRestartTime := lastCrashedAt + exponentialBackoff(actual.CrashCount)
		if nextRestartTime <= now.UnixNano() {
			return true
		}

	case actual.CrashCount < 200:
		threshhold := lastCrashedAt + MaxCrashBackoff.Nanoseconds()
		if threshhold <= now.UnixNano() {
			return true
		}
	}

	return false
}

const CrashImmediateRestartThreshold = 3
const CrashBackoffBaseDuration = 30 * time.Second

func exponentialBackoff(crashCount int) int64 {
	pow := crashCount - CrashImmediateRestartThreshold
	return CrashBackoffBaseDuration.Nanoseconds() * powerOfTwo(pow)
}

func powerOfTwo(pow int) int64 {
	if pow < 0 {
		panic("pow cannot be negative")
	}
	return 1 << uint(pow)
}

func (before ActualLRP) AllowsTransitionTo(lrpKey ActualLRPKey, containerKey ActualLRPContainerKey, newState ActualLRPState) bool {
	if before.ActualLRPKey != lrpKey {
		return false
	}

	if before.State == ActualLRPStateClaimed && newState == ActualLRPStateRunning {
		return true
	}

	if (before.State == ActualLRPStateClaimed || before.State == ActualLRPStateRunning) &&
		(newState == ActualLRPStateClaimed || newState == ActualLRPStateRunning) &&
		(before.ActualLRPContainerKey != containerKey) {
		return false
	}

	return true
}

func (actual ActualLRP) Validate() error {
	var validationError ValidationError

	err := actual.ActualLRPKey.Validate()
	if err != nil {
		validationError = validationError.Append(err)
	}

	if actual.Since == 0 {
		validationError = validationError.Append(ErrInvalidField{"since"})
	}

	switch actual.State {
	case ActualLRPStateUnclaimed:
		if !actual.ActualLRPContainerKey.Empty() {
			validationError = validationError.Append(errors.New("container key cannot be set when state is unclaimed"))
		}
		if !actual.ActualLRPNetInfo.Empty() {
			validationError = validationError.Append(errors.New("net info cannot be set when state is unclaimed"))
		}

	case ActualLRPStateClaimed:
		if err := actual.ActualLRPContainerKey.Validate(); err != nil {
			validationError = validationError.Append(err)
		}
		if !actual.ActualLRPNetInfo.Empty() {
			validationError = validationError.Append(errors.New("net info cannot be set when state is claimed"))
		}

	case ActualLRPStateRunning:
		if err := actual.ActualLRPContainerKey.Validate(); err != nil {
			validationError = validationError.Append(err)
		}
		if err := actual.ActualLRPNetInfo.Validate(); err != nil {
			validationError = validationError.Append(err)
		}

	case ActualLRPStateCrashed:
		if !actual.ActualLRPContainerKey.Empty() {
			validationError = validationError.Append(errors.New("container key cannot be set when state is crashed"))
		}
		if !actual.ActualLRPNetInfo.Empty() {
			validationError = validationError.Append(errors.New("net info cannot be set when state is crashed"))
		}

	default:
		validationError = validationError.Append(ErrInvalidField{"state"})
	}

	if !validationError.Empty() {
		return validationError
	}

	return nil
}
<|endoftext|>


<commit_before>package main

const tkStmtLoadChecks = `
SELECT check_id,
       bucket_id,
       source_check_id,
       source_object_type,
       source_object_id,
       configuration_id,
       capability_id,
       object_id,
       object_type
FROM   soma.checks
WHERE  repository_id = $1::uuid
AND    check_id = source_check_id
AND    source_object_type = $2::varchar;`

const tkStmtLoadInheritedChecks = `
SELECT check_id,
       object_id,
       object_type
FROM   soma.checks
WHERE  repository_id = $1::uuid
AND    source_check_id = $2::uuid
AND    source_check_id != check_id;`

const tkStmtLoadChecksForType = `
SELECT check_id,
       object_id
FROM   soma.checks
WHERE  repository_id = $1::uuid
AND    object_type = $2::varchar;`

const tkStmtLoadCheckConfiguration = `
SELECT bucket_id,
       configuration_name,
       configuration_object,
       configuration_object_type,
       configuration_active,
       inheritance_enabled,
       children_only,
       capability_id,
       interval,
       enabled,
       external_id
FROM   soma.check_configurations
WHERE  configuration_id = $1::uuid
AND    repository_id = $2::uuid;`

const tkStmtLoadCheckThresholds = `
SELECT sct.predicate,
       sct.threshold,
       snl.level_name,
       snl.level_shortname,
       snl.level_numeric
FROM   soma.configuration_thresholds sct
JOIN   soma.notification_levels snl
ON     sct.notification_level = snl.level_name
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintCustom = `
SELECT sccp.custom_property_id,
       scp.custom_property,
       sccp.property_value
FROM   soma.constraints_custom_property sccp
JOIN   soma.custom_properties scp
ON     sccp.custom_property_id = scp.custom_property_id
AND    sccp.repository_id = scp.repository_id
WHERE  configuration_id = $1::uuid;`

// do not get distracted by the squirrels! All constraint
// statements are constructed to use three result variables,
// so they can be loaded in one unified loop.
const tkStmtLoadCheckConstraintNative = `
SELECT native_property,
       property_value,
       'squirrel'
FROM   soma.constraints_native_property
WHERE  configuration_id = $1::uuid;`

// return configuration id: every constraint query has 2 columns
const tkStmtLoadCheckConstraintOncall = `
SELECT scop.oncall_duty_id,
       oncall_duty_name,
       oncall_duty_phone_number
FROM   soma.constraints_oncall_property scop
JOIN   inventory.oncall_duty_teams iodt
ON     scop.oncall_duty_id = iodt.oncall_duty_id
WHERE  scop.configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintAttribute = `
SELECT service_property_attribute,
       attribute_value,
       'squirrel'
FROM   soma.constraints_service_attribute
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintService = `
SELECT organizational_team_id,
       service_property,
       'squirrel'
FROM   soma.constraints_service_property
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintSystem = `
SELECT system_property,
       property_value,
       'squirrel'
FROM   soma.constraints_system_property
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckInstances = `
SELECT check_instance_id,
       check_configuration_id
FROM   soma.check_instances
WHERE  check_id = $1::uuid
AND    NOT deleted;`

// load the most recent configuration for this instance, which is
// not always the current one, since a newer version could be blocked
// by the current versions rollout
const tkStmtLoadCheckInstanceConfiguration = `
SELECT check_instance_config_id,
       version,
       monitoring_id,
       constraint_hash,
       constraint_val_hash,
       instance_service,
       instance_service_cfg_hash,
       instance_service_cfg
FROM   soma.check_instance_configurations
WHERE  check_instance_id = $1::uuid
ORDER  BY created DESC
LIMIT  1;`

const tkStmtLoadCheckGroupState = `
SELECT sg.group_id,
       sg.object_state
FROM   soma.buckets sb
JOIN   soma.groups  sg
ON     sb.bucket_id = sg.bucket_id
WHERE  sb.repository_id = $1::uuid;`

const tkStmtLoadCheckGroupRelations = `
SELECT sgmg.group_id,
       sgmg.child_group_id
FROM   soma.buckets sb
JOIN   soma.group_membership_groups sgmg
ON     sb.bucket_id = sgmg.bucket_id
WHERE  sb.repository_id = $1::uuid;`

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<commit_msg>Do not load deleted check data on startup<commit_after>package main

const tkStmtLoadChecks = `
SELECT check_id,
       bucket_id,
       source_check_id,
       source_object_type,
       source_object_id,
       configuration_id,
       capability_id,
       object_id,
       object_type
FROM   soma.checks
WHERE  repository_id = $1::uuid
AND    check_id = source_check_id
AND    source_object_type = $2::varchar
AND    NOT deleted;`

const tkStmtLoadInheritedChecks = `
SELECT check_id,
       object_id,
       object_type
FROM   soma.checks
WHERE  repository_id = $1::uuid
AND    source_check_id = $2::uuid
AND    source_check_id != check_id
AND    NOT deleted;`

const tkStmtLoadChecksForType = `
SELECT check_id,
       object_id
FROM   soma.checks
WHERE  repository_id = $1::uuid
AND    object_type = $2::varchar
AND    NOT deleted;`

const tkStmtLoadCheckConfiguration = `
SELECT bucket_id,
       configuration_name,
       configuration_object,
       configuration_object_type,
       configuration_active,
       inheritance_enabled,
       children_only,
       capability_id,
       interval,
       enabled,
       external_id
FROM   soma.check_configurations
WHERE  configuration_id = $1::uuid
AND    repository_id = $2::uuid
AND    NOT deleted;`

const tkStmtLoadCheckThresholds = `
SELECT sct.predicate,
       sct.threshold,
       snl.level_name,
       snl.level_shortname,
       snl.level_numeric
FROM   soma.configuration_thresholds sct
JOIN   soma.notification_levels snl
ON     sct.notification_level = snl.level_name
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintCustom = `
SELECT sccp.custom_property_id,
       scp.custom_property,
       sccp.property_value
FROM   soma.constraints_custom_property sccp
JOIN   soma.custom_properties scp
ON     sccp.custom_property_id = scp.custom_property_id
AND    sccp.repository_id = scp.repository_id
WHERE  configuration_id = $1::uuid;`

// do not get distracted by the squirrels! All constraint
// statements are constructed to use three result variables,
// so they can be loaded in one unified loop.
const tkStmtLoadCheckConstraintNative = `
SELECT native_property,
       property_value,
       'squirrel'
FROM   soma.constraints_native_property
WHERE  configuration_id = $1::uuid;`

// return configuration id: every constraint query has 2 columns
const tkStmtLoadCheckConstraintOncall = `
SELECT scop.oncall_duty_id,
       oncall_duty_name,
       oncall_duty_phone_number
FROM   soma.constraints_oncall_property scop
JOIN   inventory.oncall_duty_teams iodt
ON     scop.oncall_duty_id = iodt.oncall_duty_id
WHERE  scop.configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintAttribute = `
SELECT service_property_attribute,
       attribute_value,
       'squirrel'
FROM   soma.constraints_service_attribute
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintService = `
SELECT organizational_team_id,
       service_property,
       'squirrel'
FROM   soma.constraints_service_property
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckConstraintSystem = `
SELECT system_property,
       property_value,
       'squirrel'
FROM   soma.constraints_system_property
WHERE  configuration_id = $1::uuid;`

const tkStmtLoadCheckInstances = `
SELECT check_instance_id,
       check_configuration_id
FROM   soma.check_instances
WHERE  check_id = $1::uuid
AND    NOT deleted;`

// load the most recent configuration for this instance, which is
// not always the current one, since a newer version could be blocked
// by the current versions rollout
const tkStmtLoadCheckInstanceConfiguration = `
SELECT check_instance_config_id,
       version,
       monitoring_id,
       constraint_hash,
       constraint_val_hash,
       instance_service,
       instance_service_cfg_hash,
       instance_service_cfg
FROM   soma.check_instance_configurations
WHERE  check_instance_id = $1::uuid
ORDER  BY created DESC
LIMIT  1;`

const tkStmtLoadCheckGroupState = `
SELECT sg.group_id,
       sg.object_state
FROM   soma.buckets sb
JOIN   soma.groups  sg
ON     sb.bucket_id = sg.bucket_id
WHERE  sb.repository_id = $1::uuid;`

const tkStmtLoadCheckGroupRelations = `
SELECT sgmg.group_id,
       sgmg.child_group_id
FROM   soma.buckets sb
JOIN   soma.group_membership_groups sgmg
ON     sb.bucket_id = sgmg.bucket_id
WHERE  sb.repository_id = $1::uuid;`

// vim: ts=4 sw=4 sts=4 noet fenc=utf-8 ffs=unix
<|endoftext|>


<commit_before>package menu

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strings"

	"github.com/fatih/color"
	"github.com/miquella/ask"
	"golang.org/x/crypto/ssh"

	"github.com/miquella/vaulted/lib"
)

type SSHKeyMenu struct {
	*Menu
}

func (m *SSHKeyMenu) Help() {
	menuColor.Set()
	defer color.Unset()

	fmt.Println("a,add      - Add")
	fmt.Println("D,delete   - Delete")
	fmt.Println("g,generate - Generate Key")
	fmt.Println("v          - HashiCorp Vault Signing URL")
	fmt.Println("u,users    - HashiCorp Vault User Principals")
	fmt.Println("E          - Expose External SSH Agent")
	fmt.Println("?,help     - Help")
	fmt.Println("b,back     - Back")
	fmt.Println("q,quit     - Quit")
}

func (m *SSHKeyMenu) Handler() error {
	for {
		var err error
		m.Printer()
		input, err := interaction.ReadMenu("Edit ssh keys: [a,D,g,v,u,E,b]: ")
		if err != nil {
			return err
		}
		switch input {
		case "a", "add", "key", "keys":
			err = m.AddSSHKey()
		case "D", "delete", "remove":
			var key string
			key, err = interaction.ReadValue("Key: ")
			if err == nil {
				if _, exists := m.Vault.SSHKeys[key]; exists {
					delete(m.Vault.SSHKeys, key)
				} else {
					color.Red("Key '%s' not found", key)
				}
			}
		case "g", "generate":
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			m.Vault.SSHOptions.GenerateRSAKey = !m.Vault.SSHOptions.GenerateRSAKey
		case "v":
			signingUrl, err := interaction.ReadValue("HashiCorp Vault signing URL: ")
			if err != nil {
				return err
			}
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			m.Vault.SSHOptions.VaultSigningUrl = signingUrl

			if signingUrl != "" && !m.Vault.SSHOptions.GenerateRSAKey && len(m.Vault.SSHKeys) == 0 {
				generateKey, _ := interaction.ReadValue("Would you like to enable RSA key generation (y/n): ")
				if generateKey == "y" {
					m.Vault.SSHOptions.GenerateRSAKey = true
				}
			}
		case "u", "users":
			userPrincipals, err := interaction.ReadValue("HashiCorp Vault user principals (comma separated): ")
			if err != nil {
				return err
			}
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			if userPrincipals != "" {
				m.Vault.SSHOptions.ValidPrincipals = strings.Split(userPrincipals, ",")
			} else {
				m.Vault.SSHOptions.ValidPrincipals = []string{}
			}
		case "E":
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			m.Vault.SSHOptions.DisableProxy = !m.Vault.SSHOptions.DisableProxy
		case "b", "back":
			return nil
		case "q", "quit", "exit":
			var confirm string
			confirm, err = interaction.ReadValue("Are you sure you wish to save and exit the vault? (y/n): ")
			if err == nil {
				if confirm == "y" {
					return ErrSaveAndExit
				}
			}
		case "?", "help":
			m.Help()
		default:
			color.Red("Command not recognized")
		}

		if err != nil {
			return err
		}
	}
}

func (m *SSHKeyMenu) AddSSHKey() error {
	var err error

	homeDir := ""
	user, err := user.Current()
	if err == nil {
		homeDir = user.HomeDir
	} else {
		homeDir = os.Getenv("HOME")
	}

	defaultFilename := ""
	filename := ""
	if homeDir != "" {
		defaultFilename = filepath.Join(homeDir, ".ssh", "id_rsa")
		filename, err = interaction.ReadValue(fmt.Sprintf("Key file (default: %s): ", defaultFilename))
		if err != nil {
			return err
		}
		if filename == "" {
			filename = defaultFilename
		}
		if !filepath.IsAbs(filename) {
			filename = filepath.Join(filepath.Join(homeDir, ".ssh"), filename)
		}
	} else {
		filename, err = interaction.ReadValue("Key file: ")
		if err != nil {
			return err
		}
	}

	decryptedBlock, err := loadAndDecryptKey(filename)
	if err != nil {
		color.Red("%v", err)
		return nil
	}

	comment := loadPublicKeyComment(filename + ".pub")
	var name string
	if comment != "" {
		name, err = interaction.ReadValue(fmt.Sprintf("Name (default: %s): ", comment))
		if err != nil {
			return err
		}
		if name == "" {
			name = comment
		}
	} else {
		name, err = interaction.ReadValue("Name: ")
		if err != nil {
			return err
		}
		if name == "" {
			name = filename
		}
	}

	if m.Vault.SSHKeys == nil {
		m.Vault.SSHKeys = make(map[string]string)
	}
	m.Vault.SSHKeys[name] = string(pem.EncodeToMemory(decryptedBlock))

	return nil
}

func loadAndDecryptKey(filename string) (*pem.Block, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	data, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	block, _ := pem.Decode(data)
	if block == nil {
		return nil, err
	}

	if x509.IsEncryptedPEMBlock(block) {
		var passphrase string
		var decryptedBytes []byte
		for i := 0; i < 3; i++ {
			passphrase, err = ask.HiddenAsk("Passphrase: ")
			if err != nil {
				return nil, err
			}

			decryptedBytes, err = x509.DecryptPEMBlock(block, []byte(passphrase))
			if err == nil {
				break
			}
			if err != x509.IncorrectPasswordError {
				return nil, err
			}
		}

		if err != nil {
			return nil, err
		}

		return &pem.Block{
			Type:  block.Type,
			Bytes: decryptedBytes,
		}, nil
	}
	return block, nil
}

func loadPublicKeyComment(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		return ""
	}
	defer f.Close()

	data, err := ioutil.ReadAll(f)
	if err != nil {
		return ""
	}

	_, comment, _, _, err := ssh.ParseAuthorizedKey(data)
	if err != nil {
		return ""
	}
	return comment
}

func (m *SSHKeyMenu) Printer() {
	color.Cyan("\nSSH Agent:")
	color.Cyan("  Keys:")
	if len(m.Vault.SSHKeys) > 0 || m.Vault.SSHOptions != nil && m.Vault.SSHOptions.GenerateRSAKey {
		keys := []string{}
		for key := range m.Vault.SSHKeys {
			keys = append(keys, key)
		}
		sort.Strings(keys)

		for _, key := range keys {
			green.Printf("    %s\n", key)
		}

		if m.Vault.SSHOptions != nil && m.Vault.SSHOptions.GenerateRSAKey {
			faintColor.Print("    <generated RSA key>\n")
		}
	} else {
		fmt.Println("    [Empty]")
	}

	if m.Vault.SSHOptions != nil {
		if m.Vault.SSHOptions.VaultSigningUrl != "" || len(m.Vault.SSHOptions.ValidPrincipals) > 0 {
			color.Cyan("\n  Signing (HashiCorp Vault):")
			if m.Vault.SSHOptions.VaultSigningUrl != "" {
				green.Printf("    URL: ")
				fmt.Printf("%s\n", m.Vault.SSHOptions.VaultSigningUrl)
			}

			if len(m.Vault.SSHOptions.ValidPrincipals) > 0 {
				green.Printf("    User(s): ")
				fmt.Printf("%s\n", strings.Join(m.Vault.SSHOptions.ValidPrincipals, ", "))
			}
		}

		cyan.Print("\n  Expose external SSH agent: ")
		fmt.Printf("%t\n", !m.Vault.SSHOptions.DisableProxy)
	}
}
<commit_msg>update key generation prompt wording<commit_after>package menu

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strings"

	"github.com/fatih/color"
	"github.com/miquella/ask"
	"golang.org/x/crypto/ssh"

	"github.com/miquella/vaulted/lib"
)

type SSHKeyMenu struct {
	*Menu
}

func (m *SSHKeyMenu) Help() {
	menuColor.Set()
	defer color.Unset()

	fmt.Println("a,add      - Add")
	fmt.Println("D,delete   - Delete")
	fmt.Println("g,generate - Generate Key")
	fmt.Println("v          - HashiCorp Vault Signing URL")
	fmt.Println("u,users    - HashiCorp Vault User Principals")
	fmt.Println("E          - Expose External SSH Agent")
	fmt.Println("?,help     - Help")
	fmt.Println("b,back     - Back")
	fmt.Println("q,quit     - Quit")
}

func (m *SSHKeyMenu) Handler() error {
	for {
		var err error
		m.Printer()
		input, err := interaction.ReadMenu("Edit ssh keys: [a,D,g,v,u,E,b]: ")
		if err != nil {
			return err
		}
		switch input {
		case "a", "add", "key", "keys":
			err = m.AddSSHKey()
		case "D", "delete", "remove":
			var key string
			key, err = interaction.ReadValue("Key: ")
			if err == nil {
				if _, exists := m.Vault.SSHKeys[key]; exists {
					delete(m.Vault.SSHKeys, key)
				} else {
					color.Red("Key '%s' not found", key)
				}
			}
		case "g", "generate":
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			m.Vault.SSHOptions.GenerateRSAKey = !m.Vault.SSHOptions.GenerateRSAKey
		case "v":
			signingUrl, err := interaction.ReadValue("HashiCorp Vault signing URL: ")
			if err != nil {
				return err
			}
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			m.Vault.SSHOptions.VaultSigningUrl = signingUrl

			if signingUrl != "" && !m.Vault.SSHOptions.GenerateRSAKey && len(m.Vault.SSHKeys) == 0 {
				generateKey, _ := interaction.ReadValue("Would you like to enable per-session, dynamic RSA key generation (y/n): ")
				if generateKey == "y" {
					m.Vault.SSHOptions.GenerateRSAKey = true
				}
			}
		case "u", "users":
			userPrincipals, err := interaction.ReadValue("HashiCorp Vault user principals (comma separated): ")
			if err != nil {
				return err
			}
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			if userPrincipals != "" {
				m.Vault.SSHOptions.ValidPrincipals = strings.Split(userPrincipals, ",")
			} else {
				m.Vault.SSHOptions.ValidPrincipals = []string{}
			}
		case "E":
			if m.Vault.SSHOptions == nil {
				m.Vault.SSHOptions = &vaulted.SSHOptions{}
			}
			m.Vault.SSHOptions.DisableProxy = !m.Vault.SSHOptions.DisableProxy
		case "b", "back":
			return nil
		case "q", "quit", "exit":
			var confirm string
			confirm, err = interaction.ReadValue("Are you sure you wish to save and exit the vault? (y/n): ")
			if err == nil {
				if confirm == "y" {
					return ErrSaveAndExit
				}
			}
		case "?", "help":
			m.Help()
		default:
			color.Red("Command not recognized")
		}

		if err != nil {
			return err
		}
	}
}

func (m *SSHKeyMenu) AddSSHKey() error {
	var err error

	homeDir := ""
	user, err := user.Current()
	if err == nil {
		homeDir = user.HomeDir
	} else {
		homeDir = os.Getenv("HOME")
	}

	defaultFilename := ""
	filename := ""
	if homeDir != "" {
		defaultFilename = filepath.Join(homeDir, ".ssh", "id_rsa")
		filename, err = interaction.ReadValue(fmt.Sprintf("Key file (default: %s): ", defaultFilename))
		if err != nil {
			return err
		}
		if filename == "" {
			filename = defaultFilename
		}
		if !filepath.IsAbs(filename) {
			filename = filepath.Join(filepath.Join(homeDir, ".ssh"), filename)
		}
	} else {
		filename, err = interaction.ReadValue("Key file: ")
		if err != nil {
			return err
		}
	}

	decryptedBlock, err := loadAndDecryptKey(filename)
	if err != nil {
		color.Red("%v", err)
		return nil
	}

	comment := loadPublicKeyComment(filename + ".pub")
	var name string
	if comment != "" {
		name, err = interaction.ReadValue(fmt.Sprintf("Name (default: %s): ", comment))
		if err != nil {
			return err
		}
		if name == "" {
			name = comment
		}
	} else {
		name, err = interaction.ReadValue("Name: ")
		if err != nil {
			return err
		}
		if name == "" {
			name = filename
		}
	}

	if m.Vault.SSHKeys == nil {
		m.Vault.SSHKeys = make(map[string]string)
	}
	m.Vault.SSHKeys[name] = string(pem.EncodeToMemory(decryptedBlock))

	return nil
}

func loadAndDecryptKey(filename string) (*pem.Block, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	data, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	block, _ := pem.Decode(data)
	if block == nil {
		return nil, err
	}

	if x509.IsEncryptedPEMBlock(block) {
		var passphrase string
		var decryptedBytes []byte
		for i := 0; i < 3; i++ {
			passphrase, err = ask.HiddenAsk("Passphrase: ")
			if err != nil {
				return nil, err
			}

			decryptedBytes, err = x509.DecryptPEMBlock(block, []byte(passphrase))
			if err == nil {
				break
			}
			if err != x509.IncorrectPasswordError {
				return nil, err
			}
		}

		if err != nil {
			return nil, err
		}

		return &pem.Block{
			Type:  block.Type,
			Bytes: decryptedBytes,
		}, nil
	}
	return block, nil
}

func loadPublicKeyComment(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		return ""
	}
	defer f.Close()

	data, err := ioutil.ReadAll(f)
	if err != nil {
		return ""
	}

	_, comment, _, _, err := ssh.ParseAuthorizedKey(data)
	if err != nil {
		return ""
	}
	return comment
}

func (m *SSHKeyMenu) Printer() {
	color.Cyan("\nSSH Agent:")
	color.Cyan("  Keys:")
	if len(m.Vault.SSHKeys) > 0 || m.Vault.SSHOptions != nil && m.Vault.SSHOptions.GenerateRSAKey {
		keys := []string{}
		for key := range m.Vault.SSHKeys {
			keys = append(keys, key)
		}
		sort.Strings(keys)

		for _, key := range keys {
			green.Printf("    %s\n", key)
		}

		if m.Vault.SSHOptions != nil && m.Vault.SSHOptions.GenerateRSAKey {
			faintColor.Print("    <generated RSA key>\n")
		}
	} else {
		fmt.Println("    [Empty]")
	}

	if m.Vault.SSHOptions != nil {
		if m.Vault.SSHOptions.VaultSigningUrl != "" || len(m.Vault.SSHOptions.ValidPrincipals) > 0 {
			color.Cyan("\n  Signing (HashiCorp Vault):")
			if m.Vault.SSHOptions.VaultSigningUrl != "" {
				green.Printf("    URL: ")
				fmt.Printf("%s\n", m.Vault.SSHOptions.VaultSigningUrl)
			}

			if len(m.Vault.SSHOptions.ValidPrincipals) > 0 {
				green.Printf("    User(s): ")
				fmt.Printf("%s\n", strings.Join(m.Vault.SSHOptions.ValidPrincipals, ", "))
			}
		}

		cyan.Print("\n  Expose external SSH agent: ")
		fmt.Printf("%t\n", !m.Vault.SSHOptions.DisableProxy)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package outboundttl

import (
	"strings"
	"time"

	"github.com/yarpc/yarpc-go/crossdock-go"
	"github.com/yarpc/yarpc-go/crossdock/client/rpc"
	"github.com/yarpc/yarpc-go/encoding/raw"
	"github.com/yarpc/yarpc-go/transport"

	"golang.org/x/net/context"
)

// Run tests the behavior of any outbound transport to verify that it will
// timeout if the server fails to respond in a timely fashion.
func Run(t crossdock.T) {
	assert := crossdock.Assert(t)
	fatals := crossdock.Fatals(t)

	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()

	rpc := rpc.Create(t)
	ch := raw.New(rpc.Channel("yarpc-test"))
	_, _, err := ch.Call(&raw.ReqMeta{
		Context:   ctx,
		Procedure: "sleep/raw",
	}, nil)
	fatals.Error(err, "expected a failure for timeout")

	if err, ok := err.(transport.BadRequestError); !ok {
		t.Skipf("sleep/raw method not implemented: %v", err)
		return
	}

	form := strings.HasPrefix(err.Error(), `timeout for procedure "sleep/raw" of service "yarpc-test" after`)
	assert.True(form, "error message has expected prefix for timeouts, got %q", err.Error())
	_, ok := err.(transport.TimeoutError)
	assert.True(ok, "error should be a TimeoutError, got %T", err)
}
<commit_msg>Fix logic for identifying unimplemented outboundttl test subject (#204)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package outboundttl

import (
	"strings"
	"time"

	"github.com/yarpc/yarpc-go/crossdock-go"
	"github.com/yarpc/yarpc-go/crossdock/client/rpc"
	"github.com/yarpc/yarpc-go/encoding/raw"
	"github.com/yarpc/yarpc-go/transport"

	"golang.org/x/net/context"
)

// Run tests the behavior of any outbound transport to verify that it will
// timeout if the server fails to respond in a timely fashion.
func Run(t crossdock.T) {
	assert := crossdock.Assert(t)
	fatals := crossdock.Fatals(t)

	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()

	rpc := rpc.Create(t)
	ch := raw.New(rpc.Channel("yarpc-test"))
	_, _, err := ch.Call(&raw.ReqMeta{
		Context:   ctx,
		Procedure: "sleep/raw",
	}, nil)
	fatals.Error(err, "expected a failure for timeout")

	if err, ok := err.(transport.BadRequestError); ok {
		t.Skipf("sleep/raw method not implemented: %v", err)
		return
	}

	form := strings.HasPrefix(err.Error(), `timeout for procedure "sleep/raw" of service "yarpc-test" after`)
	assert.True(form, "error message has expected prefix for timeouts, got %q", err.Error())
	_, ok := err.(transport.TimeoutError)
	assert.True(ok, "error should be a TimeoutError, got %T", err)
}
<|endoftext|>


<commit_before>package unit

import (
	"bytes"
	"database/sql/driver"
	"errors"
	"strings"
	"sync"
)

var (
	mux                 *sync.RWMutex = new(sync.RWMutex)
	currencyMappingFunc func(string) string
	currencyMapping     map[string]string = nil
	currencyUnMapping   map[string]string = nil
)

func SetCurrencyMappingFunc(f func(string) string) {
	currencyMappingFunc = f
}

func SetCurrencyMapping(s, mapping string) {
	if len(s) != 0 && len(mapping) != 0 && s != mapping {
		mux.Lock()
		if currencyMapping == nil {
			currencyMapping = make(map[string]string)
		}
		currencyMapping[s] = mapping
		if currencyUnMapping == nil {
			currencyUnMapping = make(map[string]string)
		}
		currencyUnMapping[mapping] = s
		mux.Unlock()
	}
}

func CurrencyMapping(s string) string {
	if len(s) != 0 {
		if currencyMappingFunc != nil {
			return currencyMappingFunc(s)
		} else if currencyMapping != nil {
			mux.RLock()
			mapping, ok := currencyMapping[s]
			mux.RUnlock()
			if ok {
				return mapping
			}
		}
	}
	return s
}

func CurrencyUnMapping(mapping string) string {
	if len(mapping) != 0 {
		if currencyMappingFunc != nil {
			return currencyMappingFunc(mapping)
		} else if currencyMapping != nil {
			mux.RLock()
			s, ok := currencyUnMapping[mapping]
			mux.RUnlock()
			if ok {
				return s
			}
		}
	}
	return mapping
}

type Currency string

func NewCurrency(s string) Currency {
	return Currency(CurrencyMapping(s))
}

func (c *Currency) MarshalJSON() ([]byte, error) {
	if c == nil {
		return nil, errors.New("MarshalJSON on nil pointer")
	}
	var b bytes.Buffer
	b.WriteByte('"')
	b.WriteString(CurrencyUnMapping(c.String()))
	b.WriteByte('"')
	return b.Bytes(), nil
}

func (c *Currency) UnmarshalJSON(data []byte) error {
	if c == nil {
		return errors.New("UnmarshalJSON on nil pointer")
	}
	c.SetString(strings.Replace(string(data), `"`, ``, -1))
	return nil
}

func (c Currency) Value() (driver.Value, error) {
	return c.String(), nil
}

func (c *Currency) Scan(src interface{}) error {
	if src == nil {
		return nil
	}
	switch o := src.(type) {
	case string:
		c.SetString(o)
	case []byte:
		c.SetString(string(o))
	default:
		return errors.New("Incompatible type for Currency")
	}
	return nil
}

func (c Currency) String() string {
	return string(c)
}

func (c *Currency) SetString(s string) *Currency {
	*c = NewCurrency(s)
	return c
}

func (c *Currency) Equal(s *Currency) bool {
	return c.String() == s.String()
}
<commit_msg>feat: unit currency equal<commit_after>package unit

import (
	"bytes"
	"database/sql/driver"
	"errors"
	"strings"
	"sync"
)

var (
	mux                 *sync.RWMutex = new(sync.RWMutex)
	currencyMappingFunc func(string) string
	currencyMapping     map[string]string = nil
	currencyUnMapping   map[string]string = nil
)

func SetCurrencyMappingFunc(f func(string) string) {
	currencyMappingFunc = f
}

func SetCurrencyMapping(s, mapping string) {
	if len(s) != 0 && len(mapping) != 0 && s != mapping {
		mux.Lock()
		if currencyMapping == nil {
			currencyMapping = make(map[string]string)
		}
		currencyMapping[s] = mapping
		if currencyUnMapping == nil {
			currencyUnMapping = make(map[string]string)
		}
		currencyUnMapping[mapping] = s
		mux.Unlock()
	}
}

func CurrencyMapping(s string) string {
	if len(s) != 0 {
		if currencyMappingFunc != nil {
			return currencyMappingFunc(s)
		} else if currencyMapping != nil {
			mux.RLock()
			mapping, ok := currencyMapping[s]
			mux.RUnlock()
			if ok {
				return mapping
			}
		}
	}
	return s
}

func CurrencyUnMapping(mapping string) string {
	if len(mapping) != 0 {
		if currencyMappingFunc != nil {
			return currencyMappingFunc(mapping)
		} else if currencyMapping != nil {
			mux.RLock()
			s, ok := currencyUnMapping[mapping]
			mux.RUnlock()
			if ok {
				return s
			}
		}
	}
	return mapping
}

type Currency string

func NewCurrency(s string) Currency {
	return Currency(CurrencyMapping(s))
}

func (c *Currency) MarshalJSON() ([]byte, error) {
	if c == nil {
		return nil, errors.New("MarshalJSON on nil pointer")
	}
	var b bytes.Buffer
	b.WriteByte('"')
	b.WriteString(CurrencyUnMapping(c.String()))
	b.WriteByte('"')
	return b.Bytes(), nil
}

func (c *Currency) UnmarshalJSON(data []byte) error {
	if c == nil {
		return errors.New("UnmarshalJSON on nil pointer")
	}
	c.SetString(strings.Replace(string(data), `"`, ``, -1))
	return nil
}

func (c Currency) Value() (driver.Value, error) {
	return c.String(), nil
}

func (c *Currency) Scan(src interface{}) error {
	if src == nil {
		return nil
	}
	switch o := src.(type) {
	case string:
		c.SetString(o)
	case []byte:
		c.SetString(string(o))
	default:
		return errors.New("Incompatible type for Currency")
	}
	return nil
}

func (c Currency) String() string {
	return string(c)
}

func (c *Currency) SetString(s string) *Currency {
	*c = NewCurrency(s)
	return c
}

func (c *Currency) Equal(s *Currency) bool {
	return c.String() == s.String()
}

func (c *Currency) EqualString(s string) bool {
	return c.String() == s
}
<|endoftext|>


<commit_before>package upload

import (
	"crypto/md5"
	"fmt"
	"io"
	"net/http"
	"os"

	"github.com/subutai-io/base/agent/log"
)

var (
	path = "/tmp/"
)

func Page(repo string) string {
	return `

  <title>Go upload</title>

  <form action="/` + repo + `/upload" method="post" enctype="multipart/form-data">
  Filename:
  <input type="file" name="file" id="file">
  <input type="submit" name="submit" value="Submit">
  </form>

`
}

func Handler(w http.ResponseWriter, r *http.Request) string {
	// hash := genHash()
	file, header, err := r.FormFile("file")
	if err != nil {
		log.Warn(err.Error())
		return ""
	}
	defer file.Close()
	out, err := os.Create(path + header.Filename)
	if err != nil {
		log.Warn("Unable to create the file for writing. Check your write access privilege")
		return ""
	}
	defer out.Close()
	// write the content from POST to the file
	_, err = io.Copy(out, file)
	if err != nil {
		log.Warn(err.Error())
	}
	hash := genHash(path + header.Filename)
	os.Rename(path+header.Filename, path+hash)
	log.Info("File uploaded successfully: " + header.Filename + "(" + hash + ")")
	return hash
}

func genHash(file string) string {
	f, err := os.Open(file)
	log.Check(log.FatalLevel, "Opening file"+file, err)
	defer f.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, f); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}
<commit_msg>Upload with token only<commit_after>package upload

import (
	"crypto/md5"
	"fmt"
	"io"
	"net/http"
	"os"

	"github.com/optdyn/gorjun/db"

	"github.com/subutai-io/base/agent/log"
)

var (
	path = "/tmp/"
)

func Page(repo string) string {
	return `

  <title>Go upload</title>

  <form action="/` + repo + `/upload" method="post" enctype="multipart/form-data">
  Filename:
  <input type="file" name="file" id="file">
  <input type="submit" name="submit" value="Submit">
  </form>

`
}

func Handler(w http.ResponseWriter, r *http.Request) string {
	token := r.PostFormValue("token")
	if len(token) == 0 || len(db.CheckToken(token)) == 0 {
		w.WriteHeader(http.StatusUnauthorized)
		w.Write([]byte("Invalid token"))
		return
	}
	file, header, err := r.FormFile("file")
	if err != nil {
		log.Warn(err.Error())
		return ""
	}
	defer file.Close()
	out, err := os.Create(path + header.Filename)
	if err != nil {
		log.Warn("Unable to create the file for writing. Check your write access privilege")
		return ""
	}
	defer out.Close()
	// write the content from POST to the file
	_, err = io.Copy(out, file)
	if err != nil {
		log.Warn(err.Error())
	}
	hash := genHash(path + header.Filename)
	os.Rename(path+header.Filename, path+hash)
	log.Info("File uploaded successfully: " + header.Filename + "(" + hash + ")")
	return hash
}

func genHash(file string) string {
	f, err := os.Open(file)
	log.Check(log.FatalLevel, "Opening file"+file, err)
	defer f.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, f); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/gorilla/mux"
)

func NewRouter() *mux.Router {

	router := mux.NewRouter().StrictSlash(true)
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("./static/"))))
	for _, route := range routes {
		var handler http.Handler

		handler = route.HandlerFunc

		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(handler)

	}

	return router
}

type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type Routes []Route

var routes = Routes{
	Route{
		"Index",
		"GET",
		"/",
		IndexHandler,
	},
	Route{
		"Scan",
		"POST",
		"/api/v1/scan",
		ScanHandler,
	},
	Route{
		"Results",
		"GET",
		"/api/v1/results",
		ResultHandler,
	},
	Route{
		"Certificate",
		"GET",
		"/api/v1/certificate",
		CertificateHandler,
	},
	Route{
		"Certificate",
		"POST",
		"/api/v1/certificate",
		PostCertificateHandler,
	},
	Route{
		"Paths",
		"GET",
		"/api/v1/paths",
		PathsHandler,
	},
	Route{
		"Truststore",
		"GET",
		"/api/v1/truststore",
		TruststoreHandler,
	},
	Route{
		"IssuerEECount",
		"GET",
		"/api/v1/issuereecount",
		IssuerEECountHandler,
	},
	// CORS preflight endpoints
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/scan",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/results",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/certificate",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/paths",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/truststore",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/issuerStatus",
		PreflightHandler,
	},
	Route{
		"Heartbeat",
		"GET",
		"/api/v1/__heartbeat__",
		HeartbeatHandler,
	},
}
<commit_msg>addressed feedback<commit_after>package main

import (
	"net/http"

	"github.com/gorilla/mux"
)

func NewRouter() *mux.Router {

	router := mux.NewRouter().StrictSlash(true)
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("./static/"))))
	for _, route := range routes {
		var handler http.Handler

		handler = route.HandlerFunc

		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(handler)

	}

	return router
}

type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type Routes []Route

var routes = Routes{
	Route{
		"Index",
		"GET",
		"/",
		IndexHandler,
	},
	Route{
		"Scan",
		"POST",
		"/api/v1/scan",
		ScanHandler,
	},
	Route{
		"Results",
		"GET",
		"/api/v1/results",
		ResultHandler,
	},
	Route{
		"Certificate",
		"GET",
		"/api/v1/certificate",
		CertificateHandler,
	},
	Route{
		"Certificate",
		"POST",
		"/api/v1/certificate",
		PostCertificateHandler,
	},
	Route{
		"Paths",
		"GET",
		"/api/v1/paths",
		PathsHandler,
	},
	Route{
		"Truststore",
		"GET",
		"/api/v1/truststore",
		TruststoreHandler,
	},
	Route{
		"IssuerEECount",
		"GET",
		"/api/v1/issuereecount",
		IssuerEECountHandler,
	},
	// CORS preflight endpoints
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/scan",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/results",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/certificate",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/paths",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/truststore",
		PreflightHandler,
	},
	Route{
		"CORS Preflight",
		"OPTIONS",
		"/api/v1/issuereecount",
		PreflightHandler,
	},
	Route{
		"Heartbeat",
		"GET",
		"/api/v1/__heartbeat__",
		HeartbeatHandler,
	},
}
<|endoftext|>


<commit_before>package useful

import (
	. "github.com/SimonRichardson/wishful/wishful"
)

type Tuple2 struct {
	_1 Any
	_2 Any
}

func NewTuple2(a Any, b Any) Tuple2 {
	return Tuple2{_1: a, _2: b}
}

func (t Tuple2) Fst() Any {
	return t._1
}

func (t Tuple2) Snd() Any {
	return t._2
}

type Tuple3 struct {
	Tuple2
	_3 Any
}

func NewTuple3(a Any, b Any, c Any) Tuple3 {
	return Tuple3{
		Tuple2: Tuple2{_1: a, _2: b},
		_3:     c,
	}
}

func (t Tuple3) Trd() Any {
	return t._3
}

type Tuple4 struct {
	Tuple3
	_4 Any
}

func NewTuple4(a Any, b Any, c Any, d Any) Tuple4 {
	return Tuple4{
		Tuple3: Tuple3{
			Tuple2: Tuple2{_1: a, _2: b},
			_3:     c,
		},
		_4: d,
	}
}

func (t Tuple4) Fth() Any {
	return t._4
}
<commit_msg>Make tuples more useful<commit_after>package useful

import (
	. "github.com/SimonRichardson/wishful/wishful"
)

type Tuple2 struct {
	_1 Any
	_2 Any
}

func NewTuple2(a Any, b Any) Tuple2 {
	return Tuple2{_1: a, _2: b}
}

func (t Tuple2) Fst() Any {
	return t._1
}

func (t Tuple2) Snd() Any {
	return t._2
}

func (t Tuple2) MapFst(f func(Any) Any) Tuple2 {
	return NewTuple2(f(t._1), t._2)
}

func (t Tuple2) MapSnd(f func(Any) Any) Tuple2 {
	return NewTuple2(t._1, f(t._2))
}

func (t Tuple2) Bimap(f func(Any) Any, g func(Any) Any) Tuple2 {
	return NewTuple2(f(t._1), g(t._2))
}

func (t Tuple2) Append(x Any) Tuple3 {
	return NewTuple3(t._1, t._2, x)
}

func (t Tuple2) Slice() []Any {
	return []Any{t._1, t._2}
}

type Tuple3 struct {
	Tuple2
	_3 Any
}

func NewTuple3(a Any, b Any, c Any) Tuple3 {
	return Tuple3{
		Tuple2: Tuple2{_1: a, _2: b},
		_3:     c,
	}
}

func (t Tuple3) Trd() Any {
	return t._3
}

func (t Tuple3) MapTrd(f func(Any) Any) Tuple3 {
	return NewTuple3(t._1, t._2, f(t._3))
}

func (t Tuple3) Append(x Any) Tuple4 {
	return NewTuple4(t._1, t._2, t._3, x)
}

func (t Tuple3) Slice() []Any {
	return []Any{t._1, t._2, t._3}
}

type Tuple4 struct {
	Tuple3
	_4 Any
}

func NewTuple4(a Any, b Any, c Any, d Any) Tuple4 {
	return Tuple4{
		Tuple3: Tuple3{
			Tuple2: Tuple2{_1: a, _2: b},
			_3:     c,
		},
		_4: d,
	}
}

func (t Tuple4) Fth() Any {
	return t._4
}

func (t Tuple4) MapFth(f func(Any) Any) Tuple4 {
	return NewTuple4(t._1, t._2, t._3, f(t._4))
}
<|endoftext|>


<commit_before>package dfbuilder

import (
	"context"
	"fmt"
	"net/url"
	"sync"
	"time"

	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/query"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/perf/go/btts"
	"go.skia.org/infra/perf/go/cid"
	"go.skia.org/infra/perf/go/dataframe"
	"go.skia.org/infra/perf/go/types"
	"golang.org/x/sync/errgroup"
)

// builder implements DataFrameBuilder using btts.
type builder struct {
	vcs   vcsinfo.VCS
	store *btts.BigTableTraceStore
}

func NewDataFrameBuilderFromBTTS(vcs vcsinfo.VCS, store *btts.BigTableTraceStore) dataframe.DataFrameBuilder {
	return &builder{
		vcs:   vcs,
		store: store,
	}
}

// fromIndexCommit returns the slices of ColumnHeader and index.
// The slices are populated from the given vcsinfo.IndexCommits.
//
// The value for 'skip', the number of commits skipped, is passed through to
// the return values.
func fromIndexCommit(resp []*vcsinfo.IndexCommit, skip int) ([]*dataframe.ColumnHeader, []int32, int) {
	headers := []*dataframe.ColumnHeader{}
	indices := []int32{}
	for _, r := range resp {
		headers = append(headers, &dataframe.ColumnHeader{
			Source:    "master",
			Offset:    int64(r.Index),
			Timestamp: r.Timestamp.Unix(),
		})
		indices = append(indices, int32(r.Index))
	}
	return headers, indices, skip
}

// lastNCommits returns the slices of ColumnHeader and cid.CommitID that are
// needed by DataFrame and ptracestore.PTraceStore, respectively. The slices
// are for the last N commits in the repo.
//
// Returns 0 for 'skip', the number of commits skipped.
func lastNCommits(vcs vcsinfo.VCS, n int) ([]*dataframe.ColumnHeader, []int32, int) {
	return fromIndexCommit(vcs.LastNIndex(n), 0)
}

// fromTimeRange returns the slices of ColumnHeader and int32. The slices
// are for the commits that fall in the given time range [begin, end).
//
// If 'downsample' is true then the number of commits returned is limited
// to MAX_SAMPLE_SIZE.
//
// The value for 'skip', the number of commits skipped, is also returned.
func fromTimeRange(vcs vcsinfo.VCS, begin, end time.Time, downsample bool) ([]*dataframe.ColumnHeader, []int32, int) {
	commits := vcs.Range(begin, end)
	skip := 0
	if downsample {
		commits, skip = dataframe.DownSample(commits, dataframe.MAX_SAMPLE_SIZE)
	}
	return fromIndexCommit(commits, skip)
}

// tileMapOffsetToIndex maps the offset of each point in a tile to the index it
// should appear in the resulting Trace.
type tileMapOffsetToIndex map[btts.TileKey]map[int32]int32

// buildTileMapOffsetToIndex returns a tileMapOffsetToIndex for the given indices and the given BigTableTraceStore.
//
// The returned map is used when loading traces out of tiles.
func buildTileMapOffsetToIndex(indices []int32, store *btts.BigTableTraceStore) tileMapOffsetToIndex {
	ret := tileMapOffsetToIndex{}
	for targetIndex, sourceIndex := range indices {
		tileKey := store.TileKey(sourceIndex)
		if traceMap, ok := ret[tileKey]; !ok {
			ret[tileKey] = map[int32]int32{
				store.OffsetFromIndex(sourceIndex): int32(targetIndex),
			}
		} else {
			traceMap[store.OffsetFromIndex(sourceIndex)] = int32(targetIndex)
		}
	}
	return ret
}

// new builds a DataFrame for the given columns and populates it with traces that match the given query.
//
// The progress callback is triggered once for every tile.
func (b *builder) new(colHeaders []*dataframe.ColumnHeader, indices []int32, q *query.Query, progress types.Progress, skip int) (*dataframe.DataFrame, error) {
	// TODO tickle progress as each Go routine completes.
	defer timer.New("btts_new").Stop()
	// Determine which tiles we are querying over, and how each tile maps into our results.
	mapper := buildTileMapOffsetToIndex(indices, b.store)

	var mutex sync.Mutex // mutex protects traceSet, paramSet, and stepsCompleted.
	traceSet := types.TraceSet{}
	paramSet := paramtools.ParamSet{}
	stepsCompleted := 0
	// triggerProgress must only be called when the caller has mutex locked.
	triggerProgress := func() {
		stepsCompleted += 1
		if progress != nil {
			progress(stepsCompleted, len(mapper))
		}
	}

	var g errgroup.Group
	// For each tile.
	for tileKey, traceMap := range mapper {
		tileKey := tileKey
		traceMap := traceMap
		// TODO(jcgregorio) If we query across a large number of tiles N then this will spawn N*8 Go routines
		// all hitting the backend at the same time. Maybe we need a worker pool if this becomes a problem.
		g.Go(func() error {
			defer timer.New("btts_by_tile").Stop()
			// Get the OPS, which we need to encode the query, and decode the traceids of the results.
			ops, err := b.store.GetOrderedParamSet(tileKey)
			if err != nil {
				return err
			}
			// Convert query to regex.
			r, err := q.Regexp(ops)
			if err != nil {
				sklog.Infof("Failed to compile query regex: %s", err)
				// Not an error, we just won't match anything in this tile.
				return nil
			}
			// Query for matching traces in the given tile.
			traces, err := b.store.QueryTraces(tileKey, r)
			if err != nil {
				return err
			}
			mutex.Lock()
			defer mutex.Unlock()
			// For each trace, convert the encodedKey to a structured key
			// and copy the trace values into their final destination.
			for encodedKey, tileTrace := range traces {
				p, err := ops.DecodeParamsFromString(encodedKey)
				if err != nil {
					// It is possible we matched a trace that appeared after we grabbed the OPS,
					// so just ignore it.
					continue
				}
				paramSet.AddParams(p)
				key, err := query.MakeKey(p)
				if err != nil {
					return err
				}
				trace, ok := traceSet[key]
				if !ok {
					trace = types.NewTrace(len(indices))
				}
				for srcIndex, dstIndex := range traceMap {
					trace[dstIndex] = tileTrace[srcIndex]
				}
				traceSet[key] = trace
			}
			triggerProgress()
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, fmt.Errorf("Failed while querying: %s", err)
	}
	d := &dataframe.DataFrame{
		TraceSet: traceSet,
		Header:   colHeaders,
		ParamSet: paramSet,
		Skip:     skip,
	}
	return d, nil
}

// See DataFrameBuilder.
func (b *builder) New(progress types.Progress) (*dataframe.DataFrame, error) {
	return b.NewN(progress, dataframe.DEFAULT_NUM_COMMITS)
}

// See DataFrameBuilder.
func (b *builder) NewN(progress types.Progress, n int) (*dataframe.DataFrame, error) {
	colHeaders, indices, skip := lastNCommits(b.vcs, n)
	q, err := query.New(url.Values{})
	if err != nil {
		return nil, err
	}
	return b.new(colHeaders, indices, q, progress, skip)
}

// See DataFrameBuilder.
func (b *builder) NewFromQueryAndRange(begin, end time.Time, q *query.Query, progress types.Progress) (*dataframe.DataFrame, error) {
	colHeaders, indices, skip := fromTimeRange(b.vcs, begin, end, true)
	return b.new(colHeaders, indices, q, progress, skip)
}

// See DataFrameBuilder.
func (b *builder) NewFromKeysAndRange(keys []string, begin, end time.Time, progress types.Progress) (*dataframe.DataFrame, error) {
	// TODO tickle progress as each Go routine completes.
	defer timer.New("NewFromKeysAndRange").Stop()
	colHeaders, indices, skip := fromTimeRange(b.vcs, begin, end, true)

	// Determine which tiles we are querying over, and how each tile maps into our results.
	mapper := buildTileMapOffsetToIndex(indices, b.store)

	var mutex sync.Mutex // mutex protects traceSet and paramSet.
	traceSet := types.TraceSet{}
	paramSet := paramtools.ParamSet{}
	stepsCompleted := 0
	// triggerProgress must only be called when the caller has mutex locked.
	triggerProgress := func() {
		stepsCompleted += 1
		if progress != nil {
			progress(stepsCompleted, len(mapper))
		}
	}

	var g errgroup.Group
	// For each tile.
	for tileKey, traceMap := range mapper {
		tileKey := tileKey
		traceMap := traceMap
		g.Go(func() error {
			// Read the traces for the given keys.
			traces, err := b.store.ReadTraces(tileKey, keys)
			if err != nil {
				return err
			}
			mutex.Lock()
			defer mutex.Unlock()
			// For each trace, convert the encodedKey to a structured key
			// and copy the trace values into their final destination.
			for key, tileTrace := range traces {
				trace, ok := traceSet[key]
				if !ok {
					trace = types.NewTrace(len(indices))
				}
				for srcIndex, dstIndex := range traceMap {
					trace[dstIndex] = tileTrace[srcIndex]
				}
				traceSet[key] = trace
			}
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, fmt.Errorf("Failed while querying: %s", err)
	}
	d := &dataframe.DataFrame{
		TraceSet: traceSet,
		Header:   colHeaders,
		ParamSet: paramSet,
		Skip:     skip,
	}
	triggerProgress()
	return d, nil
}

// See DataFrameBuilder.
func (b *builder) NewFromCommitIDsAndQuery(ctx context.Context, cids []*cid.CommitID, cidl *cid.CommitIDLookup, q *query.Query, progress types.Progress) (*dataframe.DataFrame, error) {
	details, err := cidl.Lookup(ctx, cids)
	if err != nil {
		return nil, fmt.Errorf("Failed to look up CommitIDs: %s", err)
	}
	colHeaders := []*dataframe.ColumnHeader{}
	indices := []int32{}
	for _, d := range details {
		colHeaders = append(colHeaders, &dataframe.ColumnHeader{
			Source:    d.Source,
			Offset:    int64(d.Offset),
			Timestamp: d.Timestamp,
		})
		indices = append(indices, int32(d.Offset))
	}
	return b.new(colHeaders, indices, q, progress, 0)
}

// Validate that the concrete bttsDataFrameBuilder faithfully implements the DataFrameBuidler interface.
var _ dataframe.DataFrameBuilder = (*builder)(nil)
<commit_msg>Perf - Sort the values for the UI.<commit_after>package dfbuilder

import (
	"context"
	"fmt"
	"net/url"
	"sync"
	"time"

	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/query"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/perf/go/btts"
	"go.skia.org/infra/perf/go/cid"
	"go.skia.org/infra/perf/go/dataframe"
	"go.skia.org/infra/perf/go/types"
	"golang.org/x/sync/errgroup"
)

// builder implements DataFrameBuilder using btts.
type builder struct {
	vcs   vcsinfo.VCS
	store *btts.BigTableTraceStore
}

func NewDataFrameBuilderFromBTTS(vcs vcsinfo.VCS, store *btts.BigTableTraceStore) dataframe.DataFrameBuilder {
	return &builder{
		vcs:   vcs,
		store: store,
	}
}

// fromIndexCommit returns the slices of ColumnHeader and index.
// The slices are populated from the given vcsinfo.IndexCommits.
//
// The value for 'skip', the number of commits skipped, is passed through to
// the return values.
func fromIndexCommit(resp []*vcsinfo.IndexCommit, skip int) ([]*dataframe.ColumnHeader, []int32, int) {
	headers := []*dataframe.ColumnHeader{}
	indices := []int32{}
	for _, r := range resp {
		headers = append(headers, &dataframe.ColumnHeader{
			Source:    "master",
			Offset:    int64(r.Index),
			Timestamp: r.Timestamp.Unix(),
		})
		indices = append(indices, int32(r.Index))
	}
	return headers, indices, skip
}

// lastNCommits returns the slices of ColumnHeader and cid.CommitID that are
// needed by DataFrame and ptracestore.PTraceStore, respectively. The slices
// are for the last N commits in the repo.
//
// Returns 0 for 'skip', the number of commits skipped.
func lastNCommits(vcs vcsinfo.VCS, n int) ([]*dataframe.ColumnHeader, []int32, int) {
	return fromIndexCommit(vcs.LastNIndex(n), 0)
}

// fromTimeRange returns the slices of ColumnHeader and int32. The slices
// are for the commits that fall in the given time range [begin, end).
//
// If 'downsample' is true then the number of commits returned is limited
// to MAX_SAMPLE_SIZE.
//
// The value for 'skip', the number of commits skipped, is also returned.
func fromTimeRange(vcs vcsinfo.VCS, begin, end time.Time, downsample bool) ([]*dataframe.ColumnHeader, []int32, int) {
	commits := vcs.Range(begin, end)
	skip := 0
	if downsample {
		commits, skip = dataframe.DownSample(commits, dataframe.MAX_SAMPLE_SIZE)
	}
	return fromIndexCommit(commits, skip)
}

// tileMapOffsetToIndex maps the offset of each point in a tile to the index it
// should appear in the resulting Trace.
type tileMapOffsetToIndex map[btts.TileKey]map[int32]int32

// buildTileMapOffsetToIndex returns a tileMapOffsetToIndex for the given indices and the given BigTableTraceStore.
//
// The returned map is used when loading traces out of tiles.
func buildTileMapOffsetToIndex(indices []int32, store *btts.BigTableTraceStore) tileMapOffsetToIndex {
	ret := tileMapOffsetToIndex{}
	for targetIndex, sourceIndex := range indices {
		tileKey := store.TileKey(sourceIndex)
		if traceMap, ok := ret[tileKey]; !ok {
			ret[tileKey] = map[int32]int32{
				store.OffsetFromIndex(sourceIndex): int32(targetIndex),
			}
		} else {
			traceMap[store.OffsetFromIndex(sourceIndex)] = int32(targetIndex)
		}
	}
	return ret
}

// new builds a DataFrame for the given columns and populates it with traces that match the given query.
//
// The progress callback is triggered once for every tile.
func (b *builder) new(colHeaders []*dataframe.ColumnHeader, indices []int32, q *query.Query, progress types.Progress, skip int) (*dataframe.DataFrame, error) {
	// TODO tickle progress as each Go routine completes.
	defer timer.New("btts_new").Stop()
	// Determine which tiles we are querying over, and how each tile maps into our results.
	mapper := buildTileMapOffsetToIndex(indices, b.store)

	var mutex sync.Mutex // mutex protects traceSet, paramSet, and stepsCompleted.
	traceSet := types.TraceSet{}
	paramSet := paramtools.ParamSet{}
	stepsCompleted := 0
	// triggerProgress must only be called when the caller has mutex locked.
	triggerProgress := func() {
		stepsCompleted += 1
		if progress != nil {
			progress(stepsCompleted, len(mapper))
		}
	}

	var g errgroup.Group
	// For each tile.
	for tileKey, traceMap := range mapper {
		tileKey := tileKey
		traceMap := traceMap
		// TODO(jcgregorio) If we query across a large number of tiles N then this will spawn N*8 Go routines
		// all hitting the backend at the same time. Maybe we need a worker pool if this becomes a problem.
		g.Go(func() error {
			defer timer.New("btts_by_tile").Stop()
			// Get the OPS, which we need to encode the query, and decode the traceids of the results.
			ops, err := b.store.GetOrderedParamSet(tileKey)
			if err != nil {
				return err
			}
			// Convert query to regex.
			r, err := q.Regexp(ops)
			if err != nil {
				sklog.Infof("Failed to compile query regex: %s", err)
				// Not an error, we just won't match anything in this tile.
				return nil
			}
			// Query for matching traces in the given tile.
			traces, err := b.store.QueryTraces(tileKey, r)
			if err != nil {
				return err
			}
			mutex.Lock()
			defer mutex.Unlock()
			// For each trace, convert the encodedKey to a structured key
			// and copy the trace values into their final destination.
			for encodedKey, tileTrace := range traces {
				p, err := ops.DecodeParamsFromString(encodedKey)
				if err != nil {
					// It is possible we matched a trace that appeared after we grabbed the OPS,
					// so just ignore it.
					continue
				}
				paramSet.AddParams(p)
				key, err := query.MakeKey(p)
				if err != nil {
					return err
				}
				trace, ok := traceSet[key]
				if !ok {
					trace = types.NewTrace(len(indices))
				}
				for srcIndex, dstIndex := range traceMap {
					trace[dstIndex] = tileTrace[srcIndex]
				}
				traceSet[key] = trace
			}
			triggerProgress()
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, fmt.Errorf("Failed while querying: %s", err)
	}
	paramSet.Normalize()
	d := &dataframe.DataFrame{
		TraceSet: traceSet,
		Header:   colHeaders,
		ParamSet: paramSet,
		Skip:     skip,
	}
	return d, nil
}

// See DataFrameBuilder.
func (b *builder) New(progress types.Progress) (*dataframe.DataFrame, error) {
	return b.NewN(progress, dataframe.DEFAULT_NUM_COMMITS)
}

// See DataFrameBuilder.
func (b *builder) NewN(progress types.Progress, n int) (*dataframe.DataFrame, error) {
	colHeaders, indices, skip := lastNCommits(b.vcs, n)
	q, err := query.New(url.Values{})
	if err != nil {
		return nil, err
	}
	return b.new(colHeaders, indices, q, progress, skip)
}

// See DataFrameBuilder.
func (b *builder) NewFromQueryAndRange(begin, end time.Time, q *query.Query, progress types.Progress) (*dataframe.DataFrame, error) {
	colHeaders, indices, skip := fromTimeRange(b.vcs, begin, end, true)
	return b.new(colHeaders, indices, q, progress, skip)
}

// See DataFrameBuilder.
func (b *builder) NewFromKeysAndRange(keys []string, begin, end time.Time, progress types.Progress) (*dataframe.DataFrame, error) {
	// TODO tickle progress as each Go routine completes.
	defer timer.New("NewFromKeysAndRange").Stop()
	colHeaders, indices, skip := fromTimeRange(b.vcs, begin, end, true)

	// Determine which tiles we are querying over, and how each tile maps into our results.
	mapper := buildTileMapOffsetToIndex(indices, b.store)

	var mutex sync.Mutex // mutex protects traceSet and paramSet.
	traceSet := types.TraceSet{}
	paramSet := paramtools.ParamSet{}
	stepsCompleted := 0
	// triggerProgress must only be called when the caller has mutex locked.
	triggerProgress := func() {
		stepsCompleted += 1
		if progress != nil {
			progress(stepsCompleted, len(mapper))
		}
	}

	var g errgroup.Group
	// For each tile.
	for tileKey, traceMap := range mapper {
		tileKey := tileKey
		traceMap := traceMap
		g.Go(func() error {
			// Read the traces for the given keys.
			traces, err := b.store.ReadTraces(tileKey, keys)
			if err != nil {
				return err
			}
			mutex.Lock()
			defer mutex.Unlock()
			// For each trace, convert the encodedKey to a structured key
			// and copy the trace values into their final destination.
			for key, tileTrace := range traces {
				trace, ok := traceSet[key]
				if !ok {
					trace = types.NewTrace(len(indices))
				}
				for srcIndex, dstIndex := range traceMap {
					trace[dstIndex] = tileTrace[srcIndex]
				}
				traceSet[key] = trace
			}
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, fmt.Errorf("Failed while querying: %s", err)
	}
	d := &dataframe.DataFrame{
		TraceSet: traceSet,
		Header:   colHeaders,
		ParamSet: paramSet,
		Skip:     skip,
	}
	triggerProgress()
	return d, nil
}

// See DataFrameBuilder.
func (b *builder) NewFromCommitIDsAndQuery(ctx context.Context, cids []*cid.CommitID, cidl *cid.CommitIDLookup, q *query.Query, progress types.Progress) (*dataframe.DataFrame, error) {
	details, err := cidl.Lookup(ctx, cids)
	if err != nil {
		return nil, fmt.Errorf("Failed to look up CommitIDs: %s", err)
	}
	colHeaders := []*dataframe.ColumnHeader{}
	indices := []int32{}
	for _, d := range details {
		colHeaders = append(colHeaders, &dataframe.ColumnHeader{
			Source:    d.Source,
			Offset:    int64(d.Offset),
			Timestamp: d.Timestamp,
		})
		indices = append(indices, int32(d.Offset))
	}
	return b.new(colHeaders, indices, q, progress, 0)
}

// Validate that the concrete bttsDataFrameBuilder faithfully implements the DataFrameBuidler interface.
var _ dataframe.DataFrameBuilder = (*builder)(nil)
<|endoftext|>


<commit_before>package trion

import (
	. "polydawn.net/gosh/psh"
	"polydawn.net/dockctrl/crocker"
	. "fmt"
	"os"
	"path/filepath"
	"strings"
)

//Default docker command template
var docker = crocker.NewDock("dock").Client()

//Where to place & call CIDfiles
const TempDir    = "/tmp"
const TempPrefix = "trion-"

//Default tar filename amd image tag
const TarFile    = "image.tar"
const DefaultTag = "latest"

//Executes 'docker run' and returns the container's CID.
func Run(config TrionConfig) string {
	dockRun := docker("run")

	//Find the absolute path for each host mount
	for i, j := range config.Mount {
		cwd, err := filepath.Abs(j[0])
		if err != nil {
			Println("Fatal: Cannot determine absolute path:", j[0])
			os.Exit(1)
		}

		config.Mount[i][0] = cwd
	}

	//Where should docker write the new CID?
	CIDfilename := createCIDfile()
	dockRun = dockRun("-cidfile", CIDfilename)

	//Where should the container start?
	dockRun = dockRun("-w", config.StartIn)

	//Is the docker in privleged (pwn ur box) mode?
	if (config.Privileged) {
		dockRun = dockRun("-privileged")
	}

	//Custom DNS servers?
	for i := range config.DNS {
		dockRun = dockRun ("-dns", config.DNS[i])
	}

	//What folders get mounted?
	for i := range config.Mount {
		dockRun = dockRun("-v", config.Mount[i][0] + ":" + config.Mount[i][1] + ":" + config.Mount[i][2])
	}

	//What environment variables are set?
	for i:= range config.Environment {
		dockRun = dockRun("-e", config.Environment[i][0] + "=" + config.Environment[i][1])
	}

	//Are we attaching?
	if config.Attach {
		dockRun = dockRun("-i", "-t")
	}

	//Add image name
	dockRun = dockRun(config.Image)

	//What command should it run?
	for i := range config.Command {
		dockRun = dockRun(config.Command[i])
	}

	//Poll for the CID and run the docker
	dockRun()
	getCID := pollCid(CIDfilename)
	return <- getCID
}

//Executes 'docker wait'
func Wait(CID string) {
	docker("wait", CID)()
}

//Executes 'docker rm'
func Purge(CID string) {
	docker("rm", CID)()
}

//Executes 'docker export', after ensuring there is no image.tar in the way.
//	This is because docker will *happily* export into an existing tar.
func Export(CID, path string) {
	tar := path + TarFile

	//Check for existing file
	file, _ := os.Open(tar)
	_, err  := file.Stat()
	file.Close()

	//Delete tar if it exists
	if err == nil {
		Println("Warning: Output image.tar already exists. Overwriting...")
		err = os.Remove("./image.tar")
		if err != nil {
			Println("Fatal: Could not delete tar file.")
			os.Exit(1)
		}
	}

	out, err := os.OpenFile(tar, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err);
	}

	docker("export", CID)(Opts{Out: out})()
}

//Import an image into docker's repository.
func Import(config TrionConfig, path string) {
	tar := path + TarFile

	//Open the file
	in, err := os.Open(tar)
	if err != nil {
		Println("Fatal: Could not open file for import:", tar)
	}

	//Get the repository and tag from the config
	repo, tag := "", ""
	sp := strings.Split(config.Image, ":")

	//If both a name and version are specified, use them, otherwise just tag it as 'latest'
	if len(sp) == 2 {
		repo = sp[0]
		tag = sp[1]
	} else {
		repo = config.Image
		tag = DefaultTag
	}

	Println("Importing", repo + ":" + tag)
	docker("import", "-", repo, tag)(Opts{In: in, Out: os.Stdout })()
}
<commit_msg>Don't try to stat image.tar if it does not exist<commit_after>package trion

import (
	. "polydawn.net/gosh/psh"
	"polydawn.net/dockctrl/crocker"
	. "fmt"
	"os"
	"path/filepath"
	"strings"
)

//Default docker command template
var docker = crocker.NewDock("dock").Client()

//Where to place & call CIDfiles
const TempDir    = "/tmp"
const TempPrefix = "trion-"

//Default tar filename amd image tag
const TarFile    = "image.tar"
const DefaultTag = "latest"

//Executes 'docker run' and returns the container's CID.
func Run(config TrionConfig) string {
	dockRun := docker("run")

	//Find the absolute path for each host mount
	for i, j := range config.Mount {
		cwd, err := filepath.Abs(j[0])
		if err != nil {
			Println("Fatal: Cannot determine absolute path:", j[0])
			os.Exit(1)
		}

		config.Mount[i][0] = cwd
	}

	//Where should docker write the new CID?
	CIDfilename := createCIDfile()
	dockRun = dockRun("-cidfile", CIDfilename)

	//Where should the container start?
	dockRun = dockRun("-w", config.StartIn)

	//Is the docker in privleged (pwn ur box) mode?
	if (config.Privileged) {
		dockRun = dockRun("-privileged")
	}

	//Custom DNS servers?
	for i := range config.DNS {
		dockRun = dockRun ("-dns", config.DNS[i])
	}

	//What folders get mounted?
	for i := range config.Mount {
		dockRun = dockRun("-v", config.Mount[i][0] + ":" + config.Mount[i][1] + ":" + config.Mount[i][2])
	}

	//What environment variables are set?
	for i:= range config.Environment {
		dockRun = dockRun("-e", config.Environment[i][0] + "=" + config.Environment[i][1])
	}

	//Are we attaching?
	if config.Attach {
		dockRun = dockRun("-i", "-t")
	}

	//Add image name
	dockRun = dockRun(config.Image)

	//What command should it run?
	for i := range config.Command {
		dockRun = dockRun(config.Command[i])
	}

	//Poll for the CID and run the docker
	dockRun()
	getCID := pollCid(CIDfilename)
	return <- getCID
}

//Executes 'docker wait'
func Wait(CID string) {
	docker("wait", CID)()
}

//Executes 'docker rm'
func Purge(CID string) {
	docker("rm", CID)()
}

//Executes 'docker export', after ensuring there is no image.tar in the way.
//	This is because docker will *happily* export into an existing tar.
func Export(CID, path string) {
	tar := path + TarFile

	//Check for existing file
	file, err := os.Open(tar)
	if (err == nil) {
		_, err  = file.Stat()
		file.Close()
	}

	//Delete tar if it exists
	if err == nil {
		Println("Warning: Output image.tar already exists. Overwriting...")
		err = os.Remove("./image.tar")
		if err != nil {
			Println("Fatal: Could not delete tar file.")
			os.Exit(1)
		}
	}

	out, err := os.OpenFile(tar, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err);
	}

	docker("export", CID)(Opts{Out: out})()
}

//Import an image into docker's repository.
func Import(config TrionConfig, path string) {
	tar := path + TarFile

	//Open the file
	in, err := os.Open(tar)
	if err != nil {
		Println("Fatal: Could not open file for import:", tar)
	}

	//Get the repository and tag from the config
	repo, tag := "", ""
	sp := strings.Split(config.Image, ":")

	//If both a name and version are specified, use them, otherwise just tag it as 'latest'
	if len(sp) == 2 {
		repo = sp[0]
		tag = sp[1]
	} else {
		repo = config.Image
		tag = DefaultTag
	}

	Println("Importing", repo + ":" + tag)
	docker("import", "-", repo, tag)(Opts{In: in, Out: os.Stdout })()
}
<|endoftext|>


<commit_before>// +build darwin

package xattr

import (
	"syscall"
	"unsafe"
)

func getxattr(path string, name string, value *byte, size int, pos int, options int) (int, error) {

	r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options))
	if e1 != syscall.Errno(0) {
		return int(r0), e1
	}
	return int(r0), nil
}

func listxattr(path string, namebuf *byte, size int, options int) (int, error) {
	r0, _, e1 := syscall.Syscall6(syscall.SYS_LISTXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(namebuf)), uintptr(size), uintptr(options), 0, 0)
	return int(r0), e1
}

func setxattr(path string, name string, value *byte, size int, pos int, options int) error {
	if _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options)); e1 != syscall.Errno(0) {
		return e1
	}
	return nil
}

func removexattr(path string, name string, options int) error {
	if _, _, e1 := syscall.Syscall(syscall.SYS_REMOVEXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(options)); e1 != syscall.Errno(0) {
		return e1
	}
	return nil
}
<commit_msg>listxattr: Test errno != 0 before returning error<commit_after>// +build darwin

package xattr

import (
	"syscall"
	"unsafe"
)

func getxattr(path string, name string, value *byte, size int, pos int, options int) (int, error) {

	r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options))
	if e1 != syscall.Errno(0) {
		return int(r0), e1
	}
	return int(r0), nil
}

func listxattr(path string, namebuf *byte, size int, options int) (int, error) {
	r0, _, e1 := syscall.Syscall6(syscall.SYS_LISTXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(namebuf)), uintptr(size), uintptr(options), 0, 0)
	if e1 != syscall.Errno(0) {
		return int(r0), e1
	}
	return int(r0), nil
}

func setxattr(path string, name string, value *byte, size int, pos int, options int) error {
	if _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options)); e1 != syscall.Errno(0) {
		return e1
	}
	return nil
}

func removexattr(path string, name string, options int) error {
	if _, _, e1 := syscall.Syscall(syscall.SYS_REMOVEXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(options)); e1 != syscall.Errno(0) {
		return e1
	}
	return nil
}
<|endoftext|>


<commit_before>// +build linux

package water

import (
	"os"
	"strings"
	"syscall"
	"unsafe"
)

const (
	cIFF_TUN         = 0x0001
	cIFF_TAP         = 0x0002
	cIFF_NO_PI       = 0x1000
	cIFF_MULTI_QUEUE = 0x0100
)

type ifReq struct {
	Name  [0x10]byte
	Flags uint16
	pad   [0x28 - 0x10 - 2]byte
}

func ioctl(fd uintptr, request uint, argp uintptr) error {
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(request), argp)
	if errno != 0 {
		return os.NewSyscallError("ioctl", errno)
	}
	return nil
}

func newTAP(config Config) (ifce *Interface, err error) {
	file, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}

	var flags uint16
	flags = cIFF_TUN | cIFF_NO_PI
	if config.PlatformSpecificParams.MultiQueue {
		flags |= cIFF_MULTI_QUEUE
	}
	name, err := createInterface(file.Fd(), config.Name, flags)
	if err != nil {
		return nil, err
	}

	if err = setDeviceOptions(file.Fd(), config); err != nil {
		return nil, err
	}

	ifce = &Interface{isTAP: true, ReadWriteCloser: file, name: name}
	return
}

func newTUN(config Config) (ifce *Interface, err error) {
	file, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}

	var flags uint16
	flags = cIFF_TUN | cIFF_NO_PI
	if config.PlatformSpecificParams.MultiQueue {
		flags |= cIFF_MULTI_QUEUE
	}
	name, err := createInterface(file.Fd(), config.Name, flags)
	if err != nil {
		return nil, err
	}

	if err = setDeviceOptions(file.Fd(), config); err != nil {
		return nil, err
	}

	ifce = &Interface{isTAP: false, ReadWriteCloser: file, name: name}
	return
}

func createInterface(fd uintptr, ifName string, flags uint16) (createdIFName string, err error) {
	var req ifReq
	req.Flags = flags
	copy(req.Name[:], ifName)

	err = ioctl(fd, syscall.TUNSETIFF, uintptr(unsafe.Pointer(&req)))
	if err != nil {
		return
	}

	createdIFName = strings.Trim(string(req.Name[:]), "\x00")
	return
}

func setDeviceOptions(fd uintptr, config Config) (err error) {

	// Device Permissions
	if config.Permissions != nil {

		// Set Owner
		if err = ioctl(fd, syscall.TUNSETOWNER, uintptr(config.Permissions.Owner)); err != nil {
			return
		}

		// Set Group
		if err = ioctl(fd, syscall.TUNSETGROUP, uintptr(config.Permissions.Group)); err != nil {
			return
		}
	}

	// Set/Clear Persist Device Flag
	value := 0
	if config.Persist {
		value = 1
	}
	return ioctl(fd, syscall.TUNSETPERSIST, uintptr(value))

}
<commit_msg>use uintptr instand of uint<commit_after>// +build linux

package water

import (
	"os"
	"strings"
	"syscall"
	"unsafe"
)

const (
	cIFF_TUN         = 0x0001
	cIFF_TAP         = 0x0002
	cIFF_NO_PI       = 0x1000
	cIFF_MULTI_QUEUE = 0x0100
)

type ifReq struct {
	Name  [0x10]byte
	Flags uint16
	pad   [0x28 - 0x10 - 2]byte
}

func ioctl(fd uintptr, request uintptr, argp uintptr) error {
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(request), argp)
	if errno != 0 {
		return os.NewSyscallError("ioctl", errno)
	}
	return nil
}

func newTAP(config Config) (ifce *Interface, err error) {
	file, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}

	var flags uint16
	flags = cIFF_TUN | cIFF_NO_PI
	if config.PlatformSpecificParams.MultiQueue {
		flags |= cIFF_MULTI_QUEUE
	}
	name, err := createInterface(file.Fd(), config.Name, flags)
	if err != nil {
		return nil, err
	}

	if err = setDeviceOptions(file.Fd(), config); err != nil {
		return nil, err
	}

	ifce = &Interface{isTAP: true, ReadWriteCloser: file, name: name}
	return
}

func newTUN(config Config) (ifce *Interface, err error) {
	file, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}

	var flags uint16
	flags = cIFF_TUN | cIFF_NO_PI
	if config.PlatformSpecificParams.MultiQueue {
		flags |= cIFF_MULTI_QUEUE
	}
	name, err := createInterface(file.Fd(), config.Name, flags)
	if err != nil {
		return nil, err
	}

	if err = setDeviceOptions(file.Fd(), config); err != nil {
		return nil, err
	}

	ifce = &Interface{isTAP: false, ReadWriteCloser: file, name: name}
	return
}

func createInterface(fd uintptr, ifName string, flags uint16) (createdIFName string, err error) {
	var req ifReq
	req.Flags = flags
	copy(req.Name[:], ifName)

	err = ioctl(fd, syscall.TUNSETIFF, uintptr(unsafe.Pointer(&req)))
	if err != nil {
		return
	}

	createdIFName = strings.Trim(string(req.Name[:]), "\x00")
	return
}

func setDeviceOptions(fd uintptr, config Config) (err error) {

	// Device Permissions
	if config.Permissions != nil {

		// Set Owner
		if err = ioctl(fd, syscall.TUNSETOWNER, uintptr(config.Permissions.Owner)); err != nil {
			return
		}

		// Set Group
		if err = ioctl(fd, syscall.TUNSETGROUP, uintptr(config.Permissions.Group)); err != nil {
			return
		}
	}

	// Set/Clear Persist Device Flag
	value := 0
	if config.Persist {
		value = 1
	}
	return ioctl(fd, syscall.TUNSETPERSIST, uintptr(value))

}
<|endoftext|>


<commit_before>package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	iconv "github.com/djimenez/iconv-go"
	"github.com/toomore/gogrs/utils"
)

// TWSECLASS is a class list of TWSE.
var TWSECLASS = map[string]string{
	"MS":         "大盤統計資訊",
	"0049":       "封閉式基金",
	"0099P":      "ETF",
	"01":         "水泥工業",
	"019919T":    "受益證券",
	"02":         "食品工業",
	"03":         "塑膠工業",
	"04":         "紡織纖維",
	"05":         "電機機械",
	"06":         "電器電纜",
	"07":         "化學生技醫療",
	"08":         "玻璃陶瓷",
	"09":         "造紙工業",
	"0999":       "認購權證", //(不含牛證)
	"0999B":      "熊證",
	"0999C":      "牛證",
	"0999G9":     "認股權憑證",
	"0999GA":     "附認股權特別股",
	"0999GD":     "附認股權公司債",
	"0999P":      "認售權證", //(不含熊證)
	"0999X":      "可展延牛證",
	"0999Y":      "可展延熊證",
	"10":         "鋼鐵工業",
	"11":         "橡膠工業",
	"12":         "汽車工業",
	"13":         "電子工業",
	"14":         "建材營造",
	"15":         "航運業",
	"16":         "觀光事業",
	"17":         "金融保險",
	"18":         "貿易百貨",
	"19":         "綜合",
	"20":         "其他",
	"21":         "化學工業",
	"22":         "生技醫療業",
	"23":         "油電燃氣業",
	"24":         "半導體業",
	"25":         "電腦及週邊設備業",
	"26":         "光電業",
	"27":         "通信網路業",
	"28":         "電子零組件業",
	"29":         "電子通路業",
	"30":         "資訊服務業",
	"31":         "其他電子業",
	"9299":       "存託憑證",
	"ALL":        "全部",
	"ALLBUT0999": "全部(不含權證、牛熊證、可展延牛熊證)",
	"CB":         "可轉換公司債",
}

// StockInfo is simple stock info for no, name.
type StockInfo struct {
	No   string
	Name string
}

// Lists is to get TWSE list.
type Lists struct {
	Date            time.Time
	FmtData         map[string]FmtListData
	categoryRawData map[string][][]string
	categoryNoList  map[string][]StockInfo
}

// Get is to get csv data.
func (l *Lists) Get(category string) ([][]string, error) {
	if TWSECLASS[category] == "" {
		return nil, errors.New("Not support.")
	}

	year, month, day := l.Date.Date()
	data, err := http.PostForm(fmt.Sprintf("%s%s", utils.TWSEHOST, utils.TWSELISTCSV),
		url.Values{"download": {"csv"}, "selectType": {category},
			"qdate": {fmt.Sprintf("%d/%02d/%02d", year-1911, month, day)}})
	defer data.Body.Close()

	if err != nil {
		return nil, fmt.Errorf("Network fail: %s", err)
	}

	dataContentBig5, _ := ioutil.ReadAll(data.Body)
	dataContent, _ := iconv.ConvertString(string(dataContentBig5), "big5", "utf-8")
	csvArrayContent := strings.Split(dataContent, "\n")

	if category == "MS" {
		if len(csvArrayContent) > 6 {
			csvReader := csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:51], "\n")))
			returnData, err := csvReader.ReadAll()
			return returnData, err
		}
	} else {
		if len(csvArrayContent) > 9 {
			csvReader := csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:len(csvArrayContent)-7], "\n")))
			returnData, err := csvReader.ReadAll()
			if err == nil {
				if l.categoryRawData == nil {
					l.categoryRawData = make(map[string][][]string)
				}
				l.categoryRawData[category] = returnData
				l.formatData(category)
			}
			return returnData, err
		}
	}
	return nil, errors.New("Not enough data.")
}

// GetCategoryList 取得分類的股票代碼與名稱列表
func (l Lists) GetCategoryList(category string) []StockInfo {
	if _, ok := l.categoryNoList[category]; !ok {
		l.Get(category)
	}
	return l.categoryNoList[category]
}

// FmtListData 格式化個股的資料資訊
type FmtListData struct {
	No             string
	Name           string
	Volume         uint64  //成交股數
	TotalPrice     uint64  //成交金額
	Open           float64 //開盤價
	High           float64 //最高價
	Low            float64 //最低價
	Price          float64 //收盤價
	Range          float64 //漲跌價差
	Totalsale      uint64  //成交筆數
	LastBuyPrice   float64 //最後揭示買價
	LastBuyVolume  uint64  //最後揭示買量
	LastSellPrice  float64 //最後揭示賣價
	LastSellVolume uint64  //最後揭示賣量
	PERatio        float64 //本益比
}

func (l *Lists) formatData(categoryNo string) {
	if l.FmtData == nil {
		l.FmtData = make(map[string]FmtListData)
	}

	if l.categoryNoList == nil {
		l.categoryNoList = make(map[string][]StockInfo)
	}

	if _, ok := l.categoryNoList[categoryNo]; !ok {
		l.categoryNoList[categoryNo] = make([]StockInfo, len(l.categoryRawData[categoryNo]))
	}

	for i, v := range l.categoryRawData[categoryNo] {
		var data FmtListData
		data.No = strings.Trim(v[0], " ")
		data.Name = strings.Trim(v[1], " ")
		data.Volume, _ = strconv.ParseUint(v[2], 10, 32)
		data.Totalsale, _ = strconv.ParseUint(v[3], 10, 32)
		data.TotalPrice, _ = strconv.ParseUint(v[4], 10, 32)
		data.Open, _ = strconv.ParseFloat(v[5], 64)
		data.High, _ = strconv.ParseFloat(v[6], 64)
		data.Low, _ = strconv.ParseFloat(v[7], 64)
		data.Price, _ = strconv.ParseFloat(v[8], 64)
		data.Range, _ = strconv.ParseFloat(fmt.Sprintf("%s%s", v[9], v[10]), 64)
		data.LastBuyPrice, _ = strconv.ParseFloat(v[11], 64)
		data.LastBuyVolume, _ = strconv.ParseUint(v[12], 10, 32)
		data.LastSellPrice, _ = strconv.ParseFloat(v[13], 64)
		data.LastSellVolume, _ = strconv.ParseUint(v[14], 10, 32)
		data.PERatio, _ = strconv.ParseFloat(v[15], 64)

		l.FmtData[data.No] = data
		l.categoryNoList[categoryNo][i] = StockInfo{No: data.No, Name: data.Name}
	}
}
<commit_msg>Using cp950.<commit_after>package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	iconv "github.com/djimenez/iconv-go"
	"github.com/toomore/gogrs/utils"
)

// TWSECLASS is a class list of TWSE.
var TWSECLASS = map[string]string{
	"MS":         "大盤統計資訊",
	"0049":       "封閉式基金",
	"0099P":      "ETF",
	"01":         "水泥工業",
	"019919T":    "受益證券",
	"02":         "食品工業",
	"03":         "塑膠工業",
	"04":         "紡織纖維",
	"05":         "電機機械",
	"06":         "電器電纜",
	"07":         "化學生技醫療",
	"08":         "玻璃陶瓷",
	"09":         "造紙工業",
	"0999":       "認購權證", //(不含牛證)
	"0999B":      "熊證",
	"0999C":      "牛證",
	"0999G9":     "認股權憑證",
	"0999GA":     "附認股權特別股",
	"0999GD":     "附認股權公司債",
	"0999P":      "認售權證", //(不含熊證)
	"0999X":      "可展延牛證",
	"0999Y":      "可展延熊證",
	"10":         "鋼鐵工業",
	"11":         "橡膠工業",
	"12":         "汽車工業",
	"13":         "電子工業",
	"14":         "建材營造",
	"15":         "航運業",
	"16":         "觀光事業",
	"17":         "金融保險",
	"18":         "貿易百貨",
	"19":         "綜合",
	"20":         "其他",
	"21":         "化學工業",
	"22":         "生技醫療業",
	"23":         "油電燃氣業",
	"24":         "半導體業",
	"25":         "電腦及週邊設備業",
	"26":         "光電業",
	"27":         "通信網路業",
	"28":         "電子零組件業",
	"29":         "電子通路業",
	"30":         "資訊服務業",
	"31":         "其他電子業",
	"9299":       "存託憑證",
	"ALL":        "全部",
	"ALLBUT0999": "全部(不含權證、牛熊證、可展延牛熊證)",
	"CB":         "可轉換公司債",
}

// StockInfo is simple stock info for no, name.
type StockInfo struct {
	No   string
	Name string
}

// Lists is to get TWSE list.
type Lists struct {
	Date            time.Time
	FmtData         map[string]FmtListData
	categoryRawData map[string][][]string
	categoryNoList  map[string][]StockInfo
}

// Get is to get csv data.
func (l *Lists) Get(category string) ([][]string, error) {
	if TWSECLASS[category] == "" {
		return nil, errors.New("Not support.")
	}

	year, month, day := l.Date.Date()
	data, err := http.PostForm(fmt.Sprintf("%s%s", utils.TWSEHOST, utils.TWSELISTCSV),
		url.Values{"download": {"csv"}, "selectType": {category},
			"qdate": {fmt.Sprintf("%d/%02d/%02d", year-1911, month, day)}})
	defer data.Body.Close()

	if err != nil {
		return nil, fmt.Errorf("Network fail: %s", err)
	}

	dataContentCp950, _ := ioutil.ReadAll(data.Body)
	dataContent, _ := iconv.ConvertString(string(dataContentCp950), "cp950", "utf-8")
	csvArrayContent := strings.Split(dataContent, "\n")

	if category == "MS" {
		if len(csvArrayContent) > 6 {
			csvReader := csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:51], "\n")))
			returnData, err := csvReader.ReadAll()
			return returnData, err
		}
	} else {
		if len(csvArrayContent) > 9 {
			csvReader := csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[4:len(csvArrayContent)-7], "\n")))
			returnData, err := csvReader.ReadAll()
			if err == nil {
				if l.categoryRawData == nil {
					l.categoryRawData = make(map[string][][]string)
				}
				l.categoryRawData[category] = returnData
				l.formatData(category)
			}
			return returnData, err
		}
	}
	return nil, errors.New("Not enough data.")
}

// GetCategoryList 取得分類的股票代碼與名稱列表
func (l Lists) GetCategoryList(category string) []StockInfo {
	if _, ok := l.categoryNoList[category]; !ok {
		l.Get(category)
	}
	return l.categoryNoList[category]
}

// FmtListData 格式化個股的資料資訊
type FmtListData struct {
	No             string
	Name           string
	Volume         uint64  //成交股數
	TotalPrice     uint64  //成交金額
	Open           float64 //開盤價
	High           float64 //最高價
	Low            float64 //最低價
	Price          float64 //收盤價
	Range          float64 //漲跌價差
	Totalsale      uint64  //成交筆數
	LastBuyPrice   float64 //最後揭示買價
	LastBuyVolume  uint64  //最後揭示買量
	LastSellPrice  float64 //最後揭示賣價
	LastSellVolume uint64  //最後揭示賣量
	PERatio        float64 //本益比
}

func (l *Lists) formatData(categoryNo string) {
	if l.FmtData == nil {
		l.FmtData = make(map[string]FmtListData)
	}

	if l.categoryNoList == nil {
		l.categoryNoList = make(map[string][]StockInfo)
	}

	if _, ok := l.categoryNoList[categoryNo]; !ok {
		l.categoryNoList[categoryNo] = make([]StockInfo, len(l.categoryRawData[categoryNo]))
	}

	for i, v := range l.categoryRawData[categoryNo] {
		var data FmtListData
		data.No = strings.Trim(v[0], " ")
		data.Name = strings.Trim(v[1], " ")
		data.Volume, _ = strconv.ParseUint(v[2], 10, 32)
		data.Totalsale, _ = strconv.ParseUint(v[3], 10, 32)
		data.TotalPrice, _ = strconv.ParseUint(v[4], 10, 32)
		data.Open, _ = strconv.ParseFloat(v[5], 64)
		data.High, _ = strconv.ParseFloat(v[6], 64)
		data.Low, _ = strconv.ParseFloat(v[7], 64)
		data.Price, _ = strconv.ParseFloat(v[8], 64)
		data.Range, _ = strconv.ParseFloat(fmt.Sprintf("%s%s", v[9], v[10]), 64)
		data.LastBuyPrice, _ = strconv.ParseFloat(v[11], 64)
		data.LastBuyVolume, _ = strconv.ParseUint(v[12], 10, 32)
		data.LastSellPrice, _ = strconv.ParseFloat(v[13], 64)
		data.LastSellVolume, _ = strconv.ParseUint(v[14], 10, 32)
		data.PERatio, _ = strconv.ParseFloat(v[15], 64)

		l.FmtData[data.No] = data
		l.categoryNoList[categoryNo][i] = StockInfo{No: data.No, Name: data.Name}
	}
}
<|endoftext|>