<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package harnesses

import (
	"os/exec"
	"path/filepath"

	"golang.org/x/benchmarks/sweet/common"
	"golang.org/x/benchmarks/sweet/common/log"
)

const (
	server = "tile38-server"
)

type Tile38 struct{}

func (h Tile38) CheckPrerequisites() error {
	return nil
}

func (h Tile38) Get(srcDir string) error {
	return gitShallowClone(
		srcDir,
		"https://github.com/tidwall/tile38",
		"1.25.3",
	)
}

func (h Tile38) Build(cfg *common.Config, bcfg *common.BuildConfig) error {
	env := cfg.BuildEnv.Env

	// Add the Go tool to PATH, since tile38's Makefile doesn't provide enough
	// visibility into how tile38 is built to allow us to pass this information
	// directly. Also set the GOROOT explicitly because it might have propagated
	// differently from the environment.
	env = env.Prefix("PATH", filepath.Join(cfg.GoRoot, "bin")+":")
	env = env.MustSet("GOROOT=" + cfg.GoRoot)

	cmd := exec.Command("make", "-C", bcfg.SrcDir)
	cmd.Env = env.Collapse()
	log.TraceCommand(cmd, false)
	// Call Output here to get an *ExitError with a populated Stderr field.
	if _, err := cmd.Output(); err != nil {
		return err
	}
	// Note that no matter what we do, the build script insists on putting the
	// binaries into the source directory, so copy the one we care about into
	// BinDir.
	if err := copyFile(filepath.Join(bcfg.BinDir, server), filepath.Join(bcfg.SrcDir, server)); err != nil {
		return err
	}
	return cfg.GoTool().BuildPath(bcfg.BenchDir, filepath.Join(bcfg.BinDir, "tile38-bench"))
}

func (h Tile38) Run(cfg *common.Config, rcfg *common.RunConfig) error {
	// Make sure all the data passed to the server is writable.
	// The server needs to be able to open its persistent storage as read-write.
	dataPath := filepath.Join(rcfg.AssetsDir, "data")
	if err := makeWriteable(dataPath); err != nil {
		return err
	}
	args := append(rcfg.Args, []string{
		"-host", "127.0.0.1",
		"-port", "9851",
		"-server", filepath.Join(rcfg.BinDir, server),
		"-data", dataPath,
		"-tmp", rcfg.TmpDir,
	}...)
	if rcfg.Short {
		args = append(args, "-short")
	}
	cmd := exec.Command(
		filepath.Join(rcfg.BinDir, "tile38-bench"),
		args...,
	)
	cmd.Env = cfg.ExecEnv.Collapse()
	cmd.Stdout = rcfg.Results
	cmd.Stderr = rcfg.Results
	log.TraceCommand(cmd, false)
	return cmd.Run()
}
<commit_msg>sweet: upgrade tile38 to 1.29.1<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package harnesses

import (
	"os/exec"
	"path/filepath"

	"golang.org/x/benchmarks/sweet/common"
	"golang.org/x/benchmarks/sweet/common/log"
)

const (
	server = "tile38-server"
)

type Tile38 struct{}

func (h Tile38) CheckPrerequisites() error {
	return nil
}

func (h Tile38) Get(srcDir string) error {
	return gitShallowClone(
		srcDir,
		"https://github.com/tidwall/tile38",
		"1.29.1",
	)
}

func (h Tile38) Build(cfg *common.Config, bcfg *common.BuildConfig) error {
	env := cfg.BuildEnv.Env

	// Add the Go tool to PATH, since tile38's Makefile doesn't provide enough
	// visibility into how tile38 is built to allow us to pass this information
	// directly. Also set the GOROOT explicitly because it might have propagated
	// differently from the environment.
	env = env.Prefix("PATH", filepath.Join(cfg.GoRoot, "bin")+":")
	env = env.MustSet("GOROOT=" + cfg.GoRoot)

	cmd := exec.Command("make", "-C", bcfg.SrcDir)
	cmd.Env = env.Collapse()
	log.TraceCommand(cmd, false)
	// Call Output here to get an *ExitError with a populated Stderr field.
	if _, err := cmd.Output(); err != nil {
		return err
	}
	// Note that no matter what we do, the build script insists on putting the
	// binaries into the source directory, so copy the one we care about into
	// BinDir.
	if err := copyFile(filepath.Join(bcfg.BinDir, server), filepath.Join(bcfg.SrcDir, server)); err != nil {
		return err
	}
	return cfg.GoTool().BuildPath(bcfg.BenchDir, filepath.Join(bcfg.BinDir, "tile38-bench"))
}

func (h Tile38) Run(cfg *common.Config, rcfg *common.RunConfig) error {
	// Make sure all the data passed to the server is writable.
	// The server needs to be able to open its persistent storage as read-write.
	dataPath := filepath.Join(rcfg.AssetsDir, "data")
	if err := makeWriteable(dataPath); err != nil {
		return err
	}
	args := append(rcfg.Args, []string{
		"-host", "127.0.0.1",
		"-port", "9851",
		"-server", filepath.Join(rcfg.BinDir, server),
		"-data", dataPath,
		"-tmp", rcfg.TmpDir,
	}...)
	if rcfg.Short {
		args = append(args, "-short")
	}
	cmd := exec.Command(
		filepath.Join(rcfg.BinDir, "tile38-bench"),
		args...,
	)
	cmd.Env = cfg.ExecEnv.Collapse()
	cmd.Stdout = rcfg.Results
	cmd.Stderr = rcfg.Results
	log.TraceCommand(cmd, false)
	return cmd.Run()
}
<|endoftext|>


<commit_before>package organization

import (
	"fmt"
	"path/filepath"

	"github.com/pivotalservices/cf-mgmt/cloudcontroller"
	"github.com/pivotalservices/cf-mgmt/ldap"
	"github.com/pivotalservices/cf-mgmt/uaac"
	"github.com/pivotalservices/cf-mgmt/utils"
	"github.com/xchapter7x/lo"
)

//NewManager -
func NewManager(sysDomain, token, uaacToken string) (mgr Manager) {

	cloudController := cloudcontroller.NewManager(fmt.Sprintf("https://api.%s", sysDomain), token)
	ldapMgr := ldap.NewManager()
	uaacMgr := uaac.NewManager(sysDomain, uaacToken)
	UserMgr := NewUserManager(cloudController, ldapMgr, uaacMgr)

	return &DefaultOrgManager{
		CloudController: cloudController,
		UAACMgr:         uaacMgr,
		UtilsMgr:        utils.NewDefaultManager(),
		LdapMgr:         ldapMgr,
		UserMgr:         UserMgr,
	}
}

func (m *DefaultOrgManager) GetOrgConfigs(configDir string) ([]*InputUpdateOrgs, error) {
	orgConfigs := []*InputUpdateOrgs{}
	files, err := m.UtilsMgr.FindFiles(configDir, "orgConfig.yml")
	if err != nil {
		return nil, err
	}
	for _, f := range files {
		input := &InputUpdateOrgs{
			AppInstanceLimit:        -1,
			TotalReservedRoutePorts: 0,
			TotalPrivateDomains:     -1,
			TotalServiceKeys:        -1,
		}
		if err = m.UtilsMgr.LoadFile(f, input); err != nil {
			lo.G.Error(err)
			return nil, err
		}
		orgConfigs = append(orgConfigs, input)
	}
	return orgConfigs, nil
}

//CreateQuotas -
func (m *DefaultOrgManager) CreateQuotas(configDir string) error {
	orgs, err := m.GetOrgConfigs(configDir)
	if err != nil {
		return err
	}

	quotas, err := m.CloudController.ListAllOrgQuotas()
	if err != nil {
		return err
	}

	for _, input := range orgs {
		if !input.EnableOrgQuota {
			continue
		}

		org, err := m.FindOrg(input.Org)
		if err != nil {
			return err
		}
		quotaName := org.Entity.Name
		quota := cloudcontroller.QuotaEntity{
			Name:                    quotaName,
			MemoryLimit:             input.MemoryLimit,
			InstanceMemoryLimit:     input.InstanceMemoryLimit,
			TotalRoutes:             input.TotalRoutes,
			TotalServices:           input.TotalServices,
			PaidServicePlansAllowed: input.PaidServicePlansAllowed,
			TotalPrivateDomains:     input.TotalPrivateDomains,
			TotalReservedRoutePorts: input.TotalReservedRoutePorts,
			TotalServiceKeys:        input.TotalServiceKeys,
			AppInstanceLimit:        input.AppInstanceLimit,
		}
		if quotaGUID, ok := quotas[quotaName]; ok {
			lo.G.Info("Updating quota", quotaName)

			if err = m.CloudController.UpdateQuota(quotaGUID, quota); err != nil {
				return err
			}
			lo.G.Info("Assigning", quotaName, "to", org.Entity.Name)
			if err = m.CloudController.AssignQuotaToOrg(org.MetaData.GUID, quotaGUID); err != nil {
				return err
			}
		} else {
			lo.G.Info("Creating quota", quotaName)
			targetQuotaGUID, err := m.CloudController.CreateQuota(quota)
			if err != nil {
				return err
			}
			lo.G.Info("Assigning", quotaName, "to", org.Entity.Name)
			if err := m.CloudController.AssignQuotaToOrg(org.MetaData.GUID, targetQuotaGUID); err != nil {
				return err
			}
		}
	}
	return nil
}

func (m *DefaultOrgManager) GetOrgGUID(orgName string) (string, error) {
	org, err := m.FindOrg(orgName)
	if err != nil {
		return "", err
	}
	return org.MetaData.GUID, nil
}

//CreateOrgs -
func (m *DefaultOrgManager) CreateOrgs(configDir string) error {
	configFile := filepath.Join(configDir, "orgs.yml")
	lo.G.Info("Processing org file", configFile)
	input := &InputOrgs{}
	if err := m.UtilsMgr.LoadFile(configFile, input); err != nil {
		return err
	}
	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return err
	}

	for _, orgName := range input.Orgs {
		if m.DoesOrgExist(orgName, orgs) {
			lo.G.Infof("[%s] org already exists", orgName)
			continue
		}
		lo.G.Infof("Creating [%s] org", orgName)
		if err := m.CloudController.CreateOrg(orgName); err != nil {
			return err
		}
	}
	return nil
}

func (m *DefaultOrgManager) CreatePrivateDomains(configDir string) error {

	orgConfigs, err := m.GetOrgConfigs(configDir)
	if err != nil {
		lo.G.Error(err)
		return err
	}

	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return err
	}
	allPrivateDomains, err := m.CloudController.ListAllPrivateDomains()
	if err != nil {
		return err
	}
	for _, orgConfig := range orgConfigs {
		orgGUID, err := m.getOrgGUID(orgs, orgConfig.Org)
		if err != nil {
			return err
		}
		privateDomainMap := make(map[string]string)
		for _, privateDomain := range orgConfig.PrivateDomains {
			if existingOrgGUID, ok := allPrivateDomains[privateDomain]; ok {
				if orgGUID != existingOrgGUID {
					existingOrgName, _ := m.getOrgName(orgs, existingOrgGUID)
					msg := fmt.Sprintf("Private Domain %s already exists in org [%s]", privateDomain, existingOrgName)
					lo.G.Error(msg)
					return fmt.Errorf(msg)
				}
				lo.G.Infof("Private Domain %s already exists for Org %s", privateDomain, orgConfig.Org)
			} else {
				lo.G.Infof("Creating Private Domain %s for Org %s", privateDomain, orgConfig.Org)
				err = m.CloudController.CreatePrivateDomain(orgGUID, privateDomain)
				if err != nil {
					return err
				}
				allPrivateDomains[privateDomain] = orgGUID
			}
			privateDomainMap[privateDomain] = privateDomain
		}

		if orgConfig.RemovePrivateDomains {
			lo.G.Infof("Looking for private domains to remove for org [%s]", orgConfig.Org)
			orgPrivateDomains, err := m.CloudController.ListOrgPrivateDomains(orgGUID)
			if err != nil {
				return err
			}
			for existingPrivateDomain, privateDomainGUID := range orgPrivateDomains {
				if _, ok := privateDomainMap[existingPrivateDomain]; !ok {
					lo.G.Infof("Removing Private Domain %s for Org %s", existingPrivateDomain, orgConfig.Org)
					err = m.CloudController.DeletePrivateDomain(privateDomainGUID)
					if err != nil {
						return err
					}
				}
			}
		} else {
			lo.G.Infof("Private domains will not be removed for org [%s], must set enable-remove-private-domains: true in orgConfig.yml", orgConfig.Org)
		}
	}

	return nil
}

func (m *DefaultOrgManager) getOrgName(orgs []*cloudcontroller.Org, orgGUID string) (string, error) {
	for _, org := range orgs {
		if org.MetaData.GUID == orgGUID {
			return org.Entity.Name, nil
		}
	}
	return "", fmt.Errorf("Org for GUID %s does not exist", orgGUID)
}

func (m *DefaultOrgManager) getOrgGUID(orgs []*cloudcontroller.Org, orgName string) (string, error) {
	for _, org := range orgs {
		if org.Entity.Name == orgName {
			return org.MetaData.GUID, nil
		}
	}
	return "", fmt.Errorf("Org %s does not exist", orgName)
}

//DeleteOrgs -
func (m *DefaultOrgManager) DeleteOrgs(configDir string, peekDeletion bool) error {
	configFile := filepath.Join(configDir, "orgs.yml")
	lo.G.Info("Processing org file", configFile)
	input := &InputOrgs{}
	if err := m.UtilsMgr.LoadFile(configFile, input); err != nil {
		return err
	}

	if !input.EnableDeleteOrgs {
		lo.G.Info("Org deletion is not enabled.  Set enable-delete-orgs: true")
		return nil
	}

	configuredOrgs := make(map[string]bool)
	for _, orgName := range input.Orgs {
		configuredOrgs[orgName] = true
	}
	protectedOrgs := make(map[string]bool)
	//never allow accidental deletion of system org
	protectedOrgs["system"] = true
	for _, orgName := range input.ProtectedOrgs {
		protectedOrgs[orgName] = true
	}

	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return err
	}

	orgsToDelete := make([]*cloudcontroller.Org, 0)
	for _, org := range orgs {
		if _, exists := configuredOrgs[org.Entity.Name]; !exists {
			if _, protected := protectedOrgs[org.Entity.Name]; !protected {
				orgsToDelete = append(orgsToDelete, org)
			} else {
				lo.G.Info(fmt.Sprintf("Protected org [%s] - will not be deleted", org.Entity.Name))
			}
		}
	}

	if peekDeletion {
		for _, org := range orgsToDelete {
			lo.G.Info(fmt.Sprintf("Peek - Would Delete [%s] org", org.Entity.Name))
		}
	} else {
		for _, org := range orgsToDelete {
			lo.G.Info(fmt.Sprintf("Deleting [%s] org", org.Entity.Name))
			if err := m.CloudController.DeleteOrg(org.MetaData.GUID); err != nil {
				return err
			}
		}
	}

	return nil
}

func (m *DefaultOrgManager) DoesOrgExist(orgName string, orgs []*cloudcontroller.Org) bool {
	for _, org := range orgs {
		if org.Entity.Name == orgName {
			return true
		}
	}
	return false
}

//FindOrg -
func (m *DefaultOrgManager) FindOrg(orgName string) (*cloudcontroller.Org, error) {
	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return nil, err
	}
	for _, theOrg := range orgs {
		if theOrg.Entity.Name == orgName {
			return theOrg, nil
		}
	}
	return nil, fmt.Errorf("org %q not found", orgName)
}

//UpdateOrgUsers -
func (m *DefaultOrgManager) UpdateOrgUsers(configDir, ldapBindPassword string) error {
	config, err := m.LdapMgr.GetConfig(configDir, ldapBindPassword)
	if err != nil {
		lo.G.Error(err)
		return err
	}

	uaacUsers, err := m.UAACMgr.ListUsers()
	if err != nil {
		lo.G.Error(err)
		return err
	}

	orgConfigs, err := m.GetOrgConfigs(configDir)
	if err != nil {
		lo.G.Error(err)
		return err
	}

	for _, input := range orgConfigs {
		if err := m.updateOrgUsers(config, input, uaacUsers); err != nil {
			return err
		}
	}
	return nil
}

func (m *DefaultOrgManager) updateOrgUsers(config *ldap.Config, input *InputUpdateOrgs, uaacUsers map[string]string) error {
	org, err := m.FindOrg(input.Org)
	if err != nil {
		return err
	}

	err = m.UserMgr.UpdateOrgUsers(
		config, uaacUsers, UpdateUsersInput{
			OrgName:        org.Entity.Name,
			OrgGUID:        org.MetaData.GUID,
			Role:           "billing_managers",
			LdapGroupNames: input.GetBillingManagerGroups(),
			LdapUsers:      input.BillingManager.LdapUsers,
			Users:          input.BillingManager.Users,
			RemoveUsers:    input.RemoveUsers,
		})
	if err != nil {
		return err
	}

	err = m.UserMgr.UpdateOrgUsers(
		config, uaacUsers, UpdateUsersInput{
			OrgName:        org.Entity.Name,
			OrgGUID:        org.MetaData.GUID,
			Role:           "auditors",
			LdapGroupNames: input.GetAuditorGroups(),
			LdapUsers:      input.Auditor.LdapUsers,
			Users:          input.Auditor.Users,
			RemoveUsers:    input.RemoveUsers,
		})
	if err != nil {
		return err
	}

	err = m.UserMgr.UpdateOrgUsers(
		config, uaacUsers, UpdateUsersInput{
			OrgName:        org.Entity.Name,
			OrgGUID:        org.MetaData.GUID,
			Role:           "managers",
			LdapGroupNames: input.GetManagerGroups(),
			LdapUsers:      input.Manager.LdapUsers,
			Users:          input.Manager.Users,
			RemoveUsers:    input.RemoveUsers,
		})
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Minor simplifications<commit_after>package organization

import (
	"fmt"
	"path/filepath"

	"github.com/pivotalservices/cf-mgmt/cloudcontroller"
	"github.com/pivotalservices/cf-mgmt/ldap"
	"github.com/pivotalservices/cf-mgmt/uaac"
	"github.com/pivotalservices/cf-mgmt/utils"
	"github.com/xchapter7x/lo"
)

func NewManager(sysDomain, token, uaacToken string) (mgr Manager) {
	cloudController := cloudcontroller.NewManager(fmt.Sprintf("https://api.%s", sysDomain), token)
	ldapMgr := ldap.NewManager()
	uaacMgr := uaac.NewManager(sysDomain, uaacToken)
	UserMgr := NewUserManager(cloudController, ldapMgr, uaacMgr)

	return &DefaultOrgManager{
		CloudController: cloudController,
		UAACMgr:         uaacMgr,
		UtilsMgr:        utils.NewDefaultManager(),
		LdapMgr:         ldapMgr,
		UserMgr:         UserMgr,
	}
}

func (m *DefaultOrgManager) GetOrgConfigs(configDir string) ([]*InputUpdateOrgs, error) {
	orgConfigs := []*InputUpdateOrgs{}
	files, err := m.UtilsMgr.FindFiles(configDir, "orgConfig.yml")
	if err != nil {
		return nil, err
	}
	for _, f := range files {
		input := &InputUpdateOrgs{
			AppInstanceLimit:        -1,
			TotalReservedRoutePorts: 0,
			TotalPrivateDomains:     -1,
			TotalServiceKeys:        -1,
		}
		if err = m.UtilsMgr.LoadFile(f, input); err != nil {
			lo.G.Error(err)
			return nil, err
		}
		orgConfigs = append(orgConfigs, input)
	}
	return orgConfigs, nil
}

//CreateQuotas -
func (m *DefaultOrgManager) CreateQuotas(configDir string) error {
	orgs, err := m.GetOrgConfigs(configDir)
	if err != nil {
		return err
	}

	quotas, err := m.CloudController.ListAllOrgQuotas()
	if err != nil {
		return err
	}

	for _, input := range orgs {
		if !input.EnableOrgQuota {
			continue
		}

		org, err := m.FindOrg(input.Org)
		if err != nil {
			return err
		}
		quotaName := org.Entity.Name
		quota := cloudcontroller.QuotaEntity{
			Name:                    quotaName,
			MemoryLimit:             input.MemoryLimit,
			InstanceMemoryLimit:     input.InstanceMemoryLimit,
			TotalRoutes:             input.TotalRoutes,
			TotalServices:           input.TotalServices,
			PaidServicePlansAllowed: input.PaidServicePlansAllowed,
			TotalPrivateDomains:     input.TotalPrivateDomains,
			TotalReservedRoutePorts: input.TotalReservedRoutePorts,
			TotalServiceKeys:        input.TotalServiceKeys,
			AppInstanceLimit:        input.AppInstanceLimit,
		}
		if quotaGUID, ok := quotas[quotaName]; ok {
			lo.G.Info("Updating quota", quotaName)

			if err = m.CloudController.UpdateQuota(quotaGUID, quota); err != nil {
				return err
			}
			lo.G.Info("Assigning", quotaName, "to", org.Entity.Name)
			if err = m.CloudController.AssignQuotaToOrg(org.MetaData.GUID, quotaGUID); err != nil {
				return err
			}
		} else {
			lo.G.Info("Creating quota", quotaName)
			targetQuotaGUID, err := m.CloudController.CreateQuota(quota)
			if err != nil {
				return err
			}
			lo.G.Info("Assigning", quotaName, "to", org.Entity.Name)
			if err := m.CloudController.AssignQuotaToOrg(org.MetaData.GUID, targetQuotaGUID); err != nil {
				return err
			}
		}
	}
	return nil
}

func (m *DefaultOrgManager) GetOrgGUID(orgName string) (string, error) {
	org, err := m.FindOrg(orgName)
	if err != nil {
		return "", err
	}
	return org.MetaData.GUID, nil
}

//CreateOrgs -
func (m *DefaultOrgManager) CreateOrgs(configDir string) error {
	configFile := filepath.Join(configDir, "orgs.yml")
	lo.G.Info("Processing org file", configFile)
	input := &InputOrgs{}
	if err := m.UtilsMgr.LoadFile(configFile, input); err != nil {
		return err
	}
	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return err
	}

	for _, orgName := range input.Orgs {
		if m.DoesOrgExist(orgName, orgs) {
			lo.G.Infof("[%s] org already exists", orgName)
			continue
		}
		lo.G.Infof("Creating [%s] org", orgName)
		if err := m.CloudController.CreateOrg(orgName); err != nil {
			return err
		}
	}
	return nil
}

func (m *DefaultOrgManager) CreatePrivateDomains(configDir string) error {
	orgConfigs, err := m.GetOrgConfigs(configDir)
	if err != nil {
		lo.G.Error(err)
		return err
	}

	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return err
	}
	allPrivateDomains, err := m.CloudController.ListAllPrivateDomains()
	if err != nil {
		return err
	}
	for _, orgConfig := range orgConfigs {
		orgGUID, err := m.getOrgGUID(orgs, orgConfig.Org)
		if err != nil {
			return err
		}
		privateDomainMap := make(map[string]string)
		for _, privateDomain := range orgConfig.PrivateDomains {
			if existingOrgGUID, ok := allPrivateDomains[privateDomain]; ok {
				if orgGUID != existingOrgGUID {
					existingOrgName, _ := m.getOrgName(orgs, existingOrgGUID)
					msg := fmt.Sprintf("Private Domain %s already exists in org [%s]", privateDomain, existingOrgName)
					lo.G.Error(msg)
					return fmt.Errorf(msg)
				}
				lo.G.Infof("Private Domain %s already exists for Org %s", privateDomain, orgConfig.Org)
			} else {
				lo.G.Infof("Creating Private Domain %s for Org %s", privateDomain, orgConfig.Org)
				err = m.CloudController.CreatePrivateDomain(orgGUID, privateDomain)
				if err != nil {
					return err
				}
				allPrivateDomains[privateDomain] = orgGUID
			}
			privateDomainMap[privateDomain] = privateDomain
		}

		if orgConfig.RemovePrivateDomains {
			lo.G.Infof("Looking for private domains to remove for org [%s]", orgConfig.Org)
			orgPrivateDomains, err := m.CloudController.ListOrgPrivateDomains(orgGUID)
			if err != nil {
				return err
			}
			for existingPrivateDomain, privateDomainGUID := range orgPrivateDomains {
				if _, ok := privateDomainMap[existingPrivateDomain]; !ok {
					lo.G.Infof("Removing Private Domain %s for Org %s", existingPrivateDomain, orgConfig.Org)
					err = m.CloudController.DeletePrivateDomain(privateDomainGUID)
					if err != nil {
						return err
					}
				}
			}
		} else {
			lo.G.Infof("Private domains will not be removed for org [%s], must set enable-remove-private-domains: true in orgConfig.yml", orgConfig.Org)
		}
	}

	return nil
}

func (m *DefaultOrgManager) getOrgName(orgs []*cloudcontroller.Org, orgGUID string) (string, error) {
	for _, org := range orgs {
		if org.MetaData.GUID == orgGUID {
			return org.Entity.Name, nil
		}
	}
	return "", fmt.Errorf("org for GUID %s does not exist", orgGUID)
}

func (m *DefaultOrgManager) getOrgGUID(orgs []*cloudcontroller.Org, orgName string) (string, error) {
	for _, org := range orgs {
		if org.Entity.Name == orgName {
			return org.MetaData.GUID, nil
		}
	}
	return "", fmt.Errorf("org %s does not exist", orgName)
}

//DeleteOrgs -
func (m *DefaultOrgManager) DeleteOrgs(configDir string, peekDeletion bool) error {
	configFile := filepath.Join(configDir, "orgs.yml")
	lo.G.Info("Processing org file", configFile)
	input := &InputOrgs{}
	if err := m.UtilsMgr.LoadFile(configFile, input); err != nil {
		return err
	}

	if !input.EnableDeleteOrgs {
		lo.G.Info("Org deletion is not enabled.  Set enable-delete-orgs: true")
		return nil
	}

	configuredOrgs := make(map[string]bool)
	for _, orgName := range input.Orgs {
		configuredOrgs[orgName] = true
	}

	protectedOrgs := make(map[string]bool)
	//never allow accidental deletion of system org
	protectedOrgs["system"] = true
	for _, orgName := range input.ProtectedOrgs {
		protectedOrgs[orgName] = true
	}

	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return err
	}

	orgsToDelete := make([]*cloudcontroller.Org, 0)
	for _, org := range orgs {
		if _, exists := configuredOrgs[org.Entity.Name]; !exists {
			if _, protected := protectedOrgs[org.Entity.Name]; !protected {
				orgsToDelete = append(orgsToDelete, org)
			} else {
				lo.G.Info(fmt.Sprintf("Protected org [%s] - will not be deleted", org.Entity.Name))
			}
		}
	}

	if peekDeletion {
		for _, org := range orgsToDelete {
			lo.G.Info(fmt.Sprintf("Peek - Would Delete [%s] org", org.Entity.Name))
		}
	} else {
		for _, org := range orgsToDelete {
			lo.G.Info(fmt.Sprintf("Deleting [%s] org", org.Entity.Name))
			if err := m.CloudController.DeleteOrg(org.MetaData.GUID); err != nil {
				return err
			}
		}
	}

	return nil
}

func (m *DefaultOrgManager) DoesOrgExist(orgName string, orgs []*cloudcontroller.Org) bool {
	for _, org := range orgs {
		if org.Entity.Name == orgName {
			return true
		}
	}
	return false
}

//FindOrg -
func (m *DefaultOrgManager) FindOrg(orgName string) (*cloudcontroller.Org, error) {
	orgs, err := m.CloudController.ListOrgs()
	if err != nil {
		return nil, err
	}
	for _, theOrg := range orgs {
		if theOrg.Entity.Name == orgName {
			return theOrg, nil
		}
	}
	return nil, fmt.Errorf("org %q not found", orgName)
}

//UpdateOrgUsers -
func (m *DefaultOrgManager) UpdateOrgUsers(configDir, ldapBindPassword string) error {
	config, err := m.LdapMgr.GetConfig(configDir, ldapBindPassword)
	if err != nil {
		lo.G.Error(err)
		return err
	}

	uaacUsers, err := m.UAACMgr.ListUsers()
	if err != nil {
		lo.G.Error(err)
		return err
	}

	orgConfigs, err := m.GetOrgConfigs(configDir)
	if err != nil {
		lo.G.Error(err)
		return err
	}

	for _, input := range orgConfigs {
		if err := m.updateOrgUsers(config, input, uaacUsers); err != nil {
			return err
		}
	}
	return nil
}

func (m *DefaultOrgManager) updateOrgUsers(config *ldap.Config, input *InputUpdateOrgs, uaacUsers map[string]string) error {
	org, err := m.FindOrg(input.Org)
	if err != nil {
		return err
	}

	err = m.UserMgr.UpdateOrgUsers(
		config, uaacUsers, UpdateUsersInput{
			OrgName:        org.Entity.Name,
			OrgGUID:        org.MetaData.GUID,
			Role:           "billing_managers",
			LdapGroupNames: input.GetBillingManagerGroups(),
			LdapUsers:      input.BillingManager.LdapUsers,
			Users:          input.BillingManager.Users,
			RemoveUsers:    input.RemoveUsers,
		})
	if err != nil {
		return err
	}

	err = m.UserMgr.UpdateOrgUsers(
		config, uaacUsers, UpdateUsersInput{
			OrgName:        org.Entity.Name,
			OrgGUID:        org.MetaData.GUID,
			Role:           "auditors",
			LdapGroupNames: input.GetAuditorGroups(),
			LdapUsers:      input.Auditor.LdapUsers,
			Users:          input.Auditor.Users,
			RemoveUsers:    input.RemoveUsers,
		})
	if err != nil {
		return err
	}

	return m.UserMgr.UpdateOrgUsers(
		config, uaacUsers, UpdateUsersInput{
			OrgName:        org.Entity.Name,
			OrgGUID:        org.MetaData.GUID,
			Role:           "managers",
			LdapGroupNames: input.GetManagerGroups(),
			LdapUsers:      input.Manager.LdapUsers,
			Users:          input.Manager.Users,
			RemoveUsers:    input.RemoveUsers,
		})
}
<|endoftext|>


<commit_before>// Package ini provides helpers for reading ini file
package ini

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

// WalkFunc is the type of the function called for each key visited by Walk.
type WalkFunc func(section, name, value []byte) error

// Walk walks all items from r, calling walkFn for each item.
func Walk(r io.Reader, walkFn WalkFunc) error {
	var section []byte
	sep := []byte{'='}

	scanner := bufio.NewScanner(r)
	for lineNo := 1; scanner.Scan(); lineNo++ {
		line := bytes.TrimSpace(scanner.Bytes())
		if len(line) == 0 || line[0] == '#' || line[0] == ';' {
			// Skip comments
			continue
		}
		if line[0] == '[' && line[len(line)-1] == ']' {
			// Section
			section = duplicate(section, bytes.TrimSpace(line[1:len(line)-1]))
			continue
		}
		if line[len(line)-1] == '\\' {
			// Duplicate data from Scanner to append next lines
			line = append([]byte(nil), line[:len(line)-1]...)

			// Continuation lines
			for scanner.Scan() {
				lineNo++
				l := bytes.TrimSpace(scanner.Bytes())
				if l[len(l)-1] == '\\' {
					line = append(line, l[:len(l)-1]...)
				} else {
					line = append(line, l...)
					break
				}
			}
		}
		key := bytes.SplitN(line, sep, 2)
		if len(key) != 2 {
			return fmt.Errorf("missing delimiter (line %d)", lineNo)
		}
		name := bytes.TrimSpace(key[0])
		if len(name) == 0 {
			return fmt.Errorf("empty name (line %d)", lineNo)
		}
		value := bytes.TrimSpace(key[1])
		err := walkFn(section, name, value)
		if err != nil {
			return err
		}
	}
	return nil
}

// duplicate copies data from src to dest. It returns a new slice if dest capacity does not
// fit src length.
func duplicate(dest, src []byte) []byte {
	if cap(dest) < len(src) {
		return append([]byte(nil), src...)
	}
	dest = dest[:len(src)]
	copy(dest, src)
	return dest
}
<commit_msg>Use Index to split name & value<commit_after>// Package ini provides helpers for reading ini file
package ini

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
)

// WalkFunc is the type of the function called for each key visited by Walk.
type WalkFunc func(section, name, value []byte) error

// Walk walks all items from r, calling walkFn for each item.
func Walk(r io.Reader, walkFn WalkFunc) error {
	var section []byte
	sep := []byte{'='}

	scanner := bufio.NewScanner(r)
	for lineNo := 1; scanner.Scan(); lineNo++ {
		line := bytes.TrimSpace(scanner.Bytes())
		if len(line) == 0 || line[0] == '#' || line[0] == ';' {
			// Skip comments
			continue
		}
		if line[0] == '[' && line[len(line)-1] == ']' {
			// Section
			section = duplicate(section, bytes.TrimSpace(line[1:len(line)-1]))
			continue
		}
		if line[len(line)-1] == '\\' {
			// Duplicate data from Scanner to append next lines
			line = append([]byte(nil), line[:len(line)-1]...)

			// Continuation lines
			for scanner.Scan() {
				lineNo++
				l := bytes.TrimSpace(scanner.Bytes())
				if l[len(l)-1] == '\\' {
					line = append(line, l[:len(l)-1]...)
				} else {
					line = append(line, l...)
					break
				}
			}
		}
		i := bytes.Index(line, sep)
		if i < 0 {
			return fmt.Errorf("missing delimiter (line %d)", lineNo)
		}
		name := bytes.TrimSpace(line[:i])
		if len(name) == 0 {
			return fmt.Errorf("empty name (line %d)", lineNo)
		}
		value := bytes.TrimSpace(line[i+1:])
		err := walkFn(section, name, value)
		if err != nil {
			return err
		}
	}
	return nil
}

// duplicate copies data from src to dest. It returns a new slice if dest capacity does not
// fit src length.
func duplicate(dest, src []byte) []byte {
	if cap(dest) < len(src) {
		return append([]byte(nil), src...)
	}
	dest = dest[:len(src)]
	copy(dest, src)
	return dest
}
<|endoftext|>


<commit_before>package spider

import (
	"sort"
	"time"
)

// InMemory is the default scheduler
type InMemory struct {
	entries Entries
	addCh   chan *Entry
	stopCh  chan struct{}
	running bool
}

// NewScheduler returns a new InMemory scheduler
func NewScheduler() *InMemory {
	return &InMemory{
		addCh:   make(chan *Entry),
		stopCh:  make(chan struct{}),
		entries: nil,
	}
}

// Entry groups a spider, its root context, a Schedule and the Next time the spider must be launched
type Entry struct {
	Spider   Spider
	Schedule Schedule
	Ctx      *Context
	Next     time.Time
}

// Entries is a collection of Entry.
// Sortable by time.
type Entries []*Entry

func (e Entries) Len() int      { return len(e) }
func (e Entries) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
func (e Entries) Less(i, j int) bool {
	if e[i].Next.IsZero() {
		return false
	}
	if e[j].Next.IsZero() {
		return true
	}
	return e[i].Next.Before(e[j].Next)
}

// Add adds a spider using a nil root Context
func (in *InMemory) Add(sched Schedule, spider Spider) {
	in.AddWithCtx(sched, spider, nil)
}

// AddWithCtx adds a spider with a root Context passed in the arguments
func (in *InMemory) AddWithCtx(sched Schedule, spider Spider, ctx *Context) {
	entry := &Entry{
		Spider:   spider,
		Schedule: sched,
		Ctx:      ctx,
	}
	if !in.running {
		in.entries = append(in.entries, entry)
		return
	}
	in.addCh <- entry
}

// AddFunc allows to add a spider using an url and a closure.
// It is by default using the GET HTTP method.
func (in *InMemory) AddFunc(sched Schedule, url string, fn func(*Context) error) {
	s := Get(url, fn)
	in.AddWithCtx(sched, s, nil)
}

// Start launch the scheduler.
// It will run in its own goroutine.
// Your code will continue to be execute after calling this function.
func (in *InMemory) Start() {
	in.running = true
	go in.start()
}

func (in *InMemory) start() {
	now := time.Now().Local()
	for _, e := range in.entries {
		e.Next = e.Schedule.Next(now)
	}
	for {
		sort.Sort(in.entries)
		var nextRun time.Time

		if len(in.entries) == 0 {
			// Wait 1 day if there is no spiders to run
			nextRun = now.Add(24 * time.Hour)
		} else {
			nextRun = in.entries[0].Next
		}

		select {
		case <-time.After(nextRun.Sub(now)):
			for _, e := range in.entries {
				if e.Next != nextRun {
					break
				}
				go func(e *Entry) {
					ctx, _ := e.Spider.Setup(e.Ctx)
					go e.Spider.Spin(ctx)
				}(e)
				e.Next = e.Schedule.Next(nextRun)
			}
			continue
		case e := <-in.addCh:
			in.entries = append(in.entries, e)
			e.Next = e.Schedule.Next(now)
		case <-in.stopCh:
			return
		}
		now = time.Now().Local()
	}
}

// Stop the scheduler.
// Should be called after Start.
func (in *InMemory) Stop() {
	in.stopCh <- struct{}{}
	in.running = false
}

// Standard Scheduler
var stdSched = NewScheduler()

// Add adds a spider to the standard scheduler
func Add(sched Schedule, spider Spider) {
	stdSched.Add(sched, spider)
}

// AddFunc allows to add a spider to the standard scheduler using an url and a closure.
func AddFunc(sched Schedule, url string, fn func(*Context) error) {
	stdSched.AddFunc(sched, url, fn)
}

// Start starts the standard scheduler
func Start() {
	stdSched.Start()
}

// Stop stops the standard scheduler
func Stop() {
	stdSched.Stop()
}
<commit_msg>Fix updating effective time<commit_after>package spider

import (
	"sort"
	"time"
)

// InMemory is the default scheduler
type InMemory struct {
	entries Entries
	addCh   chan *Entry
	stopCh  chan struct{}
	running bool
}

// NewScheduler returns a new InMemory scheduler
func NewScheduler() *InMemory {
	return &InMemory{
		addCh:   make(chan *Entry),
		stopCh:  make(chan struct{}),
		entries: nil,
	}
}

// Entry groups a spider, its root context, a Schedule and the Next time the spider must be launched
type Entry struct {
	Spider   Spider
	Schedule Schedule
	Ctx      *Context
	Next     time.Time
}

// Entries is a collection of Entry.
// Sortable by time.
type Entries []*Entry

func (e Entries) Len() int      { return len(e) }
func (e Entries) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
func (e Entries) Less(i, j int) bool {
	if e[i].Next.IsZero() {
		return false
	}
	if e[j].Next.IsZero() {
		return true
	}
	return e[i].Next.Before(e[j].Next)
}

// Add adds a spider using a nil root Context
func (in *InMemory) Add(sched Schedule, spider Spider) {
	in.AddWithCtx(sched, spider, nil)
}

// AddWithCtx adds a spider with a root Context passed in the arguments
func (in *InMemory) AddWithCtx(sched Schedule, spider Spider, ctx *Context) {
	entry := &Entry{
		Spider:   spider,
		Schedule: sched,
		Ctx:      ctx,
	}
	if !in.running {
		in.entries = append(in.entries, entry)
		return
	}
	in.addCh <- entry
}

// AddFunc allows to add a spider using an url and a closure.
// It is by default using the GET HTTP method.
func (in *InMemory) AddFunc(sched Schedule, url string, fn func(*Context) error) {
	s := Get(url, fn)
	in.AddWithCtx(sched, s, nil)
}

// Start launch the scheduler.
// It will run in its own goroutine.
// Your code will continue to be execute after calling this function.
func (in *InMemory) Start() {
	in.running = true
	go in.start()
}

func (in *InMemory) start() {
	now := time.Now().Local()
	for _, e := range in.entries {
		e.Next = e.Schedule.Next(now)
	}
	for {
		sort.Sort(in.entries)
		var nextRun time.Time

		if len(in.entries) == 0 {
			// Wait 1 day if there is no spiders to run
			nextRun = now.Add(24 * time.Hour)
		} else {
			nextRun = in.entries[0].Next
		}

		select {
		case now = <-time.After(nextRun.Sub(now)):
			for _, e := range in.entries {
				if e.Next != nextRun {
					break
				}
				go in.runEntry(e)
				e.Next = e.Schedule.Next(nextRun)
			}
			continue
		case e := <-in.addCh:
			in.entries = append(in.entries, e)
			e.Next = e.Schedule.Next(now)
		case <-in.stopCh:
			return
		}
		now = time.Now().Local()
	}
}

func (in *InMemory) runEntry(e *Entry) {
	ctx, _ := e.Spider.Setup(e.Ctx)
	e.Spider.Spin(ctx)
}

// Stop the scheduler.
// Should be called after Start.
func (in *InMemory) Stop() {
	in.stopCh <- struct{}{}
	in.running = false
}

// Standard Scheduler
var stdSched = NewScheduler()

// Add adds a spider to the standard scheduler
func Add(sched Schedule, spider Spider) {
	stdSched.Add(sched, spider)
}

// AddFunc allows to add a spider to the standard scheduler using an url and a closure.
func AddFunc(sched Schedule, url string, fn func(*Context) error) {
	stdSched.AddFunc(sched, url, fn)
}

// Start starts the standard scheduler
func Start() {
	stdSched.Start()
}

// Stop stops the standard scheduler
func Stop() {
	stdSched.Stop()
}
<|endoftext|>


<commit_before>package structs

import (
	"testing"
)

type champStatus struct {
  Banned  bool
  Picked  bool
  Won     bool
}

func outcomes() map[RiotID]champStatus {
  return map[RiotID]champStatus{
    122: champStatus{ Banned: true, Picked: false, Won: false },
    164: champStatus{ Banned: true, Picked: false, Won: false },
    23: champStatus{ Banned: true, Picked: false, Won: false },
    50: champStatus{ Banned: false, Picked: false, Won: false },
    3: champStatus{ Banned: false, Picked: true, Won: true },
    19: champStatus{ Banned: true, Picked: false, Won: false },
  }
}

// Make sure we are encoding + decoding to storage format correctly.
func TestToFromProto(t *testing.T) {
  samples := rawSamples()

  for _, sample := range samples {
    match := ToMatch(sample)

    buf := match.Bytes()
    m2 := MakeMatch(buf)

  	if match.GameID != m2.GameID {
  		t.Fail()
  	}

  	if match.SeasonID != m2.SeasonID {
  		t.Fail()
  	}

  	if match.GameCreation != m2.GameCreation {
  		t.Fail()
  	}

  	if match.GameDuration != m2.GameDuration {
  		t.Fail()
  	}

    // TODO: Check participants and ban lists as well.
  }
}

// TODO: save some raw json match data for next tests

// Ensure we correctly parse raw API responses into Match's.
func TestCanParse(t *testing.T) {
	matches := rawSamples()

	for _, sample := range matches {
		match := ToMatch(sample)

		if match.GameID != sample.GameID {
			t.Fail()
		}

		if match.SeasonID != sample.SeasonID {
			t.Fail()
		}

		if match.GameCreation != sample.GameCreation {
			t.Fail()
		}

		if match.GameDuration != sample.GameDuration {
			t.Fail()
		}
	}
}

// Test Banned(), Picked(), and Won() with unpacked matches.
func TestCountsUnpacked(t *testing.T) {
  samples := rawSamples()
  matches := make([]Match, 0, len(samples))

  for _, sample := range samples {
    matches = append(matches, ToMatch(sample))
  }

  oc := outcomes()

  for id, outcome := range oc {
    if matches[0].Banned(id) != outcome.Banned {
      t.Fail()
    }

    if matches[0].Picked(id) != outcome.Picked {
      t.Fail()
    }

    if matches[0].Won(id) != outcome.Won {
      t.Fail()
    }
  }
}

// TODO: test Banned(), Picked(), and Won() both packed and unpacked
<commit_msg>A few placeholder test files.<commit_after>package structs

import (
	"testing"
)

// Included in a map[RiotID] to indicate whether an individual champion is
// { banned, picked, won } in a match.
type champStatus struct {
  Banned  bool
  Picked  bool
  Won     bool
}

func outcomes() map[RiotID]champStatus {
  return map[RiotID]champStatus{
    122: champStatus{ Banned: true, Picked: false, Won: false },
    164: champStatus{ Banned: true, Picked: false, Won: false },
    23: champStatus{ Banned: true, Picked: false, Won: false },
    50: champStatus{ Banned: false, Picked: false, Won: false },
    3: champStatus{ Banned: false, Picked: true, Won: true },
    19: champStatus{ Banned: true, Picked: false, Won: false },
  }
}

// Make sure we are encoding + decoding to storage format correctly.
func TestToFromProto(t *testing.T) {
  samples := rawSamples()

  for _, sample := range samples {
    match := ToMatch(sample)

    buf := match.Bytes()
    m2 := MakeMatch(buf)

  	if match.GameID != m2.GameID {
  		t.Fail()
  	}

  	if match.SeasonID != m2.SeasonID {
  		t.Fail()
  	}

  	if match.GameCreation != m2.GameCreation {
  		t.Fail()
  	}

  	if match.GameDuration != m2.GameDuration {
  		t.Fail()
  	}

    // TODO: Check participants and ban lists as well.
  }
}

// TODO: save some raw json match data for next tests

// Ensure we correctly parse raw API responses into Match's.
func TestCanParse(t *testing.T) {
	matches := rawSamples()

	for _, sample := range matches {
		match := ToMatch(sample)

		if match.GameID != sample.GameID {
			t.Fail()
		}

		if match.SeasonID != sample.SeasonID {
			t.Fail()
		}

		if match.GameCreation != sample.GameCreation {
			t.Fail()
		}

		if match.GameDuration != sample.GameDuration {
			t.Fail()
		}
	}
}

// Test Banned(), Picked(), and Won() with unpacked matches.
func TestCountsUnpacked(t *testing.T) {
  samples := rawSamples()
  matches := make([]Match, 0, len(samples))

  for _, sample := range samples {
    matches = append(matches, ToMatch(sample))
  }

  oc := outcomes()

  for id, outcome := range oc {
    if matches[0].Banned(id) != outcome.Banned {
      t.Fail()
    }

    if matches[0].Picked(id) != outcome.Picked {
      t.Fail()
    }

    if matches[0].Won(id) != outcome.Won {
      t.Fail()
    }
  }
}

// TODO: test Banned(), Picked(), and Won() both packed and unpacked
<|endoftext|>


<commit_before>package unit

import (
	"bytes"
	"fmt"
	"sort"
)

// Uniter is an interface representing a type that can be converted
// to a unit.
type Uniter interface {
	Unit() *Unit
}

// Dimension is a type representing an SI base dimension or other
// orthogonal dimension. If a new dimension is desired for a
// domain-specific problem, NewDimension should be used. Integers
// should never be cast as type dimension
//	// Good: Create a package constant with an init function
//	var MyDimension unit.Dimension
//	init(){
//		MyDimension = NewDimension("my")
//	}
//	main(){
//		var := MyDimension(28.2)
//	}
type Dimension int

func (d Dimension) String() string {
	switch {
	case d == reserved:
		return "reserved"
	case d < Dimension(len(symbols)):
		return symbols[d]
	default:
		panic("unit: illegal dimension")
	}
}

const (
	// SI Base Units
	reserved Dimension = iota
	CurrentDim
	LengthDim
	LuminousIntensityDim
	MassDim
	TemperatureDim
	TimeDim
	// Other common SI Dimensions
	AngleDim // e.g. radians
)

var (
	symbols = []string{
		CurrentDim:           "A",
		LengthDim:            "m",
		LuminousIntensityDim: "cd",
		MassDim:              "kg",
		TemperatureDim:       "K",
		TimeDim:              "s",
		AngleDim:             "rad",
	}

	// for guaranteeing there aren't two identical symbols
	dimensions = map[string]Dimension{
		"A":   CurrentDim,
		"m":   LengthDim,
		"cd":  LuminousIntensityDim,
		"kg":  MassDim,
		"K":   TemperatureDim,
		"s":   TimeDim,
		"rad": AngleDim,

		// Reserve common SI symbols
		// base units
		"mol": reserved,
		// prefixes
		"Y":  reserved,
		"Z":  reserved,
		"E":  reserved,
		"P":  reserved,
		"T":  reserved,
		"G":  reserved,
		"M":  reserved,
		"k":  reserved,
		"h":  reserved,
		"da": reserved,
		"d":  reserved,
		"c":  reserved,
		"μ":  reserved,
		"n":  reserved,
		"p":  reserved,
		"f":  reserved,
		"a":  reserved,
		"z":  reserved,
		"y":  reserved,
		// SI Derived units with special symbols
		"sr":  reserved,
		"F":   reserved,
		"C":   reserved,
		"S":   reserved,
		"H":   reserved,
		"V":   reserved,
		"Ω":   reserved,
		"J":   reserved,
		"N":   reserved,
		"Hz":  reserved,
		"lx":  reserved,
		"lm":  reserved,
		"Wb":  reserved,
		"W":   reserved,
		"Pa":  reserved,
		"Bq":  reserved,
		"Gy":  reserved,
		"Sv":  reserved,
		"kat": reserved,
		// Units in use with SI
		"ha": reserved,
		"L":  reserved,
		"l":  reserved,
		// Units in Use Temporarily with SI
		"bar": reserved,
		"b":   reserved,
		"Ci":  reserved,
		"R":   reserved,
		"rd":  reserved,
		"rem": reserved,
	}
)

// TODO: Should we actually reserve "common" SI unit symbols ("N", "J", etc.) so there isn't confusion
// TODO: If we have a fancier ParseUnit, maybe the 'reserved' symbols should be a different map
// 		map[string]string which says how they go?

// Dimensions represent the dimensionality of the unit in powers
// of that dimension. If a key is not present, the power of that
// dimension is zero. Dimensions is used in conjuction with New.
type Dimensions map[Dimension]int

func (d Dimensions) String() string {
	// Map iterates randomly, but print should be in a fixed order. Can't use
	// dimension number, because for user-defined dimension that number may
	// not be fixed from run to run.
	atoms := make(unitPrinters, 0, len(d))
	for dimension, power := range d {
		if power != 0 {
			atoms = append(atoms, atom{dimension, power})
		}
	}
	sort.Sort(atoms)
	var b bytes.Buffer
	for i, a := range atoms {
		if i > 0 {
			b.WriteByte(' ')
		}
		fmt.Fprintf(&b, "%s", a.Dimension)
		if a.pow != 1 {
			fmt.Fprintf(&b, "^%d", a.pow)
		}
	}

	return b.String()
}

type atom struct {
	Dimension
	pow int
}

type unitPrinters []atom

func (u unitPrinters) Len() int {
	return len(u)
}

func (u unitPrinters) Less(i, j int) bool {
	return (u[i].pow > 0 && u[j].pow < 0) || u[i].String() < u[j].String()
}

func (u unitPrinters) Swap(i, j int) {
	u[i], u[j] = u[j], u[i]
}

// NewDimension returns a new dimension variable which will have a
// unique representation across packages to prevent accidental overlap.
// The input string represents a symbol name which will be used for printing
// Unit types. This symbol may not overlap with any of the SI base units
// or other symbols of common use in SI ("kg", "J", "μ", etc.). A list of
// such symbols can be found at http://lamar.colostate.edu/~hillger/basic.htm or
// by consulting the package source. Furthermore, the provided symbol is also
// forbidden from overlapping with other packages calling NewDimension. NewDimension
// is expecting to be used only during initialization, and as such it will panic
// if the symbol matching an existing symbol
// NewDimension should only be called for unit types that are actually orthogonal
// to the base dimensions defined in this package. Please see the package-level
// documentation for further explanation. Calls to NewDimension are not thread safe.
func NewDimension(symbol string) Dimension {
	_, ok := dimensions[symbol]
	if ok {
		panic("unit: dimension string \"" + symbol + "\" already used")
	}
	symbols = append(symbols, symbol)
	d := Dimension(len(symbols))
	dimensions[symbol] = d
	return d
}

// Unit is a type a value with generic SI units. Most useful for
// translating between dimensions, for example, by multiplying
// an acceleration with a mass to get a force. Please see the
// package documentation for further explanation.
type Unit struct {
	dimensions Dimensions // Map for custom dimensions
	formatted  string
	value      float64
}

// New creates a new variable of type Unit which has the value
// specified by value and the dimensions specified by the
// base units struct. The value is always in SI Units.
//
// Example: To create an acceleration of 3 m/s^2, one could do
// myvar := CreateUnit(3.0, &Dimensions{unit.LengthDim: 1, unit.TimeDim: -2})
func New(value float64, d Dimensions) *Unit {
	u := &Unit{
		dimensions: make(map[Dimension]int),
		value:      value,
	}
	for key, val := range d {
		if val != 0 {
			u.dimensions[key] = val
		}
	}
	return u
}

// DimensionsMatch checks if the dimensions of two Uniters are the same.
func DimensionsMatch(a, b Uniter) bool {
	aUnit := a.Unit()
	bUnit := b.Unit()
	if len(aUnit.dimensions) != len(bUnit.dimensions) {
		return false
	}
	for key, val := range aUnit.dimensions {
		if bUnit.dimensions[key] != val {
			return false
		}
	}
	return true
}

// Add adds the function argument to the reciever. Panics if the units of
// the receiver and the argument don't match.
func (u *Unit) Add(uniter Uniter) *Unit {
	a := uniter.Unit()
	if !DimensionsMatch(u, a) {
		panic("unit: mismatched dimensions in addition")
	}
	u.value += a.value
	return u
}

// Unit implements the Uniter interface
func (u *Unit) Unit() *Unit {
	return u
}

// Mul multiply the receiver by the input changing the dimensions
// of the receiver as appropriate. The input is not changed.
func (u *Unit) Mul(uniter Uniter) *Unit {
	a := uniter.Unit()
	for key, val := range a.dimensions {
		if u.dimensions[key] == -val {
			delete(u.dimensions, key)
		} else {
			u.dimensions[key] += val
		}
	}
	u.formatted = ""
	u.value *= a.value
	return u
}

// Div divides the receiver by the argument changing the
// dimensions of the receiver as appropriate.
func (u *Unit) Div(uniter Uniter) *Unit {
	a := uniter.Unit()
	u.value /= a.value
	for key, val := range a.dimensions {
		if u.dimensions[key] == val {
			delete(u.dimensions, key)
		} else {
			u.dimensions[key] -= val
		}
	}
	u.formatted = ""
	return u
}

// Value return the raw value of the unit as a float64. Use of this
// method is, in general, not recommended, though it can be useful
// for printing. Instead, the From type of a specific dimension
// should be used to guarantee dimension consistency.
func (u *Unit) Value() float64 {
	return u.value
}

// Format makes Unit satisfy the fmt.Formatter interface. The unit is formatted
// with dimensions appended. If the power if the dimension is not zero or one,
// symbol^power is appended, if the power is one, just the symbol is appended
// and if the power is zero, nothing is appended. Dimensions are appended
// in order by symbol name with positive powers ahead of negative powers.
func (u *Unit) Format(fs fmt.State, c rune) {
	if u == nil {
		fmt.Fprint(fs, "<nil>")
	}
	switch c {
	case 'v':
		if fs.Flag('#') {
			fmt.Fprintf(fs, "&%#v", *u)
			return
		}
		fallthrough
	case 'e', 'E', 'f', 'F', 'g', 'G':
		p, pOk := fs.Precision()
		if !pOk {
			p = -1
		}
		w, wOk := fs.Width()
		if !wOk {
			w = -1
		}
		fmt.Fprintf(fs, "%*.*"+string(c), w, p, u.value)
	default:
		fmt.Fprintf(fs, "%%!%c(*Unit=%g)", c, u)
		return
	}
	if u.formatted == "" && len(u.dimensions) > 0 {
		u.formatted = u.dimensions.String()
	}
	fmt.Fprintf(fs, " %s", u.formatted)
}
<commit_msg>Avoid unnecessary map accesses<commit_after>package unit

import (
	"bytes"
	"fmt"
	"sort"
)

// Uniter is an interface representing a type that can be converted
// to a unit.
type Uniter interface {
	Unit() *Unit
}

// Dimension is a type representing an SI base dimension or other
// orthogonal dimension. If a new dimension is desired for a
// domain-specific problem, NewDimension should be used. Integers
// should never be cast as type dimension
//	// Good: Create a package constant with an init function
//	var MyDimension unit.Dimension
//	init(){
//		MyDimension = NewDimension("my")
//	}
//	main(){
//		var := MyDimension(28.2)
//	}
type Dimension int

func (d Dimension) String() string {
	switch {
	case d == reserved:
		return "reserved"
	case d < Dimension(len(symbols)):
		return symbols[d]
	default:
		panic("unit: illegal dimension")
	}
}

const (
	// SI Base Units
	reserved Dimension = iota
	CurrentDim
	LengthDim
	LuminousIntensityDim
	MassDim
	TemperatureDim
	TimeDim
	// Other common SI Dimensions
	AngleDim // e.g. radians
)

var (
	symbols = []string{
		CurrentDim:           "A",
		LengthDim:            "m",
		LuminousIntensityDim: "cd",
		MassDim:              "kg",
		TemperatureDim:       "K",
		TimeDim:              "s",
		AngleDim:             "rad",
	}

	// for guaranteeing there aren't two identical symbols
	dimensions = map[string]Dimension{
		"A":   CurrentDim,
		"m":   LengthDim,
		"cd":  LuminousIntensityDim,
		"kg":  MassDim,
		"K":   TemperatureDim,
		"s":   TimeDim,
		"rad": AngleDim,

		// Reserve common SI symbols
		// base units
		"mol": reserved,
		// prefixes
		"Y":  reserved,
		"Z":  reserved,
		"E":  reserved,
		"P":  reserved,
		"T":  reserved,
		"G":  reserved,
		"M":  reserved,
		"k":  reserved,
		"h":  reserved,
		"da": reserved,
		"d":  reserved,
		"c":  reserved,
		"μ":  reserved,
		"n":  reserved,
		"p":  reserved,
		"f":  reserved,
		"a":  reserved,
		"z":  reserved,
		"y":  reserved,
		// SI Derived units with special symbols
		"sr":  reserved,
		"F":   reserved,
		"C":   reserved,
		"S":   reserved,
		"H":   reserved,
		"V":   reserved,
		"Ω":   reserved,
		"J":   reserved,
		"N":   reserved,
		"Hz":  reserved,
		"lx":  reserved,
		"lm":  reserved,
		"Wb":  reserved,
		"W":   reserved,
		"Pa":  reserved,
		"Bq":  reserved,
		"Gy":  reserved,
		"Sv":  reserved,
		"kat": reserved,
		// Units in use with SI
		"ha": reserved,
		"L":  reserved,
		"l":  reserved,
		// Units in Use Temporarily with SI
		"bar": reserved,
		"b":   reserved,
		"Ci":  reserved,
		"R":   reserved,
		"rd":  reserved,
		"rem": reserved,
	}
)

// TODO: Should we actually reserve "common" SI unit symbols ("N", "J", etc.) so there isn't confusion
// TODO: If we have a fancier ParseUnit, maybe the 'reserved' symbols should be a different map
// 		map[string]string which says how they go?

// Dimensions represent the dimensionality of the unit in powers
// of that dimension. If a key is not present, the power of that
// dimension is zero. Dimensions is used in conjuction with New.
type Dimensions map[Dimension]int

func (d Dimensions) String() string {
	// Map iterates randomly, but print should be in a fixed order. Can't use
	// dimension number, because for user-defined dimension that number may
	// not be fixed from run to run.
	atoms := make(unitPrinters, 0, len(d))
	for dimension, power := range d {
		if power != 0 {
			atoms = append(atoms, atom{dimension, power})
		}
	}
	sort.Sort(atoms)
	var b bytes.Buffer
	for i, a := range atoms {
		if i > 0 {
			b.WriteByte(' ')
		}
		fmt.Fprintf(&b, "%s", a.Dimension)
		if a.pow != 1 {
			fmt.Fprintf(&b, "^%d", a.pow)
		}
	}

	return b.String()
}

type atom struct {
	Dimension
	pow int
}

type unitPrinters []atom

func (u unitPrinters) Len() int {
	return len(u)
}

func (u unitPrinters) Less(i, j int) bool {
	return (u[i].pow > 0 && u[j].pow < 0) || u[i].String() < u[j].String()
}

func (u unitPrinters) Swap(i, j int) {
	u[i], u[j] = u[j], u[i]
}

// NewDimension returns a new dimension variable which will have a
// unique representation across packages to prevent accidental overlap.
// The input string represents a symbol name which will be used for printing
// Unit types. This symbol may not overlap with any of the SI base units
// or other symbols of common use in SI ("kg", "J", "μ", etc.). A list of
// such symbols can be found at http://lamar.colostate.edu/~hillger/basic.htm or
// by consulting the package source. Furthermore, the provided symbol is also
// forbidden from overlapping with other packages calling NewDimension. NewDimension
// is expecting to be used only during initialization, and as such it will panic
// if the symbol matching an existing symbol
// NewDimension should only be called for unit types that are actually orthogonal
// to the base dimensions defined in this package. Please see the package-level
// documentation for further explanation. Calls to NewDimension are not thread safe.
func NewDimension(symbol string) Dimension {
	_, ok := dimensions[symbol]
	if ok {
		panic("unit: dimension string \"" + symbol + "\" already used")
	}
	symbols = append(symbols, symbol)
	d := Dimension(len(symbols))
	dimensions[symbol] = d
	return d
}

// Unit is a type a value with generic SI units. Most useful for
// translating between dimensions, for example, by multiplying
// an acceleration with a mass to get a force. Please see the
// package documentation for further explanation.
type Unit struct {
	dimensions Dimensions // Map for custom dimensions
	formatted  string
	value      float64
}

// New creates a new variable of type Unit which has the value
// specified by value and the dimensions specified by the
// base units struct. The value is always in SI Units.
//
// Example: To create an acceleration of 3 m/s^2, one could do
// myvar := CreateUnit(3.0, &Dimensions{unit.LengthDim: 1, unit.TimeDim: -2})
func New(value float64, d Dimensions) *Unit {
	u := &Unit{
		dimensions: make(map[Dimension]int),
		value:      value,
	}
	for key, val := range d {
		if val != 0 {
			u.dimensions[key] = val
		}
	}
	return u
}

// DimensionsMatch checks if the dimensions of two Uniters are the same.
func DimensionsMatch(a, b Uniter) bool {
	aUnit := a.Unit()
	bUnit := b.Unit()
	if len(aUnit.dimensions) != len(bUnit.dimensions) {
		return false
	}
	for key, val := range aUnit.dimensions {
		if bUnit.dimensions[key] != val {
			return false
		}
	}
	return true
}

// Add adds the function argument to the reciever. Panics if the units of
// the receiver and the argument don't match.
func (u *Unit) Add(uniter Uniter) *Unit {
	a := uniter.Unit()
	if !DimensionsMatch(u, a) {
		panic("unit: mismatched dimensions in addition")
	}
	u.value += a.value
	return u
}

// Unit implements the Uniter interface
func (u *Unit) Unit() *Unit {
	return u
}

// Mul multiply the receiver by the input changing the dimensions
// of the receiver as appropriate. The input is not changed.
func (u *Unit) Mul(uniter Uniter) *Unit {
	a := uniter.Unit()
	for key, val := range a.dimensions {
		if d := u.dimensions[key]; d == -val {
			delete(u.dimensions, key)
		} else {
			u.dimensions[key] = d + val
		}
	}
	u.formatted = ""
	u.value *= a.value
	return u
}

// Div divides the receiver by the argument changing the
// dimensions of the receiver as appropriate.
func (u *Unit) Div(uniter Uniter) *Unit {
	a := uniter.Unit()
	u.value /= a.value
	for key, val := range a.dimensions {
		if d := u.dimensions[key]; d == val {
			delete(u.dimensions, key)
		} else {
			u.dimensions[key] = d - val
		}
	}
	u.formatted = ""
	return u
}

// Value return the raw value of the unit as a float64. Use of this
// method is, in general, not recommended, though it can be useful
// for printing. Instead, the From type of a specific dimension
// should be used to guarantee dimension consistency.
func (u *Unit) Value() float64 {
	return u.value
}

// Format makes Unit satisfy the fmt.Formatter interface. The unit is formatted
// with dimensions appended. If the power if the dimension is not zero or one,
// symbol^power is appended, if the power is one, just the symbol is appended
// and if the power is zero, nothing is appended. Dimensions are appended
// in order by symbol name with positive powers ahead of negative powers.
func (u *Unit) Format(fs fmt.State, c rune) {
	if u == nil {
		fmt.Fprint(fs, "<nil>")
	}
	switch c {
	case 'v':
		if fs.Flag('#') {
			fmt.Fprintf(fs, "&%#v", *u)
			return
		}
		fallthrough
	case 'e', 'E', 'f', 'F', 'g', 'G':
		p, pOk := fs.Precision()
		if !pOk {
			p = -1
		}
		w, wOk := fs.Width()
		if !wOk {
			w = -1
		}
		fmt.Fprintf(fs, "%*.*"+string(c), w, p, u.value)
	default:
		fmt.Fprintf(fs, "%%!%c(*Unit=%g)", c, u)
		return
	}
	if u.formatted == "" && len(u.dimensions) > 0 {
		u.formatted = u.dimensions.String()
	}
	fmt.Fprintf(fs, " %s", u.formatted)
}
<|endoftext|>


<commit_before>package unix

import (
	"syscall"
	"os"
	"time"
	"github.com/golang/glog"
)

type WAL struct {
  fd int
  mode string
}

func openWriteAheadFile(filename string, mode string) WAL {
	var file int
	var err error
	switch mode {
	case "osync":
		file, err = syscall.Open(filename, syscall.O_SYNC|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case "dsync":
		file, err = syscall.Open(filename, syscall.O_DSYNC|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case "none", "fsync":
		file, err = syscall.Open(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	default:
		glog.Fatal("PersistenceMode not reconised")
	}
	if err != nil {
		glog.Fatal(err)
	}
  // TOD0: remove hardcoded filesize
  err = syscall.Fallocate(file,0,0,int64( 64 * 1000 * 1000)) // 64MB
  if err != nil {
    glog.Fatal(err)
  }
	return WAL{file,mode}
}

func (w WAL) writeAhead(bytes []byte) {
	startTime := time.Now()
  _, err := syscall.Write(w.fd,bytes[:10])
	if err != nil {
		glog.Fatal(err)
	}
	if w.mode=="fsync" {
  	err = syscall.Fdatasync(w.fd)
		if err != nil {
			glog.Fatal(err)
		}
	}
	if time.Since(startTime) > time.Millisecond {
		glog.Info(" bytes written & synced to persistent log in ", time.Since(startTime).String())
	}
	glog.V(1).Info(" bytes written & synced to persistent log in ", time.Since(startTime).String())
}
<commit_msg>adding option for directIO<commit_after>package unix

import (
	"syscall"
	"os"
	"time"
	"github.com/golang/glog"
)

type WAL struct {
  fd int
  mode string
}

func openWriteAheadFile(filename string, mode string) WAL {
	var file int
	var err error
	switch mode {
	case "osync":
		file, err = syscall.Open(filename, syscall.O_SYNC|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case "dsync":
		file, err = syscall.Open(filename, syscall.O_DSYNC|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case "direct":
		file, err = syscall.Open(filename, syscall.O_DIRECT|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	case "none", "fsync":
		file, err = syscall.Open(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	default:
		glog.Fatal("PersistenceMode not reconised")
	}
	if err != nil {
		glog.Fatal(err)
	}
  // TOD0: remove hardcoded filesize
  err = syscall.Fallocate(file,0,0,int64( 64 * 1000 * 1000)) // 64MB
  if err != nil {
    glog.Fatal(err)
  }
	return WAL{file,mode}
}

func (w WAL) writeAhead(bytes []byte) {
	startTime := time.Now()
  _, err := syscall.Write(w.fd,bytes[:10])
	if err != nil {
		glog.Fatal(err)
	}
	if w.mode=="fsync" || w.mode=="direct" {
  	err = syscall.Fdatasync(w.fd)
		if err != nil {
			glog.Fatal(err)
		}
	}
	if time.Since(startTime) > time.Millisecond {
		glog.Info(" bytes written & synced to persistent log in ", time.Since(startTime).String())
	}
	glog.V(1).Info(" bytes written & synced to persistent log in ", time.Since(startTime).String())
}
<|endoftext|>


<commit_before>package file

import (
	"encoding/json"
	"log"
	"os"

	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/mitchellh/mapstructure"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

func init() {
	outputs.Register("file", func() outputs.Output {
		return &File{
			Cfg: &Config{},
			metrics: []prometheus.Collector{
				NumberOfWrittenBytes,
				NumberOfReceivedMsgs,
				NumberOfWrittenMsgs,
			},
		}
	})
}

// File //
type File struct {
	Cfg     *Config
	file    *os.File
	logger  *log.Logger
	metrics []prometheus.Collector
}

// Config //
type Config struct {
	FileName string `mapstructure:"filename,omitempty"`
	FileType string `mapstructure:"file-type,omitempty"`
	Format   string `mapstructure:"format,omitempty"`
}

func (f *File) String() string {
	b, err := json.Marshal(f)
	if err != nil {
		return ""
	}
	return string(b)
}

// Init //
func (f *File) Init(cfg map[string]interface{}, logger *log.Logger) error {
	c := new(Config)
	err := mapstructure.Decode(cfg, c)
	if err != nil {
		return err
	}
	f.Cfg = c

	var file *os.File
	switch f.Cfg.FileType {
	case "stdout":
		file = os.Stdout
	case "stderr":
		file = os.Stderr
	default:
		file, err = os.OpenFile(f.Cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
	}
	f.file = file
	f.logger = log.New(os.Stderr, "file_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		f.logger.SetOutput(logger.Writer())
		f.logger.SetFlags(logger.Flags())
	}
	if f.Cfg.Format == "" {
		f.Cfg.Format = "json"
	}
	f.logger.Printf("initialized file output: %s", f.String())
	return nil
}

// Write //
func (f *File) Write(rsp proto.Message, meta outputs.Meta) {
	if rsp == nil {
		return
	}
	NumberOfReceivedMsgs.WithLabelValues(f.file.Name()).Inc()
	var multiline bool
	var indent string
	if (f.Cfg.Format == "json" || f.Cfg.Format == "textproto") && (f.Cfg.FileType == "stdout" || f.Cfg.FileType == "stderr") {
		indent = "  "
		multiline = true
	}
	b, err := collector.Marshal(rsp, f.Cfg.Format, meta, multiline, indent)
	if err != nil {
		f.logger.Printf("failed marshaling proto msg: %v", err)
		return
	}
	n, err := f.file.Write(append(b, []byte("\n")...))
	if err != nil {
		f.logger.Printf("failed to write to file '%s': %v", f.file.Name(), err)
		return
	}
	NumberOfWrittenBytes.WithLabelValues(f.file.Name()).Add(float64(n))
	NumberOfWrittenMsgs.WithLabelValues(f.file.Name()).Inc()
}

// Close //
func (f *File) Close() error {
	f.logger.Printf("closing file '%s' output", f.file.Name())
	return f.file.Close()
}

// Metrics //
func (f *File) Metrics() []prometheus.Collector { return f.metrics }
<commit_msg>add file output formatting options<commit_after>package file

import (
	"encoding/json"
	"log"
	"os"

	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/mitchellh/mapstructure"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

func init() {
	outputs.Register("file", func() outputs.Output {
		return &File{
			Cfg: &Config{},
			metrics: []prometheus.Collector{
				NumberOfWrittenBytes,
				NumberOfReceivedMsgs,
				NumberOfWrittenMsgs,
			},
		}
	})
}

// File //
type File struct {
	Cfg     *Config
	file    *os.File
	logger  *log.Logger
	metrics []prometheus.Collector
}

// Config //
type Config struct {
	FileName  string `mapstructure:"filename,omitempty"`
	FileType  string `mapstructure:"file-type,omitempty"`
	Format    string `mapstructure:"format,omitempty"`
	Multiline bool   `mapstructure:"multiline,omitempty"`
	Indent    string `mapstructure:"indent,omitempty"`
}

func (f *File) String() string {
	b, err := json.Marshal(f)
	if err != nil {
		return ""
	}
	return string(b)
}

// Init //
func (f *File) Init(cfg map[string]interface{}, logger *log.Logger) error {
	c := new(Config)
	err := mapstructure.Decode(cfg, c)
	if err != nil {
		return err
	}
	f.Cfg = c

	var file *os.File
	switch f.Cfg.FileType {
	case "stdout":
		file = os.Stdout
	case "stderr":
		file = os.Stderr
	default:
		file, err = os.OpenFile(f.Cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err != nil {
			return err
		}
	}
	f.file = file
	f.logger = log.New(os.Stderr, "file_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		f.logger.SetOutput(logger.Writer())
		f.logger.SetFlags(logger.Flags())
	}
	if f.Cfg.Format == "" {
		f.Cfg.Format = "json"
	}
	if (f.Cfg.Format == "json" || f.Cfg.Format == "textproto") && (f.Cfg.FileType == "stdout" || f.Cfg.FileType == "stderr") {
		f.Cfg.Indent = "  "
		f.Cfg.Multiline = true
	}
	f.logger.Printf("initialized file output: %s", f.String())
	return nil
}

// Write //
func (f *File) Write(rsp proto.Message, meta outputs.Meta) {
	if rsp == nil {
		return
	}
	NumberOfReceivedMsgs.WithLabelValues(f.file.Name()).Inc()
	b, err := collector.Marshal(rsp, f.Cfg.Format, meta, f.Cfg.Multiline, f.Cfg.Indent)
	if err != nil {
		f.logger.Printf("failed marshaling proto msg: %v", err)
		return
	}
	n, err := f.file.Write(append(b, []byte("\n")...))
	if err != nil {
		f.logger.Printf("failed to write to file '%s': %v", f.file.Name(), err)
		return
	}
	NumberOfWrittenBytes.WithLabelValues(f.file.Name()).Add(float64(n))
	NumberOfWrittenMsgs.WithLabelValues(f.file.Name()).Inc()
}

// Close //
func (f *File) Close() error {
	f.logger.Printf("closing file '%s' output", f.file.Name())
	return f.file.Close()
}

// Metrics //
func (f *File) Metrics() []prometheus.Collector { return f.metrics }
<|endoftext|>


<commit_before>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
	"golang.org/x/sync/errgroup"
)

func TestMoveLeader(t *testing.T)        { testMoveLeader(t, true) }
func TestMoveLeaderService(t *testing.T) { testMoveLeader(t, false) }

func testMoveLeader(t *testing.T, auto bool) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	oldLeadIdx := clus.WaitLeader(t)
	oldLeadID := uint64(clus.Members[oldLeadIdx].s.ID())

	// ensure followers go through leader transition while leadership transfer
	idc := make(chan uint64)
	stopc := make(chan struct{})
	defer close(stopc)

	for i := range clus.Members {
		if oldLeadIdx != i {
			go func(m *member) {
				select {
				case idc <- checkLeaderTransition(m, oldLeadID):
				case <-stopc:
				}
			}(clus.Members[i])
		}
	}

	target := uint64(clus.Members[(oldLeadIdx+1)%3].s.ID())
	if auto {
		err := clus.Members[oldLeadIdx].s.TransferLeadership()
		if err != nil {
			t.Fatal(err)
		}
	} else {
		mvc := toGRPC(clus.Client(oldLeadIdx)).Maintenance
		_, err := mvc.MoveLeader(context.TODO(), &pb.MoveLeaderRequest{TargetID: target})
		if err != nil {
			t.Fatal(err)
		}
	}

	// wait until leader transitions have happened
	var newLeadIDs [2]uint64
	for i := range newLeadIDs {
		select {
		case newLeadIDs[i] = <-idc:
		case <-time.After(time.Second):
			t.Fatal("timed out waiting for leader transition")
		}
	}

	// remaining members must agree on the same leader
	if newLeadIDs[0] != newLeadIDs[1] {
		t.Fatalf("expected same new leader %d == %d", newLeadIDs[0], newLeadIDs[1])
	}

	// new leader must be different than the old leader
	if oldLeadID == newLeadIDs[0] {
		t.Fatalf("expected old leader %d != new leader %d", oldLeadID, newLeadIDs[0])
	}

	// if move-leader were used, new leader must match transferee
	if !auto {
		if newLeadIDs[0] != target {
			t.Fatalf("expected new leader %d != target %d", newLeadIDs[0], target)
		}
	}
}

// TestMoveLeaderError ensures that request to non-leader fail.
func TestMoveLeaderError(t *testing.T) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	oldLeadIdx := clus.WaitLeader(t)
	followerIdx := (oldLeadIdx + 1) % 3

	target := uint64(clus.Members[(oldLeadIdx+2)%3].s.ID())

	mvc := toGRPC(clus.Client(followerIdx)).Maintenance
	_, err := mvc.MoveLeader(context.TODO(), &pb.MoveLeaderRequest{TargetID: target})
	if !eqErrGRPC(err, rpctypes.ErrGRPCNotLeader) {
		t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCNotLeader)
	}
}

// TestMoveLeaderToLearnerError ensures that leader transfer to learner member will fail.
func TestMoveLeaderToLearnerError(t *testing.T) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	// we have to add and launch learner member after initial cluster was created, because
	// bootstrapping a cluster with learner member is not supported.
	clus.AddAndLaunchLearnerMember(t)

	learners, err := clus.GetLearnerMembers()
	if err != nil {
		t.Fatalf("failed to get the learner members in cluster: %v", err)
	}
	if len(learners) != 1 {
		t.Fatalf("added 1 learner to cluster, got %d", len(learners))
	}

	learnerID := learners[0].ID
	leaderIdx := clus.WaitLeader(t)
	cli := clus.Client(leaderIdx)
	_, err = cli.MoveLeader(context.Background(), learnerID)
	if err == nil {
		t.Fatalf("expecting leader transfer to learner to fail, got no error")
	}
	expectedErrKeywords := "bad leader transferee"
	if !strings.Contains(err.Error(), expectedErrKeywords) {
		t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error())
	}
}

// TestTransferLeadershipWithLearner ensures TransferLeadership does not timeout due to learner is
// automatically picked by leader as transferee.
func TestTransferLeadershipWithLearner(t *testing.T) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	clus.AddAndLaunchLearnerMember(t)

	learners, err := clus.GetLearnerMembers()
	if err != nil {
		t.Fatalf("failed to get the learner members in cluster: %v", err)
	}
	if len(learners) != 1 {
		t.Fatalf("added 1 learner to cluster, got %d", len(learners))
	}

	leaderIdx := clus.WaitLeader(t)
	errCh := make(chan error, 1)
	go func() {
		// note that this cluster has 1 leader and 1 learner. TransferLeadership should return nil.
		// Leadership transfer is skipped in cluster with 1 voting member.
		errCh <- clus.Members[leaderIdx].s.TransferLeadership()
	}()
	select {
	case err := <-errCh:
		if err != nil {
			t.Errorf("got error during leadership transfer: %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Error("timed out waiting for leader transition")
	}
}

func TestFirstCommitNotification(t *testing.T) {
	BeforeTest(t)
	clusterSize := 3
	cluster := NewClusterV3(t, &ClusterConfig{Size: clusterSize})
	defer cluster.Terminate(t)

	oldLeaderIdx := cluster.WaitLeader(t)
	oldLeaderClient := cluster.Client(oldLeaderIdx)

	newLeaderIdx := (oldLeaderIdx + 1) % clusterSize
	newLeaderId := uint64(cluster.Members[newLeaderIdx].ID())

	notifiers := make(map[int]<-chan struct{}, clusterSize)
	for i, clusterMember := range cluster.Members {
		notifiers[i] = clusterMember.s.FirstCommitInTermNotify()
	}

	_, err := oldLeaderClient.MoveLeader(context.Background(), newLeaderId)

	if err != nil {
		t.Errorf("got error during leadership transfer: %v", err)
	}

	leaderAppliedIndex := cluster.Members[newLeaderIdx].s.AppliedIndex()

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	group, groupContext := errgroup.WithContext(ctx)

	for i, notifier := range notifiers {
		member, notifier := cluster.Members[i], notifier
		group.Go(func() error {
			return checkFirstCommitNotification(groupContext, member, leaderAppliedIndex, notifier)
		})
	}

	err = group.Wait()
	if err != nil {
		t.Error(err)
	}
}

func checkFirstCommitNotification(
	ctx context.Context,
	member *member,
	leaderAppliedIndex uint64,
	notifier <-chan struct{},
) error {
	// wait until server applies all the changes of leader
	for member.s.AppliedIndex() < leaderAppliedIndex {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
			time.Sleep(100 * time.Millisecond)
		}
	}

	select {
	case msg, ok := <-notifier:
		if ok {
			return fmt.Errorf(
				"member with ID %d got message via notifier, msg: %v",
				member.ID(),
				msg,
			)
		}
	default:
		return fmt.Errorf(
			"notification was not triggered, member ID: %d",
			member.ID(),
		)
	}

	return nil
}
<commit_msg>Deflake: TestFirstCommitNotification<commit_after>// Copyright 2017 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
	"golang.org/x/sync/errgroup"
)

func TestMoveLeader(t *testing.T)        { testMoveLeader(t, true) }
func TestMoveLeaderService(t *testing.T) { testMoveLeader(t, false) }

func testMoveLeader(t *testing.T, auto bool) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	oldLeadIdx := clus.WaitLeader(t)
	oldLeadID := uint64(clus.Members[oldLeadIdx].s.ID())

	// ensure followers go through leader transition while leadership transfer
	idc := make(chan uint64)
	stopc := make(chan struct{})
	defer close(stopc)

	for i := range clus.Members {
		if oldLeadIdx != i {
			go func(m *member) {
				select {
				case idc <- checkLeaderTransition(m, oldLeadID):
				case <-stopc:
				}
			}(clus.Members[i])
		}
	}

	target := uint64(clus.Members[(oldLeadIdx+1)%3].s.ID())
	if auto {
		err := clus.Members[oldLeadIdx].s.TransferLeadership()
		if err != nil {
			t.Fatal(err)
		}
	} else {
		mvc := toGRPC(clus.Client(oldLeadIdx)).Maintenance
		_, err := mvc.MoveLeader(context.TODO(), &pb.MoveLeaderRequest{TargetID: target})
		if err != nil {
			t.Fatal(err)
		}
	}

	// wait until leader transitions have happened
	var newLeadIDs [2]uint64
	for i := range newLeadIDs {
		select {
		case newLeadIDs[i] = <-idc:
		case <-time.After(time.Second):
			t.Fatal("timed out waiting for leader transition")
		}
	}

	// remaining members must agree on the same leader
	if newLeadIDs[0] != newLeadIDs[1] {
		t.Fatalf("expected same new leader %d == %d", newLeadIDs[0], newLeadIDs[1])
	}

	// new leader must be different than the old leader
	if oldLeadID == newLeadIDs[0] {
		t.Fatalf("expected old leader %d != new leader %d", oldLeadID, newLeadIDs[0])
	}

	// if move-leader were used, new leader must match transferee
	if !auto {
		if newLeadIDs[0] != target {
			t.Fatalf("expected new leader %d != target %d", newLeadIDs[0], target)
		}
	}
}

// TestMoveLeaderError ensures that request to non-leader fail.
func TestMoveLeaderError(t *testing.T) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	oldLeadIdx := clus.WaitLeader(t)
	followerIdx := (oldLeadIdx + 1) % 3

	target := uint64(clus.Members[(oldLeadIdx+2)%3].s.ID())

	mvc := toGRPC(clus.Client(followerIdx)).Maintenance
	_, err := mvc.MoveLeader(context.TODO(), &pb.MoveLeaderRequest{TargetID: target})
	if !eqErrGRPC(err, rpctypes.ErrGRPCNotLeader) {
		t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCNotLeader)
	}
}

// TestMoveLeaderToLearnerError ensures that leader transfer to learner member will fail.
func TestMoveLeaderToLearnerError(t *testing.T) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	// we have to add and launch learner member after initial cluster was created, because
	// bootstrapping a cluster with learner member is not supported.
	clus.AddAndLaunchLearnerMember(t)

	learners, err := clus.GetLearnerMembers()
	if err != nil {
		t.Fatalf("failed to get the learner members in cluster: %v", err)
	}
	if len(learners) != 1 {
		t.Fatalf("added 1 learner to cluster, got %d", len(learners))
	}

	learnerID := learners[0].ID
	leaderIdx := clus.WaitLeader(t)
	cli := clus.Client(leaderIdx)
	_, err = cli.MoveLeader(context.Background(), learnerID)
	if err == nil {
		t.Fatalf("expecting leader transfer to learner to fail, got no error")
	}
	expectedErrKeywords := "bad leader transferee"
	if !strings.Contains(err.Error(), expectedErrKeywords) {
		t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error())
	}
}

// TestTransferLeadershipWithLearner ensures TransferLeadership does not timeout due to learner is
// automatically picked by leader as transferee.
func TestTransferLeadershipWithLearner(t *testing.T) {
	BeforeTest(t)

	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	clus.AddAndLaunchLearnerMember(t)

	learners, err := clus.GetLearnerMembers()
	if err != nil {
		t.Fatalf("failed to get the learner members in cluster: %v", err)
	}
	if len(learners) != 1 {
		t.Fatalf("added 1 learner to cluster, got %d", len(learners))
	}

	leaderIdx := clus.WaitLeader(t)
	errCh := make(chan error, 1)
	go func() {
		// note that this cluster has 1 leader and 1 learner. TransferLeadership should return nil.
		// Leadership transfer is skipped in cluster with 1 voting member.
		errCh <- clus.Members[leaderIdx].s.TransferLeadership()
	}()
	select {
	case err := <-errCh:
		if err != nil {
			t.Errorf("got error during leadership transfer: %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Error("timed out waiting for leader transition")
	}
}

func TestFirstCommitNotification(t *testing.T) {
	BeforeTest(t)
	ctx := context.Background()
	clusterSize := 3
	cluster := NewClusterV3(t, &ClusterConfig{Size: clusterSize})
	defer cluster.Terminate(t)

	oldLeaderIdx := cluster.WaitLeader(t)
	oldLeaderClient := cluster.Client(oldLeaderIdx)

	newLeaderIdx := (oldLeaderIdx + 1) % clusterSize
	newLeaderId := uint64(cluster.Members[newLeaderIdx].ID())

	notifiers := make(map[int]<-chan struct{}, clusterSize)
	for i, clusterMember := range cluster.Members {
		notifiers[i] = clusterMember.s.FirstCommitInTermNotify()
	}

	_, err := oldLeaderClient.MoveLeader(context.Background(), newLeaderId)

	if err != nil {
		t.Errorf("got error during leadership transfer: %v", err)
	}

	t.Logf("Leadership transferred.")
	t.Logf("Submitting write to make sure empty and 'foo' index entry was already flushed")
	cli := cluster.RandClient()

	if _, err := cli.Put(ctx, "foo", "bar"); err != nil {
		t.Fatalf("Failed to put kv pair.")
	}

	// It's guaranteed now that leader contains the 'foo'->'bar' index entry.
	leaderAppliedIndex := cluster.Members[newLeaderIdx].s.AppliedIndex()

	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	group, groupContext := errgroup.WithContext(ctx)

	for i, notifier := range notifiers {
		member, notifier := cluster.Members[i], notifier
		group.Go(func() error {
			return checkFirstCommitNotification(groupContext, t, member, leaderAppliedIndex, notifier)
		})
	}

	err = group.Wait()
	if err != nil {
		t.Error(err)
	}
}

func checkFirstCommitNotification(
	ctx context.Context,
	t testing.TB,
	member *member,
	leaderAppliedIndex uint64,
	notifier <-chan struct{},
) error {
	// wait until server applies all the changes of leader
	for member.s.AppliedIndex() < leaderAppliedIndex {
		t.Logf("member.s.AppliedIndex():%v <= leaderAppliedIndex:%v", member.s.AppliedIndex(), leaderAppliedIndex)
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
			time.Sleep(100 * time.Millisecond)
		}
	}
	select {
	case msg, ok := <-notifier:
		if ok {
			return fmt.Errorf(
				"member with ID %d got message via notifier, msg: %v",
				member.ID(),
				msg,
			)
		}
	default:
		t.Logf("member.s.AppliedIndex():%v >= leaderAppliedIndex:%v", member.s.AppliedIndex(), leaderAppliedIndex)
		return fmt.Errorf(
			"notification was not triggered, member ID: %d",
			member.ID(),
		)
	}

	return nil
}
<|endoftext|>