<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package local_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"
	"launchpad.net/golxc"

	coreCloudinit "launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/container"
	"launchpad.net/juju-core/container/lxc"
	containertesting "launchpad.net/juju-core/container/testing"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/cloudinit"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/environs/jujutest"
	envtesting "launchpad.net/juju-core/environs/testing"
	"launchpad.net/juju-core/environs/tools"
	"launchpad.net/juju-core/juju/arch"
	"launchpad.net/juju-core/juju/osenv"
	"launchpad.net/juju-core/provider/local"
	"launchpad.net/juju-core/state/api/params"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/upstart"
)

const echoCommandScript = "#!/bin/sh\necho $0 \"$@\" >> $0.args"

type environSuite struct {
	baseProviderSuite
	envtesting.ToolsFixture
}

var _ = gc.Suite(&environSuite{})

func (s *environSuite) SetUpTest(c *gc.C) {
	s.baseProviderSuite.SetUpTest(c)
	s.ToolsFixture.SetUpTest(c)
}

func (s *environSuite) TearDownTest(c *gc.C) {
	s.ToolsFixture.TearDownTest(c)
	s.baseProviderSuite.TearDownTest(c)
}

func (*environSuite) TestOpenFailsWithProtectedDirectories(c *gc.C) {
	testConfig := minimalConfig(c)
	testConfig, err := testConfig.Apply(map[string]interface{}{
		"root-dir": "/usr/lib/juju",
	})
	c.Assert(err, gc.IsNil)

	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.ErrorMatches, "mkdir .* permission denied")
	c.Assert(environ, gc.IsNil)
}

func (s *environSuite) TestNameAndStorage(c *gc.C) {
	testConfig := minimalConfig(c)
	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.IsNil)
	c.Assert(environ.Name(), gc.Equals, "test")
	c.Assert(environ.Storage(), gc.NotNil)
}

func (s *environSuite) TestGetToolsMetadataSources(c *gc.C) {
	testConfig := minimalConfig(c)
	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.IsNil)
	sources, err := tools.GetMetadataSources(environ)
	c.Assert(err, gc.IsNil)
	c.Assert(len(sources), gc.Equals, 1)
	url, err := sources[0].URL("")
	c.Assert(err, gc.IsNil)
	c.Assert(strings.Contains(url, "/tools"), jc.IsTrue)
}

func (*environSuite) TestSupportedArchitectures(c *gc.C) {
	testConfig := minimalConfig(c)
	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.IsNil)
	c.Assert(err, gc.IsNil)
	arches, err := environ.SupportedArchitectures()
	c.Assert(err, gc.IsNil)
	for _, a := range arches {
		c.Assert(arch.IsSupportedArch(a), jc.IsTrue)
	}
}

type localJujuTestSuite struct {
	baseProviderSuite
	jujutest.Tests
	oldUpstartLocation string
	testPath           string
	dbServiceName      string
	fakesudo           string
}

func (s *localJujuTestSuite) SetUpTest(c *gc.C) {
	s.baseProviderSuite.SetUpTest(c)
	// Construct the directories first.
	err := local.CreateDirs(c, minimalConfig(c))
	c.Assert(err, gc.IsNil)
	s.testPath = c.MkDir()
	s.fakesudo = filepath.Join(s.testPath, "sudo")
	s.PatchEnvPathPrepend(s.testPath)

	// Write a fake "sudo" which records its args to sudo.args.
	err = ioutil.WriteFile(s.fakesudo, []byte(echoCommandScript), 0755)
	c.Assert(err, gc.IsNil)

	// Add in an admin secret
	s.Tests.TestConfig["admin-secret"] = "sekrit"
	s.PatchValue(local.CheckIfRoot, func() bool { return false })
	s.Tests.SetUpTest(c)

	cfg, err := config.New(config.NoDefaults, s.TestConfig)
	c.Assert(err, gc.IsNil)
	s.dbServiceName = "juju-db-" + local.ConfigNamespace(cfg)

	s.PatchValue(local.FinishBootstrap, func(mcfg *cloudinit.MachineConfig, cloudcfg *coreCloudinit.Config, ctx environs.BootstrapContext) error {
		return nil
	})
}

func (s *localJujuTestSuite) TearDownTest(c *gc.C) {
	s.Tests.TearDownTest(c)
	s.baseProviderSuite.TearDownTest(c)
}

func (s *localJujuTestSuite) MakeTool(c *gc.C, name, script string) {
	path := filepath.Join(s.testPath, name)
	script = "#!/bin/bash\n" + script
	err := ioutil.WriteFile(path, []byte(script), 0755)
	c.Assert(err, gc.IsNil)
}

func (s *localJujuTestSuite) StoppedStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service stop/waiting"`)
}

func (s *localJujuTestSuite) RunningStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service start/running, process 123"`)
}

var _ = gc.Suite(&localJujuTestSuite{
	Tests: jujutest.Tests{
		TestConfig: minimalConfigValues(),
	},
})

func (s *localJujuTestSuite) TestStartStop(c *gc.C) {
	c.Skip("StartInstance not implemented yet.")
}

func (s *localJujuTestSuite) testBootstrap(c *gc.C, cfg *config.Config) (env environs.Environ) {
	ctx := coretesting.Context(c)
	environ, err := local.Provider.Prepare(ctx, cfg)
	c.Assert(err, gc.IsNil)
	envtesting.UploadFakeTools(c, environ.Storage())
	defer environ.Storage().RemoveAll()
	err = environ.Bootstrap(ctx, constraints.Value{})
	c.Assert(err, gc.IsNil)
	return environ
}

func (s *localJujuTestSuite) TestBootstrap(c *gc.C) {
	s.PatchValue(local.FinishBootstrap, func(mcfg *cloudinit.MachineConfig, cloudcfg *coreCloudinit.Config, ctx environs.BootstrapContext) error {
		c.Assert(cloudcfg.AptUpdate(), jc.IsFalse)
		c.Assert(cloudcfg.AptUpgrade(), jc.IsFalse)
		c.Assert(cloudcfg.Packages(), gc.HasLen, 0)
		c.Assert(mcfg.AgentEnvironment, gc.Not(gc.IsNil))
		// local does not allow machine-0 to host units
		c.Assert(mcfg.Jobs, gc.DeepEquals, []params.MachineJob{params.JobManageEnviron})
		return nil
	})
	s.testBootstrap(c, minimalConfig(c))
}

func (s *localJujuTestSuite) TestDestroy(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	err := env.Destroy()
	// Succeeds because there's no "agents" directory,
	// so destroy will just return without attempting
	// sudo or anything.
	c.Assert(err, gc.IsNil)
	c.Assert(s.fakesudo+".args", jc.DoesNotExist)
}

func (s *localJujuTestSuite) makeAgentsDir(c *gc.C, env environs.Environ) {
	rootDir := env.Config().AllAttrs()["root-dir"].(string)
	agentsDir := filepath.Join(rootDir, "agents")
	err := os.Mkdir(agentsDir, 0755)
	c.Assert(err, gc.IsNil)
}

func (s *localJujuTestSuite) TestDestroyCallSudo(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	s.makeAgentsDir(c, env)
	err := env.Destroy()
	c.Assert(err, gc.IsNil)
	data, err := ioutil.ReadFile(s.fakesudo + ".args")
	c.Assert(err, gc.IsNil)
	expected := []string{
		s.fakesudo,
		"JUJU_HOME=" + osenv.JujuHome(),
		os.Args[0],
		"destroy-environment",
		"-y",
		"--force",
		env.Config().Name(),
	}
	c.Assert(string(data), gc.Equals, strings.Join(expected, " ")+"\n")
}

func (s *localJujuTestSuite) makeFakeUpstartScripts(c *gc.C, env environs.Environ,
) (mongo *upstart.Service, machineAgent *upstart.Service) {
	upstartDir := c.MkDir()
	s.PatchValue(&upstart.InitDir, upstartDir)
	s.MakeTool(c, "start", `echo "some-service start/running, process 123"`)

	namespace := env.Config().AllAttrs()["namespace"].(string)
	mongo = upstart.NewService(fmt.Sprintf("juju-db-%s", namespace))
	mongoConf := upstart.Conf{
		Service: *mongo,
		Desc:    "fake mongo",
		Cmd:     "echo FAKE",
	}
	err := mongoConf.Install()
	c.Assert(err, gc.IsNil)
	c.Assert(mongo.Installed(), jc.IsTrue)

	machineAgent = upstart.NewService(fmt.Sprintf("juju-agent-%s", namespace))
	agentConf := upstart.Conf{
		Service: *machineAgent,
		Desc:    "fake agent",
		Cmd:     "echo FAKE",
	}
	err = agentConf.Install()
	c.Assert(err, gc.IsNil)
	c.Assert(machineAgent.Installed(), jc.IsTrue)

	return mongo, machineAgent
}

func (s *localJujuTestSuite) TestDestroyRemovesUpstartServices(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	s.makeAgentsDir(c, env)
	mongo, machineAgent := s.makeFakeUpstartScripts(c, env)
	s.PatchValue(local.CheckIfRoot, func() bool { return true })

	err := env.Destroy()
	c.Assert(err, gc.IsNil)

	c.Assert(mongo.Installed(), jc.IsFalse)
	c.Assert(machineAgent.Installed(), jc.IsFalse)
}

func (s *localJujuTestSuite) TestDestroyRemovesContainers(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	s.makeAgentsDir(c, env)
	s.PatchValue(local.CheckIfRoot, func() bool { return true })

	namespace := env.Config().AllAttrs()["namespace"].(string)
	manager, err := lxc.NewContainerManager(container.ManagerConfig{
		container.ConfigName:   namespace,
		container.ConfigLogDir: "logdir",
	})
	c.Assert(err, gc.IsNil)

	machine1 := containertesting.CreateContainer(c, manager, "1")

	err = env.Destroy()
	c.Assert(err, gc.IsNil)

	container := s.Factory.New(string(machine1.Id()))
	c.Assert(container.IsConstructed(), jc.IsFalse)
}

func (s *localJujuTestSuite) TestBootstrapRemoveLeftovers(c *gc.C) {
	cfg := minimalConfig(c)
	rootDir := cfg.AllAttrs()["root-dir"].(string)

	// Create a dir inside local/log that should be removed by Bootstrap.
	logThings := filepath.Join(rootDir, "log", "things")
	err := os.MkdirAll(logThings, 0755)
	c.Assert(err, gc.IsNil)

	// Create a cloud-init-output.log in root-dir that should be
	// removed/truncated by Bootstrap.
	cloudInitOutputLog := filepath.Join(rootDir, "cloud-init-output.log")
	err = ioutil.WriteFile(cloudInitOutputLog, []byte("ohai"), 0644)
	c.Assert(err, gc.IsNil)

	s.testBootstrap(c, cfg)
	c.Assert(logThings, jc.DoesNotExist)
	c.Assert(cloudInitOutputLog, jc.DoesNotExist)
	c.Assert(filepath.Join(rootDir, "log"), jc.IsSymlink)
}
<commit_msg>Remove unused import.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package local_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	coreCloudinit "launchpad.net/juju-core/cloudinit"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/container"
	"launchpad.net/juju-core/container/lxc"
	containertesting "launchpad.net/juju-core/container/testing"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/cloudinit"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/environs/jujutest"
	envtesting "launchpad.net/juju-core/environs/testing"
	"launchpad.net/juju-core/environs/tools"
	"launchpad.net/juju-core/juju/arch"
	"launchpad.net/juju-core/juju/osenv"
	"launchpad.net/juju-core/provider/local"
	"launchpad.net/juju-core/state/api/params"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/upstart"
)

const echoCommandScript = "#!/bin/sh\necho $0 \"$@\" >> $0.args"

type environSuite struct {
	baseProviderSuite
	envtesting.ToolsFixture
}

var _ = gc.Suite(&environSuite{})

func (s *environSuite) SetUpTest(c *gc.C) {
	s.baseProviderSuite.SetUpTest(c)
	s.ToolsFixture.SetUpTest(c)
}

func (s *environSuite) TearDownTest(c *gc.C) {
	s.ToolsFixture.TearDownTest(c)
	s.baseProviderSuite.TearDownTest(c)
}

func (*environSuite) TestOpenFailsWithProtectedDirectories(c *gc.C) {
	testConfig := minimalConfig(c)
	testConfig, err := testConfig.Apply(map[string]interface{}{
		"root-dir": "/usr/lib/juju",
	})
	c.Assert(err, gc.IsNil)

	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.ErrorMatches, "mkdir .* permission denied")
	c.Assert(environ, gc.IsNil)
}

func (s *environSuite) TestNameAndStorage(c *gc.C) {
	testConfig := minimalConfig(c)
	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.IsNil)
	c.Assert(environ.Name(), gc.Equals, "test")
	c.Assert(environ.Storage(), gc.NotNil)
}

func (s *environSuite) TestGetToolsMetadataSources(c *gc.C) {
	testConfig := minimalConfig(c)
	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.IsNil)
	sources, err := tools.GetMetadataSources(environ)
	c.Assert(err, gc.IsNil)
	c.Assert(len(sources), gc.Equals, 1)
	url, err := sources[0].URL("")
	c.Assert(err, gc.IsNil)
	c.Assert(strings.Contains(url, "/tools"), jc.IsTrue)
}

func (*environSuite) TestSupportedArchitectures(c *gc.C) {
	testConfig := minimalConfig(c)
	environ, err := local.Provider.Open(testConfig)
	c.Assert(err, gc.IsNil)
	c.Assert(err, gc.IsNil)
	arches, err := environ.SupportedArchitectures()
	c.Assert(err, gc.IsNil)
	for _, a := range arches {
		c.Assert(arch.IsSupportedArch(a), jc.IsTrue)
	}
}

type localJujuTestSuite struct {
	baseProviderSuite
	jujutest.Tests
	oldUpstartLocation string
	testPath           string
	dbServiceName      string
	fakesudo           string
}

func (s *localJujuTestSuite) SetUpTest(c *gc.C) {
	s.baseProviderSuite.SetUpTest(c)
	// Construct the directories first.
	err := local.CreateDirs(c, minimalConfig(c))
	c.Assert(err, gc.IsNil)
	s.testPath = c.MkDir()
	s.fakesudo = filepath.Join(s.testPath, "sudo")
	s.PatchEnvPathPrepend(s.testPath)

	// Write a fake "sudo" which records its args to sudo.args.
	err = ioutil.WriteFile(s.fakesudo, []byte(echoCommandScript), 0755)
	c.Assert(err, gc.IsNil)

	// Add in an admin secret
	s.Tests.TestConfig["admin-secret"] = "sekrit"
	s.PatchValue(local.CheckIfRoot, func() bool { return false })
	s.Tests.SetUpTest(c)

	cfg, err := config.New(config.NoDefaults, s.TestConfig)
	c.Assert(err, gc.IsNil)
	s.dbServiceName = "juju-db-" + local.ConfigNamespace(cfg)

	s.PatchValue(local.FinishBootstrap, func(mcfg *cloudinit.MachineConfig, cloudcfg *coreCloudinit.Config, ctx environs.BootstrapContext) error {
		return nil
	})
}

func (s *localJujuTestSuite) TearDownTest(c *gc.C) {
	s.Tests.TearDownTest(c)
	s.baseProviderSuite.TearDownTest(c)
}

func (s *localJujuTestSuite) MakeTool(c *gc.C, name, script string) {
	path := filepath.Join(s.testPath, name)
	script = "#!/bin/bash\n" + script
	err := ioutil.WriteFile(path, []byte(script), 0755)
	c.Assert(err, gc.IsNil)
}

func (s *localJujuTestSuite) StoppedStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service stop/waiting"`)
}

func (s *localJujuTestSuite) RunningStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service start/running, process 123"`)
}

var _ = gc.Suite(&localJujuTestSuite{
	Tests: jujutest.Tests{
		TestConfig: minimalConfigValues(),
	},
})

func (s *localJujuTestSuite) TestStartStop(c *gc.C) {
	c.Skip("StartInstance not implemented yet.")
}

func (s *localJujuTestSuite) testBootstrap(c *gc.C, cfg *config.Config) (env environs.Environ) {
	ctx := coretesting.Context(c)
	environ, err := local.Provider.Prepare(ctx, cfg)
	c.Assert(err, gc.IsNil)
	envtesting.UploadFakeTools(c, environ.Storage())
	defer environ.Storage().RemoveAll()
	err = environ.Bootstrap(ctx, constraints.Value{})
	c.Assert(err, gc.IsNil)
	return environ
}

func (s *localJujuTestSuite) TestBootstrap(c *gc.C) {
	s.PatchValue(local.FinishBootstrap, func(mcfg *cloudinit.MachineConfig, cloudcfg *coreCloudinit.Config, ctx environs.BootstrapContext) error {
		c.Assert(cloudcfg.AptUpdate(), jc.IsFalse)
		c.Assert(cloudcfg.AptUpgrade(), jc.IsFalse)
		c.Assert(cloudcfg.Packages(), gc.HasLen, 0)
		c.Assert(mcfg.AgentEnvironment, gc.Not(gc.IsNil))
		// local does not allow machine-0 to host units
		c.Assert(mcfg.Jobs, gc.DeepEquals, []params.MachineJob{params.JobManageEnviron})
		return nil
	})
	s.testBootstrap(c, minimalConfig(c))
}

func (s *localJujuTestSuite) TestDestroy(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	err := env.Destroy()
	// Succeeds because there's no "agents" directory,
	// so destroy will just return without attempting
	// sudo or anything.
	c.Assert(err, gc.IsNil)
	c.Assert(s.fakesudo+".args", jc.DoesNotExist)
}

func (s *localJujuTestSuite) makeAgentsDir(c *gc.C, env environs.Environ) {
	rootDir := env.Config().AllAttrs()["root-dir"].(string)
	agentsDir := filepath.Join(rootDir, "agents")
	err := os.Mkdir(agentsDir, 0755)
	c.Assert(err, gc.IsNil)
}

func (s *localJujuTestSuite) TestDestroyCallSudo(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	s.makeAgentsDir(c, env)
	err := env.Destroy()
	c.Assert(err, gc.IsNil)
	data, err := ioutil.ReadFile(s.fakesudo + ".args")
	c.Assert(err, gc.IsNil)
	expected := []string{
		s.fakesudo,
		"JUJU_HOME=" + osenv.JujuHome(),
		os.Args[0],
		"destroy-environment",
		"-y",
		"--force",
		env.Config().Name(),
	}
	c.Assert(string(data), gc.Equals, strings.Join(expected, " ")+"\n")
}

func (s *localJujuTestSuite) makeFakeUpstartScripts(c *gc.C, env environs.Environ,
) (mongo *upstart.Service, machineAgent *upstart.Service) {
	upstartDir := c.MkDir()
	s.PatchValue(&upstart.InitDir, upstartDir)
	s.MakeTool(c, "start", `echo "some-service start/running, process 123"`)

	namespace := env.Config().AllAttrs()["namespace"].(string)
	mongo = upstart.NewService(fmt.Sprintf("juju-db-%s", namespace))
	mongoConf := upstart.Conf{
		Service: *mongo,
		Desc:    "fake mongo",
		Cmd:     "echo FAKE",
	}
	err := mongoConf.Install()
	c.Assert(err, gc.IsNil)
	c.Assert(mongo.Installed(), jc.IsTrue)

	machineAgent = upstart.NewService(fmt.Sprintf("juju-agent-%s", namespace))
	agentConf := upstart.Conf{
		Service: *machineAgent,
		Desc:    "fake agent",
		Cmd:     "echo FAKE",
	}
	err = agentConf.Install()
	c.Assert(err, gc.IsNil)
	c.Assert(machineAgent.Installed(), jc.IsTrue)

	return mongo, machineAgent
}

func (s *localJujuTestSuite) TestDestroyRemovesUpstartServices(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	s.makeAgentsDir(c, env)
	mongo, machineAgent := s.makeFakeUpstartScripts(c, env)
	s.PatchValue(local.CheckIfRoot, func() bool { return true })

	err := env.Destroy()
	c.Assert(err, gc.IsNil)

	c.Assert(mongo.Installed(), jc.IsFalse)
	c.Assert(machineAgent.Installed(), jc.IsFalse)
}

func (s *localJujuTestSuite) TestDestroyRemovesContainers(c *gc.C) {
	env := s.testBootstrap(c, minimalConfig(c))
	s.makeAgentsDir(c, env)
	s.PatchValue(local.CheckIfRoot, func() bool { return true })

	namespace := env.Config().AllAttrs()["namespace"].(string)
	manager, err := lxc.NewContainerManager(container.ManagerConfig{
		container.ConfigName:   namespace,
		container.ConfigLogDir: "logdir",
	})
	c.Assert(err, gc.IsNil)

	machine1 := containertesting.CreateContainer(c, manager, "1")

	err = env.Destroy()
	c.Assert(err, gc.IsNil)

	container := s.Factory.New(string(machine1.Id()))
	c.Assert(container.IsConstructed(), jc.IsFalse)
}

func (s *localJujuTestSuite) TestBootstrapRemoveLeftovers(c *gc.C) {
	cfg := minimalConfig(c)
	rootDir := cfg.AllAttrs()["root-dir"].(string)

	// Create a dir inside local/log that should be removed by Bootstrap.
	logThings := filepath.Join(rootDir, "log", "things")
	err := os.MkdirAll(logThings, 0755)
	c.Assert(err, gc.IsNil)

	// Create a cloud-init-output.log in root-dir that should be
	// removed/truncated by Bootstrap.
	cloudInitOutputLog := filepath.Join(rootDir, "cloud-init-output.log")
	err = ioutil.WriteFile(cloudInitOutputLog, []byte("ohai"), 0644)
	c.Assert(err, gc.IsNil)

	s.testBootstrap(c, cfg)
	c.Assert(logThings, jc.DoesNotExist)
	c.Assert(cloudInitOutputLog, jc.DoesNotExist)
	c.Assert(filepath.Join(rootDir, "log"), jc.IsSymlink)
}
<|endoftext|>


<commit_before>package dnsimple

import (
	"context"
	"log"
	"regexp"
	"strconv"

	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2"

	"github.com/ShowMax/go-fqdn"
	"github.com/arnested/sshfpgo/providers"
	"github.com/arnested/sshfpgo/sshkeygen"
	"github.com/dnsimple/dnsimple-go/dnsimple"
	"github.com/urfave/cli"
)

func init() {
	providers.Register(dnsimpleCommand)
}

func dnsimpleCommand() cli.Command {
	defaultHostname := fqdn.Get()
	apex, err := publicsuffix.EffectiveTLDPlusOne(defaultHostname)
	if err != nil {
		log.Fatal(err)
	}
	return cli.Command{
		Name:  "dnsimple",
		Usage: "Update SSHFP DNS records for DNSimple provider",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "zone",
				Usage: "DNSimple `ZONE`",
				Value: apex,
			},
			cli.StringFlag{
				Name:   "token",
				Usage:  "DNSimple Oauth `TOKEN`",
				EnvVar: "DNSIMPLE_TOKEN",
			},
			cli.BoolFlag{
				Name:  "sandbox",
				Usage: "Run against DNSimples sandbox environment",
			},
		},
		Action: action,
	}
}

func action(c *cli.Context) error {
	verbose := c.GlobalBool("verbose")
	dryRun := c.GlobalBool("dry-run")

	recordMap := sshkeygen.SshfpRecords

	oauthToken := c.String("token")
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken})
	tc := oauth2.NewClient(context.Background(), ts)

	// new client
	client := dnsimple.NewClient(tc)

	if c.Bool("sandbox") {
		client.BaseURL = "https://api.sandbox.dnsimple.com"
	} else {
		client.BaseURL = "https://api.dnsimple.com"
	}

	account, err := client.Accounts.ListAccounts(&dnsimple.ListOptions{})
	if err != nil {
		log.Fatal(err)
	}

	// Define an account id
	accountID := strconv.FormatInt(account.Data[0].ID, 10)

	re := regexp.MustCompile("\\.?" + regexp.QuoteMeta(c.String("zone")) + "$")
	recordName := re.ReplaceAllString(c.GlobalString("hostname"), "")

	zoneResponse, err := client.Zones.ListRecords(accountID, c.String("zone"), &dnsimple.ZoneRecordListOptions{Name: recordName, Type: "SSHFP", ListOptions: dnsimple.ListOptions{}})

	if err != nil {
		log.Fatal(err)
	}

	for _, record := range zoneResponse.Data {
		// Filter by record type SSHFP. ListRecords above
		// should have done that already but didn't.
		if record.Type != "SSHFP" {
			continue
		}

		_, exists := recordMap[record.Content]

		delete(recordMap, record.Content)

		if exists {
			if verbose {
				log.Printf("Skipping up to date SSHFP record: '%v'...", record.Content)
			}

			continue
		}

		if verbose {
			log.Printf("Deletes incorrect SSHFP record: '%v'...", record.Content)
		}

		if dryRun {
			continue
		}

		_, err := client.Zones.DeleteRecord(accountID, record.ZoneID, record.ID)
		if err != nil {
			log.Fatal(err)
		}
	}

	for _, record := range recordMap {
		if verbose {
			log.Printf("Creates new SSHFP record: '%v %v %v'...", record.Algorithm, record.FingerprintType, record.Fingerprint)
		}

		if dryRun {
			continue
		}

		zoneRecord := dnsimple.ZoneRecord{
			Name:    recordName,
			Type:    "SSHFP",
			Content: record.Algorithm + " " + record.FingerprintType + " " + record.Fingerprint,
			Regions: nil,
		}

		_, err := client.Zones.CreateRecord(accountID, c.String("zone"), zoneRecord)

		if err != nil {
			log.Fatal(err)
		}
	}

	return nil
}
<commit_msg>Actually do this check after all<commit_after>package dnsimple

import (
	"context"
	"log"
	"regexp"
	"strconv"

	"golang.org/x/net/publicsuffix"
	"golang.org/x/oauth2"

	"github.com/ShowMax/go-fqdn"
	"github.com/arnested/sshfpgo/providers"
	"github.com/arnested/sshfpgo/sshkeygen"
	"github.com/dnsimple/dnsimple-go/dnsimple"
	"github.com/urfave/cli"
)

func init() {
	providers.Register(dnsimpleCommand)
}

func dnsimpleCommand() cli.Command {
	defaultHostname := fqdn.Get()
	apex, err := publicsuffix.EffectiveTLDPlusOne(defaultHostname)
	if err != nil {
		log.Fatal(err)
	}
	return cli.Command{
		Name:  "dnsimple",
		Usage: "Update SSHFP DNS records for DNSimple provider",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "zone",
				Usage: "DNSimple `ZONE`",
				Value: apex,
			},
			cli.StringFlag{
				Name:   "token",
				Usage:  "DNSimple Oauth `TOKEN`",
				EnvVar: "DNSIMPLE_TOKEN",
			},
			cli.BoolFlag{
				Name:  "sandbox",
				Usage: "Run against DNSimples sandbox environment",
			},
		},
		Action: action,
	}
}

func action(c *cli.Context) error {
	if c.String("token") == "" {
		_ = cli.ShowCommandHelp(c, "dnsimple")
		return cli.NewExitError("You need to provide a DNSimple token", 0)
	}

	verbose := c.GlobalBool("verbose")
	dryRun := c.GlobalBool("dry-run")

	recordMap := sshkeygen.SshfpRecords

	oauthToken := c.String("token")
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken})
	tc := oauth2.NewClient(context.Background(), ts)

	// new client
	client := dnsimple.NewClient(tc)

	if c.Bool("sandbox") {
		client.BaseURL = "https://api.sandbox.dnsimple.com"
	} else {
		client.BaseURL = "https://api.dnsimple.com"
	}

	account, err := client.Accounts.ListAccounts(&dnsimple.ListOptions{})
	if err != nil {
		log.Fatal(err)
	}

	// Define an account id
	accountID := strconv.FormatInt(account.Data[0].ID, 10)

	re := regexp.MustCompile("\\.?" + regexp.QuoteMeta(c.String("zone")) + "$")
	recordName := re.ReplaceAllString(c.GlobalString("hostname"), "")

	zoneResponse, err := client.Zones.ListRecords(accountID, c.String("zone"), &dnsimple.ZoneRecordListOptions{Name: recordName, Type: "SSHFP", ListOptions: dnsimple.ListOptions{}})

	if err != nil {
		log.Fatal(err)
	}

	for _, record := range zoneResponse.Data {
		// Filter by record type SSHFP. ListRecords above
		// should have done that already but didn't.
		if record.Type != "SSHFP" {
			continue
		}

		_, exists := recordMap[record.Content]

		delete(recordMap, record.Content)

		if exists {
			if verbose {
				log.Printf("Skipping up to date SSHFP record: '%v'...", record.Content)
			}

			continue
		}

		if verbose {
			log.Printf("Deletes incorrect SSHFP record: '%v'...", record.Content)
		}

		if dryRun {
			continue
		}

		_, err := client.Zones.DeleteRecord(accountID, record.ZoneID, record.ID)
		if err != nil {
			log.Fatal(err)
		}
	}

	for _, record := range recordMap {
		if verbose {
			log.Printf("Creates new SSHFP record: '%v %v %v'...", record.Algorithm, record.FingerprintType, record.Fingerprint)
		}

		if dryRun {
			continue
		}

		zoneRecord := dnsimple.ZoneRecord{
			Name:    recordName,
			Type:    "SSHFP",
			Content: record.Algorithm + " " + record.FingerprintType + " " + record.Fingerprint,
			Regions: nil,
		}

		_, err := client.Zones.CreateRecord(accountID, c.String("zone"), zoneRecord)

		if err != nil {
			log.Fatal(err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package securepassctl

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const (
	// DefaultRemote is the default Content-Type header used in HTTP requests
	DefaultRemote = "https://beta.secure-pass.net"
	// ContentType is the default Content-Type header used in HTTP requests
	ContentType = "application/json"
	// UserAgent contains the default User-Agent value used in HTTP requests
	UserAgent = "SecurePass CLI"
)

// DebugLogger collects all debug messages
var DebugLogger = log.New(ioutil.Discard, "", log.LstdFlags)

// SecurePass main object type
type SecurePass struct {
	AppID     string `ini:"APP_ID"`
	AppSecret string `ini:"APP_SECRET"`
	Endpoint  string
}

func (s *SecurePass) setupRequestFieds(req *http.Request) {
	req.Header.Set("Accept", ContentType)
	req.Header.Set("Content-Type", ContentType)
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("X-SecurePass-App-ID", s.AppID)
	req.Header.Set("X-SecurePass-App-Secret", s.AppSecret)
}

func (s *SecurePass) makeRequestURL(path string) (string, error) {
	baseURL, _ := url.Parse(s.Endpoint)
	URL, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(URL).String(), nil
}

// NewRequest initializes and issues an HTTP request to the SecurePass endpoint
func (s *SecurePass) NewRequest(method, path string, buf *bytes.Buffer) (*http.Request, error) {
	var err error
	var req *http.Request

	URL, err := s.makeRequestURL(path)
	if err != nil {
		return nil, err
	}
	if buf != nil {
		req, err = http.NewRequest(method, URL, buf)
	} else {
		req, err = http.NewRequest(method, URL, nil)
	}
	if err != nil {
		return nil, err
	}
	s.setupRequestFieds(req)
	return req, nil
}

// DoRequest issues an HTTP request
func (s *SecurePass) DoRequest(req *http.Request, obj APIResponse, expstatus int) error {
	client := NewClient(nil)
	DebugLogger.Printf("Sending %s request to %s", req.Method, req.URL)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != expstatus {
		return fmt.Errorf("%s", resp.Status)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	if obj.ErrorCode() != 0 {
		return fmt.Errorf("%d: %s", obj.ErrorCode(), obj.ErrorMessage())
	}
	return nil
}

// NewClient initialize http.Client with a certain http.Transport
func NewClient(tr *http.Transport) *http.Client {
	// Skip SSL certificate verification
	if tr == nil {
		tr = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	}
	return &http.Client{Transport: tr}
}

// AppInfo retrieves information on a SecurePass application
func (s *SecurePass) AppInfo(app string) (*AppInfoResponse, error) {
	var obj AppInfoResponse

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/info", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppAdd represents /api/v1/apps/add
func (s *SecurePass) AppAdd(app *ApplicationDescriptor) (*AppAddResponse, error) {
	var obj AppAddResponse

	data := url.Values{}
	data.Set("LABEL", app.Label)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}
	if app.Realm != "" {
		data.Set("REALM", app.Realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/add", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppDel deletes an application from SecurePass
func (s *SecurePass) AppDel(app string) (*Response, error) {
	var obj Response

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/delete", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppMod represents /api/v1/apps/modify
func (s *SecurePass) AppMod(appID string, app *ApplicationDescriptor) (*AppInfoResponse, error) {
	var obj AppInfoResponse

	data := url.Values{}
	data.Set("APP_ID", appID)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.Label != "" {
		data.Set("LABEL", app.Label)
	}
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}

	fmt.Println(data)

	req, err := s.NewRequest("POST", "/api/v1/apps/modify", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppList retrieves the list of applications available in SecurePass
func (s *SecurePass) AppList(app *ApplicationDescriptor) (*AppListResponse, error) {
	var obj AppListResponse

	data := url.Values{}
	if app.Realm != "" {
		data.Set("REALM", app.Realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/list", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Logs retrieves application logs
func (s *SecurePass) Logs(realm, start, end string) (*LogsResponse, error) {
	var obj LogsResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}
	if start != "" {
		data.Set("START", start)
	}
	if end != "" {
		data.Set("END", end)
	}

	req, err := s.NewRequest("POST", "/api/v1/logs/get", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// GroupMember issues requests to /api/v1/groups/member
func (s *SecurePass) GroupMember(user, group string) (*GroupMemberResponse, error) {
	var obj GroupMemberResponse

	data := url.Values{}
	if user != "" {
		data.Set("USERNAME", user)
	}
	if group != "" {
		data.Set("GROUP", group)
	}

	req, err := s.NewRequest("POST", "/api/v1/groups/member", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserInfo issues requests to /api/v1/users/info
func (s *SecurePass) UserInfo(username string) (*UserInfoResponse, error) {
	var obj UserInfoResponse

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/info", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserList issues requests to /api/v1/users/list
func (s *SecurePass) UserList(realm string) (*UserListResponse, error) {
	var obj UserListResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/users/list", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAuth issues requests to /api/v1/users/auth
func (s *SecurePass) UserAuth(username, secret string) (*UserAuthResponse, error) {
	var obj UserAuthResponse

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("SECRET", secret)

	req, err := s.NewRequest("POST", "/api/v1/users/auth", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAdd issues requests to /api/v1/users/add
func (s *SecurePass) UserAdd(user *UserDescriptor) (*UserAddResponse, error) {
	var obj UserAddResponse

	data := url.Values{}
	data.Set("USERNAME", user.Username)
	data.Set("NAME", user.Name)
	data.Set("SURNAME", user.Surname)
	data.Set("EMAIL", user.Email)
	data.Set("MOBILE", user.Mobile)
	data.Set("NIN", user.Nin)
	data.Set("RFID", user.Rfid)
	data.Set("MANAGER", user.Manager)

	req, err := s.NewRequest("POST", "/api/v1/users/add", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Ping reprenets the /api/v1/ping API call
func (s *SecurePass) Ping() (*PingResponse, error) {
	var obj PingResponse

	req, err := s.NewRequest("GET", "/api/v1/ping", nil)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}
<commit_msg>Add UserDel API call<commit_after>package securepassctl

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const (
	// DefaultRemote is the default Content-Type header used in HTTP requests
	DefaultRemote = "https://beta.secure-pass.net"
	// ContentType is the default Content-Type header used in HTTP requests
	ContentType = "application/json"
	// UserAgent contains the default User-Agent value used in HTTP requests
	UserAgent = "SecurePass CLI"
)

// DebugLogger collects all debug messages
var DebugLogger = log.New(ioutil.Discard, "", log.LstdFlags)

// SecurePass main object type
type SecurePass struct {
	AppID     string `ini:"APP_ID"`
	AppSecret string `ini:"APP_SECRET"`
	Endpoint  string
}

func (s *SecurePass) setupRequestFieds(req *http.Request) {
	req.Header.Set("Accept", ContentType)
	req.Header.Set("Content-Type", ContentType)
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("X-SecurePass-App-ID", s.AppID)
	req.Header.Set("X-SecurePass-App-Secret", s.AppSecret)
}

func (s *SecurePass) makeRequestURL(path string) (string, error) {
	baseURL, _ := url.Parse(s.Endpoint)
	URL, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(URL).String(), nil
}

// NewRequest initializes and issues an HTTP request to the SecurePass endpoint
func (s *SecurePass) NewRequest(method, path string, buf *bytes.Buffer) (*http.Request, error) {
	var err error
	var req *http.Request

	URL, err := s.makeRequestURL(path)
	if err != nil {
		return nil, err
	}
	if buf != nil {
		req, err = http.NewRequest(method, URL, buf)
	} else {
		req, err = http.NewRequest(method, URL, nil)
	}
	if err != nil {
		return nil, err
	}
	s.setupRequestFieds(req)
	return req, nil
}

// DoRequest issues an HTTP request
func (s *SecurePass) DoRequest(req *http.Request, obj APIResponse, expstatus int) error {
	client := NewClient(nil)
	DebugLogger.Printf("Sending %s request to %s", req.Method, req.URL)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != expstatus {
		return fmt.Errorf("%s", resp.Status)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	if obj.ErrorCode() != 0 {
		return fmt.Errorf("%d: %s", obj.ErrorCode(), obj.ErrorMessage())
	}
	return nil
}

// NewClient initialize http.Client with a certain http.Transport
func NewClient(tr *http.Transport) *http.Client {
	// Skip SSL certificate verification
	if tr == nil {
		tr = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	}
	return &http.Client{Transport: tr}
}

// AppInfo retrieves information on a SecurePass application
func (s *SecurePass) AppInfo(app string) (*AppInfoResponse, error) {
	var obj AppInfoResponse

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/info", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppAdd represents /api/v1/apps/add
func (s *SecurePass) AppAdd(app *ApplicationDescriptor) (*AppAddResponse, error) {
	var obj AppAddResponse

	data := url.Values{}
	data.Set("LABEL", app.Label)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}
	if app.Realm != "" {
		data.Set("REALM", app.Realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/add", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppDel deletes an application from SecurePass
func (s *SecurePass) AppDel(app string) (*Response, error) {
	var obj Response

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/delete", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppMod represents /api/v1/apps/modify
func (s *SecurePass) AppMod(appID string, app *ApplicationDescriptor) (*AppInfoResponse, error) {
	var obj AppInfoResponse

	data := url.Values{}
	data.Set("APP_ID", appID)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.Label != "" {
		data.Set("LABEL", app.Label)
	}
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}

	fmt.Println(data)

	req, err := s.NewRequest("POST", "/api/v1/apps/modify", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppList retrieves the list of applications available in SecurePass
func (s *SecurePass) AppList(app *ApplicationDescriptor) (*AppListResponse, error) {
	var obj AppListResponse

	data := url.Values{}
	if app.Realm != "" {
		data.Set("REALM", app.Realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/list", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Logs retrieves application logs
func (s *SecurePass) Logs(realm, start, end string) (*LogsResponse, error) {
	var obj LogsResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}
	if start != "" {
		data.Set("START", start)
	}
	if end != "" {
		data.Set("END", end)
	}

	req, err := s.NewRequest("POST", "/api/v1/logs/get", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// GroupMember issues requests to /api/v1/groups/member
func (s *SecurePass) GroupMember(user, group string) (*GroupMemberResponse, error) {
	var obj GroupMemberResponse

	data := url.Values{}
	if user != "" {
		data.Set("USERNAME", user)
	}
	if group != "" {
		data.Set("GROUP", group)
	}

	req, err := s.NewRequest("POST", "/api/v1/groups/member", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserInfo issues requests to /api/v1/users/info
func (s *SecurePass) UserInfo(username string) (*UserInfoResponse, error) {
	var obj UserInfoResponse

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/info", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserList issues requests to /api/v1/users/list
func (s *SecurePass) UserList(realm string) (*UserListResponse, error) {
	var obj UserListResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/users/list", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAuth issues requests to /api/v1/users/auth
func (s *SecurePass) UserAuth(username, secret string) (*UserAuthResponse, error) {
	var obj UserAuthResponse

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("SECRET", secret)

	req, err := s.NewRequest("POST", "/api/v1/users/auth", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAdd issues requests to /api/v1/users/add
func (s *SecurePass) UserAdd(user *UserDescriptor) (*UserAddResponse, error) {
	var obj UserAddResponse

	data := url.Values{}
	data.Set("USERNAME", user.Username)
	data.Set("NAME", user.Name)
	data.Set("SURNAME", user.Surname)
	data.Set("EMAIL", user.Email)
	data.Set("MOBILE", user.Mobile)
	data.Set("NIN", user.Nin)
	data.Set("RFID", user.Rfid)
	data.Set("MANAGER", user.Manager)

	req, err := s.NewRequest("POST", "/api/v1/users/add", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserDel deletes a user from SecurePass
func (s *SecurePass) UserDel(username string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/delete", bytes.NewBufferString(data.Encode()))
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Ping reprenets the /api/v1/ping API call
func (s *SecurePass) Ping() (*PingResponse, error) {
	var obj PingResponse

	req, err := s.NewRequest("GET", "/api/v1/ping", nil)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}
<|endoftext|>


<commit_before>package srcgraph

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"sourcegraph.com/sourcegraph/util"

	"github.com/aybabtme/color/brush"
	"github.com/kr/fs"
)

var keepTestFiles = flag.Bool("keep", false, "keep test files around after test is run")
var generate = flag.Bool("generate", false, "generate new expected test data")
var match = flag.String("match", "", "run only test cases that contain this string")

func Test_SrcgraphCmd(t *testing.T) {
	testdir, _ := filepath.Abs("testdata")
	testCases := getTestCases(testdir, *match)
	testwd, _ := os.Getwd()
	var testCaseErrs map[testCase]error

	for _, testCase := range testCases {
		t.Logf("Running test case %+v", testCase)
		err := os.Chdir(testCase.Dir)
		if err != nil {
			t.Fatalf("Could not chdir: %s", err)
		}

		cmd := []string{"-v", "-commit=expected-test", "-conf.cache=false"}
		if !*generate {
			cmd = append(cmd, "-test")
			if *keepTestFiles {
				cmd = append(cmd, "-test-keep")
			}
		}

		if err = make_(cmd); err != nil {
			testCaseErrs[testCase] = err
		}
	}
	os.Chdir(testwd)

	t.Logf("Ran test cases %+v", testCases)
	if len(testCaseErrs) == 0 {
		t.Log(brush.Green("** ALL PASS **").String())
	} else {
		t.Log(brush.Red("** ERRORS **").String())
		for testCase, err := range testCaseErrs {
			t.Log(brush.Red(fmt.Sprintf("Test case %+v: %s", testCase, err)).String())
		}
	}
}

type testCase struct {
	Dir string
}

func getTestCases(testdir string, match string) []testCase {
	var testCases []testCase
	walker := fs.Walk(testdir)
	for walker.Step() {
		path := walker.Path()
		if walker.Stat().IsDir() && util.IsFile(filepath.Join(path, ".git/config")) {
			if strings.Contains(path, match) {
				testCases = append(testCases, testCase{Dir: path})
			}
		}
	}
	return testCases
}
<commit_msg>fix test<commit_after>package srcgraph

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"sourcegraph.com/sourcegraph/util"

	"github.com/aybabtme/color/brush"
	"github.com/kr/fs"
)

var keepTestFiles = flag.Bool("keep", false, "keep test files around after test is run")
var generate = flag.Bool("generate", false, "generate new expected test data")
var match = flag.String("match", "", "run only test cases that contain this string")

func Test_SrcgraphCmd(t *testing.T) {
	testdir, _ := filepath.Abs("testdata")
	testCases := getTestCases(testdir, *match)
	testwd, _ := os.Getwd()
	var testCaseErrs = make(map[testCase]error)

	for _, testCase := range testCases {
		t.Logf("Running test case %+v", testCase)
		err := os.Chdir(testCase.Dir)
		if err != nil {
			t.Fatalf("Could not chdir: %s", err)
		}

		cmd := []string{"-v", "-commit=expected-test", "-conf.cache=false"}
		if !*generate {
			cmd = append(cmd, "-test")
			if *keepTestFiles {
				cmd = append(cmd, "-test-keep")
			}
		}

		if err = make_(cmd); err != nil {
			testCaseErrs[testCase] = err
		}
	}
	os.Chdir(testwd)

	t.Logf("Ran test cases %+v", testCases)
	if len(testCaseErrs) == 0 {
		t.Log(brush.Green("** ALL PASS **").String())
	} else {
		t.Log(brush.Red("** ERRORS **").String())
		for testCase, err := range testCaseErrs {
			t.Log(brush.Red(fmt.Sprintf("Test case %+v: %s", testCase, err)).String())
		}
	}
}

type testCase struct {
	Dir string
}

func getTestCases(testdir string, match string) []testCase {
	var testCases []testCase
	walker := fs.Walk(testdir)
	for walker.Step() {
		path := walker.Path()
		if walker.Stat().IsDir() && util.IsFile(filepath.Join(path, ".git/config")) {
			if strings.Contains(path, match) {
				testCases = append(testCases, testCase{Dir: path})
			}
		}
	}
	return testCases
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package codehost defines the interface implemented by a code hosting source,
// along with support code for use by implementations.
package codehost

import (
	"bytes"
	"cmd/go/internal/str"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// Downloaded size limits.
const (
	MaxGoMod   = 16 << 20  // maximum size of go.mod file
	MaxLICENSE = 16 << 20  // maximum size of LICENSE file
	MaxZipFile = 100 << 20 // maximum size of downloaded zip file
)

// A Repo represents a code hosting source.
// Typical implementations include local version control repositories,
// remote version control servers, and code hosting sites.
type Repo interface {
	// Root returns the import path of the root directory of the repository.
	Root() string

	// List lists all tags with the given prefix.
	Tags(prefix string) (tags []string, err error)

	// Stat returns information about the revision rev.
	// A revision can be any identifier known to the underlying service:
	// commit hash, branch, tag, and so on.
	Stat(rev string) (*RevInfo, error)

	// Latest returns the latest revision on the default branch,
	// whatever that means in the underlying implementation.
	Latest() (*RevInfo, error)

	// ReadFile reads the given file in the file tree corresponding to revision rev.
	// It should refuse to read more than maxSize bytes.
	ReadFile(rev, file string, maxSize int64) (data []byte, err error)

	// ReadZip downloads a zip file for the subdir subdirectory
	// of the given revision to a new file in a given temporary directory.
	// It should refuse to read more than maxSize bytes.
	// It returns a ReadCloser for a streamed copy of the zip file,
	// along with the actual subdirectory (possibly shorter than subdir)
	// contained in the zip file. All files in the zip file are expected to be
	// nested in a single top-level directory, whose name is not specified.
	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error)
}

// A Rev describes a single revision in a source code repository.
type RevInfo struct {
	Name    string    // complete ID in underlying repository
	Short   string    // shortened ID, for use in pseudo-version
	Version string    // TODO what is this?
	Time    time.Time // commit time
}

// AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
func AllHex(rev string) bool {
	for i := 0; i < len(rev); i++ {
		c := rev[i]
		if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
			continue
		}
		return false
	}
	return true
}

// ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
// used in pseudo-versions (12 hex digits).
func ShortenSHA1(rev string) string {
	if AllHex(rev) && len(rev) == 40 {
		return rev[:12]
	}
	return rev
}

// WorkRoot is the root of the cached work directory.
// It is set by cmd/go/internal/vgo.InitMod.
var WorkRoot string

// WorkDir returns the name of the cached work directory to use for the
// given repository type and name.
func WorkDir(typ, name string) (string, error) {
	if WorkRoot == "" {
		return "", fmt.Errorf("codehost.WorkRoot not set")
	}

	// We name the work directory for the SHA256 hash of the type and name.
	// We intentionally avoid the actual name both because of possible
	// conflicts with valid file system paths and because we want to ensure
	// that one checkout is never nested inside another. That nesting has
	// led to security problems in the past.
	if strings.Contains(typ, ":") {
		return "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
	}
	key := typ + ":" + name
	dir := filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
	data, err := ioutil.ReadFile(dir + ".info")
	if err == nil {
		have := strings.TrimSuffix(string(data), "\n")
		if have != key {
			return "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
		}
		_, err := os.Stat(dir)
		if err != nil {
			return "", fmt.Errorf("%s exists but %s does not", dir+".info", dir)
		}
		return dir, nil
	}

	os.RemoveAll(dir)
	if err := os.MkdirAll(dir, 0777); err != nil {
		return "", err
	}
	if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil {
		os.RemoveAll(dir)
		return "", err
	}
	return dir, nil
}

type RunError struct {
	Cmd    string
	Err    error
	Stderr []byte
}

func (e *RunError) Error() string {
	text := e.Cmd + ": " + e.Err.Error()
	stderr := bytes.TrimRight(e.Stderr, "\n")
	if len(stderr) > 0 {
		text += ":\n\t" + strings.Replace(string(stderr), "\n", "\n\t", -1)
	}
	return text
}

var DebugRun bool

// Run runs the command line in the given directory
// (an empty dir means the current directory).
// It returns the standard output and, for a non-zero exit,
// a *RunError indicating the command, exit status, and standard error.
// Standard error is unavailable for commands that exit successfully.
func Run(dir string, cmdline ...interface{}) ([]byte, error) {
	cmd := str.StringList(cmdline...)
	if DebugRun {
		fmt.Fprintf(os.Stderr, "codehost.Run[%s]: %s\n", dir, strings.Join(cmd, " "))
	}
	// TODO: Impose limits on command output size.
	// TODO: Set environment to get English error messages.
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	c := exec.Command(cmd[0], cmd[1:]...)
	c.Dir = dir
	c.Stderr = &stderr
	c.Stdout = &stdout
	err := c.Run()
	if err != nil {
		err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
	}
	return stdout.Bytes(), err
}
<commit_msg>cmd/go/internal/modfetch/codehost: raise maximum repo size limit<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package codehost defines the interface implemented by a code hosting source,
// along with support code for use by implementations.
package codehost

import (
	"bytes"
	"cmd/go/internal/str"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// Downloaded size limits.
const (
	MaxGoMod   = 16 << 20  // maximum size of go.mod file
	MaxLICENSE = 16 << 20  // maximum size of LICENSE file
	MaxZipFile = 500 << 20 // maximum size of downloaded zip file
)

// A Repo represents a code hosting source.
// Typical implementations include local version control repositories,
// remote version control servers, and code hosting sites.
type Repo interface {
	// Root returns the import path of the root directory of the repository.
	Root() string

	// List lists all tags with the given prefix.
	Tags(prefix string) (tags []string, err error)

	// Stat returns information about the revision rev.
	// A revision can be any identifier known to the underlying service:
	// commit hash, branch, tag, and so on.
	Stat(rev string) (*RevInfo, error)

	// Latest returns the latest revision on the default branch,
	// whatever that means in the underlying implementation.
	Latest() (*RevInfo, error)

	// ReadFile reads the given file in the file tree corresponding to revision rev.
	// It should refuse to read more than maxSize bytes.
	ReadFile(rev, file string, maxSize int64) (data []byte, err error)

	// ReadZip downloads a zip file for the subdir subdirectory
	// of the given revision to a new file in a given temporary directory.
	// It should refuse to read more than maxSize bytes.
	// It returns a ReadCloser for a streamed copy of the zip file,
	// along with the actual subdirectory (possibly shorter than subdir)
	// contained in the zip file. All files in the zip file are expected to be
	// nested in a single top-level directory, whose name is not specified.
	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error)
}

// A Rev describes a single revision in a source code repository.
type RevInfo struct {
	Name    string    // complete ID in underlying repository
	Short   string    // shortened ID, for use in pseudo-version
	Version string    // TODO what is this?
	Time    time.Time // commit time
}

// AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
func AllHex(rev string) bool {
	for i := 0; i < len(rev); i++ {
		c := rev[i]
		if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
			continue
		}
		return false
	}
	return true
}

// ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
// used in pseudo-versions (12 hex digits).
func ShortenSHA1(rev string) string {
	if AllHex(rev) && len(rev) == 40 {
		return rev[:12]
	}
	return rev
}

// WorkRoot is the root of the cached work directory.
// It is set by cmd/go/internal/vgo.InitMod.
var WorkRoot string

// WorkDir returns the name of the cached work directory to use for the
// given repository type and name.
func WorkDir(typ, name string) (string, error) {
	if WorkRoot == "" {
		return "", fmt.Errorf("codehost.WorkRoot not set")
	}

	// We name the work directory for the SHA256 hash of the type and name.
	// We intentionally avoid the actual name both because of possible
	// conflicts with valid file system paths and because we want to ensure
	// that one checkout is never nested inside another. That nesting has
	// led to security problems in the past.
	if strings.Contains(typ, ":") {
		return "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
	}
	key := typ + ":" + name
	dir := filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
	data, err := ioutil.ReadFile(dir + ".info")
	if err == nil {
		have := strings.TrimSuffix(string(data), "\n")
		if have != key {
			return "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
		}
		_, err := os.Stat(dir)
		if err != nil {
			return "", fmt.Errorf("%s exists but %s does not", dir+".info", dir)
		}
		return dir, nil
	}

	os.RemoveAll(dir)
	if err := os.MkdirAll(dir, 0777); err != nil {
		return "", err
	}
	if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil {
		os.RemoveAll(dir)
		return "", err
	}
	return dir, nil
}

type RunError struct {
	Cmd    string
	Err    error
	Stderr []byte
}

func (e *RunError) Error() string {
	text := e.Cmd + ": " + e.Err.Error()
	stderr := bytes.TrimRight(e.Stderr, "\n")
	if len(stderr) > 0 {
		text += ":\n\t" + strings.Replace(string(stderr), "\n", "\n\t", -1)
	}
	return text
}

var DebugRun bool

// Run runs the command line in the given directory
// (an empty dir means the current directory).
// It returns the standard output and, for a non-zero exit,
// a *RunError indicating the command, exit status, and standard error.
// Standard error is unavailable for commands that exit successfully.
func Run(dir string, cmdline ...interface{}) ([]byte, error) {
	cmd := str.StringList(cmdline...)
	if DebugRun {
		fmt.Fprintf(os.Stderr, "codehost.Run[%s]: %s\n", dir, strings.Join(cmd, " "))
	}
	// TODO: Impose limits on command output size.
	// TODO: Set environment to get English error messages.
	var stderr bytes.Buffer
	var stdout bytes.Buffer
	c := exec.Command(cmd[0], cmd[1:]...)
	c.Dir = dir
	c.Stderr = &stderr
	c.Stdout = &stdout
	err := c.Run()
	if err != nil {
		err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
	}
	return stdout.Bytes(), err
}
<|endoftext|>


<commit_before>package signalfx

import (
	"encoding/json"
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"bytes"
	"context"

	"github.com/gorilla/mux"
	"github.com/signalfx/com_signalfx_metrics_protobuf"
	"github.com/signalfx/gateway/logkey"
	"github.com/signalfx/gateway/protocol"
	"github.com/signalfx/gateway/protocol/collectd"
	"github.com/signalfx/gateway/protocol/signalfx/tagreplace"
	"github.com/signalfx/gateway/protocol/zipper"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/datapoint/dpsink"
	"github.com/signalfx/golib/errors"
	"github.com/signalfx/golib/log"
	"github.com/signalfx/golib/pointer"
	"github.com/signalfx/golib/sfxclient"
	"github.com/signalfx/golib/web"
)

// ListenerServer controls listening on a socket for SignalFx connections
type ListenerServer struct {
	protocol.CloseableHealthCheck
	listener net.Listener
	logger   log.Logger

	internalCollectors sfxclient.Collector
	metricHandler      metricHandler
}

// Close the exposed socket listening for new connections
func (streamer *ListenerServer) Close() error {
	return streamer.listener.Close()
}

// Addr returns the currently listening address
func (streamer *ListenerServer) Addr() net.Addr {
	return streamer.listener.Addr()
}

// Datapoints returns the datapoints about various internal endpoints
func (streamer *ListenerServer) Datapoints() []*datapoint.Datapoint {
	return append(streamer.internalCollectors.Datapoints(), streamer.HealthDatapoints()...)
}

// MericTypeGetter is an old metric interface that returns the type of a metric name
type MericTypeGetter interface {
	GetMetricTypeFromMap(metricName string) com_signalfx_metrics_protobuf.MetricType
}

// ErrorReader are datapoint streamers that read from a HTTP request and return errors if
// the stream is invalid
type ErrorReader interface {
	Read(ctx context.Context, req *http.Request) error
}

// ErrorTrackerHandler behaves like a http handler, but tracks error returns from a ErrorReader
type ErrorTrackerHandler struct {
	TotalErrors int64
	reader      ErrorReader
	Logger      log.Logger
}

// Datapoints gets TotalErrors stats
func (e *ErrorTrackerHandler) Datapoints() []*datapoint.Datapoint {
	return []*datapoint.Datapoint{
		sfxclient.Cumulative("total_errors", nil, atomic.LoadInt64(&e.TotalErrors)),
	}
}

// ServeHTTPC will serve the wrapped ErrorReader and return the error (if any) to rw if ErrorReader
// fails
func (e *ErrorTrackerHandler) ServeHTTPC(ctx context.Context, rw http.ResponseWriter, req *http.Request) {
	if err := e.reader.Read(ctx, req); err != nil {
		atomic.AddInt64(&e.TotalErrors, 1)
		rw.WriteHeader(http.StatusBadRequest)
		_, err = rw.Write([]byte(err.Error()))
		log.IfErr(e.Logger, err)
		return
	}
	_, err := rw.Write([]byte(`"OK"`))
	log.IfErr(e.Logger, err)
}

// ListenerConfig controls optional parameters for the listener
type ListenerConfig struct {
	ListenAddr                         *string
	HealthCheck                        *string
	Timeout                            *time.Duration
	Logger                             log.Logger
	RootContext                        context.Context
	JSONMarshal                        func(v interface{}) ([]byte, error)
	DebugContext                       *web.HeaderCtxFlag
	HTTPChain                          web.NextConstructor
	SpanNameReplacementRules           []string
	SpanNameReplacementBreakAfterMatch *bool
}

var defaultListenerConfig = &ListenerConfig{
	ListenAddr:                         pointer.String("127.0.0.1:12345"),
	HealthCheck:                        pointer.String("/healthz"),
	Timeout:                            pointer.Duration(time.Second * 30),
	Logger:                             log.Discard,
	RootContext:                        context.Background(),
	JSONMarshal:                        json.Marshal,
	SpanNameReplacementRules:           []string{},
	SpanNameReplacementBreakAfterMatch: pointer.Bool(true),
}

type metricHandler struct {
	metricCreationsMapMutex sync.Mutex
	metricCreationsMap      map[string]com_signalfx_metrics_protobuf.MetricType
	jsonMarshal             func(v interface{}) ([]byte, error)
	logger                  log.Logger
}

func (handler *metricHandler) ServeHTTP(writter http.ResponseWriter, req *http.Request) {
	dec := json.NewDecoder(req.Body)
	var d []MetricCreationStruct
	if err := dec.Decode(&d); err != nil {
		handler.logger.Log(log.Err, err, "Invalid metric creation request")
		writter.WriteHeader(http.StatusBadRequest)
		_, err = writter.Write([]byte(`{msg:"Invalid creation request"}`))
		log.IfErr(handler.logger, err)
		return
	}
	handler.metricCreationsMapMutex.Lock()
	defer handler.metricCreationsMapMutex.Unlock()
	ret := []MetricCreationResponse{}
	for _, m := range d {
		metricType, ok := com_signalfx_metrics_protobuf.MetricType_value[m.MetricType]
		if !ok {
			writter.WriteHeader(http.StatusBadRequest)
			_, err := writter.Write([]byte(`{msg:"Invalid metric type"}`))
			log.IfErr(handler.logger, err)
			return
		}
		handler.metricCreationsMap[m.MetricName] = com_signalfx_metrics_protobuf.MetricType(metricType)
		ret = append(ret, MetricCreationResponse{Code: 409})
	}
	toWrite, err := handler.jsonMarshal(ret)
	if err != nil {
		handler.logger.Log(log.Err, err, "Unable to marshal json")
		writter.WriteHeader(http.StatusBadRequest)
		_, err = writter.Write([]byte(`{msg:"Unable to marshal json!"}`))
		log.IfErr(handler.logger, err)
		return
	}
	writter.WriteHeader(http.StatusOK)
	_, err = writter.Write(toWrite)
	log.IfErr(handler.logger, err)
}

func (handler *metricHandler) GetMetricTypeFromMap(metricName string) com_signalfx_metrics_protobuf.MetricType {
	handler.metricCreationsMapMutex.Lock()
	defer handler.metricCreationsMapMutex.Unlock()
	mt, ok := handler.metricCreationsMap[metricName]
	if !ok {
		return com_signalfx_metrics_protobuf.MetricType_GAUGE
	}
	return mt
}

// NewListener servers http requests for Signalfx datapoints
func NewListener(sink Sink, conf *ListenerConfig) (*ListenerServer, error) {
	conf = pointer.FillDefaultFrom(conf, defaultListenerConfig).(*ListenerConfig)

	listener, err := net.Listen("tcp", *conf.ListenAddr)
	if err != nil {
		return nil, errors.Annotatef(err, "cannot open listening address %s", *conf.ListenAddr)
	}
	r := mux.NewRouter()

	server := http.Server{
		Handler:      r,
		Addr:         *conf.ListenAddr,
		ReadTimeout:  *conf.Timeout,
		WriteTimeout: *conf.Timeout,
	}
	listenServer := ListenerServer{
		listener: listener,
		logger:   conf.Logger,
		metricHandler: metricHandler{
			metricCreationsMap: make(map[string]com_signalfx_metrics_protobuf.MetricType),
			logger:             log.NewContext(conf.Logger).With(logkey.Struct, "metricHandler"),
			jsonMarshal:        conf.JSONMarshal,
		},
	}
	listenServer.SetupHealthCheck(conf.HealthCheck, r, conf.Logger)

	r.Handle("/v1/metric", &listenServer.metricHandler)
	r.Handle("/metric", &listenServer.metricHandler)

	traceSink := sink
	if len(conf.SpanNameReplacementRules) > 0 {
		var err error
		traceSink, err = tagreplace.New(conf.SpanNameReplacementRules, *conf.SpanNameReplacementBreakAfterMatch, sink)
		if err != nil {
			return nil, errors.Annotatef(err, "cannot parse tag replacement rules %v", conf.SpanNameReplacementRules)
		}
	}

	listenServer.internalCollectors = sfxclient.NewMultiCollector(
		setupNotFoundHandler(conf.RootContext, r),
		setupProtobufV1(conf.RootContext, r, sink, &listenServer.metricHandler, conf.Logger, conf.HTTPChain),
		setupJSONV1(conf.RootContext, r, sink, &listenServer.metricHandler, conf.Logger, conf.HTTPChain),
		setupProtobufV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupProtobufEventV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupJSONV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupJSONEventV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupCollectd(conf.RootContext, r, sink, conf.DebugContext, conf.HTTPChain, conf.Logger),
		setupThriftTraceV1(conf.RootContext, r, traceSink, conf.Logger, conf.HTTPChain),
		setupJSONTraceV1(conf.RootContext, r, traceSink, conf.Logger, conf.HTTPChain),
	)

	go func() {
		log.IfErr(conf.Logger, server.Serve(listener))
	}()
	return &listenServer, err
}

func setupNotFoundHandler(ctx context.Context, r *mux.Router) sfxclient.Collector {
	metricTracking := web.RequestCounter{}
	r.NotFoundHandler = web.NewHandler(ctx, web.FromHTTP(http.NotFoundHandler())).Add(web.NextHTTP(metricTracking.ServeHTTP))
	return &sfxclient.WithDimensions{
		Dimensions: map[string]string{"http_endpoint": "http404"},
		Collector:  &metricTracking,
	}
}

// SetupChain wraps the reader returned by getReader in an http.Handler along
// with some middleware that calculates internal metrics about requests.
func SetupChain(ctx context.Context, sink Sink, chainType string, getReader func(Sink) ErrorReader, httpChain web.NextConstructor, logger log.Logger, moreConstructors ...web.Constructor) (http.Handler, sfxclient.Collector) {
	zippers := zipper.NewZipper()

	counter := &dpsink.Counter{
		Logger: logger,
	}

	ucount := UnifyNextSinkWrap(counter)
	finalSink := FromChain(sink, NextWrap(ucount))
	errReader := getReader(finalSink)
	errorTracker := ErrorTrackerHandler{
		reader: errReader,
		Logger: logger,
	}
	metricTracking := web.RequestCounter{}
	handler := web.NewHandler(ctx, &errorTracker).Add(web.NextHTTP(metricTracking.ServeHTTP)).Add(httpChain)
	for _, c := range moreConstructors {
		handler.Add(c)
	}
	st := &sfxclient.WithDimensions{
		Collector: sfxclient.NewMultiCollector(
			&metricTracking,
			&errorTracker,
			counter,
			zippers,
		),
		Dimensions: map[string]string{
			"http_endpoint": "sfx_" + chainType,
		},
	}
	return zippers.GzipHandler(handler), st
}

// SetupJSONByPaths tells the router which paths the given handler (which should handle the given
// endpoint) should see
func SetupJSONByPaths(r *mux.Router, handler http.Handler, endpoint string) {
	r.Path(endpoint).Methods("POST").Headers("Content-Type", "application/json").Handler(handler)
	r.Path(endpoint).Methods("POST").Headers("Content-Type", "application/json; charset=UTF-8").Handler(handler)
	r.Path(endpoint).Methods("POST").Headers("Content-Type", "").HandlerFunc(web.InvalidContentType)
	r.Path(endpoint).Methods("POST").Handler(handler)
}

func setupCollectd(ctx context.Context, r *mux.Router, sink dpsink.Sink, debugContext *web.HeaderCtxFlag, httpChain web.NextConstructor, logger log.Logger) sfxclient.Collector {
	counter := &dpsink.Counter{
		Logger: logger,
	}
	finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter))
	decoder := collectd.JSONDecoder{
		Logger: logger,
		SendTo: finalSink,
	}
	metricTracking := &web.RequestCounter{}
	httpHandler := web.NewHandler(ctx, &decoder).Add(web.NextHTTP(metricTracking.ServeHTTP), debugContext, httpChain)
	collectd.SetupCollectdPaths(r, httpHandler, "/v1/collectd")
	return &sfxclient.WithDimensions{
		Collector: sfxclient.NewMultiCollector(
			metricTracking,
			counter,
			&decoder,
		),
		Dimensions: map[string]string{
			"type": "collectd",
		},
	}
}

func readFromRequest(jeff *bytes.Buffer, req *http.Request, logger log.Logger) error {
	// for compressed transactions, contentLength isn't trustworthy
	readLen, err := jeff.ReadFrom(req.Body)
	if err != nil {
		logger.Log(log.Err, err, logkey.ReadLen, readLen, logkey.ContentLength, req.ContentLength, "Unable to fully read from buffer")
		return err
	}
	return nil
}
<commit_msg>Update proxy to use same nomenclature as sbingest http_endpoint -> protocol<commit_after>package signalfx

import (
	"encoding/json"
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"bytes"
	"context"

	"github.com/gorilla/mux"
	"github.com/signalfx/com_signalfx_metrics_protobuf"
	"github.com/signalfx/gateway/logkey"
	"github.com/signalfx/gateway/protocol"
	"github.com/signalfx/gateway/protocol/collectd"
	"github.com/signalfx/gateway/protocol/signalfx/tagreplace"
	"github.com/signalfx/gateway/protocol/zipper"
	"github.com/signalfx/golib/datapoint"
	"github.com/signalfx/golib/datapoint/dpsink"
	"github.com/signalfx/golib/errors"
	"github.com/signalfx/golib/log"
	"github.com/signalfx/golib/pointer"
	"github.com/signalfx/golib/sfxclient"
	"github.com/signalfx/golib/web"
)

// ListenerServer controls listening on a socket for SignalFx connections
type ListenerServer struct {
	protocol.CloseableHealthCheck
	listener net.Listener
	logger   log.Logger

	internalCollectors sfxclient.Collector
	metricHandler      metricHandler
}

// Close the exposed socket listening for new connections
func (streamer *ListenerServer) Close() error {
	return streamer.listener.Close()
}

// Addr returns the currently listening address
func (streamer *ListenerServer) Addr() net.Addr {
	return streamer.listener.Addr()
}

// Datapoints returns the datapoints about various internal endpoints
func (streamer *ListenerServer) Datapoints() []*datapoint.Datapoint {
	return append(streamer.internalCollectors.Datapoints(), streamer.HealthDatapoints()...)
}

// MericTypeGetter is an old metric interface that returns the type of a metric name
type MericTypeGetter interface {
	GetMetricTypeFromMap(metricName string) com_signalfx_metrics_protobuf.MetricType
}

// ErrorReader are datapoint streamers that read from a HTTP request and return errors if
// the stream is invalid
type ErrorReader interface {
	Read(ctx context.Context, req *http.Request) error
}

// ErrorTrackerHandler behaves like a http handler, but tracks error returns from a ErrorReader
type ErrorTrackerHandler struct {
	TotalErrors int64
	reader      ErrorReader
	Logger      log.Logger
}

// Datapoints gets TotalErrors stats
func (e *ErrorTrackerHandler) Datapoints() []*datapoint.Datapoint {
	return []*datapoint.Datapoint{
		sfxclient.Cumulative("total_errors", nil, atomic.LoadInt64(&e.TotalErrors)),
	}
}

// ServeHTTPC will serve the wrapped ErrorReader and return the error (if any) to rw if ErrorReader
// fails
func (e *ErrorTrackerHandler) ServeHTTPC(ctx context.Context, rw http.ResponseWriter, req *http.Request) {
	if err := e.reader.Read(ctx, req); err != nil {
		atomic.AddInt64(&e.TotalErrors, 1)
		rw.WriteHeader(http.StatusBadRequest)
		_, err = rw.Write([]byte(err.Error()))
		log.IfErr(e.Logger, err)
		return
	}
	_, err := rw.Write([]byte(`"OK"`))
	log.IfErr(e.Logger, err)
}

// ListenerConfig controls optional parameters for the listener
type ListenerConfig struct {
	ListenAddr                         *string
	HealthCheck                        *string
	Timeout                            *time.Duration
	Logger                             log.Logger
	RootContext                        context.Context
	JSONMarshal                        func(v interface{}) ([]byte, error)
	DebugContext                       *web.HeaderCtxFlag
	HTTPChain                          web.NextConstructor
	SpanNameReplacementRules           []string
	SpanNameReplacementBreakAfterMatch *bool
}

var defaultListenerConfig = &ListenerConfig{
	ListenAddr:                         pointer.String("127.0.0.1:12345"),
	HealthCheck:                        pointer.String("/healthz"),
	Timeout:                            pointer.Duration(time.Second * 30),
	Logger:                             log.Discard,
	RootContext:                        context.Background(),
	JSONMarshal:                        json.Marshal,
	SpanNameReplacementRules:           []string{},
	SpanNameReplacementBreakAfterMatch: pointer.Bool(true),
}

type metricHandler struct {
	metricCreationsMapMutex sync.Mutex
	metricCreationsMap      map[string]com_signalfx_metrics_protobuf.MetricType
	jsonMarshal             func(v interface{}) ([]byte, error)
	logger                  log.Logger
}

func (handler *metricHandler) ServeHTTP(writter http.ResponseWriter, req *http.Request) {
	dec := json.NewDecoder(req.Body)
	var d []MetricCreationStruct
	if err := dec.Decode(&d); err != nil {
		handler.logger.Log(log.Err, err, "Invalid metric creation request")
		writter.WriteHeader(http.StatusBadRequest)
		_, err = writter.Write([]byte(`{msg:"Invalid creation request"}`))
		log.IfErr(handler.logger, err)
		return
	}
	handler.metricCreationsMapMutex.Lock()
	defer handler.metricCreationsMapMutex.Unlock()
	ret := []MetricCreationResponse{}
	for _, m := range d {
		metricType, ok := com_signalfx_metrics_protobuf.MetricType_value[m.MetricType]
		if !ok {
			writter.WriteHeader(http.StatusBadRequest)
			_, err := writter.Write([]byte(`{msg:"Invalid metric type"}`))
			log.IfErr(handler.logger, err)
			return
		}
		handler.metricCreationsMap[m.MetricName] = com_signalfx_metrics_protobuf.MetricType(metricType)
		ret = append(ret, MetricCreationResponse{Code: 409})
	}
	toWrite, err := handler.jsonMarshal(ret)
	if err != nil {
		handler.logger.Log(log.Err, err, "Unable to marshal json")
		writter.WriteHeader(http.StatusBadRequest)
		_, err = writter.Write([]byte(`{msg:"Unable to marshal json!"}`))
		log.IfErr(handler.logger, err)
		return
	}
	writter.WriteHeader(http.StatusOK)
	_, err = writter.Write(toWrite)
	log.IfErr(handler.logger, err)
}

func (handler *metricHandler) GetMetricTypeFromMap(metricName string) com_signalfx_metrics_protobuf.MetricType {
	handler.metricCreationsMapMutex.Lock()
	defer handler.metricCreationsMapMutex.Unlock()
	mt, ok := handler.metricCreationsMap[metricName]
	if !ok {
		return com_signalfx_metrics_protobuf.MetricType_GAUGE
	}
	return mt
}

// NewListener servers http requests for Signalfx datapoints
func NewListener(sink Sink, conf *ListenerConfig) (*ListenerServer, error) {
	conf = pointer.FillDefaultFrom(conf, defaultListenerConfig).(*ListenerConfig)

	listener, err := net.Listen("tcp", *conf.ListenAddr)
	if err != nil {
		return nil, errors.Annotatef(err, "cannot open listening address %s", *conf.ListenAddr)
	}
	r := mux.NewRouter()

	server := http.Server{
		Handler:      r,
		Addr:         *conf.ListenAddr,
		ReadTimeout:  *conf.Timeout,
		WriteTimeout: *conf.Timeout,
	}
	listenServer := ListenerServer{
		listener: listener,
		logger:   conf.Logger,
		metricHandler: metricHandler{
			metricCreationsMap: make(map[string]com_signalfx_metrics_protobuf.MetricType),
			logger:             log.NewContext(conf.Logger).With(logkey.Struct, "metricHandler"),
			jsonMarshal:        conf.JSONMarshal,
		},
	}
	listenServer.SetupHealthCheck(conf.HealthCheck, r, conf.Logger)

	r.Handle("/v1/metric", &listenServer.metricHandler)
	r.Handle("/metric", &listenServer.metricHandler)

	traceSink := sink
	if len(conf.SpanNameReplacementRules) > 0 {
		var err error
		traceSink, err = tagreplace.New(conf.SpanNameReplacementRules, *conf.SpanNameReplacementBreakAfterMatch, sink)
		if err != nil {
			return nil, errors.Annotatef(err, "cannot parse tag replacement rules %v", conf.SpanNameReplacementRules)
		}
	}

	listenServer.internalCollectors = sfxclient.NewMultiCollector(
		setupNotFoundHandler(conf.RootContext, r),
		setupProtobufV1(conf.RootContext, r, sink, &listenServer.metricHandler, conf.Logger, conf.HTTPChain),
		setupJSONV1(conf.RootContext, r, sink, &listenServer.metricHandler, conf.Logger, conf.HTTPChain),
		setupProtobufV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupProtobufEventV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupJSONV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupJSONEventV2(conf.RootContext, r, sink, conf.Logger, conf.DebugContext, conf.HTTPChain),
		setupCollectd(conf.RootContext, r, sink, conf.DebugContext, conf.HTTPChain, conf.Logger),
		setupThriftTraceV1(conf.RootContext, r, traceSink, conf.Logger, conf.HTTPChain),
		setupJSONTraceV1(conf.RootContext, r, traceSink, conf.Logger, conf.HTTPChain),
	)

	go func() {
		log.IfErr(conf.Logger, server.Serve(listener))
	}()
	return &listenServer, err
}

func setupNotFoundHandler(ctx context.Context, r *mux.Router) sfxclient.Collector {
	metricTracking := web.RequestCounter{}
	r.NotFoundHandler = web.NewHandler(ctx, web.FromHTTP(http.NotFoundHandler())).Add(web.NextHTTP(metricTracking.ServeHTTP))
	return &sfxclient.WithDimensions{
		Dimensions: map[string]string{"protocol": "http404"},
		Collector:  &metricTracking,
	}
}

// SetupChain wraps the reader returned by getReader in an http.Handler along
// with some middleware that calculates internal metrics about requests.
func SetupChain(ctx context.Context, sink Sink, chainType string, getReader func(Sink) ErrorReader, httpChain web.NextConstructor, logger log.Logger, moreConstructors ...web.Constructor) (http.Handler, sfxclient.Collector) {
	zippers := zipper.NewZipper()

	counter := &dpsink.Counter{
		Logger: logger,
	}

	ucount := UnifyNextSinkWrap(counter)
	finalSink := FromChain(sink, NextWrap(ucount))
	errReader := getReader(finalSink)
	errorTracker := ErrorTrackerHandler{
		reader: errReader,
		Logger: logger,
	}
	metricTracking := web.RequestCounter{}
	handler := web.NewHandler(ctx, &errorTracker).Add(web.NextHTTP(metricTracking.ServeHTTP)).Add(httpChain)
	for _, c := range moreConstructors {
		handler.Add(c)
	}
	st := &sfxclient.WithDimensions{
		Collector: sfxclient.NewMultiCollector(
			&metricTracking,
			&errorTracker,
			counter,
			zippers,
		),
		Dimensions: map[string]string{
			"protocol": "sfx_" + chainType,
		},
	}
	return zippers.GzipHandler(handler), st
}

// SetupJSONByPaths tells the router which paths the given handler (which should handle the given
// endpoint) should see
func SetupJSONByPaths(r *mux.Router, handler http.Handler, endpoint string) {
	r.Path(endpoint).Methods("POST").Headers("Content-Type", "application/json").Handler(handler)
	r.Path(endpoint).Methods("POST").Headers("Content-Type", "application/json; charset=UTF-8").Handler(handler)
	r.Path(endpoint).Methods("POST").Headers("Content-Type", "").HandlerFunc(web.InvalidContentType)
	r.Path(endpoint).Methods("POST").Handler(handler)
}

func setupCollectd(ctx context.Context, r *mux.Router, sink dpsink.Sink, debugContext *web.HeaderCtxFlag, httpChain web.NextConstructor, logger log.Logger) sfxclient.Collector {
	counter := &dpsink.Counter{
		Logger: logger,
	}
	finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter))
	decoder := collectd.JSONDecoder{
		Logger: logger,
		SendTo: finalSink,
	}
	metricTracking := &web.RequestCounter{}
	httpHandler := web.NewHandler(ctx, &decoder).Add(web.NextHTTP(metricTracking.ServeHTTP), debugContext, httpChain)
	collectd.SetupCollectdPaths(r, httpHandler, "/v1/collectd")
	return &sfxclient.WithDimensions{
		Collector: sfxclient.NewMultiCollector(
			metricTracking,
			counter,
			&decoder,
		),
		Dimensions: map[string]string{
			"type": "collectd",
		},
	}
}

func readFromRequest(jeff *bytes.Buffer, req *http.Request, logger log.Logger) error {
	// for compressed transactions, contentLength isn't trustworthy
	readLen, err := jeff.ReadFrom(req.Body)
	if err != nil {
		logger.Log(log.Err, err, logkey.ReadLen, readLen, logkey.ContentLength, req.ContentLength, "Unable to fully read from buffer")
		return err
	}
	return nil
}
<|endoftext|>