<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package experiment

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"testing"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gocql/gocql"
	"github.com/intelsdi-x/swan/integration_tests/test_helpers"
	"github.com/intelsdi-x/swan/pkg/experiment"
	"github.com/intelsdi-x/swan/pkg/experiment/sensitivity"
	"github.com/intelsdi-x/swan/pkg/workloads/low_level/stressng"
	. "github.com/smartystreets/goconvey/convey"
)

func getUUID(outs []byte) string {
	So(outs, ShouldNotBeNil)
	lines := strings.Split(string(outs), "\n")
	So(len(lines), ShouldBeGreaterThan, 0)
	return string(lines[0])
}

func runExp(command string, dumpOutputOnError bool, args ...string) (string, error) {
	env := ""
	for _, e := range os.Environ() {
		if strings.Contains(e, "SWAN_") {
			env += e + " "
		}
	}
	fullCommand := "sudo -E env PATH=$PATH " + env + " " + command + " " + strings.Join(args, " ")
	// Extra logs vs asked explicitly.
	log.Debugf("[FullCommand]==> %q", fullCommand)

	c := exec.Command(command, args...)
	b := &bytes.Buffer{}
	c.Stderr = b
	out, err := c.Output()

	log.Debugf("[Out]==> %s", string(out))
	log.Debugf("[Err]==> %s", b.String())
	log.Debugf("[Warning]==> %s", err)

	if err != nil {
		if dumpOutputOnError {
			Printf("[FullCommand]==> %q", fullCommand)
			Printf("[Out]==> %s", string(out))
			Printf("[Err]==> %s", b.String())
			Printf("[Warning]==> %s", err)
		}
		return "", err
	}

	return getUUID(out), nil
}

func loadDataFromCassandra(session *gocql.Session, experimentID string) (tags map[string]string, swanRepetitions, swanAggressorsNames, swanPhases []string, metricsCount int) {
	time.Sleep(5 * time.Second)
	var ns string
	iter := session.Query(`SELECT ns, tags FROM swan.metrics WHERE tags['swan_experiment'] = ? ALLOW FILTERING`, experimentID).Iter()
	for iter.Scan(&ns, &tags) {
		metricsCount++
		swanAggressorsNames = append(swanAggressorsNames, tags["swan_aggressor_name"])
		swanPhases = append(swanPhases, tags["swan_phase"])
		swanRepetitions = append(swanRepetitions, tags["swan_repetition"])
	}
	err := iter.Close()
	if err != nil {
		panic(err)
	}
	log.Debugf("experimentID=%s metrics=%d ns=%s tags=%#v", experimentID, metricsCount, ns, tags)

	return
}

// Use experiment binaries from build directory to simplify development flow (doesn't required make bist install).
var memcachedSensitivityProfileBin = path.Join(testhelpers.SwanPath, "build/experiments/memcached/memcached-sensitivity-profile")

func TestExperimentConfiguration(t *testing.T) {
	log.SetLevel(log.DebugLevel)
	const confFilename = "temp_new_config"

	Convey("generated config should contain some flags", t, func() {
		config, err := exec.Command(memcachedSensitivityProfileBin, "-config-dump").Output()
		So(err, ShouldBeNil)

		So(string(config), ShouldContainSubstring, "KUBERNETES=false")

		Convey("and after replace new value is dumped", func() {
			newConfig := strings.Replace(string(config), "KUBERNETES=false", "KUBERNETES=true", -1)

			err = ioutil.WriteFile(confFilename, []byte(newConfig), os.ModePerm)
			So(err, ShouldBeNil)

			reloadedConfig, err := exec.Command(memcachedSensitivityProfileBin, "-config", confFilename, "-config-dump").CombinedOutput()
			So(err, ShouldBeNil)
			So(string(reloadedConfig), ShouldContainSubstring, "KUBERNETES=true")

			Reset(func() { os.Remove(confFilename) })
		})

	})

}

func TestExperiment(t *testing.T) {

	log.SetLevel(log.ErrorLevel)

	envs := map[string]string{
		"SWAN_LOG_LEVEL":                           "debug",
		"SWAN_EXPERIMENT_HP_WORKLOAD_CPU_RANGE":    "0",
		"SWAN_EXPERIMENT_BE_WORKLOAD_L1_CPU_RANGE": "0",
		"SWAN_EXPERIMENT_BE_WORKLOAD_L3_CPU_RANGE": "0",
		"SWAN_EXPERIMENT_REPETITIONS":              "1",
		"SWAN_EXPERIMENT_LOAD_POINTS":              "1",
		"SWAN_EXPERIMENT_PEAK_LOAD":                "5000",
		"SWAN_EXPERIMENT_LOAD_DURATION":            "1s",
		"SWAN_MUTILATE_RECORDS":                    "10000",
		"SWAN_MUTILATE_AGENT_CONNECTIONS":          "1",
		"SWAN_MUTILATE_AGENT_AFFINITY":             "false",
		"SWAN_MUTILATE_MASTER_AFFINITY":            "false",
	}

	Convey("With environment prepared for experiment", t, func() {
		for k, v := range envs {
			os.Setenv(k, v)
		}

		session, err := getCassandraSession()
		So(err, ShouldBeNil)
		defer session.Close()

		Convey("With proper configuration and without aggressor phases", func() {
			_, err := runExp(memcachedSensitivityProfileBin, true, "-experiment_be_workloads", sensitivity.NoneAggressorID)

			Convey("Experiment should return with no errors", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("With just caffe aggressor", func() {
			args := []string{"-experiment_be_workloads", "caffe"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				_, _, swanAggressorsNames, _, _ := loadDataFromCassandra(session, experimentID)
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)
				So("Caffe", ShouldBeIn, swanAggressorsNames)
			})
		})

		Convey("With proper configuration and with l1d aggressors", func() {
			args := []string{"-experiment_be_workloads", "stress-ng-cache-l1"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				_, _, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(stressng.IDCacheL1, ShouldBeIn, swanAggressorsNames)

				// Check metadata was saved.
				var (
					metadata     = make(map[string]string)
					iterMetadata map[string]string
				)

				iter := session.Query(`SELECT metadata FROM swan.metadata WHERE experiment_id = ? ALLOW FILTERING`, experimentID).Iter()
				for iter.Scan(&iterMetadata) {
					So(iterMetadata, ShouldNotBeEmpty)
					for k, v := range iterMetadata {
						metadata[k] = v
					}
				}
				So(err, ShouldBeNil)
				So(metadata, ShouldNotBeEmpty)
				So(metadata["SWAN_EXPERIMENT_PEAK_LOAD"], ShouldEqual, "5000")
				So(metadata["load_points"], ShouldEqual, "1")
				So(metadata["load_duration"], ShouldEqual, "1s")
				So(metadata[experiment.CPUModelNameKey], ShouldNotEqual, "")
			})

			Convey("While having two repetitions to phase", func() {
				os.Setenv("SWAN_EXPERIMENT_REPETITIONS", "2")
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				_, swanRepetitions, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)

				So(stressng.IDCacheL1, ShouldBeIn, swanAggressorsNames)
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)

				So("0", ShouldBeIn, swanRepetitions)
				So("1", ShouldBeIn, swanRepetitions)

				So(swanAggressorsNames, ShouldHaveLength, 18)
				So(swanRepetitions, ShouldHaveLength, 18)

			})

			Convey("Experiment should succeed also with 2 load points", func() {
				os.Setenv("SWAN_EXPERIMENT_LOAD_POINTS", "2")
				fmt.Println(args)
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, swanAggressorsNames, swanPhases, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)

				So(tags["swan_repetition"], ShouldEqual, "0")

				So(swanAggressorsNames, ShouldHaveLength, 18)
				So(swanPhases, ShouldHaveLength, 18)

				So("stress-ng-cache-l1", ShouldBeIn, swanAggressorsNames)
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)

				So("Aggressor stress-ng-cache-l1; load point 0;", ShouldBeIn, swanPhases)

			})
		})

		Convey("With proper kubernetes configuration and without phases", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads=None", "-kubernetes_hp_memory_resource=1000000000"}
			_, err := runExp(memcachedSensitivityProfileBin, true, args...)
			Convey("Experiment should return with no errors", func() {
				So(err, ShouldBeNil)
			})
		})

		Convey("With proper kubernetes configuration and with l1d aggressor", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads", "stress-ng-cache-l1", "-kubernetes_hp_memory_resource", "1000000000"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(tags["swan_aggressor_name"], ShouldEqual, "stress-ng-cache-l1")
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)
			})
		})

		Convey("With proper kubernetes configuration and with stress-ng-stream aggressor", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads", "stress-ng-stream", "-kubernetes_hp_memory_resource", "1000000000"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(tags["swan_aggressor_name"], ShouldEqual, "stress-ng-stream")
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)
			})
		})

		Convey("With proper kubernetes and caffe", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads", "caffe", "-kubernetes_hp_memory_resource", "1000000000"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, _, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(tags["swan_aggressor_name"], ShouldEqual, "Caffe")
			})
		})

		Convey("With invalid configuration stop experiment if error", func() {
			os.Setenv("SWAN_EXPERIMENT_LOAD_POINTS", "abc")
			_, err := runExp(memcachedSensitivityProfileBin, false)
			So(err, ShouldNotBeNil)
		})

		Convey("While setting zero repetitions to phase", func() {
			args := []string{"-experiment_be_workloads", "stress-ng-cache-l1"}
			os.Setenv("SWAN_EXPERIMENT_LOAD_POINTS", "1")
			os.Setenv("SWAN_EXPERIMENT_REPETITIONS", "0")
			Convey("Experiment should pass with no errors", func() {
				_, err := runExp(memcachedSensitivityProfileBin, false, args...)
				So(err, ShouldBeNil)
			})
		})

		Convey("With wrong aggresor name", func() {
			args := []string{"-experiment_be_workloads", "not-existing-aggressor"}
			_, err := runExp(memcachedSensitivityProfileBin, false, args...)
			So(err, ShouldNotBeNil)
		})
	})
}

func getCassandraSession() (*gocql.Session, error) {
	cluster := gocql.NewCluster("127.0.0.1")
	cluster.Keyspace = "swan"
	cluster.ProtoVersion = 4
	cluster.Timeout = 100 * time.Second
	session, err := cluster.CreateSession()
	return session, err
}
<commit_msg>Improving experiment test (#703)<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package experiment

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"testing"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gocql/gocql"
	"github.com/intelsdi-x/swan/integration_tests/test_helpers"
	"github.com/intelsdi-x/swan/pkg/experiment"
	"github.com/intelsdi-x/swan/pkg/experiment/sensitivity"
	"github.com/intelsdi-x/swan/pkg/workloads/low_level/stressng"
	. "github.com/smartystreets/goconvey/convey"
)

func getUUID(outs []byte) string {
	So(outs, ShouldNotBeNil)
	lines := strings.Split(string(outs), "\n")
	So(len(lines), ShouldBeGreaterThan, 0)
	return string(lines[0])
}

func runExp(command string, dumpOutputOnError bool, args ...string) (string, error) {
	env := ""
	for _, e := range os.Environ() {
		if strings.Contains(e, "SWAN_") {
			env += e + " "
		}
	}
	fullCommand := "sudo -E env PATH=$PATH " + env + " " + command + " " + strings.Join(args, " ")
	// Extra logs vs asked explicitly.
	log.Debugf("[FullCommand]==> %q", fullCommand)

	c := exec.Command(command, args...)
	b := &bytes.Buffer{}
	c.Stderr = b
	out, err := c.Output()

	log.Debugf("[Out]==> %s", string(out))
	log.Debugf("[Err]==> %s", b.String())
	log.Debugf("[Warning]==> %s", err)

	if err != nil {
		if dumpOutputOnError {
			Printf("[FullCommand]==> %q", fullCommand)
			Printf("[Out]==> %s", string(out))
			Printf("[Err]==> %s", b.String())
			Printf("[Warning]==> %s", err)
		}
		return "", err
	}

	return getUUID(out), nil
}

func loadDataFromCassandra(session *gocql.Session, experimentID string) (tags map[string]string, swanRepetitions, swanAggressorsNames, swanPhases []string, metricsCount int) {
	time.Sleep(5 * time.Second)
	var ns string
	iter := session.Query(`SELECT ns, tags FROM swan.metrics WHERE tags['swan_experiment'] = ? ALLOW FILTERING`, experimentID).Iter()
	for iter.Scan(&ns, &tags) {
		metricsCount++
		swanAggressorsNames = append(swanAggressorsNames, tags["swan_aggressor_name"])
		swanPhases = append(swanPhases, tags["swan_phase"])
		swanRepetitions = append(swanRepetitions, tags["swan_repetition"])
	}
	err := iter.Close()
	if err != nil {
		panic(err)
	}
	log.Debugf("experimentID=%s metrics=%d ns=%s tags=%#v", experimentID, metricsCount, ns, tags)

	return
}

// Use experiment binaries from build directory to simplify development flow (doesn't required make bist install).
var memcachedSensitivityProfileBin = path.Join(testhelpers.SwanPath, "build/experiments/memcached/memcached-sensitivity-profile")

func TestExperimentConfiguration(t *testing.T) {
	log.SetLevel(log.DebugLevel)
	const confFilename = "temp_new_config"

	Convey("generated config should contain some flags", t, func() {
		config, err := exec.Command(memcachedSensitivityProfileBin, "-config-dump").Output()
		So(err, ShouldBeNil)

		So(string(config), ShouldContainSubstring, "KUBERNETES=false")

		Convey("and after replace new value is dumped", func() {
			newConfig := strings.Replace(string(config), "KUBERNETES=false", "KUBERNETES=true", -1)

			err = ioutil.WriteFile(confFilename, []byte(newConfig), os.ModePerm)
			So(err, ShouldBeNil)

			reloadedConfig, err := exec.Command(memcachedSensitivityProfileBin, "-config", confFilename, "-config-dump").CombinedOutput()
			So(err, ShouldBeNil)
			So(string(reloadedConfig), ShouldContainSubstring, "KUBERNETES=true")

			Reset(func() { os.Remove(confFilename) })
		})

	})

}

func TestExperiment(t *testing.T) {

	log.SetLevel(log.ErrorLevel)

	envs := map[string]string{
		"SWAN_LOG_LEVEL":                           "debug",
		"SWAN_EXPERIMENT_HP_WORKLOAD_CPU_RANGE":    "0",
		"SWAN_EXPERIMENT_BE_WORKLOAD_L1_CPU_RANGE": "0",
		"SWAN_EXPERIMENT_BE_WORKLOAD_L3_CPU_RANGE": "0",
		"SWAN_EXPERIMENT_REPETITIONS":              "1",
		"SWAN_EXPERIMENT_LOAD_POINTS":              "1",
		"SWAN_EXPERIMENT_PEAK_LOAD":                "5000",
		"SWAN_EXPERIMENT_LOAD_DURATION":            "1s",
		"SWAN_MUTILATE_RECORDS":                    "10000",
		"SWAN_MUTILATE_AGENT_CONNECTIONS":          "1",
		"SWAN_MUTILATE_AGENT_AFFINITY":             "false",
		"SWAN_MUTILATE_MASTER_AFFINITY":            "false",
		"SWAN_EXPERIMENT_STOP_ON_ERROR":            "true",
		"SWAN_KUBERNETES_HP_MEMORY_RESOURCE":       "1000000000",
	}

	Convey("With environment prepared for experiment", t, func() {
		for k, v := range envs {
			os.Setenv(k, v)
		}

		session, err := getCassandraSession()
		So(err, ShouldBeNil)
		defer session.Close()

		Convey("With proper configuration and without aggressor phases", func() {
			experimentID, err := runExp(memcachedSensitivityProfileBin, true, "-experiment_be_workloads", sensitivity.NoneAggressorID)

			Convey("Experiment should return with no errors and there should be 9 metrics in Cassandra", func() {
				_, _, _, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldEqual, 9)
				So(err, ShouldBeNil)
			})
		})

		Convey("With just caffe aggressor", func() {
			args := []string{"-experiment_be_workloads", "caffe"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				_, _, swanAggressorsNames, _, _ := loadDataFromCassandra(session, experimentID)
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)
				So("Caffe", ShouldBeIn, swanAggressorsNames)
			})
		})

		Convey("With proper configuration and with l1d aggressors", func() {
			args := []string{"-experiment_be_workloads", "stress-ng-cache-l1"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				_, _, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(stressng.IDCacheL1, ShouldBeIn, swanAggressorsNames)

				// Check metadata was saved.
				var (
					metadata     = make(map[string]string)
					iterMetadata map[string]string
				)

				iter := session.Query(`SELECT metadata FROM swan.metadata WHERE experiment_id = ? ALLOW FILTERING`, experimentID).Iter()
				for iter.Scan(&iterMetadata) {
					So(iterMetadata, ShouldNotBeEmpty)
					for k, v := range iterMetadata {
						metadata[k] = v
					}
				}
				So(err, ShouldBeNil)
				So(metadata, ShouldNotBeEmpty)
				So(metadata["SWAN_EXPERIMENT_PEAK_LOAD"], ShouldEqual, "5000")
				So(metadata["load_points"], ShouldEqual, "1")
				So(metadata["load_duration"], ShouldEqual, "1s")
				So(metadata[experiment.CPUModelNameKey], ShouldNotEqual, "")
			})

			Convey("While having two repetitions to phase", func() {
				os.Setenv("SWAN_EXPERIMENT_REPETITIONS", "2")
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				_, swanRepetitions, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)

				So(stressng.IDCacheL1, ShouldBeIn, swanAggressorsNames)
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)

				So("0", ShouldBeIn, swanRepetitions)
				So("1", ShouldBeIn, swanRepetitions)

				So(swanAggressorsNames, ShouldHaveLength, 18)
				So(swanRepetitions, ShouldHaveLength, 18)

			})

			Convey("Experiment should succeed also with 2 load points", func() {
				os.Setenv("SWAN_EXPERIMENT_LOAD_POINTS", "2")
				fmt.Println(args)
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, swanAggressorsNames, swanPhases, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)

				So(tags["swan_repetition"], ShouldEqual, "0")

				So(swanAggressorsNames, ShouldHaveLength, 18)
				So(swanPhases, ShouldHaveLength, 18)

				So("stress-ng-cache-l1", ShouldBeIn, swanAggressorsNames)
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)

				So("Aggressor stress-ng-cache-l1; load point 0;", ShouldBeIn, swanPhases)

			})
		})

		Convey("With proper kubernetes configuration and without aggressor phases", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads=None"}
			experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
			Convey("Experiment should return with no errors and there should be 9 metrics in Cassandra", func() {
				So(err, ShouldBeNil)
				_, _, _, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldEqual, 9)
			})
		})

		Convey("With proper kubernetes configuration and with l1d aggressor", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads", "stress-ng-cache-l1"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(tags["swan_aggressor_name"], ShouldEqual, "stress-ng-cache-l1")
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)
			})
		})

		Convey("With proper kubernetes configuration and with stress-ng-stream aggressor", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads", "stress-ng-stream"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, swanAggressorsNames, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(tags["swan_aggressor_name"], ShouldEqual, "stress-ng-stream")
				So(sensitivity.NoneAggressorID, ShouldNotBeIn, swanAggressorsNames)
			})
		})

		Convey("With proper kubernetes and caffe", func() {
			args := []string{"-kubernetes", "-experiment_be_workloads", "caffe"}
			Convey("Experiment should run with no errors and results should be stored in a Cassandra DB", func() {
				experimentID, err := runExp(memcachedSensitivityProfileBin, true, args...)
				So(err, ShouldBeNil)

				tags, _, _, _, metricsCount := loadDataFromCassandra(session, experimentID)
				So(metricsCount, ShouldBeGreaterThan, 0)
				So(tags["swan_aggressor_name"], ShouldEqual, "Caffe")
			})
		})

		Convey("With invalid configuration stop experiment if error", func() {
			os.Setenv("SWAN_EXPERIMENT_LOAD_POINTS", "abc")
			_, err := runExp(memcachedSensitivityProfileBin, false)
			So(err, ShouldNotBeNil)
		})

		Convey("While setting zero repetitions to phase", func() {
			args := []string{"-experiment_be_workloads", "stress-ng-cache-l1"}
			os.Setenv("SWAN_EXPERIMENT_LOAD_POINTS", "1")
			os.Setenv("SWAN_EXPERIMENT_REPETITIONS", "0")
			Convey("Experiment should pass with no errors", func() {
				_, err := runExp(memcachedSensitivityProfileBin, false, args...)
				So(err, ShouldBeNil)
			})
		})

		Convey("With wrong aggresor name", func() {
			args := []string{"-experiment_be_workloads", "not-existing-aggressor"}
			_, err := runExp(memcachedSensitivityProfileBin, false, args...)
			So(err, ShouldNotBeNil)
		})
	})
}

func getCassandraSession() (*gocql.Session, error) {
	cluster := gocql.NewCluster("127.0.0.1")
	cluster.Keyspace = "swan"
	cluster.ProtoVersion = 4
	cluster.Timeout = 100 * time.Second
	session, err := cluster.CreateSession()
	return session, err
}
<|endoftext|>


<commit_before>package conditionalaccess_test

import (
	"context"
	"fmt"
	"net/http"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	"github.com/manicminer/hamilton/odata"

	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance"
	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance/check"
	"github.com/hashicorp/terraform-provider-azuread/internal/clients"
	"github.com/hashicorp/terraform-provider-azuread/internal/utils"
)

type ConditionalAccessPolicyResource struct{}

func TestAccConditionalAccessPolicy_basic(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_complete(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_update(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_sessionControls(t *testing.T) {
	// This should continue to pass when https://github.com/microsoftgraph/msgraph-metadata/issues/93
	// is resolved and the conditional ForceNew workaround has been removed

	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.sessionControls(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.sessionControls(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.sessionControls(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_sessionControlsDisabled(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.sessionControlsDisabled(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.sessionControlsDisabled(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func (r ConditionalAccessPolicyResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
	var id *string

	app, status, err := clients.ConditionalAccess.PoliciesClient.Get(ctx, state.ID, odata.Query{})
	if err != nil {
		if status == http.StatusNotFound {
			return nil, fmt.Errorf("Conditional Access Policy with ID %q does not exist", state.ID)
		}
		return nil, fmt.Errorf("failed to retrieve Conditional Access Policy with ID %q: %+v", state.ID, err)
	}
	id = app.ID

	return utils.Bool(id != nil && *id == state.ID), nil
}

func (ConditionalAccessPolicyResource) basic(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types = ["browser"]

    applications {
      included_applications = ["All"]
    }

    locations {
      included_locations = ["All"]
    }

    platforms {
      included_platforms = ["all"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["block"]
  }
}
`, data.RandomInteger)
}

func (ConditionalAccessPolicyResource) complete(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types    = ["all"]
    sign_in_risk_levels = ["medium"]
    user_risk_levels    = ["medium"]

    applications {
      included_applications = ["All"]
      excluded_applications = ["00000004-0000-0ff1-ce00-000000000000"]
    }

    locations {
      included_locations = ["All"]
      excluded_locations = ["AllTrusted"]
    }

    platforms {
      included_platforms = ["android"]
      excluded_platforms = ["iOS"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["mfa"]
  }

  session_controls {
    application_enforced_restrictions_enabled = true
    cloud_app_security_policy                 = "monitorOnly"
    sign_in_frequency                         = 10
    sign_in_frequency_period                  = "hours"
		persisten_browser_mode										= "never"
  }
}
`, data.RandomInteger)
}

func (ConditionalAccessPolicyResource) sessionControls(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types = ["browser"]

    applications {
      included_applications = ["All"]
    }

    locations {
      included_locations = ["All"]
    }

    platforms {
      included_platforms = ["all"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["block"]
  }

  session_controls {
    cloud_app_security_policy = "monitorOnly"
  }
}
`, data.RandomInteger)
}

func (ConditionalAccessPolicyResource) sessionControlsDisabled(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types = ["browser"]

    applications {
      included_applications = ["All"]
    }

    locations {
      included_locations = ["All"]
    }

    platforms {
      included_platforms = ["all"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["block"]
  }

  session_controls {
    application_enforced_restrictions_enabled = false
  }
}
`, data.RandomInteger)
}
<commit_msg>Apply suggestions from code review<commit_after>package conditionalaccess_test

import (
	"context"
	"fmt"
	"net/http"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
	"github.com/manicminer/hamilton/odata"

	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance"
	"github.com/hashicorp/terraform-provider-azuread/internal/acceptance/check"
	"github.com/hashicorp/terraform-provider-azuread/internal/clients"
	"github.com/hashicorp/terraform-provider-azuread/internal/utils"
)

type ConditionalAccessPolicyResource struct{}

func TestAccConditionalAccessPolicy_basic(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_complete(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_update(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_sessionControls(t *testing.T) {
	// This should continue to pass when https://github.com/microsoftgraph/msgraph-metadata/issues/93
	// is resolved and the conditional ForceNew workaround has been removed

	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.sessionControls(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.sessionControls(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
		{
			Config: r.basic(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.sessionControls(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func TestAccConditionalAccessPolicy_sessionControlsDisabled(t *testing.T) {
	data := acceptance.BuildTestData(t, "azuread_conditional_access_policy", "test")
	r := ConditionalAccessPolicyResource{}

	data.ResourceTest(t, r, []resource.TestStep{
		{
			Config: r.sessionControlsDisabled(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
		{
			Config: r.complete(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
			),
		},
		data.ImportStep(),
		{
			Config: r.sessionControlsDisabled(data),
			Check: resource.ComposeTestCheckFunc(
				check.That(data.ResourceName).ExistsInAzure(r),
				check.That(data.ResourceName).Key("id").Exists(),
				check.That(data.ResourceName).Key("display_name").HasValue(fmt.Sprintf("acctest-CONPOLICY-%d", data.RandomInteger)),
				check.That(data.ResourceName).Key("state").HasValue("disabled"),
			),
		},
		data.ImportStep(),
	})
}

func (r ConditionalAccessPolicyResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
	var id *string

	app, status, err := clients.ConditionalAccess.PoliciesClient.Get(ctx, state.ID, odata.Query{})
	if err != nil {
		if status == http.StatusNotFound {
			return nil, fmt.Errorf("Conditional Access Policy with ID %q does not exist", state.ID)
		}
		return nil, fmt.Errorf("failed to retrieve Conditional Access Policy with ID %q: %+v", state.ID, err)
	}
	id = app.ID

	return utils.Bool(id != nil && *id == state.ID), nil
}

func (ConditionalAccessPolicyResource) basic(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types = ["browser"]

    applications {
      included_applications = ["All"]
    }

    locations {
      included_locations = ["All"]
    }

    platforms {
      included_platforms = ["all"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["block"]
  }
}
`, data.RandomInteger)
}

func (ConditionalAccessPolicyResource) complete(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types    = ["all"]
    sign_in_risk_levels = ["medium"]
    user_risk_levels    = ["medium"]

    applications {
      included_applications = ["All"]
      excluded_applications = ["00000004-0000-0ff1-ce00-000000000000"]
    }

    locations {
      included_locations = ["All"]
      excluded_locations = ["AllTrusted"]
    }

    platforms {
      included_platforms = ["android"]
      excluded_platforms = ["iOS"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["mfa"]
  }

  session_controls {
    application_enforced_restrictions_enabled = true
    cloud_app_security_policy                 = "monitorOnly"
    sign_in_frequency                         = 10
    sign_in_frequency_period                  = "hours"
		persistent_browser_mode										= "never"
  }
}
`, data.RandomInteger)
}

func (ConditionalAccessPolicyResource) sessionControls(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types = ["browser"]

    applications {
      included_applications = ["All"]
    }

    locations {
      included_locations = ["All"]
    }

    platforms {
      included_platforms = ["all"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["block"]
  }

  session_controls {
    cloud_app_security_policy = "monitorOnly"
  }
}
`, data.RandomInteger)
}

func (ConditionalAccessPolicyResource) sessionControlsDisabled(data acceptance.TestData) string {
	return fmt.Sprintf(`
resource "azuread_conditional_access_policy" "test" {
  display_name = "acctest-CONPOLICY-%[1]d"
  state        = "disabled"

  conditions {
    client_app_types = ["browser"]

    applications {
      included_applications = ["All"]
    }

    locations {
      included_locations = ["All"]
    }

    platforms {
      included_platforms = ["all"]
    }

    users {
      included_users = ["All"]
      excluded_users = ["GuestsOrExternalUsers"]
    }
  }

  grant_controls {
    operator          = "OR"
    built_in_controls = ["block"]
  }

  session_controls {
    application_enforced_restrictions_enabled = false
  }
}
`, data.RandomInteger)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2016, Simon J Mudd
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

// Package stopwatch implements simple stopwatch functionality
package stopwatch

import (
	"fmt"
	"sync"
	"time"
)

// NamedStopwatch holds a map of string named stopwatches. Intended
// to be used when several Stopwatches are being used at once, and
// easy to use as they are name based.
type NamedStopwatch struct {
	sync.RWMutex
	stopwatches map[string](*Stopwatch)
}

// NewNamedStopwatch creates an empty Stopwatch list
func NewNamedStopwatch() *NamedStopwatch {
	return new(NamedStopwatch)
}

// Add adds a single Stopwatch name with the given name.
func (ns *NamedStopwatch) Add(name string) error {
	ns.Lock()
	defer ns.Unlock()

	return ns.add(name)
}

// Add adds a single Stopwatch name with the given name.
// The caller is assumed to have locked the structure.
func (ns *NamedStopwatch) add(name string) error {
	if ns.stopwatches == nil {
		// create structure
		ns.stopwatches = make(map[string](*Stopwatch))
	} else {
		// check for existing name
		if _, ok := ns.stopwatches[name]; ok {
			return fmt.Errorf("NamedStopwatch.add() Stopwatch name %q already exists", name)
		}
	}
	ns.stopwatches[name] = New(nil)

	return nil
}

// AddMany adds several named stopwatches in one go
func (ns *NamedStopwatch) AddMany(names []string) error {
	ns.Lock()
	defer ns.Unlock()

	for _, name := range names {
		if err := ns.add(name); err != nil {
			return err
		}
	}
	return nil
}

// Delete removes a Stopwatch with the given name (if it exists)
func (ns *NamedStopwatch) Delete(name string) {
	ns.Lock()
	defer ns.Unlock()

	if ns.stopwatches == nil {
		return
	}

	delete(ns.stopwatches, name) // check if it exists in case the user did the wrong thing
}

// Exists returns true if the NamedStopwatch exists
func (ns *NamedStopwatch) Exists(name string) bool {
	ns.RLock()
	defer ns.RUnlock()

	if ns == nil {
		return false
	}

	_, found := ns.stopwatches[name]

	return found
}

// Start starts a NamedStopwatch if it exists
func (ns *NamedStopwatch) Start(name string) {
	ns.Lock()
	defer ns.Unlock()

	ns.start(name)
}

// start starts a NamedStopwatch if it exists. The structure is expected to be locked.
func (ns *NamedStopwatch) start(name string) {
	if ns == nil {
		return
	}
	if s, ok := ns.stopwatches[name]; ok {
		s.Start()
	}
}

// StartMany allows you to start several stopwatches in one go
func (ns *NamedStopwatch) StartMany(names []string) {
	ns.Lock()
	defer ns.Unlock()

	for _, name := range names {
		ns.start(name)
	}
}

// Stop stops a NamedStopwatch if it exists
func (ns *NamedStopwatch) Stop(name string) {
	ns.Lock()
	defer ns.Unlock()

	ns.stop(name)
}

// stop stops a NamedStopwatch if it exists and expects the structure to be locked.
func (ns *NamedStopwatch) stop(name string) {
	if ns == nil {
		return
	}
	if s, ok := ns.stopwatches[name]; ok {
		if s.IsRunning() {
			s.Stop()
		} else {
			fmt.Printf("WARNING: NamedStopwatch.Stop(%q) IsRunning is false\n", name)
		}
	}
}

// StopMany allows you to stop several stopwatches in one go
func (ns *NamedStopwatch) StopMany(names []string) {
	ns.Lock()
	defer ns.Unlock()

	for _, name := range names {
		ns.stop(name)
	}
}

// Reset resets a NamedStopwatch if it exists
func (ns *NamedStopwatch) Reset(name string) {
	ns.Lock()
	defer ns.Unlock()

	if ns == nil {
		return
	}
	if s, ok := ns.stopwatches[name]; ok {
		s.Reset()
	}
}

// Keys returns the known names of Stopwatches
func (ns *NamedStopwatch) Keys() []string {
	ns.RLock()
	defer ns.RUnlock()

	if ns == nil {
		return nil
	}
	keys := []string{}
	for k := range ns.stopwatches {
		keys = append(keys, k)
	}
	return keys
}

// Elapsed returns the elapsed time.Duration of the named stopwatch if it exists or 0
func (ns *NamedStopwatch) Elapsed(name string) time.Duration {
	ns.RLock()
	defer ns.RUnlock()

	if s, ok := ns.stopwatches[name]; ok {
		return s.Elapsed()
	}
	return time.Duration(0)
}

// ElapsedSeconds returns the elapsed time in seconds of the named
// stopwatch if it exists or 0.
func (ns *NamedStopwatch) ElapsedSeconds(name string) float64 {
	ns.RLock()
	defer ns.RUnlock()

	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedSeconds()
	}
	return float64(0)
}

// ElapsedMilliSeconds returns the elapsed time in milliseconds of
// the named stopwatch if it exists or 0.
func (ns *NamedStopwatch) ElapsedMilliSeconds(name string) float64 {
	ns.RLock()
	defer ns.RUnlock()

	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedMilliSeconds()
	}
	return float64(0)
}

// AddElapsedSince adds the duration since the reference time to the given named stopwatch.
func (ns *NamedStopwatch) AddElapsedSince(name string, t time.Time) {
	ns.Lock()
	defer ns.Unlock()

	if s, ok := ns.stopwatches[name]; ok {
		s.AddElapsedSince(t)
	}
}
<commit_msg>Allow code may call NamedStopwatch with a nil pointer. (null action)<commit_after>/*
Copyright (c) 2016, Simon J Mudd
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

// Package stopwatch implements simple stopwatch functionality
package stopwatch

import (
	"fmt"
	"sync"
	"time"
)

// NamedStopwatch holds a map of string named stopwatches. Intended
// to be used when several Stopwatches are being used at once, and
// easy to use as they are name based.
type NamedStopwatch struct {
	sync.RWMutex
	stopwatches map[string](*Stopwatch)
}

// NewNamedStopwatch creates an empty Stopwatch list
func NewNamedStopwatch() *NamedStopwatch {
	return new(NamedStopwatch)
}

// Add adds a single Stopwatch name with the given name.
func (ns *NamedStopwatch) Add(name string) error {
	ns.Lock()
	defer ns.Unlock()

	return ns.add(name)
}

// Add adds a single Stopwatch name with the given name.
// The caller is assumed to have locked the structure.
func (ns *NamedStopwatch) add(name string) error {
	if ns.stopwatches == nil {
		// create structure
		ns.stopwatches = make(map[string](*Stopwatch))
	} else {
		// check for existing name
		if _, ok := ns.stopwatches[name]; ok {
			return fmt.Errorf("NamedStopwatch.add() Stopwatch name %q already exists", name)
		}
	}
	ns.stopwatches[name] = New(nil)

	return nil
}

// AddMany adds several named stopwatches in one go
func (ns *NamedStopwatch) AddMany(names []string) error {
	ns.Lock()
	defer ns.Unlock()

	for _, name := range names {
		if err := ns.add(name); err != nil {
			return err
		}
	}
	return nil
}

// Delete removes a Stopwatch with the given name (if it exists)
func (ns *NamedStopwatch) Delete(name string) {
	ns.Lock()
	defer ns.Unlock()

	if ns.stopwatches == nil {
		return
	}

	delete(ns.stopwatches, name) // check if it exists in case the user did the wrong thing
}

// Exists returns true if the NamedStopwatch exists
func (ns *NamedStopwatch) Exists(name string) bool {
	ns.RLock()
	defer ns.RUnlock()

	if ns == nil {
		return false
	}

	_, found := ns.stopwatches[name]

	return found
}

// Start starts a NamedStopwatch if it exists
func (ns *NamedStopwatch) Start(name string) {
	if ns == nil {
		return // if we're not using stopwatches we just do nothing
	}
	ns.Lock()
	defer ns.Unlock()

	ns.start(name)
}

// start starts a NamedStopwatch if it exists. The structure is expected to be locked.
func (ns *NamedStopwatch) start(name string) {
	if ns == nil {
		return
	}
	if s, ok := ns.stopwatches[name]; ok {
		s.Start()
	}
}

// StartMany allows you to start several stopwatches in one go
func (ns *NamedStopwatch) StartMany(names []string) {
	if ns == nil {
		return // if we're not using stopwatches we just do nothing
	}
	ns.Lock()
	defer ns.Unlock()

	for _, name := range names {
		ns.start(name)
	}
}

// Stop stops a NamedStopwatch if it exists
func (ns *NamedStopwatch) Stop(name string) {
	if ns == nil {
		return // if we're not using stopwatches we just do nothing
	}
	ns.Lock()
	defer ns.Unlock()

	ns.stop(name)
}

// stop stops a NamedStopwatch if it exists and expects the structure to be locked.
func (ns *NamedStopwatch) stop(name string) {
	if ns == nil {
		return
	}
	if s, ok := ns.stopwatches[name]; ok {
		if s.IsRunning() {
			s.Stop()
		} else {
			fmt.Printf("WARNING: NamedStopwatch.Stop(%q) IsRunning is false\n", name)
		}
	}
}

// StopMany allows you to stop several stopwatches in one go
func (ns *NamedStopwatch) StopMany(names []string) {
	if ns == nil {
		return // if we're not using stopwatches we just do nothing
	}
	ns.Lock()
	defer ns.Unlock()

	for _, name := range names {
		ns.stop(name)
	}
}

// Reset resets a NamedStopwatch if it exists
func (ns *NamedStopwatch) Reset(name string) {
	if ns == nil {
		return // if we're not using stopwatches we just do nothing
	}
	ns.Lock()
	defer ns.Unlock()

	if ns == nil {
		return
	}
	if s, ok := ns.stopwatches[name]; ok {
		s.Reset()
	}
}

// Keys returns the known names of Stopwatches
func (ns *NamedStopwatch) Keys() []string {
	if ns == nil {
		return nil
	}

	ns.RLock()
	defer ns.RUnlock()

	keys := []string{}
	for k := range ns.stopwatches {
		keys = append(keys, k)
	}
	return keys
}

// Elapsed returns the elapsed time.Duration of the named stopwatch if it exists or 0
func (ns *NamedStopwatch) Elapsed(name string) time.Duration {
	if ns == nil {
		return time.Duration(0)
	}
	ns.RLock()
	defer ns.RUnlock()

	if s, ok := ns.stopwatches[name]; ok {
		return s.Elapsed()
	}
	return time.Duration(0)
}

// ElapsedSeconds returns the elapsed time in seconds of the named
// stopwatch if it exists or 0.
func (ns *NamedStopwatch) ElapsedSeconds(name string) float64 {
	if ns == nil {
		return float64(0)
	}
	ns.RLock()
	defer ns.RUnlock()

	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedSeconds()
	}
	return float64(0)
}

// ElapsedMilliSeconds returns the elapsed time in milliseconds of
// the named stopwatch if it exists or 0.
func (ns *NamedStopwatch) ElapsedMilliSeconds(name string) float64 {
	if ns == nil {
		return float64(0)
	}
	ns.RLock()
	defer ns.RUnlock()

	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedMilliSeconds()
	}
	return float64(0)
}
<|endoftext|>


<commit_before>package napcomp

// napcomp takes input file from student registration
// and output file from results reporting
// and does a comparison to find students not accounted for
// in both files.

import (
	gocsv "encoding/csv"
	goxml "encoding/xml"
	"fmt"
	"log"
	"os"
	"path/filepath"

	ms "github.com/mitchellh/mapstructure"
	"github.com/wildducktheories/go-csv"
	"gopkg.in/fatih/set.v0"

	"github.com/nsip/nias2/lib"
	"github.com/nsip/nias2/naprrql"
	"github.com/nsip/nias2/xml"
	"github.com/syndtr/goleveldb/leveldb"
)

var resultsKeys, registrationKeys *set.Set

//
// iterate & load any r/r data files and
// registration data files provided
//
func IngestData() {
	// ingest the data
	log.Println("invoking data ingest...")

	clearDBWorkingDirectory()

	registrationKeys = set.New()
	resultsKeys = set.New()

	log.Println("reading results data files...")
	resultsFiles := parseResultsFileDirectory()
	for _, resultsFile := range resultsFiles {
		ingestResultsFile(resultsFile)
	}

	log.Println("reading registration data files...")
	registrationFiles := parseRegistrationFileDirectory()
	for _, regFile := range registrationFiles {
		ingestRegistrationFile(regFile)
	}

}

//
// reads in student information from a results reporting dataset file
//
func ingestResultsFile(resultsFilePath string) {

	db := naprrql.GetDB(true)
	ge := naprrql.GobEncoder{}

	// open the data file for streaming read
	xmlFile, err := naprrql.OpenResultsFile(resultsFilePath)
	if err != nil {
		log.Fatalln("unable to open results file")
	}

	log.Printf("Reading data file [%s]", resultsFilePath)

	batch := new(leveldb.Batch)

	decoder := goxml.NewDecoder(xmlFile)
	totalStudents := 0
	var inElement string
	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case goxml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...handle by type
			switch inElement {
			case "StudentPersonal":
				var sp xml.RegistrationRecord
				decoder.DecodeElement(&sp, &se)
				sp.Flatten() // align structure to registration record, explicit other-ids
				gsp, err := ge.Encode(sp)
				if err != nil {
					log.Println("Unable to gob-encode studentpersonal: ", err)
				}

				key := makeComparisonKey(&sp)
				// log.Println("Result Key: ", key)

				// store object in db
				// {StudentPersonal-id} = object
				batch.Put([]byte("res:"+key), gsp)
				// keep the key for comparisons
				resultsKeys.Add(key)

				totalStudents++

			}
		default:
		}
	}

	// commit database entries
	batcherr := db.Write(batch, nil)
	if batcherr != nil {
		log.Fatalln("batch error: ", batcherr)
	}

	log.Println("Data file read complete...")
	log.Printf("No. report data students found: %d \n", totalStudents)
	log.Printf("ingestion complete for [%s]", resultsFilePath)

}

//
// reads in student information from a registraiton data file
//
func ingestRegistrationFile(regFilePath string) {

	db := naprrql.GetDB(true)
	ge := naprrql.GobEncoder{}

	log.Printf("Reading data file [%s]", regFilePath)

	batch := new(leveldb.Batch)

	regFile, err := os.Open(regFilePath)
	if err != nil {
		log.Fatalln("Unable to open: ", regFilePath)
	}
	defer regFile.Close()
	reader := csv.WithIoReader(regFile)
	defer reader.Close()

	totalStudents := 0
	for record := range reader.C() {
		regr := &xml.RegistrationRecord{}
		r := lib.RemoveBlanks(record.AsMap())
		decode_err := ms.Decode(r, regr)
		regr.Unflatten() // make equivalent to xml record for other-ids
		if decode_err != nil {
			continue
		}
		gsp, err := ge.Encode(regr)
		if err != nil {
			log.Println("Unable to gob-encode studentpersonal: ", err)
		}
		key := makeComparisonKey(regr)
		// log.Println("Reg Key: ", key)

		// store object in db
		// {StudentPersonal-id} = object
		batch.Put([]byte("reg:"+key), gsp)
		// keep the key for comparisons
		registrationKeys.Add(key)

		totalStudents++
	}

	// commit database entries
	batcherr := db.Write(batch, nil)
	if batcherr != nil {
		log.Fatalln("batch error: ", batcherr)
	}

	log.Println("Data file read complete...")
	log.Printf("No. registration data students found: %d \n", totalStudents)
	log.Printf("ingestion complete for [%s]", regFilePath)

}

//
// create key to use in record comparisons from selected
// data fields
//
func makeComparisonKey(r *xml.RegistrationRecord) string {

	key := fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s",
		r.FamilyName,
		r.GivenName,
		r.MiddleName,
		// r.PreferredName,
		r.LocalId,
		// r.StateProvinceId,
		// r.DiocesanId,
		// r.NationalId,
		r.PlatformId,
		r.ASLSchoolId,
		r.BirthDate,
		// r.SchoolLocalId,
	)

	return key
}

//
// create .csv reports
//
func WriteReports() {

	clearReportsDirectory()
	db := naprrql.GetDB(false)
	ge := naprrql.GobEncoder{}

	log.Println("generating difference reports...")

	log.Println()
	regbutnotres := set.Difference(registrationKeys, resultsKeys)
	log.Printf("Students registred  but not in results: %d", regbutnotres.Size())

	resbutnotreg := set.Difference(resultsKeys, registrationKeys)
	log.Printf("Students in results but not in registration: %d", resbutnotreg.Size())

	symdiff := set.SymmetricDifference(resultsKeys, registrationKeys)
	log.Printf("Total students not in both files: %d", symdiff.Size())
	log.Println()

	// registered not in results...
	f, err := os.Create("./out/RegisteredButNotInResults.csv")
	if err != nil {
		log.Fatalln("Cannot open file to publish report: ", err)
	}
	defer f.Close()
	w := gocsv.NewWriter(f)
	// header
	hdr := xml.RegistrationRecord{}
	w.Write(hdr.GetHeaders())
	for _, key := range set.StringSlice(regbutnotres) {
		k := "reg:" + key
		val, err := db.Get([]byte(k), nil)
		if err != nil {
			log.Println("Cannot retrieve object for id: ", k)
		}
		var rrObj interface{}
		err = ge.Decode(val, &rrObj)
		if err != nil {
			log.Println("Cannot gob-decode object.")
		}
		sp, ok := rrObj.(xml.RegistrationRecord)
		if !ok {
			log.Println("Cannot assert object as Student Personal.")
		}
		w.Write(sp.GetSlice())
	}
	w.Flush()

	// results but not registered...
	f, err = os.Create("./out/ResultsButNotInRegister.csv")
	if err != nil {
		log.Fatalln("Cannot open file to publish report: ", err)
	}
	defer f.Close()
	w = gocsv.NewWriter(f)
	// header
	w.Write(hdr.GetHeaders())
	for _, key := range set.StringSlice(resbutnotreg) {
		k := "res:" + key
		val, err := db.Get([]byte(k), nil)
		if err != nil {
			log.Println("Cannot retrieve object for id: ", k)
		}
		var rrObj interface{}
		err = ge.Decode(val, &rrObj)
		if err != nil {
			log.Println("Cannot gob-decode object.")
		}
		sp, ok := rrObj.(xml.RegistrationRecord)
		if !ok {
			log.Println("Cannot assert object as Student Personal.")
		}
		w.Write(sp.GetSlice())
	}
	w.Flush()

	log.Println("reports generated to /out folder.")
}

//
// look for results data files
//
func parseResultsFileDirectory() []string {

	files := make([]string, 0)

	zipFiles, err := filepath.Glob("./in/results/*.zip")
	xmlFiles, err := filepath.Glob("./in/results/*.xml")

	files = append(files, zipFiles...)
	files = append(files, xmlFiles...)
	if len(files) == 0 {
		log.Fatalln("No results data *.zip *.xml.zip or *.xml files found in input folder /in/results.", err)
	}

	return files

}

//
// look for registration data files
//
func parseRegistrationFileDirectory() []string {

	files := make([]string, 0)

	csvFiles, err := filepath.Glob("./in/registration/*.csv")

	files = append(files, csvFiles...)
	if len(files) == 0 {
		log.Fatalln("No registration data *.csv files found in input folder /in/registration.", err)
	}

	return files

}

//
// ensure clean shutdown of data store
//
func CloseDB() {
	log.Println("Closing datastore...")
	naprrql.GetDB(true).Close()
	log.Println("Datastore closed.")
}

//
// remove working files of datastore
//
func clearDBWorkingDirectory() {

	// remove existing logs and recreate the directory
	err := os.RemoveAll("kvs")
	if err != nil {
		log.Println("Error trying to reset datastore working directory: ", err)
	}
	createDBWorkingDirectory()
}

//
// remove reports working directory
//
func clearReportsDirectory() {
	// remove existing logs and recreate the directory
	err := os.RemoveAll("out")
	if err != nil {
		log.Println("Error trying to reset reports directory: ", err)
	}
	createReportsDirectory()

}

//
// create folder for .csv reports
//
func createReportsDirectory() {
	err := os.Mkdir("out", os.ModePerm)
	if !os.IsExist(err) && err != nil {
		log.Fatalln("Error trying to create reports directory: ", err)
	}

}

//
// create folder for datastore
//
func createDBWorkingDirectory() {
	err := os.Mkdir("kvs", os.ModePerm)
	if !os.IsExist(err) && err != nil {
		log.Fatalln("Error trying to create datastore working directory: ", err)
	}

}
<commit_msg>Restored access api<commit_after>package napcomp

// napcomp takes input file from student registration
// and output file from results reporting
// and does a comparison to find students not accounted for
// in both files.

import (
	gocsv "encoding/csv"
	goxml "encoding/xml"
	"fmt"
	"log"
	"os"
	"path/filepath"

	ms "github.com/mitchellh/mapstructure"
	"github.com/wildducktheories/go-csv"
	"gopkg.in/fatih/set.v0"

	"github.com/nsip/nias2/lib"
	"github.com/nsip/nias2/naprrql"
	"github.com/nsip/nias2/xml"
	"github.com/syndtr/goleveldb/leveldb"
)

var resultsKeys, registrationKeys *set.Set

//
// iterate & load any r/r data files and
// registration data files provided
//
func IngestData() {
	// ingest the data
	log.Println("invoking data ingest...")

	clearDBWorkingDirectory()

	registrationKeys = set.New()
	resultsKeys = set.New()

	log.Println("reading results data files...")
	resultsFiles := parseResultsFileDirectory()
	for _, resultsFile := range resultsFiles {
		ingestResultsFile(resultsFile)
	}

	log.Println("reading registration data files...")
	registrationFiles := parseRegistrationFileDirectory()
	for _, regFile := range registrationFiles {
		ingestRegistrationFile(regFile)
	}

}

//
// reads in student information from a results reporting dataset file
//
func ingestResultsFile(resultsFilePath string) {

	db := naprrql.GetDB()
	ge := naprrql.GobEncoder{}

	// open the data file for streaming read
	xmlFile, err := naprrql.OpenResultsFile(resultsFilePath)
	if err != nil {
		log.Fatalln("unable to open results file")
	}

	log.Printf("Reading data file [%s]", resultsFilePath)

	batch := new(leveldb.Batch)

	decoder := goxml.NewDecoder(xmlFile)
	totalStudents := 0
	var inElement string
	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case goxml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...handle by type
			switch inElement {
			case "StudentPersonal":
				var sp xml.RegistrationRecord
				decoder.DecodeElement(&sp, &se)
				sp.Flatten() // align structure to registration record, explicit other-ids
				gsp, err := ge.Encode(sp)
				if err != nil {
					log.Println("Unable to gob-encode studentpersonal: ", err)
				}

				key := makeComparisonKey(&sp)
				// log.Println("Result Key: ", key)

				// store object in db
				// {StudentPersonal-id} = object
				batch.Put([]byte("res:"+key), gsp)
				// keep the key for comparisons
				resultsKeys.Add(key)

				totalStudents++

			}
		default:
		}
	}

	// commit database entries
	batcherr := db.Write(batch, nil)
	if batcherr != nil {
		log.Fatalln("batch error: ", batcherr)
	}

	log.Println("Data file read complete...")
	log.Printf("No. report data students found: %d \n", totalStudents)
	log.Printf("ingestion complete for [%s]", resultsFilePath)

}

//
// reads in student information from a registraiton data file
//
func ingestRegistrationFile(regFilePath string) {

	db := naprrql.GetDB()
	ge := naprrql.GobEncoder{}

	log.Printf("Reading data file [%s]", regFilePath)

	batch := new(leveldb.Batch)

	regFile, err := os.Open(regFilePath)
	if err != nil {
		log.Fatalln("Unable to open: ", regFilePath)
	}
	defer regFile.Close()
	reader := csv.WithIoReader(regFile)
	defer reader.Close()

	totalStudents := 0
	for record := range reader.C() {
		regr := &xml.RegistrationRecord{}
		r := lib.RemoveBlanks(record.AsMap())
		decode_err := ms.Decode(r, regr)
		regr.Unflatten() // make equivalent to xml record for other-ids
		if decode_err != nil {
			continue
		}
		gsp, err := ge.Encode(regr)
		if err != nil {
			log.Println("Unable to gob-encode studentpersonal: ", err)
		}
		key := makeComparisonKey(regr)
		// log.Println("Reg Key: ", key)

		// store object in db
		// {StudentPersonal-id} = object
		batch.Put([]byte("reg:"+key), gsp)
		// keep the key for comparisons
		registrationKeys.Add(key)

		totalStudents++
	}

	// commit database entries
	batcherr := db.Write(batch, nil)
	if batcherr != nil {
		log.Fatalln("batch error: ", batcherr)
	}

	log.Println("Data file read complete...")
	log.Printf("No. registration data students found: %d \n", totalStudents)
	log.Printf("ingestion complete for [%s]", regFilePath)

}

//
// create key to use in record comparisons from selected
// data fields
//
func makeComparisonKey(r *xml.RegistrationRecord) string {

	key := fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s",
		r.FamilyName,
		r.GivenName,
		r.MiddleName,
		// r.PreferredName,
		r.LocalId,
		// r.StateProvinceId,
		// r.DiocesanId,
		// r.NationalId,
		r.PlatformId,
		r.ASLSchoolId,
		r.BirthDate,
		// r.SchoolLocalId,
	)

	return key
}

//
// create .csv reports
//
func WriteReports() {

	clearReportsDirectory()
	db := naprrql.GetDB()
	ge := naprrql.GobEncoder{}

	log.Println("generating difference reports...")

	log.Println()
	regbutnotres := set.Difference(registrationKeys, resultsKeys)
	log.Printf("Students registred  but not in results: %d", regbutnotres.Size())

	resbutnotreg := set.Difference(resultsKeys, registrationKeys)
	log.Printf("Students in results but not in registration: %d", resbutnotreg.Size())

	symdiff := set.SymmetricDifference(resultsKeys, registrationKeys)
	log.Printf("Total students not in both files: %d", symdiff.Size())
	log.Println()

	// registered not in results...
	f, err := os.Create("./out/RegisteredButNotInResults.csv")
	if err != nil {
		log.Fatalln("Cannot open file to publish report: ", err)
	}
	defer f.Close()
	w := gocsv.NewWriter(f)
	// header
	hdr := xml.RegistrationRecord{}
	w.Write(hdr.GetHeaders())
	for _, key := range set.StringSlice(regbutnotres) {
		k := "reg:" + key
		val, err := db.Get([]byte(k), nil)
		if err != nil {
			log.Println("Cannot retrieve object for id: ", k)
		}
		var rrObj interface{}
		err = ge.Decode(val, &rrObj)
		if err != nil {
			log.Println("Cannot gob-decode object.")
		}
		sp, ok := rrObj.(xml.RegistrationRecord)
		if !ok {
			log.Println("Cannot assert object as Student Personal.")
		}
		w.Write(sp.GetSlice())
	}
	w.Flush()

	// results but not registered...
	f, err = os.Create("./out/ResultsButNotInRegister.csv")
	if err != nil {
		log.Fatalln("Cannot open file to publish report: ", err)
	}
	defer f.Close()
	w = gocsv.NewWriter(f)
	// header
	w.Write(hdr.GetHeaders())
	for _, key := range set.StringSlice(resbutnotreg) {
		k := "res:" + key
		val, err := db.Get([]byte(k), nil)
		if err != nil {
			log.Println("Cannot retrieve object for id: ", k)
		}
		var rrObj interface{}
		err = ge.Decode(val, &rrObj)
		if err != nil {
			log.Println("Cannot gob-decode object.")
		}
		sp, ok := rrObj.(xml.RegistrationRecord)
		if !ok {
			log.Println("Cannot assert object as Student Personal.")
		}
		w.Write(sp.GetSlice())
	}
	w.Flush()

	log.Println("reports generated to /out folder.")
}

//
// look for results data files
//
func parseResultsFileDirectory() []string {

	files := make([]string, 0)

	zipFiles, err := filepath.Glob("./in/results/*.zip")
	xmlFiles, err := filepath.Glob("./in/results/*.xml")

	files = append(files, zipFiles...)
	files = append(files, xmlFiles...)
	if len(files) == 0 {
		log.Fatalln("No results data *.zip *.xml.zip or *.xml files found in input folder /in/results.", err)
	}

	return files

}

//
// look for registration data files
//
func parseRegistrationFileDirectory() []string {

	files := make([]string, 0)

	csvFiles, err := filepath.Glob("./in/registration/*.csv")

	files = append(files, csvFiles...)
	if len(files) == 0 {
		log.Fatalln("No registration data *.csv files found in input folder /in/registration.", err)
	}

	return files

}

//
// ensure clean shutdown of data store
//
func CloseDB() {
	log.Println("Closing datastore...")
	naprrql.GetDB().Close()
	log.Println("Datastore closed.")
}

//
// remove working files of datastore
//
func clearDBWorkingDirectory() {

	// remove existing logs and recreate the directory
	err := os.RemoveAll("kvs")
	if err != nil {
		log.Println("Error trying to reset datastore working directory: ", err)
	}
	createDBWorkingDirectory()
}

//
// remove reports working directory
//
func clearReportsDirectory() {
	// remove existing logs and recreate the directory
	err := os.RemoveAll("out")
	if err != nil {
		log.Println("Error trying to reset reports directory: ", err)
	}
	createReportsDirectory()

}

//
// create folder for .csv reports
//
func createReportsDirectory() {
	err := os.Mkdir("out", os.ModePerm)
	if !os.IsExist(err) && err != nil {
		log.Fatalln("Error trying to create reports directory: ", err)
	}

}

//
// create folder for datastore
//
func createDBWorkingDirectory() {
	err := os.Mkdir("kvs", os.ModePerm)
	if !os.IsExist(err) && err != nil {
		log.Fatalln("Error trying to create datastore working directory: ", err)
	}

}
<|endoftext|>


<commit_before>package hooks

import (
	"regexp"
	"sync"
	"time"

	"github.com/danielkrainas/csense/api/v1"
	"github.com/danielkrainas/csense/context"
	"github.com/danielkrainas/csense/storage"
	"github.com/danielkrainas/csense/uuid"
)

type Filter interface {
	Match(hook *v1.Hook, c *v1.ContainerInfo) bool
}

type CriteriaFilter struct{}

func (f *CriteriaFilter) Match(hook *v1.Hook, c *v1.ContainerInfo) bool {
	crit := hook.Criteria

	if c.Name != "" && IsValid(crit.Name, c.Name) {
		return true
	}

	if c.ImageName != "" && IsValid(crit.ImageName, c.ImageName) {
		return true
	}

	for k, v := range c.Labels {
		if x, ok := c.Labels[k]; ok && x == v {
			return true
		}
	}

	return false
}

func IsValid(c *v1.Condition, v string) bool {
	switch c.Op {
	case v1.OperandEqual:
		return c.Value == v
	case v1.OperandNotEqual:
		return c.Value != v
	case v1.OperandMatch:
		ok, err := regexp.MatchString(c.Value, v)
		return err == nil && ok
	}

	return false
}

func DefaultHook() *v1.Hook {
	return &v1.Hook{
		ID:      uuid.Generate(),
		Events:  make([]v1.EventType, 0),
		TTL:     -1,
		Created: time.Now().Unix(),
		Format:  v1.FormatJSON,
	}
}

func FilterAll(hooks []*v1.Hook, c *v1.ContainerInfo, f Filter) []*v1.Hook {
	results := make([]*v1.Hook, 0)
	for _, hook := range hooks {
		if f.Match(hook, c) {
			results = append(results, hook)
		}
	}

	return results
}

type Cache struct {
	ticker *time.Ticker
	update sync.Mutex
	hooks  []*v1.Hook
}

func (c *Cache) Hooks() []*v1.Hook {
	c.update.Lock()
	defer c.update.Unlock()
	return c.hooks
}

func NewCache(ctx context.Context, d time.Duration, store storage.HookStore) *Cache {
	c := &Cache{
		ticker: time.NewTicker(d),
		hooks:  []*v1.Hook{},
	}

	go func() {
		for {
			<-c.ticker.C
			hooks, err := store.GetAll(ctx)
			if err != nil {
				context.GetLogger(ctx).Warnf("error caching hooks: %v", err)
				continue
			}

			c.update.Lock()
			c.hooks = hooks
			c.update.Unlock()
		}
	}()

	return c
}
<commit_msg>ignore invalid conditions<commit_after>package hooks

import (
	"regexp"
	"sync"
	"time"

	"github.com/danielkrainas/csense/api/v1"
	"github.com/danielkrainas/csense/context"
	"github.com/danielkrainas/csense/storage"
	"github.com/danielkrainas/csense/uuid"
)

type Filter interface {
	Match(hook *v1.Hook, c *v1.ContainerInfo) bool
}

type CriteriaFilter struct{}

func (f *CriteriaFilter) Match(hook *v1.Hook, c *v1.ContainerInfo) bool {
	crit := hook.Criteria

	if c.Name != "" && IsValid(crit.Name, c.Name) {
		return true
	}

	if c.ImageName != "" && IsValid(crit.ImageName, c.ImageName) {
		return true
	}

	for k, v := range c.Labels {
		if x, ok := c.Labels[k]; ok && x == v {
			return true
		}
	}

	return false
}

func IsValid(c *v1.Condition, v string) bool {
	if c == nil {
		return false
	}

	switch c.Op {
	case v1.OperandEqual:
		return c.Value == v
	case v1.OperandNotEqual:
		return c.Value != v
	case v1.OperandMatch:
		ok, err := regexp.MatchString(c.Value, v)
		return err == nil && ok
	}

	return false
}

func DefaultHook() *v1.Hook {
	return &v1.Hook{
		ID:      uuid.Generate(),
		Events:  make([]v1.EventType, 0),
		TTL:     -1,
		Created: time.Now().Unix(),
		Format:  v1.FormatJSON,
	}
}

func FilterAll(hooks []*v1.Hook, c *v1.ContainerInfo, f Filter) []*v1.Hook {
	results := make([]*v1.Hook, 0)
	for _, hook := range hooks {
		if f.Match(hook, c) {
			results = append(results, hook)
		}
	}

	return results
}

type Cache struct {
	ticker *time.Ticker
	update sync.Mutex
	hooks  []*v1.Hook
}

func (c *Cache) Hooks() []*v1.Hook {
	c.update.Lock()
	defer c.update.Unlock()
	return c.hooks
}

func NewCache(ctx context.Context, d time.Duration, store storage.HookStore) *Cache {
	c := &Cache{
		ticker: time.NewTicker(d),
		hooks:  []*v1.Hook{},
	}

	go func() {
		for {
			<-c.ticker.C
			hooks, err := store.GetAll(ctx)
			if err != nil {
				context.GetLogger(ctx).Warnf("error caching hooks: %v", err)
				continue
			}

			c.update.Lock()
			c.hooks = hooks
			c.update.Unlock()
		}
	}()

	return c
}
<|endoftext|>


<commit_before>package pgx_test

import (
	"github.com/jackc/pgx"
	"testing"
)

func TestHstoreTranscode(t *testing.T) {
	t.Parallel()

	conn := mustConnect(t, *defaultConnConfig)
	defer closeConn(t, conn)

	type test struct {
		hstore      pgx.Hstore
		description string
	}

	tests := []test{
		{pgx.Hstore{}, "empty"},
		{pgx.Hstore{"foo": "bar"}, "single key/value"},
		{pgx.Hstore{"foo": "bar", "baz": "quz"}, "multiple key/values"},
		{pgx.Hstore{"NULL": "bar"}, `string "NULL" key`},
		{pgx.Hstore{"foo": "NULL"}, `string "NULL" value`},
	}

	specialStringTests := []struct {
		input       string
		description string
	}{
		{`"`, `double quote (")`},
		{`'`, `single quote (')`},
		{`\`, `backslash (\)`},
		{`\\`, `multiple backslashes (\\)`},
		{`=>`, `separator (=>)`},
		{` `, `space`},
		{`\ / / \\ => " ' " '`, `multiple special characters`},
	}
	for _, sst := range specialStringTests {
		tests = append(tests, test{pgx.Hstore{sst.input + "foo": "bar"}, "key with " + sst.description + " at beginning"})
		tests = append(tests, test{pgx.Hstore{"foo" + sst.input + "foo": "bar"}, "key with " + sst.description + " in middle"})
		tests = append(tests, test{pgx.Hstore{"foo" + sst.input: "bar"}, "key with " + sst.description + " at end"})
		tests = append(tests, test{pgx.Hstore{sst.input: "bar"}, "key is " + sst.description})

		tests = append(tests, test{pgx.Hstore{"foo": sst.input + "bar"}, "value with " + sst.description + " at beginning"})
		tests = append(tests, test{pgx.Hstore{"foo": "bar" + sst.input + "bar"}, "value with " + sst.description + " in middle"})
		tests = append(tests, test{pgx.Hstore{"foo": "bar" + sst.input}, "value with " + sst.description + " at end"})
		tests = append(tests, test{pgx.Hstore{"foo": sst.input}, "value is " + sst.description})
	}

	for _, tt := range tests {
		var result pgx.Hstore
		err := conn.QueryRow("select $1::hstore", tt.hstore).Scan(&result)
		if err != nil {
			t.Errorf(`%s: QueryRow.Scan returned an error: %v`, tt.description, err)
		}

		for key, inValue := range tt.hstore {
			outValue, ok := result[key]
			if ok {
				if inValue != outValue {
					t.Errorf(`%s: Key %s mismatch - expected %s, received %s`, tt.description, key, inValue, outValue)
				}
			} else {
				t.Errorf(`%s: Missing key %s`, tt.description, key)
			}
		}

		ensureConnValid(t, conn)
	}
}
<commit_msg>Add tests for NullHstore<commit_after>package pgx_test

import (
	"github.com/jackc/pgx"
	"testing"
)

func TestHstoreTranscode(t *testing.T) {
	t.Parallel()

	conn := mustConnect(t, *defaultConnConfig)
	defer closeConn(t, conn)

	type test struct {
		hstore      pgx.Hstore
		description string
	}

	tests := []test{
		{pgx.Hstore{}, "empty"},
		{pgx.Hstore{"foo": "bar"}, "single key/value"},
		{pgx.Hstore{"foo": "bar", "baz": "quz"}, "multiple key/values"},
		{pgx.Hstore{"NULL": "bar"}, `string "NULL" key`},
		{pgx.Hstore{"foo": "NULL"}, `string "NULL" value`},
	}

	specialStringTests := []struct {
		input       string
		description string
	}{
		{`"`, `double quote (")`},
		{`'`, `single quote (')`},
		{`\`, `backslash (\)`},
		{`\\`, `multiple backslashes (\\)`},
		{`=>`, `separator (=>)`},
		{` `, `space`},
		{`\ / / \\ => " ' " '`, `multiple special characters`},
	}
	for _, sst := range specialStringTests {
		tests = append(tests, test{pgx.Hstore{sst.input + "foo": "bar"}, "key with " + sst.description + " at beginning"})
		tests = append(tests, test{pgx.Hstore{"foo" + sst.input + "foo": "bar"}, "key with " + sst.description + " in middle"})
		tests = append(tests, test{pgx.Hstore{"foo" + sst.input: "bar"}, "key with " + sst.description + " at end"})
		tests = append(tests, test{pgx.Hstore{sst.input: "bar"}, "key is " + sst.description})

		tests = append(tests, test{pgx.Hstore{"foo": sst.input + "bar"}, "value with " + sst.description + " at beginning"})
		tests = append(tests, test{pgx.Hstore{"foo": "bar" + sst.input + "bar"}, "value with " + sst.description + " in middle"})
		tests = append(tests, test{pgx.Hstore{"foo": "bar" + sst.input}, "value with " + sst.description + " at end"})
		tests = append(tests, test{pgx.Hstore{"foo": sst.input}, "value is " + sst.description})
	}

	for _, tt := range tests {
		var result pgx.Hstore
		err := conn.QueryRow("select $1::hstore", tt.hstore).Scan(&result)
		if err != nil {
			t.Errorf(`%s: QueryRow.Scan returned an error: %v`, tt.description, err)
		}

		for key, inValue := range tt.hstore {
			outValue, ok := result[key]
			if ok {
				if inValue != outValue {
					t.Errorf(`%s: Key %s mismatch - expected %s, received %s`, tt.description, key, inValue, outValue)
				}
			} else {
				t.Errorf(`%s: Missing key %s`, tt.description, key)
			}
		}

		ensureConnValid(t, conn)
	}
}

func TestNullHstoreTranscode(t *testing.T) {
	t.Parallel()

	conn := mustConnect(t, *defaultConnConfig)
	defer closeConn(t, conn)

	type test struct {
		nullHstore  pgx.NullHstore
		description string
	}

	tests := []test{
		{pgx.NullHstore{}, "null"},
		{pgx.NullHstore{Valid: true}, "empty"},
		{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{"bar", true}},
			Valid:  true},
			"single key/value"},
		{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{"bar", true}, "baz": pgx.NullString{"quz", true}},
			Valid:  true},
			"multiple key/values"},
		{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"NULL": pgx.NullString{"bar", true}},
			Valid:  true},
			`string "NULL" key`},
		{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{"NULL", true}},
			Valid:  true},
			`string "NULL" value`},
		{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{"", false}},
			Valid:  true},
			`NULL value`},
	}

	specialStringTests := []struct {
		input       string
		description string
	}{
		{`"`, `double quote (")`},
		{`'`, `single quote (')`},
		{`\`, `backslash (\)`},
		{`\\`, `multiple backslashes (\\)`},
		{`=>`, `separator (=>)`},
		{` `, `space`},
		{`\ / / \\ => " ' " '`, `multiple special characters`},
	}
	for _, sst := range specialStringTests {
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{sst.input + "foo": pgx.NullString{"bar", true}},
			Valid:  true},
			"key with " + sst.description + " at beginning"})
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo" + sst.input + "foo": pgx.NullString{"bar", true}},
			Valid:  true},
			"key with " + sst.description + " in middle"})
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo" + sst.input: pgx.NullString{"bar", true}},
			Valid:  true},
			"key with " + sst.description + " at end"})
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{sst.input: pgx.NullString{"bar", true}},
			Valid:  true},
			"key is " + sst.description})

		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{sst.input + "bar", true}},
			Valid:  true},
			"value with " + sst.description + " at beginning"})
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{"bar" + sst.input + "bar", true}},
			Valid:  true},
			"value with " + sst.description + " in middle"})
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{"bar" + sst.input, true}},
			Valid:  true},
			"value with " + sst.description + " at end"})
		tests = append(tests, test{pgx.NullHstore{
			Hstore: map[string]pgx.NullString{"foo": pgx.NullString{sst.input, true}},
			Valid:  true},
			"value is " + sst.description})
	}

	for _, tt := range tests {
		var result pgx.NullHstore
		err := conn.QueryRow("select $1::hstore", tt.nullHstore).Scan(&result)
		if err != nil {
			t.Errorf(`%s: QueryRow.Scan returned an error: %v`, tt.description, err)
		}

		if result.Valid != tt.nullHstore.Valid {
			t.Errorf(`%s: Valid mismatch - expected %v, received %v`, tt.description, tt.nullHstore.Valid, result.Valid)
		}

		for key, inValue := range tt.nullHstore.Hstore {
			outValue, ok := result.Hstore[key]
			if ok {
				if inValue != outValue {
					t.Errorf(`%s: Key %s mismatch - expected %s, received %s`, tt.description, key, inValue, outValue)
				}
			} else {
				t.Errorf(`%s: Missing key %s`, tt.description, key)
			}
		}

		ensureConnValid(t, conn)
	}
}
<|endoftext|>


<commit_before>package httptoo

import (
	"net/http"
	"net/url"
	"path"
)

// Deep copies a URL. I could call it DeepCopyURL, but what else would you be
// copying when you have a *url.URL? Of note is that the Userinfo is deep
// copied. The returned URL shares no references with the original.
func CopyURL(u *url.URL) (ret *url.URL) {
	ret = new(url.URL)
	*ret = *u
	if u.User != nil {
		ret.User = new(url.Userinfo)
		*ret.User = *u.User
	}
	return
}

// Reconstructs the URL that would have produced the given Request.
// Request.URLs are not fully populated in http.Server handlers.
func RequestedURL(r *http.Request) (ret *url.URL) {
	ret = CopyURL(r.URL)
	ret.Host = r.Host
	ret.Scheme = OriginatingProtocol(r)
	return
}

// The official URL struct parameters, for tracking changes and reference
// here.
//
// 	Scheme     string
// 	Opaque     string    // encoded opaque data
// 	User       *Userinfo // username and password information
// 	Host       string    // host or host:port
// 	Path       string
// 	RawPath    string // encoded path hint (Go 1.5 and later only; see EscapedPath method)
// 	ForceQuery bool   // append a query ('?') even if RawQuery is empty
// 	RawQuery   string // encoded query values, without '?'
// 	Fragment   string // fragment for references, without '#'

// Return the first URL extended with elements of the second, in the manner
// that occurs throughout my projects. Noteworthy difference from
// url.URL.ResolveReference is that if the reference has a scheme, the base is
// not completely ignored.
func AppendURL(u, v *url.URL) *url.URL {
	u = CopyURL(u)
	clobberString(&u.Scheme, v.Scheme)
	clobberString(&u.Host, v.Host)
	u.Path = path.Join(u.Path, v.Path)
	q := u.Query()
	for k, v := range v.Query() {
		q[k] = append(q[k], v...)
	}
	u.RawQuery = q.Encode()
	return u
}

func clobberString(s *string, value string) {
	if value != "" {
		*s = value
	}
}
<commit_msg>httptoo.AppendURL: Don't use path.Join as it trims trailing slashes<commit_after>package httptoo

import (
	"net/http"
	"net/url"
)

// Deep copies a URL. I could call it DeepCopyURL, but what else would you be
// copying when you have a *url.URL? Of note is that the Userinfo is deep
// copied. The returned URL shares no references with the original.
func CopyURL(u *url.URL) (ret *url.URL) {
	ret = new(url.URL)
	*ret = *u
	if u.User != nil {
		ret.User = new(url.Userinfo)
		*ret.User = *u.User
	}
	return
}

// Reconstructs the URL that would have produced the given Request.
// Request.URLs are not fully populated in http.Server handlers.
func RequestedURL(r *http.Request) (ret *url.URL) {
	ret = CopyURL(r.URL)
	ret.Host = r.Host
	ret.Scheme = OriginatingProtocol(r)
	return
}

// The official URL struct parameters, for tracking changes and reference
// here.
//
// 	Scheme     string
// 	Opaque     string    // encoded opaque data
// 	User       *Userinfo // username and password information
// 	Host       string    // host or host:port
// 	Path       string
// 	RawPath    string // encoded path hint (Go 1.5 and later only; see EscapedPath method)
// 	ForceQuery bool   // append a query ('?') even if RawQuery is empty
// 	RawQuery   string // encoded query values, without '?'
// 	Fragment   string // fragment for references, without '#'

// Return the first URL extended with elements of the second, in the manner
// that occurs throughout my projects. Noteworthy difference from
// url.URL.ResolveReference is that if the reference has a scheme, the base is
// not completely ignored.
func AppendURL(u, v *url.URL) *url.URL {
	u = CopyURL(u)
	clobberString(&u.Scheme, v.Scheme)
	clobberString(&u.Host, v.Host)
	u.Path += v.Path
	q := u.Query()
	for k, v := range v.Query() {
		q[k] = append(q[k], v...)
	}
	u.RawQuery = q.Encode()
	return u
}

func clobberString(s *string, value string) {
	if value != "" {
		*s = value
	}
}
<|endoftext|>