<commit_before>// mouseAnalyzer determines vesicle release events and latencies for our
// mouse NMJ 6 AZ model with two synaptic vesicles each according to the
// second sensor faciliation model (see Ma et al., J. Neurophys, 2014)
package main

import (
	"flag"
	"fmt"
	"github.com/haskelladdict/mbdr/libmbd"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	numPulses      int     // number of AP pulses
	energyModel    bool    // use the energy model
	sytEnergy      int     // energy of activated synaptotagmin toward vesicle fusion
	yEnergy        int     // energy of activated Y sites toward vesicle fusion
	numActiveSites int     // number of simultaneously active sites required for release
	isiValue       float64 // interstimulus interval

)

func init() {
	flag.IntVar(&numPulses, "p", 1, "number of AP pulses in the model")
	flag.IntVar(&sytEnergy, "s", -1, "energy of active synaptotagmin sites "+
		"(required with -e flag)")
	flag.IntVar(&yEnergy, "y", -1, "energy of active y sites "+
		"(required with -e flag")
	flag.BoolVar(&energyModel, "e", false, "use the energy model instead of "+
		"deterministic model")
	flag.IntVar(&numActiveSites, "n", 0, "number of sites required for activation"+
		"for deterministic model")
	flag.Float64Var(&isiValue, "i", -1.0, "pulse duration in [s] for analysis multi "+
		"pulse data")
}

// usage prints a brief usage information to stdout
func usage() {
	fmt.Println("usage: mouseAnalyzer [options] <binary mcell files>")
	fmt.Println("\noptions:")
	flag.PrintDefaults()
}

// extractSeed attempts to extract the seed from the filename of the provided
// binary mcell data file.
// NOTE: the following filenaming convention is assumed *.<seedIDString>.bin.(gz|bz2)
func extractSeed(fileName string) (int, error) {
	items := strings.Split(fileName, ".")
	if len(items) <= 3 {
		return -1, fmt.Errorf("incorrectly formatted fileName. " +
			"Expected *.<seedIDString>.bin.(gz|bz2)")
	}

	for i := len(items) - 1; i >= 0; i-- {
		if items[i] == "bin" && i >= 1 {
			seed, err := strconv.Atoi(items[i-1])
			if err != nil {
				return -1, err
			}
			return seed, nil
		}
	}
	return -1, fmt.Errorf("Unable to extract seed id from filename ", fileName)
}

// printHeader prints and informative header file with date and commandline
// options requested for analysis
func printHeader() {
	fmt.Println("mouseAnalyzer ran on", time.Now())
	if host, err := os.Hostname(); err == nil {
		fmt.Println("on ", host)
	}
	fmt.Println("\n-------------- parameters --------------")
	fmt.Println("number of pulses       :", numPulses)
	if numPulses > 1 {
		fmt.Println("ISI                    :", isiValue)
	}
	if energyModel {
		fmt.Println("model                  : energy model")
		fmt.Println("syt energy             :", sytEnergy)
		fmt.Println("y energy               :", yEnergy)
	} else {
		fmt.Println("model                  : deterministic model")
		fmt.Println("number of active sites :", numActiveSites)
	}
	fmt.Println("-------------- data --------------------\n")
}

// printReleases prints a summary statistic for all released vesicle for a
// given seed
func printReleases(data *libmbd.MCellData, seed int, rel []*ReleaseEvent) {
	timeStep := data.StepSize()
	for _, r := range rel {

		channels, err := determineCaChanContrib(data, r)
		if err != nil {
			log.Fatal(err)
		}

		eventTime := float64(r.eventIter) * timeStep
		pulseID := int(math.Floor(eventTime/isiValue)) + 1

		fmt.Printf("seed : %d   AZ : %d   ves : %d   time : %e   pulseID : %d", seed,
			r.azId+1, r.vesicleID+1, eventTime, pulseID)
		fmt.Printf("  sensors: [")
		for _, s := range r.sensors {
			fmt.Printf("%d ", s)
		}
		fmt.Printf("]")
		fmt.Printf("  channels: [")
		for n, c := range channels {
			fmt.Printf("%s : %d  ", n, int(c))
		}
		fmt.Printf("]")
		fmt.Printf("\n")
	}
}

// determineCaContrib determines which Ca channels contributed to the release
// of a particular vesicle.
// NOTE: We try to be as agnostic as we can in terms of the particular
// nomenclature used for naming the channels. However, the expectation is
// that data files tracking Ca binding to vesicles are named
// vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func determineCaChanContrib(data *libmbd.MCellData, rel *ReleaseEvent) (map[string]float64, error) {
	channels := make(map[string]float64)
	// the az/channel counting is 1 based whereas our internal counting is 0 based
	regexString := fmt.Sprintf("vesicle_%d_%d_ca_.*", rel.azId+1, rel.vesicleID+1)
	counts, err := data.BlockDataByRegex(regexString)
	if err != nil {
		return nil, err
	}
	for k, c := range counts {
		if len(c.Col) != 1 {
			return nil, fmt.Errorf("data set %s has more than the expected 1 column",
				k)
		}
		if c.Col[0][rel.eventIter] > 0 {
			// need to subtract 2 from regexString due to the extra ".*"
			caString, err := extractCaChanName(k, len(regexString)-2)
			if err != nil {
				return nil, err
			}
			channels[caString] = c.Col[0][rel.eventIter]
		}
	}

	return channels, nil
}

// extractCaChanName attempts to extract the name of the calcium channel based
// on the expected data name pattern vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func extractCaChanName(name string, prefixLength int) (string, error) {
	caName := name[prefixLength:]
	items := strings.Split(caName, ".")
	if len(items) == 0 {
		return "", fmt.Errorf("Could not determine Ca channel name from data set %s", name)
	}
	return items[0], nil
}

// main entry point
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		usage()
		return
	}

	// some sanity checks
	if energyModel && (sytEnergy < 0 || yEnergy < 0) {
		log.Fatal("Please provide a non-negative synaptotagmin and y site energy")
	}

	if !energyModel && numActiveSites == 0 {
		log.Fatal("Please provide a positive count for the number of required active sites")
	}

	if numPulses > 1 && isiValue <= 0 {
		log.Fatal("Analysis multi-pulse data requires a non-zero ISI value.")
	}

	printHeader()
	// loop over all provided data sets
	for _, fileName := range flag.Args() {

		seed, err := extractSeed(fileName)
		if err != nil {
			log.Fatal(err)
		}

		data, err := libmbd.Read(fileName)
		if err != nil {
			log.Fatal(err)
		}

		err = analyze(data, energyModel, seed, numPulses, numActiveSites, sytEnergy,
			yEnergy)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Added code to force a GC cycle before loading the next dataset.<commit_after>// mouseAnalyzer determines vesicle release events and latencies for our
// mouse NMJ 6 AZ model with two synaptic vesicles each according to the
// second sensor faciliation model (see Ma et al., J. Neurophys, 2014)
package main

import (
	"flag"
	"fmt"
	"github.com/haskelladdict/mbdr/libmbd"
	"log"
	"math"
	"os"
	"runtime/debug"
	"strconv"
	"strings"
	"time"
)

var (
	numPulses      int     // number of AP pulses
	energyModel    bool    // use the energy model
	sytEnergy      int     // energy of activated synaptotagmin toward vesicle fusion
	yEnergy        int     // energy of activated Y sites toward vesicle fusion
	numActiveSites int     // number of simultaneously active sites required for release
	isiValue       float64 // interstimulus interval

)

func init() {
	flag.IntVar(&numPulses, "p", 1, "number of AP pulses in the model")
	flag.IntVar(&sytEnergy, "s", -1, "energy of active synaptotagmin sites "+
		"(required with -e flag)")
	flag.IntVar(&yEnergy, "y", -1, "energy of active y sites "+
		"(required with -e flag")
	flag.BoolVar(&energyModel, "e", false, "use the energy model instead of "+
		"deterministic model")
	flag.IntVar(&numActiveSites, "n", 0, "number of sites required for activation"+
		"for deterministic model")
	flag.Float64Var(&isiValue, "i", -1.0, "pulse duration in [s] for analysis multi "+
		"pulse data")
}

// usage prints a brief usage information to stdout
func usage() {
	fmt.Println("usage: mouseAnalyzer [options] <binary mcell files>")
	fmt.Println("\noptions:")
	flag.PrintDefaults()
}

// extractSeed attempts to extract the seed from the filename of the provided
// binary mcell data file.
// NOTE: the following filenaming convention is assumed *.<seedIDString>.bin.(gz|bz2)
func extractSeed(fileName string) (int, error) {
	items := strings.Split(fileName, ".")
	if len(items) <= 3 {
		return -1, fmt.Errorf("incorrectly formatted fileName. " +
			"Expected *.<seedIDString>.bin.(gz|bz2)")
	}

	for i := len(items) - 1; i >= 0; i-- {
		if items[i] == "bin" && i >= 1 {
			seed, err := strconv.Atoi(items[i-1])
			if err != nil {
				return -1, err
			}
			return seed, nil
		}
	}
	return -1, fmt.Errorf("Unable to extract seed id from filename ", fileName)
}

// printHeader prints and informative header file with date and commandline
// options requested for analysis
func printHeader() {
	fmt.Println("mouseAnalyzer ran on", time.Now())
	if host, err := os.Hostname(); err == nil {
		fmt.Println("on ", host)
	}
	fmt.Println("\n-------------- parameters --------------")
	fmt.Println("number of pulses       :", numPulses)
	if numPulses > 1 {
		fmt.Println("ISI                    :", isiValue, "s")
	}
	if energyModel {
		fmt.Println("model                  : energy model")
		fmt.Println("syt energy             :", sytEnergy)
		fmt.Println("y energy               :", yEnergy)
	} else {
		fmt.Println("model                  : deterministic model")
		fmt.Println("number of active sites :", numActiveSites)
	}
	fmt.Println("-------------- data --------------------\n")
}

// printReleases prints a summary statistic for all released vesicle for a
// given seed
func printReleases(data *libmbd.MCellData, seed int, rel []*ReleaseEvent) {
	timeStep := data.StepSize()
	for _, r := range rel {

		channels, err := determineCaChanContrib(data, r)
		if err != nil {
			log.Fatal(err)
		}

		eventTime := float64(r.eventIter) * timeStep
		pulseID := int(math.Floor(eventTime/isiValue)) + 1

		fmt.Printf("seed : %d   AZ : %d   ves : %d   time : %e   pulseID : %d", seed,
			r.azId+1, r.vesicleID+1, eventTime, pulseID)
		fmt.Printf("  sensors: [")
		for _, s := range r.sensors {
			fmt.Printf("%d ", s)
		}
		fmt.Printf("]")
		fmt.Printf("  channels: [")
		for n, c := range channels {
			fmt.Printf("%s : %d  ", n, int(c))
		}
		fmt.Printf("]")
		fmt.Printf("\n")
	}
}

// determineCaContrib determines which Ca channels contributed to the release
// of a particular vesicle.
// NOTE: We try to be as agnostic as we can in terms of the particular
// nomenclature used for naming the channels. However, the expectation is
// that data files tracking Ca binding to vesicles are named
// vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func determineCaChanContrib(data *libmbd.MCellData, rel *ReleaseEvent) (map[string]float64, error) {
	channels := make(map[string]float64)
	// the az/channel counting is 1 based whereas our internal counting is 0 based
	regexString := fmt.Sprintf("vesicle_%d_%d_ca_.*", rel.azId+1, rel.vesicleID+1)
	counts, err := data.BlockDataByRegex(regexString)
	if err != nil {
		return nil, err
	}
	for k, c := range counts {
		if len(c.Col) != 1 {
			return nil, fmt.Errorf("data set %s has more than the expected 1 column",
				k)
		}
		if c.Col[0][rel.eventIter] > 0 {
			// need to subtract 2 from regexString due to the extra ".*"
			caString, err := extractCaChanName(k, len(regexString)-2)
			if err != nil {
				return nil, err
			}
			channels[caString] = c.Col[0][rel.eventIter]
		}
	}

	return channels, nil
}

// extractCaChanName attempts to extract the name of the calcium channel based
// on the expected data name pattern vesicle_<az>_<1|2>_ca_<ca naming>.<seed>.dat
func extractCaChanName(name string, prefixLength int) (string, error) {
	caName := name[prefixLength:]
	items := strings.Split(caName, ".")
	if len(items) == 0 {
		return "", fmt.Errorf("Could not determine Ca channel name from data set %s", name)
	}
	return items[0], nil
}

// main entry point
func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
		usage()
		return
	}

	// some sanity checks
	if energyModel && (sytEnergy < 0 || yEnergy < 0) {
		log.Fatal("Please provide a non-negative synaptotagmin and y site energy")
	}

	if !energyModel && numActiveSites == 0 {
		log.Fatal("Please provide a positive count for the number of required active sites")
	}

	if numPulses > 1 && isiValue <= 0 {
		log.Fatal("Analysis multi-pulse data requires a non-zero ISI value.")
	}

	printHeader()
	// loop over all provided data sets
	for _, fileName := range flag.Args() {

		seed, err := extractSeed(fileName)
		if err != nil {
			log.Fatal(err)
		}

		data, err := libmbd.Read(fileName)
		if err != nil {
			log.Fatal(err)
		}

		err = analyze(data, energyModel, seed, numPulses, numActiveSites, sytEnergy,
			yEnergy)
		if err != nil {
			log.Fatal(err)
		}
		// NOTE: This is a bit of a hack but since we're dealing with potentially
		// large data sets we need to make sure to free memory before we start
		// working on the next one
		debug.FreeOSMemory()
	}
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package loader

import (
	"net/url"
	"testing"

	"github.com/spf13/afero"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCDNJS(t *testing.T) {
	paths := map[string]struct {
		parts []string
		src   string
	}{
		"cdnjs.com/libraries/Faker": {
			[]string{"Faker", "", ""},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/[\d\.]+/faker.min.js$`,
		},
		"cdnjs.com/libraries/Faker/faker.js": {
			[]string{"Faker", "", "faker.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/[\d\.]+/faker.js$`,
		},
		"cdnjs.com/libraries/Faker/locales/en_AU/faker.en_AU.min.js": {
			[]string{"Faker", "", "locales/en_AU/faker.en_AU.min.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/[\d\.]+/locales/en_AU/faker.en_AU.min.js$`,
		},
		"cdnjs.com/libraries/Faker/3.1.0": {
			[]string{"Faker", "3.1.0", ""},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/faker.min.js$`,
		},
		"cdnjs.com/libraries/Faker/3.1.0/faker.js": {
			[]string{"Faker", "3.1.0", "faker.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/faker.js$`,
		},
		"cdnjs.com/libraries/Faker/3.1.0/locales/en_AU/faker.en_AU.min.js": {
			[]string{"Faker", "3.1.0", "locales/en_AU/faker.en_AU.min.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/locales/en_AU/faker.en_AU.min.js$`,
		},
		"cdnjs.com/libraries/Faker/0.7.2": {
			[]string{"Faker", "0.7.2", ""},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/0.7.2/MinFaker.js$`,
		},
	}

	var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"}
	for path, expected := range paths {
		path, expected := path, expected
		t.Run(path, func(t *testing.T) {
			name, loader, parts := pickLoader(path)
			assert.Equal(t, "cdnjs", name)
			assert.Equal(t, expected.parts, parts)

			src, err := loader(path, parts)
			require.NoError(t, err)
			assert.Regexp(t, expected.src, src)

			resolvedURL, err := Resolve(root, path)
			require.NoError(t, err)
			require.Empty(t, resolvedURL.Scheme)
			require.Equal(t, path, resolvedURL.Opaque)

			data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path)
			require.NoError(t, err)
			assert.Equal(t, resolvedURL, data.URL)
			assert.NotEmpty(t, data.Data)
		})
	}

	t.Run("cdnjs.com/libraries/nonexistent", func(t *testing.T) {
		path := "cdnjs.com/libraries/nonexistent"
		name, loader, parts := pickLoader(path)
		assert.Equal(t, "cdnjs", name)
		assert.Equal(t, []string{"nonexistent", "", ""}, parts)
		_, err := loader(path, parts)
		assert.EqualError(t, err, "cdnjs: no such library: nonexistent")
	})

	t.Run("cdnjs.com/libraries/Faker/3.1.0/nonexistent.js", func(t *testing.T) {
		path := "cdnjs.com/libraries/Faker/3.1.0/nonexistent.js"
		name, loader, parts := pickLoader(path)
		assert.Equal(t, "cdnjs", name)
		assert.Equal(t, []string{"Faker", "3.1.0", "nonexistent.js"}, parts)
		src, err := loader(path, parts)
		require.NoError(t, err)
		assert.Equal(t, "https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js", src)

		pathURL, err := url.Parse(src)
		require.NoError(t, err)

		_, err = Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path)
		require.Error(t, err)
		assert.Contains(t, err.Error(), "not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js")
	})
}
<commit_msg>Skip TestCDNJS<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package loader

import (
	"net/url"
	"testing"

	"github.com/spf13/afero"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCDNJS(t *testing.T) {
	t.Skip("skipped to avoid inconsistent API responses")

	paths := map[string]struct {
		parts []string
		src   string
	}{
		"cdnjs.com/libraries/Faker": {
			[]string{"Faker", "", ""},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/[\d\.]+/faker.min.js$`,
		},
		"cdnjs.com/libraries/Faker/faker.js": {
			[]string{"Faker", "", "faker.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/[\d\.]+/faker.js$`,
		},
		"cdnjs.com/libraries/Faker/locales/en_AU/faker.en_AU.min.js": {
			[]string{"Faker", "", "locales/en_AU/faker.en_AU.min.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/[\d\.]+/locales/en_AU/faker.en_AU.min.js$`,
		},
		"cdnjs.com/libraries/Faker/3.1.0": {
			[]string{"Faker", "3.1.0", ""},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/faker.min.js$`,
		},
		"cdnjs.com/libraries/Faker/3.1.0/faker.js": {
			[]string{"Faker", "3.1.0", "faker.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/faker.js$`,
		},
		"cdnjs.com/libraries/Faker/3.1.0/locales/en_AU/faker.en_AU.min.js": {
			[]string{"Faker", "3.1.0", "locales/en_AU/faker.en_AU.min.js"},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/locales/en_AU/faker.en_AU.min.js$`,
		},
		"cdnjs.com/libraries/Faker/0.7.2": {
			[]string{"Faker", "0.7.2", ""},
			`^https://cdnjs.cloudflare.com/ajax/libs/Faker/0.7.2/MinFaker.js$`,
		},
	}

	var root = &url.URL{Scheme: "https", Host: "example.com", Path: "/something/"}
	for path, expected := range paths {
		path, expected := path, expected
		t.Run(path, func(t *testing.T) {
			name, loader, parts := pickLoader(path)
			assert.Equal(t, "cdnjs", name)
			assert.Equal(t, expected.parts, parts)

			src, err := loader(path, parts)
			require.NoError(t, err)
			assert.Regexp(t, expected.src, src)

			resolvedURL, err := Resolve(root, path)
			require.NoError(t, err)
			require.Empty(t, resolvedURL.Scheme)
			require.Equal(t, path, resolvedURL.Opaque)

			data, err := Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, resolvedURL, path)
			require.NoError(t, err)
			assert.Equal(t, resolvedURL, data.URL)
			assert.NotEmpty(t, data.Data)
		})
	}

	t.Run("cdnjs.com/libraries/nonexistent", func(t *testing.T) {
		path := "cdnjs.com/libraries/nonexistent"
		name, loader, parts := pickLoader(path)
		assert.Equal(t, "cdnjs", name)
		assert.Equal(t, []string{"nonexistent", "", ""}, parts)
		_, err := loader(path, parts)
		assert.EqualError(t, err, "cdnjs: no such library: nonexistent")
	})

	t.Run("cdnjs.com/libraries/Faker/3.1.0/nonexistent.js", func(t *testing.T) {
		path := "cdnjs.com/libraries/Faker/3.1.0/nonexistent.js"
		name, loader, parts := pickLoader(path)
		assert.Equal(t, "cdnjs", name)
		assert.Equal(t, []string{"Faker", "3.1.0", "nonexistent.js"}, parts)
		src, err := loader(path, parts)
		require.NoError(t, err)
		assert.Equal(t, "https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js", src)

		pathURL, err := url.Parse(src)
		require.NoError(t, err)

		_, err = Load(map[string]afero.Fs{"https": afero.NewMemMapFs()}, pathURL, path)
		require.Error(t, err)
		assert.Contains(t, err.Error(), "not found: https://cdnjs.cloudflare.com/ajax/libs/Faker/3.1.0/nonexistent.js")
	})
}
<|endoftext|>


<commit_before>package template

import (
	"fmt"

	"os"

	"github.com/devimteam/microgen/generator/write_strategy"
	"github.com/devimteam/microgen/util"
	"github.com/vetcher/godecl/types"
	. "github.com/vetcher/jennifer/jen"
)

const (
	GolangProtobufPtypesTimestamp = "github.com/golang/protobuf/ptypes/timestamp"
	JsonbPackage                  = "github.com/sas1024/gorm-jsonb/jsonb"
)

func specialTypeConverter(p *types.Type) *Statement {
	// error -> string
	if p.Name == "error" && p.Import == nil {
		return (&Statement{}).Id("string")
	}
	// time.Time -> timestamp.Timestamp
	if p.Name == "Time" && p.Import != nil && p.Import.Package == "time" {
		return (&Statement{}).Qual(GolangProtobufPtypesTimestamp, "Timestamp")
	}
	// jsonb.JSONB -> string
	if p.Name == "JSONB" && p.Import != nil && p.Import.Package == JsonbPackage {
		return (&Statement{}).Id("string")
	}
	return nil
}

func converterToProtoBody(field *types.Variable) Code {
	s := &Statement{}
	switch typeToProto(&field.Type, 0) {
	case "ErrorToProto":
		s.If().Id(util.ToLowerFirst(field.Name)).Op("==").Nil().Block(
			Return().List(Lit(""), Nil()),
		).Line()
		s.Return().List(Id(util.ToLowerFirst(field.Name)).Dot("Error").Call(), Nil())
	case "ByteListToProto":
		s.Return().List(Id(util.ToLowerFirst(field.Name)), Nil())
	default:
		s.Panic(Lit("method not provided"))
	}
	return s
}

func converterProtoToBody(field *types.Variable) Code {
	s := &Statement{}
	switch protoToType(&field.Type, 0) {
	case "ProtoToError":
		s.If().Id("proto" + util.ToUpperFirst(field.Name)).Op("==").Lit("").Block(
			Return().List(Nil(), Nil()),
		).Line()
		s.Return().List(Qual("errors", "New").Call(Id("proto"+util.ToUpperFirst(field.Name))), Nil())
	case "ProtoToByteList":
		s.Return().List(Id("proto"+util.ToLowerFirst(field.Name)), Nil())
	default:
		s.Panic(Lit("method not provided"))
	}
	return s
}

type stubGRPCTypeConverterTemplate struct {
	Info                      *GenerationInfo
	alreadyRenderedConverters []string
	state                     WriteStrategyState
}

func NewStubGRPCTypeConverterTemplate(info *GenerationInfo) Template {
	return &stubGRPCTypeConverterTemplate{
		Info: info.Duplicate(),
	}
}

// Render whole file with protobuf converters.
//
//		// This file was automatically generated by "microgen" utility.
//		package protobuf
//
//		func IntListToProto(positions []int) (protoPositions []int64, convPositionsErr error) {
//			panic("method not provided")
//		}
//
//		func ProtoToIntList(protoPositions []int64) (positions []int, convPositionsErr error) {
//			panic("method not provided")
//		}
//
func (t *stubGRPCTypeConverterTemplate) Render() write_strategy.Renderer {
	f := &Statement{}

	for _, signature := range t.Info.Iface.Methods {
		args := append(removeContextIfFirst(signature.Args), removeContextIfFirst(signature.Results)...)
		for _, field := range args {
			if _, ok := golangTypeToProto("", &field); !ok && !util.IsInStringSlice(typeToProto(&field.Type, 0), t.alreadyRenderedConverters) {
				f.Add(t.stubConverterToProto(&field)).Line().Line()
				t.alreadyRenderedConverters = append(t.alreadyRenderedConverters, typeToProto(&field.Type, 0))
				f.Add(t.stubConverterProtoTo(&field)).Line().Line()
				t.alreadyRenderedConverters = append(t.alreadyRenderedConverters, protoToType(&field.Type, 0))
			}
		}
	}

	if t.state == AppendStrat {
		return f
	}

	file := NewFile(t.Info.ServiceImportPackageName)
	file.PackageComment(FileHeader)
	file.PackageComment(`It is better for you if you do not change functions names!`)
	file.PackageComment(`This file will never be overwritten.`)
	file.Add(f)

	return file
}

func (stubGRPCTypeConverterTemplate) DefaultPath() string {
	return "./transport/converter/protobuf/type_converters.go"
}

func (t *stubGRPCTypeConverterTemplate) Prepare() error {
	if t.Info.ProtobufPackage == "" {
		return fmt.Errorf("protobuf package is empty")
	}
	return nil
}

func (t *stubGRPCTypeConverterTemplate) ChooseStrategy() (write_strategy.Strategy, error) {
	if err := util.TryToOpenFile(t.Info.AbsOutPath, t.DefaultPath()); os.IsNotExist(err) {
		t.state = FileStrat
		return write_strategy.NewFileMethod(t.Info.AbsOutPath, t.DefaultPath()), nil
	}
	t.state = AppendStrat
	return write_strategy.AppendToFileStrategy(t.Info.AbsOutPath, t.DefaultPath()), nil
}

// Render stub method for golang to protobuf converter.
//
//		func IntListToProto(positions []int) (protoPositions []int64, convPositionsErr error) {
//			return
//		}
//
func (t *stubGRPCTypeConverterTemplate) stubConverterToProto(field *types.Variable) *Statement {
	return Func().Id(typeToProto(&field.Type, 0)).
		Params(Id(util.ToLowerFirst(field.Name)).Add(fieldType(&field.Type))).
		Params(Id("proto"+util.ToUpperFirst(field.Name)).Add(t.protoFieldType(&field.Type)), Id("conv"+util.ToUpperFirst(field.Name)+"Err").Error()).
		Block(converterToProtoBody(field))
}

// Render stub method for protobuf to golang converter.
//
//		func ProtoToIntList(protoPositions []int64) (positions []int, convPositionsErr error) {
//			return
//		}
//
func (t *stubGRPCTypeConverterTemplate) stubConverterProtoTo(field *types.Variable) *Statement {
	return Func().Id(protoToType(&field.Type, 0)).
		Params(Id("proto"+util.ToUpperFirst(field.Name)).Add(t.protoFieldType(&field.Type))).
		Params(Id(util.ToLowerFirst(field.Name)).Add(fieldType(&field.Type)), Id("conv"+util.ToUpperFirst(field.Name)+"Err").Error()).
		Block(converterProtoToBody(field))
}

// Render protobuf field type for given func field.
//
//  	*repository.Visit
//
func (t *stubGRPCTypeConverterTemplate) protoFieldType(field *types.Type) *Statement {
	c := &Statement{}

	if field.IsArray {
		c.Index()
	}

	if field.IsPointer {
		c.Op("*")
	}

	if field.IsMap {
		m := field.Map()
		return c.Map(t.protoFieldType(&m.Key)).Add(t.protoFieldType(&m.Value))
	}
	protoType := field.Name
	if tmp, ok := goToProtoTypesMap[field.Name]; ok {
		protoType = tmp
	}
	if code := specialTypeConverter(field); code != nil {
		return c.Add(code)
	}
	if field.Import != nil {
		c.Qual(t.Info.ProtobufPackage, protoType)
	} else {
		c.Id(protoType)
	}

	return c
}
<commit_msg>feat(template): add fine append to file mechanism, part 2<commit_after>package template

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/devimteam/microgen/generator/write_strategy"
	"github.com/devimteam/microgen/util"
	"github.com/vetcher/godecl/types"
	. "github.com/vetcher/jennifer/jen"
)

const (
	GolangProtobufPtypesTimestamp = "github.com/golang/protobuf/ptypes/timestamp"
	JsonbPackage                  = "github.com/sas1024/gorm-jsonb/jsonb"
)

func specialTypeConverter(p *types.Type) *Statement {
	// error -> string
	if p.Name == "error" && p.Import == nil {
		return (&Statement{}).Id("string")
	}
	// time.Time -> timestamp.Timestamp
	if p.Name == "Time" && p.Import != nil && p.Import.Package == "time" {
		return (&Statement{}).Qual(GolangProtobufPtypesTimestamp, "Timestamp")
	}
	// jsonb.JSONB -> string
	if p.Name == "JSONB" && p.Import != nil && p.Import.Package == JsonbPackage {
		return (&Statement{}).Id("string")
	}
	return nil
}

func converterToProtoBody(field *types.Variable) Code {
	s := &Statement{}
	switch typeToProto(&field.Type, 0) {
	case "ErrorToProto":
		s.If().Id(util.ToLowerFirst(field.Name)).Op("==").Nil().Block(
			Return().List(Lit(""), Nil()),
		).Line()
		s.Return().List(Id(util.ToLowerFirst(field.Name)).Dot("Error").Call(), Nil())
	case "ByteListToProto":
		s.Return().List(Id(util.ToLowerFirst(field.Name)), Nil())
	default:
		s.Panic(Lit("method not provided"))
	}
	return s
}

func converterProtoToBody(field *types.Variable) Code {
	s := &Statement{}
	switch protoToType(&field.Type, 0) {
	case "ProtoToError":
		s.If().Id("proto" + util.ToUpperFirst(field.Name)).Op("==").Lit("").Block(
			Return().List(Nil(), Nil()),
		).Line()
		s.Return().List(Qual("errors", "New").Call(Id("proto"+util.ToUpperFirst(field.Name))), Nil())
	case "ProtoToByteList":
		s.Return().List(Id("proto"+util.ToLowerFirst(field.Name)), Nil())
	default:
		s.Panic(Lit("method not provided"))
	}
	return s
}

type stubGRPCTypeConverterTemplate struct {
	Info                      *GenerationInfo
	alreadyRenderedConverters []string
	state                     WriteStrategyState
}

func NewStubGRPCTypeConverterTemplate(info *GenerationInfo) Template {
	return &stubGRPCTypeConverterTemplate{
		Info: info.Duplicate(),
	}
}

// Render whole file with protobuf converters.
//
//		// This file was automatically generated by "microgen" utility.
//		package protobuf
//
//		func IntListToProto(positions []int) (protoPositions []int64, convPositionsErr error) {
//			panic("method not provided")
//		}
//
//		func ProtoToIntList(protoPositions []int64) (positions []int, convPositionsErr error) {
//			panic("method not provided")
//		}
//
func (t *stubGRPCTypeConverterTemplate) Render() write_strategy.Renderer {
	f := &Statement{}

	for _, signature := range t.Info.Iface.Methods {
		args := append(removeContextIfFirst(signature.Args), removeContextIfFirst(signature.Results)...)
		for _, field := range args {
			if _, ok := golangTypeToProto("", &field); !ok && !util.IsInStringSlice(typeToProto(&field.Type, 0), t.alreadyRenderedConverters) {
				f.Add(t.stubConverterToProto(&field)).Line().Line()
				t.alreadyRenderedConverters = append(t.alreadyRenderedConverters, typeToProto(&field.Type, 0))
				f.Add(t.stubConverterProtoTo(&field)).Line().Line()
				t.alreadyRenderedConverters = append(t.alreadyRenderedConverters, protoToType(&field.Type, 0))
			}
		}
	}

	if t.state == AppendStrat {
		return f
	}

	file := NewFile(t.Info.ServiceImportPackageName)
	file.PackageComment(FileHeader)
	file.PackageComment(`It is better for you if you do not change functions names!`)
	file.PackageComment(`This file will never be overwritten.`)
	file.Add(f)

	return file
}

func (stubGRPCTypeConverterTemplate) DefaultPath() string {
	return "./transport/converter/protobuf/type_converters.go"
}

func (t *stubGRPCTypeConverterTemplate) Prepare() error {
	if t.Info.ProtobufPackage == "" {
		return fmt.Errorf("protobuf package is empty")
	}
	return nil
}

func (t *stubGRPCTypeConverterTemplate) ChooseStrategy() (write_strategy.Strategy, error) {
	if err := util.TryToOpenFile(t.Info.AbsOutPath, t.DefaultPath()); os.IsNotExist(err) {
		t.state = FileStrat
		return write_strategy.NewFileMethod(t.Info.AbsOutPath, t.DefaultPath()), nil
	}
	file, err := util.ParseFile(filepath.Join(t.Info.AbsOutPath, t.DefaultPath()))
	if err != nil {
		return nil, err
	}

	for i := range file.Functions {
		t.alreadyRenderedConverters = append(t.alreadyRenderedConverters, file.Functions[i].Name)
	}

	t.state = AppendStrat
	return write_strategy.AppendToFileStrategy(t.Info.AbsOutPath, t.DefaultPath()), nil
}

// Render stub method for golang to protobuf converter.
//
//		func IntListToProto(positions []int) (protoPositions []int64, convPositionsErr error) {
//			return
//		}
//
func (t *stubGRPCTypeConverterTemplate) stubConverterToProto(field *types.Variable) *Statement {
	return Func().Id(typeToProto(&field.Type, 0)).
		Params(Id(util.ToLowerFirst(field.Name)).Add(fieldType(&field.Type))).
		Params(Id("proto"+util.ToUpperFirst(field.Name)).Add(t.protoFieldType(&field.Type)), Id("conv"+util.ToUpperFirst(field.Name)+"Err").Error()).
		Block(converterToProtoBody(field))
}

// Render stub method for protobuf to golang converter.
//
//		func ProtoToIntList(protoPositions []int64) (positions []int, convPositionsErr error) {
//			return
//		}
//
func (t *stubGRPCTypeConverterTemplate) stubConverterProtoTo(field *types.Variable) *Statement {
	return Func().Id(protoToType(&field.Type, 0)).
		Params(Id("proto"+util.ToUpperFirst(field.Name)).Add(t.protoFieldType(&field.Type))).
		Params(Id(util.ToLowerFirst(field.Name)).Add(fieldType(&field.Type)), Id("conv"+util.ToUpperFirst(field.Name)+"Err").Error()).
		Block(converterProtoToBody(field))
}

// Render protobuf field type for given func field.
//
//  	*repository.Visit
//
func (t *stubGRPCTypeConverterTemplate) protoFieldType(field *types.Type) *Statement {
	c := &Statement{}

	if field.IsArray {
		c.Index()
	}

	if field.IsPointer {
		c.Op("*")
	}

	if field.IsMap {
		m := field.Map()
		return c.Map(t.protoFieldType(&m.Key)).Add(t.protoFieldType(&m.Value))
	}
	protoType := field.Name
	if tmp, ok := goToProtoTypesMap[field.Name]; ok {
		protoType = tmp
	}
	if code := specialTypeConverter(field); code != nil {
		return c.Add(code)
	}
	if field.Import != nil {
		c.Qual(t.Info.ProtobufPackage, protoType)
	} else {
		c.Id(protoType)
	}

	return c
}
<|endoftext|>


<commit_before>package loopback

import (
	"sync"
	"io"
	"os"
	"time"
)

// TODO implement CloseWithError

type block struct {
	t    int64
	data []byte
	prev *block
	next *block
}

type streamReader stream
type streamWriter stream

func (r *streamReader) Read(data []byte) (int, os.Error) {
	return (*stream)(r).Read(data)
}

func (r *streamReader) Close() os.Error {
	return (*stream)(r).closeInput()
}

func (w *streamWriter) Write(data []byte) (int, os.Error) {
	return (*stream)(w).Write(data)
}

func (w *streamWriter) Close() os.Error {
	return (*stream)(w).closeOutput()
}

type stream struct {
	mu sync.Mutex

	outClosed bool
	inClosed  bool

	outTail     *block // sentinel.
	outHead     *block // also transitTail.
	transitHead *block // also inTail.
	inHead      *block // overall head of list.

	outLimit int // total size of output queue.
	outAvail int // free bytes in output queue.

	inLimit int // total size of input queue.
	inAvail int // free bytes in input queue.

	byteDelay int64
	latency   int64
	mtu       int

	rwait sync.Cond
	wwait  sync.Cond
}

// Loopback options for use with Pipe.
type Options struct {
	// ByteDelay controls the time a packet takes in the link.  A packet
	// n bytes long takes ByteDelay * n nanoseconds to exit
	// the output queue and is available for reading Latency
	// nanoseconds later.
	ByteDelay int64
	Latency   int64

	// MTU gives the maximum packet size that can
	// be tranferred atomically across the link.
	// Larger packet will be split.
	// If this is zero, a default of 32768 is assumed
	MTU int

	// InLimit and OutLimit gives the size of the input and output queues.
	// If either is zero, a default of MTU is assumed.
	InLimit  int
	OutLimit int
}

// Pipe creates an asynchronous in-memory pipe,
// Writes are divided into packets of at most opts.MTU bytes
// written to a flow-controlled output queue, transferred across the link,
// and put into an input queue where it is readable with the r.
// The options determine when and how the data will be transferred.
func Pipe(opt Options) (r io.ReadCloser, w io.WriteCloser) {
	if opt.MTU == 0 {
		opt.MTU = 32768
	}
	if opt.InLimit == 0 {
		opt.InLimit = opt.MTU
	}
	if opt.OutLimit == 0 {
		opt.OutLimit = opt.MTU
	}
	if opt.InLimit < opt.MTU {
		opt.InLimit = opt.MTU
	}
	if opt.OutLimit < opt.MTU {
		opt.OutLimit = opt.MTU
	}
	sentinel := &block{}
	s := &stream{
		outLimit:    opt.OutLimit,
		outAvail:    opt.OutLimit,
		inLimit:     opt.InLimit,
		inAvail:     opt.InLimit,
		mtu:         opt.MTU,
		byteDelay:   opt.ByteDelay,
		latency:     opt.Latency,
		outTail:     sentinel,
		outHead:     sentinel,
		transitHead: sentinel,
		inHead:      sentinel,
	}
	s.rwait.L = &s.mu
	s.wwait.L = &s.mu
	return (*streamReader)(s), (*streamWriter)(s)
}

// Dodgy heuristic:
// If there's stuff in the transit queue that's ready to
// enter the input queue, but the input queue is full
// and it's been waiting for at least latency ns,
// then we block the output queue.
// TODO what do we do about latency for
// blocked packets - as it is a blocked packet
// will incur less latency.
func (s *stream) outBlocked(now int64) bool {
	return s.transitHead != s.outHead &&
		now >= s.transitHead.t+s.latency &&
		s.inAvail < len(s.transitHead.data)
}

func (s *stream) closeInput() os.Error {
	s.mu.Lock()
	s.inClosed = true
	s.rwait.Broadcast()
	s.wwait.Broadcast()
	s.mu.Unlock()
	return nil
}

func (s *stream) closeOutput() os.Error {
	s.mu.Lock()
	s.outClosed = true
	s.rwait.Broadcast()
	s.wwait.Broadcast()
	s.mu.Unlock()
	return nil
}

func (s *stream) pushLink(now int64) {
	if !s.outBlocked(now) {
		// move blocks from out queue to transit queue.
		for s.outTail != s.outHead && now >= s.outHead.t {
			s.outHead.t += s.latency
			s.outAvail += len(s.outHead.data)
			s.outHead = s.outHead.next
		}
	}
	// move blocks from transit queue to input queue
	for s.transitHead != s.outHead && now >= s.transitHead.t {
		if s.inAvail < len(s.transitHead.data) {
			break // or discard packet
		}
		s.inAvail -= len(s.transitHead.data)
		s.transitHead = s.transitHead.next
	}
}

func (s *stream) Write(data []byte) (int, os.Error) {
	// split the packet into MTU-sized portions if necessary.
	tot := 0
	for len(data) > s.mtu {
		n, err := s.Write(data[0:s.mtu])
		tot += n
		if err != nil {
			return tot, err
		}
		data = data[s.mtu:]
	}
	s.mu.Lock()
	now := time.Nanoseconds()
	for {
		s.pushLink(now)
		if s.outAvail >= len(data) || s.outClosed {
			break
		}
		if s.outBlocked(time.Nanoseconds()) {
			if s.inClosed {
				s.mu.Unlock()
				return 0, os.EPIPE
			}
			s.wwait.Wait()
			continue
		}
		t := s.earliestWriteTime(len(data))
		now = s.sleepUntil(t)
	}
	if s.outClosed {
		s.mu.Unlock()
		return 0, os.EPIPE
	}
	delay := int64(len(data)) * s.byteDelay
	var t int64
	// If there's a block in the queue that's not yet due
	// for transit, then this block leaves delay ns after
	// that one.
	if s.outHead != s.outTail && now < s.outTail.prev.t {
		t = s.outTail.prev.t + delay
	} else {
		t = now + delay
	}
	s.addBlock(t, s.copy(data))
	s.outAvail -= len(data)

	s.rwait.Broadcast()
	s.mu.Unlock()
	// TODO runtime.Gosched() ?
	return len(data), nil
}

func (s *stream) Read(buf []byte) (int, os.Error) {
	s.mu.Lock()
	// Loop until there's something to read from the input queue.
	now := time.Nanoseconds()
	for {
		s.pushLink(now)
		if s.inHead != s.transitHead {
			break
		}
		if s.inHead == s.outTail {
			// No data at all in the queue.
			// If the queue is empty and the output queue is closed,
			// then we see EOF.
			if s.outClosed {
				s.mu.Unlock()
				return 0, os.EOF
			}
			s.rwait.Wait()
			continue
		}
		now = s.sleepUntil(s.earliestReadTime())
	}
	if s.inClosed {
		// input queue has been forcibly closed:
		// TODO is os.EOF the right error here?
		return 0, os.EOF
	}
	b := s.inHead
	n := copy(buf, b.data)
	b.data = b.data[n:]
	s.inAvail += n
	if len(b.data) == 0 {
		s.removeBlock()
	}
	// Wake up any writers blocked on a full queue.
	s.wwait.Broadcast()
	s.mu.Unlock()
	return n, nil
}

// earliestReadTime returns the earliest time that
// some data might arrive into the input queue.
// It assumes that there is some data in the system.
func (s *stream) earliestReadTime() int64 {
	if s.inAvail < s.inLimit {
		// data is available right now.
		return 0
	}
	if s.transitHead != s.outHead {
		return s.transitHead.t
	}
	if s.outHead != s.outTail {
		return s.outHead.t + s.latency
	}
	panic("no data")
}

// earliestWriteTime returns the earliest time that
// there may be space for n bytes of data to be
// placed into the output queue (it might be later
// if packets are dropped).
func (s *stream) earliestWriteTime(n int) int64 {
	if s.outAvail < s.outLimit {
		// space is available now.
		return 0
	}
	tot := s.outAvail
	for b := s.outHead; b != s.outTail; b = b.next {
		tot += len(b.data)
		if tot >= n {
			return b.t
		}
	}
	panic("write limit exceeded by block size")
}

// sleep until the absolute time t.
// Called with lock held.
func (s *stream) sleepUntil(t int64) int64 {
	now := time.Nanoseconds()
	if now >= t {
		return now
	}
	s.mu.Unlock()
	time.Sleep(t - now)
	s.mu.Lock()
	return time.Nanoseconds()
}

func (s *stream) copy(x []byte) []byte {
	y := make([]byte, len(x))
	copy(y, x)
	return y
}

// addBlock adds a block to the head of the queue.
// It does not adjust queue stats.
func (s *stream) addBlock(t int64, data []byte) {
	// If there are no items in output queue, replace sentinel block
	// so that other pointers into queue do not need
	// to change.
	if s.outHead == s.outTail {
		s.outHead.t = t
		s.outHead.data = data
		s.outHead.next = &block{prev: s.outHead} // new sentinel
		s.outTail = s.outHead.next
		return
	}

	// Add a new block just after the sentinel.
	b := &block{
		t:    t,
		data: data,
	}
	b.next = s.outTail
	b.prev = s.outTail.prev

	s.outTail.prev = b
	b.prev.next = b
}

// Remove the block from the front of the queue.
// (assumes that there is such a block to remove)
func (s *stream) removeBlock() {
	b := s.inHead
	s.inHead = b.next
	if s.inHead != nil {
		s.inHead.prev = nil
	}
	// help garbage collector
	b.next = nil
	b.prev = nil
}
<commit_msg>add loopback close bug message<commit_after>package loopback

import (
	"sync"
	"io"
	"os"
	"time"
)

// TODO implement CloseWithError.
// BUG close propagates faster than latency time.
//	send close as a block with nil data.

type block struct {
	t    int64
	data []byte
	prev *block
	next *block
}

type streamReader stream
type streamWriter stream

func (r *streamReader) Read(data []byte) (int, os.Error) {
	return (*stream)(r).Read(data)
}

func (r *streamReader) Close() os.Error {
	return (*stream)(r).closeInput()
}

func (w *streamWriter) Write(data []byte) (int, os.Error) {
	return (*stream)(w).Write(data)
}

func (w *streamWriter) Close() os.Error {
	return (*stream)(w).closeOutput()
}

type stream struct {
	mu sync.Mutex

	outClosed bool
	inClosed  bool

	outTail     *block // sentinel.
	outHead     *block // also transitTail.
	transitHead *block // also inTail.
	inHead      *block // overall head of list.

	outLimit int // total size of output queue.
	outAvail int // free bytes in output queue.

	inLimit int // total size of input queue.
	inAvail int // free bytes in input queue.

	byteDelay int64
	latency   int64
	mtu       int

	rwait sync.Cond
	wwait  sync.Cond
}

// Loopback options for use with Pipe.
type Options struct {
	// ByteDelay controls the time a packet takes in the link.  A packet
	// n bytes long takes ByteDelay * n nanoseconds to exit
	// the output queue and is available for reading Latency
	// nanoseconds later.
	ByteDelay int64
	Latency   int64

	// MTU gives the maximum packet size that can
	// be tranferred atomically across the link.
	// Larger packet will be split.
	// If this is zero, a default of 32768 is assumed
	MTU int

	// InLimit and OutLimit gives the size of the input and output queues.
	// If either is zero, a default of MTU is assumed.
	InLimit  int
	OutLimit int
}

// Pipe creates an asynchronous in-memory pipe,
// Writes are divided into packets of at most opts.MTU bytes
// written to a flow-controlled output queue, transferred across the link,
// and put into an input queue where it is readable with the r.
// The options determine when and how the data will be transferred.
func Pipe(opt Options) (r io.ReadCloser, w io.WriteCloser) {
	if opt.MTU == 0 {
		opt.MTU = 32768
	}
	if opt.InLimit == 0 {
		opt.InLimit = opt.MTU
	}
	if opt.OutLimit == 0 {
		opt.OutLimit = opt.MTU
	}
	if opt.InLimit < opt.MTU {
		opt.InLimit = opt.MTU
	}
	if opt.OutLimit < opt.MTU {
		opt.OutLimit = opt.MTU
	}
	sentinel := &block{}
	s := &stream{
		outLimit:    opt.OutLimit,
		outAvail:    opt.OutLimit,
		inLimit:     opt.InLimit,
		inAvail:     opt.InLimit,
		mtu:         opt.MTU,
		byteDelay:   opt.ByteDelay,
		latency:     opt.Latency,
		outTail:     sentinel,
		outHead:     sentinel,
		transitHead: sentinel,
		inHead:      sentinel,
	}
	s.rwait.L = &s.mu
	s.wwait.L = &s.mu
	return (*streamReader)(s), (*streamWriter)(s)
}

// Dodgy heuristic:
// If there's stuff in the transit queue that's ready to
// enter the input queue, but the input queue is full
// and it's been waiting for at least latency ns,
// then we block the output queue.
// TODO what do we do about latency for
// blocked packets - as it is a blocked packet
// will incur less latency.
func (s *stream) outBlocked(now int64) bool {
	return s.transitHead != s.outHead &&
		now >= s.transitHead.t+s.latency &&
		s.inAvail < len(s.transitHead.data)
}

func (s *stream) closeInput() os.Error {
	s.mu.Lock()
	s.inClosed = true
	s.rwait.Broadcast()
	s.wwait.Broadcast()
	s.mu.Unlock()
	return nil
}

func (s *stream) closeOutput() os.Error {
	s.mu.Lock()
	s.outClosed = true
	s.rwait.Broadcast()
	s.wwait.Broadcast()
	s.mu.Unlock()
	return nil
}

func (s *stream) pushLink(now int64) {
	if !s.outBlocked(now) {
		// move blocks from out queue to transit queue.
		for s.outTail != s.outHead && now >= s.outHead.t {
			s.outHead.t += s.latency
			s.outAvail += len(s.outHead.data)
			s.outHead = s.outHead.next
		}
	}
	// move blocks from transit queue to input queue
	for s.transitHead != s.outHead && now >= s.transitHead.t {
		if s.inAvail < len(s.transitHead.data) {
			break // or discard packet
		}
		s.inAvail -= len(s.transitHead.data)
		s.transitHead = s.transitHead.next
	}
}

func (s *stream) Write(data []byte) (int, os.Error) {
	// split the packet into MTU-sized portions if necessary.
	tot := 0
	for len(data) > s.mtu {
		n, err := s.Write(data[0:s.mtu])
		tot += n
		if err != nil {
			return tot, err
		}
		data = data[s.mtu:]
	}
	s.mu.Lock()
	now := time.Nanoseconds()
	for {
		s.pushLink(now)
		if s.outAvail >= len(data) || s.outClosed {
			break
		}
		if s.outBlocked(time.Nanoseconds()) {
			if s.inClosed {
				s.mu.Unlock()
				return 0, os.EPIPE
			}
			s.wwait.Wait()
			continue
		}
		t := s.earliestWriteTime(len(data))
		now = s.sleepUntil(t)
	}
	if s.outClosed {
		s.mu.Unlock()
		return 0, os.EPIPE
	}
	delay := int64(len(data)) * s.byteDelay
	var t int64
	// If there's a block in the queue that's not yet due
	// for transit, then this block leaves delay ns after
	// that one.
	if s.outHead != s.outTail && now < s.outTail.prev.t {
		t = s.outTail.prev.t + delay
	} else {
		t = now + delay
	}
	s.addBlock(t, s.copy(data))
	s.outAvail -= len(data)

	s.rwait.Broadcast()
	s.mu.Unlock()
	// TODO runtime.Gosched() ?
	return len(data), nil
}

func (s *stream) Read(buf []byte) (int, os.Error) {
	s.mu.Lock()
	// Loop until there's something to read from the input queue.
	now := time.Nanoseconds()
	for {
		s.pushLink(now)
		if s.inHead != s.transitHead {
			break
		}
		if s.inHead == s.outTail {
			// No data at all in the queue.
			// If the queue is empty and the output queue is closed,
			// then we see EOF.
			if s.outClosed {
				s.mu.Unlock()
				return 0, os.EOF
			}
			s.rwait.Wait()
			continue
		}
		now = s.sleepUntil(s.earliestReadTime())
	}
	if s.inClosed {
		// input queue has been forcibly closed:
		// TODO is os.EOF the right error here?
		return 0, os.EOF
	}
	b := s.inHead
	n := copy(buf, b.data)
	b.data = b.data[n:]
	s.inAvail += n
	if len(b.data) == 0 {
		s.removeBlock()
	}
	// Wake up any writers blocked on a full queue.
	s.wwait.Broadcast()
	s.mu.Unlock()
	return n, nil
}

// earliestReadTime returns the earliest time that
// some data might arrive into the input queue.
// It assumes that there is some data in the system.
func (s *stream) earliestReadTime() int64 {
	if s.inAvail < s.inLimit {
		// data is available right now.
		return 0
	}
	if s.transitHead != s.outHead {
		return s.transitHead.t
	}
	if s.outHead != s.outTail {
		return s.outHead.t + s.latency
	}
	panic("no data")
}

// earliestWriteTime returns the earliest time that
// there may be space for n bytes of data to be
// placed into the output queue (it might be later
// if packets are dropped).
func (s *stream) earliestWriteTime(n int) int64 {
	if s.outAvail < s.outLimit {
		// space is available now.
		return 0
	}
	tot := s.outAvail
	for b := s.outHead; b != s.outTail; b = b.next {
		tot += len(b.data)
		if tot >= n {
			return b.t
		}
	}
	panic("write limit exceeded by block size")
}

// sleep until the absolute time t.
// Called with lock held.
func (s *stream) sleepUntil(t int64) int64 {
	now := time.Nanoseconds()
	if now >= t {
		return now
	}
	s.mu.Unlock()
	time.Sleep(t - now)
	s.mu.Lock()
	return time.Nanoseconds()
}

func (s *stream) copy(x []byte) []byte {
	y := make([]byte, len(x))
	copy(y, x)
	return y
}

// addBlock adds a block to the head of the queue.
// It does not adjust queue stats.
func (s *stream) addBlock(t int64, data []byte) {
	// If there are no items in output queue, replace sentinel block
	// so that other pointers into queue do not need
	// to change.
	if s.outHead == s.outTail {
		s.outHead.t = t
		s.outHead.data = data
		s.outHead.next = &block{prev: s.outHead} // new sentinel
		s.outTail = s.outHead.next
		return
	}

	// Add a new block just after the sentinel.
	b := &block{
		t:    t,
		data: data,
	}
	b.next = s.outTail
	b.prev = s.outTail.prev

	s.outTail.prev = b
	b.prev.next = b
}

// Remove the block from the front of the queue.
// (assumes that there is such a block to remove)
func (s *stream) removeBlock() {
	b := s.inHead
	s.inHead = b.next
	if s.inHead != nil {
		s.inHead.prev = nil
	}
	// help garbage collector
	b.next = nil
	b.prev = nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This tool displays the content of the export file.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/google/exposure-notifications-server/internal/export"
	exportpb "github.com/google/exposure-notifications-server/internal/pb/export"
	"github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/hashicorp/go-multierror"
)

var (
	filePath       = flag.String("file", "", "path to the export zip file.")
	printJSON      = flag.Bool("json", true, "print a JSON representation of the output")
	quiet          = flag.Bool("q", false, "run in quiet mode")
	allowedTEKAge  = flag.Duration("tek-age", 14*24*time.Hour, "max TEK age in checks")
	symptomDayLmit = flag.Int("symptom-days", 14, "magnitude of expected symptom onset day range")
)

func main() {
	flag.Parse()
	if *filePath == "" {
		log.Fatal("--file is required.")
	}
	if *allowedTEKAge < time.Duration(0) {
		log.Fatalf("--tek-age must be a positive duration, got: %v", *allowedTEKAge)
	}
	if *symptomDayLmit < 0 {
		log.Fatalf("--symptom-days must be >=0, got: %v", *symptomDayLmit)
	}

	blob, err := ioutil.ReadFile(*filePath)
	if err != nil {
		log.Fatalf("can't read export file: %v", err)
	}

	keyExport, err := export.UnmarshalExportFile(blob)
	if err != nil {
		log.Fatalf("error unmarshaling export file: %v", err)
	}

	// Do some basic data validation.
	success := true
	if err := checkExportFile(keyExport); err != nil {
		success = false
		if !*quiet {
			log.Printf("export file contains errors: %v", err)
		}
	}

	if *printJSON {
		prettyJSON, err := json.MarshalIndent(keyExport, "", "  ")
		if err != nil {
			log.Fatalf("error pretty printing export: %v", err)
		}
		log.Printf("Export file contents:\n%v", string(prettyJSON))
	}

	if !success {
		// return a non zero code if there are issues with the export file.
		os.Exit(1)
	}
}

func checkExportFile(export *exportpb.TemporaryExposureKeyExport) error {
	now := time.Now().UTC()
	floor := model.IntervalNumber(now.Add(*allowedTEKAge))
	ceiling := model.IntervalNumber(now)

	var errors *multierror.Error
	if err := checkKeys("keys", export.Keys, floor, ceiling); err != nil {
		errors = multierror.Append(errors, err)
	}
	if err := checkKeys("revisedKeys", export.RevisedKeys, floor, ceiling); err != nil {
		errors = multierror.Append(errors, err)
	}
	return errors
}

func checkKeys(kType string, keys []*exportpb.TemporaryExposureKey, floor, ceiling int32) error {
	symptomDays := int32(*symptomDayLmit)
	var errors *multierror.Error
	for i, k := range keys {
		if l := len(k.KeyData); l != 16 {
			errors = multierror.Append(fmt.Errorf("%s #%d: invald key length: want 16, got: %v", kType, i, l))
		}
		if s := k.GetRollingStartIntervalNumber(); s < floor {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling interval start number is > %v ago, want >= %d, got %d", kType, i, *allowedTEKAge, floor, s))
		} else if s > ceiling {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling interval start number in the future, want < %d, got %d", kType, i, ceiling, s))
		}
		if r := k.GetRollingPeriod(); r < 1 || r > 144 {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling period invalid, want >= 1 && <= 144, got %d", kType, i, r))
		}
		if k.DaysSinceOnsetOfSymptoms != nil {
			if d := k.GetDaysSinceOnsetOfSymptoms(); d < -symptomDays || d > symptomDays {
				errors = multierror.Append(fmt.Errorf("%s #%d: days_since_onset_of_symptoms is outside of expectd range, -%d..%d, got: %d", kType, i, symptomDays, symptomDays, d))
			}
		}
	}
	return errors
}
<commit_msg>add missing -1 (#889)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This tool displays the content of the export file.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/google/exposure-notifications-server/internal/export"
	exportpb "github.com/google/exposure-notifications-server/internal/pb/export"
	"github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/hashicorp/go-multierror"
)

var (
	filePath       = flag.String("file", "", "path to the export zip file.")
	printJSON      = flag.Bool("json", true, "print a JSON representation of the output")
	quiet          = flag.Bool("q", false, "run in quiet mode")
	allowedTEKAge  = flag.Duration("tek-age", 14*24*time.Hour, "max TEK age in checks")
	symptomDayLmit = flag.Int("symptom-days", 14, "magnitude of expected symptom onset day range")
)

func main() {
	flag.Parse()
	if *filePath == "" {
		log.Fatal("--file is required.")
	}
	if *allowedTEKAge < time.Duration(0) {
		log.Fatalf("--tek-age must be a positive duration, got: %v", *allowedTEKAge)
	}
	if *symptomDayLmit < 0 {
		log.Fatalf("--symptom-days must be >=0, got: %v", *symptomDayLmit)
	}

	blob, err := ioutil.ReadFile(*filePath)
	if err != nil {
		log.Fatalf("can't read export file: %v", err)
	}

	keyExport, err := export.UnmarshalExportFile(blob)
	if err != nil {
		log.Fatalf("error unmarshaling export file: %v", err)
	}

	// Do some basic data validation.
	success := true
	if err := checkExportFile(keyExport); err != nil {
		success = false
		if !*quiet {
			log.Printf("export file contains errors: %v", err)
		}
	}

	if *printJSON {
		prettyJSON, err := json.MarshalIndent(keyExport, "", "  ")
		if err != nil {
			log.Fatalf("error pretty printing export: %v", err)
		}
		log.Printf("Export file contents:\n%v", string(prettyJSON))
	}

	if !success {
		// return a non zero code if there are issues with the export file.
		os.Exit(1)
	}
}

func checkExportFile(export *exportpb.TemporaryExposureKeyExport) error {
	now := time.Now().UTC()
	floor := model.IntervalNumber(now.Add(-1 * *allowedTEKAge))
	ceiling := model.IntervalNumber(now)

	var errors *multierror.Error
	if err := checkKeys("keys", export.Keys, floor, ceiling); err != nil {
		errors = multierror.Append(errors, err)
	}
	if err := checkKeys("revisedKeys", export.RevisedKeys, floor, ceiling); err != nil {
		errors = multierror.Append(errors, err)
	}
	return errors
}

func checkKeys(kType string, keys []*exportpb.TemporaryExposureKey, floor, ceiling int32) error {
	symptomDays := int32(*symptomDayLmit)
	var errors *multierror.Error
	for i, k := range keys {
		if l := len(k.KeyData); l != 16 {
			errors = multierror.Append(fmt.Errorf("%s #%d: invald key length: want 16, got: %v", kType, i, l))
		}
		if s := k.GetRollingStartIntervalNumber(); s < floor {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling interval start number is > %v ago, want >= %d, got %d", kType, i, *allowedTEKAge, floor, s))
		} else if s > ceiling {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling interval start number in the future, want < %d, got %d", kType, i, ceiling, s))
		}
		if r := k.GetRollingPeriod(); r < 1 || r > 144 {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling period invalid, want >= 1 && <= 144, got %d", kType, i, r))
		}
		if k.DaysSinceOnsetOfSymptoms != nil {
			if d := k.GetDaysSinceOnsetOfSymptoms(); d < -symptomDays || d > symptomDays {
				errors = multierror.Append(fmt.Errorf("%s #%d: days_since_onset_of_symptoms is outside of expectd range, -%d..%d, got: %d", kType, i, symptomDays, symptomDays, d))
			}
		}
	}
	return errors
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/zlib"
	"crypto/tls"
	"encoding/binary"
	"log"
	"net"
	"time"
)

type LumberjackClient struct {
	options *LumberjackClientOptions

	conn     net.Conn
	sequence uint32
}

type LumberjackClientOptions struct {
	Network           string
	Address           string
	ConnectionTimeout time.Duration

	TLSConfig *tls.Config

	WriteTimeout time.Duration
	ReadTimeout  time.Duration
}

func NewLumberjackClient(options *LumberjackClientOptions) *LumberjackClient {
	return &LumberjackClient{
		options: options,
	}
}

func (c *LumberjackClient) ensureConnected() error {
	if c.conn == nil {
		var conn net.Conn

		if c.options.TLSConfig != nil {
			conn = tls.Client(conn, c.options.TLSConfig)
		}

		conn, err := net.DialTimeout(c.options.Network, c.options.Address, c.options.ConnectionTimeout)
		if err != nil {
			return err
		}

		c.conn = conn
	}

	return nil
}

func (c *LumberjackClient) Disconnect() error {
	var err error
	if c.conn != nil {
		err = c.conn.Close()
		c.conn = nil
	}

	c.sequence = 0
	return err
}

func (c *LumberjackClient) Send(lines []Data) error {
	err := c.ensureConnected()
	if err != nil {
		return err
	}

	// Serialize (w/ compression)
	linesBuf := c.serialize(lines)
	linesBytes := linesBuf.Bytes()

	buf := new(bytes.Buffer)

	// Window size
	buf.WriteString("1W")
	binary.Write(buf, binary.BigEndian, uint32(len(lines)))

	// Compressed size
	buf.WriteString("1C")
	log.Printf("linesBytes: %d\n", len(linesBytes))
	binary.Write(buf, binary.BigEndian, uint32(len(linesBytes)))

	// Actual lines
	buf.Write(linesBytes)

	c.conn.SetWriteDeadline(time.Now().Add(c.options.WriteTimeout))
	_, err = c.conn.Write(buf.Bytes())
	if err != nil {
		c.Disconnect()
		return err
	}

	// Wait for ACK (6 bytes)
	// This is pretty weird, but is mirroring what logstash-forwarder does
	c.conn.SetReadDeadline(time.Now().Add(c.options.ReadTimeout))

	ack := make([]byte, 6)
	ackBytes := 0
	for ackBytes < 6 {
		n, err := c.conn.Read(ack[ackBytes:len(ack)])
		if n > 0 {
			ackBytes += n
		} else if err != nil {
			c.Disconnect()
			return err
		}
	}

	return nil
}

func (c *LumberjackClient) serialize(lines []Data) *bytes.Buffer {
	buf := new(bytes.Buffer)
	compressor := zlib.NewWriter(buf)

	for _, data := range lines {
		c.sequence += 1

		compressor.Write([]byte("1D"))
		binary.Write(compressor, binary.BigEndian, uint32(c.sequence))
		binary.Write(compressor, binary.BigEndian, uint32(len(data)))
		for k, v := range data {
			binary.Write(compressor, binary.BigEndian, uint32(len(k)))
			compressor.Write([]byte(k))
			binary.Write(compressor, binary.BigEndian, uint32(len(v)))
			compressor.Write([]byte(v))
		}
	}

	compressor.Close()
	return buf
}
<commit_msg>Removes debugging statement<commit_after>package main

import (
	"bytes"
	"compress/zlib"
	"crypto/tls"
	"encoding/binary"
	"net"
	"time"
)

type LumberjackClient struct {
	options *LumberjackClientOptions

	conn     net.Conn
	sequence uint32
}

type LumberjackClientOptions struct {
	Network           string
	Address           string
	ConnectionTimeout time.Duration

	TLSConfig *tls.Config

	WriteTimeout time.Duration
	ReadTimeout  time.Duration
}

func NewLumberjackClient(options *LumberjackClientOptions) *LumberjackClient {
	return &LumberjackClient{
		options: options,
	}
}

func (c *LumberjackClient) ensureConnected() error {
	if c.conn == nil {
		var conn net.Conn

		if c.options.TLSConfig != nil {
			conn = tls.Client(conn, c.options.TLSConfig)
		}

		conn, err := net.DialTimeout(c.options.Network, c.options.Address, c.options.ConnectionTimeout)
		if err != nil {
			return err
		}

		c.conn = conn
	}

	return nil
}

func (c *LumberjackClient) Disconnect() error {
	var err error
	if c.conn != nil {
		err = c.conn.Close()
		c.conn = nil
	}

	c.sequence = 0
	return err
}

func (c *LumberjackClient) Send(lines []Data) error {
	err := c.ensureConnected()
	if err != nil {
		return err
	}

	// Serialize (w/ compression)
	linesBuf := c.serialize(lines)
	linesBytes := linesBuf.Bytes()

	buf := new(bytes.Buffer)

	// Window size
	buf.WriteString("1W")
	binary.Write(buf, binary.BigEndian, uint32(len(lines)))

	// Compressed size
	buf.WriteString("1C")
	binary.Write(buf, binary.BigEndian, uint32(len(linesBytes)))

	// Actual lines
	buf.Write(linesBytes)

	c.conn.SetWriteDeadline(time.Now().Add(c.options.WriteTimeout))
	_, err = c.conn.Write(buf.Bytes())
	if err != nil {
		c.Disconnect()
		return err
	}

	// Wait for ACK (6 bytes)
	// This is pretty weird, but is mirroring what logstash-forwarder does
	c.conn.SetReadDeadline(time.Now().Add(c.options.ReadTimeout))

	ack := make([]byte, 6)
	ackBytes := 0
	for ackBytes < 6 {
		n, err := c.conn.Read(ack[ackBytes:len(ack)])
		if n > 0 {
			ackBytes += n
		} else if err != nil {
			c.Disconnect()
			return err
		}
	}

	return nil
}

func (c *LumberjackClient) serialize(lines []Data) *bytes.Buffer {
	buf := new(bytes.Buffer)
	compressor := zlib.NewWriter(buf)

	for _, data := range lines {
		c.sequence += 1

		compressor.Write([]byte("1D"))
		binary.Write(compressor, binary.BigEndian, uint32(c.sequence))
		binary.Write(compressor, binary.BigEndian, uint32(len(data)))
		for k, v := range data {
			binary.Write(compressor, binary.BigEndian, uint32(len(k)))
			compressor.Write([]byte(k))
			binary.Write(compressor, binary.BigEndian, uint32(len(v)))
			compressor.Write([]byte(v))
		}
	}

	compressor.Close()
	return buf
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"syscall"
	"time"
)

// Waits up to a minute for all processes to die.
func waitForDeath() error {
	for i := 0; i < 30; i++ {
		pids, err := getAllProcesses()
		if err != nil {
			return err
		}
		if len(pids) == 0 {
			return nil
		}

		fmt.Fprintf(os.Stderr, "Waiting for processes to die (%d processes left)..\n", len(pids))
		time.Sleep(2 * time.Second)
	}
	return fmt.Errorf("processes did not die after a minute")
}

// KillAll processes on the system.
func KillAll() {
	fmt.Fprintf(os.Stderr, "Killing system processes..\n")
	// Try to send a sigterm
	pids, err := getAllProcesses()
	if err != nil {
		log.Println(err)
		return
	}
	for _, proc := range pids {
		proc.Signal(syscall.SIGTERM)
	}

	if err := waitForDeath(); err != nil {
		log.Println(err)
	}
	// They didn't respond to sigterm after a minute, so be mean and send a SIGKILL
	pids, _ = getAllProcesses()
	for _, proc := range pids {
		proc.Signal(syscall.SIGKILL)
	}
	if len(pids) > 0 {
		fmt.Fprintf(os.Stderr, "Sent kill signal to %d processes that didn't respond to term..\n", len(pids))
		time.Sleep(2 * time.Second)
	}

	if err := waitForDeath(); err != nil {
		log.Println(err, " :(")
	}
}

// Get a list of all processes on the system by checking /proc/*/cmdline files
func getAllProcesses() ([]*os.Process, error) {
	procs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, err
	}
	rprocs := make([]*os.Process, 0)
	for _, f := range procs {
		if !f.IsDir() {
			continue
		}
		pid, err := strconv.Atoi(f.Name())
		if err != nil {
			// The directory wasn't an integer, so it wasn't a pid.
			continue
		}
		if pid < 2 {
			// don't include the init system in the procs that get killed.
			continue
		}
		cmdline := fmt.Sprintf("/proc/%d/cmdline", pid)
		if _, err := os.Stat(cmdline); os.IsNotExist(err) {
			// There was no command line, it's not a process to kill
			continue
		}
		contents, err := ioutil.ReadFile(cmdline)
		if len(contents) == 0 {
			// the cmdline file was empty, it's not a real command
			continue
		}
		proc, err := os.FindProcess(pid)
		if err != nil {
			log.Println(err)
			continue
		}
		rprocs = append(rprocs, proc)
	}
	return rprocs, nil
}
<commit_msg>When shutdown print processes names to waiting message<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"syscall"
	"time"
	"github.com/mitchellh/go-ps"
)

// Waits up to a minute for all processes to die.
func waitForDeath() error {
	for i := 0; i < 30; i++ {
		pids, err := getAllProcesses()
		if err != nil {
			return err
		}
		if len(pids) == 0 {
			return nil
		}

		fmt.Fprintf(os.Stderr, "Waiting for processes to die (%d processes left)..\n", len(pids))

		for _, pid := range pids {
			p, err := ps.FindProcess(pid.Pid)
			if err != nil {
				return err
			}
			fmt.Fprintf(os.Stderr, "PID %d with name %s is still alive.\n", p.Pid(), p.Executable())
		}

		time.Sleep(2 * time.Second)
	}
	return fmt.Errorf("processes did not die after a minute")
}

// KillAll processes on the system.
func KillAll() {
	fmt.Fprintf(os.Stderr, "Killing system processes..\n")
	// Try to send a sigterm
	pids, err := getAllProcesses()
	if err != nil {
		log.Println(err)
		return
	}
	for _, proc := range pids {
		proc.Signal(syscall.SIGTERM)
	}

	if err := waitForDeath(); err != nil {
		log.Println(err)
	}
	// They didn't respond to sigterm after a minute, so be mean and send a SIGKILL
	pids, _ = getAllProcesses()
	for _, proc := range pids {
		proc.Signal(syscall.SIGKILL)
	}
	if len(pids) > 0 {
		fmt.Fprintf(os.Stderr, "Sent kill signal to %d processes that didn't respond to term..\n", len(pids))
		time.Sleep(2 * time.Second)
	}

	if err := waitForDeath(); err != nil {
		log.Println(err, " :(")
	}
}

// Get a list of all processes on the system by checking /proc/*/cmdline files
func getAllProcesses() ([]*os.Process, error) {
	procs, err := ioutil.ReadDir("/proc")
	if err != nil {
		return nil, err
	}
	rprocs := make([]*os.Process, 0)
	for _, f := range procs {
		if !f.IsDir() {
			continue
		}
		pid, err := strconv.Atoi(f.Name())
		if err != nil {
			// The directory wasn't an integer, so it wasn't a pid.
			continue
		}
		if pid < 2 {
			// don't include the init system in the procs that get killed.
			continue
		}
		cmdline := fmt.Sprintf("/proc/%d/cmdline", pid)
		if _, err := os.Stat(cmdline); os.IsNotExist(err) {
			// There was no command line, it's not a process to kill
			continue
		}
		contents, err := ioutil.ReadFile(cmdline)
		if len(contents) == 0 {
			// the cmdline file was empty, it's not a real command
			continue
		}
		proc, err := os.FindProcess(pid)
		if err != nil {
			log.Println(err)
			continue
		}
		rprocs = append(rprocs, proc)
	}
	return rprocs, nil
}
<|endoftext|>


<commit_before>package events

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/pborman/uuid"

	"github.com/gorilla/websocket"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// EventSource indicates the source of an event.
type EventSource uint8

// EventSourceLocal indicates the event was generated locally.
const EventSourceLocal = 0

// EventSourcePull indicates the event was received from an outbound event listener stream.
const EventSourcePull = 1

// EventSourcePush indicates the event was received from an event listener client connected to us.
const EventSourcePush = 2

// Server represents an instance of an event server.
type Server struct {
	serverCommon

	listeners map[string]*Listener
}

// NewServer returns a new event server.
func NewServer(debug bool, verbose bool) *Server {
	server := &Server{
		serverCommon: serverCommon{
			debug:   debug,
			verbose: verbose,
		},
		listeners: map[string]*Listener{},
	}

	return server
}

// AddListener creates and returns a new event listener.
func (s *Server) AddListener(projectName string, allProjects bool, connection *websocket.Conn, messageTypes []string, location string, excludeSources []EventSource, recvFunc EventHandler) (*Listener, error) {
	if allProjects && projectName != "" {
		return nil, fmt.Errorf("Cannot specify project name when listening for events on all projects")
	}

	ctx, ctxCancel := context.WithCancel(context.Background())

	listener := &Listener{
		listenerCommon: listenerCommon{
			Conn:         connection,
			messageTypes: messageTypes,
			ctx:          ctx,
			ctxCancel:    ctxCancel,
			id:           uuid.New(),
			recvFunc:     recvFunc,
		},

		location:       location,
		allProjects:    allProjects,
		projectName:    projectName,
		excludeSources: excludeSources,
	}

	s.lock.Lock()
	defer s.lock.Unlock()

	if s.listeners[listener.id] != nil {
		return nil, fmt.Errorf("A listener with ID %q already exists", listener.id)
	}

	s.listeners[listener.id] = listener

	go listener.heartbeat()

	return listener, nil
}

// SendLifecycle broadcasts a lifecycle event.
func (s *Server) SendLifecycle(projectName string, event api.EventLifecycle) {
	s.Send(projectName, "lifecycle", event)
}

// Send broadcasts a custom event.
func (s *Server) Send(projectName string, eventType string, eventMessage interface{}) error {
	encodedMessage, err := json.Marshal(eventMessage)
	if err != nil {
		return err
	}
	event := api.Event{
		Type:      eventType,
		Timestamp: time.Now(),
		Metadata:  encodedMessage,
		Project:   projectName,
	}

	return s.broadcast(event, EventSourceLocal)
}

// Forward to the local events dispatcher an event received from another node.
func (s *Server) Forward(id int64, event api.Event) {
	if event.Type == "logging" {
		// Parse the message
		logEntry := api.EventLogging{}
		err := json.Unmarshal(event.Metadata, &logEntry)
		if err != nil {
			return
		}

		if !s.debug && logEntry.Level == "dbug" {
			return
		}

		if !s.debug && !s.verbose && logEntry.Level == "info" {
			return
		}
	}

	err := s.broadcast(event, EventSourcePull)
	if err != nil {
		logger.Warnf("Failed to forward event from member %d: %v", id, err)
	}
}

func (s *Server) broadcast(event api.Event, eventSource EventSource) error {
	sourceInSlice := func(source EventSource, sources []EventSource) bool {
		for _, i := range sources {
			if source == i {
				return true
			}
		}

		return false
	}

	s.lock.Lock()
	listeners := s.listeners
	for _, listener := range listeners {
		// If the event is project specific, check if the listener is requesting events from that project.
		if event.Project != "" && !listener.allProjects && event.Project != listener.projectName {
			continue
		}

		if sourceInSlice(eventSource, listener.excludeSources) {
			continue
		}

		if !shared.StringInSlice(event.Type, listener.messageTypes) {
			continue
		}

		go func(listener *Listener, event api.Event) {
			// Check that the listener still exists
			if listener == nil {
				return
			}

			// Make sure we're not done already
			if listener.IsClosed() {
				return
			}

			// Set the Location to the expected serverName
			if event.Location == "" {
				eventCopy := api.Event{}
				err := shared.DeepCopy(&event, &eventCopy)
				if err != nil {
					return
				}
				eventCopy.Location = listener.location

				event = eventCopy
			}

			listener.SetWriteDeadline(time.Now().Add(5 * time.Second))
			err := listener.WriteJSON(event)
			if err != nil {
				// Remove the listener from the list
				s.lock.Lock()
				delete(s.listeners, listener.id)
				s.lock.Unlock()

				listener.Close()
			}
		}(listener, event)
	}
	s.lock.Unlock()

	return nil
}

// Listener describes an event listener.
type Listener struct {
	listenerCommon

	location       string
	allProjects    bool
	projectName    string
	excludeSources []EventSource
}
<commit_msg>lxd/events/events: Removes listener level location concept and replaces with server location concept<commit_after>package events

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/pborman/uuid"

	"github.com/gorilla/websocket"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// EventSource indicates the source of an event.
type EventSource uint8

// EventSourceLocal indicates the event was generated locally.
const EventSourceLocal = 0

// EventSourcePull indicates the event was received from an outbound event listener stream.
const EventSourcePull = 1

// EventSourcePush indicates the event was received from an event listener client connected to us.
const EventSourcePush = 2

// Server represents an instance of an event server.
type Server struct {
	serverCommon

	listeners map[string]*Listener
	location  string
}

// NewServer returns a new event server.
func NewServer(debug bool, verbose bool) *Server {
	server := &Server{
		serverCommon: serverCommon{
			debug:   debug,
			verbose: verbose,
		},
		listeners: map[string]*Listener{},
	}

	return server
}

// SetLocalLocation sets the local location of this member.
// This value will be added to the Location event field if not populated from another member.
func (s *Server) SetLocalLocation(location string) {
	s.lock.Lock()
	defer s.lock.Unlock()

	s.location = location
}

// AddListener creates and returns a new event listener.
func (s *Server) AddListener(projectName string, allProjects bool, connection *websocket.Conn, messageTypes []string, excludeSources []EventSource, recvFunc EventHandler) (*Listener, error) {
	if allProjects && projectName != "" {
		return nil, fmt.Errorf("Cannot specify project name when listening for events on all projects")
	}

	ctx, ctxCancel := context.WithCancel(context.Background())

	listener := &Listener{
		listenerCommon: listenerCommon{
			Conn:         connection,
			messageTypes: messageTypes,
			ctx:          ctx,
			ctxCancel:    ctxCancel,
			id:           uuid.New(),
			recvFunc:     recvFunc,
		},

		allProjects:    allProjects,
		projectName:    projectName,
		excludeSources: excludeSources,
	}

	s.lock.Lock()
	defer s.lock.Unlock()

	if s.listeners[listener.id] != nil {
		return nil, fmt.Errorf("A listener with ID %q already exists", listener.id)
	}

	s.listeners[listener.id] = listener

	go listener.heartbeat()

	return listener, nil
}

// SendLifecycle broadcasts a lifecycle event.
func (s *Server) SendLifecycle(projectName string, event api.EventLifecycle) {
	s.Send(projectName, "lifecycle", event)
}

// Send broadcasts a custom event.
func (s *Server) Send(projectName string, eventType string, eventMessage interface{}) error {
	encodedMessage, err := json.Marshal(eventMessage)
	if err != nil {
		return err
	}
	event := api.Event{
		Type:      eventType,
		Timestamp: time.Now(),
		Metadata:  encodedMessage,
		Project:   projectName,
	}

	return s.broadcast(event, EventSourceLocal)
}

// Forward to the local events dispatcher an event received from another node.
func (s *Server) Forward(id int64, event api.Event) {
	if event.Type == "logging" {
		// Parse the message
		logEntry := api.EventLogging{}
		err := json.Unmarshal(event.Metadata, &logEntry)
		if err != nil {
			return
		}

		if !s.debug && logEntry.Level == "dbug" {
			return
		}

		if !s.debug && !s.verbose && logEntry.Level == "info" {
			return
		}
	}

	err := s.broadcast(event, EventSourcePull)
	if err != nil {
		logger.Warnf("Failed to forward event from member %d: %v", id, err)
	}
}

func (s *Server) broadcast(event api.Event, eventSource EventSource) error {
	sourceInSlice := func(source EventSource, sources []EventSource) bool {
		for _, i := range sources {
			if source == i {
				return true
			}
		}

		return false
	}

	s.lock.Lock()

	// Set the Location for local events to the local serverName if not already populated (do it here rather
	// than in Send as the lock to read s.location has been taken here already).
	if eventSource == EventSourceLocal && event.Location == "" {
		event.Location = s.location
	}

	listeners := s.listeners
	for _, listener := range listeners {
		// If the event is project specific, check if the listener is requesting events from that project.
		if event.Project != "" && !listener.allProjects && event.Project != listener.projectName {
			continue
		}

		if sourceInSlice(eventSource, listener.excludeSources) {
			continue
		}

		if !shared.StringInSlice(event.Type, listener.messageTypes) {
			continue
		}

		go func(listener *Listener, event api.Event) {
			// Check that the listener still exists
			if listener == nil {
				return
			}

			// Make sure we're not done already
			if listener.IsClosed() {
				return
			}

			listener.SetWriteDeadline(time.Now().Add(5 * time.Second))
			err := listener.WriteJSON(event)
			if err != nil {
				// Remove the listener from the list
				s.lock.Lock()
				delete(s.listeners, listener.id)
				s.lock.Unlock()

				listener.Close()
			}
		}(listener, event)
	}
	s.lock.Unlock()

	return nil
}

// Listener describes an event listener.
type Listener struct {
	listenerCommon

	location       string
	allProjects    bool
	projectName    string
	excludeSources []EventSource
}
<|endoftext|>


<commit_before>package mailserver

// This file describes two routes that are used throughout the Melange System.

import (
	"airdispat.ch/identity"
	"airdispat.ch/routing"
	"airdispat.ch/tracker"
	"errors"
	"strings"
)

var ServerKey *identity.Identity

func InitRouter() {
	if ServerKey == nil {
		ServerKey, _ = identity.CreateIdentity()
	}
	if RegistrationRouter == nil {
		RegistrationRouter = &tracker.TrackerRouter{
			tracker.GetTrackingServerLocationFromURL("airdispat.ch"),
			ServerKey,
		}
	}
	if LookupRouter == nil {
		LookupRouter = &Router{
			Origin:      ServerKey,
			TrackerList: []string{"mailserver.airdispat.ch:5000"},
		}
	}
}

var RegistrationRouter routing.Router
var LookupRouter routing.Router

type Router struct {
	Origin      *identity.Identity
	TrackerList []string
}

func (a *Router) LookupAlias(from string) (*identity.Address, error) {
	if from[0] == '/' {
		return a.Lookup(from[1:])
	}
	comp := strings.Split(from, "@")
	if len(comp) != 2 {
		return nil, errors.New("Can't use lookup router without tracker address.")
	}

	url := tracker.GetTrackingServerLocationFromURL(comp[1])
	t := &tracker.TrackerRouter{url, a.Origin}

	return t.LookupAlias(comp[0])
}

func (a *Router) Lookup(from string) (*identity.Address, error) {
	t := tracker.CreateTrackerListRouterWithStrings(a.Origin, a.TrackerList...)
	return t.Lookup(from)
}

func (a *Router) Register(key *identity.Identity, alias string) error {
	return errors.New("Can't use LookupRouter for registration.")
}
<commit_msg>Added Routing Caching<commit_after>package mailserver

// This file describes two routes that are used throughout the Melange System.

import (
	"airdispat.ch/identity"
	"airdispat.ch/routing"
	"airdispat.ch/tracker"
	"errors"
	"strings"
	"sync"
)

var cache map[string]*identity.Address
var cLock sync.RWMutex

var ServerKey *identity.Identity

func InitRouter() {
	if cache == nil {
		cache = make(map[string]*identity.Address)
	}
	if ServerKey == nil {
		ServerKey, _ = identity.CreateIdentity()
	}
	if RegistrationRouter == nil {
		RegistrationRouter = &tracker.TrackerRouter{
			tracker.GetTrackingServerLocationFromURL("airdispat.ch"),
			ServerKey,
		}
	}
	if LookupRouter == nil {
		LookupRouter = &Router{
			Origin:      ServerKey,
			TrackerList: []string{"mailserver.airdispat.ch:5000"},
		}
	}
}

var RegistrationRouter routing.Router
var LookupRouter routing.Router

type Router struct {
	Origin      *identity.Identity
	TrackerList []string
}

func (a *Router) LookupAlias(from string) (*identity.Address, error) {
	cLock.RLock()
	test, ok := cache[from]
	if ok {
		return test, nil
	}
	cLock.RUnlock()

	if from[0] == '/' {
		return a.Lookup(from[1:])
	}
	comp := strings.Split(from, "@")
	if len(comp) != 2 {
		return nil, errors.New("Can't use lookup router without tracker address.")
	}

	url := tracker.GetTrackingServerLocationFromURL(comp[1])
	t := &tracker.TrackerRouter{url, a.Origin}

	addr, err := t.LookupAlias(comp[0])
	if err == nil {
		cLock.Lock()
		cache[from] = addr
		cache[addr.String()] = addr
		cLock.Unlock()
		return addr, nil
	}
	return nil, err
}

func (a *Router) Lookup(from string) (*identity.Address, error) {
	cLock.RLock()
	test, ok := cache[from]
	if ok {
		return test, nil
	}
	cLock.RUnlock()

	for _, v := range a.TrackerList {
		a, err := (&tracker.TrackerRouter{v, ServerKey}).Lookup(from)
		if err == nil {
			cLock.Lock()
			cache[from] = a
			cache[a.String()] = a
			cLock.Unlock()
			return a, nil
		}
	}
	return nil, errors.New("Couldn't find address in Trackers.")
}

func (a *Router) Register(key *identity.Identity, alias string) error {
	return errors.New("Can't use LookupRouter for registration.")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func testableRemover(s *[]string) func(string) error {
	return func(p string) error {
		*s = append(*s, p)
		return nil
	}
}

type testableService struct {
	Uninstalled bool
}

func (u *testableService) Uninstall() error {
	u.Uninstalled = true
	return nil
}

func (u *testableService) Stop() error {
	return nil
}

func TestRemoveKiteKey(t *testing.T) {
	Convey("Given the KiteKeyDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KiteKeyFilename: "bar.key",
				remover:         testableRemover(&removed),
			}
			So(u.RemoveKiteKey(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KiteKeyFilename is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KiteKeyDirectory: "foo",
				remover:          testableRemover(&removed),
			}
			So(u.RemoveKiteKey(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given that everything is configured properly", t, func() {
		Convey("Then remove the expected files", func() {
			var removed []string
			u := &Uninstall{
				KiteKeyDirectory: "foo",
				KiteKeyFilename:  "bar.key",
				remover:          testableRemover(&removed),
			}
			So(u.RemoveKiteKey(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo/bar.key",
				"foo",
			})
		})
	})
}

func TestRemoveKlientFiles(t *testing.T) {
	Convey("Given the KlientDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientFilename:        "baz",
				KlientshFilename:      "baz.sh",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientParentDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientDirectory:  "foo",
				KlientFilename:   "baz",
				KlientshFilename: "baz.sh",
				remover:          testableRemover(&removed),
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientFilename is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar",
				KlientshFilename:      "baz.sh",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientshFilename is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar",
				KlientFilename:        "baz",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given that everything is configured properly", t, func() {
		Convey("Then remove the klient files", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar",
				KlientFilename:        "baz",
				KlientshFilename:      "baz.sh",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientFiles(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo/bar/baz",
				"foo/bar/baz.sh",
			})
		})
	})
}

func TestRemoveKlientDirectories(t *testing.T) {
	Convey("Given the KlientDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientDirectories(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientParentDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientDirectory: "foo",
				remover:         testableRemover(&removed),
			}
			So(u.RemoveKlientDirectories(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientDirectory is absolute", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "/bar",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientDirectories(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given everything is configured properly", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar/baz",
				remover:               testableRemover(&removed),
			}
			So(u.RemoveKlientDirectories(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo/bar/baz",
				"foo/bar",
			})
		})

		Convey("And a directory cannot be removed", func() {
			Convey("Then return an error", func() {
				var removed []string
				u := &Uninstall{
					KlientParentDirectory: "foo",
					KlientDirectory:       "bar/baz/boop",
					remover: func(p string) error {
						// error on foo/bar to simulate a failure
						if p == "foo/bar" {
							return errors.New("Testing failure, cannot remove foo/bar")
						}
						removed = append(removed, p)
						return nil
					},
				}
				So(u.RemoveKlientDirectories(), ShouldNotBeNil)
				So(removed, ShouldResemble, []string{
					"foo/bar/baz/boop",
					"foo/bar/baz",
				})
			})
		})
	})
}

func TestRemoveKlientctl(t *testing.T) {
	Convey("Given the KlientctlPath is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				remover: testableRemover(&removed),
			}
			So(u.RemoveKlientctl(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given that the KlientctlPath is not empty", t, func() {
		Convey("Then remove the give path", func() {
			var removed []string
			u := &Uninstall{
				KlientctlPath: "foo",
				remover:       testableRemover(&removed),
			}
			So(u.RemoveKlientctl(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo",
			})
		})
	})
}

func TestUninstall(t *testing.T) {
	Convey("Given the it is configured properly", t, func() {
		var (
			removed []string
			service = &testableService{}
		)
		u := &Uninstall{
			remover:               testableRemover(&removed),
			ServiceUninstaller:    service,
			KiteKeyDirectory:      "/etc/kite",
			KiteKeyFilename:       "kite.key",
			KlientctlPath:         "/usr/local/bin/kd",
			KlientParentDirectory: "/opt",
			KlientDirectory:       "kite/klient",
			KlientFilename:        "klient",
			KlientshFilename:      "klient.sh",
		}

		Convey("Then remove the expected files", func() {
			u.Uninstall()
			So(service.Uninstalled, ShouldBeTrue)

			// This is a bit brittle due to execution order, but most of the order
			// is important since files need to go before dirs, and etc. Therefor
			// some brittleness is acceptable, i think.
			So(removed, ShouldResemble, []string{
				"/etc/kite/kite.key",
				"/etc/kite",
				"/opt/kite/klient/klient",
				"/opt/kite/klient/klient.sh",
				"/opt/kite/klient",
				"/opt/kite",
				"/usr/local/bin/kd",
			})
		})
	})
}
<commit_msg>kd: remove kite.key file-related tests<commit_after>package main

import (
	"errors"
	"testing"

	"github.com/koding/logging"
	. "github.com/smartystreets/goconvey/convey"
)

func testableRemover(s *[]string) func(string) error {
	return func(p string) error {
		*s = append(*s, p)
		return nil
	}
}

type testableService struct {
	Uninstalled bool
}

func (u *testableService) Uninstall() error {
	u.Uninstalled = true
	return nil
}

func (u *testableService) Stop() error {
	return nil
}

var testLog = logging.NewCustom("test-uninstall", true)

func TestRemoveKlientFiles(t *testing.T) {
	Convey("Given the KlientDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientFilename:        "baz",
				KlientshFilename:      "baz.sh",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientParentDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientDirectory:  "foo",
				KlientFilename:   "baz",
				KlientshFilename: "baz.sh",
				remover:          testableRemover(&removed),
				log:              testLog,
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientFilename is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar",
				KlientshFilename:      "baz.sh",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientshFilename is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar",
				KlientFilename:        "baz",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientFiles(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given that everything is configured properly", t, func() {
		Convey("Then remove the klient files", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar",
				KlientFilename:        "baz",
				KlientshFilename:      "baz.sh",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientFiles(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo/bar/baz",
				"foo/bar/baz.sh",
			})
		})
	})
}

func TestRemoveKlientDirectories(t *testing.T) {
	Convey("Given the KlientDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientDirectories(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientParentDirectory is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientDirectory: "foo",
				remover:         testableRemover(&removed),
				log:             testLog,
			}
			So(u.RemoveKlientDirectories(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given the KlientDirectory is absolute", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "/bar",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientDirectories(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given everything is configured properly", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				KlientParentDirectory: "foo",
				KlientDirectory:       "bar/baz",
				remover:               testableRemover(&removed),
				log:                   testLog,
			}
			So(u.RemoveKlientDirectories(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo/bar/baz",
				"foo/bar",
			})
		})

		Convey("And a directory cannot be removed", func() {
			Convey("Then return an error", func() {
				var removed []string
				u := &Uninstall{
					KlientParentDirectory: "foo",
					KlientDirectory:       "bar/baz/boop",
					remover: func(p string) error {
						// error on foo/bar to simulate a failure
						if p == "foo/bar" {
							return errors.New("Testing failure, cannot remove foo/bar")
						}
						removed = append(removed, p)
						return nil
					},
					log: testLog,
				}
				So(u.RemoveKlientDirectories(), ShouldNotBeNil)
				So(removed, ShouldResemble, []string{
					"foo/bar/baz/boop",
					"foo/bar/baz",
				})
			})
		})
	})
}

func TestRemoveKlientctl(t *testing.T) {
	Convey("Given the KlientctlPath is empty", t, func() {
		Convey("Then return an error", func() {
			var removed []string
			u := &Uninstall{
				remover: testableRemover(&removed),
				log:     testLog,
			}
			So(u.RemoveKlientctl(), ShouldNotBeNil)
			So(removed, ShouldBeNil)
		})
	})

	Convey("Given that the KlientctlPath is not empty", t, func() {
		Convey("Then remove the give path", func() {
			var removed []string
			u := &Uninstall{
				KlientctlPath: "foo",
				remover:       testableRemover(&removed),
				log:           testLog,
			}
			So(u.RemoveKlientctl(), ShouldBeNil)
			So(removed, ShouldResemble, []string{
				"foo",
			})
		})
	})
}

func TestUninstall(t *testing.T) {
	Convey("Given the it is configured properly", t, func() {
		var (
			removed []string
			service = &testableService{}
		)
		u := &Uninstall{
			ServiceUninstaller:    service,
			KlientctlPath:         "/usr/local/bin/kd",
			KlientParentDirectory: "/opt",
			KlientDirectory:       "kite/klient",
			KlientFilename:        "klient",
			KlientshFilename:      "klient.sh",
			remover:               testableRemover(&removed),
			log:                   testLog,
		}

		Convey("Then remove the expected files", func() {
			u.Uninstall()
			So(service.Uninstalled, ShouldBeTrue)

			// This is a bit brittle due to execution order, but most of the order
			// is important since files need to go before dirs, and etc. Therefor
			// some brittleness is acceptable, i think.
			So(removed, ShouldResemble, []string{
				"/opt/kite/klient/klient",
				"/opt/kite/klient/klient.sh",
				"/opt/kite/klient",
				"/opt/kite",
				"/usr/local/bin/kd",
			})
		})
	})
}
<|endoftext|>