<commit_before>package hammer

import (
	"errors"
	"github.com/Sirupsen/logrus"
	shlex "github.com/anmitsu/go-shlex"
	"io/ioutil"
	"os/exec"
	"path"
)

var (
	// ErrFieldRequired is returned when a required field is not set.
	ErrFieldRequired = errors.New("field is required")

	// ErrInvalidScriptName is returned when a bad script name is set and passed
	// to FPM.
	ErrInvalidScriptName = errors.New("invalid script name")
)

// FPM is a wrapper around the Ruby FPM tool, and will call it in a subprocess.
type FPM struct {
	Package *Package

	baseOpts []string
	baseArgs []string
}

// NewFPM does all necessary setup to run an FPM instance
func NewFPM(p *Package) (*FPM, error) {
	fpm := &FPM{
		Package: p,
	}

	err := fpm.setBaseArgs()
	if err != nil {
		return nil, err
	}

	err = fpm.setBaseOpts()
	if err != nil {
		return nil, err
	}

	return fpm, nil
}

// PackageFor runs the packaging process on a given out type ("rpm", for
// instance). It returns a string of the command combined output.
func (f *FPM) PackageFor(outType string) (string, error) {
	// put args and opts all together
	extra, err := f.extraArgs()
	if err != nil {
		return "", err
	}

	arguments := []string{}
	arguments = append(arguments, f.baseOpts...)
	arguments = append(arguments, f.optsForType(outType)...)
	arguments = append(arguments, extra...)
	arguments = append(arguments, f.baseArgs...)

	f.Package.logger.WithField("args", arguments).Debug("running FPM with args")
	fpm := exec.Command("fpm", arguments...)
	out, err := fpm.CombinedOutput()

	if fpm.ProcessState != nil {
		f.Package.logger.WithFields(logrus.Fields{
			"systemTime": fpm.ProcessState.SystemTime(),
			"userTime":   fpm.ProcessState.UserTime(),
			"success":    fpm.ProcessState.Success(),
		}).Debug("package command exited")
	} else {
		f.Package.logger.Debug("package command exited")
	}

	return string(out), err
}

func (f *FPM) setBaseArgs() error {
	args := []string{}

	p := f.Package

	// targets
	for i, target := range p.Targets {
		srcBuf, err := p.template.Render(target.Src)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target source name")
			return err
		}
		src := srcBuf.String()

		dest, err := p.template.Render(target.Dest)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target destination")
			return err
		}

		// opt-in templating. We don't want to template *every* file because some
		// things look like Go templates and aren't (see for example every other
		// kind of mustache template)
		if !target.Template {
			args = append(args, src+"="+dest.String())
		} else {
			var content []byte
			rawContent, err := ioutil.ReadFile(src)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error reading content")
				return err
			}

			contentBuf, err := p.template.Render(string(rawContent))
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error templating content")
				return err
			}
			content = contentBuf.Bytes()

			_, name := path.Split(src)
			contentDest := path.Join(p.TargetRoot, name)
			err = ioutil.WriteFile(contentDest, content, 0777)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error writing content")
				return err
			}

			args = append(args, contentDest+"="+dest.String())
		}
	}

	f.baseArgs = args
	return nil
}

func (f *FPM) setBaseOpts() error {
	opts := []string{
		"-s", "dir",
		"-p", f.Package.OutputRoot,
	}

	type Source func() ([]string, error)
	fieldSources := []Source{
		f.baseFields,
		f.baseDependencies,
		f.baseScripts,
		f.baseConfigs,
	}

	for _, source := range fieldSources {
		newOpts, err := source()
		if err != nil {
			return err
		}
		opts = append(opts, newOpts...)
	}

	f.baseOpts = opts
	return nil
}

func (f *FPM) baseFields() ([]string, error) {
	opts := []string{}

	type field struct {
		Name     string
		Value    string
		Required bool
	}
	fields := []field{
		{"name", f.Package.Name, true},
		{"version", f.Package.Version, true},
		{"iteration", f.Package.Iteration, true},
		{"epoch", f.Package.Epoch, false},
		{"license", f.Package.License, false},
		{"vendor", f.Package.Vendor, false},
		{"description", f.Package.Description, false},
		{"url", f.Package.URL, false},
		{"architecture", f.Package.Architecture, false},
	}

	for _, field := range fields {
		if field.Value == "" {
			if !field.Required {
				continue
			} else {
				f.Package.logger.WithField("field", field.Name).Error(ErrFieldRequired)
				return opts, ErrFieldRequired
			}
		}

		templated, err := f.Package.template.Render(field.Value)
		if err != nil {
			f.Package.logger.WithFields(logrus.Fields{
				"field": field.Name,
				"error": err,
			}).Error("failed to render field as template")
			return opts, err
		}

		opts = append(opts, "--"+field.Name, templated.String())
	}

	return opts, nil
}

func (f *FPM) baseDependencies() ([]string, error) {
	opts := []string{}

	for _, rawDepend := range f.Package.Depends {
		depend, err := f.Package.template.Render(rawDepend)
		if err != nil {
			f.Package.logger.WithFields(logrus.Fields{
				"error": err,
				"raw":   rawDepend,
			}).Error("failed to render dependency as template")
			return opts, err
		}
		opts = append(opts, "--depends", depend.String())
	}

	return opts, nil
}

func (f *FPM) baseScripts() ([]string, error) {
	opts := []string{}

	for name, location := range f.Package.scriptLocations {
		if name == "build" {
			continue
		}

		if name != "before-install" && name != "after-install" && name != "before-remove" && name != "after-remove" && name != "before-upgrade" && name != "after-upgrade" {
			f.Package.logger.WithFields(logrus.Fields{
				"script": name,
			}).Error(ErrInvalidScriptName)
			return opts, ErrInvalidScriptName
		}

		opts = append(opts, "--"+name, location)
	}

	return opts, nil
}

func (f *FPM) baseConfigs() ([]string, error) {
	opts := []string{}

	for i, target := range f.Package.Targets {
		if !target.Config {
			continue
		}

		dest, err := f.Package.template.Render(target.Dest)
		if err != nil {
			f.Package.logger.WithField("index", i).Error("error templating target destination")
			return opts, err
		}

		opts = append(opts, "--config-files", dest.String())
	}

	return opts, nil
}

func (f *FPM) optsForType(t string) []string {
	opts := []string{
		"-t", t,
	}

	return opts
}

func (f *FPM) extraArgs() ([]string, error) {
	extra, err := shlex.Split(f.Package.ExtraArgs, true)
	if err != nil {
		f.Package.logger.WithField("error", err).Error("failed to parse extra args")
	}
	return extra, err
}
<commit_msg>Use `-s empty` when there are no targets in the spec file.<commit_after>package hammer

import (
	"errors"
	"github.com/Sirupsen/logrus"
	shlex "github.com/anmitsu/go-shlex"
	"io/ioutil"
	"os/exec"
	"path"
)

var (
	// ErrFieldRequired is returned when a required field is not set.
	ErrFieldRequired = errors.New("field is required")

	// ErrInvalidScriptName is returned when a bad script name is set and passed
	// to FPM.
	ErrInvalidScriptName = errors.New("invalid script name")
)

// FPM is a wrapper around the Ruby FPM tool, and will call it in a subprocess.
type FPM struct {
	Package *Package

	baseOpts []string
	baseArgs []string
}

// NewFPM does all necessary setup to run an FPM instance
func NewFPM(p *Package) (*FPM, error) {
	fpm := &FPM{
		Package: p,
	}

	err := fpm.setBaseArgs()
	if err != nil {
		return nil, err
	}

	err = fpm.setBaseOpts()
	if err != nil {
		return nil, err
	}

	return fpm, nil
}

// PackageFor runs the packaging process on a given out type ("rpm", for
// instance). It returns a string of the command combined output.
func (f *FPM) PackageFor(outType string) (string, error) {
	// put args and opts all together
	extra, err := f.extraArgs()
	if err != nil {
		return "", err
	}

	arguments := []string{}
	arguments = append(arguments, f.baseOpts...)
	arguments = append(arguments, f.optsForType(outType)...)
	arguments = append(arguments, extra...)
	arguments = append(arguments, f.baseArgs...)

	f.Package.logger.WithField("args", arguments).Debug("running FPM with args")
	fpm := exec.Command("fpm", arguments...)
	out, err := fpm.CombinedOutput()

	if fpm.ProcessState != nil {
		f.Package.logger.WithFields(logrus.Fields{
			"systemTime": fpm.ProcessState.SystemTime(),
			"userTime":   fpm.ProcessState.UserTime(),
			"success":    fpm.ProcessState.Success(),
		}).Debug("package command exited")
	} else {
		f.Package.logger.Debug("package command exited")
	}

	return string(out), err
}

func (f *FPM) setBaseArgs() error {
	args := []string{}

	p := f.Package

	// targets
	for i, target := range p.Targets {
		srcBuf, err := p.template.Render(target.Src)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target source name")
			return err
		}
		src := srcBuf.String()

		dest, err := p.template.Render(target.Dest)
		if err != nil {
			p.logger.WithField("index", i).Error("error templating target destination")
			return err
		}

		// opt-in templating. We don't want to template *every* file because some
		// things look like Go templates and aren't (see for example every other
		// kind of mustache template)
		if !target.Template {
			args = append(args, src+"="+dest.String())
		} else {
			var content []byte
			rawContent, err := ioutil.ReadFile(src)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error reading content")
				return err
			}

			contentBuf, err := p.template.Render(string(rawContent))
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error templating content")
				return err
			}
			content = contentBuf.Bytes()

			_, name := path.Split(src)
			contentDest := path.Join(p.TargetRoot, name)
			err = ioutil.WriteFile(contentDest, content, 0777)
			if err != nil {
				p.logger.WithFields(logrus.Fields{
					"name":  src,
					"error": err,
				}).Error("error writing content")
				return err
			}

			args = append(args, contentDest+"="+dest.String())
		}
	}

	f.baseArgs = args
	return nil
}

func (f *FPM) setBaseOpts() error {
	var opts []string

	if len(f.Package.Targets) == 0 {
		opts = []string{
			"-s", "empty",
			"-p", f.Package.OutputRoot,
		}
	} else {
		opts = []string{
			"-s", "dir",
			"-p", f.Package.OutputRoot,
		}
	}

	type Source func() ([]string, error)
	fieldSources := []Source{
		f.baseFields,
		f.baseDependencies,
		f.baseScripts,
		f.baseConfigs,
	}

	for _, source := range fieldSources {
		newOpts, err := source()
		if err != nil {
			return err
		}
		opts = append(opts, newOpts...)
	}

	f.baseOpts = opts
	return nil
}

func (f *FPM) baseFields() ([]string, error) {
	opts := []string{}

	type field struct {
		Name     string
		Value    string
		Required bool
	}
	fields := []field{
		{"name", f.Package.Name, true},
		{"version", f.Package.Version, true},
		{"iteration", f.Package.Iteration, true},
		{"epoch", f.Package.Epoch, false},
		{"license", f.Package.License, false},
		{"vendor", f.Package.Vendor, false},
		{"description", f.Package.Description, false},
		{"url", f.Package.URL, false},
		{"architecture", f.Package.Architecture, false},
	}

	for _, field := range fields {
		if field.Value == "" {
			if !field.Required {
				continue
			} else {
				f.Package.logger.WithField("field", field.Name).Error(ErrFieldRequired)
				return opts, ErrFieldRequired
			}
		}

		templated, err := f.Package.template.Render(field.Value)
		if err != nil {
			f.Package.logger.WithFields(logrus.Fields{
				"field": field.Name,
				"error": err,
			}).Error("failed to render field as template")
			return opts, err
		}

		opts = append(opts, "--"+field.Name, templated.String())
	}

	return opts, nil
}

func (f *FPM) baseDependencies() ([]string, error) {
	opts := []string{}

	for _, rawDepend := range f.Package.Depends {
		depend, err := f.Package.template.Render(rawDepend)
		if err != nil {
			f.Package.logger.WithFields(logrus.Fields{
				"error": err,
				"raw":   rawDepend,
			}).Error("failed to render dependency as template")
			return opts, err
		}
		opts = append(opts, "--depends", depend.String())
	}

	return opts, nil
}

func (f *FPM) baseScripts() ([]string, error) {
	opts := []string{}

	for name, location := range f.Package.scriptLocations {
		if name == "build" {
			continue
		}

		if name != "before-install" && name != "after-install" && name != "before-remove" && name != "after-remove" && name != "before-upgrade" && name != "after-upgrade" {
			f.Package.logger.WithFields(logrus.Fields{
				"script": name,
			}).Error(ErrInvalidScriptName)
			return opts, ErrInvalidScriptName
		}

		opts = append(opts, "--"+name, location)
	}

	return opts, nil
}

func (f *FPM) baseConfigs() ([]string, error) {
	opts := []string{}

	for i, target := range f.Package.Targets {
		if !target.Config {
			continue
		}

		dest, err := f.Package.template.Render(target.Dest)
		if err != nil {
			f.Package.logger.WithField("index", i).Error("error templating target destination")
			return opts, err
		}

		opts = append(opts, "--config-files", dest.String())
	}

	return opts, nil
}

func (f *FPM) optsForType(t string) []string {
	opts := []string{
		"-t", t,
	}

	return opts
}

func (f *FPM) extraArgs() ([]string, error) {
	extra, err := shlex.Split(f.Package.ExtraArgs, true)
	if err != nil {
		f.Package.logger.WithField("error", err).Error("failed to parse extra args")
	}
	return extra, err
}
<|endoftext|>


<commit_before>// Copyright 2016 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nvim

import (
	"fmt"
	"log"
	"reflect"
	"strings"
	"testing"
)

func newEmbeddedNvim(t *testing.T) (*Nvim, func()) {
	v, err := NewEmbedded(&EmbedOptions{
		Args: []string{"-u", "NONE", "-n"},
		Env:  []string{},
		Logf: t.Logf,
	})
	if err != nil {
		t.Fatal(err)
	}

	done := make(chan error, 1)
	go func() {
		done <- v.Serve()
	}()

	return v, func() {
		e1 := v.Close()
		e2 := <-done
		if e1 != nil {
			t.Fatal(e1)
		}
		if e2 != nil {
			t.Fatal(e2)
		}
	}
}

func helloHandler(s string) (string, error) {
	return "Hello, " + s, nil
}

func TestAPI(t *testing.T) {
	v, cleanup := newEmbeddedNvim(t)
	defer cleanup()
	cid := v.ChannelID()
	if cid <= 0 {
		t.Fatal("could not get channel id")
	}

	t.Run("simpleHandler", func(t *testing.T) {
		if err := v.RegisterHandler("hello", helloHandler); err != nil {
			t.Fatal(err)
		}
		var result string
		if err := v.Call("rpcrequest", &result, cid, "hello", "world"); err != nil {
			t.Fatal(err)
		}
		expected := "Hello, world"
		if result != expected {
			t.Errorf("hello returned %q, want %q", result, expected)
		}
	})

	t.Run("buffer", func(t *testing.T) {
		bufs, err := v.Buffers()
		if err != nil {
			t.Fatal(err)
		}
		if len(bufs) != 1 {
			t.Errorf("expected one buf, found %d bufs", len(bufs))
		}
		if bufs[0] == 0 {
			t.Errorf("bufs[0] == 0")
		}
		buf, err := v.CurrentBuffer()
		if err != nil {
			t.Fatal(err)
		}
		if buf != bufs[0] {
			t.Fatalf("buf %v != bufs[0] %v", buf, bufs[0])
		}
		err = v.SetCurrentBuffer(buf)
		if err != nil {
			t.Fatal(err)
		}

		err = v.SetBufferVar(buf, "bvar", "bval")
		if err != nil {
			t.Fatal(err)
		}

		var s string
		err = v.BufferVar(buf, "bvar", &s)
		if err != nil {
			t.Fatal(err)
		}
		if s != "bval" {
			t.Fatalf("expected bvar=bval, got %s", s)
		}

		err = v.DeleteBufferVar(buf, "bvar")
		if err != nil {
			t.Fatal(err)
		}

		s = ""
		err = v.BufferVar(buf, "bvar", &s)
		if err == nil {
			t.Errorf("expected key not found error")
		}
	})

	t.Run("window", func(t *testing.T) {
		wins, err := v.Windows()
		if err != nil {
			t.Fatal(err)
		}
		if len(wins) != 1 {
			t.Errorf("expected one win, found %d wins", len(wins))
		}
		if wins[0] == 0 {
			t.Errorf("wins[0] == 0")
		}
		win, err := v.CurrentWindow()
		if err != nil {
			t.Fatal(err)
		}
		if win != wins[0] {
			t.Fatalf("win %v != wins[0] %v", win, wins[0])
		}
		err = v.SetCurrentWindow(win)
		if err != nil {
			t.Fatal(err)
		}
	})

	t.Run("tabpage", func(t *testing.T) {
		pages, err := v.Tabpages()
		if err != nil {
			t.Fatal(err)
		}
		if len(pages) != 1 {
			t.Errorf("expected one page, found %d pages", len(pages))
		}
		if pages[0] == 0 {
			t.Errorf("pages[0] == 0")
		}
		page, err := v.CurrentTabpage()
		if err != nil {
			t.Fatal(err)
		}
		if page != pages[0] {
			t.Fatalf("page %v != pages[0] %v", page, pages[0])
		}
		err = v.SetCurrentTabpage(page)
		if err != nil {
			t.Fatal(err)
		}
	})

	t.Run("lines", func(t *testing.T) {
		buf, err := v.CurrentBuffer()
		if err != nil {
			t.Fatal(err)
		}
		lines := [][]byte{[]byte("hello"), []byte("world")}
		if err := v.SetBufferLines(buf, 0, -1, true, lines); err != nil {
			t.Fatal(err)
		}
		lines2, err := v.BufferLines(buf, 0, -1, true)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(lines2, lines) {
			t.Fatalf("lines = %+v, want %+v", lines2, lines)
		}
	})

	t.Run("var", func(t *testing.T) {
		if err := v.SetVar("gvar", "gval"); err != nil {
			t.Fatal(err)
		}
		var value interface{}
		if err := v.Var("gvar", &value); err != nil {
			t.Fatal(err)
		}
		if value != "gval" {
			t.Errorf("got %v, want %q", value, "gval")
		}
		if err := v.SetVar("gvar", ""); err != nil {
			t.Fatal(err)
		}
		value = nil
		if err := v.Var("gvar", &value); err != nil {
			t.Fatal(err)
		}
		if value != "" {
			t.Errorf("got %v, want %q", value, "")
		}
	})

	t.Run("structValue", func(t *testing.T) {
		var expected, actual struct {
			Str string
			Num int
		}
		expected.Str = "Hello"
		expected.Num = 42
		if err := v.SetVar("structvar", &expected); err != nil {
			t.Fatal(err)
		}
		if err := v.Var("structvar", &actual); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(&actual, &expected) {
			t.Errorf("got %+v, want %+v", &actual, &expected)
		}
	})

	t.Run("eval", func(t *testing.T) {
		var a, b string
		if err := v.Eval(`["hello", "world"]`, []*string{&a, &b}); err != nil {
			t.Error(err)
		}
		if a != "hello" || b != "world" {
			t.Errorf("a=%q b=%q, want a=hello b=world", a, b)
		}
	})

	t.Run("batch", func(t *testing.T) {
		b := v.NewBatch()
		results := make([]int, 128)

		for i := range results {
			b.SetVar(fmt.Sprintf("batch%d", i), i)
		}

		for i := range results {
			b.Var(fmt.Sprintf("batch%d", i), &results[i])
		}

		if err := b.Execute(); err != nil {
			t.Fatal(err)
		}

		for i := range results {
			if results[i] != i {
				t.Fatalf("result[i] = %d, want %d", results[i], i)
			}
		}

		// Reuse batch

		var i int
		b.Var("batch3", &i)
		if err := b.Execute(); err != nil {
			log.Fatal(err)
		}
		if i != 3 {
			t.Fatalf("i = %d, want %d", i, 3)
		}

		// Check for *BatchError

		const errorIndex = 3

		for i := range results {
			results[i] = -1
		}

		for i := range results {
			if i == errorIndex {
				b.Var("batch_bad_var", &results[i])
			} else {
				b.Var(fmt.Sprintf("batch%d", i), &results[i])
			}
		}
		err := b.Execute()
		if e, ok := err.(*BatchError); !ok || e.Index != errorIndex {
			t.Errorf("unxpected error %T %v", e, e)
		}
		// Expect results proceeding error.
		for i := 0; i < errorIndex; i++ {
			if results[i] != i {
				t.Errorf("result[i] = %d, want %d", results[i], i)
				break
			}
		}
		// No results after error.
		for i := errorIndex; i < len(results); i++ {
			if results[i] != -1 {
				t.Errorf("result[i] = %d, want %d", results[i], -1)
				break
			}
		}

		// Execute should return marshal error for argument that cannot be marshaled.
		b.SetVar("batch0", make(chan bool))
		err = b.Execute()
		if err == nil || !strings.Contains(err.Error(), "chan bool") {
			t.Errorf("err = nil, expect error containing text 'chan bool'")
		}

		// Test call with empty argument list.
		var buf Buffer
		b.CurrentBuffer(&buf)
		err = b.Execute()
		if err != nil {
			t.Errorf("GetCurrentBuffer returns err %s: %#v", err, err)
		}
	})

	t.Run("callWithNoArgs", func(t *testing.T) {
		var wd string
		err := v.Call("getcwd", &wd)
		if err != nil {
			t.Fatal(err)
		}
	})

	/*
		t.Run("mode", func(t *testing.T) {
			m, err := v.Mode()
			if err != nil {
				t.Fatal(err)
			}
			if m.Mode != "n" {
				t.Errorf("Mode() returned %s, want n", m.Mode)
			}
		})

		t.Run("exeuteLua", func(t *testing.T) {
			var n int
			err := v.ExecuteLua("local a, b = ... return a + b", &n, 1, 2)
			if err != nil {
				t.Fatal(err)
			}
			if n != 3 {
				t.Errorf("Mode() returned %v, want 3", n)
			}
		})
	*/
}
<commit_msg>Test error propagation<commit_after>// Copyright 2016 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nvim

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
	"testing"
)

func newEmbeddedNvim(t *testing.T) (*Nvim, func()) {
	v, err := NewEmbedded(&EmbedOptions{
		Args: []string{"-u", "NONE", "-n"},
		Env:  []string{},
		Logf: t.Logf,
	})
	if err != nil {
		t.Fatal(err)
	}

	done := make(chan error, 1)
	go func() {
		done <- v.Serve()
	}()

	return v, func() {
		e1 := v.Close()
		e2 := <-done
		if e1 != nil {
			t.Fatal(e1)
		}
		if e2 != nil {
			t.Fatal(e2)
		}
	}
}

func helloHandler(s string) (string, error) {
	return "Hello, " + s, nil
}

func errorHandler() error {
	return errors.New("ouch")
}

func TestAPI(t *testing.T) {
	v, cleanup := newEmbeddedNvim(t)
	defer cleanup()
	cid := v.ChannelID()
	if cid <= 0 {
		t.Fatal("could not get channel id")
	}

	t.Run("simpleHandler", func(t *testing.T) {
		if err := v.RegisterHandler("hello", helloHandler); err != nil {
			t.Fatal(err)
		}
		if err := v.RegisterHandler("error", errorHandler); err != nil {
			t.Fatal(err)
		}
		var result string
		if err := v.Call("rpcrequest", &result, cid, "hello", "world"); err != nil {
			t.Fatal(err)
		}
		if expected := "Hello, world"; result != expected {
			t.Errorf("hello returned %q, want %q", result, expected)
		}

		// Test errors.
		if err := v.Call("execute", &result, fmt.Sprintf("silent! call rpcrequest(%d, 'error')", cid)); err != nil {
			t.Fatal(err)
		}
		if expected := "\nouch"; result != expected {
			t.Errorf("got error %q, want %q", result, expected)
		}
	})

	t.Run("buffer", func(t *testing.T) {
		bufs, err := v.Buffers()
		if err != nil {
			t.Fatal(err)
		}
		if len(bufs) != 1 {
			t.Errorf("expected one buf, found %d bufs", len(bufs))
		}
		if bufs[0] == 0 {
			t.Errorf("bufs[0] == 0")
		}
		buf, err := v.CurrentBuffer()
		if err != nil {
			t.Fatal(err)
		}
		if buf != bufs[0] {
			t.Fatalf("buf %v != bufs[0] %v", buf, bufs[0])
		}
		err = v.SetCurrentBuffer(buf)
		if err != nil {
			t.Fatal(err)
		}

		err = v.SetBufferVar(buf, "bvar", "bval")
		if err != nil {
			t.Fatal(err)
		}

		var s string
		err = v.BufferVar(buf, "bvar", &s)
		if err != nil {
			t.Fatal(err)
		}
		if s != "bval" {
			t.Fatalf("expected bvar=bval, got %s", s)
		}

		err = v.DeleteBufferVar(buf, "bvar")
		if err != nil {
			t.Fatal(err)
		}

		s = ""
		err = v.BufferVar(buf, "bvar", &s)
		if err == nil {
			t.Errorf("expected key not found error")
		}
	})

	t.Run("window", func(t *testing.T) {
		wins, err := v.Windows()
		if err != nil {
			t.Fatal(err)
		}
		if len(wins) != 1 {
			t.Errorf("expected one win, found %d wins", len(wins))
		}
		if wins[0] == 0 {
			t.Errorf("wins[0] == 0")
		}
		win, err := v.CurrentWindow()
		if err != nil {
			t.Fatal(err)
		}
		if win != wins[0] {
			t.Fatalf("win %v != wins[0] %v", win, wins[0])
		}
		err = v.SetCurrentWindow(win)
		if err != nil {
			t.Fatal(err)
		}
	})

	t.Run("tabpage", func(t *testing.T) {
		pages, err := v.Tabpages()
		if err != nil {
			t.Fatal(err)
		}
		if len(pages) != 1 {
			t.Errorf("expected one page, found %d pages", len(pages))
		}
		if pages[0] == 0 {
			t.Errorf("pages[0] == 0")
		}
		page, err := v.CurrentTabpage()
		if err != nil {
			t.Fatal(err)
		}
		if page != pages[0] {
			t.Fatalf("page %v != pages[0] %v", page, pages[0])
		}
		err = v.SetCurrentTabpage(page)
		if err != nil {
			t.Fatal(err)
		}
	})

	t.Run("lines", func(t *testing.T) {
		buf, err := v.CurrentBuffer()
		if err != nil {
			t.Fatal(err)
		}
		lines := [][]byte{[]byte("hello"), []byte("world")}
		if err := v.SetBufferLines(buf, 0, -1, true, lines); err != nil {
			t.Fatal(err)
		}
		lines2, err := v.BufferLines(buf, 0, -1, true)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(lines2, lines) {
			t.Fatalf("lines = %+v, want %+v", lines2, lines)
		}
	})

	t.Run("var", func(t *testing.T) {
		if err := v.SetVar("gvar", "gval"); err != nil {
			t.Fatal(err)
		}
		var value interface{}
		if err := v.Var("gvar", &value); err != nil {
			t.Fatal(err)
		}
		if value != "gval" {
			t.Errorf("got %v, want %q", value, "gval")
		}
		if err := v.SetVar("gvar", ""); err != nil {
			t.Fatal(err)
		}
		value = nil
		if err := v.Var("gvar", &value); err != nil {
			t.Fatal(err)
		}
		if value != "" {
			t.Errorf("got %v, want %q", value, "")
		}
	})

	t.Run("structValue", func(t *testing.T) {
		var expected, actual struct {
			Str string
			Num int
		}
		expected.Str = "Hello"
		expected.Num = 42
		if err := v.SetVar("structvar", &expected); err != nil {
			t.Fatal(err)
		}
		if err := v.Var("structvar", &actual); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(&actual, &expected) {
			t.Errorf("got %+v, want %+v", &actual, &expected)
		}
	})

	t.Run("eval", func(t *testing.T) {
		var a, b string
		if err := v.Eval(`["hello", "world"]`, []*string{&a, &b}); err != nil {
			t.Error(err)
		}
		if a != "hello" || b != "world" {
			t.Errorf("a=%q b=%q, want a=hello b=world", a, b)
		}
	})

	t.Run("batch", func(t *testing.T) {
		b := v.NewBatch()
		results := make([]int, 128)

		for i := range results {
			b.SetVar(fmt.Sprintf("batch%d", i), i)
		}

		for i := range results {
			b.Var(fmt.Sprintf("batch%d", i), &results[i])
		}

		if err := b.Execute(); err != nil {
			t.Fatal(err)
		}

		for i := range results {
			if results[i] != i {
				t.Fatalf("result[i] = %d, want %d", results[i], i)
			}
		}

		// Reuse batch

		var i int
		b.Var("batch3", &i)
		if err := b.Execute(); err != nil {
			log.Fatal(err)
		}
		if i != 3 {
			t.Fatalf("i = %d, want %d", i, 3)
		}

		// Check for *BatchError

		const errorIndex = 3

		for i := range results {
			results[i] = -1
		}

		for i := range results {
			if i == errorIndex {
				b.Var("batch_bad_var", &results[i])
			} else {
				b.Var(fmt.Sprintf("batch%d", i), &results[i])
			}
		}
		err := b.Execute()
		if e, ok := err.(*BatchError); !ok || e.Index != errorIndex {
			t.Errorf("unxpected error %T %v", e, e)
		}
		// Expect results proceeding error.
		for i := 0; i < errorIndex; i++ {
			if results[i] != i {
				t.Errorf("result[i] = %d, want %d", results[i], i)
				break
			}
		}
		// No results after error.
		for i := errorIndex; i < len(results); i++ {
			if results[i] != -1 {
				t.Errorf("result[i] = %d, want %d", results[i], -1)
				break
			}
		}

		// Execute should return marshal error for argument that cannot be marshaled.
		b.SetVar("batch0", make(chan bool))
		err = b.Execute()
		if err == nil || !strings.Contains(err.Error(), "chan bool") {
			t.Errorf("err = nil, expect error containing text 'chan bool'")
		}

		// Test call with empty argument list.
		var buf Buffer
		b.CurrentBuffer(&buf)
		err = b.Execute()
		if err != nil {
			t.Errorf("GetCurrentBuffer returns err %s: %#v", err, err)
		}
	})

	t.Run("callWithNoArgs", func(t *testing.T) {
		var wd string
		err := v.Call("getcwd", &wd)
		if err != nil {
			t.Fatal(err)
		}
	})

	/*
		t.Run("mode", func(t *testing.T) {
			m, err := v.Mode()
			if err != nil {
				t.Fatal(err)
			}
			if m.Mode != "n" {
				t.Errorf("Mode() returned %s, want n", m.Mode)
			}
		})

		t.Run("exeuteLua", func(t *testing.T) {
			var n int
			err := v.ExecuteLua("local a, b = ... return a + b", &n, 1, 2)
			if err != nil {
				t.Fatal(err)
			}
			if n != 3 {
				t.Errorf("Mode() returned %v, want 3", n)
			}
		})
	*/
}
<|endoftext|>


<commit_before>package assets

import (
	"fmt"
)

const (
	analyticsScript = `<script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
	(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
	ga('create', '%s');
	ga('send', 'pageview');</script>`
)

func googleAnalytics(m *Manager, names []string, options Options) ([]*Asset, error) {
	key := names[0]
	if key == "" {
		return nil, nil
	}
	return []*Asset{
		&Asset{
			Name:     "google-analytics.js",
			Position: Bottom,
			HTML:     fmt.Sprintf(analyticsScript, key),
		},
	}, nil
}

func init() {
	Register("analytics", googleAnalytics)
}
<commit_msg>Add support for including the domain the the analytics asset<commit_after>package assets

import (
	"fmt"
)

const (
	analyticsScript = `<script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
	(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
	ga('create', %s);
	ga('send', 'pageview');</script>`
)

func googleAnalytics(m *Manager, names []string, options Options) ([]*Asset, error) {
	if len(names) != 1 && len(names) != 2 {
		return nil, fmt.Errorf("analytics requires either 1 or 2 arguments (either \"UA-XXXXXX-YY, mysite.com\" or just \"UA-XXXXXX-YY\" - without quotes in both cases")
	}
	key := names[0]
	if key == "" {
		return nil, nil
	}
	var arg string
	if len(names) == 2 {
		arg = fmt.Sprintf("'%s', '%s'", key, names[1])
	} else {
		arg = fmt.Sprintf("'%s'", key)
	}
	return []*Asset{
		&Asset{
			Name:     "google-analytics.js",
			Position: Bottom,
			HTML:     fmt.Sprintf(analyticsScript, arg),
		},
	}, nil
}

func init() {
	Register("analytics", googleAnalytics)
}
<|endoftext|>


<commit_before>package tls

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/dynamiclistener"
	"github.com/rancher/dynamiclistener/cert"
	"github.com/rancher/dynamiclistener/server"
	"github.com/rancher/dynamiclistener/storage/kubernetes"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
	corev1controllers "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/net"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/rest"
)

const (
	rancherCertFile    = "/etc/rancher/ssl/cert.pem"
	rancherKeyFile     = "/etc/rancher/ssl/key.pem"
	rancherCACertsFile = "/etc/rancher/ssl/cacerts.pem"
)

func ListenAndServe(ctx context.Context, restConfig *rest.Config, handler http.Handler, httpsPort, httpPort int, acmeDomains []string, noCACerts bool) error {
	restConfig = rest.CopyConfig(restConfig)
	restConfig.Timeout = 10 * time.Minute

	core, err := core.NewFactoryFromConfig(restConfig)
	if err != nil {
		return err
	}

	opts, err := SetupListener(core.Core().V1().Secret(), acmeDomains, noCACerts)
	if err != nil {
		return err
	}

	migrateConfig(ctx, restConfig, opts)

	if err := server.ListenAndServe(ctx, httpsPort, httpPort, handler, opts); err != nil {
		return err
	}

	if err := core.Start(ctx, 5); err != nil {
		return err
	}

	<-ctx.Done()
	return ctx.Err()

}

func migrateConfig(ctx context.Context, restConfig *rest.Config, opts *server.ListenOpts) {
	defer func() {
		opts.TLSListenerConfig.MaxSANs += len(opts.TLSListenerConfig.SANs)
	}()

	c, err := dynamic.NewForConfig(restConfig)
	if err != nil {
		return
	}

	config, err := c.Resource(schema.GroupVersionResource{
		Group:    "management.cattle.io",
		Version:  "v3",
		Resource: "listenconfigs",
	}).Get(ctx, "cli-config", metav1.GetOptions{})
	if err != nil {
		return
	}

	known := convert.ToStringSlice(config.Object["knownIps"])
	for k := range convert.ToMapInterface(config.Object["generatedCerts"]) {
		if strings.HasPrefix(k, "local/") {
			continue
		}
		known = append(known, k)
	}

	for _, k := range known {
		found := false
		for _, san := range opts.TLSListenerConfig.SANs {
			if san == k {
				found = true
				break
			}
		}
		if !found {
			opts.TLSListenerConfig.SANs = append(opts.TLSListenerConfig.SANs, k)
		}
	}
}

func SetupListener(secrets corev1controllers.SecretController, acmeDomains []string, noCACerts bool) (*server.ListenOpts, error) {
	caForAgent, noCACerts, opts, err := readConfig(secrets, acmeDomains, noCACerts)
	if err != nil {
		return nil, err
	}

	if noCACerts {
		caForAgent = ""
	} else if caForAgent == "" {
		caCert, caKey, err := kubernetes.LoadOrGenCA(secrets, opts.CANamespace, opts.CAName)
		if err != nil {
			return nil, err
		}
		caForAgent = string(cert.EncodeCertPEM(caCert))
		opts.CA = caCert
		opts.CAKey = caKey
	}

	caForAgent = strings.TrimSpace(caForAgent)
	if settings.CACerts.Get() != caForAgent {
		if err := settings.CACerts.Set(caForAgent); err != nil {
			return nil, err
		}
	}

	return opts, nil
}

func readConfig(secrets corev1controllers.SecretController, acmeDomains []string, noCACerts bool) (string, bool, *server.ListenOpts, error) {
	var (
		ca  string
		err error
	)

	tlsConfig, err := BaseTLSConfig()
	if err != nil {
		return "", noCACerts, nil, err
	}

	expiration, err := strconv.Atoi(settings.RotateCertsIfExpiringInDays.Get())
	if err != nil {
		return "", noCACerts, nil, errors.Wrapf(err, "parsing %s", settings.RotateCertsIfExpiringInDays.Get())
	}

	sans := []string{"localhost", "127.0.0.1"}
	ip, err := net.ChooseHostInterface()
	if err == nil {
		sans = append(sans, ip.String())
	}

	opts := &server.ListenOpts{
		Secrets:       secrets,
		CAName:        "tls-rancher",
		CANamespace:   "cattle-system",
		CertNamespace: "cattle-system",
		AcmeDomains:   acmeDomains,
		TLSListenerConfig: dynamiclistener.Config{
			TLSConfig:             tlsConfig,
			ExpirationDaysCheck:   expiration,
			SANs:                  sans,
			MaxSANs:               20,
			CloseConnOnCertChange: true,
		},
	}

	// ACME / Let's Encrypt
	// If --acme-domain is set, configure and return
	if len(acmeDomains) > 0 {
		return "", true, opts, nil
	}

	// Mounted certificates
	// If certificate file/key are set
	certFileExists := fileExists(rancherCertFile)
	keyFileExists := fileExists(rancherKeyFile)

	// If certificate file exists but not certificate key, or other way around, error out
	if (certFileExists && !keyFileExists) || (!certFileExists && keyFileExists) {
		return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set both certificate file and certificate key file (one is missing)")
	}

	caFileExists := fileExists(rancherCACertsFile)

	// If certificate file and certificate key file exists, load files into listenConfig
	if certFileExists && keyFileExists {
		cert, err := tls.LoadX509KeyPair(rancherCertFile, rancherKeyFile)
		if err != nil {
			return "", noCACerts, nil, err
		}
		opts.TLSListenerConfig.TLSConfig.Certificates = []tls.Certificate{cert}

		// Selfsigned needs cacerts, recognized CA needs --no-cacerts but can't be used together
		if (caFileExists && noCACerts) || (!caFileExists && !noCACerts) {
			return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set cacerts when using self signed certificates or use --no-cacerts when using certificates from a recognized Certificate Authority, do not use both at the same time")
		}
		// Load cacerts if exists
		if caFileExists {
			ca, err = readPEM(rancherCACertsFile)
			if err != nil {
				return "", noCACerts, nil, err
			}
		}
		return ca, noCACerts, opts, nil
	}

	// External termination
	// We need to check if cacerts is passed or if --no-cacerts is used (when not providing certificate file and key)
	// If cacerts is passed
	if caFileExists {
		// We can't have --no-cacerts
		if noCACerts {
			return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set cacerts when using self signed certificates or use --no-cacerts when using certificates from a recognized Certificate Authority, do not use both at the same time")
		}
		ca, err = readPEM(rancherCACertsFile)
		if err != nil {
			return "", noCACerts, nil, err
		}
	}

	// No certificates mounted or only --no-cacerts used
	return ca, noCACerts, opts, nil
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

func readPEM(path string) (string, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return "", err
	}

	return string(content), nil
}
<commit_msg>Lock dynamic TLS to server-url value<commit_after>package tls

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/dynamiclistener"
	"github.com/rancher/dynamiclistener/cert"
	"github.com/rancher/dynamiclistener/server"
	"github.com/rancher/dynamiclistener/storage/kubernetes"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
	corev1controllers "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/net"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/rest"
)

const (
	rancherCertFile    = "/etc/rancher/ssl/cert.pem"
	rancherKeyFile     = "/etc/rancher/ssl/key.pem"
	rancherCACertsFile = "/etc/rancher/ssl/cacerts.pem"
)

func ListenAndServe(ctx context.Context, restConfig *rest.Config, handler http.Handler, httpsPort, httpPort int, acmeDomains []string, noCACerts bool) error {
	restConfig = rest.CopyConfig(restConfig)
	restConfig.Timeout = 10 * time.Minute

	core, err := core.NewFactoryFromConfig(restConfig)
	if err != nil {
		return err
	}

	opts, err := SetupListener(core.Core().V1().Secret(), acmeDomains, noCACerts)
	if err != nil {
		return err
	}

	migrateConfig(ctx, restConfig, opts)

	if err := server.ListenAndServe(ctx, httpsPort, httpPort, handler, opts); err != nil {
		return err
	}

	if err := core.Start(ctx, 5); err != nil {
		return err
	}

	<-ctx.Done()
	return ctx.Err()

}

func migrateConfig(ctx context.Context, restConfig *rest.Config, opts *server.ListenOpts) {
	defer func() {
		opts.TLSListenerConfig.MaxSANs += len(opts.TLSListenerConfig.SANs)
	}()

	c, err := dynamic.NewForConfig(restConfig)
	if err != nil {
		return
	}

	config, err := c.Resource(schema.GroupVersionResource{
		Group:    "management.cattle.io",
		Version:  "v3",
		Resource: "listenconfigs",
	}).Get(ctx, "cli-config", metav1.GetOptions{})
	if err != nil {
		return
	}

	known := convert.ToStringSlice(config.Object["knownIps"])
	for k := range convert.ToMapInterface(config.Object["generatedCerts"]) {
		if strings.HasPrefix(k, "local/") {
			continue
		}
		known = append(known, k)
	}

	for _, k := range known {
		k = strings.SplitN(k, ":", 2)[0]
		found := false
		for _, san := range opts.TLSListenerConfig.SANs {
			if san == k {
				found = true
				break
			}
		}
		if !found {
			opts.TLSListenerConfig.SANs = append(opts.TLSListenerConfig.SANs, k)
		}
	}
}

func SetupListener(secrets corev1controllers.SecretController, acmeDomains []string, noCACerts bool) (*server.ListenOpts, error) {
	caForAgent, noCACerts, opts, err := readConfig(secrets, acmeDomains, noCACerts)
	if err != nil {
		return nil, err
	}

	if noCACerts {
		caForAgent = ""
	} else if caForAgent == "" {
		caCert, caKey, err := kubernetes.LoadOrGenCA(secrets, opts.CANamespace, opts.CAName)
		if err != nil {
			return nil, err
		}
		caForAgent = string(cert.EncodeCertPEM(caCert))
		opts.CA = caCert
		opts.CAKey = caKey
	}

	caForAgent = strings.TrimSpace(caForAgent)
	if settings.CACerts.Get() != caForAgent {
		if err := settings.CACerts.Set(caForAgent); err != nil {
			return nil, err
		}
	}

	return opts, nil
}

func readConfig(secrets corev1controllers.SecretController, acmeDomains []string, noCACerts bool) (string, bool, *server.ListenOpts, error) {
	var (
		ca  string
		err error
	)

	tlsConfig, err := BaseTLSConfig()
	if err != nil {
		return "", noCACerts, nil, err
	}

	expiration, err := strconv.Atoi(settings.RotateCertsIfExpiringInDays.Get())
	if err != nil {
		return "", noCACerts, nil, errors.Wrapf(err, "parsing %s", settings.RotateCertsIfExpiringInDays.Get())
	}

	sans := []string{"localhost", "127.0.0.1"}
	ip, err := net.ChooseHostInterface()
	if err == nil {
		sans = append(sans, ip.String())
	}

	opts := &server.ListenOpts{
		Secrets:       secrets,
		CAName:        "tls-rancher",
		CANamespace:   "cattle-system",
		CertNamespace: "cattle-system",
		AcmeDomains:   acmeDomains,
		TLSListenerConfig: dynamiclistener.Config{
			TLSConfig:             tlsConfig,
			ExpirationDaysCheck:   expiration,
			SANs:                  sans,
			FilterCN:              filterCN,
			CloseConnOnCertChange: true,
		},
	}

	// ACME / Let's Encrypt
	// If --acme-domain is set, configure and return
	if len(acmeDomains) > 0 {
		return "", true, opts, nil
	}

	// Mounted certificates
	// If certificate file/key are set
	certFileExists := fileExists(rancherCertFile)
	keyFileExists := fileExists(rancherKeyFile)

	// If certificate file exists but not certificate key, or other way around, error out
	if (certFileExists && !keyFileExists) || (!certFileExists && keyFileExists) {
		return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set both certificate file and certificate key file (one is missing)")
	}

	caFileExists := fileExists(rancherCACertsFile)

	// If certificate file and certificate key file exists, load files into listenConfig
	if certFileExists && keyFileExists {
		cert, err := tls.LoadX509KeyPair(rancherCertFile, rancherKeyFile)
		if err != nil {
			return "", noCACerts, nil, err
		}
		opts.TLSListenerConfig.TLSConfig.Certificates = []tls.Certificate{cert}

		// Selfsigned needs cacerts, recognized CA needs --no-cacerts but can't be used together
		if (caFileExists && noCACerts) || (!caFileExists && !noCACerts) {
			return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set cacerts when using self signed certificates or use --no-cacerts when using certificates from a recognized Certificate Authority, do not use both at the same time")
		}
		// Load cacerts if exists
		if caFileExists {
			ca, err = readPEM(rancherCACertsFile)
			if err != nil {
				return "", noCACerts, nil, err
			}
		}
		return ca, noCACerts, opts, nil
	}

	// External termination
	// We need to check if cacerts is passed or if --no-cacerts is used (when not providing certificate file and key)
	// If cacerts is passed
	if caFileExists {
		// We can't have --no-cacerts
		if noCACerts {
			return "", noCACerts, nil, fmt.Errorf("invalid SSL configuration found, please set cacerts when using self signed certificates or use --no-cacerts when using certificates from a recognized Certificate Authority, do not use both at the same time")
		}
		ca, err = readPEM(rancherCACertsFile)
		if err != nil {
			return "", noCACerts, nil, err
		}
	}

	// No certificates mounted or only --no-cacerts used
	return ca, noCACerts, opts, nil
}

func filterCN(cns ...string) []string {
	serverURL := settings.ServerURL.Get()
	if serverURL == "" {
		return cns
	}
	u, err := url.Parse(serverURL)
	if err != nil {
		logrus.Errorf("invalid server-url, can not parse %s: %v", serverURL, err)
		return cns
	}
	host := u.Hostname()
	for _, cn := range cns {
		if cn == host {
			return []string{host}
		}
	}
	return nil
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

func readPEM(path string) (string, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return "", err
	}

	return string(content), nil
}
<|endoftext|>


<commit_before>package transactionpool

import (
	"github.com/rivine/rivine/modules"
	"github.com/rivine/rivine/types"
)

type (
	transactionCache struct {
		cache map[TransactionSetID]*broadcastInfo
	}

	broadcastInfo struct {
		// originalSubmit is the original block height in which a transacton was submitted
		// in our cache
		originalSubmit types.BlockHeight
		// broadcasts is the amount of times we have rebroadcast a transaction
		broadcasts uint32
	}
)

func newTransactionCache() transactionCache {
	return transactionCache{
		cache: make(map[TransactionSetID]*broadcastInfo),
	}
}

// add a new transaction ID to the cache
func (tc *transactionCache) add(id TransactionSetID, currentHeight types.BlockHeight) {
	// Because the update method of the transaction pool purges the tp, and then adds all the
	// transactions again which it finds are still unconfirmed, we need to check if the id we
	// add is not already known to us
	if _, exists := tc.cache[id]; !exists {
		tc.cache[id] = &broadcastInfo{originalSubmit: currentHeight, broadcasts: 0}
	}
}

// delete ensures the transaction ID is no longer present in the cache. If it is not present in the first place,
// no action is taken.
func (tc *transactionCache) delete(id TransactionSetID) {
	if _, exists := tc.cache[id]; exists {
		delete(tc.cache, id)
	}
}

func (tc *transactionCache) getTransactionsToBroadcast(height types.BlockHeight) []TransactionSetID {
	txnIDs := []TransactionSetID{}
	for id, info := range tc.cache {
		if info.shouldRebroadcast(height) {
			txnIDs = append(txnIDs, id)
		}
	}
	return txnIDs
}

// shouldRebroadcast decides if the txninfo indicates that the transaction should
// be rebroadcast. If this should be the case, the broadcasts coutner will be updated.
func (bci *broadcastInfo) shouldRebroadcast(height types.BlockHeight) bool {
	if (height-bci.originalSubmit)%modules.TransactionPoolRebroadcastDelay == 0 &&
		bci.broadcasts <= modules.TransactionPoolMaxRebroadcasts {
		bci.broadcasts++
		return true
	}
	return false
}
<commit_msg>Remove unnecessary check in tpcache<commit_after>package transactionpool

import (
	"github.com/rivine/rivine/modules"
	"github.com/rivine/rivine/types"
)

type (
	transactionCache struct {
		cache map[TransactionSetID]*broadcastInfo
	}

	broadcastInfo struct {
		// originalSubmit is the original block height in which a transacton was submitted
		// in our cache
		originalSubmit types.BlockHeight
		// broadcasts is the amount of times we have rebroadcast a transaction
		broadcasts uint32
	}
)

func newTransactionCache() transactionCache {
	return transactionCache{
		cache: make(map[TransactionSetID]*broadcastInfo),
	}
}

// add a new transaction ID to the cache
func (tc *transactionCache) add(id TransactionSetID, currentHeight types.BlockHeight) {
	// Because the update method of the transaction pool purges the tp, and then adds all the
	// transactions again which it finds are still unconfirmed, we need to check if the id we
	// add is not already known to us
	if _, exists := tc.cache[id]; !exists {
		tc.cache[id] = &broadcastInfo{originalSubmit: currentHeight, broadcasts: 0}
	}
}

// delete ensures the transaction ID is no longer present in the cache. If it is not present in the first place,
// no action is taken.
func (tc *transactionCache) delete(id TransactionSetID) {
	delete(tc.cache, id)
}

func (tc *transactionCache) getTransactionsToBroadcast(height types.BlockHeight) []TransactionSetID {
	txnIDs := []TransactionSetID{}
	for id, info := range tc.cache {
		if info.shouldRebroadcast(height) {
			txnIDs = append(txnIDs, id)
		}
	}
	return txnIDs
}

// shouldRebroadcast decides if the txninfo indicates that the transaction should
// be rebroadcast. If this should be the case, the broadcasts coutner will be updated.
func (bci *broadcastInfo) shouldRebroadcast(height types.BlockHeight) bool {
	if (height-bci.originalSubmit)%modules.TransactionPoolRebroadcastDelay == 0 &&
		bci.broadcasts <= modules.TransactionPoolMaxRebroadcasts {
		bci.broadcasts++
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package output

import (
	"encoding/json"
	"strings"

	"github.com/r3labs/graph"
	"github.com/r3labs/workflow"
)

// FSMMessage is the JSON payload that will be sent to the FSM to create a
// service.
type FSMMessage struct {
	ID            string   `json:"id"`
	Body          string   `json:"body"`
	Endpoint      string   `json:"endpoint"`
	Service       string   `json:"service"`
	Bootstrapping string   `json:"bootstrapping"`
	ErnestIP      []string `json:"ernest_ip"`
	ServiceIP     string   `json:"service_ip"`
	Parent        string   `json:"existing_service"`
	Workflow      struct {
		Arcs []graph.Edge `json:"arcs"`
	} `json:"workflow"`
	ServiceName string `json:"name"`
	Client      string `json:"client"` // TODO: Use client or client_id not both!
	ClientID    string `json:"client_id"`
	ClientName  string `json:"client_name"`
	Started     string `json:"started"`
	Finished    string `json:"finished"`
	Status      string `json:"status"`
	Type        string `json:"type"`
	Datacenters struct {
		Started  string       `json:"started"`
		Finished string       `json:"finished"`
		Status   string       `json:"status"`
		Items    []Datacenter `json:"items"`
	} `json:"datacenters"`
	Networks struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks"`
	NetworksToCreate struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_create"`
	NetworksToDelete struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_delete"`
	Instances struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances"`
	InstancesToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_create"`
	InstancesToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_update"`
	InstancesToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_delete"`
	Firewalls struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls"`
	FirewallsToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_create"`
	FirewallsToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_update"`
	FirewallsToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_delete"`
	Nats struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats"`
	NatsToCreate struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_create"`
	NatsToUpdate struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_update"`
	NatsToDelete struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_delete"`
}

// Diff compares against an existing FSMMessage from a previous fsm message
func (m *FSMMessage) Diff(om FSMMessage) {
	// build new networks
	for _, network := range m.Networks.Items {
		if o := om.FindNetwork(network.Name); o == nil {
			m.NetworksToCreate.Items = append(m.NetworksToCreate.Items, network)
		}
	}

	// build old networks to delete
	for _, network := range om.Networks.Items {
		if m.FindNetwork(network.Name) == nil {
			m.NetworksToDelete.Items = append(m.NetworksToDelete.Items, network)
		}
	}

	// build new and existing instances
	for _, instance := range m.Instances.Items {
		if oi := om.FindInstance(instance.Name); oi == nil {
			m.InstancesToCreate.Items = append(m.InstancesToCreate.Items, instance)
		} else if instance.HasChanged(oi) {
			m.InstancesToUpdate.Items = append(m.InstancesToUpdate.Items, instance)
		}
	}

	// build old instances to delete
	for _, instance := range om.Instances.Items {
		if m.FindInstance(instance.Name) == nil {
			m.InstancesToDelete.Items = append(m.InstancesToDelete.Items, instance)
		}
	}

	// build new and existing firewalls
	for _, firewall := range m.Firewalls.Items {
		if of := om.FindFirewall(firewall.Name); of == nil {
			m.FirewallsToCreate.Items = append(m.FirewallsToCreate.Items, firewall)
		} else if firewall.HasChanged(of) {
			m.FirewallsToUpdate.Items = append(m.FirewallsToUpdate.Items, firewall)
		}
	}

	// build old firewalls to delete
	for _, firewall := range om.Firewalls.Items {
		if m.FindFirewall(firewall.Name) == nil {
			m.FirewallsToDelete.Items = append(m.FirewallsToDelete.Items, firewall)
		}
	}

	// build new and existing nats
	for _, nat := range m.Nats.Items {
		if on := om.FindNat(nat.Name); on == nil {
			m.NatsToCreate.Items = append(m.NatsToCreate.Items, nat)
		} else if nat.HasChanged(on) {
			m.NatsToUpdate.Items = append(m.NatsToUpdate.Items, nat)
		}
	}

	// build old nats to delete
	for _, nat := range om.Nats.Items {
		if m.FindNat(nat.Name) == nil {
			m.NatsToDelete.Items = append(m.NatsToDelete.Items, nat)
		}
	}
}

// GenerateWorkflow creates a fsm workflow based upon actionable tasks, such as creation or deletion of an entity.
func (m *FSMMessage) GenerateWorkflow(path string) error {
	w := workflow.New()
	err := w.LoadFile("./output/arcs/" + path)
	if err != nil {
		return err
	}

	// Set network items
	w.SetCount("creating_networks", len(m.NetworksToCreate.Items))
	w.SetCount("networks_created", len(m.NetworksToCreate.Items))
	w.SetCount("deleting_networks", len(m.NetworksToDelete.Items))
	w.SetCount("networks_deleted", len(m.NetworksToDelete.Items))

	// Set instance items
	w.SetCount("creating_instances", len(m.InstancesToCreate.Items))
	w.SetCount("instances_created", len(m.InstancesToCreate.Items))
	w.SetCount("updating_instances", len(m.InstancesToUpdate.Items))
	w.SetCount("instances_updated", len(m.InstancesToUpdate.Items))
	w.SetCount("deleting_instances", len(m.InstancesToDelete.Items))
	w.SetCount("instances_deleted", len(m.InstancesToDelete.Items))

	// Set firewall items
	w.SetCount("creating_firewalls", len(m.FirewallsToCreate.Items))
	w.SetCount("firewalls_created", len(m.FirewallsToCreate.Items))
	w.SetCount("updating_firewalls", len(m.FirewallsToUpdate.Items))
	w.SetCount("firewalls_updated", len(m.FirewallsToUpdate.Items))
	w.SetCount("deleting_firewalls", len(m.FirewallsToDelete.Items))
	w.SetCount("firewalls_deleted", len(m.FirewallsToDelete.Items))

	// Set nat items
	w.SetCount("creating_nats", len(m.NatsToCreate.Items))
	w.SetCount("nats_created", len(m.NatsToCreate.Items))
	w.SetCount("updating_nats", len(m.NatsToUpdate.Items))
	w.SetCount("nats_updated", len(m.NatsToUpdate.Items))
	w.SetCount("deleting_nats", len(m.NatsToDelete.Items))
	w.SetCount("nats_deleted", len(m.NatsToDelete.Items))

	// Optimize the graph, removing unused arcs/verticies
	w.Optimize()

	m.Workflow.Arcs = w.Arcs()

	return nil
}

// FindNetwork returns true if a router with a given name exists
func (m *FSMMessage) FindNetwork(name string) *Network {
	for i, network := range m.Networks.Items {
		if network.Name == name {
			return &m.Networks.Items[i]
		}
	}
	return nil
}

// FindInstance returns true if a router with a given name exists
func (m *FSMMessage) FindInstance(name string) *Instance {
	for i, instance := range m.Instances.Items {
		if instance.Name == name {
			return &m.Instances.Items[i]
		}
	}
	return nil
}

// FindFirewall returns true if a router with a given name exists
func (m *FSMMessage) FindFirewall(name string) *Firewall {
	for i, firewall := range m.Firewalls.Items {
		if firewall.Name == name {
			return &m.Firewalls.Items[i]
		}
	}
	return nil
}

// FindNat returns true if a router with a given name exists
func (m *FSMMessage) FindNat(name string) *Nat {
	for i, nat := range m.Nats.Items {
		if nat.Name == name {
			return &m.Nats.Items[i]
		}
	}
	return nil
}

// FilterNewInstances will return any new instances that match a certain pattern
func (m *FSMMessage) FilterNewInstances(name string) []Instance {
	var instances []Instance
	for _, instance := range m.InstancesToCreate.Items {
		if strings.Contains(instance.Name, name) {
			instances = append(instances, instance)
		}
	}
	return instances
}

// ToJSON : Get this service as a json
func (m *FSMMessage) ToJSON() []byte {
	json, _ := json.Marshal(m)

	return json
}
<commit_msg>[#ERNEST-2069] : Do not send repeated lists on components and components_to_create<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package output

import (
	"encoding/json"
	"strings"

	"github.com/r3labs/graph"
	"github.com/r3labs/workflow"
)

// FSMMessage is the JSON payload that will be sent to the FSM to create a
// service.
type FSMMessage struct {
	ID            string   `json:"id"`
	Body          string   `json:"body"`
	Endpoint      string   `json:"endpoint"`
	Service       string   `json:"service"`
	Bootstrapping string   `json:"bootstrapping"`
	ErnestIP      []string `json:"ernest_ip"`
	ServiceIP     string   `json:"service_ip"`
	Parent        string   `json:"existing_service"`
	Workflow      struct {
		Arcs []graph.Edge `json:"arcs"`
	} `json:"workflow"`
	ServiceName string `json:"name"`
	Client      string `json:"client"` // TODO: Use client or client_id not both!
	ClientID    string `json:"client_id"`
	ClientName  string `json:"client_name"`
	Started     string `json:"started"`
	Finished    string `json:"finished"`
	Status      string `json:"status"`
	Type        string `json:"type"`
	Datacenters struct {
		Started  string       `json:"started"`
		Finished string       `json:"finished"`
		Status   string       `json:"status"`
		Items    []Datacenter `json:"items"`
	} `json:"datacenters"`
	Networks struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks"`
	NetworksToCreate struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_create"`
	NetworksToDelete struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_delete"`
	Instances struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances"`
	InstancesToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_create"`
	InstancesToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_update"`
	InstancesToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_delete"`
	Firewalls struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls"`
	FirewallsToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_create"`
	FirewallsToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_update"`
	FirewallsToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_delete"`
	Nats struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats"`
	NatsToCreate struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_create"`
	NatsToUpdate struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_update"`
	NatsToDelete struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_delete"`
}

// Diff compares against an existing FSMMessage from a previous fsm message
func (m *FSMMessage) Diff(om FSMMessage) {
	// build new networks
	for _, network := range m.Networks.Items {
		if o := om.FindNetwork(network.Name); o == nil {
			m.NetworksToCreate.Items = append(m.NetworksToCreate.Items, network)
		}
	}

	// build old networks to delete
	for _, network := range om.Networks.Items {
		if m.FindNetwork(network.Name) == nil {
			m.NetworksToDelete.Items = append(m.NetworksToDelete.Items, network)
		}
	}

	// remove items to be created from the base
	networks := []Network{}
	for _, e := range m.Networks.Items {
		toBeCreated := false
		for _, c := range m.NetworksToCreate.Items {
			if e.Name == c.Name {
				toBeCreated = true
			}
		}
		if toBeCreated == false {
			networks = append(networks, e)
		}
	}
	m.Networks.Items = networks

	// build new and existing instances
	for _, instance := range m.Instances.Items {
		if oi := om.FindInstance(instance.Name); oi == nil {
			m.InstancesToCreate.Items = append(m.InstancesToCreate.Items, instance)
		} else if instance.HasChanged(oi) {
			m.InstancesToUpdate.Items = append(m.InstancesToUpdate.Items, instance)
		}
	}

	// build old instances to delete
	for _, instance := range om.Instances.Items {
		if m.FindInstance(instance.Name) == nil {
			m.InstancesToDelete.Items = append(m.InstancesToDelete.Items, instance)
		}
	}

	// remove items to be created from the base
	instances := []Instance{}
	for _, e := range m.Instances.Items {
		toBeCreated := false
		for _, c := range m.InstancesToCreate.Items {
			if e.Name == c.Name {
				toBeCreated = true
			}
		}
		if toBeCreated == false {
			instances = append(instances, e)
		}
	}
	m.Instances.Items = instances

	// build new and existing firewalls
	for _, firewall := range m.Firewalls.Items {
		if of := om.FindFirewall(firewall.Name); of == nil {
			m.FirewallsToCreate.Items = append(m.FirewallsToCreate.Items, firewall)
		} else if firewall.HasChanged(of) {
			m.FirewallsToUpdate.Items = append(m.FirewallsToUpdate.Items, firewall)
		}
	}

	// build old firewalls to delete
	for _, firewall := range om.Firewalls.Items {
		if m.FindFirewall(firewall.Name) == nil {
			m.FirewallsToDelete.Items = append(m.FirewallsToDelete.Items, firewall)
		}
	}

	// remove items to be created from the base
	firewalls := []Firewall{}
	for _, e := range m.Firewalls.Items {
		toBeCreated := false
		for _, c := range m.FirewallsToCreate.Items {
			if e.Name == c.Name {
				toBeCreated = true
			}
		}
		if toBeCreated == false {
			firewalls = append(firewalls, e)
		}
	}
	m.Firewalls.Items = firewalls

	// build new and existing nats
	for _, nat := range m.Nats.Items {
		if on := om.FindNat(nat.Name); on == nil {
			m.NatsToCreate.Items = append(m.NatsToCreate.Items, nat)
		} else if nat.HasChanged(on) {
			m.NatsToUpdate.Items = append(m.NatsToUpdate.Items, nat)
		}
	}

	// build old nats to delete
	for _, nat := range om.Nats.Items {
		if m.FindNat(nat.Name) == nil {
			m.NatsToDelete.Items = append(m.NatsToDelete.Items, nat)
		}
	}

	// remove items to be created from the base
	nats := []Nat{}
	for _, e := range m.Nats.Items {
		toBeCreated := false
		for _, c := range m.NatsToCreate.Items {
			if e.Name == c.Name {
				toBeCreated = true
			}
		}
		if toBeCreated == false {
			nats = append(nats, e)
		}
	}
	m.Nats.Items = nats

}

// GenerateWorkflow creates a fsm workflow based upon actionable tasks, such as creation or deletion of an entity.
func (m *FSMMessage) GenerateWorkflow(path string) error {
	w := workflow.New()
	err := w.LoadFile("./output/arcs/" + path)
	if err != nil {
		return err
	}

	// Set network items
	w.SetCount("creating_networks", len(m.NetworksToCreate.Items))
	w.SetCount("networks_created", len(m.NetworksToCreate.Items))
	w.SetCount("deleting_networks", len(m.NetworksToDelete.Items))
	w.SetCount("networks_deleted", len(m.NetworksToDelete.Items))

	// Set instance items
	w.SetCount("creating_instances", len(m.InstancesToCreate.Items))
	w.SetCount("instances_created", len(m.InstancesToCreate.Items))
	w.SetCount("updating_instances", len(m.InstancesToUpdate.Items))
	w.SetCount("instances_updated", len(m.InstancesToUpdate.Items))
	w.SetCount("deleting_instances", len(m.InstancesToDelete.Items))
	w.SetCount("instances_deleted", len(m.InstancesToDelete.Items))

	// Set firewall items
	w.SetCount("creating_firewalls", len(m.FirewallsToCreate.Items))
	w.SetCount("firewalls_created", len(m.FirewallsToCreate.Items))
	w.SetCount("updating_firewalls", len(m.FirewallsToUpdate.Items))
	w.SetCount("firewalls_updated", len(m.FirewallsToUpdate.Items))
	w.SetCount("deleting_firewalls", len(m.FirewallsToDelete.Items))
	w.SetCount("firewalls_deleted", len(m.FirewallsToDelete.Items))

	// Set nat items
	w.SetCount("creating_nats", len(m.NatsToCreate.Items))
	w.SetCount("nats_created", len(m.NatsToCreate.Items))
	w.SetCount("updating_nats", len(m.NatsToUpdate.Items))
	w.SetCount("nats_updated", len(m.NatsToUpdate.Items))
	w.SetCount("deleting_nats", len(m.NatsToDelete.Items))
	w.SetCount("nats_deleted", len(m.NatsToDelete.Items))

	// Optimize the graph, removing unused arcs/verticies
	w.Optimize()

	m.Workflow.Arcs = w.Arcs()

	return nil
}

// FindNetwork returns true if a router with a given name exists
func (m *FSMMessage) FindNetwork(name string) *Network {
	for i, network := range m.Networks.Items {
		if network.Name == name {
			return &m.Networks.Items[i]
		}
	}
	return nil
}

// FindInstance returns true if a router with a given name exists
func (m *FSMMessage) FindInstance(name string) *Instance {
	for i, instance := range m.Instances.Items {
		if instance.Name == name {
			return &m.Instances.Items[i]
		}
	}
	return nil
}

// FindFirewall returns true if a router with a given name exists
func (m *FSMMessage) FindFirewall(name string) *Firewall {
	for i, firewall := range m.Firewalls.Items {
		if firewall.Name == name {
			return &m.Firewalls.Items[i]
		}
	}
	return nil
}

// FindNat returns true if a router with a given name exists
func (m *FSMMessage) FindNat(name string) *Nat {
	for i, nat := range m.Nats.Items {
		if nat.Name == name {
			return &m.Nats.Items[i]
		}
	}
	return nil
}

// FilterNewInstances will return any new instances that match a certain pattern
func (m *FSMMessage) FilterNewInstances(name string) []Instance {
	var instances []Instance
	for _, instance := range m.InstancesToCreate.Items {
		if strings.Contains(instance.Name, name) {
			instances = append(instances, instance)
		}
	}
	return instances
}

// ToJSON : Get this service as a json
func (m *FSMMessage) ToJSON() []byte {
	json, _ := json.Marshal(m)

	return json
}
<|endoftext|>


<commit_before>package ozinit

import (
	"fmt"
	"os"
	"os/user"
	"path"
	"strconv"
	"syscall"

	"github.com/naegelejd/go-acl"
	"github.com/op/go-logging"

	"github.com/subgraph/oz/fs"
)

var basicBindDirs = []string{
	"/bin", "/lib", "/lib64", "/usr", "/etc",
}

var basicEmptyDirs = []string{
	"/boot", "/dev", "/home", "/media", "/mnt",
	"/opt", "/proc", "/root", "/run", "/run/lock", "/run/user",
	"/sbin", "/srv", "/sys", "/tmp", "/var", "/var/lib", "/var/lib/dbus",
	"/var/cache", "/var/crash", "/run/resolvconf",
}

var basicEmptyUserDirs = []string{
	"/run/dbus",
}

var basicSymlinks = [][2]string{
	{"/run", "/var/run"},
	{"/tmp", "/var/tmp"},
	{"/run/lock", "/var/lock"},
	{"/dev/shm", "/run/shm"},
}

var deviceSymlinks = [][2]string{
	{"/proc/self/fd", "/dev/fd"},
	{"/proc/self/fd/2", "/dev/stderr"},
	{"/proc/self/fd/0", "/dev/stdin"},
	{"/proc/self/fd/1", "/dev/stdout"},
	{"/dev/pts/ptmx", "/dev/ptmx"},
}

var basicBlacklist = []string{
	/*"${PATH}/dbus-daemon", "${PATH}/dbus-launch", "${PATH}/pulseaudio",*/
	"/usr/lib/gvfs",

	"/usr/sbin", "/sbin",

	"/etc/machine-id", "/etc/shadow", "/etc/shadow-",
	"${PATH}/sudo", "${PATH}/su",
	"${PATH}/xinput", "${PATH}/strace",
	"${PATH}/mount", "${PATH}/umount",
	"${PATH}/fusermount",
}

var basicWhiteList = []string{
	"${HOME}/.config/mimeapps.list",
}

/*	"/etc/X11",*/

type fsDeviceDefinition struct {
	path string
	mode uint32
	dev  int
	gid  int
}

const ugorw = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP | syscall.S_IWGRP | syscall.S_IROTH | syscall.S_IWOTH
const urwgr = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP
const urwgw = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IWGRP
const urw = syscall.S_IRUSR | syscall.S_IWUSR

var basicDevices = []fsDeviceDefinition{
	{path: "/dev/full", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 7)},
	{path: "/dev/null", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 3)},
	{path: "/dev/random", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 8)},

	{path: "/dev/console", mode: syscall.S_IFCHR | urw, dev: _makedev(5, 1)},
	{path: "/dev/tty", mode: syscall.S_IFCHR | ugorw, dev: _makedev(5, 0), gid: 5},
	{path: "/dev/tty0", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 0), gid: 5},
	{path: "/dev/tty1", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 1), gid: 5},
	{path: "/dev/tty2", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 2), gid: 5},
	{path: "/dev/tty3", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 3), gid: 5},
	{path: "/dev/tty4", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 4), gid: 5},
	{path: "/dev/tty5", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 5), gid: 5},

	{path: "/dev/urandom", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 9)},
	{path: "/dev/zero", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 5)},
}

func _makedev(x, y int) int {
	return (((x) << 8) | (y))
}

func setupRootfs(fsys *fs.Filesystem, user *user.User, uid, gid uint32, display int, useFullDev bool, log *logging.Logger) error {
	if err := os.MkdirAll(fsys.Root(), 0755); err != nil {
		return fmt.Errorf("could not create rootfs path '%s': %v", fsys.Root(), err)
	}

	if err := syscall.Mount("", "/", "", syscall.MS_PRIVATE|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("failed to set MS_PRIVATE on '%s': %v", "/", err)
	}

	flags := uintptr(syscall.MS_NOSUID | syscall.MS_NOEXEC | syscall.MS_NODEV)
	if err := syscall.Mount("", fsys.Root(), "tmpfs", flags, "mode=755,gid=0"); err != nil {
		return fmt.Errorf("failed to mount tmpfs on '%s': %v", fsys.Root(), err)
	}

	if err := syscall.Mount("", fsys.Root(), "", syscall.MS_PRIVATE, ""); err != nil {
		return fmt.Errorf("failed to set MS_PRIVATE on '%s': %v", fsys.Root(), err)
	}

	for _, p := range basicBindDirs {
		if err := fsys.BindPath(p, fs.BindReadOnly, display); err != nil {
			return fmt.Errorf("failed to bind directory '%s': %v", p, err)
		}
	}

	userMountDir := path.Join("/media", user.Username)
	basicEmptyDirs = append(basicEmptyDirs, userMountDir)
	for _, p := range basicEmptyDirs {
		//log.Debug("Creating empty dir: %s", p)
		if err := fsys.CreateEmptyDir(p); err != nil {
			return fmt.Errorf("failed to create empty directory '%s': %v", p, err)
		}
	}

	if err := setupMountDirectory(fsys, userMountDir); err != nil {
		return fmt.Errorf("failed to create mount directory: %v", err)
	}

	basicEmptyUserDirs = append(basicEmptyUserDirs, user.HomeDir)
	for _, p := range basicEmptyUserDirs {
		//log.Debug("Creating empty user dir: %s", p)
		if err := fsys.CreateEmptyDir(p); err != nil {
			return fmt.Errorf("failed to create empty user directory '%s': %v", p, err)
		}
		if err := os.Chown(path.Join(fsys.Root(), p), int(uid), int(gid)); err != nil {
			return fmt.Errorf("failed to chown user dir: %v", err)
		}
	}

	rup := path.Join(fsys.Root(), "/run/user", strconv.FormatUint(uint64(uid), 10))
	if err := os.MkdirAll(rup, 0700); err != nil {
		return fmt.Errorf("failed to create user rundir: %v", err)
	}
	if err := os.Chown(rup, int(uid), int(gid)); err != nil {
		return fmt.Errorf("failed to chown user rundir: %v", err)
	}

	dp := path.Join(fsys.Root(), "dev")
	if err := syscall.Mount("", dp, "tmpfs", syscall.MS_NOSUID|syscall.MS_NOEXEC, "mode=755"); err != nil {
		return err
	}

	if !useFullDev {
		for _, d := range basicDevices {
			if err := fsys.CreateDevice(d.path, d.dev, d.mode, d.gid); err != nil {
				return err
			}
		}

		smp := path.Join(fsys.Root(), "/dev", "/shm")
		smflags := uintptr(syscall.MS_NOSUID | syscall.MS_NOEXEC | syscall.MS_REC)
		if err := os.MkdirAll(smp, 0755); err != nil {
			return err
		}
		if err := syscall.Mount("", smp, "tmpfs", smflags, "mode=755"); err != nil {
			return err
		}
	}

	tp := path.Join(fsys.Root(), "/tmp")
	tflags := uintptr(syscall.MS_NODEV | syscall.MS_NOSUID | syscall.MS_NOEXEC | syscall.MS_REC)
	if err := syscall.Mount("", tp, "tmpfs", tflags, "mode=777"); err != nil {
		return err
	}

	for _, sl := range append(basicSymlinks, deviceSymlinks...) {
		if err := fsys.CreateSymlink(sl[0], sl[1]); err != nil {
			return err
		}
	}

	if err := fsys.CreateBlacklistPaths(); err != nil {
		return err
	}

	for _, bl := range basicBlacklist {
		if err := fsys.BlacklistPath(bl, display); err != nil {
			return err
			//log.Warning("Unable to blacklist %s: %v", bl, err)
		}
	}
	return nil
}

func setupMountDirectory(fsys *fs.Filesystem, src string) error {
	acls, err := acl.GetFileAccess(src)
	if err != nil {
		return err
	}
	defer acls.Free()

	target := path.Join(fsys.Root(), src)
	if err := acls.SetFileAccess(target); err != nil {
		return fmt.Errorf("%v on %s", err, target)
	}
	return nil
}
<commit_msg>Added /etc/fstab to basic blacklist<commit_after>package ozinit

import (
	"fmt"
	"os"
	"os/user"
	"path"
	"strconv"
	"syscall"

	"github.com/naegelejd/go-acl"
	"github.com/op/go-logging"

	"github.com/subgraph/oz/fs"
)

var basicBindDirs = []string{
	"/bin", "/lib", "/lib64", "/usr", "/etc",
}

var basicEmptyDirs = []string{
	"/boot", "/dev", "/home", "/media", "/mnt",
	"/opt", "/proc", "/root", "/run", "/run/lock", "/run/user",
	"/sbin", "/srv", "/sys", "/tmp", "/var", "/var/lib", "/var/lib/dbus",
	"/var/cache", "/var/crash", "/run/resolvconf",
}

var basicEmptyUserDirs = []string{
	"/run/dbus",
}

var basicSymlinks = [][2]string{
	{"/run", "/var/run"},
	{"/tmp", "/var/tmp"},
	{"/run/lock", "/var/lock"},
	{"/dev/shm", "/run/shm"},
}

var deviceSymlinks = [][2]string{
	{"/proc/self/fd", "/dev/fd"},
	{"/proc/self/fd/2", "/dev/stderr"},
	{"/proc/self/fd/0", "/dev/stdin"},
	{"/proc/self/fd/1", "/dev/stdout"},
	{"/dev/pts/ptmx", "/dev/ptmx"},
}

var basicBlacklist = []string{
	/*"${PATH}/dbus-daemon", "${PATH}/dbus-launch", "${PATH}/pulseaudio",*/
	"/usr/lib/gvfs",

	"/usr/sbin", "/sbin",

	"/etc/machine-id", "/etc/shadow", "/etc/shadow-", "/etc/fstab",
	"${PATH}/sudo", "${PATH}/su",
	"${PATH}/xinput", "${PATH}/strace",
	"${PATH}/mount", "${PATH}/umount",
	"${PATH}/fusermount",
}

var basicWhiteList = []string{
	"${HOME}/.config/mimeapps.list",
}

/*	"/etc/X11",*/

type fsDeviceDefinition struct {
	path string
	mode uint32
	dev  int
	gid  int
}

const ugorw = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP | syscall.S_IWGRP | syscall.S_IROTH | syscall.S_IWOTH
const urwgr = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IRGRP
const urwgw = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IWGRP
const urw = syscall.S_IRUSR | syscall.S_IWUSR

var basicDevices = []fsDeviceDefinition{
	{path: "/dev/full", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 7)},
	{path: "/dev/null", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 3)},
	{path: "/dev/random", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 8)},

	{path: "/dev/console", mode: syscall.S_IFCHR | urw, dev: _makedev(5, 1)},
	{path: "/dev/tty", mode: syscall.S_IFCHR | ugorw, dev: _makedev(5, 0), gid: 5},
	{path: "/dev/tty0", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 0), gid: 5},
	{path: "/dev/tty1", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 1), gid: 5},
	{path: "/dev/tty2", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 2), gid: 5},
	{path: "/dev/tty3", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 3), gid: 5},
	{path: "/dev/tty4", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 4), gid: 5},
	{path: "/dev/tty5", mode: syscall.S_IFCHR | urwgw, dev: _makedev(4, 5), gid: 5},

	{path: "/dev/urandom", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 9)},
	{path: "/dev/zero", mode: syscall.S_IFCHR | ugorw, dev: _makedev(1, 5)},
}

func _makedev(x, y int) int {
	return (((x) << 8) | (y))
}

func setupRootfs(fsys *fs.Filesystem, user *user.User, uid, gid uint32, display int, useFullDev bool, log *logging.Logger) error {
	if err := os.MkdirAll(fsys.Root(), 0755); err != nil {
		return fmt.Errorf("could not create rootfs path '%s': %v", fsys.Root(), err)
	}

	if err := syscall.Mount("", "/", "", syscall.MS_PRIVATE|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("failed to set MS_PRIVATE on '%s': %v", "/", err)
	}

	flags := uintptr(syscall.MS_NOSUID | syscall.MS_NOEXEC | syscall.MS_NODEV)
	if err := syscall.Mount("", fsys.Root(), "tmpfs", flags, "mode=755,gid=0"); err != nil {
		return fmt.Errorf("failed to mount tmpfs on '%s': %v", fsys.Root(), err)
	}

	if err := syscall.Mount("", fsys.Root(), "", syscall.MS_PRIVATE, ""); err != nil {
		return fmt.Errorf("failed to set MS_PRIVATE on '%s': %v", fsys.Root(), err)
	}

	for _, p := range basicBindDirs {
		if err := fsys.BindPath(p, fs.BindReadOnly, display); err != nil {
			return fmt.Errorf("failed to bind directory '%s': %v", p, err)
		}
	}

	userMountDir := path.Join("/media", user.Username)
	basicEmptyDirs = append(basicEmptyDirs, userMountDir)
	for _, p := range basicEmptyDirs {
		//log.Debug("Creating empty dir: %s", p)
		if err := fsys.CreateEmptyDir(p); err != nil {
			return fmt.Errorf("failed to create empty directory '%s': %v", p, err)
		}
	}

	if err := setupMountDirectory(fsys, userMountDir); err != nil {
		return fmt.Errorf("failed to create mount directory: %v", err)
	}

	basicEmptyUserDirs = append(basicEmptyUserDirs, user.HomeDir)
	for _, p := range basicEmptyUserDirs {
		//log.Debug("Creating empty user dir: %s", p)
		if err := fsys.CreateEmptyDir(p); err != nil {
			return fmt.Errorf("failed to create empty user directory '%s': %v", p, err)
		}
		if err := os.Chown(path.Join(fsys.Root(), p), int(uid), int(gid)); err != nil {
			return fmt.Errorf("failed to chown user dir: %v", err)
		}
	}

	rup := path.Join(fsys.Root(), "/run/user", strconv.FormatUint(uint64(uid), 10))
	if err := os.MkdirAll(rup, 0700); err != nil {
		return fmt.Errorf("failed to create user rundir: %v", err)
	}
	if err := os.Chown(rup, int(uid), int(gid)); err != nil {
		return fmt.Errorf("failed to chown user rundir: %v", err)
	}

	dp := path.Join(fsys.Root(), "dev")
	if err := syscall.Mount("", dp, "tmpfs", syscall.MS_NOSUID|syscall.MS_NOEXEC, "mode=755"); err != nil {
		return err
	}

	if !useFullDev {
		for _, d := range basicDevices {
			if err := fsys.CreateDevice(d.path, d.dev, d.mode, d.gid); err != nil {
				return err
			}
		}

		smp := path.Join(fsys.Root(), "/dev", "/shm")
		smflags := uintptr(syscall.MS_NOSUID | syscall.MS_NOEXEC | syscall.MS_REC)
		if err := os.MkdirAll(smp, 0755); err != nil {
			return err
		}
		if err := syscall.Mount("", smp, "tmpfs", smflags, "mode=755"); err != nil {
			return err
		}
	}

	tp := path.Join(fsys.Root(), "/tmp")
	tflags := uintptr(syscall.MS_NODEV | syscall.MS_NOSUID | syscall.MS_NOEXEC | syscall.MS_REC)
	if err := syscall.Mount("", tp, "tmpfs", tflags, "mode=777"); err != nil {
		return err
	}

	for _, sl := range append(basicSymlinks, deviceSymlinks...) {
		if err := fsys.CreateSymlink(sl[0], sl[1]); err != nil {
			return err
		}
	}

	if err := fsys.CreateBlacklistPaths(); err != nil {
		return err
	}

	for _, bl := range basicBlacklist {
		if err := fsys.BlacklistPath(bl, display); err != nil {
			return err
			//log.Warning("Unable to blacklist %s: %v", bl, err)
		}
	}
	return nil
}

func setupMountDirectory(fsys *fs.Filesystem, src string) error {
	acls, err := acl.GetFileAccess(src)
	if err != nil {
		return err
	}
	defer acls.Free()

	target := path.Join(fsys.Root(), src)
	if err := acls.SetFileAccess(target); err != nil {
		return fmt.Errorf("%v on %s", err, target)
	}
	return nil
}
<|endoftext|>


<commit_before>package topology

import (
	"context"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/storage/needle"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

func batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
	ch := make(chan int, locationlist.Length())
	errCount := int32(0)
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
					VolumeId: uint32(vid),
				})
				if err != nil {
					atomic.AddInt32(&errCount, 1)
					ch <- -1
					return err
				}
				if resp.GarbageRatio >= garbageThreshold {
					ch <- index
				} else {
					ch <- -1
				}
				return nil
			})
			if err != nil {
				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
			}
		}(index, dn.Url(), vid)
	}
	vacuumLocationList := NewVolumeLocationList()
	for range locationlist.list {
		select {
		case index := <-ch:
			if index != -1 {
				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
			}
		case <-time.After(30 * time.Minute):
			return vacuumLocationList, false
		}
	}
	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
}
func batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, preallocate int64) bool {
	vl.accessLock.Lock()
	vl.removeFromWritable(vid)
	vl.accessLock.Unlock()

	ch := make(chan bool, locationlist.Length())
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				_, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
					VolumeId: uint32(vid),
				})
				return err
			})
			if err != nil {
				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
				ch <- false
			} else {
				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
				ch <- true
			}
		}(index, dn.Url(), vid)
	}
	isVacuumSuccess := true
	for range locationlist.list {
		select {
		case canCommit := <-ch:
			isVacuumSuccess = isVacuumSuccess && canCommit
		case <-time.After(30 * time.Minute):
			return false
		}
	}
	return isVacuumSuccess
}
func batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
	isCommitSuccess := true
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
			isCommitSuccess = false
		} else {
			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
		}
	}
	if isCommitSuccess {
		for _, dn := range locationlist.list {
			vl.SetVolumeAvailable(dn, vid)
		}
	}
	return isCommitSuccess
}
func batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
		} else {
			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
		}
	}
}

func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) int {

	// if there is vacuum going on, return immediately
	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
	if !swapped {
		return 0
	}
	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)

	// now only one vacuum process going on

	glog.V(1).Infof("Start vacuum on demand with threshold: %f", garbageThreshold)
	for _, col := range t.collectionMap.Items() {
		c := col.(*Collection)
		for _, vl := range c.storageType2VolumeLayout.Items() {
			if vl != nil {
				volumeLayout := vl.(*VolumeLayout)
				vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
			}
		}
	}
	return 0
}

func vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

	volumeLayout.accessLock.RLock()
	tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
	for vid, locationList := range volumeLayout.vid2location {
		tmpMap[vid] = locationList
	}
	volumeLayout.accessLock.RUnlock()

	for vid, locationList := range tmpMap {

		volumeLayout.accessLock.RLock()
		isReadOnly, hasValue := volumeLayout.readonlyVolumes[vid]
		volumeLayout.accessLock.RUnlock()

		if hasValue && isReadOnly {
			continue
		}

		glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
		if vacuumLocationList, needVacuum := batchVacuumVolumeCheck(
			grpcDialOption, volumeLayout, vid, locationList, garbageThreshold); needVacuum {
			if batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
				batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			} else {
				batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			}
		}
	}
}
<commit_msg>volume: vacuum pass preallocate variable<commit_after>package topology

import (
	"context"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/storage/needle"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

func batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
	ch := make(chan int, locationlist.Length())
	errCount := int32(0)
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
					VolumeId: uint32(vid),
				})
				if err != nil {
					atomic.AddInt32(&errCount, 1)
					ch <- -1
					return err
				}
				if resp.GarbageRatio >= garbageThreshold {
					ch <- index
				} else {
					ch <- -1
				}
				return nil
			})
			if err != nil {
				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
			}
		}(index, dn.Url(), vid)
	}
	vacuumLocationList := NewVolumeLocationList()
	for range locationlist.list {
		select {
		case index := <-ch:
			if index != -1 {
				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
			}
		case <-time.After(30 * time.Minute):
			return vacuumLocationList, false
		}
	}
	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
}
func batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, preallocate int64) bool {
	vl.accessLock.Lock()
	vl.removeFromWritable(vid)
	vl.accessLock.Unlock()

	ch := make(chan bool, locationlist.Length())
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				_, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
					VolumeId:    uint32(vid),
					Preallocate: preallocate,
				})
				return err
			})
			if err != nil {
				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
				ch <- false
			} else {
				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
				ch <- true
			}
		}(index, dn.Url(), vid)
	}
	isVacuumSuccess := true
	for range locationlist.list {
		select {
		case canCommit := <-ch:
			isVacuumSuccess = isVacuumSuccess && canCommit
		case <-time.After(30 * time.Minute):
			return false
		}
	}
	return isVacuumSuccess
}
func batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
	isCommitSuccess := true
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
			isCommitSuccess = false
		} else {
			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
		}
	}
	if isCommitSuccess {
		for _, dn := range locationlist.list {
			vl.SetVolumeAvailable(dn, vid)
		}
	}
	return isCommitSuccess
}
func batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
		} else {
			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
		}
	}
}

func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) int {

	// if there is vacuum going on, return immediately
	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
	if !swapped {
		return 0
	}
	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)

	// now only one vacuum process going on

	glog.V(1).Infof("Start vacuum on demand with threshold: %f", garbageThreshold)
	for _, col := range t.collectionMap.Items() {
		c := col.(*Collection)
		for _, vl := range c.storageType2VolumeLayout.Items() {
			if vl != nil {
				volumeLayout := vl.(*VolumeLayout)
				vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
			}
		}
	}
	return 0
}

func vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

	volumeLayout.accessLock.RLock()
	tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
	for vid, locationList := range volumeLayout.vid2location {
		tmpMap[vid] = locationList
	}
	volumeLayout.accessLock.RUnlock()

	for vid, locationList := range tmpMap {

		volumeLayout.accessLock.RLock()
		isReadOnly, hasValue := volumeLayout.readonlyVolumes[vid]
		volumeLayout.accessLock.RUnlock()

		if hasValue && isReadOnly {
			continue
		}

		glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
		if vacuumLocationList, needVacuum := batchVacuumVolumeCheck(
			grpcDialOption, volumeLayout, vid, locationList, garbageThreshold); needVacuum {
			if batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
				batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			} else {
				batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			}
		}
	}
}
<|endoftext|>