<commit_before>package manifest_test

import (
	"cf/manifest"
	"generic"
	"github.com/stretchr/testify/assert"
	"testing"
	testassert "testhelpers/assert"
	"strings"
)

func TestManifestWithGlobalAndAppSpecificProperties(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512M",
		"applications": []interface{}{
			map[string]interface{}{
				"name":     "bitcoin-miner",
				"no-route": true,
			},
		},
	}))
	assert.NoError(t, err)

	apps := m.Applications
	assert.Equal(t, apps[0].Get("instances"), 3)
	assert.Equal(t, apps[0].Get("memory").(uint64), uint64(512))
	assert.True(t, apps[0].Get("no-route").(bool))
}

func TestManifestWithInvalidMemory(t *testing.T) {
	_, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512",
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bitcoin-miner",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "memory")
}

func TestParsingManifestWithTimeoutSetsHealthCheckTimeout(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"name":    "bitcoin-miner",
				"timeout": "360",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("health_check_timeout"), 360)
	assert.False(t, m.Applications[0].Has("timeout"))
}

func TestParsingManifestWithEmptyEnvVarIsInvalid(t *testing.T) {
	_, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"env": map[string]interface{}{
			"bar": nil,
		},
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bad app",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "env var 'bar' should not be null")
}

func TestParsingManifestWithNulls(t *testing.T) {
	_, errs := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"buildpack": nil,
				"disk_quota": nil,
				"domain": nil,
				"host": nil,
				"name": nil,
				"path": nil,
				"stack": nil,
				"memory": nil,
				"instances": nil,
				"timeout": nil,
				"no-route": nil,
				"services": nil,
				"env": nil,
			},
		},
	}))

	assert.Error(t, errs)
	errorSlice  := strings.Split(errs.Error(), "\n")
	manifestKeys := []string{"buildpack", "disk_quota", "domain", "host", "name", "path", "stack",
							 "memory", "instances", "timeout", "no-route", "services", "env"}

	for _, key := range manifestKeys {
		testassert.SliceContains(t, errorSlice, testassert.Lines{{key, "not be null"}})
	}
}

func TestParsingManifestWithPropertiesReturnsErrors(t *testing.T) {
	_, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"env": map[string]interface{}{
					"bar": "many-${foo}-are-cool",
				},
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "Properties are not supported. Found property '${foo}'")
}

func TestParsingManifestWithNullCommand(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"command": nil,
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("command"), "")
}

func TestParsingEmptyManifestDoesNotSetCommand(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{},
		},
	}))

	assert.NoError(t, err)
	assert.False(t, m.Applications[0].Has("command"))
}
<commit_msg>go fmt<commit_after>package manifest_test

import (
	"cf/manifest"
	"generic"
	"github.com/stretchr/testify/assert"
	"strings"
	testassert "testhelpers/assert"
	"testing"
)

func TestManifestWithGlobalAndAppSpecificProperties(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512M",
		"applications": []interface{}{
			map[string]interface{}{
				"name":     "bitcoin-miner",
				"no-route": true,
			},
		},
	}))
	assert.NoError(t, err)

	apps := m.Applications
	assert.Equal(t, apps[0].Get("instances"), 3)
	assert.Equal(t, apps[0].Get("memory").(uint64), uint64(512))
	assert.True(t, apps[0].Get("no-route").(bool))
}

func TestManifestWithInvalidMemory(t *testing.T) {
	_, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"instances": "3",
		"memory":    "512",
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bitcoin-miner",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "memory")
}

func TestParsingManifestWithTimeoutSetsHealthCheckTimeout(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"name":    "bitcoin-miner",
				"timeout": "360",
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("health_check_timeout"), 360)
	assert.False(t, m.Applications[0].Has("timeout"))
}

func TestParsingManifestWithEmptyEnvVarIsInvalid(t *testing.T) {
	_, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"env": map[string]interface{}{
			"bar": nil,
		},
		"applications": []interface{}{
			map[string]interface{}{
				"name": "bad app",
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "env var 'bar' should not be null")
}

func TestParsingManifestWithNulls(t *testing.T) {
	_, errs := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"buildpack":  nil,
				"disk_quota": nil,
				"domain":     nil,
				"host":       nil,
				"name":       nil,
				"path":       nil,
				"stack":      nil,
				"memory":     nil,
				"instances":  nil,
				"timeout":    nil,
				"no-route":   nil,
				"services":   nil,
				"env":        nil,
			},
		},
	}))

	assert.Error(t, errs)
	errorSlice := strings.Split(errs.Error(), "\n")
	manifestKeys := []string{"buildpack", "disk_quota", "domain", "host", "name", "path", "stack",
		"memory", "instances", "timeout", "no-route", "services", "env"}

	for _, key := range manifestKeys {
		testassert.SliceContains(t, errorSlice, testassert.Lines{{key, "not be null"}})
	}
}

func TestParsingManifestWithPropertiesReturnsErrors(t *testing.T) {
	_, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"env": map[string]interface{}{
					"bar": "many-${foo}-are-cool",
				},
			},
		},
	}))

	assert.Error(t, err)
	assert.Contains(t, err.Error(), "Properties are not supported. Found property '${foo}'")
}

func TestParsingManifestWithNullCommand(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{
				"command": nil,
			},
		},
	}))

	assert.NoError(t, err)
	assert.Equal(t, m.Applications[0].Get("command"), "")
}

func TestParsingEmptyManifestDoesNotSetCommand(t *testing.T) {
	m, err := manifest.NewManifest(generic.NewMap(map[string]interface{}{
		"applications": []interface{}{
			map[string]interface{}{},
		},
	}))

	assert.NoError(t, err)
	assert.False(t, m.Applications[0].Has("command"))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// KubectlBuilder install kubectl
type KubectlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &KubectlBuilder{}

func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
	if !b.IsMaster {
		// We don't have the configuration on the machines, so it only works on the master anyway
		return nil
	}

	// Add kubectl file as an asset
	{
		// TODO: Extract to common function?
		assetName := "kubectl"
		assetPath := ""
		asset, err := b.Assets.Find(assetName, assetPath)
		if err != nil {
			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
		}
		if asset == nil {
			return fmt.Errorf("unable to locate asset %q", assetName)
		}

		t := &nodetasks.File{
			Path:     b.kubectlPath(),
			Contents: asset,
			Type:     nodetasks.FileType_File,
			Mode:     s("0755"),
		}
		c.AddTask(t)
	}

	// Add kubeconfig
	{
		kubeconfig, err := b.buildPKIKubeconfig("kubecfg")
		if err != nil {
			return err
		}

		t := &nodetasks.File{
			Path:     "/var/lib/kubectl/kubeconfig",
			Contents: fi.NewStringResource(kubeconfig),
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		}
		c.AddTask(t)

		switch b.Distribution {
		case distros.DistributionJessie:
			c.AddTask(&nodetasks.File{
				Path:     "/home/admin/.kube/config",
				Contents: fi.NewStringResource(kubeconfig),
				Type:     nodetasks.FileType_File,
				Mode:     s("0400"),
				Owner:    s("admin"),
				Group:    s("admin"),
			})

		default:
			glog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
		}
	}

	return nil
}

func (b *KubectlBuilder) kubectlPath() string {
	kubeletCommand := "/usr/local/bin/kubectl"
	if b.Distribution == distros.DistributionCoreOS {
		kubeletCommand = "/opt/bin/kubectl"
	}
	if b.Distribution == distros.DistributionContainerOS {
		kubeletCommand = "/home/kubernetes/bin/kubectl"
	}
	return kubeletCommand
}
<commit_msg>[Fixes #2466] Creates .kube dir at /home/admin with group/owner as admin<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// KubectlBuilder install kubectl
type KubectlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &KubectlBuilder{}

func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
	if !b.IsMaster {
		// We don't have the configuration on the machines, so it only works on the master anyway
		return nil
	}

	// Add kubectl file as an asset
	{
		// TODO: Extract to common function?
		assetName := "kubectl"
		assetPath := ""
		asset, err := b.Assets.Find(assetName, assetPath)
		if err != nil {
			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
		}
		if asset == nil {
			return fmt.Errorf("unable to locate asset %q", assetName)
		}

		t := &nodetasks.File{
			Path:     b.kubectlPath(),
			Contents: asset,
			Type:     nodetasks.FileType_File,
			Mode:     s("0755"),
		}
		c.AddTask(t)
	}

	// Add kubeconfig
	{
		kubeconfig, err := b.buildPKIKubeconfig("kubecfg")
		if err != nil {
			return err
		}

		t := &nodetasks.File{
			Path:     "/var/lib/kubectl/kubeconfig",
			Contents: fi.NewStringResource(kubeconfig),
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		}
		c.AddTask(t)

		switch b.Distribution {
		case distros.DistributionJessie:
			c.AddTask(&nodetasks.File{
				Path:  "/home/admin/.kube/",
				Type:  nodetasks.FileType_Directory,
				Mode:  s("0400"),
				Owner: s("admin"),
				Group: s("admin"),
			})

			c.AddTask(&nodetasks.File{
				Path:     "/home/admin/.kube/config",
				Contents: fi.NewStringResource(kubeconfig),
				Type:     nodetasks.FileType_File,
				Mode:     s("0400"),
				Owner:    s("admin"),
				Group:    s("admin"),
			})

		default:
			glog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
		}
	}

	return nil
}

func (b *KubectlBuilder) kubectlPath() string {
	kubeletCommand := "/usr/local/bin/kubectl"
	if b.Distribution == distros.DistributionCoreOS {
		kubeletCommand = "/opt/bin/kubectl"
	}
	if b.Distribution == distros.DistributionContainerOS {
		kubeletCommand = "/home/kubernetes/bin/kubectl"
	}
	return kubeletCommand
}
<|endoftext|>


<commit_before>// Copyright ©2011-2012 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package matrix provides basic linear algebra operations.
package matrix

import (
	"math"
)

const (
	Cols = true
	Rows = !Cols
)

const (
	Inf = int(^uint(0) >> 1)
	Fro = -Inf - 1
)

type FloatFunc func() float64
type FilterFunc func(r, c int, v float64) bool
type ApplyFunc func(r, c int, v float64) float64

type Matrix interface {
	Clone(c Matrix) Matrix
	Dims() (int, int)
	At(r, c int) float64
	Norm(int) float64
	T(c Matrix) Matrix
	Det() float64
	Add(b, c Matrix) Matrix
	Sub(b, c Matrix) Matrix
	MulElem(b, c Matrix) Matrix
	Equals(b Matrix) bool
	EqualsApprox(b Matrix, epsilon float64) bool
	Scalar(f float64, c Matrix) Matrix
	Sum() (s float64)
	Dot(b, c Matrix) Matrix
	Inner(b Matrix) float64
	Stack(b, c Matrix) Matrix
	Augment(b, c Matrix) Matrix
	Apply(f ApplyFunc, c Matrix) Matrix
	ApplyAll(f ApplyFunc, c Matrix) Matrix
	Filter(f FilterFunc, c Matrix) Matrix
	Trace() float64
	U(Matrix) Matrix
	L(Matrix) Matrix
	Sparse(*Sparse) *Sparse
	Dense(*Dense) *Dense
}

type Mutable interface {
	Matrix
	New(r, c int) (Matrix, error)
	Set(r, c int, v float64)
}

// A Panicker is a function that returns a matrix and may panic.
type Panicker func() Matrix

// Maybe will recover a panic with a type matrix.Error from fn, and return this error.
// Any other error is re-panicked.
func Maybe(fn Panicker) (m Matrix, err error) {
	defer func() {
		if r := recover(); r != nil {
			var ok bool
			if err, ok = r.(Error); ok {
				return
			}
			panic(r)
		}
	}()
	return fn(), nil
}

// A FloatPanicker is a function that returns a float64 and may panic.
type FloatPanicker func() float64

// MaybeFloat will recover a panic with a type matrix.Error from fn, and return this error.
// Any other error is re-panicked.
func MaybeFloat(fn FloatPanicker) (f float64, err error) {
	defer func() {
		if r := recover(); r != nil {
			if e, ok := r.(Error); ok {
				err = e
				return
			}
			panic(r)
		}
	}()
	return fn(), nil
}

// Must can be used to wrap a function returning a matrix and an error.
// If the returned error is not nil, Must will panic.
func Must(m Matrix, err error) Matrix {
	if err != nil {
		panic(err)
	}
	return m
}

// Type Error represents matrix package errors. These errors can be recovered by Maybe wrappers.
type Error string

func (err Error) Error() string { return string(err) }

const (
	ErrIndexOutOfRange = Error("matrix: index out of range")
	ErrZeroLength      = Error("matrix: zero length in matrix definition")
	ErrRowLength       = Error("matrix: row length mismatch")
	ErrColLength       = Error("matrix: col length mismatch")
	ErrSquare          = Error("matrix: expect square matrix")
	ErrNormOrder       = Error("matrix: invalid norm order for matrix")
	ErrShape           = Error("matrix: dimension mismatch")
	ErrPivot           = Error("matrix: malformed pivot list")
)

// ElementsVector returns the matrix's elements concatenated, row-wise, into a float slice.
func ElementsVector(mats ...Matrix) []float64 {
	var length int
	for _, m := range mats {
		switch m := m.(type) {
		case *Dense:
			length += len(m.matrix)
		case *Sparse:
			for _, row := range m.matrix {
				length += len(row)
			}
		}
	}

	v := make([]float64, 0, length)
	for _, m := range mats {
		switch m := m.(type) {
		case *Dense:
			v = append(v, m.matrix...)
		case *Sparse:
			for _, row := range m.matrix {
				for _, e := range row {
					if e.value != 0 {
						v = append(v, e.value)
					}
				}
			}
		case Matrix:
			rows, cols := m.Dims()
			for r := 0; r < rows; r++ {
				for c := 0; c < cols; c++ {
					v = append(v, m.At(r, c))
				}
			}
		}
	}

	return v
}

// Determine a variety of norms on a vector.
func Norm(v []float64, ord int) float64 {
	var n float64
	switch ord {
	case 0:
		for _, e := range v {
			if e != 0 {
				n += e
			}
		}
	case Inf:
		for _, e := range v {
			n = math.Max(math.Abs(e), n)
		}
	case -Inf:
		n = math.MaxFloat64
		for _, e := range v {
			n = math.Min(math.Abs(e), n)
		}
	case Fro, 2:
		for _, e := range v {
			n += e * e
		}
		return math.Sqrt(n)
	default:
		ord := float64(ord)
		for _, e := range v {
			n += math.Pow(math.Abs(e), ord)
		}
		return math.Pow(n, 1/ord)
	}
	return n
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>Missed in previous commit<commit_after>// Copyright ©2011-2012 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package matrix provides basic linear algebra operations.
package matrix

import (
	"math"
)

const (
	Cols = true
	Rows = !Cols
)

const (
	Inf = int(^uint(0) >> 1)
	Fro = -Inf - 1
)

type FloatFunc func() float64
type FilterFunc func(r, c int, v float64) bool
type ApplyFunc func(r, c int, v float64) float64

type Matrix interface {
	Clone(c Matrix) Matrix
	Dims() (int, int)
	At(r, c int) float64
	Norm(int) float64
	T(c Matrix) Matrix
	Det() float64
	Add(b, c Matrix) Matrix
	Sub(b, c Matrix) Matrix
	MulElem(b, c Matrix) Matrix
	Equals(b Matrix) bool
	EqualsApprox(b Matrix, epsilon float64) bool
	Scalar(f float64, c Matrix) Matrix
	Sum() (s float64)
	Dot(b, c Matrix) Matrix
	Inner(b Matrix) float64
	Stack(b, c Matrix) Matrix
	Augment(b, c Matrix) Matrix
	Apply(f ApplyFunc, c Matrix) Matrix
	ApplyAll(f ApplyFunc, c Matrix) Matrix
	Filter(f FilterFunc, c Matrix) Matrix
	Trace() float64
	U(Matrix) Matrix
	L(Matrix) Matrix
	Sparse(*Sparse) *Sparse
	Dense(*Dense) *Dense
}

type Mutable interface {
	Matrix
	New(r, c int) (Matrix, error)
	Set(r, c int, v float64)
}

// A Panicker is a function that returns a matrix and may panic.
type Panicker func() Matrix

// Maybe will recover a panic with a type matrix.Error from fn, and return this error.
// Any other error is re-panicked.
func Maybe(fn Panicker) (m Matrix, err error) {
	defer func() {
		if r := recover(); r != nil {
			var ok bool
			if err, ok = r.(Error); ok {
				return
			}
			panic(r)
		}
	}()
	return fn(), nil
}

// A FloatPanicker is a function that returns a float64 and may panic.
type FloatPanicker func() float64

// MaybeFloat will recover a panic with a type matrix.Error from fn, and return this error.
// Any other error is re-panicked.
func MaybeFloat(fn FloatPanicker) (f float64, err error) {
	defer func() {
		if r := recover(); r != nil {
			if e, ok := r.(Error); ok {
				err = e
				return
			}
			panic(r)
		}
	}()
	return fn(), nil
}

// Must can be used to wrap a function returning a matrix and an error.
// If the returned error is not nil, Must will panic.
func Must(m Matrix, err error) Matrix {
	if err != nil {
		panic(err)
	}
	return m
}

// Type Error represents matrix package errors. These errors can be recovered by Maybe wrappers.
type Error string

func (err Error) Error() string { return string(err) }

const (
	ErrIndexOutOfRange = Error("matrix: index out of range")
	ErrZeroLength      = Error("matrix: zero length in matrix definition")
	ErrRowLength       = Error("matrix: row length mismatch")
	ErrColLength       = Error("matrix: col length mismatch")
	ErrSquare          = Error("matrix: expect square matrix")
	ErrNormOrder       = Error("matrix: invalid norm order for matrix")
	ErrShape           = Error("matrix: dimension mismatch")
	ErrIllegalStride   = Error("matrix: illegal stride")
	ErrPivot           = Error("matrix: malformed pivot list")
)

// ElementsVector returns the matrix's elements concatenated, row-wise, into a float slice.
func ElementsVector(mats ...Matrix) []float64 {
	var length int
	for _, m := range mats {
		switch m := m.(type) {
		case *Dense:
			length += len(m.matrix)
		case *Sparse:
			for _, row := range m.matrix {
				length += len(row)
			}
		}
	}

	v := make([]float64, 0, length)
	for _, m := range mats {
		switch m := m.(type) {
		case *Dense:
			v = append(v, m.matrix...)
		case *Sparse:
			for _, row := range m.matrix {
				for _, e := range row {
					if e.value != 0 {
						v = append(v, e.value)
					}
				}
			}
		case Matrix:
			rows, cols := m.Dims()
			for r := 0; r < rows; r++ {
				for c := 0; c < cols; c++ {
					v = append(v, m.At(r, c))
				}
			}
		}
	}

	return v
}

// Determine a variety of norms on a vector.
func Norm(v []float64, ord int) float64 {
	var n float64
	switch ord {
	case 0:
		for _, e := range v {
			if e != 0 {
				n += e
			}
		}
	case Inf:
		for _, e := range v {
			n = math.Max(math.Abs(e), n)
		}
	case -Inf:
		n = math.MaxFloat64
		for _, e := range v {
			n = math.Min(math.Abs(e), n)
		}
	case Fro, 2:
		for _, e := range v {
			n += e * e
		}
		return math.Sqrt(n)
	default:
		ord := float64(ord)
		for _, e := range v {
			n += math.Pow(math.Abs(e), ord)
		}
		return math.Pow(n, 1/ord)
	}
	return n
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package gxutil

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
)

type PackageBase struct {
	Name         string        `json:"name,omitempty"`
	Author       string        `json:"author,omitempty"`
	Description  string        `json:"description,omitempty"`
	Keywords     string        `json:"keywords,omitempty"`
	Version      string        `json:"version,omitempty"`
	Dependencies []*Dependency `json:"gxDependencies,omitempty"`
	Bin          string        `json:"bin,omitempty"`
	Build        string        `json:"build,omitempty"`
	Test         string        `json:"test,omitempty"`
	Language     string        `json:"language,omitempty"`
	Copyright    string        `json:"copyright,omitempty"`
	Issues       string        `json:"issues_url"`
}

type Package struct {
	PackageBase

	Gx json.RawMessage `json:"gx,omitempty"`
}

// Dependency represents a dependency of a package
type Dependency struct {
	Author  string `json:"author,omitempty"`
	Name    string `json:"name,omitempty"`
	Hash    string `json:"hash"`
	Version string `json:"version,omitempty"`
}

func LoadPackageFile(pkg interface{}, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}

	dec := json.NewDecoder(fi)
	err = dec.Decode(pkg)
	if err != nil {
		return err
	}

	return nil
}

func SavePackageFile(pkg interface{}, fname string) error {
	fi, err := os.Create(fname)
	if err != nil {
		return err
	}
	defer fi.Close()

	out, err := json.MarshalIndent(pkg, "", "  ")
	if err != nil {
		return err
	}
	_, err = fi.Write(out)
	return err
}

func (pkg *Package) FindDep(ref string) *Dependency {
	for _, d := range pkg.Dependencies {
		if d.Hash == ref || d.Name == ref {
			return d
		}
	}
	return nil
}

func (pkg *Package) ForEachDep(cb func(dep *Dependency, pkg *Package) error) error {
	local, err := InstallPath(pkg.Language, "", false)
	if err != nil {
		return err
	}

	global, err := InstallPath(pkg.Language, "", true)
	if err != nil {
		return err
	}

	for _, dep := range pkg.Dependencies {
		var pkg Package
		err := FindPackageInDir(&pkg, filepath.Join(local, "gx", "ipfs", dep.Hash))
		if err == nil {
			err := cb(dep, &pkg)
			if err != nil {
				return err
			}

			continue
		}
		err = FindPackageInDir(&pkg, filepath.Join(global, "gx", "ipfs", dep.Hash))
		if err != nil {
			return fmt.Errorf("package %s (%s) not found", dep.Name, dep.Hash)
		}

		err = cb(dep, &pkg)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>move package methods to base struct<commit_after>package gxutil

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
)

type PackageBase struct {
	Name         string        `json:"name,omitempty"`
	Author       string        `json:"author,omitempty"`
	Description  string        `json:"description,omitempty"`
	Keywords     string        `json:"keywords,omitempty"`
	Version      string        `json:"version,omitempty"`
	Dependencies []*Dependency `json:"gxDependencies,omitempty"`
	Bin          string        `json:"bin,omitempty"`
	Build        string        `json:"build,omitempty"`
	Test         string        `json:"test,omitempty"`
	Language     string        `json:"language,omitempty"`
	Copyright    string        `json:"copyright,omitempty"`
	Issues       string        `json:"issues_url"`
}

type Package struct {
	PackageBase

	Gx json.RawMessage `json:"gx,omitempty"`
}

// Dependency represents a dependency of a package
type Dependency struct {
	Author  string `json:"author,omitempty"`
	Name    string `json:"name,omitempty"`
	Hash    string `json:"hash"`
	Version string `json:"version,omitempty"`
}

func LoadPackageFile(pkg interface{}, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}

	dec := json.NewDecoder(fi)
	err = dec.Decode(pkg)
	if err != nil {
		return err
	}

	return nil
}

func SavePackageFile(pkg interface{}, fname string) error {
	fi, err := os.Create(fname)
	if err != nil {
		return err
	}
	defer fi.Close()

	out, err := json.MarshalIndent(pkg, "", "  ")
	if err != nil {
		return err
	}
	_, err = fi.Write(out)
	return err
}

func (pkg *PackageBase) FindDep(ref string) *Dependency {
	for _, d := range pkg.Dependencies {
		if d.Hash == ref || d.Name == ref {
			return d
		}
	}
	return nil
}

func (pkg *PackageBase) ForEachDep(cb func(dep *Dependency, pkg *Package) error) error {
	local, err := InstallPath(pkg.Language, "", false)
	if err != nil {
		return err
	}

	global, err := InstallPath(pkg.Language, "", true)
	if err != nil {
		return err
	}

	for _, dep := range pkg.Dependencies {
		var pkg Package
		err := FindPackageInDir(&pkg, filepath.Join(local, "gx", "ipfs", dep.Hash))
		if err == nil {
			err := cb(dep, &pkg)
			if err != nil {
				return err
			}

			continue
		}
		err = FindPackageInDir(&pkg, filepath.Join(global, "gx", "ipfs", dep.Hash))
		if err != nil {
			return fmt.Errorf("package %s (%s) not found", dep.Name, dep.Hash)
		}

		err = cb(dep, &pkg)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package mitm

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"sync"
	"time"
)

type ServerParam struct {
	CA        *tls.Certificate // the Root CA for generatng on the fly MITM certificates
	TLSConfig *tls.Config      // a template TLS config for the server.
}

// A ServerConn is a net.Conn that holds its clients SNI header in ServerName
// after the handshake.
type ServerConn struct {
	*tls.Conn

	// ServerName is set during Conn's handshake to the client's requested
	// server name set in the SNI header. It is not safe to access across
	// multiple goroutines while Conn is performing the handshake.
	ServerName string
}

// Server wraps cn with a ServerConn configured with p so that during its
// Handshake, it will generate a new certificate using p.CA. After a successful
// Handshake, its ServerName field will be set to the clients requested
// ServerName in the SNI header.
func Server(cn net.Conn, p ServerParam) *ServerConn {
	conf := new(tls.Config)
	if p.TLSConfig != nil {
		*conf = *p.TLSConfig
	}
	sc := new(ServerConn)
	conf.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		sc.ServerName = hello.ServerName
		return GenerateCert(p.CA, hello.ServerName)
	}
	sc.Conn = tls.Server(cn, conf)
	return sc
}

type listener struct {
	net.Listener
	ca   *tls.Certificate
	conf *tls.Config
}

// NewListener returns a net.Listener that generates a new cert from ca for
// each new Accept. It uses SNI to generate the cert, and herefore only
// works with clients that send SNI headers.
//
// This is useful for building transparent MITM proxies.
func NewListener(inner net.Listener, ca *tls.Certificate, conf *tls.Config) net.Listener {
	return &listener{inner, ca, conf}
}

func (l *listener) Accept() (net.Conn, error) {
	cn, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}
	sc := Server(cn, ServerParam{
		CA:        l.ca,
		TLSConfig: l.conf,
	})
	return sc, nil
}

// Proxy is a forward proxy that substitutes its own certificate
// for incoming TLS connections in place of the upstream server's
// certificate.
type Proxy struct {
	// Wrap specifies a function for optionally wrapping upstream for
	// inspecting the decrypted HTTP request and response.
	Wrap func(upstream http.Handler) http.Handler

	// CA specifies the root CA for generating leaf certs for each incoming
	// TLS request.
	CA *tls.Certificate

	// TLSServerConfig specifies the tls.Config to use when generating leaf
	// cert using CA.
	TLSServerConfig *tls.Config

	// TLSClientConfig specifies the tls.Config to use when establishing
	// an upstream connection for proxying.
	TLSClientConfig *tls.Config

	// FlushInterval specifies the flush interval
	// to flush to the client while copying the
	// response body.
	// If zero, no periodic flushing is done.
	FlushInterval time.Duration
}

var (
	okHeader           = "HTTP/1.1 200 OK\r\n\r\n"
	noUpstreamHeader   = "HTTP/1.1 503 No Upstream\r\n\r\n"
	noDownstreamHeader = "HTTP/1.1 503 No Downstream\r\n\r\n"
	errHeader          = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
)

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if req.Method != "CONNECT" {
		rp := &httputil.ReverseProxy{
			Director:      httpDirector,
			FlushInterval: p.FlushInterval,
		}
		p.Wrap(rp).ServeHTTP(w, req)
		return
	}

	cn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Println("Hijack:", err)
		http.Error(w, "No Upstream", 503)
		return
	}
	defer cn.Close()

	_, err = io.WriteString(cn, okHeader)
	if err != nil {
		log.Println("Write:", err)
		return
	}

	sc, ok := cn.(*ServerConn)
	if !ok {
		name := dnsName(req.Host)
		if name == "" {
			log.Println("cannot determine cert name for " + req.Host)
			io.WriteString(cn, noDownstreamHeader)
			return
		}
		sc = Server(cn, ServerParam{
			CA:        p.CA,
			TLSConfig: p.TLSServerConfig,
		})
		if err := sc.Handshake(); err != nil {
			log.Println("Server Handshake:", err)
			return
		}
	}

	cc, err := p.tlsDial(req.Host, sc.ServerName)
	if err != nil {
		log.Println("tlsDial:", err)
		io.WriteString(cn, noUpstreamHeader)
		return
	}
	p.proxyMITM(sc, cc)
}

func (p *Proxy) tlsDial(addr, serverName string) (net.Conn, error) {
	conf := new(tls.Config)
	if p.TLSClientConfig != nil {
		*conf = *p.TLSClientConfig
	}
	conf.ServerName = serverName
	return tls.Dial("tcp", addr, conf)
}

func (p *Proxy) proxyMITM(upstream, downstream net.Conn) {
	var mu sync.Mutex
	dial := func(network, addr string) (net.Conn, error) {
		mu.Lock()
		defer mu.Unlock()
		if downstream == nil {
			return nil, io.EOF
		}
		cn := downstream
		downstream = nil
		return cn, nil
	}
	rp := &httputil.ReverseProxy{
		Director:      httpsDirector,
		Transport:     &http.Transport{DialTLS: dial},
		FlushInterval: p.FlushInterval,
	}
	ch := make(chan struct{})
	wc := &onCloseConn{upstream, func() { ch <- struct{}{} }}
	http.Serve(&oneShotListener{wc}, p.Wrap(rp))
	<-ch
}

func httpDirector(r *http.Request) {
	r.URL.Host = r.Host
	r.URL.Scheme = "http"
}

func httpsDirector(req *http.Request) {
	req.URL.Host = req.Host
	req.URL.Scheme = "https"
}

// A oneShotListener implements net.Listener whos Accept only returns a
// net.Conn as specified by c followed by an error for each subsequent Accept.
type oneShotListener struct {
	c net.Conn
}

func (l *oneShotListener) Accept() (net.Conn, error) {
	if l.c == nil {
		return nil, errors.New("closed")
	}
	c := l.c
	l.c = nil
	return c, nil
}

func (l *oneShotListener) Close() error {
	return nil
}

func (l *oneShotListener) Addr() net.Addr {
	return l.c.LocalAddr()
}

// A onCloseConn implements net.Conn and calls its f on Close.
type onCloseConn struct {
	net.Conn
	f func()
}

func (c *onCloseConn) Close() error {
	if c.f != nil {
		c.f()
		c.f = nil
	}
	return c.Conn.Close()
}

// dnsName returns the DNS name in addr, if any.
func dnsName(addr string) string {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return ""
	}
	return host
}
<commit_msg>Parameterize proxy director<commit_after>package mitm

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"sync"
	"time"
)

type ServerParam struct {
	CA        *tls.Certificate // the Root CA for generatng on the fly MITM certificates
	TLSConfig *tls.Config      // a template TLS config for the server.
}

// A ServerConn is a net.Conn that holds its clients SNI header in ServerName
// after the handshake.
type ServerConn struct {
	*tls.Conn

	// ServerName is set during Conn's handshake to the client's requested
	// server name set in the SNI header. It is not safe to access across
	// multiple goroutines while Conn is performing the handshake.
	ServerName string
}

// Server wraps cn with a ServerConn configured with p so that during its
// Handshake, it will generate a new certificate using p.CA. After a successful
// Handshake, its ServerName field will be set to the clients requested
// ServerName in the SNI header.
func Server(cn net.Conn, p ServerParam) *ServerConn {
	conf := new(tls.Config)
	if p.TLSConfig != nil {
		*conf = *p.TLSConfig
	}
	sc := new(ServerConn)
	conf.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		sc.ServerName = hello.ServerName
		return GenerateCert(p.CA, hello.ServerName)
	}
	sc.Conn = tls.Server(cn, conf)
	return sc
}

type listener struct {
	net.Listener
	ca   *tls.Certificate
	conf *tls.Config
}

// NewListener returns a net.Listener that generates a new cert from ca for
// each new Accept. It uses SNI to generate the cert, and herefore only
// works with clients that send SNI headers.
//
// This is useful for building transparent MITM proxies.
func NewListener(inner net.Listener, ca *tls.Certificate, conf *tls.Config) net.Listener {
	return &listener{inner, ca, conf}
}

func (l *listener) Accept() (net.Conn, error) {
	cn, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}
	sc := Server(cn, ServerParam{
		CA:        l.ca,
		TLSConfig: l.conf,
	})
	return sc, nil
}

// Proxy is a forward proxy that substitutes its own certificate
// for incoming TLS connections in place of the upstream server's
// certificate.
type Proxy struct {
	// Wrap specifies a function for optionally wrapping upstream for
	// inspecting the decrypted HTTP request and response.
	Wrap func(upstream http.Handler) http.Handler

	// CA specifies the root CA for generating leaf certs for each incoming
	// TLS request.
	CA *tls.Certificate

	// TLSServerConfig specifies the tls.Config to use when generating leaf
	// cert using CA.
	TLSServerConfig *tls.Config

	// TLSClientConfig specifies the tls.Config to use when establishing
	// an upstream connection for proxying.
	TLSClientConfig *tls.Config

	// FlushInterval specifies the flush interval
	// to flush to the client while copying the
	// response body.
	// If zero, no periodic flushing is done.
	FlushInterval time.Duration

	// Director is function which modifies the request into a new
	// request to be sent using Transport. See the documentation for
	// httputil.ReverseProxy for more details. For mitm proxies, the
	// director defaults to HTTPDirector, but for transparent TLS
	// proxies it should be set to HTTPSDirector.
	Director func(*http.Request)
}

var (
	okHeader           = "HTTP/1.1 200 OK\r\n\r\n"
	noUpstreamHeader   = "HTTP/1.1 503 No Upstream\r\n\r\n"
	noDownstreamHeader = "HTTP/1.1 503 No Downstream\r\n\r\n"
	errHeader          = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
)

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if req.Method != "CONNECT" {
		rp := &httputil.ReverseProxy{
			Director:      p.Director,
			FlushInterval: p.FlushInterval,
		}
		if rp.Director == nil {
			rp.Director = HTTPDirector
		}
		p.Wrap(rp).ServeHTTP(w, req)
		return
	}

	cn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Println("Hijack:", err)
		http.Error(w, "No Upstream", 503)
		return
	}
	defer cn.Close()

	_, err = io.WriteString(cn, okHeader)
	if err != nil {
		log.Println("Write:", err)
		return
	}

	sc, ok := cn.(*ServerConn)
	if !ok {
		name := dnsName(req.Host)
		if name == "" {
			log.Println("cannot determine cert name for " + req.Host)
			io.WriteString(cn, noDownstreamHeader)
			return
		}
		sc = Server(cn, ServerParam{
			CA:        p.CA,
			TLSConfig: p.TLSServerConfig,
		})
		if err := sc.Handshake(); err != nil {
			log.Println("Server Handshake:", err)
			return
		}
	}

	cc, err := p.tlsDial(req.Host, sc.ServerName)
	if err != nil {
		log.Println("tlsDial:", err)
		io.WriteString(cn, noUpstreamHeader)
		return
	}
	p.proxyMITM(sc, cc)
}

func (p *Proxy) tlsDial(addr, serverName string) (net.Conn, error) {
	conf := new(tls.Config)
	if p.TLSClientConfig != nil {
		*conf = *p.TLSClientConfig
	}
	conf.ServerName = serverName
	return tls.Dial("tcp", addr, conf)
}

func (p *Proxy) proxyMITM(upstream, downstream net.Conn) {
	var mu sync.Mutex
	dial := func(network, addr string) (net.Conn, error) {
		mu.Lock()
		defer mu.Unlock()
		if downstream == nil {
			return nil, io.EOF
		}
		cn := downstream
		downstream = nil
		return cn, nil
	}
	rp := &httputil.ReverseProxy{
		Director:      HTTPSDirector,
		Transport:     &http.Transport{DialTLS: dial},
		FlushInterval: p.FlushInterval,
	}
	ch := make(chan struct{})
	wc := &onCloseConn{upstream, func() { ch <- struct{}{} }}
	http.Serve(&oneShotListener{wc}, p.Wrap(rp))
	<-ch
}

// HTTPDirector is director designed for use in Proxy for http
// proxies.
func HTTPDirector(r *http.Request) {
	r.URL.Host = r.Host
	r.URL.Scheme = "http"
}

// HTTPSDirector is a director designed for use in Proxy for
// transparent TLS proxies.
func HTTPSDirector(req *http.Request) {
	req.URL.Host = req.Host
	req.URL.Scheme = "https"
}

// A oneShotListener implements net.Listener whos Accept only returns a
// net.Conn as specified by c followed by an error for each subsequent Accept.
type oneShotListener struct {
	c net.Conn
}

func (l *oneShotListener) Accept() (net.Conn, error) {
	if l.c == nil {
		return nil, errors.New("closed")
	}
	c := l.c
	l.c = nil
	return c, nil
}

func (l *oneShotListener) Close() error {
	return nil
}

func (l *oneShotListener) Addr() net.Addr {
	return l.c.LocalAddr()
}

// A onCloseConn implements net.Conn and calls its f on Close.
type onCloseConn struct {
	net.Conn
	f func()
}

func (c *onCloseConn) Close() error {
	if c.f != nil {
		c.f()
		c.f = nil
	}
	return c.Conn.Close()
}

// dnsName returns the DNS name in addr, if any.
func dnsName(addr string) string {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return ""
	}
	return host
}
<|endoftext|>


<commit_before>package member

import (
	"errors"
	"github.com/watermint/toolbox/domain/model/mo_group_member"
	"github.com/watermint/toolbox/domain/service/sv_group"
	"github.com/watermint/toolbox/domain/service/sv_group_member"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recpie/app_conn"
	"github.com/watermint/toolbox/infra/recpie/app_kitchen"
	"github.com/watermint/toolbox/infra/recpie/app_test"
	"github.com/watermint/toolbox/infra/recpie/app_vo"
	"github.com/watermint/toolbox/infra/ui/app_msg"
)

type ListVO struct {
	PeerName app_conn.ConnBusinessInfo
}

type List struct {
}

func (z *List) Test(c app_control.Control) error {
	lvo := &ListVO{}
	if !app_test.ApplyTestPeers(c, lvo) {
		return nil
	}
	if err := z.Exec(app_kitchen.NewKitchen(c, lvo)); err != nil {
		return err
	}
	return app_test.TestRows(c, "group_member", func(cols map[string]string) error {
		if _, ok := cols["group_id"]; !ok {
			return errors.New("group_id is not found")
		}
		if _, ok := cols["team_member_id"]; !ok {
			return errors.New("team_member_id is not found")
		}
		return nil
	})
}

func (*List) Requirement() app_vo.ValueObject {
	return &ListVO{}
}

func (z *List) Exec(k app_kitchen.Kitchen) error {
	ui := k.UI()
	var vo interface{} = k.Value()
	lvo := vo.(*ListVO)
	connInfo, err := lvo.PeerName.Connect(k.Control())
	if err != nil {
		return err
	}

	gsv := sv_group.New(connInfo)
	groups, err := gsv.List()
	if err != nil {
		return err
	}

	rep, err := k.Report("group_member", &mo_group_member.GroupMember{})
	if err != nil {
		return err
	}
	defer rep.Close()

	for _, group := range groups {
		ui.Info("recipe.group.member.progress.scan", app_msg.P{"Group": group.GroupName})

		msv := sv_group_member.New(connInfo, group)
		members, err := msv.List()
		if err != nil {
			return err
		}
		for _, m := range members {
			row := mo_group_member.NewGroupMember(group, m)
			rep.Row(row)
		}
	}
	return nil
}
<commit_msg>fix test<commit_after>package member

import (
	"errors"
	"github.com/watermint/toolbox/domain/model/mo_group_member"
	"github.com/watermint/toolbox/domain/service/sv_group"
	"github.com/watermint/toolbox/domain/service/sv_group_member"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recpie/app_conn"
	"github.com/watermint/toolbox/infra/recpie/app_kitchen"
	"github.com/watermint/toolbox/infra/recpie/app_test"
	"github.com/watermint/toolbox/infra/recpie/app_vo"
	"github.com/watermint/toolbox/infra/ui/app_msg"
)

type ListVO struct {
	PeerName app_conn.ConnBusinessInfo
}

type List struct {
}

func (z *List) Test(c app_control.Control) error {
	lvo := &ListVO{}
	if !app_test.ApplyTestPeers(c, lvo) {
		return nil
	}
	if err := z.Exec(app_kitchen.NewKitchen(c, lvo)); err != nil {
		return err
	}
	return app_test.TestRows(c, "group_member", func(cols map[string]string) error {
		if _, ok := cols["group_id"]; !ok {
			return errors.New("group_id is not found")
		}
		if _, ok := cols["team_member_id"]; !ok {
			return errors.New("team_member_id is not found")
		}
		return nil
	})
}

func (*List) Requirement() app_vo.ValueObject {
	return &ListVO{}
}

func (z *List) Exec(k app_kitchen.Kitchen) error {
	ui := k.UI()
	var vo interface{} = k.Value()
	lvo := vo.(*ListVO)
	connInfo, err := lvo.PeerName.Connect(k.Control())
	if err != nil {
		return err
	}

	gsv := sv_group.New(connInfo)
	groups, err := gsv.List()
	if err != nil {
		return err
	}

	rep, err := k.Report("group_member", &mo_group_member.GroupMember{})
	if err != nil {
		return err
	}
	defer rep.Close()

	for _, group := range groups {
		ui.Info("recipe.group.member.list.progress.scan", app_msg.P{"Group": group.GroupName})

		msv := sv_group_member.New(connInfo, group)
		members, err := msv.List()
		if err != nil {
			return err
		}
		for _, m := range members {
			row := mo_group_member.NewGroupMember(group, m)
			rep.Row(row)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package raftgorums

import (
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	pb "github.com/relab/raft/raftgorums/raftpb"
)

func (r *Raft) handleOutgoing() error {
	// January 1, 1970 UTC.
	var lastCuReq time.Time

	for {
		select {
		case <-r.stop:
			return nil
		case err := <-r.mem.get().SubError():
			// TODO If a node becomes unavailable and there is a
			// backup available in the same or an alternate region,
			// instantiate reconfiguratior. TODO How many errors
			// before a node is considered unavailable? If there is
			// no backup node available, don't do anything, but
			// schedule the reconfiguratior.
			r.logger.WithField("nodeid", err.NodeID).Warnln("Node unavailable")
		case req := <-r.cureqout:
			// TODO Use config.
			if time.Since(lastCuReq) < 100*time.Millisecond {
				continue
			}
			lastCuReq = time.Now()

			r.logger.WithField("matchindex", req.matchIndex).Warnln("Sending catch-up")
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			leader := r.mem.getNode(req.leaderID)
			_, err := leader.RaftClient.CatchMeUp(ctx, &pb.CatchMeUpRequest{
				FollowerID: r.id,
				NextIndex:  req.matchIndex + 1,
			})
			cancel()

			if err != nil {
				r.logger.WithError(err).Warnln("CatchMeUp failed")
			}
		case req := <-r.rvreqout:
			conf := r.mem.get()

			r.logger.WithField("conf", conf.NodeIDs()).Println("Sending request for vote")

			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := conf.RequestVote(ctx, req)
			cancel()

			if err != nil {
				r.logger.WithError(err).Warnln("RequestVote failed")
			}

			if res == nil {
				continue
			}

			r.HandleRequestVoteResponse(res)

		case req := <-r.aereqout:
			next := make(map[uint32]uint64)
			nextIndex := req.PrevLogIndex + 1

			for nodeID, ch := range r.match {
				select {
				case index := <-ch:
					// TODO Acessing maxAppendEntries, safe but needs fix.
					atLeastMaxEntries := req.PrevLogIndex+1 > r.maxAppendEntries
					lessThenMaxEntriesBehind := index < req.PrevLogIndex+1-r.maxAppendEntries

					if atLeastMaxEntries && lessThenMaxEntriesBehind {
						r.logger.WithFields(logrus.Fields{
							"gorumsid": nodeID,
						}).Warnln("Server too far behind")
						index = req.PrevLogIndex + 1
					}
					next[nodeID] = index
					if index < nextIndex {
						nextIndex = index
					}
				default:
				}
			}

			// TODO This should be safe as it only accesses storage
			// which uses transactions. TODO It accesses
			// maxAppendEntries but this on does not change after
			// startup.
			entries := r.getNextEntries(nextIndex)
			e := uint64(len(entries))
			maxIndex := nextIndex + e - 1

			conf := r.mem.get()
			r.logger.WithField("conf", conf.NodeIDs()).Println("Sending append entries request")

			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := conf.AppendEntries(ctx, req,
				// These functions will be executed concurrently.
				func(req pb.AppendEntriesRequest, nodeID uint32) *pb.AppendEntriesRequest {
					if index, ok := next[nodeID]; ok {
						req.PrevLogIndex = index - 1
						// TODO This should be safe as
						// it only accesses storage
						// which uses transactions.
						req.PrevLogTerm = r.logTerm(index - 1)
					}

					need := maxIndex - req.PrevLogIndex
					req.Entries = entries[e-need:]

					r.logger.WithFields(logrus.Fields{
						"prevlogindex": req.PrevLogIndex,
						"prevlogterm":  req.PrevLogTerm,
						"commitindex":  req.CommitIndex,
						"currentterm":  req.Term,
						"lenentries":   len(req.Entries),
						"gorumsid":     nodeID,
					}).Infoln("Sending AppendEntries")

					return &req
				},
			)

			if err != nil {
				r.logger.WithError(err).Warnln("AppendEntries failed")
			}

			if res == nil {
				continue
			}

			// Cancel on abort.
			if !res.Success {
				cancel()
			}

			r.HandleAppendEntriesResponse(res, res.Replies)
		}
	}
}
<commit_msg>raftgorums/outgoing.go: Cause leader to step down on failed AE<commit_after>package raftgorums

import (
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	pb "github.com/relab/raft/raftgorums/raftpb"
)

func (r *Raft) handleOutgoing() error {
	// January 1, 1970 UTC.
	var lastCuReq time.Time

	for {
		select {
		case <-r.stop:
			return nil
		case err := <-r.mem.get().SubError():
			// TODO If a node becomes unavailable and there is a
			// backup available in the same or an alternate region,
			// instantiate reconfiguratior. TODO How many errors
			// before a node is considered unavailable? If there is
			// no backup node available, don't do anything, but
			// schedule the reconfiguratior.
			r.logger.WithField("nodeid", err.NodeID).Warnln("Node unavailable")
		case req := <-r.cureqout:
			// TODO Use config.
			if time.Since(lastCuReq) < 100*time.Millisecond {
				continue
			}
			lastCuReq = time.Now()

			r.logger.WithField("matchindex", req.matchIndex).Warnln("Sending catch-up")
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			leader := r.mem.getNode(req.leaderID)
			_, err := leader.RaftClient.CatchMeUp(ctx, &pb.CatchMeUpRequest{
				FollowerID: r.id,
				NextIndex:  req.matchIndex + 1,
			})
			cancel()

			if err != nil {
				r.logger.WithError(err).Warnln("CatchMeUp failed")
			}
		case req := <-r.rvreqout:
			conf := r.mem.get()

			r.logger.WithField("conf", conf.NodeIDs()).Println("Sending request for vote")

			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := conf.RequestVote(ctx, req)
			cancel()

			if err != nil {
				r.logger.WithError(err).Warnln("RequestVote failed")
			}

			if res == nil {
				continue
			}

			r.HandleRequestVoteResponse(res)

		case req := <-r.aereqout:
			next := make(map[uint32]uint64)
			nextIndex := req.PrevLogIndex + 1

			for nodeID, ch := range r.match {
				select {
				case index := <-ch:
					// TODO Acessing maxAppendEntries, safe but needs fix.
					atLeastMaxEntries := req.PrevLogIndex+1 > r.maxAppendEntries
					lessThenMaxEntriesBehind := index < req.PrevLogIndex+1-r.maxAppendEntries

					if atLeastMaxEntries && lessThenMaxEntriesBehind {
						r.logger.WithFields(logrus.Fields{
							"gorumsid": nodeID,
						}).Warnln("Server too far behind")
						index = req.PrevLogIndex + 1
					}
					next[nodeID] = index
					if index < nextIndex {
						nextIndex = index
					}
				default:
				}
			}

			// TODO This should be safe as it only accesses storage
			// which uses transactions. TODO It accesses
			// maxAppendEntries but this on does not change after
			// startup.
			entries := r.getNextEntries(nextIndex)
			e := uint64(len(entries))
			maxIndex := nextIndex + e - 1

			conf := r.mem.get()
			r.logger.WithField("conf", conf.NodeIDs()).Println("Sending append entries request")

			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := conf.AppendEntries(ctx, req,
				// These functions will be executed concurrently.
				func(req pb.AppendEntriesRequest, nodeID uint32) *pb.AppendEntriesRequest {
					if index, ok := next[nodeID]; ok {
						req.PrevLogIndex = index - 1
						// TODO This should be safe as
						// it only accesses storage
						// which uses transactions.
						req.PrevLogTerm = r.logTerm(index - 1)
					}

					need := maxIndex - req.PrevLogIndex
					req.Entries = entries[e-need:]

					r.logger.WithFields(logrus.Fields{
						"prevlogindex": req.PrevLogIndex,
						"prevlogterm":  req.PrevLogTerm,
						"commitindex":  req.CommitIndex,
						"currentterm":  req.Term,
						"lenentries":   len(req.Entries),
						"gorumsid":     nodeID,
					}).Infoln("Sending AppendEntries")

					return &req
				},
			)

			if err != nil {
				r.logger.WithError(err).Warnln("AppendEntries failed")
			}

			if res == nil {
				// This will cause the leader to step down as
				// len(res.Replies) == 0.
				res = &pb.AppendEntriesQFResponse{
					Term: req.Term,
				}
			}

			// Cancel on abort.
			if !res.Success {
				cancel()
			}

			r.HandleAppendEntriesResponse(res, res.Replies)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Evan Shaw. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file defines the common package interface and contains a little bit of
// factored out logic.

// Package mmap allows mapping files into memory. It tries to provide a simple, reasonably portable interface,
// but doesn't go out of its way to abstract away every little platform detail.
// This specifically means:
//	* forked processes may or may not inherit mappings
//	* a file's timestamp may or may not be updated by writes through mappings
//	* specifying a size larger than the file's actual size can increase the file's size
//	* If the mapped file is being modified by another process while your program's running, don't expect consistent results between platforms
package mmap

import (
	"errors"
	"os"
	"reflect"
	"unsafe"
)

const (
	// RDONLY maps the memory read-only.
	// Attempts to write to the MMap object will result in undefined behavior.
	RDONLY = 0
	// RDWR maps the memory as read-write. Writes to the MMap object will update the
	// underlying file.
	RDWR = 1 << iota
	// COPY maps the memory as copy-on-write. Writes to the MMap object will affect
	// memory, but the underlying file will remain unchanged.
	COPY
	// If EXEC is set, the mapped memory is marked as executable.
	EXEC
)

const (
	// If the ANON flag is set, the mapped memory will not be backed by a file.
	ANON = 1 << iota
)

// MMap represents a file mapped into memory.
type MMap []byte

// Map maps an entire file into memory.
// Note that because of runtime limitations, no file larger than about 2GB can
// be completely mapped into memory.
// If ANON is set in flags, f is ignored.
func Map(f *os.File, prot, flags int) (MMap, error) {
	return MapRegion(f, -1, prot, flags, 0)
}

// MapRegion maps part of a file into memory.
// The offset parameter must be a multiple of the system's page size.
// If length < 0, the entire file will be mapped.
// Note that because of runtime limitations, no file larger than about 2GB can
// be completely mapped into memory.
// If ANON is set in flags, f is ignored.
func MapRegion(f *os.File, length int, prot, flags int, offset int64) (MMap, error) {
	var fd uintptr
	if flags&ANON == 0 {
		fd = uintptr(f.Fd())
		if length < 0 {
			fi, err := f.Stat()
			if err != nil {
				return nil, err
			}
			length = int(fi.Size())
		}
	} else {
		if length <= 0 {
			return nil, errors.New("anonymous mapping requires non-zero length")
		}
		fd = ^uintptr(0)
	}
	return mmap(length, uintptr(prot), uintptr(flags), fd, offset)
}

func (m *MMap) header() *reflect.SliceHeader {
	return (*reflect.SliceHeader)(unsafe.Pointer(m))
}

// Lock keeps the mapped region in physical memory, ensuring that it will not be
// swapped out.
func (m MMap) Lock() error {
	dh := m.header()
	return lock(dh.Data, uintptr(dh.Len))
}

// Unlock reverses the effect of Lock, allowing the mapped region to potentially
// be swapped out.
// If m is already unlocked, aan error will result.
func (m MMap) Unlock() error {
	dh := m.header()
	return unlock(dh.Data, uintptr(dh.Len))
}

// Flush synchronizes the mapping's contents to the file's contents on disk.
func (m MMap) Flush() error {
	dh := m.header()
	return flush(dh.Data, uintptr(dh.Len))
}

// Unmap deletes the memory mapped region, flushes any remaining changes, and sets
// m to nil.
// Trying to read or write any remaining references to m after Unmap is called will
// result in undefined behavior.
// Unmap should only be called on the slice value that was originally returned from
// a call to Map. Calling Unmap on a derived slice may cause errors.
func (m *MMap) Unmap() error {
	dh := m.header()
	err := unmap(dh.Data, uintptr(dh.Len))
	*m = nil
	return err
}
<commit_msg>64-bit ints in Go 1.1+ allow files larger than 2GB<commit_after>// Copyright 2011 Evan Shaw. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file defines the common package interface and contains a little bit of
// factored out logic.

// Package mmap allows mapping files into memory. It tries to provide a simple, reasonably portable interface,
// but doesn't go out of its way to abstract away every little platform detail.
// This specifically means:
//	* forked processes may or may not inherit mappings
//	* a file's timestamp may or may not be updated by writes through mappings
//	* specifying a size larger than the file's actual size can increase the file's size
//	* If the mapped file is being modified by another process while your program's running, don't expect consistent results between platforms
package mmap

import (
	"errors"
	"os"
	"reflect"
	"unsafe"
)

const (
	// RDONLY maps the memory read-only.
	// Attempts to write to the MMap object will result in undefined behavior.
	RDONLY = 0
	// RDWR maps the memory as read-write. Writes to the MMap object will update the
	// underlying file.
	RDWR = 1 << iota
	// COPY maps the memory as copy-on-write. Writes to the MMap object will affect
	// memory, but the underlying file will remain unchanged.
	COPY
	// If EXEC is set, the mapped memory is marked as executable.
	EXEC
)

const (
	// If the ANON flag is set, the mapped memory will not be backed by a file.
	ANON = 1 << iota
)

// MMap represents a file mapped into memory.
type MMap []byte

// Map maps an entire file into memory.
// If ANON is set in flags, f is ignored.
func Map(f *os.File, prot, flags int) (MMap, error) {
	return MapRegion(f, -1, prot, flags, 0)
}

// MapRegion maps part of a file into memory.
// The offset parameter must be a multiple of the system's page size.
// If length < 0, the entire file will be mapped.
// If ANON is set in flags, f is ignored.
func MapRegion(f *os.File, length int, prot, flags int, offset int64) (MMap, error) {
	var fd uintptr
	if flags&ANON == 0 {
		fd = uintptr(f.Fd())
		if length < 0 {
			fi, err := f.Stat()
			if err != nil {
				return nil, err
			}
			length = int(fi.Size())
		}
	} else {
		if length <= 0 {
			return nil, errors.New("anonymous mapping requires non-zero length")
		}
		fd = ^uintptr(0)
	}
	return mmap(length, uintptr(prot), uintptr(flags), fd, offset)
}

func (m *MMap) header() *reflect.SliceHeader {
	return (*reflect.SliceHeader)(unsafe.Pointer(m))
}

// Lock keeps the mapped region in physical memory, ensuring that it will not be
// swapped out.
func (m MMap) Lock() error {
	dh := m.header()
	return lock(dh.Data, uintptr(dh.Len))
}

// Unlock reverses the effect of Lock, allowing the mapped region to potentially
// be swapped out.
// If m is already unlocked, aan error will result.
func (m MMap) Unlock() error {
	dh := m.header()
	return unlock(dh.Data, uintptr(dh.Len))
}

// Flush synchronizes the mapping's contents to the file's contents on disk.
func (m MMap) Flush() error {
	dh := m.header()
	return flush(dh.Data, uintptr(dh.Len))
}

// Unmap deletes the memory mapped region, flushes any remaining changes, and sets
// m to nil.
// Trying to read or write any remaining references to m after Unmap is called will
// result in undefined behavior.
// Unmap should only be called on the slice value that was originally returned from
// a call to Map. Calling Unmap on a derived slice may cause errors.
func (m *MMap) Unmap() error {
	dh := m.header()
	err := unmap(dh.Data, uintptr(dh.Len))
	*m = nil
	return err
}
<|endoftext|>


<commit_before>package dotweb

const(
	requestHeaderUseMockKey = "dotweb_req_mock"
	requestHeaderUseMockFlag = "true"
)

// MockHandle the handle define on mock module
type MockHandle func(ctx Context)

// Mock the define Mock module
type Mock interface{
	// Register register MockHandle on route
	Register(route string, handler MockHandle)
	// RegisterString register return mock string on route
	RegisterString(route string, resData interface{})
	// RegisterJson register return mock json on route
	RegisterJson(route string, resData interface{})
	// CheckNeedMock check is need do mock logic
	CheckNeedMock(Context) bool
	// Do do mock logic
	Do(Context)
}

// StandardMock standard mock implement for Mock interface
type StandardMock struct{
	routeMap map[string]MockHandle
}

// NewStandardMock create new StandardMock
func NewStandardMock() *StandardMock{
	return &StandardMock{routeMap:make(map[string]MockHandle)}
}

// CheckNeedMock check is need do mock logic
func (m *StandardMock) CheckNeedMock(ctx Context) bool{
	if ctx.Request().QueryHeader(requestHeaderUseMockKey) == requestHeaderUseMockFlag{
		return true
	}
	return false
}

// Do do mock logic
func (m *StandardMock) Do(ctx Context){
	handler, exists:=m.routeMap[ctx.RouterNode().Node().fullPath]
	if exists{
		handler(ctx)
	}
}

// Register register MockHandle on route
func (m *StandardMock) Register(route string, handler MockHandle){
	m.routeMap[route] = handler
}

// RegisterString register return mock string on route
func (m *StandardMock) RegisterString(route string, resData interface{}){
	m.routeMap[route] = func(ctx Context) {
		ctx.Response().SetHeader(requestHeaderUseMockKey, requestHeaderUseMockFlag)
		ctx.WriteString(resData)
		ctx.End()
	}
}

// RegisterJson register return mock json on route
func (m *StandardMock) RegisterJson(route string, resData interface{}){
	m.routeMap[route] = func(ctx Context) {
		ctx.Response().SetHeader(requestHeaderUseMockKey, requestHeaderUseMockFlag)
		ctx.WriteJson(resData)
		ctx.End()
	}
}<commit_msg>#### Version 1.5.7.1 * Add DotWeb Mock log * Mock support Register\RegisterString\RegisterJson * Update README * 2018-08-22 10:00<commit_after>package dotweb

const(
	requestHeaderUseMockKey = "dotweb_req_mock"
	requestHeaderUseMockFlag = "true"
)

// MockHandle the handle define on mock module
type MockHandle func(ctx Context)

// Mock the define Mock module
type Mock interface{
	// Register register MockHandle on route
	Register(route string, handler MockHandle)
	// RegisterString register return mock string on route
	RegisterString(route string, resData interface{})
	// RegisterJSON register return mock json on route
	RegisterJSON(route string, resData interface{})
	// CheckNeedMock check is need do mock logic
	CheckNeedMock(Context) bool
	// Do do mock logic
	Do(Context)
}

// StandardMock standard mock implement for Mock interface
type StandardMock struct{
	routeMap map[string]MockHandle
}

// NewStandardMock create new StandardMock
func NewStandardMock() *StandardMock{
	return &StandardMock{routeMap:make(map[string]MockHandle)}
}

// CheckNeedMock check is need do mock logic
func (m *StandardMock) CheckNeedMock(ctx Context) bool{
	if ctx.Request().QueryHeader(requestHeaderUseMockKey) == requestHeaderUseMockFlag{
		return true
	}
	return false
}

// Do do mock logic
func (m *StandardMock) Do(ctx Context){
	handler, exists:=m.routeMap[ctx.RouterNode().Node().fullPath]
	if exists{
		handler(ctx)
	}
}

// Register register MockHandle on route
func (m *StandardMock) Register(route string, handler MockHandle){
	m.routeMap[route] = handler
}

// RegisterString register return mock string on route
func (m *StandardMock) RegisterString(route string, resData interface{}){
	m.routeMap[route] = func(ctx Context) {
		ctx.Response().SetHeader(requestHeaderUseMockKey, requestHeaderUseMockFlag)
		ctx.WriteString(resData)
		ctx.End()
	}
}

// RegisterJSON register return mock json on route
func (m *StandardMock) RegisterJSON(route string, resData interface{}){
	m.routeMap[route] = func(ctx Context) {
		ctx.Response().SetHeader(requestHeaderUseMockKey, requestHeaderUseMockFlag)
		ctx.WriteJson(resData)
		ctx.End()
	}
}<|endoftext|>


<commit_before>package utils

import (
	"github.com/onsi/ginkgo"
	storagev1 "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

// cdi-file-host pod/service relative values
const (
	//RegistryHostName provides a deploymnet and service name for registry
	RegistryHostName = "cdi-docker-registry-host"
	// FileHostName provides a deployment and service name for tests
	FileHostName = "cdi-file-host"
	// FileHostS3Bucket provides an S3 bucket name for tests (e.g. http://<serviceIP:port>/FileHostS3Bucket/image)
	FileHostS3Bucket = "images"
	// AccessKeyValue provides a username to use for http and S3 (see hack/build/docker/cdi-func-test-file-host-http/htpasswd)
	AccessKeyValue = "admin"
	// SecretKeyValue provides a password to use for http and S3 (see hack/build/docker/cdi-func-test-file-host-http/htpasswd)
	SecretKeyValue = "password"
	// HttpAuthPort provides a cdi-file-host service auth port for tests
	HTTPAuthPort = 81
	// HttpNoAuthPort provides a cdi-file-host service no-auth port for tests, requires AccessKeyValue and SecretKeyValue
	HTTPNoAuthPort = 80
	// HTTPRateLimitPort provides a cdi-file-host service rate limit port for tests, speed is limited to 25k/s to allow for testing slow connection behavior. No auth.
	HTTPRateLimitPort = 82
	// S3Port provides a cdi-file-host service S3 port, requires AccessKey and SecretKeyValue
	S3Port = 9000
	// HTTPSPort is the https port of cdi-file-host
	HTTPSNoAuthPort = 443
	// RegistryCertConfigMap is the ConfigMap where the cert for the docker registry is stored
	RegistryCertConfigMap = "cdi-docker-registry-host-certs"
	// FileHostCertConfigMap is the ConfigMap where the cert fir the file host is stored
	FileHostCertConfigMap = "cdi-file-host-certs"
)

var (
	// DefaultNodeName the default node to use in tests
	DefaultNodeName string
	// DefaultStorageClass the defauld storage class used in tests
	DefaultStorageClass *storagev1.StorageClass
)

func getDefaultNodeName(client *kubernetes.Clientset) string {
	nodes, err := client.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		ginkgo.Fail("Unable to list nodes")
	}
	return nodes.Items[0].Name
}

func getDefaultStorageClass(client *kubernetes.Clientset) *storagev1.StorageClass {
	storageclasses, err := client.StorageV1().StorageClasses().List(metav1.ListOptions{})
	if err != nil {
		ginkgo.Fail("Unable to list storage classes")
		return nil
	}
	for _, storageClass := range storageclasses.Items {
		if storageClass.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" {
			return &storageClass
		}
	}
	ginkgo.Fail("Unable to find default storage classes")
	return nil
}

// IsHostpathProvisioner returns true if hostpath-provisioner is the default storage class
func IsHostpathProvisioner() bool {
	if DefaultStorageClass == nil {
		return false
	}
	return DefaultStorageClass.Provisioner == "kubevirt.io/hostpath-provisioner"
}

// AddProvisionOnNodeToAnn adds 'kubevirt.io/provisionOnNode' annotaion
func AddProvisionOnNodeToAnn(pvcAnn map[string]string) {
	pvcAnn["kubevirt.io/provisionOnNode"] = DefaultNodeName
}

// CacheTestsData fetch and cache data required for tests
func CacheTestsData(client *kubernetes.Clientset) {
	DefaultNodeName = getDefaultNodeName(client)
	DefaultStorageClass = getDefaultStorageClass(client)
}
<commit_msg>Exclude master and tainted nodes from being used as node for hostpath provisioner (#1040)<commit_after>package utils

import (
	"fmt"
	"github.com/onsi/ginkgo"
	corev1 "k8s.io/api/core/v1"
	storagev1 "k8s.io/api/storage/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

// cdi-file-host pod/service relative values
const (
	//RegistryHostName provides a deploymnet and service name for registry
	RegistryHostName = "cdi-docker-registry-host"
	// FileHostName provides a deployment and service name for tests
	FileHostName = "cdi-file-host"
	// FileHostS3Bucket provides an S3 bucket name for tests (e.g. http://<serviceIP:port>/FileHostS3Bucket/image)
	FileHostS3Bucket = "images"
	// AccessKeyValue provides a username to use for http and S3 (see hack/build/docker/cdi-func-test-file-host-http/htpasswd)
	AccessKeyValue = "admin"
	// SecretKeyValue provides a password to use for http and S3 (see hack/build/docker/cdi-func-test-file-host-http/htpasswd)
	SecretKeyValue = "password"
	// HttpAuthPort provides a cdi-file-host service auth port for tests
	HTTPAuthPort = 81
	// HttpNoAuthPort provides a cdi-file-host service no-auth port for tests, requires AccessKeyValue and SecretKeyValue
	HTTPNoAuthPort = 80
	// HTTPRateLimitPort provides a cdi-file-host service rate limit port for tests, speed is limited to 25k/s to allow for testing slow connection behavior. No auth.
	HTTPRateLimitPort = 82
	// S3Port provides a cdi-file-host service S3 port, requires AccessKey and SecretKeyValue
	S3Port = 9000
	// HTTPSPort is the https port of cdi-file-host
	HTTPSNoAuthPort = 443
	// RegistryCertConfigMap is the ConfigMap where the cert for the docker registry is stored
	RegistryCertConfigMap = "cdi-docker-registry-host-certs"
	// FileHostCertConfigMap is the ConfigMap where the cert fir the file host is stored
	FileHostCertConfigMap = "cdi-file-host-certs"
)

var (
	// DefaultNodeName the default node to use in tests
	DefaultNodeName string
	// DefaultStorageClass the defauld storage class used in tests
	DefaultStorageClass *storagev1.StorageClass
)

func getDefaultNodeName(client *kubernetes.Clientset) string {
	nodes, err := client.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		ginkgo.Fail("Unable to list nodes")
	}
	for _, node := range nodes.Items {
		scheduleable := true
		if node.Spec.Taints != nil {
			for _, taint := range node.Spec.Taints {
				if taint.Effect == corev1.TaintEffectNoSchedule {
					scheduleable = false
				}
			}
		}
		if node.GetLabels() != nil {
			if val, ok := node.GetLabels()["node-role.kubernetes.io/master"]; ok && val == "true" {
				scheduleable = false
			}
		}
		if scheduleable {
			ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Using node %s as target node for hostpath provisioner if set.\n", node.Name)))
			return node.Name
		}
	}
	return ""
}

func getDefaultStorageClass(client *kubernetes.Clientset) *storagev1.StorageClass {
	storageclasses, err := client.StorageV1().StorageClasses().List(metav1.ListOptions{})
	if err != nil {
		ginkgo.Fail("Unable to list storage classes")
		return nil
	}
	for _, storageClass := range storageclasses.Items {
		if storageClass.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" {
			return &storageClass
		}
	}
	ginkgo.Fail("Unable to find default storage classes")
	return nil
}

// IsHostpathProvisioner returns true if hostpath-provisioner is the default storage class
func IsHostpathProvisioner() bool {
	if DefaultStorageClass == nil {
		return false
	}
	return DefaultStorageClass.Provisioner == "kubevirt.io/hostpath-provisioner"
}

// AddProvisionOnNodeToAnn adds 'kubevirt.io/provisionOnNode' annotaion
func AddProvisionOnNodeToAnn(pvcAnn map[string]string) {
	if DefaultNodeName == "" {
		ginkgo.Skip("No scheduleable nodes available")
	}
	pvcAnn["kubevirt.io/provisionOnNode"] = DefaultNodeName
}

// CacheTestsData fetch and cache data required for tests
func CacheTestsData(client *kubernetes.Clientset) {
	DefaultNodeName = getDefaultNodeName(client)
	DefaultStorageClass = getDefaultStorageClass(client)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"sync"

	"github.com/decred/dcrd/blockchain"
	"github.com/decred/dcrd/blockchain/stake"
	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/database"
	"github.com/decred/dcrd/dcrutil"
	"github.com/decred/dcrd/txscript"
	"github.com/decred/dcrd/wire"
)

var (
	// existsAddressIndexName is the human-readable name for the index.
	existsAddressIndexName = "exists address index"

	// existsAddrIndexKey is the key of the ever seen address index and
	// the db bucket used to house it.
	existsAddrIndexKey = []byte("existsaddridx")
)

// ExistsAddrIndex implements an "ever seen" address index.  Any address that
// is ever seen in a block or in the mempool is stored here as a key. Values
// are empty.  Once an address is seen, it is never removed from this store.
// This results in a local version of this database that is consistent only
// for this peer, but at minimum contains all the addresses seen on the
// blockchain itself.
//
// In addition, support is provided for a memory-only index of unconfirmed
// transactions such as those which are kept in the memory pool before inclusion
// in a block.
type ExistsAddrIndex struct {
	// The following fields are set when the instance is created and can't
	// be changed afterwards, so there is no need to protect them with a
	// separate mutex.
	db          database.DB
	chainParams *chaincfg.Params

	// The following fields are used to quickly link transactions and
	// addresses that have not been included into a block yet when an
	// address index is being maintained.  The are protected by the
	// unconfirmedLock field.
	//
	// The txnsByAddr field is used to keep an index of all transactions
	// which either create an output to a given address or spend from a
	// previous output to it keyed by the address.
	//
	// The addrsByTx field is essentially the reverse and is used to
	// keep an index of all addresses which a given transaction involves.
	// This allows fairly efficient updates when transactions are removed
	// once they are included into a block.
	unconfirmedLock sync.RWMutex
	mpExistsAddr    map[[addrKeySize]byte]struct{}
}

// NewExistsAddrIndex returns a new instance of an indexer that is used to
// create a mapping of all addresses ever seen.
//
// It implements the Indexer interface which plugs into the IndexManager that in
// turn is used by the blockchain package.  This allows the index to be
// seamlessly maintained along with the chain.
func NewExistsAddrIndex(db database.DB, chainParams *chaincfg.Params) *ExistsAddrIndex {
	return &ExistsAddrIndex{
		db:           db,
		chainParams:  chainParams,
		mpExistsAddr: make(map[[addrKeySize]byte]struct{}),
	}
}

// Ensure the ExistsAddrIndex type implements the Indexer interface.
var _ Indexer = (*ExistsAddrIndex)(nil)

// Ensure the ExistsAddrIndex type implements the NeedsInputser interface.
var _ NeedsInputser = (*ExistsAddrIndex)(nil)

// NeedsInputs signals that the index requires the referenced inputs in order
// to properly create the index.
//
// This implements the NeedsInputser interface.
func (idx *ExistsAddrIndex) NeedsInputs() bool {
	return false
}

// Init is only provided to satisfy the Indexer interface as there is nothing to
// initialize for this index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Init() error {
	// Nothing to do.
	return nil
}

// Key returns the database key to use for the index as a byte slice.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Key() []byte {
	return existsAddrIndexKey
}

// Name returns the human-readable name of the index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Name() string {
	return existsAddressIndexName
}

// Create is invoked when the indexer manager determines the index needs
// to be created for the first time.  It creates the bucket for the address
// index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Create(dbTx database.Tx) error {
	_, err := dbTx.Metadata().CreateBucket(existsAddrIndexKey)
	return err
}

// dbPutExistsAddr uses an existing database transaction to update or add a
// used address index to the database.
func dbPutExistsAddr(bucket database.Bucket, addrKey [addrKeySize]byte) error {
	return bucket.Put(addrKey[:], nil)
}

// existsAddress takes a bucket and key for an address and responds with
// whether or not the key exists in the database.
func (idx *ExistsAddrIndex) existsAddress(bucket database.Bucket, k [addrKeySize]byte) bool {
	if bucket.Get(k[:]) != nil {
		return true
	}

	idx.unconfirmedLock.RLock()
	_, exists := idx.mpExistsAddr[k]
	idx.unconfirmedLock.RUnlock()

	return exists
}

// ExistsAddress is the concurrency safe, exported function that returns
// whether or not an address has been seen before.
func (idx *ExistsAddrIndex) ExistsAddress(addr dcrutil.Address) (bool, error) {
	k, err := addrToKey(addr, idx.chainParams)
	if err != nil {
		return false, err
	}

	var exists bool
	err = idx.db.View(func(dbTx database.Tx) error {
		meta := dbTx.Metadata()
		existsAddrIndex := meta.Bucket(existsAddrIndexKey)
		exists = existsAddrIndex.Get(k[:]) != nil

		return nil
	})
	if err != nil {
		return false, err
	}

	// Only check the in memory map if needed.
	if !exists {
		idx.unconfirmedLock.RLock()
		_, exists = idx.mpExistsAddr[k]
		idx.unconfirmedLock.RUnlock()
	}

	return exists, nil
}

// ExistsAddresses is the concurrency safe, exported function that returns
// whether or not each address in a slice of addresses has been seen before.
func (idx *ExistsAddrIndex) ExistsAddresses(addrs []dcrutil.Address) ([]bool, error) {
	exists := make([]bool, len(addrs))
	addrKeys := make([][addrKeySize]byte, len(addrs))
	for i := range addrKeys {
		var err error
		addrKeys[i], err = addrToKey(addrs[i], idx.chainParams)
		if err != nil {
			return nil, err
		}
	}

	err := idx.db.View(func(dbTx database.Tx) error {
		for i := range addrKeys {
			meta := dbTx.Metadata()
			existsAddrIndex := meta.Bucket(existsAddrIndexKey)

			exists[i] = existsAddrIndex.Get(addrKeys[i][:]) != nil
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	idx.unconfirmedLock.RLock()
	for i := range addrKeys {
		if !exists[i] {
			_, exists[i] = idx.mpExistsAddr[addrKeys[i]]
		}
	}
	idx.unconfirmedLock.RUnlock()

	return exists, nil
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain.  This indexer adds a key for each address
// the transactions in the block involve.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) ConnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error {
	var parentTxs []*dcrutil.Tx
	if approvesParent(block) && block.Height() > 1 {
		parentTxs = parent.Transactions()
	}
	blockTxns := block.STransactions()
	allTxns := append(parentTxs, blockTxns...)

	usedAddrs := make(map[[addrKeySize]byte]struct{})

	for _, tx := range allTxns {
		msgTx := tx.MsgTx()
		isSStx := stake.IsSStx(msgTx)
		for _, txIn := range msgTx.TxIn {
			if txscript.IsMultisigSigScript(txIn.SignatureScript) {
				rs, err :=
					txscript.MultisigRedeemScriptFromScriptSig(
						txIn.SignatureScript)
				if err != nil {
					continue
				}

				class, addrs, _, err := txscript.ExtractPkScriptAddrs(
					txscript.DefaultScriptVersion, rs, idx.chainParams)
				if err != nil {
					// Non-standard outputs are skipped.
					continue
				}
				if class != txscript.MultiSigTy {
					// This should never happen, but be paranoid.
					continue
				}

				for _, addr := range addrs {
					k, err := addrToKey(addr, idx.chainParams)
					if err != nil {
						continue
					}

					usedAddrs[k] = struct{}{}
				}
			}
		}

		for _, txOut := range tx.MsgTx().TxOut {
			class, addrs, _, err := txscript.ExtractPkScriptAddrs(
				txOut.Version, txOut.PkScript, idx.chainParams)
			if err != nil {
				// Non-standard outputs are skipped.
				continue
			}

			if isSStx && class == txscript.NullDataTy {
				addr, err := stake.AddrFromSStxPkScrCommitment(txOut.PkScript,
					idx.chainParams)
				if err != nil {
					// Ignore unsupported address types.
					continue
				}

				addrs = append(addrs, addr)
			}

			for _, addr := range addrs {
				k, err := addrToKey(addr, idx.chainParams)
				if err != nil {
					// Ignore unsupported address types.
					continue
				}

				usedAddrs[k] = struct{}{}
			}
		}
	}

	// Write all the newly used addresses to the database,
	// skipping any keys that already exist. Write any
	// addresses we see in mempool at this time, too,
	// then remove them from the unconfirmed map drop
	// dropping the old map and reassigning a new map.
	idx.unconfirmedLock.Lock()
	for k := range idx.mpExistsAddr {
		usedAddrs[k] = struct{}{}
	}
	idx.mpExistsAddr = make(map[[addrKeySize]byte]struct{})
	idx.unconfirmedLock.Unlock()

	meta := dbTx.Metadata()
	existsAddrIndex := meta.Bucket(existsAddrIndexKey)
	newUsedAddrs := make(map[[addrKeySize]byte]struct{})
	for k := range usedAddrs {
		if !idx.existsAddress(existsAddrIndex, k) {
			newUsedAddrs[k] = struct{}{}
		}
	}

	for k := range newUsedAddrs {
		err := dbPutExistsAddr(existsAddrIndex, k)
		if err != nil {
			return err
		}
	}

	return nil
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain. Note that the exists address manager
// never removes addresses.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) DisconnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error {
	return nil
}

// addUnconfirmedTx adds all addresses related to the transaction to the
// unconfirmed (memory-only) exists address index.
func (idx *ExistsAddrIndex) addUnconfirmedTx(tx *wire.MsgTx) {
	isSStx := stake.IsSStx(tx)
	for _, txIn := range tx.TxIn {
		if txscript.IsMultisigSigScript(txIn.SignatureScript) {
			rs, err :=
				txscript.MultisigRedeemScriptFromScriptSig(
					txIn.SignatureScript)
			if err != nil {
				continue
			}

			class, addrs, _, err := txscript.ExtractPkScriptAddrs(
				txscript.DefaultScriptVersion, rs, idx.chainParams)
			if err != nil {
				// Non-standard outputs are skipped.
				continue
			}
			if class != txscript.MultiSigTy {
				// This should never happen, but be paranoid.
				continue
			}

			for _, addr := range addrs {
				k, err := addrToKey(addr, idx.chainParams)
				if err != nil {
					continue
				}

				if _, exists := idx.mpExistsAddr[k]; !exists {
					idx.mpExistsAddr[k] = struct{}{}
				}
			}
		}
	}

	for _, txOut := range tx.TxOut {
		class, addrs, _, err := txscript.ExtractPkScriptAddrs(txOut.Version,
			txOut.PkScript, idx.chainParams)
		if err != nil {
			// Non-standard outputs are skipped.
			continue
		}

		if isSStx && class == txscript.NullDataTy {
			addr, err := stake.AddrFromSStxPkScrCommitment(txOut.PkScript,
				idx.chainParams)
			if err != nil {
				// Ignore unsupported address types.
				continue
			}

			addrs = append(addrs, addr)
		}

		for _, addr := range addrs {
			k, err := addrToKey(addr, idx.chainParams)
			if err != nil {
				// Ignore unsupported address types.
				continue
			}

			if _, exists := idx.mpExistsAddr[k]; !exists {
				idx.mpExistsAddr[k] = struct{}{}
			}
		}
	}
}

// AddUnconfirmedTx is the exported form of addUnconfirmedTx.
//
// This function is safe for concurrent access.
func (idx *ExistsAddrIndex) AddUnconfirmedTx(tx *wire.MsgTx) {
	idx.unconfirmedLock.Lock()
	defer idx.unconfirmedLock.Unlock()

	idx.addUnconfirmedTx(tx)
}

// DropExistsAddrIndex drops the exists address index from the provided
// database if it exists.
func DropExistsAddrIndex(db database.DB, interrupt <-chan struct{}) error {
	return dropFlatIndex(db, existsAddrIndexKey, existsAddressIndexName,
		interrupt)
}

// DropIndex drops the exists address index from the provided database if it
// exists.
func (*ExistsAddrIndex) DropIndex(db database.DB, interrupt <-chan struct{}) error {
	return DropExistsAddrIndex(db, interrupt)
}
<commit_msg>indexers: Remove unneeded existsaddrindex iface.<commit_after>// Copyright (c) 2016-2017 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package indexers

import (
	"sync"

	"github.com/decred/dcrd/blockchain"
	"github.com/decred/dcrd/blockchain/stake"
	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/database"
	"github.com/decred/dcrd/dcrutil"
	"github.com/decred/dcrd/txscript"
	"github.com/decred/dcrd/wire"
)

var (
	// existsAddressIndexName is the human-readable name for the index.
	existsAddressIndexName = "exists address index"

	// existsAddrIndexKey is the key of the ever seen address index and
	// the db bucket used to house it.
	existsAddrIndexKey = []byte("existsaddridx")
)

// ExistsAddrIndex implements an "ever seen" address index.  Any address that
// is ever seen in a block or in the mempool is stored here as a key. Values
// are empty.  Once an address is seen, it is never removed from this store.
// This results in a local version of this database that is consistent only
// for this peer, but at minimum contains all the addresses seen on the
// blockchain itself.
//
// In addition, support is provided for a memory-only index of unconfirmed
// transactions such as those which are kept in the memory pool before inclusion
// in a block.
type ExistsAddrIndex struct {
	// The following fields are set when the instance is created and can't
	// be changed afterwards, so there is no need to protect them with a
	// separate mutex.
	db          database.DB
	chainParams *chaincfg.Params

	// The following fields are used to quickly link transactions and
	// addresses that have not been included into a block yet when an
	// address index is being maintained.  The are protected by the
	// unconfirmedLock field.
	//
	// The txnsByAddr field is used to keep an index of all transactions
	// which either create an output to a given address or spend from a
	// previous output to it keyed by the address.
	//
	// The addrsByTx field is essentially the reverse and is used to
	// keep an index of all addresses which a given transaction involves.
	// This allows fairly efficient updates when transactions are removed
	// once they are included into a block.
	unconfirmedLock sync.RWMutex
	mpExistsAddr    map[[addrKeySize]byte]struct{}
}

// NewExistsAddrIndex returns a new instance of an indexer that is used to
// create a mapping of all addresses ever seen.
//
// It implements the Indexer interface which plugs into the IndexManager that in
// turn is used by the blockchain package.  This allows the index to be
// seamlessly maintained along with the chain.
func NewExistsAddrIndex(db database.DB, chainParams *chaincfg.Params) *ExistsAddrIndex {
	return &ExistsAddrIndex{
		db:           db,
		chainParams:  chainParams,
		mpExistsAddr: make(map[[addrKeySize]byte]struct{}),
	}
}

// Ensure the ExistsAddrIndex type implements the Indexer interface.
var _ Indexer = (*ExistsAddrIndex)(nil)

// Init is only provided to satisfy the Indexer interface as there is nothing to
// initialize for this index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Init() error {
	// Nothing to do.
	return nil
}

// Key returns the database key to use for the index as a byte slice.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Key() []byte {
	return existsAddrIndexKey
}

// Name returns the human-readable name of the index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Name() string {
	return existsAddressIndexName
}

// Create is invoked when the indexer manager determines the index needs
// to be created for the first time.  It creates the bucket for the address
// index.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) Create(dbTx database.Tx) error {
	_, err := dbTx.Metadata().CreateBucket(existsAddrIndexKey)
	return err
}

// dbPutExistsAddr uses an existing database transaction to update or add a
// used address index to the database.
func dbPutExistsAddr(bucket database.Bucket, addrKey [addrKeySize]byte) error {
	return bucket.Put(addrKey[:], nil)
}

// existsAddress takes a bucket and key for an address and responds with
// whether or not the key exists in the database.
func (idx *ExistsAddrIndex) existsAddress(bucket database.Bucket, k [addrKeySize]byte) bool {
	if bucket.Get(k[:]) != nil {
		return true
	}

	idx.unconfirmedLock.RLock()
	_, exists := idx.mpExistsAddr[k]
	idx.unconfirmedLock.RUnlock()

	return exists
}

// ExistsAddress is the concurrency safe, exported function that returns
// whether or not an address has been seen before.
func (idx *ExistsAddrIndex) ExistsAddress(addr dcrutil.Address) (bool, error) {
	k, err := addrToKey(addr, idx.chainParams)
	if err != nil {
		return false, err
	}

	var exists bool
	err = idx.db.View(func(dbTx database.Tx) error {
		meta := dbTx.Metadata()
		existsAddrIndex := meta.Bucket(existsAddrIndexKey)
		exists = existsAddrIndex.Get(k[:]) != nil

		return nil
	})
	if err != nil {
		return false, err
	}

	// Only check the in memory map if needed.
	if !exists {
		idx.unconfirmedLock.RLock()
		_, exists = idx.mpExistsAddr[k]
		idx.unconfirmedLock.RUnlock()
	}

	return exists, nil
}

// ExistsAddresses is the concurrency safe, exported function that returns
// whether or not each address in a slice of addresses has been seen before.
func (idx *ExistsAddrIndex) ExistsAddresses(addrs []dcrutil.Address) ([]bool, error) {
	exists := make([]bool, len(addrs))
	addrKeys := make([][addrKeySize]byte, len(addrs))
	for i := range addrKeys {
		var err error
		addrKeys[i], err = addrToKey(addrs[i], idx.chainParams)
		if err != nil {
			return nil, err
		}
	}

	err := idx.db.View(func(dbTx database.Tx) error {
		for i := range addrKeys {
			meta := dbTx.Metadata()
			existsAddrIndex := meta.Bucket(existsAddrIndexKey)

			exists[i] = existsAddrIndex.Get(addrKeys[i][:]) != nil
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	idx.unconfirmedLock.RLock()
	for i := range addrKeys {
		if !exists[i] {
			_, exists[i] = idx.mpExistsAddr[addrKeys[i]]
		}
	}
	idx.unconfirmedLock.RUnlock()

	return exists, nil
}

// ConnectBlock is invoked by the index manager when a new block has been
// connected to the main chain.  This indexer adds a key for each address
// the transactions in the block involve.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) ConnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error {
	var parentTxs []*dcrutil.Tx
	if approvesParent(block) && block.Height() > 1 {
		parentTxs = parent.Transactions()
	}
	blockTxns := block.STransactions()
	allTxns := append(parentTxs, blockTxns...)

	usedAddrs := make(map[[addrKeySize]byte]struct{})

	for _, tx := range allTxns {
		msgTx := tx.MsgTx()
		isSStx := stake.IsSStx(msgTx)
		for _, txIn := range msgTx.TxIn {
			if txscript.IsMultisigSigScript(txIn.SignatureScript) {
				rs, err :=
					txscript.MultisigRedeemScriptFromScriptSig(
						txIn.SignatureScript)
				if err != nil {
					continue
				}

				class, addrs, _, err := txscript.ExtractPkScriptAddrs(
					txscript.DefaultScriptVersion, rs, idx.chainParams)
				if err != nil {
					// Non-standard outputs are skipped.
					continue
				}
				if class != txscript.MultiSigTy {
					// This should never happen, but be paranoid.
					continue
				}

				for _, addr := range addrs {
					k, err := addrToKey(addr, idx.chainParams)
					if err != nil {
						continue
					}

					usedAddrs[k] = struct{}{}
				}
			}
		}

		for _, txOut := range tx.MsgTx().TxOut {
			class, addrs, _, err := txscript.ExtractPkScriptAddrs(
				txOut.Version, txOut.PkScript, idx.chainParams)
			if err != nil {
				// Non-standard outputs are skipped.
				continue
			}

			if isSStx && class == txscript.NullDataTy {
				addr, err := stake.AddrFromSStxPkScrCommitment(txOut.PkScript,
					idx.chainParams)
				if err != nil {
					// Ignore unsupported address types.
					continue
				}

				addrs = append(addrs, addr)
			}

			for _, addr := range addrs {
				k, err := addrToKey(addr, idx.chainParams)
				if err != nil {
					// Ignore unsupported address types.
					continue
				}

				usedAddrs[k] = struct{}{}
			}
		}
	}

	// Write all the newly used addresses to the database,
	// skipping any keys that already exist. Write any
	// addresses we see in mempool at this time, too,
	// then remove them from the unconfirmed map drop
	// dropping the old map and reassigning a new map.
	idx.unconfirmedLock.Lock()
	for k := range idx.mpExistsAddr {
		usedAddrs[k] = struct{}{}
	}
	idx.mpExistsAddr = make(map[[addrKeySize]byte]struct{})
	idx.unconfirmedLock.Unlock()

	meta := dbTx.Metadata()
	existsAddrIndex := meta.Bucket(existsAddrIndexKey)
	newUsedAddrs := make(map[[addrKeySize]byte]struct{})
	for k := range usedAddrs {
		if !idx.existsAddress(existsAddrIndex, k) {
			newUsedAddrs[k] = struct{}{}
		}
	}

	for k := range newUsedAddrs {
		err := dbPutExistsAddr(existsAddrIndex, k)
		if err != nil {
			return err
		}
	}

	return nil
}

// DisconnectBlock is invoked by the index manager when a block has been
// disconnected from the main chain. Note that the exists address manager
// never removes addresses.
//
// This is part of the Indexer interface.
func (idx *ExistsAddrIndex) DisconnectBlock(dbTx database.Tx, block, parent *dcrutil.Block, view *blockchain.UtxoViewpoint) error {
	return nil
}

// addUnconfirmedTx adds all addresses related to the transaction to the
// unconfirmed (memory-only) exists address index.
func (idx *ExistsAddrIndex) addUnconfirmedTx(tx *wire.MsgTx) {
	isSStx := stake.IsSStx(tx)
	for _, txIn := range tx.TxIn {
		if txscript.IsMultisigSigScript(txIn.SignatureScript) {
			rs, err :=
				txscript.MultisigRedeemScriptFromScriptSig(
					txIn.SignatureScript)
			if err != nil {
				continue
			}

			class, addrs, _, err := txscript.ExtractPkScriptAddrs(
				txscript.DefaultScriptVersion, rs, idx.chainParams)
			if err != nil {
				// Non-standard outputs are skipped.
				continue
			}
			if class != txscript.MultiSigTy {
				// This should never happen, but be paranoid.
				continue
			}

			for _, addr := range addrs {
				k, err := addrToKey(addr, idx.chainParams)
				if err != nil {
					continue
				}

				if _, exists := idx.mpExistsAddr[k]; !exists {
					idx.mpExistsAddr[k] = struct{}{}
				}
			}
		}
	}

	for _, txOut := range tx.TxOut {
		class, addrs, _, err := txscript.ExtractPkScriptAddrs(txOut.Version,
			txOut.PkScript, idx.chainParams)
		if err != nil {
			// Non-standard outputs are skipped.
			continue
		}

		if isSStx && class == txscript.NullDataTy {
			addr, err := stake.AddrFromSStxPkScrCommitment(txOut.PkScript,
				idx.chainParams)
			if err != nil {
				// Ignore unsupported address types.
				continue
			}

			addrs = append(addrs, addr)
		}

		for _, addr := range addrs {
			k, err := addrToKey(addr, idx.chainParams)
			if err != nil {
				// Ignore unsupported address types.
				continue
			}

			if _, exists := idx.mpExistsAddr[k]; !exists {
				idx.mpExistsAddr[k] = struct{}{}
			}
		}
	}
}

// AddUnconfirmedTx is the exported form of addUnconfirmedTx.
//
// This function is safe for concurrent access.
func (idx *ExistsAddrIndex) AddUnconfirmedTx(tx *wire.MsgTx) {
	idx.unconfirmedLock.Lock()
	defer idx.unconfirmedLock.Unlock()

	idx.addUnconfirmedTx(tx)
}

// DropExistsAddrIndex drops the exists address index from the provided
// database if it exists.
func DropExistsAddrIndex(db database.DB, interrupt <-chan struct{}) error {
	return dropFlatIndex(db, existsAddrIndexKey, existsAddressIndexName,
		interrupt)
}

// DropIndex drops the exists address index from the provided database if it
// exists.
func (*ExistsAddrIndex) DropIndex(db database.DB, interrupt <-chan struct{}) error {
	return DropExistsAddrIndex(db, interrupt)
}
<|endoftext|>