<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package subnets

import (
	"github.com/juju/loggo"

	"github.com/juju/juju/api/base"
)

var logger = loggo.GetLogger("juju.api.subnets")

const subnetsFacade = "Subnets"

// API provides access to the InstancePoller API facade.
type API struct {
	facade base.FacadeCaller
}

// NewAPI creates a new client-side Subnets facade.
func NewAPI(caller base.APICaller) *API {
	if caller == nil {
		panic("caller is nil")
	}
	facadeCaller := base.NewFacadeCaller(caller, subnetsFacade)
	return &API{
		facade: facadeCaller,
	}
}
<commit_msg>AddSubnet client API method<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package subnets

import (
	"github.com/juju/loggo"
	"github.com/juju/names"

	"github.com/juju/juju/api/base"
	"github.com/juju/juju/apiserver/params"
)

var logger = loggo.GetLogger("juju.api.subnets")

const subnetsFacade = "Subnets"

// API provides access to the InstancePoller API facade.
type API struct {
	facade base.FacadeCaller
}

// NewAPI creates a new client-side Subnets facade.
func NewAPI(caller base.APICaller) *API {
	if caller == nil {
		panic("caller is nil")
	}
	facadeCaller := base.NewFacadeCaller(caller, subnetsFacade)
	return &API{
		facade: facadeCaller,
	}
}

func (api *API) AddSubnet(cidr, providerId, space string, zones []string) (params.ErrorResults, error) {
	var response params.ErrorResults
	spaceTag := names.NewSpaceTag(space).String()
	subnetTag := names.NewSubnetTag(cidr).String()
	params := params.AddSubnetsParams{
		Subnets: []params.AddSubnetParams{
			{
				SubnetTag:        subnetTag,
				SubnetProviderId: providerId,
				SpaceTag:         spaceTag,
				Zones:            zones,
			}},
	}
	err := api.facade.FacadeCall("AddSubnets", params, &response)
	return response, err
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.515"
<commit_msg>fnserver: 0.3.516 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.516"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.607"
<commit_msg>fnserver: 0.3.608 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.608"
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"fmt"
	"net"

	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup.k8s.io/v1"
	testgroupetcd "k8s.io/kubernetes/examples/apiserver/rest"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/storage/storagebackend"

	// Install the testgroup API
	_ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup.k8s.io/install"
)

const (
	// Ports on which to run the server.
	// Explicitly setting these to a different value than the default values, to prevent this from clashing with a local cluster.
	InsecurePort = 8081
	SecurePort   = 6444
)

func newStorageFactory() genericapiserver.StorageFactory {
	config := storagebackend.Config{
		Prefix:     genericapiserver.DefaultEtcdPathPrefix,
		ServerList: []string{"http://127.0.0.1:4001"},
	}
	storageFactory := genericapiserver.NewDefaultStorageFactory(config, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), genericapiserver.NewResourceConfig())

	return storageFactory
}

func NewServerRunOptions() *genericapiserver.ServerRunOptions {
	serverOptions := genericapiserver.NewServerRunOptions()
	serverOptions.InsecurePort = InsecurePort
	serverOptions.SecurePort = SecurePort
	return serverOptions
}

func Run(serverOptions *genericapiserver.ServerRunOptions) error {
	// Set ServiceClusterIPRange
	_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
	serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
	serverOptions.StorageConfig.ServerList = []string{"http://127.0.0.1:4001"}
	genericapiserver.ValidateRunOptions(serverOptions)
	config := genericapiserver.NewConfig(serverOptions)
	config.Serializer = api.Codecs
	s, err := genericapiserver.New(config)
	if err != nil {
		return fmt.Errorf("Error in bringing up the server: %v", err)
	}

	groupVersion := v1.SchemeGroupVersion
	groupName := groupVersion.Group
	groupMeta, err := registered.Group(groupName)
	if err != nil {
		return fmt.Errorf("%v", err)
	}
	storageFactory := newStorageFactory()
	storage, err := storageFactory.New(unversioned.GroupResource{Group: groupName, Resource: "testtype"})
	if err != nil {
		return fmt.Errorf("Unable to get storage: %v", err)
	}

	restStorageMap := map[string]rest.Storage{
		"testtypes": testgroupetcd.NewREST(storage, s.StorageDecorator()),
	}
	apiGroupInfo := genericapiserver.APIGroupInfo{
		GroupMeta: *groupMeta,
		VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
			groupVersion.Version: restStorageMap,
		},
		Scheme:               api.Scheme,
		NegotiatedSerializer: api.Codecs,
	}
	if err := s.InstallAPIGroups([]genericapiserver.APIGroupInfo{apiGroupInfo}); err != nil {
		return fmt.Errorf("Error in installing API: %v", err)
	}
	s.Run(serverOptions)
	return nil
}
<commit_msg>Split the storage and negotiation parts of Codecs<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"fmt"
	"net"

	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup.k8s.io/v1"
	testgroupetcd "k8s.io/kubernetes/examples/apiserver/rest"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/genericapiserver"
	"k8s.io/kubernetes/pkg/storage/storagebackend"

	// Install the testgroup API
	_ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup.k8s.io/install"
)

const (
	// Ports on which to run the server.
	// Explicitly setting these to a different value than the default values, to prevent this from clashing with a local cluster.
	InsecurePort = 8081
	SecurePort   = 6444
)

func newStorageFactory() genericapiserver.StorageFactory {
	config := storagebackend.Config{
		Prefix:     genericapiserver.DefaultEtcdPathPrefix,
		ServerList: []string{"http://127.0.0.1:4001"},
	}
	storageFactory := genericapiserver.NewDefaultStorageFactory(config, "application/json", api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), genericapiserver.NewResourceConfig())

	return storageFactory
}

func NewServerRunOptions() *genericapiserver.ServerRunOptions {
	serverOptions := genericapiserver.NewServerRunOptions()
	serverOptions.InsecurePort = InsecurePort
	serverOptions.SecurePort = SecurePort
	return serverOptions
}

func Run(serverOptions *genericapiserver.ServerRunOptions) error {
	// Set ServiceClusterIPRange
	_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
	serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
	serverOptions.StorageConfig.ServerList = []string{"http://127.0.0.1:4001"}
	genericapiserver.ValidateRunOptions(serverOptions)
	config := genericapiserver.NewConfig(serverOptions)
	config.Serializer = api.Codecs
	s, err := genericapiserver.New(config)
	if err != nil {
		return fmt.Errorf("Error in bringing up the server: %v", err)
	}

	groupVersion := v1.SchemeGroupVersion
	groupName := groupVersion.Group
	groupMeta, err := registered.Group(groupName)
	if err != nil {
		return fmt.Errorf("%v", err)
	}
	storageFactory := newStorageFactory()
	storage, err := storageFactory.New(unversioned.GroupResource{Group: groupName, Resource: "testtype"})
	if err != nil {
		return fmt.Errorf("Unable to get storage: %v", err)
	}

	restStorageMap := map[string]rest.Storage{
		"testtypes": testgroupetcd.NewREST(storage, s.StorageDecorator()),
	}
	apiGroupInfo := genericapiserver.APIGroupInfo{
		GroupMeta: *groupMeta,
		VersionedResourcesStorageMap: map[string]map[string]rest.Storage{
			groupVersion.Version: restStorageMap,
		},
		Scheme:               api.Scheme,
		NegotiatedSerializer: api.Codecs,
	}
	if err := s.InstallAPIGroups([]genericapiserver.APIGroupInfo{apiGroupInfo}); err != nil {
		return fmt.Errorf("Error in installing API: %v", err)
	}
	s.Run(serverOptions)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
)

type TNode struct {
	value interface{}
	key   int
	next  []*TNode  // List of next pointers from this.  Indexed from the bottom
	tl    *TodoList // TodoList that this is a part of
}

type TodoList struct {
	lengths  []int   // the size of the ith list
	height   int     // the height of the todolist. equal to depth - 1
	n        int     // number of things being held
	epsilon  float64 // constant
	Sentinel TNode   // sentinel[i] is the ith sentinel; access list with .next[0]
}

func NewTodoList() *TodoList {
	result := TodoList{lengths: make([]int, 1, 16)}
	result.Sentinel = TNode{next: make([]*TNode, 1, 16), tl: &result}
	result.lengths[0] = 0
	result.epsilon = 0.2
	return &result
}

func (node *TNode) Next(i int) *TNode {
	// we don't have enough allocation
	if node.tl.height-i+1 > len(node.next) {
		return nil
	}
	return node.next[node.tl.height-i]
}

func (node *TNode) SetNext(i int, next *TNode) {
	// need to allocate more by appending (maybe not the best way to do this)
	if node.tl.height-i+1 > len(node.next) {
		for len(node.next) < node.tl.height-i+1 {
			node.next = append(node.next, nil)
		}
	}
	node.next[node.tl.height-i] = next

	// txt := "nil"
	// if next != nil {
	// 	txt = fmt.Sprintf("%v", next.key)
	// }
	// fmt.Printf("L[%v]: %v -> %v created\n", i, node.key, txt)
}

// findPredecessors returns a list of *TLNodes that immediately proceed
// the key x.  result[i] = node means node is the predecessor to x in L_i
func (tl *TodoList) findPredecessors(x int) []*TNode {
	height := tl.height
	result := make([]*TNode, height+1)

	// find them
	result[0] = &tl.Sentinel // get the first sentinel
	for i := 0; i <= height; i++ {
		nextGuy := result[i].Next(i)
		if nextGuy != nil && nextGuy.key < x {
			result[i] = nextGuy
		}
		if i+1 <= height {
			result[i+1] = result[i] // go down (autoamatically)
		}
	}
	return result
}

// rebuildLayer rebuilds one layer and assumes the other layers on top will also be rebuilt.
func (tl *TodoList) rebuildLayer(i int) {
	length := 0 // new length

	// reference layer
	referenceNode := tl.Sentinel.Next(i + 1)
	iNode := &tl.Sentinel

	// delete the connections in the current layer
	for node := iNode; node != nil; {
		next := node.Next(i)
		node.SetNext(i, nil)
		node = next
	}

	second := false
	for node := referenceNode; node != nil; node = node.Next(i + 1) {
		// if we're a second thing, then build
		if second == true {
			iNode.SetNext(i, node)
			iNode = node
			length++
		}
		second = !second
	}

	// length variable changes too
	tl.lengths[i] = length
}

// rebuild does a partial rebuilding of lists L_i to L_0 modifying L_i as well
func (td *TodoList) rebuild(i int) {
	for k := i; k >= 0; k-- {
		td.rebuildLayer(k)
	}
}

func (tl *TodoList) newLayer() {
	oldHeight := tl.height

	// add new layer for sentinel
	tl.Sentinel.next = append(tl.Sentinel.next, nil)
	tl.height++

	// update lengths
	tl.lengths = append(tl.lengths, tl.lengths[oldHeight])

	// rebuild from second to last layer up
	tl.rebuild(oldHeight)
}

func (tl *TodoList) removeLayer() {
	height := tl.height
	tl.Sentinel.next = tl.Sentinel.next[:height] // removes the last thing (the first layer)
	tl.lengths = tl.lengths[1:]                  // remove the first length
	tl.height--
	tl.rebuild(height - 2)
}

func (tl *TodoList) Search(key int) (value interface{}, ok bool) {
	path := tl.findPredecessors(key)
	height := tl.height
	uh := path[height]
	nextGuy := uh.Next(height)
	if nextGuy != nil && nextGuy.key == key {
		return nextGuy.value, true
	} else {
		return nil, false
	}
}

// fixes the invariant that the first layer must have <= 1 thing in it.
// caller must check if the first layer is indeed violated
func (tl *TodoList) fixFirstLayer() {
	// first, find the smallest index i such that |L_i| <= (2-ep)^i
	i := 0
	for ; float64(tl.lengths[i]) > math.Pow(2.-tl.epsilon, float64(i)); i++ {
		// fmt.Println(i, tl.lengths[i], math.Pow(2.-tl.epsilon, float64(i)))
	}
	tl.rebuild(i - 1)
}

func (tl *TodoList) Insert(key int, value interface{}) {
	path := tl.findPredecessors(key)
	height := tl.height

	// key is already here, update the value
	if next := path[height].Next(height); next != nil && next.key == key {
		next.value = value
		return
	}

	// update internal structures
	tl.n++

	// otherwise, insert (key, value) everywhere right after path
	n := &TNode{key: key, value: value, tl: tl, next: make([]*TNode, height+1)}
	for i := height; i >= 0; i-- {
		n.SetNext(i, path[i].Next(i))
		path[i].SetNext(i, n)
		tl.lengths[i] += 1 // increase length count by 1
	}

	// rebalancing condition
	if float64(tl.lengths[height]) >= math.Ceil(math.Pow(2.0-tl.epsilon, float64(height))) {
		// fmt.Println("New Layer!")
		tl.newLayer()
	}

	// now, do partial rebuilding if there is more than 1 thing in L_0
	if tl.lengths[0] > 1 {
		// fmt.Println("Rebuild L_0 cond!")
		tl.fixFirstLayer()
	}
}

func (tl *TodoList) Delete(key int) (value interface{}, ok bool) {
	path := tl.findPredecessors(key)
	height := tl.height

	foundNode := path[height].Next(height)

	// thing wasn't found in the list
	if foundNode == nil || foundNode.key != key {
		return nil, false
	}

	// thing was found, subtract 1 from n
	tl.n--
	successorNode := foundNode.Next(height) // can be nil

	// destroy found node, create all successors
	for i := 0; i <= height; i++ {
		predecessor := path[i]

		// tNext may or may not be the found node
		tNext := predecessor.Next(i)

		// figure out chaining for L_0
		if tNext != nil && tNext.key == key {
			// if tNext is the found node, predecessor.next = tNext.next
			predecessor.SetNext(i, successorNode)
			tl.lengths[i] -= 1 // decrement by 1, found node is removed

			// make the successor appear
			tNextNext := tNext.Next(i)
			if tNextNext != nil && tNextNext.key != successorNode.key {
				successorNode.SetNext(i, tNextNext)
				tl.lengths[i] += 1
			}
		} else if tNext != nil && tNext.key != successorNode.key {
			successorNode.SetNext(i, tNext)
			tl.lengths[i] += 1
		} else if tNext == nil {
			tl.lengths[i] += 1
		}

	}

	// check to see if need to delete layers (h = depth - 1)
	if float64(tl.lengths[height]) < math.Ceil(math.Pow(2.0-tl.epsilon, float64(height-1))) {
		fmt.Println("Remove Layer!")
		tl.removeLayer()
	}

	// now, do partial rebuilding if there is more than 1 thing in L_0
	if tl.lengths[0] > 1 {
		fmt.Println("Rebuild L_0 cond!")
		tl.fixFirstLayer()
	}

	return foundNode.value, true
}

func (tl *TodoList) DebugString() string {
	result := "TodoList (debug view)\n"
	// for each L_i, print out stuff
	for i := 0; i <= tl.height; i++ {
		build := fmt.Sprintf("L[%2d]--", i)
		for node := tl.Sentinel.Next(i); node != nil; node = node.Next(i) {
			build += fmt.Sprintf("%v--", node.key)
		}
		result += build + fmt.Sprintf(" (%v)", tl.lengths[i]) + "\n"
	}
	return result
}
func (tl *TodoList) String() string {
	result := "TodoList\n"
	keys := make([]int, tl.n)

	// populate keys
	count := 0
	for node := tl.Sentinel.Next(tl.height); node != nil; node = node.Next(tl.height) {
		keys[count] = node.key
		count++
	}

	// for each L_i, print out stuff
	for i := 0; i <= tl.height; i++ {
		build := fmt.Sprintf("L[%2d]--", i)
		node := tl.Sentinel.Next(i)
		for _, v := range keys {
			if node == nil || v != node.key {
				// create string with digits equal to the number of digits in v
				digits := int(math.Floor(math.Log10(float64(v)))) + 1
				for c := 0; c < digits; c++ {
					build += " "
				}
				build += "--"
			} else {
				build += fmt.Sprintf("%d--", v)
				node = node.Next(i)
			}
		}
		result += build + fmt.Sprintf(" (%v)", tl.lengths[i]) + "\n"
	}

	return result
}

func main() {
	fmt.Printf("Hello, world!\n")

	tl := NewTodoList()
	fmt.Println(tl.String())
	// tl.Insert(0, 1)
	// fmt.Println(tl.String())
	// tl.Insert(2, 1)
	// fmt.Println(tl.String())
	// tl.Insert(3, 1)
	// fmt.Println(tl.String())
	// tl.Insert(4, 4)
	// fmt.Println(tl.String())
	// tl.Insert(5, 1)
	// fmt.Println(tl.String())
	// tl.Insert(6, 1)
	// fmt.Println(tl.String())
	// tl.Insert(7, 1)
	// fmt.Println(tl.String())
	// tl.Delete(6)
	// fmt.Println(tl.String())
	// tl.Delete(5)
	// fmt.Println(tl.String())
	// tl.Delete(7)
	// fmt.Println(tl.String())
	// tl.Delete(1)
	// fmt.Println(tl.String())
	// tl.Delete(0)
	// fmt.Println(tl.String())

	tl.Insert(1, 1)
	tl.Insert(2, 1)
	tl.Insert(3, 1)
	tl.Insert(4, 1)
	tl.Insert(5, 1)
	tl.Insert(6, 1)
	tl.Insert(7, 1)
	tl.Insert(8, 1)
	tl.Insert(9, 1)
	tl.Insert(10, 1)
	fmt.Println(tl)
	tl.Delete(3)
	fmt.Println(tl)
	tl.Delete(2)
	fmt.Println(tl)
	tl.Delete(8)
	fmt.Println(tl)
	tl.Delete(10)
	fmt.Println(tl)

	// a, ok := tl.Search(4)
	// if ok {
	// 	fmt.Println("Search returned", a)
	// } else {
	// 	fmt.Println("Alert!")
	// }
	// a, ok = tl.Search(8)
	// if !ok {
	// 	fmt.Println("Search did not return")
	// } else {
	// 	fmt.Println("Alert!")
	// }
	// fmt.Println(tl.String())
}
<commit_msg>todolist delete works<commit_after>package main

import (
	"fmt"
	"math"
	"math/rand"
	"time"
)

type TNode struct {
	value interface{}
	key   int
	next  []*TNode  // List of next pointers from this.  Indexed from the bottom
	tl    *TodoList // TodoList that this is a part of
}

type TodoList struct {
	lengths  []int   // the size of the ith list
	height   int     // the height of the todolist. equal to depth - 1
	n        int     // number of things being held
	epsilon  float64 // constant
	Sentinel TNode   // sentinel[i] is the ith sentinel; access list with .next[0]
}

func NewTodoList() *TodoList {
	result := TodoList{lengths: make([]int, 1, 16)}
	result.Sentinel = TNode{next: make([]*TNode, 1, 16), tl: &result}
	result.lengths[0] = 0
	result.epsilon = 0.2
	return &result
}

func (node *TNode) Next(i int) *TNode {
	// we don't have enough allocation
	if node.tl.height-i+1 > len(node.next) {
		return nil
	}
	return node.next[node.tl.height-i]
}

func (node *TNode) SetNext(i int, next *TNode) {
	// need to allocate more by appending (maybe not the best way to do this)
	if node.tl.height-i+1 > len(node.next) {
		for len(node.next) < node.tl.height-i+1 {
			node.next = append(node.next, nil)
		}
	}
	node.next[node.tl.height-i] = next

	// txt := "nil"
	// if next != nil {
	// 	txt = fmt.Sprintf("%v", next.key)
	// }
	// fmt.Printf("L[%v]: %v -> %v created\n", i, node.key, txt)
}

// findPredecessors returns a list of *TLNodes that immediately proceed
// the key x.  result[i] = node means node is the predecessor to x in L_i
func (tl *TodoList) findPredecessors(x int) []*TNode {
	height := tl.height
	result := make([]*TNode, height+1)

	// find them
	result[0] = &tl.Sentinel // get the first sentinel
	for i := 0; i <= height; i++ {
		nextGuy := result[i].Next(i)
		if nextGuy != nil && nextGuy.key < x {
			result[i] = nextGuy
		}
		if i+1 <= height {
			result[i+1] = result[i] // go down (autoamatically)
		}
	}
	return result
}

// rebuildLayer rebuilds one layer and assumes the other layers on top will also be rebuilt.
func (tl *TodoList) rebuildLayer(i int) {
	length := 0 // new length

	// reference layer
	referenceNode := tl.Sentinel.Next(i + 1)
	iNode := &tl.Sentinel

	// delete the connections in the current layer
	for node := iNode; node != nil; {
		next := node.Next(i)
		node.SetNext(i, nil)
		node = next
	}

	second := false
	for node := referenceNode; node != nil; node = node.Next(i + 1) {
		// if we're a second thing, then build
		if second == true {
			iNode.SetNext(i, node)
			iNode = node
			length++
			//fmt.Print(node.key, " ")
		}
		second = !second
	}
	//fmt.Print(node.key, "\n")

	// length variable changes too
	tl.lengths[i] = length
}

// rebuild does a partial rebuilding of lists L_i to L_0 modifying L_i as well
func (td *TodoList) rebuild(i int) {
	for k := i; k >= 0; k-- {
		td.rebuildLayer(k)
	}
}

func (tl *TodoList) newLayer() {
	oldHeight := tl.height

	// add new layer for sentinel
	tl.Sentinel.next = append(tl.Sentinel.next, nil)
	tl.height++

	// update lengths
	tl.lengths = append(tl.lengths, tl.lengths[oldHeight])

	// rebuild from second to last layer up
	tl.rebuild(oldHeight)
}

func (tl *TodoList) removeLayer() {
	height := tl.height
	tl.Sentinel.next = tl.Sentinel.next[:height] // removes the last thing (the first layer)
	tl.lengths = tl.lengths[1:]                  // remove the first length
	tl.height--
	tl.rebuild(height - 2)
}

func (tl *TodoList) Search(key int) (value interface{}, ok bool) {
	path := tl.findPredecessors(key)
	height := tl.height
	uh := path[height]
	nextGuy := uh.Next(height)
	if nextGuy != nil && nextGuy.key == key {
		return nextGuy.value, true
	} else {
		return nil, false
	}
}

// fixes the invariant that the first layer must have <= 1 thing in it.
// caller must check if the first layer is indeed violated
func (tl *TodoList) fixFirstLayer() {
	// first, find the smallest index i such that |L_i| <= (2-ep)^i
	i := 0
	for ; float64(tl.lengths[i]) > math.Pow(2.-tl.epsilon, float64(i)); i++ {
		// fmt.Println(i, tl.lengths[i], math.Pow(2.-tl.epsilon, float64(i)))
	}
	tl.rebuild(i - 1)
}

func (tl *TodoList) Insert(key int, value interface{}) {
	path := tl.findPredecessors(key)
	height := tl.height

	// key is already here, update the value
	if next := path[height].Next(height); next != nil && next.key == key {
		next.value = value
		return
	}

	// update internal structures
	tl.n++

	// otherwise, insert (key, value) everywhere right after path
	n := &TNode{key: key, value: value, tl: tl, next: make([]*TNode, height+1)}
	for i := height; i >= 0; i-- {
		n.SetNext(i, path[i].Next(i))
		path[i].SetNext(i, n)
		tl.lengths[i] += 1 // increase length count by 1
	}

	// rebalancing condition
	if float64(tl.lengths[height]) >= math.Ceil(math.Pow(2.0-tl.epsilon, float64(height))) {
		// fmt.Println("New Layer!")
		tl.newLayer()
	}

	// now, do partial rebuilding if there is more than 1 thing in L_0
	if tl.lengths[0] > 1 {
		// fmt.Println("Rebuild L_0 cond!")
		tl.fixFirstLayer()
	}
}

func (tl *TodoList) Delete(key int) (value interface{}, ok bool) {
	path := tl.findPredecessors(key)
	height := tl.height

	//fmt.Println("Delete", key)

	foundNode := path[height].Next(height)

	// thing wasn't found in the list
	if foundNode == nil || foundNode.key != key {
		return nil, false
	}

	// thing was found, subtract 1 from n
	tl.n--
	successorNode := foundNode.Next(height) // can be nil

	// destroy found node, create all successors
	for i := 0; i <= height; i++ {
		predecessor := path[i]

		// tNext may or may not be the found node
		tNext := predecessor.Next(i)

		// case 1
		if tNext == nil {
			predecessor.SetNext(i, successorNode)
			if successorNode != nil {
				tl.lengths[i] += 1
			}

			// case 2
		} else if successorNode != nil && tNext.key == successorNode.key {
			// do nothing

			// cases 3-5
		} else if tNext.key == foundNode.key {
			tNextNext := tNext.Next(i)
			// case 3
			if tNextNext == nil {
				predecessor.SetNext(i, successorNode)
				if successorNode == nil {
					tl.lengths[i] -= 1
				}
				// case 5
			} else if tNextNext.key == successorNode.key {
				predecessor.SetNext(i, successorNode)
				tl.lengths[i] -= 1
				// case 4
			} else {
				predecessor.SetNext(i, successorNode)
				successorNode.SetNext(i, tNextNext)
			}
			// case 6, pre -> -> -> 11
		} else {
			predecessor.SetNext(i, successorNode)
			successorNode.SetNext(i, tNext)
			tl.lengths[i] += 1
		}
	}

	//fmt.Println("after sky high", tl.String())
	// check to see if need to delete layers (h = depth - 1)
	if float64(tl.lengths[height]) < math.Ceil(math.Pow(2.0-tl.epsilon, float64(height-1))) {
		//fmt.Println("Remove Layer!")
		tl.removeLayer()
		//fmt.Println("after remove layer", tl.String())
	}

	// now, do partial rebuilding if there is more than 1 thing in L_0
	if tl.lengths[0] > 1 {
		//fmt.Println("Rebuild L_0 cond!")
		tl.fixFirstLayer()
		//fmt.Println("after rebuild L_0", tl.String())
	}

	return foundNode.value, true
}

func (tl *TodoList) DebugString() string {
	result := "TodoList (debug view)\n"
	// for each L_i, print out stuff
	for i := 0; i <= tl.height; i++ {
		build := fmt.Sprintf("L[%2d]--", i)
		for node := tl.Sentinel.Next(i); node != nil; node = node.Next(i) {
			build += fmt.Sprintf("%v--", node.key)
		}
		result += build + fmt.Sprintf(" (%v)", tl.lengths[i]) + "\n"
	}
	return result
}
func (tl *TodoList) String() string {
	result := "TodoList\n"
	keys := make([]int, tl.n)

	// populate keys
	count := 0
	for node := tl.Sentinel.Next(tl.height); node != nil; node = node.Next(tl.height) {
		keys[count] = node.key
		count++
	}

	// for each L_i, print out stuff
	for i := 0; i <= tl.height; i++ {
		build := fmt.Sprintf("L[%2d]--", i)
		node := tl.Sentinel.Next(i)
		for _, v := range keys {
			if node == nil || v != node.key {
				// create string with digits equal to the number of digits in v
				digits := int(math.Floor(math.Log10(float64(v)))) + 1
				if v == 0 {
					digits = 1
				}
				for c := 0; c < digits; c++ {
					build += " "
				}
				build += "--"
			} else {
				build += fmt.Sprintf("%d--", v)
				node = node.Next(i)
			}
		}
		result += build + fmt.Sprintf(" (%v)", tl.lengths[i]) + "\n"
	}

	return result
}

func main() {
	fmt.Printf("Hello, world!\n")

	tl := NewTodoList()
	fmt.Println(tl.String())
	// tl.Insert(0, 1)
	// fmt.Println(tl.String())
	// tl.Insert(2, 1)
	// fmt.Println(tl.String())
	// tl.Insert(3, 1)
	// fmt.Println(tl.String())
	// tl.Insert(4, 4)
	// fmt.Println(tl.String())
	// tl.Insert(5, 1)
	// fmt.Println(tl.String())
	// tl.Insert(6, 1)
	// fmt.Println(tl.String())
	// tl.Insert(7, 1)
	// fmt.Println(tl.String())
	// tl.Delete(6)
	// fmt.Println(tl.String())
	// tl.Delete(5)
	// fmt.Println(tl.String())
	// tl.Delete(7)
	// fmt.Println(tl.String())
	// tl.Delete(1)
	// fmt.Println(tl.String())
	// tl.Delete(0)
	// fmt.Println(tl.String())
	nums1 := []int{11, 0, 6, 9, 7, 4, 2, 3, 1, 8, 10, 5}
	nums2 := []int{9, 7, 6, 8, 3, 4, 11, 1, 0, 10, 5, 2}
	rand.Seed(time.Now().UTC().UnixNano())
	// for i := 0; i < 10; i++ {
	// 	nums := rand.Perm(12)
	// 	fmt.Println(nums)
	// 	for _, v := range nums {
	// 		tl.Insert(v, v)
	// 		//fmt.Println(tl.String())
	// 	}
	// 	nums = rand.Perm(12)
	// 	fmt.Println(nums)
	// 	for _, v := range nums {
	// 		tl.Delete(v)
	// 		// fmt.Println(tl.String())
	// 	}
	// }
	// nums = rand.Perm(12)
	// fmt.Println(nums)
	for _, v := range nums1 {
		tl.Insert(v, v)
		fmt.Println(tl.String())
	}
	// nums = rand.Perm(12)
	// fmt.Println(nums)
	for _, v := range nums2 {
		tl.Delete(v)
		fmt.Println(tl.String())
	}

	// a, ok := tl.Search(4)
	// if ok {
	// 	fmt.Println("Search returned", a)
	// } else {
	// 	fmt.Println("Alert!")
	// }
	// a, ok = tl.Search(8)
	// if !ok {
	// 	fmt.Println("Search did not return")
	// } else {
	// 	fmt.Println("Alert!")
	// }
	// fmt.Println(tl.String())
}
<|endoftext|>


<commit_before>package forestdb

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

//#cgo LDFLAGS: -lforestdb
//#include <stdlib.h>
//#include <libforestdb/forestdb.h>
import "C"

import (
	"unsafe"
)

// Database handle
type File struct {
	dbfile *C.fdb_file_handle
}

// Open opens the database with a given file name
func Open(filename string, config *Config) (*File, error) {

	if config == nil {
		config = DefaultConfig()
	}

	dbname := C.CString(filename)
	defer C.free(unsafe.Pointer(dbname))

	rv := File{}
	Log.Tracef("fdb_open call rv:%p dbname:%v conf:%v", &rv, dbname, config.config)
	errNo := C.fdb_open(&rv.dbfile, dbname, config.config)
	Log.Tracef("fdb_open ret rv:%p errNo:%v rv:%v", &rv, errNo, rv)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// Options to be passed to Commit()
type CommitOpt uint8

const (
	// Perform commit without any options.
	COMMIT_NORMAL CommitOpt = 0x00
	// Manually flush WAL entries even though it doesn't reach the configured threshol
	COMMIT_MANUAL_WAL_FLUSH CommitOpt = 0x01
)

// Commit all pending changes into disk.
func (f *File) Commit(opt CommitOpt) error {
	Log.Tracef("fdb_commit call f:%p dbfile:%v opt:%v", f, f.dbfile, opt)
	errNo := C.fdb_commit(f.dbfile, C.fdb_commit_opt_t(opt))
	Log.Tracef("fdb_commit retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Compact the current database file and create a new compacted file
func (f *File) Compact(newfilename string) error {

	fn := C.CString(newfilename)
	defer C.free(unsafe.Pointer(fn))

	Log.Tracef("fdb_compact call f:%p dbfile:%v fn:%v", f, f.dbfile, fn)
	errNo := C.fdb_compact(f.dbfile, fn)
	Log.Tracef("fdb_compact retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// CompactUpto compacts the current database file upto given snapshot marker
//and creates a new compacted file
func (f *File) CompactUpto(newfilename string, sm *SnapMarker) error {

	fn := C.CString(newfilename)
	defer C.free(unsafe.Pointer(fn))

	Log.Tracef("fdb_compact_upto call f:%p dbfile:%v fn:%v marker:%v", f, f.dbfile, fn, sm.marker)
	errNo := C.fdb_compact_upto(f.dbfile, fn, sm.marker)
	Log.Tracef("fdb_compact_upto retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// EstimateSpaceUsed returns the overall disk space actively used by the current database file
func (f *File) EstimateSpaceUsed() int {
	Log.Tracef("fdb_estimate_space_used call f:%p dbfile:%v", f, f.dbfile)
	rv := int(C.fdb_estimate_space_used(f.dbfile))
	Log.Tracef("fdb_estimate_space_used retn f:%p rv:%v", f, rv)
	return rv
}

// DbInfo returns the information about a given database handle
func (f *File) Info() (*FileInfo, error) {
	rv := FileInfo{}
	Log.Tracef("fdb_get_file_info call f:%p dbfile:%v", f, f.dbfile)
	errNo := C.fdb_get_file_info(f.dbfile, &rv.info)
	Log.Tracef("fdb_get_file_info retn f:%p errNo:%v, info:%v", f, errNo, rv.info)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// FIXME implement fdb_switch_compaction_mode

// Close the database file
func (f *File) Close() error {
	Log.Tracef("fdb_close call f:%p dbfile:%v", f, f.dbfile)
	errNo := C.fdb_close(f.dbfile)
	Log.Tracef("fdb_close retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// OpenKVStore opens the named KVStore within the File
// using the provided KVStoreConfig.  If config is
// nil the DefaultKVStoreConfig() will be used.
func (f *File) OpenKVStore(name string, config *KVStoreConfig) (*KVStore, error) {
	if config == nil {
		config = DefaultKVStoreConfig()
	}

	rv := KVStore{
		f: f,
	}
	kvsname := C.CString(name)
	defer C.free(unsafe.Pointer(kvsname))
	Log.Tracef("fdb_kvs_open call f:%p dbfile:%v kvsname:%v config:%v", f, f.dbfile, kvsname, config.config)
	errNo := C.fdb_kvs_open(f.dbfile, &rv.db, kvsname, config.config)
	Log.Tracef("fdb_kvs_open retn f:%p errNo:%v db:%v", f, errNo, rv.db)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// OpenKVStore opens the default KVStore within the File
// using the provided KVStoreConfig.  If config is
// nil the DefaultKVStoreConfig() will be used.
func (f *File) OpenKVStoreDefault(config *KVStoreConfig) (*KVStore, error) {
	return f.OpenKVStore("default", config)
}

// Destroy destroys all resources associated with a ForestDB file permanently
func Destroy(filename string, config *Config) error {

	if config == nil {
		config = DefaultConfig()
	}

	dbname := C.CString(filename)
	defer C.free(unsafe.Pointer(dbname))

	Log.Tracef("fdb_destroy call dbname:%v config:%v", dbname, config.config)
	errNo := C.fdb_destroy(dbname, config.config)
	Log.Tracef("fdb_destroy retn dbname:%v errNo:%v", dbname, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}
<commit_msg>add support for fdb_switch_compaction_mode<commit_after>package forestdb

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

//#cgo LDFLAGS: -lforestdb
//#include <stdlib.h>
//#include <libforestdb/forestdb.h>
import "C"

import (
	"unsafe"
)

// Database handle
type File struct {
	dbfile *C.fdb_file_handle
}

// Open opens the database with a given file name
func Open(filename string, config *Config) (*File, error) {

	if config == nil {
		config = DefaultConfig()
	}

	dbname := C.CString(filename)
	defer C.free(unsafe.Pointer(dbname))

	rv := File{}
	Log.Tracef("fdb_open call rv:%p dbname:%v conf:%v", &rv, dbname, config.config)
	errNo := C.fdb_open(&rv.dbfile, dbname, config.config)
	Log.Tracef("fdb_open ret rv:%p errNo:%v rv:%v", &rv, errNo, rv)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// Options to be passed to Commit()
type CommitOpt uint8

const (
	// Perform commit without any options.
	COMMIT_NORMAL CommitOpt = 0x00
	// Manually flush WAL entries even though it doesn't reach the configured threshol
	COMMIT_MANUAL_WAL_FLUSH CommitOpt = 0x01
)

// Commit all pending changes into disk.
func (f *File) Commit(opt CommitOpt) error {
	Log.Tracef("fdb_commit call f:%p dbfile:%v opt:%v", f, f.dbfile, opt)
	errNo := C.fdb_commit(f.dbfile, C.fdb_commit_opt_t(opt))
	Log.Tracef("fdb_commit retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Compact the current database file and create a new compacted file
func (f *File) Compact(newfilename string) error {

	fn := C.CString(newfilename)
	defer C.free(unsafe.Pointer(fn))

	Log.Tracef("fdb_compact call f:%p dbfile:%v fn:%v", f, f.dbfile, fn)
	errNo := C.fdb_compact(f.dbfile, fn)
	Log.Tracef("fdb_compact retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// CompactUpto compacts the current database file upto given snapshot marker
//and creates a new compacted file
func (f *File) CompactUpto(newfilename string, sm *SnapMarker) error {

	fn := C.CString(newfilename)
	defer C.free(unsafe.Pointer(fn))

	Log.Tracef("fdb_compact_upto call f:%p dbfile:%v fn:%v marker:%v", f, f.dbfile, fn, sm.marker)
	errNo := C.fdb_compact_upto(f.dbfile, fn, sm.marker)
	Log.Tracef("fdb_compact_upto retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// EstimateSpaceUsed returns the overall disk space actively used by the current database file
func (f *File) EstimateSpaceUsed() int {
	Log.Tracef("fdb_estimate_space_used call f:%p dbfile:%v", f, f.dbfile)
	rv := int(C.fdb_estimate_space_used(f.dbfile))
	Log.Tracef("fdb_estimate_space_used retn f:%p rv:%v", f, rv)
	return rv
}

// DbInfo returns the information about a given database handle
func (f *File) Info() (*FileInfo, error) {
	rv := FileInfo{}
	Log.Tracef("fdb_get_file_info call f:%p dbfile:%v", f, f.dbfile)
	errNo := C.fdb_get_file_info(f.dbfile, &rv.info)
	Log.Tracef("fdb_get_file_info retn f:%p errNo:%v, info:%v", f, errNo, rv.info)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// SwitchCompactionMode changes the compaction mode of a ForestDB file
// if the mode is changed to COMPACT_AUTO the compaction
// threshold is set to the threshold passed to this API
func (f *File) SwitchCompactionMode(mode CompactOpt, threshold int) error {
	errNo := C.fdb_switch_compaction_mode(f.dbfile, C.fdb_compaction_mode_t(mode), C.size_t(threshold))
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Close the database file
func (f *File) Close() error {
	Log.Tracef("fdb_close call f:%p dbfile:%v", f, f.dbfile)
	errNo := C.fdb_close(f.dbfile)
	Log.Tracef("fdb_close retn f:%p errNo:%v", f, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// OpenKVStore opens the named KVStore within the File
// using the provided KVStoreConfig.  If config is
// nil the DefaultKVStoreConfig() will be used.
func (f *File) OpenKVStore(name string, config *KVStoreConfig) (*KVStore, error) {
	if config == nil {
		config = DefaultKVStoreConfig()
	}

	rv := KVStore{
		f: f,
	}
	kvsname := C.CString(name)
	defer C.free(unsafe.Pointer(kvsname))
	Log.Tracef("fdb_kvs_open call f:%p dbfile:%v kvsname:%v config:%v", f, f.dbfile, kvsname, config.config)
	errNo := C.fdb_kvs_open(f.dbfile, &rv.db, kvsname, config.config)
	Log.Tracef("fdb_kvs_open retn f:%p errNo:%v db:%v", f, errNo, rv.db)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// OpenKVStore opens the default KVStore within the File
// using the provided KVStoreConfig.  If config is
// nil the DefaultKVStoreConfig() will be used.
func (f *File) OpenKVStoreDefault(config *KVStoreConfig) (*KVStore, error) {
	return f.OpenKVStore("default", config)
}

// Destroy destroys all resources associated with a ForestDB file permanently
func Destroy(filename string, config *Config) error {

	if config == nil {
		config = DefaultConfig()
	}

	dbname := C.CString(filename)
	defer C.free(unsafe.Pointer(dbname))

	Log.Tracef("fdb_destroy call dbname:%v config:%v", dbname, config.config)
	errNo := C.fdb_destroy(dbname, config.config)
	Log.Tracef("fdb_destroy retn dbname:%v errNo:%v", dbname, errNo)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package siv

import (
	"bytes"
	"github.com/jacobsa/aes/common"
)

var dblRb []byte

func init() {
	dblRb = append(bytes.Repeat([]byte{0x00}, 15), 0x87)
}

// Given a 128-bit binary string, shift the string left by one bit and XOR the
// result with 0x00...87 if the bit shifted off was one. This is the dbl
// function of RFC 5297.
func dbl(b []byte) []byte {
	if len(b) != 16 {
		panic("Invalid length.")
	}

	shiftedOne := common.Msb(b) == 1
	b = common.ShiftLeft(b)
	if shiftedOne {
		b = common.Xor(b, dblRb)
	}

	return b
}
<commit_msg>Fixed a bug.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package siv

import (
	"bytes"
	"github.com/jacobsa/aes/common"
)

var dblRb []byte

func init() {
	dblRb = append(bytes.Repeat([]byte{0x00}, 15), 0x87)
}

// Given a 128-bit binary string, shift the string left by one bit and XOR the
// result with 0x00...87 if the bit shifted off was one. This is the dbl
// function of RFC 5297.
func dbl(b []byte) []byte {
	if len(b) != 16 {
		panic("dbl requires a 16-byte buffer.")
	}

	shiftedOne := common.Msb(b) == 1
	b = common.ShiftLeft(b)
	if shiftedOne {
		b = common.Xor(b, dblRb)
	}

	return b
}
<|endoftext|>


<commit_before>package main

// This utility moves a network interface into a network namespace and
// configures it. The configuration is passed in as a JSON object
// (marshalled prot.NetworkAdapter).  It is necessary to implement
// this as a separate utility as in Go one does not have tight control
// over which OS thread a given Go thread/routing executes but as can
// only enter a namespace with a specific OS thread.
//
// Note, this logs to stdout so that the caller (gcs) can log the
// output itself.

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"

	"github.com/Microsoft/opengcs/service/gcs/prot"
	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

func netnsConfigMain() {
	if err := netnsConfig(); err != nil {
		log.Fatal("netnsConfig returned: ", err)
	}
	os.Exit(0)
}

func netnsConfig() error {
	ifStr := flag.String("if", "", "Interface/Adapter to move/configure")
	nspid := flag.Int("nspid", -1, "Process ID (to locate netns")
	cfgStr := flag.String("cfg", "", "Adapter configuration (json)")

	flag.Parse()
	if *ifStr == "" || *nspid == -1 || *cfgStr == "" {
		return fmt.Errorf("All three arguments must be specified")
	}

	var a prot.NetworkAdapter
	if err := json.Unmarshal([]byte(*cfgStr), &a); err != nil {
		return err
	}

	if a.NatEnabled {
		log.Infof("Configure %s in %d with: %s/%d gw=%s", *ifStr, *nspid, a.AllocatedIPAddress, a.HostIPPrefixLength, a.HostIPAddress)
	} else {
		log.Infof("Configure %s in %s with DHCP", *ifStr, *nspid)
	}

	// Lock the OS Thread so we don't accidentally switch namespaces
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// Stash current network namespace away and make sure we enter it as we leave
	origNS, err := netns.Get()
	if err != nil {
		return fmt.Errorf("netns.Get() failed: %v", err)
	}
	defer origNS.Close()

	// Get a reference to the new network namespace
	ns, err := netns.GetFromPid(*nspid)
	if err != nil {
		return fmt.Errorf("netns.GetFromPid(%d) failed: %v", *nspid, err)
	}
	defer ns.Close()

	// Get a reference to the interface and make sure it's down
	link, err := netlink.LinkByName(*ifStr)
	if err != nil {
		return fmt.Errorf("netlink.LinkByName(%s) failed: %v", *ifStr, err)
	}
	if err := netlink.LinkSetDown(link); err != nil {
		return fmt.Errorf("netlink.LinkSetDown(%#v) failed: %v", link, err)
	}

	// Move the interface to the new network namespace
	if err := netlink.LinkSetNsPid(link, *nspid); err != nil {
		return fmt.Errorf("netlink.SetNsPid(%#v, %d) failed: %v", link, *nspid, err)
	}

	log.Infof("Switching from %v to %v", origNS, ns)

	// Enter the new network namespace
	if err := netns.Set(ns); err != nil {
		return fmt.Errorf("netns.Set() failed: %v", err)
	}

	// Re-Get a reference to the interface (it may be a different ID in the new namespace)
	link, err = netlink.LinkByName(*ifStr)
	if err != nil {
		return fmt.Errorf("netlink.LinkByName(%s) failed: %v", *ifStr, err)
	}

	// User requested non-default MTU size
	if a.EncapOverhead != 0 {
		mtu := link.Attrs().MTU - int(a.EncapOverhead)
		if err = netlink.LinkSetMTU(link, mtu); err != nil {
			return fmt.Errorf("netlink.LinkSetMTU(%#v, %d) failed: %v", link, mtu, err)
		}
	}

	// Configure the interface
	if a.NatEnabled {
		metric := 1
		if a.EnableLowMetric {
			metric = 500
		}

		// Bring the interface up
		if err := netlink.LinkSetUp(link); err != nil {
			return fmt.Errorf("netlink.LinkSetUp(%#v) failed: %v", link, err)
		}
		// Set IP address
		addr := &net.IPNet{
			IP: net.ParseIP(a.AllocatedIPAddress),
			// TODO(rn): This assumes/hardcodes IPv4
			Mask: net.CIDRMask(int(a.HostIPPrefixLength), 32)}
		ipAddr := &netlink.Addr{IPNet: addr, Label: ""}
		if err := netlink.AddrAdd(link, ipAddr); err != nil {
			return fmt.Errorf("netlink.AddrAdd(%#v, %#v) failed: %v", link, ipAddr, err)
		}
		// Set gateway
		if a.HostIPAddress != "" {
			gw := net.ParseIP(a.HostIPAddress)
			route := netlink.Route{
				Scope:     netlink.SCOPE_UNIVERSE,
				LinkIndex: link.Attrs().Index,
				Gw:        gw,
				Priority:  metric, // This is what ip route add does
			}
			if err := netlink.RouteAdd(&route); err != nil {
				return fmt.Errorf("netlink.RouteAdd(%#v) failed: %v", route, err)
			}
		}
	} else {
		err := exec.Command(
			"udhcpc",
			"-q",
			"-i", *ifStr,
			"-s", "/sbin/udhcpc_config.script").Run()
		if err != nil {
			return fmt.Errorf("udhcpc failed: %v", err)
		}
	}

	// Add some debug logging
	curNS, _ := netns.Get()
	// Refresh link attributes/state
	link, _ = netlink.LinkByIndex(link.Attrs().Index)
	attr := link.Attrs()
	addrs, _ := netlink.AddrList(link, 0)
	log.Infof("%v: %s[idx=%d,type=%s] is %v", curNS, attr.Name, attr.Index, link.Type(), attr.OperState)
	for _, addr := range addrs {
		log.Infof("  %v", addr)
	}

	return nil
}
<commit_msg>added support for setting a gateway ip that's outside a subnet<commit_after>package main

// This utility moves a network interface into a network namespace and
// configures it. The configuration is passed in as a JSON object
// (marshalled prot.NetworkAdapter).  It is necessary to implement
// this as a separate utility as in Go one does not have tight control
// over which OS thread a given Go thread/routing executes but as can
// only enter a namespace with a specific OS thread.
//
// Note, this logs to stdout so that the caller (gcs) can log the
// output itself.

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"

	"github.com/Microsoft/opengcs/service/gcs/prot"
	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
)

func netnsConfigMain() {
	if err := netnsConfig(); err != nil {
		log.Fatal("netnsConfig returned: ", err)
	}
	os.Exit(0)
}

func netnsConfig() error {
	ifStr := flag.String("if", "", "Interface/Adapter to move/configure")
	nspid := flag.Int("nspid", -1, "Process ID (to locate netns")
	cfgStr := flag.String("cfg", "", "Adapter configuration (json)")

	flag.Parse()
	if *ifStr == "" || *nspid == -1 || *cfgStr == "" {
		return fmt.Errorf("All three arguments must be specified")
	}

	var a prot.NetworkAdapter
	if err := json.Unmarshal([]byte(*cfgStr), &a); err != nil {
		return err
	}

	if a.NatEnabled {
		log.Infof("Configure %s in %d with: %s/%d gw=%s", *ifStr, *nspid, a.AllocatedIPAddress, a.HostIPPrefixLength, a.HostIPAddress)
	} else {
		log.Infof("Configure %s in %s with DHCP", *ifStr, *nspid)
	}

	// Lock the OS Thread so we don't accidentally switch namespaces
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// Stash current network namespace away and make sure we enter it as we leave
	origNS, err := netns.Get()
	if err != nil {
		return fmt.Errorf("netns.Get() failed: %v", err)
	}
	defer origNS.Close()

	// Get a reference to the new network namespace
	ns, err := netns.GetFromPid(*nspid)
	if err != nil {
		return fmt.Errorf("netns.GetFromPid(%d) failed: %v", *nspid, err)
	}
	defer ns.Close()

	// Get a reference to the interface and make sure it's down
	link, err := netlink.LinkByName(*ifStr)
	if err != nil {
		return fmt.Errorf("netlink.LinkByName(%s) failed: %v", *ifStr, err)
	}
	if err := netlink.LinkSetDown(link); err != nil {
		return fmt.Errorf("netlink.LinkSetDown(%#v) failed: %v", link, err)
	}

	// Move the interface to the new network namespace
	if err := netlink.LinkSetNsPid(link, *nspid); err != nil {
		return fmt.Errorf("netlink.SetNsPid(%#v, %d) failed: %v", link, *nspid, err)
	}

	log.Infof("Switching from %v to %v", origNS, ns)

	// Enter the new network namespace
	if err := netns.Set(ns); err != nil {
		return fmt.Errorf("netns.Set() failed: %v", err)
	}

	// Re-Get a reference to the interface (it may be a different ID in the new namespace)
	link, err = netlink.LinkByName(*ifStr)
	if err != nil {
		return fmt.Errorf("netlink.LinkByName(%s) failed: %v", *ifStr, err)
	}

	// User requested non-default MTU size
	if a.EncapOverhead != 0 {
		mtu := link.Attrs().MTU - int(a.EncapOverhead)
		if err = netlink.LinkSetMTU(link, mtu); err != nil {
			return fmt.Errorf("netlink.LinkSetMTU(%#v, %d) failed: %v", link, mtu, err)
		}
	}

	// Configure the interface
	if a.NatEnabled {
		metric := 1
		if a.EnableLowMetric {
			metric = 500
		}

		// Bring the interface up
		if err := netlink.LinkSetUp(link); err != nil {
			return fmt.Errorf("netlink.LinkSetUp(%#v) failed: %v", link, err)
		}
		// Set IP address
		addr := &net.IPNet{
			IP: net.ParseIP(a.AllocatedIPAddress),
			// TODO(rn): This assumes/hardcodes IPv4
			Mask: net.CIDRMask(int(a.HostIPPrefixLength), 32)}
		ipAddr := &netlink.Addr{IPNet: addr, Label: ""}
		if err := netlink.AddrAdd(link, ipAddr); err != nil {
			return fmt.Errorf("netlink.AddrAdd(%#v, %#v) failed: %v", link, ipAddr, err)
		}
		// Set gateway
		if a.HostIPAddress != "" {
			gw := net.ParseIP(a.HostIPAddress)

			if !addr.Contains(gw) {
				// In the case that a gw is not part of the subnet we are setting gw for,
				// a new addr containing this gw address need to be added into the link to avoid getting
				// unreachable error when adding this out-of-subnet gw route
				log.Infof("gw is outside of the subnet: Configure %s in %d with: %s/%d gw=%s\n",
					   *ifStr, *nspid, a.AllocatedIPAddress, a.HostIPPrefixLength, a.HostIPAddress)
				addr2 := &net.IPNet{
						IP: net.ParseIP(a.HostIPAddress),
						Mask: net.CIDRMask(24, 32)} // This assumes/hardcodes IPv4
				ipAddr2 := &netlink.Addr{IPNet: addr2, Label: ""}
				if err := netlink.AddrAdd(link, ipAddr2); err != nil {
					return fmt.Errorf("netlink.AddrAdd(%#v, %#v) failed: %v", link, ipAddr2, err)
				}
			}
			route := netlink.Route{
				Scope:     netlink.SCOPE_UNIVERSE,
				LinkIndex: link.Attrs().Index,
				Gw:        gw,
				Priority:  metric, // This is what ip route add does
			}
			if err := netlink.RouteAdd(&route); err != nil {
				return fmt.Errorf("netlink.RouteAdd(%#v) failed: %v", route, err)
			}
		}
	} else {
		err := exec.Command(
			"udhcpc",
			"-q",
			"-i", *ifStr,
			"-s", "/sbin/udhcpc_config.script").Run()
		if err != nil {
			return fmt.Errorf("udhcpc failed: %v", err)
		}
	}

	// Add some debug logging
	curNS, _ := netns.Get()
	// Refresh link attributes/state
	link, _ = netlink.LinkByIndex(link.Attrs().Index)
	attr := link.Attrs()
	addrs, _ := netlink.AddrList(link, 0)
	log.Infof("%v: %s[idx=%d,type=%s] is %v", curNS, attr.Name, attr.Index, link.Type(), attr.OperState)
	for _, addr := range addrs {
		log.Infof("  %v", addr)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package processor

import (
	"encoding/json"
	"fmt"

	"github.com/Jeffail/benthos/lib/log"
	"github.com/Jeffail/benthos/lib/message"
	"github.com/Jeffail/benthos/lib/message/mapper"
	"github.com/Jeffail/benthos/lib/metrics"
	"github.com/Jeffail/benthos/lib/processor/condition"
	"github.com/Jeffail/benthos/lib/types"
)

//------------------------------------------------------------------------------

func init() {
	Constructors[TypeProcessMap] = TypeSpec{
		constructor: func(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
			return NewProcessMap(conf.ProcessMap, mgr, log, stats)
		},
		description: `
A processor that extracts and maps fields from the original payload into new
objects, applies a list of processors to the newly constructed objects, and
finally maps the result back into the original payload.

This processor is useful for performing processors on subsections of a payload.
For example, you could extract sections of a JSON object in order to construct
a request object for an ` + "`http`" + ` processor, then map the result back
into a field within the original object.

The order of stages of this processor are as follows:

- Conditions are applied to each _individual_ message part in the batch,
  determining whether the part will be mapped. If the conditions are empty all
  message parts will be mapped. If the field ` + "`parts`" + ` is populated the
  message parts not in this list are also excluded from mapping.
- Message parts that are flagged for mapping are mapped according to the premap
  fields, creating a new object. If the premap stage fails (targets are not
  found) the message part will not be processed.
- Message parts that are mapped are processed as a batch. You may safely break
  the batch into individual parts during processing with the ` + "`split`" + `
  processor.
- After all child processors are applied to the mapped messages they are mapped
  back into the original message parts they originated from as per your postmap.
  If the postmap stage fails the mapping is skipped and the message payload
  remains as it started.

Map paths are arbitrary dot paths, target path hierarchies are constructed if
they do not yet exist. Processing is skipped for message parts where the premap
targets aren't found, for optional premap targets use ` + "`premap_optional`" + `.

Map target paths that are parents of other map target paths will always be
mapped first, therefore it is possible to map subpath overrides.

If postmap targets are not found the merge is abandoned, for optional postmap
targets use ` + "`postmap_optional`" + `.

If the premap is empty then the full payload is sent to the processors, if the
postmap is empty then the processed result replaces the original contents
entirely.

Maps can reference the root of objects either with an empty string or '.', for
example the maps:

` + "``` yaml" + `
premap:
  .: foo.bar
postmap:
  foo.bar: .
` + "```" + `

Would create a new object where the root is the value of ` + "`foo.bar`" + ` and
would map the full contents of the result back into ` + "`foo.bar`" + `.

If the number of total message parts resulting from the processing steps does
not match the original count then this processor fails and the messages continue
unchanged. Therefore, you should avoid using batch and filter type processors in
this list.

### Batch Ordering

This processor supports batch messages. When message parts are post-mapped after
processing they will be correctly aligned with the original batch. However, the
ordering of premapped message parts as they are sent through processors are not
guaranteed to match the ordering of the original batch.`,
		sanitiseConfigFunc: func(conf Config) (interface{}, error) {
			return conf.ProcessMap.Sanitise()
		},
	}
}

//------------------------------------------------------------------------------

// ProcessMapConfig is a config struct containing fields for the
// ProcessMap processor.
type ProcessMapConfig struct {
	Parts           []int              `json:"parts" yaml:"parts"`
	Conditions      []condition.Config `json:"conditions" yaml:"conditions"`
	Premap          map[string]string  `json:"premap" yaml:"premap"`
	PremapOptional  map[string]string  `json:"premap_optional" yaml:"premap_optional"`
	Postmap         map[string]string  `json:"postmap" yaml:"postmap"`
	PostmapOptional map[string]string  `json:"postmap_optional" yaml:"postmap_optional"`
	Processors      []Config           `json:"processors" yaml:"processors"`
}

// NewProcessMapConfig returns a default ProcessMapConfig.
func NewProcessMapConfig() ProcessMapConfig {
	return ProcessMapConfig{
		Parts:           []int{},
		Conditions:      []condition.Config{},
		Premap:          map[string]string{},
		PremapOptional:  map[string]string{},
		Postmap:         map[string]string{},
		PostmapOptional: map[string]string{},
		Processors:      []Config{},
	}
}

// Sanitise the configuration into a minimal structure that can be printed
// without changing the intent.
func (p ProcessMapConfig) Sanitise() (map[string]interface{}, error) {
	var err error
	condConfs := make([]interface{}, len(p.Conditions))
	for i, cConf := range p.Conditions {
		if condConfs[i], err = condition.SanitiseConfig(cConf); err != nil {
			return nil, err
		}
	}
	procConfs := make([]interface{}, len(p.Processors))
	for i, pConf := range p.Processors {
		if procConfs[i], err = SanitiseConfig(pConf); err != nil {
			return nil, err
		}
	}
	return map[string]interface{}{
		"parts":            p.Parts,
		"conditions":       condConfs,
		"premap":           p.Premap,
		"premap_optional":  p.PremapOptional,
		"postmap":          p.Postmap,
		"postmap_optional": p.PostmapOptional,
		"processors":       procConfs,
	}, nil
}

//------------------------------------------------------------------------------

// UnmarshalJSON ensures that when parsing configs that are in a slice the
// default values are still applied.
func (p *ProcessMapConfig) UnmarshalJSON(bytes []byte) error {
	type confAlias ProcessMapConfig
	aliased := confAlias(NewProcessMapConfig())

	if err := json.Unmarshal(bytes, &aliased); err != nil {
		return err
	}

	*p = ProcessMapConfig(aliased)
	return nil
}

// UnmarshalYAML ensures that when parsing configs that are in a slice the
// default values are still applied.
func (p *ProcessMapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type confAlias ProcessMapConfig
	aliased := confAlias(NewProcessMapConfig())

	if err := unmarshal(&aliased); err != nil {
		return err
	}

	*p = ProcessMapConfig(aliased)
	return nil
}

//------------------------------------------------------------------------------

// ProcessMap is a processor that applies a list of child processors to a new
// payload mapped from the original, and after processing attempts to overlay
// the results back onto the original payloads according to more mappings.
type ProcessMap struct {
	parts []int

	mapper   *mapper.Type
	children []Type

	log log.Modular

	mCount        metrics.StatCounter
	mCountParts   metrics.StatCounter
	mSkipped      metrics.StatCounter
	mSkippedParts metrics.StatCounter
	mErr          metrics.StatCounter
	mErrPre       metrics.StatCounter
	mErrProc      metrics.StatCounter
	mErrPost      metrics.StatCounter
	mSent         metrics.StatCounter
	mSentParts    metrics.StatCounter
}

// NewProcessMap returns a ProcessField processor.
func NewProcessMap(
	conf ProcessMapConfig, mgr types.Manager, log log.Modular, stats metrics.Type,
) (*ProcessMap, error) {
	nsStats := metrics.Namespaced(stats, "processor.process_map")
	nsLog := log.NewModule(".processor.process_map")

	var children []Type
	for _, pconf := range conf.Processors {
		proc, err := New(pconf, mgr, nsLog, nsStats)
		if err != nil {
			return nil, err
		}
		children = append(children, proc)
	}

	var conditions []types.Condition
	for _, cconf := range conf.Conditions {
		cond, err := condition.New(cconf, mgr, nsLog, nsStats)
		if err != nil {
			return nil, err
		}
		conditions = append(conditions, cond)
	}

	p := &ProcessMap{
		parts: conf.Parts,

		children: children,

		log: nsLog,

		mCount:        stats.GetCounter("processor.process_map.count"),
		mCountParts:   stats.GetCounter("processor.process_map.parts.count"),
		mSkipped:      stats.GetCounter("processor.process_map.skipped"),
		mSkippedParts: stats.GetCounter("processor.process_map.parts.skipped"),
		mErr:          stats.GetCounter("processor.process_map.error"),
		mErrPre:       stats.GetCounter("processor.process_map.error.premap"),
		mErrProc:      stats.GetCounter("processor.process_map.error.processors"),
		mErrPost:      stats.GetCounter("processor.process_map.error.postmap"),
		mSent:         stats.GetCounter("processor.process_map.sent"),
		mSentParts:    stats.GetCounter("processor.process_map.parts.sent"),
	}

	var err error
	if p.mapper, err = mapper.New(
		mapper.OptSetLogger(nsLog),
		mapper.OptSetStats(nsStats),
		mapper.OptSetConditions(conditions),
		mapper.OptSetReqMap(conf.Premap),
		mapper.OptSetOptReqMap(conf.PremapOptional),
		mapper.OptSetResMap(conf.Postmap),
		mapper.OptSetOptResMap(conf.PostmapOptional),
	); err != nil {
		return nil, err
	}

	return p, nil
}

//------------------------------------------------------------------------------

// ProcessMessage applies the processor to a message, either creating >0
// resulting messages or a response to be sent back to the message source.
func (p *ProcessMap) ProcessMessage(msg types.Message) ([]types.Message, types.Response) {
	alignedResult, err := p.CreateResult(msg)
	if err != nil {
		msgs := [1]types.Message{msg}
		return msgs[:], nil
	}

	result := msg.Copy()
	if err = p.OverlayResult(result, alignedResult); err != nil {
		msgs := [1]types.Message{msg}
		return msgs[:], nil
	}

	msgs := [1]types.Message{result}
	return msgs[:], nil
}

// TargetsUsed returns a list of target dependencies of this processor derived
// from its premap and premap_optional fields.
func (p *ProcessMap) TargetsUsed() []string {
	return p.mapper.TargetsUsed()
}

// TargetsProvided returns a list of targets provided by this processor derived
// from its postmap and postmap_optional fields.
func (p *ProcessMap) TargetsProvided() []string {
	return p.mapper.TargetsProvided()
}

// CreateResult performs reduction and child processors to a payload and returns
// the result. The resulting message should have the same dimension as the
// payload, where reduced indexes are nil. This result can be overlayed onto the
// original message in order to complete the map.
func (p *ProcessMap) CreateResult(msg types.Message) (types.Message, error) {
	p.mCount.Incr(1)
	p.mCountParts.Incr(int64(msg.Len()))

	mapMsg := msg
	if len(p.parts) > 0 {
		mapMsg = message.New(make([][]byte, msg.Len()))
		for _, sel := range p.parts {
			mapMsg.Get(sel).Set(msg.Get(sel).Get())
		}
	}

	mappedMsg, skipped, err := p.mapper.MapRequests(mapMsg)
	if err != nil {
		p.mErr.Incr(1)
		p.mErrPre.Incr(1)
		p.log.Errorf("Failed to map request: %v\n", err)
		return nil, err
	}

	if mappedMsg.Len() == 0 {
		p.mSkipped.Incr(1)
		p.mSkippedParts.Incr(int64(msg.Len()))
		parts := make([]types.Part, msg.Len())
		mapMsg.SetAll(parts)
		return mapMsg, nil
	}

	var procResults []types.Message
	if procResults, err = processMap(mappedMsg, p.children); err != nil {
		p.mErrProc.Incr(1)
		p.mErr.Incr(1)
		p.log.Errorf("Processors failed: %v\n", err)
		return nil, err
	}

	i := 0
	for _, m := range procResults {
		m.Iter(func(_ int, part types.Part) error {
			p.log.Tracef("Processed request part '%v': %q\n", i, part.Get())
			i++
			return nil
		})
	}

	var alignedResult types.Message
	if alignedResult, err = p.mapper.AlignResult(msg.Len(), skipped, procResults); err != nil {
		p.mErrPost.Incr(1)
		p.mErr.Incr(1)
		p.log.Errorf("Postmap failed: %v\n", err)
		return nil, err
	}

	return alignedResult, nil
}

// OverlayResult attempts to merge the result of a process_map with the original
//  payload as per the map specified in the postmap and postmap_optional fields.
func (p *ProcessMap) OverlayResult(payload, response types.Message) error {
	if err := p.mapper.MapResponses(payload, response); err != nil {
		p.mErrPost.Incr(1)
		p.mErr.Incr(1)
		p.log.Errorf("Postmap failed: %v\n", err)
		return err
	}

	p.mSent.Incr(1)
	p.mSentParts.Incr(int64(payload.Len()))
	return nil
}

func processMap(mappedMsg types.Message, processors []Type) ([]types.Message, error) {
	requestMsgs := []types.Message{mappedMsg}
	i := 0
	for ; len(requestMsgs) > 0 && i < len(processors); i++ {
		var nextRequestMsgs []types.Message
		for _, m := range requestMsgs {
			rMsgs, _ := processors[i].ProcessMessage(m)
			nextRequestMsgs = append(nextRequestMsgs, rMsgs...)
		}
		requestMsgs = nextRequestMsgs
	}

	if len(requestMsgs) == 0 {
		return nil, fmt.Errorf("processor index '%v' returned zero messages", i)
	}

	return requestMsgs, nil
}
<commit_msg>Remove trace log from process_map<commit_after>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package processor

import (
	"encoding/json"
	"fmt"

	"github.com/Jeffail/benthos/lib/log"
	"github.com/Jeffail/benthos/lib/message"
	"github.com/Jeffail/benthos/lib/message/mapper"
	"github.com/Jeffail/benthos/lib/metrics"
	"github.com/Jeffail/benthos/lib/processor/condition"
	"github.com/Jeffail/benthos/lib/types"
)

//------------------------------------------------------------------------------

func init() {
	Constructors[TypeProcessMap] = TypeSpec{
		constructor: func(conf Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
			return NewProcessMap(conf.ProcessMap, mgr, log, stats)
		},
		description: `
A processor that extracts and maps fields from the original payload into new
objects, applies a list of processors to the newly constructed objects, and
finally maps the result back into the original payload.

This processor is useful for performing processors on subsections of a payload.
For example, you could extract sections of a JSON object in order to construct
a request object for an ` + "`http`" + ` processor, then map the result back
into a field within the original object.

The order of stages of this processor are as follows:

- Conditions are applied to each _individual_ message part in the batch,
  determining whether the part will be mapped. If the conditions are empty all
  message parts will be mapped. If the field ` + "`parts`" + ` is populated the
  message parts not in this list are also excluded from mapping.
- Message parts that are flagged for mapping are mapped according to the premap
  fields, creating a new object. If the premap stage fails (targets are not
  found) the message part will not be processed.
- Message parts that are mapped are processed as a batch. You may safely break
  the batch into individual parts during processing with the ` + "`split`" + `
  processor.
- After all child processors are applied to the mapped messages they are mapped
  back into the original message parts they originated from as per your postmap.
  If the postmap stage fails the mapping is skipped and the message payload
  remains as it started.

Map paths are arbitrary dot paths, target path hierarchies are constructed if
they do not yet exist. Processing is skipped for message parts where the premap
targets aren't found, for optional premap targets use ` + "`premap_optional`" + `.

Map target paths that are parents of other map target paths will always be
mapped first, therefore it is possible to map subpath overrides.

If postmap targets are not found the merge is abandoned, for optional postmap
targets use ` + "`postmap_optional`" + `.

If the premap is empty then the full payload is sent to the processors, if the
postmap is empty then the processed result replaces the original contents
entirely.

Maps can reference the root of objects either with an empty string or '.', for
example the maps:

` + "``` yaml" + `
premap:
  .: foo.bar
postmap:
  foo.bar: .
` + "```" + `

Would create a new object where the root is the value of ` + "`foo.bar`" + ` and
would map the full contents of the result back into ` + "`foo.bar`" + `.

If the number of total message parts resulting from the processing steps does
not match the original count then this processor fails and the messages continue
unchanged. Therefore, you should avoid using batch and filter type processors in
this list.

### Batch Ordering

This processor supports batch messages. When message parts are post-mapped after
processing they will be correctly aligned with the original batch. However, the
ordering of premapped message parts as they are sent through processors are not
guaranteed to match the ordering of the original batch.`,
		sanitiseConfigFunc: func(conf Config) (interface{}, error) {
			return conf.ProcessMap.Sanitise()
		},
	}
}

//------------------------------------------------------------------------------

// ProcessMapConfig is a config struct containing fields for the
// ProcessMap processor.
type ProcessMapConfig struct {
	Parts           []int              `json:"parts" yaml:"parts"`
	Conditions      []condition.Config `json:"conditions" yaml:"conditions"`
	Premap          map[string]string  `json:"premap" yaml:"premap"`
	PremapOptional  map[string]string  `json:"premap_optional" yaml:"premap_optional"`
	Postmap         map[string]string  `json:"postmap" yaml:"postmap"`
	PostmapOptional map[string]string  `json:"postmap_optional" yaml:"postmap_optional"`
	Processors      []Config           `json:"processors" yaml:"processors"`
}

// NewProcessMapConfig returns a default ProcessMapConfig.
func NewProcessMapConfig() ProcessMapConfig {
	return ProcessMapConfig{
		Parts:           []int{},
		Conditions:      []condition.Config{},
		Premap:          map[string]string{},
		PremapOptional:  map[string]string{},
		Postmap:         map[string]string{},
		PostmapOptional: map[string]string{},
		Processors:      []Config{},
	}
}

// Sanitise the configuration into a minimal structure that can be printed
// without changing the intent.
func (p ProcessMapConfig) Sanitise() (map[string]interface{}, error) {
	var err error
	condConfs := make([]interface{}, len(p.Conditions))
	for i, cConf := range p.Conditions {
		if condConfs[i], err = condition.SanitiseConfig(cConf); err != nil {
			return nil, err
		}
	}
	procConfs := make([]interface{}, len(p.Processors))
	for i, pConf := range p.Processors {
		if procConfs[i], err = SanitiseConfig(pConf); err != nil {
			return nil, err
		}
	}
	return map[string]interface{}{
		"parts":            p.Parts,
		"conditions":       condConfs,
		"premap":           p.Premap,
		"premap_optional":  p.PremapOptional,
		"postmap":          p.Postmap,
		"postmap_optional": p.PostmapOptional,
		"processors":       procConfs,
	}, nil
}

//------------------------------------------------------------------------------

// UnmarshalJSON ensures that when parsing configs that are in a slice the
// default values are still applied.
func (p *ProcessMapConfig) UnmarshalJSON(bytes []byte) error {
	type confAlias ProcessMapConfig
	aliased := confAlias(NewProcessMapConfig())

	if err := json.Unmarshal(bytes, &aliased); err != nil {
		return err
	}

	*p = ProcessMapConfig(aliased)
	return nil
}

// UnmarshalYAML ensures that when parsing configs that are in a slice the
// default values are still applied.
func (p *ProcessMapConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type confAlias ProcessMapConfig
	aliased := confAlias(NewProcessMapConfig())

	if err := unmarshal(&aliased); err != nil {
		return err
	}

	*p = ProcessMapConfig(aliased)
	return nil
}

//------------------------------------------------------------------------------

// ProcessMap is a processor that applies a list of child processors to a new
// payload mapped from the original, and after processing attempts to overlay
// the results back onto the original payloads according to more mappings.
type ProcessMap struct {
	parts []int

	mapper   *mapper.Type
	children []Type

	log log.Modular

	mCount        metrics.StatCounter
	mCountParts   metrics.StatCounter
	mSkipped      metrics.StatCounter
	mSkippedParts metrics.StatCounter
	mErr          metrics.StatCounter
	mErrPre       metrics.StatCounter
	mErrProc      metrics.StatCounter
	mErrPost      metrics.StatCounter
	mSent         metrics.StatCounter
	mSentParts    metrics.StatCounter
}

// NewProcessMap returns a ProcessField processor.
func NewProcessMap(
	conf ProcessMapConfig, mgr types.Manager, log log.Modular, stats metrics.Type,
) (*ProcessMap, error) {
	nsStats := metrics.Namespaced(stats, "processor.process_map")
	nsLog := log.NewModule(".processor.process_map")

	var children []Type
	for _, pconf := range conf.Processors {
		proc, err := New(pconf, mgr, nsLog, nsStats)
		if err != nil {
			return nil, err
		}
		children = append(children, proc)
	}

	var conditions []types.Condition
	for _, cconf := range conf.Conditions {
		cond, err := condition.New(cconf, mgr, nsLog, nsStats)
		if err != nil {
			return nil, err
		}
		conditions = append(conditions, cond)
	}

	p := &ProcessMap{
		parts: conf.Parts,

		children: children,

		log: nsLog,

		mCount:        stats.GetCounter("processor.process_map.count"),
		mCountParts:   stats.GetCounter("processor.process_map.parts.count"),
		mSkipped:      stats.GetCounter("processor.process_map.skipped"),
		mSkippedParts: stats.GetCounter("processor.process_map.parts.skipped"),
		mErr:          stats.GetCounter("processor.process_map.error"),
		mErrPre:       stats.GetCounter("processor.process_map.error.premap"),
		mErrProc:      stats.GetCounter("processor.process_map.error.processors"),
		mErrPost:      stats.GetCounter("processor.process_map.error.postmap"),
		mSent:         stats.GetCounter("processor.process_map.sent"),
		mSentParts:    stats.GetCounter("processor.process_map.parts.sent"),
	}

	var err error
	if p.mapper, err = mapper.New(
		mapper.OptSetLogger(nsLog),
		mapper.OptSetStats(nsStats),
		mapper.OptSetConditions(conditions),
		mapper.OptSetReqMap(conf.Premap),
		mapper.OptSetOptReqMap(conf.PremapOptional),
		mapper.OptSetResMap(conf.Postmap),
		mapper.OptSetOptResMap(conf.PostmapOptional),
	); err != nil {
		return nil, err
	}

	return p, nil
}

//------------------------------------------------------------------------------

// ProcessMessage applies the processor to a message, either creating >0
// resulting messages or a response to be sent back to the message source.
func (p *ProcessMap) ProcessMessage(msg types.Message) ([]types.Message, types.Response) {
	alignedResult, err := p.CreateResult(msg)
	if err != nil {
		msgs := [1]types.Message{msg}
		return msgs[:], nil
	}

	result := msg.Copy()
	if err = p.OverlayResult(result, alignedResult); err != nil {
		msgs := [1]types.Message{msg}
		return msgs[:], nil
	}

	msgs := [1]types.Message{result}
	return msgs[:], nil
}

// TargetsUsed returns a list of target dependencies of this processor derived
// from its premap and premap_optional fields.
func (p *ProcessMap) TargetsUsed() []string {
	return p.mapper.TargetsUsed()
}

// TargetsProvided returns a list of targets provided by this processor derived
// from its postmap and postmap_optional fields.
func (p *ProcessMap) TargetsProvided() []string {
	return p.mapper.TargetsProvided()
}

// CreateResult performs reduction and child processors to a payload and returns
// the result. The resulting message should have the same dimension as the
// payload, where reduced indexes are nil. This result can be overlayed onto the
// original message in order to complete the map.
func (p *ProcessMap) CreateResult(msg types.Message) (types.Message, error) {
	p.mCount.Incr(1)
	p.mCountParts.Incr(int64(msg.Len()))

	mapMsg := msg
	if len(p.parts) > 0 {
		mapMsg = message.New(make([][]byte, msg.Len()))
		for _, sel := range p.parts {
			mapMsg.Get(sel).Set(msg.Get(sel).Get())
		}
	}

	mappedMsg, skipped, err := p.mapper.MapRequests(mapMsg)
	if err != nil {
		p.mErr.Incr(1)
		p.mErrPre.Incr(1)
		p.log.Errorf("Failed to map request: %v\n", err)
		return nil, err
	}

	if mappedMsg.Len() == 0 {
		p.mSkipped.Incr(1)
		p.mSkippedParts.Incr(int64(msg.Len()))
		parts := make([]types.Part, msg.Len())
		mapMsg.SetAll(parts)
		return mapMsg, nil
	}

	var procResults []types.Message
	if procResults, err = processMap(mappedMsg, p.children); err != nil {
		p.mErrProc.Incr(1)
		p.mErr.Incr(1)
		p.log.Errorf("Processors failed: %v\n", err)
		return nil, err
	}

	i := 0
	for _, m := range procResults {
		m.Iter(func(_ int, part types.Part) error {
			i++
			return nil
		})
	}

	var alignedResult types.Message
	if alignedResult, err = p.mapper.AlignResult(msg.Len(), skipped, procResults); err != nil {
		p.mErrPost.Incr(1)
		p.mErr.Incr(1)
		p.log.Errorf("Postmap failed: %v\n", err)
		return nil, err
	}

	return alignedResult, nil
}

// OverlayResult attempts to merge the result of a process_map with the original
//  payload as per the map specified in the postmap and postmap_optional fields.
func (p *ProcessMap) OverlayResult(payload, response types.Message) error {
	if err := p.mapper.MapResponses(payload, response); err != nil {
		p.mErrPost.Incr(1)
		p.mErr.Incr(1)
		p.log.Errorf("Postmap failed: %v\n", err)
		return err
	}

	p.mSent.Incr(1)
	p.mSentParts.Incr(int64(payload.Len()))
	return nil
}

func processMap(mappedMsg types.Message, processors []Type) ([]types.Message, error) {
	requestMsgs := []types.Message{mappedMsg}
	i := 0
	for ; len(requestMsgs) > 0 && i < len(processors); i++ {
		var nextRequestMsgs []types.Message
		for _, m := range requestMsgs {
			rMsgs, _ := processors[i].ProcessMessage(m)
			nextRequestMsgs = append(nextRequestMsgs, rMsgs...)
		}
		requestMsgs = nextRequestMsgs
	}

	if len(requestMsgs) == 0 {
		return nil, fmt.Errorf("processor index '%v' returned zero messages", i)
	}

	return requestMsgs, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"context"
	"fmt"
	"net"
	"net/url"
	"path"
	"sync"
	"sync/atomic"
	"time"

	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
	"go.etcd.io/etcd/clientv3"
	"go.etcd.io/etcd/pkg/transport"
	"google.golang.org/grpc"

	"k8s.io/apimachinery/pkg/runtime"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/server/egressselector"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/etcd3"
	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
	"k8s.io/apiserver/pkg/storage/storagebackend"
	"k8s.io/apiserver/pkg/storage/value"
	"k8s.io/component-base/metrics/legacyregistry"
	"k8s.io/klog/v2"
)

const (
	// The short keepalive timeout and interval have been chosen to aggressively
	// detect a failed etcd server without introducing much overhead.
	keepaliveTime    = 30 * time.Second
	keepaliveTimeout = 10 * time.Second

	// dialTimeout is the timeout for failing to establish a connection.
	// It is set to 20 seconds as times shorter than that will cause TLS connections to fail
	// on heavily loaded arm64 CPUs (issue #64649)
	dialTimeout = 20 * time.Second

	dbMetricsMonitorJitter = 0.5
)

func init() {
	// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
	// using the global prometheus registry and using our own wrapped global registry,
	// we need to explicitly register these metrics to our global registry here.
	// For reference: https://github.com/kubernetes/kubernetes/pull/81387
	legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
	dbMetricsMonitors = make(map[string]struct{})
}

func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
	// constructing the etcd v3 client blocks and times out if etcd is not available.
	// retry in a loop in the background until we successfully create the client, storing the client or error encountered

	clientValue := &atomic.Value{}

	clientErrMsg := &atomic.Value{}
	clientErrMsg.Store("etcd client connection not yet established")

	go wait.PollUntil(time.Second, func() (bool, error) {
		client, err := newETCD3Client(c.Transport)
		if err != nil {
			clientErrMsg.Store(err.Error())
			return false, nil
		}
		clientValue.Store(client)
		clientErrMsg.Store("")
		return true, nil
	}, wait.NeverStop)

	return func() error {
		if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 {
			return fmt.Errorf(errMsg)
		}
		client := clientValue.Load().(*clientv3.Client)
		ctx, cancel := context.WithTimeout(context.Background(), c.HealthcheckTimeout)
		defer cancel()
		// See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118
		_, err := client.Get(ctx, path.Join("/", c.Prefix, "health"))
		if err == nil {
			return nil
		}
		return fmt.Errorf("error getting data from etcd: %v", err)
	}, nil
}

func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error) {
	tlsInfo := transport.TLSInfo{
		CertFile:      c.CertFile,
		KeyFile:       c.KeyFile,
		TrustedCAFile: c.TrustedCAFile,
	}
	tlsConfig, err := tlsInfo.ClientConfig()
	if err != nil {
		return nil, err
	}
	// NOTE: Client relies on nil tlsConfig
	// for non-secure connections, update the implicit variable
	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.TrustedCAFile) == 0 {
		tlsConfig = nil
	}
	networkContext := egressselector.Etcd.AsNetworkContext()
	var egressDialer utilnet.DialFunc
	if c.EgressLookup != nil {
		egressDialer, err = c.EgressLookup(networkContext)
		if err != nil {
			return nil, err
		}
	}
	dialOptions := []grpc.DialOption{
		grpc.WithBlock(), // block until the underlying connection is up
		grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
		grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
	}
	if egressDialer != nil {
		dialer := func(ctx context.Context, addr string) (net.Conn, error) {
			u, err := url.Parse(addr)
			if err != nil {
				return nil, err
			}
			return egressDialer(ctx, "tcp", u.Host)
		}
		dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
	}
	cfg := clientv3.Config{
		DialTimeout:          dialTimeout,
		DialKeepAliveTime:    keepaliveTime,
		DialKeepAliveTimeout: keepaliveTimeout,
		DialOptions:          dialOptions,
		Endpoints:            c.ServerList,
		TLS:                  tlsConfig,
	}

	return clientv3.New(cfg)
}

type runningCompactor struct {
	interval time.Duration
	cancel   context.CancelFunc
	client   *clientv3.Client
	refs     int
}

var (
	// compactorsMu guards access to compactors map
	compactorsMu sync.Mutex
	compactors   = map[string]*runningCompactor{}
	// dbMetricsMonitorsMu guards access to dbMetricsMonitors map
	dbMetricsMonitorsMu sync.Mutex
	dbMetricsMonitors   map[string]struct{}
)

// startCompactorOnce start one compactor per transport. If the interval get smaller on repeated calls, the
// compactor is replaced. A destroy func is returned. If all destroy funcs with the same transport are called,
// the compactor is stopped.
func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration) (func(), error) {
	compactorsMu.Lock()
	defer compactorsMu.Unlock()

	key := fmt.Sprintf("%v", c) // gives: {[server1 server2] keyFile certFile caFile}
	if compactor, foundBefore := compactors[key]; !foundBefore || compactor.interval > interval {
		compactorClient, err := newETCD3Client(c)
		if err != nil {
			return nil, err
		}

		if foundBefore {
			// replace compactor
			compactor.cancel()
			compactor.client.Close()
		} else {
			// start new compactor
			compactor = &runningCompactor{}
			compactors[key] = compactor
		}

		ctx, cancel := context.WithCancel(context.Background())

		compactor.interval = interval
		compactor.cancel = cancel
		compactor.client = compactorClient

		etcd3.StartCompactor(ctx, compactorClient, interval)
	}

	compactors[key].refs++

	return func() {
		compactorsMu.Lock()
		defer compactorsMu.Unlock()

		compactor := compactors[key]
		compactor.refs--
		if compactor.refs == 0 {
			compactor.cancel()
			compactor.client.Close()
			delete(compactors, key)
		}
	}, nil
}

func newETCD3Storage(c storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
	if err != nil {
		return nil, nil, err
	}

	client, err := newETCD3Client(c.Transport)
	if err != nil {
		stopCompactor()
		return nil, nil, err
	}

	stopDBSizeMonitor, err := startDBSizeMonitorPerEndpoint(client, c.DBMetricPollInterval)
	if err != nil {
		return nil, nil, err
	}

	var once sync.Once
	destroyFunc := func() {
		// we know that storage destroy funcs are called multiple times (due to reuse in subresources).
		// Hence, we only destroy once.
		// TODO: fix duplicated storage destroy calls higher level
		once.Do(func() {
			stopCompactor()
			stopDBSizeMonitor()
			client.Close()
		})
	}
	transformer := c.Transformer
	if transformer == nil {
		transformer = value.IdentityTransformer
	}
	return etcd3.New(client, c.Codec, newFunc, c.Prefix, transformer, c.Paging), destroyFunc, nil
}

// startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the
// corresponding metric etcd_db_total_size_in_bytes for each etcd server endpoint.
func startDBSizeMonitorPerEndpoint(client *clientv3.Client, interval time.Duration) (func(), error) {
	if interval == 0 {
		return func() {}, nil
	}
	dbMetricsMonitorsMu.Lock()
	defer dbMetricsMonitorsMu.Unlock()

	ctx, cancel := context.WithCancel(context.Background())
	for _, ep := range client.Endpoints() {
		if _, found := dbMetricsMonitors[ep]; found {
			continue
		}
		dbMetricsMonitors[ep] = struct{}{}
		endpoint := ep
		klog.V(4).Infof("Start monitoring storage db size metric for endpoint %s with polling interval %v", endpoint, interval)
		go wait.JitterUntilWithContext(ctx, func(context.Context) {
			epStatus, err := client.Maintenance.Status(ctx, endpoint)
			if err != nil {
				klog.V(4).Infof("Failed to get storage db size for ep %s: %v", endpoint, err)
				metrics.UpdateEtcdDbSize(endpoint, -1)
			} else {
				metrics.UpdateEtcdDbSize(endpoint, epStatus.DbSize)
			}
		}, interval, dbMetricsMonitorJitter, true)
	}

	return func() {
		cancel()
	}, nil
}
<commit_msg>fix case when HC timeout is 0<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"context"
	"fmt"
	"net"
	"net/url"
	"path"
	"sync"
	"sync/atomic"
	"time"

	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
	"go.etcd.io/etcd/clientv3"
	"go.etcd.io/etcd/pkg/transport"
	"google.golang.org/grpc"

	"k8s.io/apimachinery/pkg/runtime"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apiserver/pkg/server/egressselector"
	"k8s.io/apiserver/pkg/storage"
	"k8s.io/apiserver/pkg/storage/etcd3"
	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
	"k8s.io/apiserver/pkg/storage/storagebackend"
	"k8s.io/apiserver/pkg/storage/value"
	"k8s.io/component-base/metrics/legacyregistry"
	"k8s.io/klog/v2"
)

const (
	// The short keepalive timeout and interval have been chosen to aggressively
	// detect a failed etcd server without introducing much overhead.
	keepaliveTime    = 30 * time.Second
	keepaliveTimeout = 10 * time.Second

	// dialTimeout is the timeout for failing to establish a connection.
	// It is set to 20 seconds as times shorter than that will cause TLS connections to fail
	// on heavily loaded arm64 CPUs (issue #64649)
	dialTimeout = 20 * time.Second

	dbMetricsMonitorJitter = 0.5
)

func init() {
	// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
	// using the global prometheus registry and using our own wrapped global registry,
	// we need to explicitly register these metrics to our global registry here.
	// For reference: https://github.com/kubernetes/kubernetes/pull/81387
	legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
	dbMetricsMonitors = make(map[string]struct{})
}

func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
	// constructing the etcd v3 client blocks and times out if etcd is not available.
	// retry in a loop in the background until we successfully create the client, storing the client or error encountered

	clientValue := &atomic.Value{}

	clientErrMsg := &atomic.Value{}
	clientErrMsg.Store("etcd client connection not yet established")

	go wait.PollUntil(time.Second, func() (bool, error) {
		client, err := newETCD3Client(c.Transport)
		if err != nil {
			clientErrMsg.Store(err.Error())
			return false, nil
		}
		clientValue.Store(client)
		clientErrMsg.Store("")
		return true, nil
	}, wait.NeverStop)

	return func() error {
		if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 {
			return fmt.Errorf(errMsg)
		}
		client := clientValue.Load().(*clientv3.Client)
		healthcheckTimeout := storagebackend.DefaultHealthcheckTimeout
		if c.HealthcheckTimeout != time.Duration(0) {
			healthcheckTimeout = c.HealthcheckTimeout
		}
		ctx, cancel := context.WithTimeout(context.Background(), healthcheckTimeout)
		defer cancel()
		// See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118
		_, err := client.Get(ctx, path.Join("/", c.Prefix, "health"))
		if err == nil {
			return nil
		}
		return fmt.Errorf("error getting data from etcd: %v", err)
	}, nil
}

func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error) {
	tlsInfo := transport.TLSInfo{
		CertFile:      c.CertFile,
		KeyFile:       c.KeyFile,
		TrustedCAFile: c.TrustedCAFile,
	}
	tlsConfig, err := tlsInfo.ClientConfig()
	if err != nil {
		return nil, err
	}
	// NOTE: Client relies on nil tlsConfig
	// for non-secure connections, update the implicit variable
	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.TrustedCAFile) == 0 {
		tlsConfig = nil
	}
	networkContext := egressselector.Etcd.AsNetworkContext()
	var egressDialer utilnet.DialFunc
	if c.EgressLookup != nil {
		egressDialer, err = c.EgressLookup(networkContext)
		if err != nil {
			return nil, err
		}
	}
	dialOptions := []grpc.DialOption{
		grpc.WithBlock(), // block until the underlying connection is up
		grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
		grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
	}
	if egressDialer != nil {
		dialer := func(ctx context.Context, addr string) (net.Conn, error) {
			u, err := url.Parse(addr)
			if err != nil {
				return nil, err
			}
			return egressDialer(ctx, "tcp", u.Host)
		}
		dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
	}
	cfg := clientv3.Config{
		DialTimeout:          dialTimeout,
		DialKeepAliveTime:    keepaliveTime,
		DialKeepAliveTimeout: keepaliveTimeout,
		DialOptions:          dialOptions,
		Endpoints:            c.ServerList,
		TLS:                  tlsConfig,
	}

	return clientv3.New(cfg)
}

type runningCompactor struct {
	interval time.Duration
	cancel   context.CancelFunc
	client   *clientv3.Client
	refs     int
}

var (
	// compactorsMu guards access to compactors map
	compactorsMu sync.Mutex
	compactors   = map[string]*runningCompactor{}
	// dbMetricsMonitorsMu guards access to dbMetricsMonitors map
	dbMetricsMonitorsMu sync.Mutex
	dbMetricsMonitors   map[string]struct{}
)

// startCompactorOnce start one compactor per transport. If the interval get smaller on repeated calls, the
// compactor is replaced. A destroy func is returned. If all destroy funcs with the same transport are called,
// the compactor is stopped.
func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration) (func(), error) {
	compactorsMu.Lock()
	defer compactorsMu.Unlock()

	key := fmt.Sprintf("%v", c) // gives: {[server1 server2] keyFile certFile caFile}
	if compactor, foundBefore := compactors[key]; !foundBefore || compactor.interval > interval {
		compactorClient, err := newETCD3Client(c)
		if err != nil {
			return nil, err
		}

		if foundBefore {
			// replace compactor
			compactor.cancel()
			compactor.client.Close()
		} else {
			// start new compactor
			compactor = &runningCompactor{}
			compactors[key] = compactor
		}

		ctx, cancel := context.WithCancel(context.Background())

		compactor.interval = interval
		compactor.cancel = cancel
		compactor.client = compactorClient

		etcd3.StartCompactor(ctx, compactorClient, interval)
	}

	compactors[key].refs++

	return func() {
		compactorsMu.Lock()
		defer compactorsMu.Unlock()

		compactor := compactors[key]
		compactor.refs--
		if compactor.refs == 0 {
			compactor.cancel()
			compactor.client.Close()
			delete(compactors, key)
		}
	}, nil
}

func newETCD3Storage(c storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
	if err != nil {
		return nil, nil, err
	}

	client, err := newETCD3Client(c.Transport)
	if err != nil {
		stopCompactor()
		return nil, nil, err
	}

	stopDBSizeMonitor, err := startDBSizeMonitorPerEndpoint(client, c.DBMetricPollInterval)
	if err != nil {
		return nil, nil, err
	}

	var once sync.Once
	destroyFunc := func() {
		// we know that storage destroy funcs are called multiple times (due to reuse in subresources).
		// Hence, we only destroy once.
		// TODO: fix duplicated storage destroy calls higher level
		once.Do(func() {
			stopCompactor()
			stopDBSizeMonitor()
			client.Close()
		})
	}
	transformer := c.Transformer
	if transformer == nil {
		transformer = value.IdentityTransformer
	}
	return etcd3.New(client, c.Codec, newFunc, c.Prefix, transformer, c.Paging), destroyFunc, nil
}

// startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the
// corresponding metric etcd_db_total_size_in_bytes for each etcd server endpoint.
func startDBSizeMonitorPerEndpoint(client *clientv3.Client, interval time.Duration) (func(), error) {
	if interval == 0 {
		return func() {}, nil
	}
	dbMetricsMonitorsMu.Lock()
	defer dbMetricsMonitorsMu.Unlock()

	ctx, cancel := context.WithCancel(context.Background())
	for _, ep := range client.Endpoints() {
		if _, found := dbMetricsMonitors[ep]; found {
			continue
		}
		dbMetricsMonitors[ep] = struct{}{}
		endpoint := ep
		klog.V(4).Infof("Start monitoring storage db size metric for endpoint %s with polling interval %v", endpoint, interval)
		go wait.JitterUntilWithContext(ctx, func(context.Context) {
			epStatus, err := client.Maintenance.Status(ctx, endpoint)
			if err != nil {
				klog.V(4).Infof("Failed to get storage db size for ep %s: %v", endpoint, err)
				metrics.UpdateEtcdDbSize(endpoint, -1)
			} else {
				metrics.UpdateEtcdDbSize(endpoint, epStatus.DbSize)
			}
		}, interval, dbMetricsMonitorJitter, true)
	}

	return func() {
		cancel()
	}, nil
}
<|endoftext|>