<commit_before>// Package replicat is a server for n way synchronization of content (Replication for the cloud).
// Copyright 2016 Jacob Taylor jacob@replic.at       More Info: http://replic.at
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"sync"
	"time"
)

// ReplicatServer is a structure that contains the definition of the servers in a cluster. Each node has a name and this
// node (as determined by globalSettings.name at the moment) also has a StorageTracker interface.
type ReplicatServer struct {
	ClusterKey    string
	Name          string
	Address       string
	Status        string
	CurrentState  DirTreeMap
	PreviousState DirTreeMap
	Lock          sync.Mutex
	storage       StorageTracker
}

// GetStatus - get the current status of the server
func (server *ReplicatServer) GetStatus() string {
	return server.Status
}

// SetStatus - set the current status of the server
func (server *ReplicatServer) SetStatus(status string) {
	// adding an inline check to make sure the server is valid
	// if the server is being shut down or at the end of a unit test)
	// the assignment can crash without this check
	if server != nil {
		server.Status = status
		sendConfigToServer()
	}
}

const (
	// REPLICAT_STATUS_INITIAL_SCAN - The server is scanning its local storage
	REPLICAT_STATUS_INITIAL_SCAN = "Initial Scan"
	// REPLICAT_STATUS_JOINING_CLUSTER - The server has started to join the cluster and get up to date
	REPLICAT_STATUS_JOINING_CLUSTER = "Joining Cluster"
	// REPLICAT_STATUS_ONLINE - The server is up to date and part of the cluster
	REPLICAT_STATUS_ONLINE = "Online"
)

var serverMap = make(map[string]*ReplicatServer)
var serverMapLock sync.RWMutex

var lastConfigPing = time.Now()

// BootstrapAndServe - Start the server
func BootstrapAndServe(address string) {
	//trackerTestDual()
	//trackerTestSmallFileInSubfolder()
	//trackerTestEmptyDirectoryMovesInOutAround()
	//trackerTestFileChangeTrackerAddFolders()
	//trackerTestSmallFileCreationAndRename()
	//trackerTestSmallFileCreationAndUpdate()
	//trackerTestSmallFileMovesInOutAround()
	//trackerTestDirectoryCreation()
	//trackerTestNestedDirectoryCreation()
	//trackerTestDirectoryStorage()
	//trackerTestFileChangeTrackerAutoCreateFolderAndCleanup()
	//trackerTestNestedFastDirectoryCreation()

	// testing code to enable debugger use
	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))
	http.Handle("/upload/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(uploadHandler)))

	lsnr, err := net.Listen("tcp4", address)
	if err != nil {
		panic(fmt.Sprintf("Error listening: %v\nAddress: %s\n", err, address))
	}
	fmt.Println("Listening on:", lsnr.Addr().String())

	logOnlyHandler := LogOnlyChangeHandler{}
	tracker := FilesystemTracker{}
	fmt.Printf("Looking up settings for node: %s\n", globalSettings.Name)

	directory := globalSettings.Directory

	fmt.Printf("GlobalSettings directory retrieved for this node: %s\n", directory)
	server := &ReplicatServer{Name: globalSettings.Name, ClusterKey: globalSettings.ClusterKey, Address: lsnr.Addr().String(), storage: &tracker, Status: REPLICAT_STATUS_INITIAL_SCAN}
	serverMap[globalSettings.Name] = server
	tracker.init(directory, server)

	go func(tracker StorageTracker) {
		for true {
			tracker.GetStatistics()
			time.Sleep(30 * time.Second)
		}
	}(&tracker)

	var c ChangeHandler
	c = &logOnlyHandler
	tracker.watchDirectory(&c)

	go func(listener net.Listener) {
		err = http.Serve(listener, nil)
		if err != nil {
			panic(err)
		}
	}(lsnr)

	fmt.Println("Starting config update processor")
	go configUpdateProcessor(configUpdateChannel)

	if globalSettings.ManagerAddress != "" {
		fmt.Printf("about to send config to server (%s)\nOur address is: (%s)\n", globalSettings.ManagerAddress, lsnr.Addr())
	}

	// we need something to keep us running.
	keepConfigCurrent()
}

const (
	managerOverdueSeconds = 30
	managerCheckSleepTime = 45
)

func keepConfigCurrent() {
	for {
		if time.Since(lastConfigPing) > managerOverdueSeconds*time.Second {
			log.Printf("Manager Contact Overdue, attempting to contact: %s\n", globalSettings.ManagerAddress)
			sendConfigToServer()
		} else {
			fmt.Println("No Update Required")
		}
		time.Sleep(time.Duration(rand.Intn(10)-5+managerCheckSleepTime) * time.Second)
	}
}

func sendConfigToServer() {
	// This field will be empty during testing. We have to abort during testing since the manager is not reachable.
	if globalSettings.ManagerAddress == "" {
		return
	}

	url := "https://" + globalSettings.ManagerAddress + "/config/"
	log.Printf("sendConfigToServer: Manager location: %s\n", url)

	server := serverMap[globalSettings.Name]
	jsonStr, _ := json.Marshal(server)
	jsonStr2, _ := json.Marshal(server.storage.GetStatistics())

	byteArraysToMarshal := [][]byte{jsonStr, jsonStr2}
	byteData := bytes.Join(byteArraysToMarshal, []byte{})
	log.Printf("jasonstr: %s\n", string(byteData))
	jsonData := bytes.NewBuffer(byteData)

	req, err := http.NewRequest("POST", url, jsonData)
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Unable to reach server. Data could be lost: %s\n", err)
		return
	}

	//log.Printf("response: \n%#v\n", resp)

	newServerMap, err := extractServerMapFromConfig(resp.Body)
	if err == io.EOF {
		log.Printf("EOF unexpected Config update failed\n")
	} else if err != nil {
		log.Printf("Config update failed due to error: %s\n", err)
	} else {
		configUpdateChannel <- newServerMap
	}

}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		r.ParseMultipartForm(32 << 20)
		file, handler, err := r.FormFile("uploadfile")
		if err != nil {
			fmt.Println(err)
			return
		}
		defer file.Close()
		fmt.Fprint(w, handler.Header)

		hash := []byte(r.Form.Get("HASH"))

		storage := serverMap[globalSettings.Name].storage
		local, _ := storage.getEntryJSON(handler.Filename)

		if !bytes.Equal(hash, local.Hash) {
			fullPath := globalSettings.Directory + "/" + handler.Filename
			f, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0666)

			if err != nil {
				fmt.Println(err)
				return
			}

			bytesWritten, err := io.Copy(f, file)
			if err != nil {
				log.Printf("Error copying file: %s, error(%#v)\n", handler.Filename, err)
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("500 - Error copying file"))
				f.Close()
				return
			}
			fmt.Printf("Wrote out (%s) bytes (%d)\n", handler.Filename, bytesWritten)
			f.Close()

			// Get the old entry
			entryString := r.Form.Get("entryJSON")
			if entryString != "" {
				var entry EntryJSON
				err := json.Unmarshal([]byte(entryString), &entry)
				if err != nil {
					log.Fatalf("Error copying file (Entry handling): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}

				err = os.Chtimes(fullPath, time.Now(), entry.ModTime)
				if err != nil {
					log.Fatalf("Error copying file (Changing times): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}
			}

		}
	}
}

var configUpdateChannel = make(chan *map[string]*ReplicatServer, 100)

func configHandler(_ http.ResponseWriter, r *http.Request) {
	newServerMap, err := extractServerMapFromConfig(r.Body)
	if err != nil {
		fmt.Printf("Config update failed due to error: %s\n", err)
		return
	}
	configUpdateChannel <- newServerMap
}

func extractServerMapFromConfig(sourceReader io.Reader) (newServerMap *map[string]*ReplicatServer, err error) {
	decoder := json.NewDecoder(sourceReader)
	err = decoder.Decode(&newServerMap)
	return
}

func configUpdateProcessor(c chan *map[string]*ReplicatServer) {
	for {
		newServerMap := <-c
		serverMapLock.Lock()

		lastConfigPing = time.Now()
		sendData := false

		// find any nodes that have been deleted and update ones that have changed
		for name, serverData := range serverMap {
			newServerData, exists := (*newServerMap)[name]
			if !exists {
				fmt.Printf("No longer found config for: %s deleting\n", name)
				delete(serverMap, name)
				continue
			}

			if serverData.Address != newServerData.Address || serverData.Name != newServerData.Name || serverData.ClusterKey != newServerData.ClusterKey || serverData.Status != serverData.Status {
				fmt.Printf("Server data is changed. Replacing.\nold: %v\nnew: %v\n", &serverData, &newServerData)
				if serverData.Status != REPLICAT_STATUS_JOINING_CLUSTER && newServerData.Status == REPLICAT_STATUS_JOINING_CLUSTER {
					fmt.Printf("Decided to send data to: %s\n", serverData.Name)
					sendData = true
				}
				serverMap[name] = newServerData
				fmt.Println("Server data replaced with new server data")
			}
		}

		// find any new nodes
		for name, newServerData := range *newServerMap {
			_, exists := serverMap[name]
			if !exists {
				fmt.Printf("New server configuration for %s: %v\n", name, newServerData)

				// If this server map is for ourselves, build a list of folder if needed and notify others
				if name == globalSettings.Name {
					listOfFileInfo, err := scanDirectoryContents()
					if err != nil {
						log.Fatal(err)
					}
					newServerData.CurrentState = listOfFileInfo
					// Tell all of our friends that we exist and our current state for them to compare against.
					go func(tree DirTreeMap) {
						sendFolderTree(tree)
					}(listOfFileInfo)
				} else {
					fmt.Printf("New Node Decided to send data to: %s\n", name)
					sendData = true
				}

				fmt.Printf("New server configuration provided. Copying: %s\n", name)
				serverMap[name] = newServerData
			}
		}

		if sendData {
			server := serverMap[globalSettings.Name]
			fmt.Println("about to send existing files")
			server.storage.SendCatalog()
			fmt.Println("done sending existing files")
		}

		serverMapLock.Unlock()
	}
}
<commit_msg>remove unused code<commit_after>// Package replicat is a server for n way synchronization of content (Replication for the cloud).
// Copyright 2016 Jacob Taylor jacob@replic.at       More Info: http://replic.at
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"sync"
	"time"
)

// ReplicatServer is a structure that contains the definition of the servers in a cluster. Each node has a name and this
// node (as determined by globalSettings.name at the moment) also has a StorageTracker interface.
type ReplicatServer struct {
	ClusterKey    string
	Name          string
	Address       string
	Status        string
	CurrentState  DirTreeMap
	PreviousState DirTreeMap
	Lock          sync.Mutex
	storage       StorageTracker
}

// GetStatus - get the current status of the server
func (server *ReplicatServer) GetStatus() string {
	return server.Status
}

// SetStatus - set the current status of the server
func (server *ReplicatServer) SetStatus(status string) {
	// adding an inline check to make sure the server is valid
	// if the server is being shut down or at the end of a unit test)
	// the assignment can crash without this check
	if server != nil {
		server.Status = status
		sendConfigToServer()
	}
}

const (
	// REPLICAT_STATUS_INITIAL_SCAN - The server is scanning its local storage
	REPLICAT_STATUS_INITIAL_SCAN = "Initial Scan"
	// REPLICAT_STATUS_JOINING_CLUSTER - The server has started to join the cluster and get up to date
	REPLICAT_STATUS_JOINING_CLUSTER = "Joining Cluster"
	// REPLICAT_STATUS_ONLINE - The server is up to date and part of the cluster
	REPLICAT_STATUS_ONLINE = "Online"
)

var serverMap = make(map[string]*ReplicatServer)
var serverMapLock sync.RWMutex

var lastConfigPing = time.Now()

// BootstrapAndServe - Start the server
func BootstrapAndServe(address string) {
	//trackerTestDual()
	//trackerTestSmallFileInSubfolder()
	//trackerTestEmptyDirectoryMovesInOutAround()
	//trackerTestFileChangeTrackerAddFolders()
	//trackerTestSmallFileCreationAndRename()
	//trackerTestSmallFileCreationAndUpdate()
	//trackerTestSmallFileMovesInOutAround()
	//trackerTestDirectoryCreation()
	//trackerTestNestedDirectoryCreation()
	//trackerTestDirectoryStorage()
	//trackerTestFileChangeTrackerAutoCreateFolderAndCleanup()
	//trackerTestNestedFastDirectoryCreation()

	// testing code to enable debugger use
	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))
	http.Handle("/upload/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(uploadHandler)))

	lsnr, err := net.Listen("tcp4", address)
	if err != nil {
		panic(fmt.Sprintf("Error listening: %v\nAddress: %s\n", err, address))
	}
	fmt.Println("Listening on:", lsnr.Addr().String())

	logOnlyHandler := LogOnlyChangeHandler{}
	tracker := FilesystemTracker{}
	fmt.Printf("Looking up settings for node: %s\n", globalSettings.Name)

	directory := globalSettings.Directory

	fmt.Printf("GlobalSettings directory retrieved for this node: %s\n", directory)
	server := &ReplicatServer{Name: globalSettings.Name, ClusterKey: globalSettings.ClusterKey, Address: lsnr.Addr().String(), storage: &tracker, Status: REPLICAT_STATUS_INITIAL_SCAN}
	serverMap[globalSettings.Name] = server
	tracker.init(directory, server)

	go func(tracker StorageTracker) {
		for true {
			tracker.GetStatistics()
			time.Sleep(30 * time.Second)
		}
	}(&tracker)

	var c ChangeHandler
	c = &logOnlyHandler
	tracker.watchDirectory(&c)

	go func(listener net.Listener) {
		err = http.Serve(listener, nil)
		if err != nil {
			panic(err)
		}
	}(lsnr)

	fmt.Println("Starting config update processor")
	go configUpdateProcessor(configUpdateChannel)

	if globalSettings.ManagerAddress != "" {
		fmt.Printf("about to send config to server (%s)\nOur address is: (%s)\n", globalSettings.ManagerAddress, lsnr.Addr())
	}

	// we need something to keep us running.
	keepConfigCurrent()
}

const (
	managerOverdueSeconds = 30
	managerCheckSleepTime = 45
)

func keepConfigCurrent() {
	for {
		if time.Since(lastConfigPing) > managerOverdueSeconds*time.Second {
			log.Printf("Manager Contact Overdue, attempting to contact: %s\n", globalSettings.ManagerAddress)
			sendConfigToServer()
		} else {
			fmt.Println("No Update Required")
		}
		time.Sleep(time.Duration(rand.Intn(10)-5+managerCheckSleepTime) * time.Second)
	}
}

func sendConfigToServer() {
	// This field will be empty during testing. We have to abort during testing since the manager is not reachable.
	if globalSettings.ManagerAddress == "" {
		return
	}

	url := "https://" + globalSettings.ManagerAddress + "/config/"
	log.Printf("sendConfigToServer: Manager location: %s\n", url)

	server := serverMap[globalSettings.Name]
	jsonStr, _ := json.Marshal(server)
	jsonStr2, _ := json.Marshal(server.storage.GetStatistics())

	byteArraysToMarshal := [][]byte{jsonStr, jsonStr2}
	byteData := bytes.Join(byteArraysToMarshal, []byte{})
	log.Printf("jasonstr: %s\n", string(byteData))
	jsonData := bytes.NewBuffer(byteData)

	req, err := http.NewRequest("POST", url, jsonData)
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Unable to reach server. Data could be lost: %s\n", err)
		return
	}

	newServerMap, err := extractServerMapFromConfig(resp.Body)
	if err == io.EOF {
		log.Printf("EOF unexpected Config update failed\n")
	} else if err != nil {
		log.Printf("Config update failed due to error: %s\n", err)
	} else {
		configUpdateChannel <- newServerMap
	}

}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		r.ParseMultipartForm(32 << 20)
		file, handler, err := r.FormFile("uploadfile")
		if err != nil {
			fmt.Println(err)
			return
		}
		defer file.Close()
		fmt.Fprint(w, handler.Header)

		hash := []byte(r.Form.Get("HASH"))

		storage := serverMap[globalSettings.Name].storage
		local, _ := storage.getEntryJSON(handler.Filename)

		if !bytes.Equal(hash, local.Hash) {
			fullPath := globalSettings.Directory + "/" + handler.Filename
			f, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0666)

			if err != nil {
				fmt.Println(err)
				return
			}

			bytesWritten, err := io.Copy(f, file)
			if err != nil {
				log.Printf("Error copying file: %s, error(%#v)\n", handler.Filename, err)
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("500 - Error copying file"))
				f.Close()
				return
			}
			fmt.Printf("Wrote out (%s) bytes (%d)\n", handler.Filename, bytesWritten)
			f.Close()

			// Get the old entry
			entryString := r.Form.Get("entryJSON")
			if entryString != "" {
				var entry EntryJSON
				err := json.Unmarshal([]byte(entryString), &entry)
				if err != nil {
					log.Fatalf("Error copying file (Entry handling): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}

				err = os.Chtimes(fullPath, time.Now(), entry.ModTime)
				if err != nil {
					log.Fatalf("Error copying file (Changing times): %s, error(%#v)\n", handler.Filename, err)
					w.WriteHeader(http.StatusInternalServerError)
					w.Write([]byte("500 - Error copying file"))
					return
				}
			}

		}
	}
}

var configUpdateChannel = make(chan *map[string]*ReplicatServer, 100)

func configHandler(_ http.ResponseWriter, r *http.Request) {
	newServerMap, err := extractServerMapFromConfig(r.Body)
	if err != nil {
		fmt.Printf("Config update failed due to error: %s\n", err)
		return
	}
	configUpdateChannel <- newServerMap
}

func extractServerMapFromConfig(sourceReader io.Reader) (newServerMap *map[string]*ReplicatServer, err error) {
	decoder := json.NewDecoder(sourceReader)
	err = decoder.Decode(&newServerMap)
	return
}

func configUpdateProcessor(c chan *map[string]*ReplicatServer) {
	for {
		newServerMap := <-c
		serverMapLock.Lock()

		lastConfigPing = time.Now()
		sendData := false

		// find any nodes that have been deleted and update ones that have changed
		for name, serverData := range serverMap {
			newServerData, exists := (*newServerMap)[name]
			if !exists {
				fmt.Printf("No longer found config for: %s deleting\n", name)
				delete(serverMap, name)
				continue
			}

			if serverData.Address != newServerData.Address || serverData.Name != newServerData.Name || serverData.ClusterKey != newServerData.ClusterKey || serverData.Status != serverData.Status {
				fmt.Printf("Server data is changed. Replacing.\nold: %v\nnew: %v\n", &serverData, &newServerData)
				if serverData.Status != REPLICAT_STATUS_JOINING_CLUSTER && newServerData.Status == REPLICAT_STATUS_JOINING_CLUSTER {
					fmt.Printf("Decided to send data to: %s\n", serverData.Name)
					sendData = true
				}
				serverMap[name] = newServerData
				fmt.Println("Server data replaced with new server data")
			}
		}

		// find any new nodes
		for name, newServerData := range *newServerMap {
			_, exists := serverMap[name]
			if !exists {
				fmt.Printf("New server configuration for %s: %v\n", name, newServerData)

				// If this server map is for ourselves, build a list of folder if needed and notify others
				if name == globalSettings.Name {
					listOfFileInfo, err := scanDirectoryContents()
					if err != nil {
						log.Fatal(err)
					}
					newServerData.CurrentState = listOfFileInfo
					// Tell all of our friends that we exist and our current state for them to compare against.
					go func(tree DirTreeMap) {
						sendFolderTree(tree)
					}(listOfFileInfo)
				} else {
					fmt.Printf("New Node Decided to send data to: %s\n", name)
					sendData = true
				}

				fmt.Printf("New server configuration provided. Copying: %s\n", name)
				serverMap[name] = newServerData
			}
		}

		if sendData {
			server := serverMap[globalSettings.Name]
			fmt.Println("about to send existing files")
			server.storage.SendCatalog()
			fmt.Println("done sending existing files")
		}

		serverMapLock.Unlock()
	}
}
<|endoftext|>


<commit_before>package main

// import (
// 	"flag"
// 	"fmt"
// 	"log"
// 	"os"
// 	"os/exec"
// 	"runtime/debug"
// )

// var (
// 	flagCertificateAuthorityName = flag.String("cn", "", "")
// 	flagHostName                 = flag.String("h", "", "")
// )

// var usage = `Usage: EasyCert [options...]
// Options:
//   -cn Certificate Authority Name (can be any name, but should reflect your company name.)
//   -h  Hostname of TLS server to install the private cert/key
// `

// func main() {
// 	flag.Usage = func() {
// 		fmt.Fprintf(os.Stderr, usage)
// 	}

// 	flag.Parse()

// 	certName := *flagCertificateAuthorityName
// 	hostName := *flagHostName

// 	if certName == "" || hostName == "" {
// 		usageAndExit("You must supply both a -cn (certificate name) and -h (host name) parameter")
// 	}

// 	createPrivateCA(certName)
// 	createServerCertKey(hostName)

// 	log.Println("*** Operation Completed Succesfully ***")
// 	log.Println("Private root certificate created: ", "ca.pem")
// 	log.Println("Web server certificate created: ", "cert.pem")
// 	log.Println("Web server key created: ", "key.pem")
// }

// func createPrivateCA(certificateAuthorityName string) {
// 	_, err := callCommand("openssl", "genrsa", "-out", "myCA.key", "2048")
// 	if err != nil {
// 		log.Fatal("Could not create private Certificate Authority key")
// 	}

// 	_, err = callCommand("openssl", "req", "-x509", "-new", "-key", "myCA.key", "-out", "ca.pem", "-days", "730", "-subj", "/CN=\""+certificateAuthorityName+"\"")
// 	if err != nil {
// 		log.Fatal("Could not create private Certificate Authority certificate")
// 	}
// }

// func createServerCertKey(host string) {
// 	_, err := callCommand("openssl", "genrsa", "-out", "key.pem", "2048")
// 	if err != nil {
// 		log.Fatal("Could not create private server key")
// 	}

// 	_, err = callCommand("openssl", "req", "-new", "-out", "cert.pem", "-key", "key.pem", "-subj", "/CN="+host)
// 	if err != nil {
// 		log.Fatal("Could not create private server certificate signing request")
// 	}

// 	_, err = callCommand("openssl", "x509", "-req", "-in", "cert.pem", "-out", "cert.pem", "-CAkey", "myCA.key", "-CA", "ca.pem", "-days", "365", "-CAcreateserial", "-CAserial", "serial")
// 	if err != nil {
// 		log.Fatal("Could not create private server certificate")
// 	}

// }

// func callCommand(command string, arg ...string) (string, error) {
// 	out, err := exec.Command(command, arg...).Output()

// 	if err != nil {
// 		log.Println("callCommand failed!")
// 		log.Println("")
// 		log.Println(string(debug.Stack()))
// 		return "", err
// 	}
// 	return string(out), nil
// }

// func usageAndExit(message string) {
// 	if message != "" {
// 		fmt.Fprintf(os.Stderr, message)
// 		fmt.Fprintf(os.Stderr, "\n\n")
// 	}
// 	flag.Usage()
// 	fmt.Fprintf(os.Stderr, "\n")
// 	os.Exit(1)
// }

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "tlsgen"
	app.Usage = "Generates web server TLS certificates"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "org",
			Value: "",
			Usage: "Organization name",
		},
		cli.StringSliceFlag{
			Name:  "host",
			Value: &cli.StringSlice{},
			Usage: "Server Host",
		},
	}
	app.Action = func(c *cli.Context) {
		org := c.String("org")
		if org == "" {
			logrus.Fatal("org shoule not be empty")
		}
		host := c.StringSlice("host")
		if len(host) == 0 {
			logrus.Fatal("host shoule not be empty")
		}
		GenerateCACertificate("ca.pem", "ca-key.pem", org, 2048)
		GenerateCert(host, "cert.pem", "key.pem", "ca.pem", "ca-key.pem", org, 2048)
	}

	app.Run(os.Args)
}

func getTLSConfig(caCert, cert, key []byte, allowInsecure bool) (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = allowInsecure
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(caCert)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}
	if allowInsecure {
		tlsConfig.InsecureSkipVerify = true
	}

	return &tlsConfig, nil
}

func newCertificate(org string) (*x509.Certificate, error) {
	now := time.Now()
	// need to set notBefore slightly in the past to account for time
	// skew in the VMs otherwise the certs sometimes are not yet valid
	notBefore := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute()-5, 0, 0, time.Local)
	notAfter := notBefore.Add(time.Hour * 24 * 1080)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}

	return &x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{org},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}, nil

}

// GenerateCACertificate generates a new certificate authority from the specified org
// and bit size and stores the resulting certificate and key file
// in the arguments.
func GenerateCACertificate(certFile, keyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}

	template.IsCA = true
	template.KeyUsage |= x509.KeyUsageCertSign

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err
	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

// GenerateCert generates a new certificate signed using the provided
// certificate authority files and stores the result in the certificate
// file and key provided.  The provided host names are set to the
// appropriate certificate fields.
func GenerateCert(hosts []string, certFile, keyFile, caFile, caKeyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}
	// client
	if len(hosts) == 1 && hosts[0] == "" {
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
		template.KeyUsage = x509.KeyUsageDigitalSignature
	} else { // server
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}
		for _, h := range hosts {
			if ip := net.ParseIP(h); ip != nil {
				template.IPAddresses = append(template.IPAddresses, ip)

			} else {
				template.DNSNames = append(template.DNSNames, h)
			}
		}
	}

	tlsCert, err := tls.LoadX509KeyPair(caFile, caKeyFile)
	if err != nil {
		return err

	}

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err

	}

	x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err

	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

func ValidateCertificate(addr, caCertPath, serverCertPath, serverKeyPath string) (bool, error) {
	caCert, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return false, err
	}

	serverCert, err := ioutil.ReadFile(serverCertPath)
	if err != nil {
		return false, err
	}

	serverKey, err := ioutil.ReadFile(serverKeyPath)
	if err != nil {
		return false, err
	}

	tlsConfig, err := getTLSConfig(caCert, serverCert, serverKey, false)
	if err != nil {
		return false, err
	}

	dialer := &net.Dialer{
		Timeout: time.Second * 2,
	}

	_, err = tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
	if err != nil {
		return false, nil
	}

	return true, nil
}
<commit_msg>update<commit_after>package main

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"math/big"
	"net"
	"os"
	"time"
)

func main() {
	app := cli.NewApp()
	app.Name = "tlsgen"
	app.Usage = "Generates web server TLS certificates"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "org",
			Value: "",
			Usage: "Organization name",
		},
		cli.StringSliceFlag{
			Name:  "host",
			Value: &cli.StringSlice{},
			Usage: "Server Host",
		},
	}
	app.Action = func(c *cli.Context) {
		org := c.String("org")
		if org == "" {
			logrus.Fatal("org shoule not be empty")
		}
		host := c.StringSlice("host")
		if len(host) == 0 {
			logrus.Fatal("host shoule not be empty")
		}
		GenerateCACertificate("ca.pem", "ca-key.pem", org, 2048)
		GenerateCert(host, "cert.pem", "key.pem", "ca.pem", "ca-key.pem", org, 2048)
	}

	app.Run(os.Args)
}

func getTLSConfig(caCert, cert, key []byte, allowInsecure bool) (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = allowInsecure
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(caCert)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}
	if allowInsecure {
		tlsConfig.InsecureSkipVerify = true
	}

	return &tlsConfig, nil
}

func newCertificate(org string) (*x509.Certificate, error) {
	now := time.Now()
	// need to set notBefore slightly in the past to account for time
	// skew in the VMs otherwise the certs sometimes are not yet valid
	notBefore := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute()-5, 0, 0, time.Local)
	notAfter := notBefore.Add(time.Hour * 24 * 1080)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}

	return &x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{org},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}, nil

}

// GenerateCACertificate generates a new certificate authority from the specified org
// and bit size and stores the resulting certificate and key file
// in the arguments.
func GenerateCACertificate(certFile, keyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}

	template.IsCA = true
	template.KeyUsage |= x509.KeyUsageCertSign

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err
	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

// GenerateCert generates a new certificate signed using the provided
// certificate authority files and stores the result in the certificate
// file and key provided.  The provided host names are set to the
// appropriate certificate fields.
func GenerateCert(hosts []string, certFile, keyFile, caFile, caKeyFile, org string, bits int) error {
	template, err := newCertificate(org)
	if err != nil {
		return err
	}
	// client
	if len(hosts) == 1 && hosts[0] == "" {
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
		template.KeyUsage = x509.KeyUsageDigitalSignature
	} else { // server
		template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}
		for _, h := range hosts {
			if ip := net.ParseIP(h); ip != nil {
				template.IPAddresses = append(template.IPAddresses, ip)

			} else {
				template.DNSNames = append(template.DNSNames, h)
			}
		}
	}

	tlsCert, err := tls.LoadX509KeyPair(caFile, caKeyFile)
	if err != nil {
		return err

	}

	priv, err := rsa.GenerateKey(rand.Reader, bits)
	if err != nil {
		return err

	}

	x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
	if err != nil {
		return err
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey)
	if err != nil {
		return err
	}

	certOut, err := os.Create(certFile)
	if err != nil {
		return err

	}

	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	certOut.Close()

	keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err

	}

	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
	keyOut.Close()

	return nil
}

func ValidateCertificate(addr, caCertPath, serverCertPath, serverKeyPath string) (bool, error) {
	caCert, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return false, err
	}

	serverCert, err := ioutil.ReadFile(serverCertPath)
	if err != nil {
		return false, err
	}

	serverKey, err := ioutil.ReadFile(serverKeyPath)
	if err != nil {
		return false, err
	}

	tlsConfig, err := getTLSConfig(caCert, serverCert, serverKey, false)
	if err != nil {
		return false, err
	}

	dialer := &net.Dialer{
		Timeout: time.Second * 2,
	}

	_, err = tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
	if err != nil {
		return false, nil
	}

	return true, nil
}
<|endoftext|>


<commit_before>package lua

/*
#include <lua.h>
 #include <lauxlib.h>
 #include <lualib.h>

*/
import "C"

const LUA_VERSION = C.LUA_VERSION
const LUA_RELEASE = C.LUA_RELEASE
const LUA_VERSION_NUM = C.LUA_VERSION_NUM
const LUA_COPYRIGHT = C.LUA_COPYRIGHT
const LUA_AUTHORS = C.LUA_AUTHORS
const LUA_MULTRET = C.LUA_MULTRET
const LUA_REGISTRYINDEX = C.LUA_REGISTRYINDEX
const LUA_ENVIRONINDEX = C.LUA_ENVIRONINDEX
const LUA_GLOBALSINDEX = C.LUA_GLOBALSINDEX
const LUA_YIELD = C.LUA_YIELD
const LUA_ERRRUN = C.LUA_ERRRUN
const LUA_ERRSYNTAX = C.LUA_ERRSYNTAX
const LUA_ERRMEM = C.LUA_ERRMEM
const LUA_ERRERR = C.LUA_ERRERR
const LUA_TNONE = C.LUA_TNONE
const LUA_TNIL = C.LUA_TNIL
const LUA_TBOOLEAN = C.LUA_TBOOLEAN
const LUA_TLIGHTUSERDATA = C.LUA_TLIGHTUSERDATA
const LUA_TNUMBER = C.LUA_TNUMBER
const LUA_TSTRING = C.LUA_TSTRING
const LUA_TTABLE = C.LUA_TTABLE
const LUA_TFUNCTION = C.LUA_TFUNCTION
const LUA_TUSERDATA = C.LUA_TUSERDATA
const LUA_TTHREAD = C.LUA_TTHREAD
const LUA_MINSTACK = C.LUA_MINSTACK
const LUA_GCSTOP = C.LUA_GCSTOP
const LUA_GCRESTART = C.LUA_GCRESTART
const LUA_GCCOLLECT = C.LUA_GCCOLLECT
const LUA_GCCOUNT = C.LUA_GCCOUNT
const LUA_GCCOUNTB = C.LUA_GCCOUNTB
const LUA_GCSTEP = C.LUA_GCSTEP
const LUA_GCSETPAUSE = C.LUA_GCSETPAUSE
const LUA_GCSETSTEPMUL = C.LUA_GCSETSTEPMUL
const LUA_HOOKCALL = C.LUA_HOOKCALL
const LUA_HOOKRET = C.LUA_HOOKRET
const LUA_HOOKLINE = C.LUA_HOOKLINE
const LUA_HOOKCOUNT = C.LUA_HOOKCOUNT
const LUA_HOOKTAILRET = C.LUA_HOOKTAILRET
const LUA_MASKCALL = C.LUA_MASKCALL
const LUA_MASKRET = C.LUA_MASKRET
const LUA_MASKLINE = C.LUA_MASKLINE
const LUA_MASKCOUNT = C.LUA_MASKCOUNT
const LUA_ERRFILE = C.LUA_ERRFILE
const LUA_NOREF = C.LUA_NOREF
const LUA_REFNIL = C.LUA_REFNIL
const LUA_FILEHANDLE = C.LUA_FILEHANDLE
const LUA_COLIBNAME = C.LUA_COLIBNAME
const LUA_TABLIBNAME = C.LUA_TABLIBNAME
const LUA_IOLIBNAME = C.LUA_IOLIBNAME
const LUA_OSLIBNAME = C.LUA_OSLIBNAME
const LUA_STRLIBNAME = C.LUA_STRLIBNAME
const LUA_MATHLIBNAME = C.LUA_MATHLIBNAME
const LUA_DBLIBNAME = C.LUA_DBLIBNAME
const LUA_LOADLIBNAME = C.LUA_LOADLIBNAME
<commit_msg>collect all consts under a single const statement<commit_after>package lua

/*
#include <lua.h>
 #include <lauxlib.h>
 #include <lualib.h>

*/
import "C"

const (
	LUA_VERSION = C.LUA_VERSION
	LUA_RELEASE = C.LUA_RELEASE
	LUA_VERSION_NUM = C.LUA_VERSION_NUM
	LUA_COPYRIGHT = C.LUA_COPYRIGHT
	LUA_AUTHORS = C.LUA_AUTHORS
	LUA_MULTRET = C.LUA_MULTRET
	LUA_REGISTRYINDEX = C.LUA_REGISTRYINDEX
	LUA_ENVIRONINDEX = C.LUA_ENVIRONINDEX
	LUA_GLOBALSINDEX = C.LUA_GLOBALSINDEX
	LUA_YIELD = C.LUA_YIELD
	LUA_ERRRUN = C.LUA_ERRRUN
	LUA_ERRSYNTAX = C.LUA_ERRSYNTAX
	LUA_ERRMEM = C.LUA_ERRMEM
	LUA_ERRERR = C.LUA_ERRERR
	LUA_TNONE = C.LUA_TNONE
	LUA_TNIL = C.LUA_TNIL
	LUA_TBOOLEAN = C.LUA_TBOOLEAN
	LUA_TLIGHTUSERDATA = C.LUA_TLIGHTUSERDATA
	LUA_TNUMBER = C.LUA_TNUMBER
	LUA_TSTRING = C.LUA_TSTRING
	LUA_TTABLE = C.LUA_TTABLE
	LUA_TFUNCTION = C.LUA_TFUNCTION
	LUA_TUSERDATA = C.LUA_TUSERDATA
	LUA_TTHREAD = C.LUA_TTHREAD
	LUA_MINSTACK = C.LUA_MINSTACK
	LUA_GCSTOP = C.LUA_GCSTOP
	LUA_GCRESTART = C.LUA_GCRESTART
	LUA_GCCOLLECT = C.LUA_GCCOLLECT
	LUA_GCCOUNT = C.LUA_GCCOUNT
	LUA_GCCOUNTB = C.LUA_GCCOUNTB
	LUA_GCSTEP = C.LUA_GCSTEP
	LUA_GCSETPAUSE = C.LUA_GCSETPAUSE
	LUA_GCSETSTEPMUL = C.LUA_GCSETSTEPMUL
	LUA_HOOKCALL = C.LUA_HOOKCALL
	LUA_HOOKRET = C.LUA_HOOKRET
	LUA_HOOKLINE = C.LUA_HOOKLINE
	LUA_HOOKCOUNT = C.LUA_HOOKCOUNT
	LUA_HOOKTAILRET = C.LUA_HOOKTAILRET
	LUA_MASKCALL = C.LUA_MASKCALL
	LUA_MASKRET = C.LUA_MASKRET
	LUA_MASKLINE = C.LUA_MASKLINE
	LUA_MASKCOUNT = C.LUA_MASKCOUNT
	LUA_ERRFILE = C.LUA_ERRFILE
	LUA_NOREF = C.LUA_NOREF
	LUA_REFNIL = C.LUA_REFNIL
	LUA_FILEHANDLE = C.LUA_FILEHANDLE
	LUA_COLIBNAME = C.LUA_COLIBNAME
	LUA_TABLIBNAME = C.LUA_TABLIBNAME
	LUA_IOLIBNAME = C.LUA_IOLIBNAME
	LUA_OSLIBNAME = C.LUA_OSLIBNAME
	LUA_STRLIBNAME = C.LUA_STRLIBNAME
	LUA_MATHLIBNAME = C.LUA_MATHLIBNAME
	LUA_DBLIBNAME = C.LUA_DBLIBNAME
	LUA_LOADLIBNAME = C.LUA_LOADLIBNAME
)<|endoftext|>


<commit_before>package tntctl

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
)

// Box is tarantool instance. For start/stop tarantool in tests
type Box struct {
	Root     string
	Port     uint
	cmd      *exec.Cmd
	stopOnce sync.Once
	stopped  chan bool
}

type Options struct {
	Listen  uint
	PortMin uint
	PortMax uint
}

func New(config string, options *Options) (*Box, error) {
	if options == nil {
		options = &Options{}
	}

	if options.PortMin == 0 {
		options.PortMin = 8000
	}

	if options.PortMax == 0 {
		options.PortMax = 9000
	}

	if options.Listen != 0 {
		options.PortMin = options.Listen
		options.PortMax = options.Listen
	}

	var box *Box

START_LOOP:
	for port := options.PortMin; port <= options.PortMax; port += 2 {

		tmpDir, err := ioutil.TempDir("", "") //os.RemoveAll(tmpDir);
		if err != nil {
			return nil, err
		}

		tarantoolConf := `
		slab_alloc_arena = 0.1
		slab_alloc_factor = 1.04

		pid_file = {root}/box.pid
		work_dir = {root}
		snap_dir = snap
		wal_dir = wal

		snap_io_rate_limit = 10
		rows_per_wal = 1000000
		too_long_threshold = 0.025

		primary_port = {port1}
		memcached_expire = false
		memcached_port = {port2}
        `

		tarantoolConf = strings.Replace(tarantoolConf, "{port1}", fmt.Sprintf("%d", port), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port2}", fmt.Sprintf("%d", port+1), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{root}", tmpDir, -1)

		tarantoolConf = fmt.Sprintf("%s\n%s", tarantoolConf, config)

		tarantoolConfFile := path.Join(tmpDir, "tarantool.conf")
		err = ioutil.WriteFile(tarantoolConfFile, []byte(tarantoolConf), 0644)
		if err != nil {
			return nil, err
		}

		for _, subDir := range []string{"snap", "wal"} {
			err = os.Mkdir(path.Join(tmpDir, subDir), 0755)
			if err != nil {
				return nil, err
			}
		}

		cmd0 := exec.Command("tarantool_box", "-c", tarantoolConfFile, "--init-storage")
		err = cmd0.Run()
		if err != nil {
			return nil, err
		}

		cmd := exec.Command("tarantool_box", "-c", tarantoolConfFile)
		boxStderr, err := cmd.StderrPipe()
		if err != nil {
			return nil, err
		}

		err = cmd.Start()
		if err != nil {
			return nil, err
		}

		var boxStderrBuffer bytes.Buffer

		p := make([]byte, 1024)

		box = &Box{
			Root:    tmpDir,
			Port:    port,
			cmd:     cmd,
			stopped: make(chan bool),
		}

	WAIT_LOOP:
		for {
			if strings.Contains(boxStderrBuffer.String(), "entering event loop") {
				break START_LOOP
			}

			if strings.Contains(boxStderrBuffer.String(), "is already in use, will retry binding after") {
				cmd.Process.Kill()
				cmd.Process.Wait()
				break WAIT_LOOP
			}

			n, err := boxStderr.Read(p)
			if n > 0 {
				boxStderrBuffer.Write(p[:n])
			}
			if err != nil {
				fmt.Println(boxStderrBuffer.String())
				return nil, err
			}
		}

		os.RemoveAll(box.Root)
		box = nil
	}

	if box == nil {
		return nil, fmt.Errorf("Can't bind any port from %d to %d", options.PortMin, options.PortMax)
	}

	return box, nil
}

func (box *Box) Close() {
	box.stopOnce.Do(func() {
		box.cmd.Process.Kill()
		box.cmd.Process.Wait()
		os.RemoveAll(box.Root)
		close(box.stopped)
	})
	<-box.stopped
}
<commit_msg>increase slab_alloc_arena<commit_after>package tntctl

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
)

// Box is tarantool instance. For start/stop tarantool in tests
type Box struct {
	Root     string
	Port     uint
	cmd      *exec.Cmd
	stopOnce sync.Once
	stopped  chan bool
}

type Options struct {
	Listen  uint
	PortMin uint
	PortMax uint
}

func New(config string, options *Options) (*Box, error) {
	if options == nil {
		options = &Options{}
	}

	if options.PortMin == 0 {
		options.PortMin = 8000
	}

	if options.PortMax == 0 {
		options.PortMax = 9000
	}

	if options.Listen != 0 {
		options.PortMin = options.Listen
		options.PortMax = options.Listen
	}

	var box *Box

START_LOOP:
	for port := options.PortMin; port <= options.PortMax; port += 2 {

		tmpDir, err := ioutil.TempDir("", "") //os.RemoveAll(tmpDir);
		if err != nil {
			return nil, err
		}

		tarantoolConf := `
		slab_alloc_arena = 1
		slab_alloc_factor = 1.04

		pid_file = {root}/box.pid
		work_dir = {root}
		snap_dir = snap
		wal_dir = wal

		snap_io_rate_limit = 10
		rows_per_wal = 1000000
		too_long_threshold = 0.025

		primary_port = {port1}
		memcached_expire = false
		memcached_port = {port2}
        `

		tarantoolConf = strings.Replace(tarantoolConf, "{port1}", fmt.Sprintf("%d", port), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{port2}", fmt.Sprintf("%d", port+1), -1)
		tarantoolConf = strings.Replace(tarantoolConf, "{root}", tmpDir, -1)

		tarantoolConf = fmt.Sprintf("%s\n%s", tarantoolConf, config)

		tarantoolConfFile := path.Join(tmpDir, "tarantool.conf")
		err = ioutil.WriteFile(tarantoolConfFile, []byte(tarantoolConf), 0644)
		if err != nil {
			return nil, err
		}

		for _, subDir := range []string{"snap", "wal"} {
			err = os.Mkdir(path.Join(tmpDir, subDir), 0755)
			if err != nil {
				return nil, err
			}
		}

		cmd0 := exec.Command("tarantool_box", "-c", tarantoolConfFile, "--init-storage")
		err = cmd0.Run()
		if err != nil {
			return nil, err
		}

		cmd := exec.Command("tarantool_box", "-c", tarantoolConfFile)
		boxStderr, err := cmd.StderrPipe()
		if err != nil {
			return nil, err
		}

		err = cmd.Start()
		if err != nil {
			return nil, err
		}

		var boxStderrBuffer bytes.Buffer

		p := make([]byte, 1024)

		box = &Box{
			Root:    tmpDir,
			Port:    port,
			cmd:     cmd,
			stopped: make(chan bool),
		}

	WAIT_LOOP:
		for {
			if strings.Contains(boxStderrBuffer.String(), "entering event loop") {
				break START_LOOP
			}

			if strings.Contains(boxStderrBuffer.String(), "is already in use, will retry binding after") {
				cmd.Process.Kill()
				cmd.Process.Wait()
				break WAIT_LOOP
			}

			n, err := boxStderr.Read(p)
			if n > 0 {
				boxStderrBuffer.Write(p[:n])
			}
			if err != nil {
				fmt.Println(boxStderrBuffer.String())
				return nil, err
			}
		}

		os.RemoveAll(box.Root)
		box = nil
	}

	if box == nil {
		return nil, fmt.Errorf("Can't bind any port from %d to %d", options.PortMin, options.PortMax)
	}

	return box, nil
}

func (box *Box) Close() {
	box.stopOnce.Do(func() {
		box.cmd.Process.Kill()
		box.cmd.Process.Wait()
		os.RemoveAll(box.Root)
		close(box.stopped)
	})
	<-box.stopped
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"net"
	"os"
	"strings"

	"github.com/pkg/errors"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// DHCPRange represents a range of IPs from start to end.
type DHCPRange struct {
	Start net.IP
	End   net.IP
}

// common represents a generic LXD network.
type common struct {
	logger      logger.Logger
	state       *state.State
	id          int64
	name        string
	netType     string
	description string
	config      map[string]string
}

// init initialise internal variables.
func (n *common) init(state *state.State, id int64, name string, netType string, description string, config map[string]string) {
	n.logger = logging.AddContext(logger.Log, log.Ctx{"driver": netType, "network": name})
	n.id = id
	n.name = name
	n.netType = netType
	n.config = config
	n.state = state
	n.description = description
}

// fillConfig fills requested config with any default values, by default this is a no-op.
func (n *common) fillConfig(req *api.NetworksPost) error {
	return nil
}

// validationRules returns a map of config rules common to all drivers.
func (n *common) validationRules() map[string]func(string) error {
	return map[string]func(string) error{}
}

// validate a network config against common rules and optional driver specific rules.
func (n *common) validate(config map[string]string, driverRules map[string]func(value string) error) error {
	checkedFields := map[string]struct{}{}

	// Get rules common for all drivers.
	rules := n.validationRules()

	// Merge driver specific rules into common rules.
	for field, validator := range driverRules {
		rules[field] = validator
	}

	// Run the validator against each field.
	for k, validator := range rules {
		checkedFields[k] = struct{}{} //Mark field as checked.
		err := validator(config[k])
		if err != nil {
			return errors.Wrapf(err, "Invalid value for network %q option %q", n.name, k)
		}
	}

	// Look for any unchecked fields, as these are unknown fields and validation should fail.
	for k := range config {
		_, checked := checkedFields[k]
		if checked {
			continue
		}

		// User keys are not validated.
		if strings.HasPrefix(k, "user.") {
			continue
		}

		return fmt.Errorf("Invalid option for network %q option %q", n.name, k)
	}

	return nil
}

// Name returns the network name.
func (n *common) Name() string {
	return n.name
}

// Type returns the network type.
func (n *common) Type() string {
	return n.netType
}

// Config returns the network config.
func (n *common) Config() map[string]string {
	return n.config
}

// IsUsed returns whether the network is used by any instances.
func (n *common) IsUsed() bool {
	// Look for instances using the interface
	insts, err := instance.LoadFromAllProjects(n.state)
	if err != nil {
		return true
	}

	for _, inst := range insts {
		if IsInUseByInstance(inst, n.name) {
			return true
		}
	}

	return false
}

// HasDHCPv4 indicates whether the network has DHCPv4 enabled.
func (n *common) HasDHCPv4() bool {
	if n.config["ipv4.dhcp"] == "" || shared.IsTrue(n.config["ipv4.dhcp"]) {
		return true
	}

	return false
}

// HasDHCPv6 indicates whether the network has DHCPv6 enabled (includes stateless SLAAC router advertisement mode).
// Technically speaking stateless SLAAC RA mode isn't DHCPv6, but for consistency with LXD's config paradigm, DHCP
// here means "an ability to automatically allocate IPs and routes", rather than stateful DHCP with leases.
// To check if true stateful DHCPv6 is enabled check the "ipv6.dhcp.stateful" config key.
func (n *common) HasDHCPv6() bool {
	if n.config["ipv6.dhcp"] == "" || shared.IsTrue(n.config["ipv6.dhcp"]) {
		return true
	}

	return false
}

// DHCPv4Ranges returns a parsed set of DHCPv4 ranges for this network.
func (n *common) DHCPv4Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv4.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv4.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To4(),
					End:   endIP.To4(),
				})
			}
		}
	}

	return dhcpRanges
}

// DHCPv6Ranges returns a parsed set of DHCPv6 ranges for this network.
func (n *common) DHCPv6Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv6.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv6.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To16(),
					End:   endIP.To16(),
				})
			}
		}
	}

	return dhcpRanges
}

// update the internal config variables, and if not cluster notification, notifies all nodes and updates database.
func (n *common) update(applyNetwork api.NetworkPut, clusterNotification bool) error {
	// Update internal config before database has been updated (so that if update is a notification we apply
	// the config being supplied and not that in the database).
	n.description = applyNetwork.Description
	n.config = applyNetwork.Config

	// If this update isn't coming via a cluster notification itself, then notify all nodes of change and then
	// update the database.
	if !clusterNotification {
		// Notify all other nodes to update the network.
		notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), cluster.NotifyAll)
		if err != nil {
			return err
		}

		err = notifier(func(client lxd.InstanceServer) error {
			return client.UpdateNetwork(n.name, applyNetwork, "")
		})
		if err != nil {
			return err
		}

		// Update the database.
		err = n.state.Cluster.UpdateNetwork(n.name, applyNetwork.Description, applyNetwork.Config)
		if err != nil {
			return err
		}
	}

	return nil
}

// configChanged compares supplied new config with existing config. Returns a boolean indicating if differences in
// the config or description were found (and the database record needs updating), and a list of non-user config
// keys that have changed, and a copy of the current internal network config that can be used to revert if needed.
func (n *common) configChanged(newNetwork api.NetworkPut) (bool, []string, api.NetworkPut, error) {
	// Backup the current state.
	oldNetwork := api.NetworkPut{
		Description: n.description,
		Config:      map[string]string{},
	}

	err := shared.DeepCopy(&n.config, &oldNetwork.Config)
	if err != nil {
		return false, nil, oldNetwork, err
	}

	// Diff the configurations.
	changedKeys := []string{}
	dbUpdateNeeded := false

	if newNetwork.Description != n.description {
		dbUpdateNeeded = true
	}

	for k, v := range oldNetwork.Config {
		if v != newNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	for k, v := range newNetwork.Config {
		if v != oldNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	return dbUpdateNeeded, changedKeys, oldNetwork, nil
}

// rename the network directory, update database record and update internal variables.
func (n *common) rename(newName string) error {
	// Clear new directory if exists.
	if shared.PathExists(shared.VarPath("networks", newName)) {
		os.RemoveAll(shared.VarPath("networks", newName))
	}

	// Rename directory to new name.
	if shared.PathExists(shared.VarPath("networks", n.name)) {
		err := os.Rename(shared.VarPath("networks", n.name), shared.VarPath("networks", newName))
		if err != nil {
			return err
		}
	}

	// Rename the database entry.
	err := n.state.Cluster.RenameNetwork(n.name, newName)
	if err != nil {
		return err
	}

	// Reinitialise internal name variable and logger context with new name.
	n.init(n.state, n.id, newName, n.netType, n.description, n.config)

	return nil
}

// delete the network from the database if clusterNotification is false.
func (n *common) delete(clusterNotification bool) error {
	// Only delete database record if not cluster notification.
	if !clusterNotification {
		// Remove the network from the database.
		err := n.state.Cluster.DeleteNetwork(n.name)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>lxd/network/driver/common: Adds default HandleHeartbeat function<commit_after>package network

import (
	"fmt"
	"net"
	"os"
	"strings"

	"github.com/pkg/errors"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// DHCPRange represents a range of IPs from start to end.
type DHCPRange struct {
	Start net.IP
	End   net.IP
}

// common represents a generic LXD network.
type common struct {
	logger      logger.Logger
	state       *state.State
	id          int64
	name        string
	netType     string
	description string
	config      map[string]string
}

// init initialise internal variables.
func (n *common) init(state *state.State, id int64, name string, netType string, description string, config map[string]string) {
	n.logger = logging.AddContext(logger.Log, log.Ctx{"driver": netType, "network": name})
	n.id = id
	n.name = name
	n.netType = netType
	n.config = config
	n.state = state
	n.description = description
}

// fillConfig fills requested config with any default values, by default this is a no-op.
func (n *common) fillConfig(req *api.NetworksPost) error {
	return nil
}

// validationRules returns a map of config rules common to all drivers.
func (n *common) validationRules() map[string]func(string) error {
	return map[string]func(string) error{}
}

// validate a network config against common rules and optional driver specific rules.
func (n *common) validate(config map[string]string, driverRules map[string]func(value string) error) error {
	checkedFields := map[string]struct{}{}

	// Get rules common for all drivers.
	rules := n.validationRules()

	// Merge driver specific rules into common rules.
	for field, validator := range driverRules {
		rules[field] = validator
	}

	// Run the validator against each field.
	for k, validator := range rules {
		checkedFields[k] = struct{}{} //Mark field as checked.
		err := validator(config[k])
		if err != nil {
			return errors.Wrapf(err, "Invalid value for network %q option %q", n.name, k)
		}
	}

	// Look for any unchecked fields, as these are unknown fields and validation should fail.
	for k := range config {
		_, checked := checkedFields[k]
		if checked {
			continue
		}

		// User keys are not validated.
		if strings.HasPrefix(k, "user.") {
			continue
		}

		return fmt.Errorf("Invalid option for network %q option %q", n.name, k)
	}

	return nil
}

// Name returns the network name.
func (n *common) Name() string {
	return n.name
}

// Type returns the network type.
func (n *common) Type() string {
	return n.netType
}

// Config returns the network config.
func (n *common) Config() map[string]string {
	return n.config
}

// IsUsed returns whether the network is used by any instances.
func (n *common) IsUsed() bool {
	// Look for instances using the interface
	insts, err := instance.LoadFromAllProjects(n.state)
	if err != nil {
		return true
	}

	for _, inst := range insts {
		if IsInUseByInstance(inst, n.name) {
			return true
		}
	}

	return false
}

// HasDHCPv4 indicates whether the network has DHCPv4 enabled.
func (n *common) HasDHCPv4() bool {
	if n.config["ipv4.dhcp"] == "" || shared.IsTrue(n.config["ipv4.dhcp"]) {
		return true
	}

	return false
}

// HasDHCPv6 indicates whether the network has DHCPv6 enabled (includes stateless SLAAC router advertisement mode).
// Technically speaking stateless SLAAC RA mode isn't DHCPv6, but for consistency with LXD's config paradigm, DHCP
// here means "an ability to automatically allocate IPs and routes", rather than stateful DHCP with leases.
// To check if true stateful DHCPv6 is enabled check the "ipv6.dhcp.stateful" config key.
func (n *common) HasDHCPv6() bool {
	if n.config["ipv6.dhcp"] == "" || shared.IsTrue(n.config["ipv6.dhcp"]) {
		return true
	}

	return false
}

// DHCPv4Ranges returns a parsed set of DHCPv4 ranges for this network.
func (n *common) DHCPv4Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv4.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv4.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To4(),
					End:   endIP.To4(),
				})
			}
		}
	}

	return dhcpRanges
}

// DHCPv6Ranges returns a parsed set of DHCPv6 ranges for this network.
func (n *common) DHCPv6Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv6.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv6.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To16(),
					End:   endIP.To16(),
				})
			}
		}
	}

	return dhcpRanges
}

// update the internal config variables, and if not cluster notification, notifies all nodes and updates database.
func (n *common) update(applyNetwork api.NetworkPut, clusterNotification bool) error {
	// Update internal config before database has been updated (so that if update is a notification we apply
	// the config being supplied and not that in the database).
	n.description = applyNetwork.Description
	n.config = applyNetwork.Config

	// If this update isn't coming via a cluster notification itself, then notify all nodes of change and then
	// update the database.
	if !clusterNotification {
		// Notify all other nodes to update the network.
		notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), cluster.NotifyAll)
		if err != nil {
			return err
		}

		err = notifier(func(client lxd.InstanceServer) error {
			return client.UpdateNetwork(n.name, applyNetwork, "")
		})
		if err != nil {
			return err
		}

		// Update the database.
		err = n.state.Cluster.UpdateNetwork(n.name, applyNetwork.Description, applyNetwork.Config)
		if err != nil {
			return err
		}
	}

	return nil
}

// configChanged compares supplied new config with existing config. Returns a boolean indicating if differences in
// the config or description were found (and the database record needs updating), and a list of non-user config
// keys that have changed, and a copy of the current internal network config that can be used to revert if needed.
func (n *common) configChanged(newNetwork api.NetworkPut) (bool, []string, api.NetworkPut, error) {
	// Backup the current state.
	oldNetwork := api.NetworkPut{
		Description: n.description,
		Config:      map[string]string{},
	}

	err := shared.DeepCopy(&n.config, &oldNetwork.Config)
	if err != nil {
		return false, nil, oldNetwork, err
	}

	// Diff the configurations.
	changedKeys := []string{}
	dbUpdateNeeded := false

	if newNetwork.Description != n.description {
		dbUpdateNeeded = true
	}

	for k, v := range oldNetwork.Config {
		if v != newNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	for k, v := range newNetwork.Config {
		if v != oldNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	return dbUpdateNeeded, changedKeys, oldNetwork, nil
}

// rename the network directory, update database record and update internal variables.
func (n *common) rename(newName string) error {
	// Clear new directory if exists.
	if shared.PathExists(shared.VarPath("networks", newName)) {
		os.RemoveAll(shared.VarPath("networks", newName))
	}

	// Rename directory to new name.
	if shared.PathExists(shared.VarPath("networks", n.name)) {
		err := os.Rename(shared.VarPath("networks", n.name), shared.VarPath("networks", newName))
		if err != nil {
			return err
		}
	}

	// Rename the database entry.
	err := n.state.Cluster.RenameNetwork(n.name, newName)
	if err != nil {
		return err
	}

	// Reinitialise internal name variable and logger context with new name.
	n.init(n.state, n.id, newName, n.netType, n.description, n.config)

	return nil
}

// delete the network from the database if clusterNotification is false.
func (n *common) delete(clusterNotification bool) error {
	// Only delete database record if not cluster notification.
	if !clusterNotification {
		// Remove the network from the database.
		err := n.state.Cluster.DeleteNetwork(n.name)
		if err != nil {
			return err
		}
	}

	return nil
}

// HandleHeartbeat is a no-op.
func (n *common) HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error {
	return nil
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"net"
	"os"
	"strings"

	"github.com/pkg/errors"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// DHCPRange represents a range of IPs from start to end.
type DHCPRange struct {
	Start net.IP
	End   net.IP
}

// common represents a generic LXD network.
type common struct {
	logger      logger.Logger
	state       *state.State
	id          int64
	name        string
	netType     string
	description string
	config      map[string]string
	status      string
}

// init initialise internal variables.
func (n *common) init(state *state.State, id int64, name string, netType string, description string, config map[string]string, status string) {
	n.logger = logging.AddContext(logger.Log, log.Ctx{"driver": netType, "network": name})
	n.id = id
	n.name = name
	n.netType = netType
	n.config = config
	n.state = state
	n.description = description
	n.status = status
}

// fillConfig fills requested config with any default values, by default this is a no-op.
func (n *common) fillConfig(req *api.NetworksPost) error {
	return nil
}

// validationRules returns a map of config rules common to all drivers.
func (n *common) validationRules() map[string]func(string) error {
	return map[string]func(string) error{}
}

// validate a network config against common rules and optional driver specific rules.
func (n *common) validate(config map[string]string, driverRules map[string]func(value string) error) error {
	checkedFields := map[string]struct{}{}

	// Get rules common for all drivers.
	rules := n.validationRules()

	// Merge driver specific rules into common rules.
	for field, validator := range driverRules {
		rules[field] = validator
	}

	// Run the validator against each field.
	for k, validator := range rules {
		checkedFields[k] = struct{}{} //Mark field as checked.
		err := validator(config[k])
		if err != nil {
			return errors.Wrapf(err, "Invalid value for network %q option %q", n.name, k)
		}
	}

	// Look for any unchecked fields, as these are unknown fields and validation should fail.
	for k := range config {
		_, checked := checkedFields[k]
		if checked {
			continue
		}

		// User keys are not validated.
		if strings.HasPrefix(k, "user.") {
			continue
		}

		return fmt.Errorf("Invalid option for network %q option %q", n.name, k)
	}

	return nil
}

// Name returns the network name.
func (n *common) Name() string {
	return n.name
}

// Status returns the network status.
func (n *common) Status() string {
	return n.status
}

// Type returns the network type.
func (n *common) Type() string {
	return n.netType
}

// Config returns the network config.
func (n *common) Config() map[string]string {
	return n.config
}

// IsUsed returns whether the network is used by any instances.
func (n *common) IsUsed() bool {
	// Look for instances using the interface
	insts, err := instance.LoadFromAllProjects(n.state)
	if err != nil {
		return true
	}

	for _, inst := range insts {
		if IsInUseByInstance(inst, n.name) {
			return true
		}
	}

	return false
}

// HasDHCPv4 indicates whether the network has DHCPv4 enabled.
func (n *common) HasDHCPv4() bool {
	if n.config["ipv4.dhcp"] == "" || shared.IsTrue(n.config["ipv4.dhcp"]) {
		return true
	}

	return false
}

// HasDHCPv6 indicates whether the network has DHCPv6 enabled (includes stateless SLAAC router advertisement mode).
// Technically speaking stateless SLAAC RA mode isn't DHCPv6, but for consistency with LXD's config paradigm, DHCP
// here means "an ability to automatically allocate IPs and routes", rather than stateful DHCP with leases.
// To check if true stateful DHCPv6 is enabled check the "ipv6.dhcp.stateful" config key.
func (n *common) HasDHCPv6() bool {
	if n.config["ipv6.dhcp"] == "" || shared.IsTrue(n.config["ipv6.dhcp"]) {
		return true
	}

	return false
}

// DHCPv4Ranges returns a parsed set of DHCPv4 ranges for this network.
func (n *common) DHCPv4Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv4.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv4.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To4(),
					End:   endIP.To4(),
				})
			}
		}
	}

	return dhcpRanges
}

// DHCPv6Ranges returns a parsed set of DHCPv6 ranges for this network.
func (n *common) DHCPv6Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv6.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv6.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To16(),
					End:   endIP.To16(),
				})
			}
		}
	}

	return dhcpRanges
}

// update the internal config variables, and if not cluster notification, notifies all nodes and updates database.
func (n *common) update(applyNetwork api.NetworkPut, clusterNotification bool) error {
	// Update internal config before database has been updated (so that if update is a notification we apply
	// the config being supplied and not that in the database).
	n.description = applyNetwork.Description
	n.config = applyNetwork.Config

	// If this update isn't coming via a cluster notification itself, then notify all nodes of change and then
	// update the database.
	if !clusterNotification {
		// Notify all other nodes to update the network.
		notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), cluster.NotifyAll)
		if err != nil {
			return err
		}

		err = notifier(func(client lxd.InstanceServer) error {
			return client.UpdateNetwork(n.name, applyNetwork, "")
		})
		if err != nil {
			return err
		}

		// Update the database.
		err = n.state.Cluster.UpdateNetwork(n.name, applyNetwork.Description, applyNetwork.Config)
		if err != nil {
			return err
		}
	}

	return nil
}

// configChanged compares supplied new config with existing config. Returns a boolean indicating if differences in
// the config or description were found (and the database record needs updating), and a list of non-user config
// keys that have changed, and a copy of the current internal network config that can be used to revert if needed.
func (n *common) configChanged(newNetwork api.NetworkPut) (bool, []string, api.NetworkPut, error) {
	// Backup the current state.
	oldNetwork := api.NetworkPut{
		Description: n.description,
		Config:      map[string]string{},
	}

	err := shared.DeepCopy(&n.config, &oldNetwork.Config)
	if err != nil {
		return false, nil, oldNetwork, err
	}

	// Diff the configurations.
	changedKeys := []string{}
	dbUpdateNeeded := false

	if newNetwork.Description != n.description {
		dbUpdateNeeded = true
	}

	for k, v := range oldNetwork.Config {
		if v != newNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	for k, v := range newNetwork.Config {
		if v != oldNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	return dbUpdateNeeded, changedKeys, oldNetwork, nil
}

// rename the network directory, update database record and update internal variables.
func (n *common) rename(newName string) error {
	// Clear new directory if exists.
	if shared.PathExists(shared.VarPath("networks", newName)) {
		os.RemoveAll(shared.VarPath("networks", newName))
	}

	// Rename directory to new name.
	if shared.PathExists(shared.VarPath("networks", n.name)) {
		err := os.Rename(shared.VarPath("networks", n.name), shared.VarPath("networks", newName))
		if err != nil {
			return err
		}
	}

	// Rename the database entry.
	err := n.state.Cluster.RenameNetwork(n.name, newName)
	if err != nil {
		return err
	}

	// Reinitialise internal name variable and logger context with new name.
	n.init(n.state, n.id, newName, n.netType, n.description, n.config, n.status)

	return nil
}

// delete the network from the database if clusterNotification is false.
func (n *common) delete(clusterNotification bool) error {
	// Only delete database record if not cluster notification.
	if !clusterNotification {
		// Remove the network from the database.
		err := n.state.Cluster.DeleteNetwork(n.name)
		if err != nil {
			return err
		}
	}

	return nil
}

// HandleHeartbeat is a no-op.
func (n *common) HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error {
	return nil
}
<commit_msg>lxd/network/driver/common: Tweaks to update function in cluster environment<commit_after>package network

import (
	"fmt"
	"net"
	"os"
	"strings"

	"github.com/pkg/errors"

	lxd "github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// DHCPRange represents a range of IPs from start to end.
type DHCPRange struct {
	Start net.IP
	End   net.IP
}

// common represents a generic LXD network.
type common struct {
	logger      logger.Logger
	state       *state.State
	id          int64
	name        string
	netType     string
	description string
	config      map[string]string
	status      string
}

// init initialise internal variables.
func (n *common) init(state *state.State, id int64, name string, netType string, description string, config map[string]string, status string) {
	n.logger = logging.AddContext(logger.Log, log.Ctx{"driver": netType, "network": name})
	n.id = id
	n.name = name
	n.netType = netType
	n.config = config
	n.state = state
	n.description = description
	n.status = status
}

// fillConfig fills requested config with any default values, by default this is a no-op.
func (n *common) fillConfig(req *api.NetworksPost) error {
	return nil
}

// validationRules returns a map of config rules common to all drivers.
func (n *common) validationRules() map[string]func(string) error {
	return map[string]func(string) error{}
}

// validate a network config against common rules and optional driver specific rules.
func (n *common) validate(config map[string]string, driverRules map[string]func(value string) error) error {
	checkedFields := map[string]struct{}{}

	// Get rules common for all drivers.
	rules := n.validationRules()

	// Merge driver specific rules into common rules.
	for field, validator := range driverRules {
		rules[field] = validator
	}

	// Run the validator against each field.
	for k, validator := range rules {
		checkedFields[k] = struct{}{} //Mark field as checked.
		err := validator(config[k])
		if err != nil {
			return errors.Wrapf(err, "Invalid value for network %q option %q", n.name, k)
		}
	}

	// Look for any unchecked fields, as these are unknown fields and validation should fail.
	for k := range config {
		_, checked := checkedFields[k]
		if checked {
			continue
		}

		// User keys are not validated.
		if strings.HasPrefix(k, "user.") {
			continue
		}

		return fmt.Errorf("Invalid option for network %q option %q", n.name, k)
	}

	return nil
}

// Name returns the network name.
func (n *common) Name() string {
	return n.name
}

// Status returns the network status.
func (n *common) Status() string {
	return n.status
}

// Type returns the network type.
func (n *common) Type() string {
	return n.netType
}

// Config returns the network config.
func (n *common) Config() map[string]string {
	return n.config
}

// IsUsed returns whether the network is used by any instances.
func (n *common) IsUsed() bool {
	// Look for instances using the interface
	insts, err := instance.LoadFromAllProjects(n.state)
	if err != nil {
		return true
	}

	for _, inst := range insts {
		if IsInUseByInstance(inst, n.name) {
			return true
		}
	}

	return false
}

// HasDHCPv4 indicates whether the network has DHCPv4 enabled.
func (n *common) HasDHCPv4() bool {
	if n.config["ipv4.dhcp"] == "" || shared.IsTrue(n.config["ipv4.dhcp"]) {
		return true
	}

	return false
}

// HasDHCPv6 indicates whether the network has DHCPv6 enabled (includes stateless SLAAC router advertisement mode).
// Technically speaking stateless SLAAC RA mode isn't DHCPv6, but for consistency with LXD's config paradigm, DHCP
// here means "an ability to automatically allocate IPs and routes", rather than stateful DHCP with leases.
// To check if true stateful DHCPv6 is enabled check the "ipv6.dhcp.stateful" config key.
func (n *common) HasDHCPv6() bool {
	if n.config["ipv6.dhcp"] == "" || shared.IsTrue(n.config["ipv6.dhcp"]) {
		return true
	}

	return false
}

// DHCPv4Ranges returns a parsed set of DHCPv4 ranges for this network.
func (n *common) DHCPv4Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv4.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv4.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To4(),
					End:   endIP.To4(),
				})
			}
		}
	}

	return dhcpRanges
}

// DHCPv6Ranges returns a parsed set of DHCPv6 ranges for this network.
func (n *common) DHCPv6Ranges() []DHCPRange {
	dhcpRanges := make([]DHCPRange, 0)
	if n.config["ipv6.dhcp.ranges"] != "" {
		for _, r := range strings.Split(n.config["ipv6.dhcp.ranges"], ",") {
			parts := strings.SplitN(strings.TrimSpace(r), "-", 2)
			if len(parts) == 2 {
				startIP := net.ParseIP(parts[0])
				endIP := net.ParseIP(parts[1])
				dhcpRanges = append(dhcpRanges, DHCPRange{
					Start: startIP.To16(),
					End:   endIP.To16(),
				})
			}
		}
	}

	return dhcpRanges
}

// update the internal config variables, and if not cluster notification, notifies all nodes and updates database.
func (n *common) update(applyNetwork api.NetworkPut, targetNode string, clusterNotification bool) error {
	// Update internal config before database has been updated (so that if update is a notification we apply
	// the config being supplied and not that in the database).
	n.description = applyNetwork.Description
	n.config = applyNetwork.Config

	// If this update isn't coming via a cluster notification itself, then notify all nodes of change and then
	// update the database.
	if !clusterNotification {
		if targetNode == "" {
			// Notify all other nodes to update the network if no target specified.
			notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), cluster.NotifyAll)
			if err != nil {
				return err
			}

			sendNetwork := applyNetwork
			sendNetwork.Config = make(map[string]string)
			for k, v := range applyNetwork.Config {
				// Don't forward node specific keys (these will be merged in on recipient node).
				if shared.StringInSlice(k, db.NodeSpecificNetworkConfig) {
					continue
				}

				sendNetwork.Config[k] = v
			}

			err = notifier(func(client lxd.InstanceServer) error {
				return client.UpdateNetwork(n.name, sendNetwork, "")
			})
			if err != nil {
				return err
			}
		}

		// Update the database.
		err := n.state.Cluster.UpdateNetwork(n.name, applyNetwork.Description, applyNetwork.Config)
		if err != nil {
			return err
		}
	}

	return nil
}

// configChanged compares supplied new config with existing config. Returns a boolean indicating if differences in
// the config or description were found (and the database record needs updating), and a list of non-user config
// keys that have changed, and a copy of the current internal network config that can be used to revert if needed.
func (n *common) configChanged(newNetwork api.NetworkPut) (bool, []string, api.NetworkPut, error) {
	// Backup the current state.
	oldNetwork := api.NetworkPut{
		Description: n.description,
		Config:      map[string]string{},
	}

	err := shared.DeepCopy(&n.config, &oldNetwork.Config)
	if err != nil {
		return false, nil, oldNetwork, err
	}

	// Diff the configurations.
	changedKeys := []string{}
	dbUpdateNeeded := false

	if newNetwork.Description != n.description {
		dbUpdateNeeded = true
	}

	for k, v := range oldNetwork.Config {
		if v != newNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	for k, v := range newNetwork.Config {
		if v != oldNetwork.Config[k] {
			dbUpdateNeeded = true

			// Add non-user changed key to list of changed keys.
			if !strings.HasPrefix(k, "user.") && !shared.StringInSlice(k, changedKeys) {
				changedKeys = append(changedKeys, k)
			}
		}
	}

	return dbUpdateNeeded, changedKeys, oldNetwork, nil
}

// rename the network directory, update database record and update internal variables.
func (n *common) rename(newName string) error {
	// Clear new directory if exists.
	if shared.PathExists(shared.VarPath("networks", newName)) {
		os.RemoveAll(shared.VarPath("networks", newName))
	}

	// Rename directory to new name.
	if shared.PathExists(shared.VarPath("networks", n.name)) {
		err := os.Rename(shared.VarPath("networks", n.name), shared.VarPath("networks", newName))
		if err != nil {
			return err
		}
	}

	// Rename the database entry.
	err := n.state.Cluster.RenameNetwork(n.name, newName)
	if err != nil {
		return err
	}

	// Reinitialise internal name variable and logger context with new name.
	n.init(n.state, n.id, newName, n.netType, n.description, n.config, n.status)

	return nil
}

// delete the network from the database if clusterNotification is false.
func (n *common) delete(clusterNotification bool) error {
	// Only delete database record if not cluster notification.
	if !clusterNotification {
		// Remove the network from the database.
		err := n.state.Cluster.DeleteNetwork(n.name)
		if err != nil {
			return err
		}
	}

	return nil
}

// HandleHeartbeat is a no-op.
func (n *common) HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error {
	return nil
}
<|endoftext|>