<commit_before>// syncwatcher.go
package main

import (
	"code.google.com/p/go.exp/fsnotify"
	"os"
	"bufio"
	"io/ioutil"
	"net/http"
	"net/url"
	"crypto/tls"
	"encoding/json"
	"log"
	"fmt"
	"time"
	"flag"
	"runtime"
	"path/filepath"
	"strings"
	"sort"
)

type Configuration struct {
	Version			int
	Folders			[]FolderConfiguration
}

type FolderConfiguration struct {
	ID			string
	Path			string
	ReadOnly		bool
	RescanIntervalS		int
}

// HTTP Authentication
var (
	target		string
	authUser	string
	authPass	string
	csrfToken	string
	csrfFile	string
	apiKey		string
)

// HTTP Debounce
var (
	debounceTimeout = 300*time.Millisecond
	dirVsFiles = 10
)

// Main
var (
	stop = make(chan int)
	ignorePaths = []string{".stversions", ".syncthing."}
)

func init() {
	flag.StringVar(&target, "target", "localhost:8080", "Target")
	flag.StringVar(&authUser, "user", "", "Username")
	flag.StringVar(&authPass, "pass", "", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", "", "API key")
	flag.Parse()
	if !strings.Contains(target, "://") { target = "http://" + target }
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatal(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
}

func main() {

	testWebGuiPost()

	folders := getFolders()
	if (len(folders) == 0) {
		log.Fatal("No folders found");
	}
	for i := range folders {
		go watchFolder(folders[i])
	}

	code := <-stop
	println("Exiting")
	os.Exit(code)

}

func getFolders() []FolderConfiguration {
	r, err := http.NewRequest("GET", target+"/rest/config", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatal(err)
	}
	return cfg.Folders
}

func watchFolder(folder FolderConfiguration) {
	path := expandTilde(folder.Path)
	sw, err := NewSyncWatcher(ignorePaths)
	if sw == nil || err != nil {
		log.Println(err)
		return
	}
	defer sw.Close()
	err = sw.Watch(path)
	if err != nil {
		log.Println(err)
		return
	}
	informChangeDebounced := informChangeDebounce(debounceTimeout, folder.ID, path, dirVsFiles, informChange)
	log.Println("Watching " + folder.ID + ": " + path)
	if folder.RescanIntervalS < 1800 {
		log.Printf("The rescan interval of folder %s can be increased to 3600 (an hour) or even 86400 (a day) as changes should be observed immediately while syncthing-inotify is running.", folder.ID)
	}
	for {
		ev := waitForEvent(sw)
		if ev == nil {
			log.Println("Error: fsnotify event is nil")
			continue
		}
		log.Println("Change detected in " + ev.Name)
		informChangeDebounced(ev.Name)
	}
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
	var ok bool
	select {
		case ev, ok = <-sw.Event:
			if !ok {
				log.Println("Error: channel closed")
			}
		case err, eok := <-sw.Error:
			log.Println(err, eok)
	}
	return
}

func testWebGuiPost() {
	r, err := http.NewRequest("POST", target+"/rest/404", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	r.Close = true
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 404 {
		log.Fatalf("Status %d != 404 for POST\n", res.StatusCode)
	}
}

func informChange(folder string, sub string) {
	data := url.Values {}
	data.Set("folder", folder)
	data.Set("sub", sub)
	r, err := http.NewRequest("POST", target+"/rest/scan?"+data.Encode(), nil)
	if err != nil {
		log.Println(err)
		return
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	r.Close = true
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println(err)
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Error: Status %d != 200 for POST.\n" + folder + ": " + sub, res.StatusCode)
		return
	} else {
		log.Println("Syncthing is indexing change in " + folder + ": " + sub)
	}
}

func informChangeDebounce(interval time.Duration, folder string, folderPath string, dirVsFiles int, callback func(folder string, sub string)) func(string) {
	debounce := func(f func(paths []string)) func(string) {
		timer := &time.Timer{}
		subs := make([]string, 0)
		return func(sub string) {
			timer.Stop()
			subs = append(subs, sub)
			timer = time.AfterFunc(interval, func() {
				f(subs)
				subs = make([]string, 0)
			})
		}
	}

	return debounce(func(paths []string) {
		// Do not inform Syncthing immediately but wait for debounce
		// Therefore, we need to keep track of the paths that were changed
		// This function optimises tracking in two ways:
		//	- If there are more than `dirVsFiles` changes in a directory, we inform Syncthing to scan the entire directory
		//	- Directories with parent directory changes are aggregated. If A/B has 3 changes and A/C has 8, A will have 11 changes and if this is bigger than dirVsFiles we will scan A.
		if (len(paths) == 0) { return }
		trackedPaths := make(map[string]int) // Map directories to scores; if score == -1 the path is a filename
		sort.Strings(paths) // Make sure parent paths are processed first
		previousPath := "" // Filter duplicates
		for i := range paths {
			path := paths[i]
			if (path == previousPath) {
				continue
			}
			previousPath = path
			dir := filepath.Dir(path)
			score := 1 // File change counts for 1 per directory
			if dir == filepath.Clean(path) {
				score = dirVsFiles // Is directory itself, should definitely inform
			}
			// Search for existing parent directory relations in the map
			for trackedPath, _ := range trackedPaths {
				if strings.HasPrefix(dir, trackedPath) {
					// Increment score of tracked current/parent directory
					trackedPaths[trackedPath] += score
				}
			}
			_, exists := trackedPaths[dir]
			if !exists {
				trackedPaths[dir] = score
			}
			trackedPaths[path] = -1
		}
		var keys []string
		for k := range trackedPaths {
			keys = append(keys, k)
		}
		sort.Strings(keys) // Sort directories before their own files
		previousDone, previousPath := false, ""
		for i := range keys {
			trackedPath := keys[i]
			trackedPathScore, _ := trackedPaths[trackedPath]
			if previousDone && strings.HasPrefix(trackedPath, previousPath) { continue } // Already informed parent directory change
			if trackedPathScore < dirVsFiles && trackedPathScore != -1 { continue } // Not enough files for this directory or it is a file
			previousDone = trackedPathScore != -1
			previousPath = trackedPath
			sub := strings.TrimPrefix(trackedPath, folderPath)
			sub = strings.TrimPrefix(sub, string(os.PathSeparator))
			callback(folder, sub)
		}
	})
}

func getHomeDir() string {
	var home string
	switch runtime.GOOS {
		case "windows":
		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home == "" {
			home = os.Getenv("UserProfile")
		}
		default:
			home = os.Getenv("HOME")
	}
	if home == "" {
		log.Fatal("No home path found - set $HOME (or the platform equivalent).")
	}
	return home
}

func expandTilde(p string) string {
	if p == "~" {
		return getHomeDir()
	}
	p = filepath.FromSlash(p)
	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
		return p
	}
	return filepath.Join(getHomeDir(), p[2:])
}
<commit_msg>Improves debounce functionality and provides retry operations if POST failed.<commit_after>// syncwatcher.go
package main

import (
	"code.google.com/p/go.exp/fsnotify"
	"os"
	"bufio"
	"io/ioutil"
	"net/http"
	"net/url"
	"crypto/tls"
	"encoding/json"
	"log"
	"fmt"
	"time"
	"flag"
	"runtime"
	"path/filepath"
	"strings"
	"sort"
)

type Configuration struct {
	Version			int
	Folders			[]FolderConfiguration
}

type FolderConfiguration struct {
	ID			string
	Path			string
	ReadOnly		bool
	RescanIntervalS		int
}

// HTTP Authentication
var (
	target		string
	authUser	string
	authPass	string
	csrfToken	string
	csrfFile	string
	apiKey		string
)

// HTTP Debounce
var (
	debounceTimeout = 300*time.Millisecond
	dirVsFiles = 10
	maxFiles = 5000
)

// Main
var (
	stop = make(chan int)
	ignorePaths = []string{".stversions", ".syncthing."}
)

func init() {
	flag.StringVar(&target, "target", "localhost:8080", "Target")
	flag.StringVar(&authUser, "user", "", "Username")
	flag.StringVar(&authPass, "pass", "", "Password")
	flag.StringVar(&csrfFile, "csrf", "", "CSRF token file")
	flag.StringVar(&apiKey, "api", "", "API key")
	flag.Parse()
	if !strings.Contains(target, "://") { target = "http://" + target }
	if len(csrfFile) > 0 {
		fd, err := os.Open(csrfFile)
		if err != nil {
			log.Fatal(err)
		}
		s := bufio.NewScanner(fd)
		for s.Scan() {
			csrfToken = s.Text()
		}
		fd.Close()
	}
}

func main() {

	testWebGuiPost()

	folders := getFolders()
	if (len(folders) == 0) {
		log.Fatal("No folders found");
	}
	for i := range folders {
		go watchFolder(folders[i])
	}

	code := <-stop
	println("Exiting")
	os.Exit(code)

}

func getFolders() []FolderConfiguration {
	r, err := http.NewRequest("GET", target+"/rest/config", nil)
	if err != nil {
		log.Fatal(err)
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("Status %d != 200 for GET", res.StatusCode)
	}
	bs, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var cfg Configuration
	err = json.Unmarshal(bs, &cfg)
	if err != nil {
		log.Fatal(err)
	}
	return cfg.Folders
}

func watchFolder(folder FolderConfiguration) {
	path := expandTilde(folder.Path)
	sw, err := NewSyncWatcher(ignorePaths)
	if sw == nil || err != nil {
		log.Println(err)
		return
	}
	defer sw.Close()
	err = sw.Watch(path)
	if err != nil {
		log.Println(err)
		return
	}
	informChannel := make(chan string, 10)
	go informChangeAccumulator(debounceTimeout, folder.ID, path, dirVsFiles, informChannel, informChange)
	log.Println("Watching " + folder.ID + ": " + path)
	if folder.RescanIntervalS < 1800 {
		log.Printf("The rescan interval of folder %s can be increased to 3600 (an hour) or even 86400 (a day) as changes should be observed immediately while syncthing-inotify is running.", folder.ID)
	}
	for {
		ev := waitForEvent(sw)
		if ev == nil {
			log.Println("Error: fsnotify event is nil")
			continue
		}
		log.Println("Change detected in " + ev.Name)
		informChannel <- ev.Name
	}
}

func waitForEvent(sw *SyncWatcher) (ev *fsnotify.FileEvent) {
	var ok bool
	select {
		case ev, ok = <-sw.Event:
			if !ok {
				log.Println("Error: channel closed")
			}
		case err, eok := <-sw.Error:
			log.Println(err, eok)
	}
	return
}

func testWebGuiPost() (bool) {
	r, err := http.NewRequest("POST", target+"/rest/404", nil)
	if err != nil {
		log.Println("Cannot connect to Syncthing", err)
		return false
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	r.Close = true
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println("Cannot connect to Syncthing", err)
		return false
	}
	defer res.Body.Close()
	if res.StatusCode != 404 {
		log.Printf("Cannot connect to Syncthing, Status %d != 404 for POST\n", res.StatusCode)
		return false
	}
	return true
}

func informChange(folder string, sub string) (bool) {
	data := url.Values {}
	data.Set("folder", folder)
	data.Set("sub", sub)
	r, err := http.NewRequest("POST", target+"/rest/scan?"+data.Encode(), nil)
	if err != nil {
		log.Println(err)
		return false
	}
	if len(csrfToken) > 0 {
		r.Header.Set("X-CSRF-Token", csrfToken)
	}
	if len(authUser) > 0 {
		r.SetBasicAuth(authUser, authPass)
	}
	if len(apiKey) > 0 {
		r.Header.Set("X-API-Key", apiKey)
	}
	r.Close = true
	tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify : true} }
	client := &http.Client{Transport: tr, Timeout: 5*time.Second}
	res, err := client.Do(r)
	if err != nil {
		log.Println("Syncthing returned error", err)
		return false
	}
	if res.StatusCode != 200 {
		log.Printf("Error: Status %d != 200 for POST.\n" + folder + ": " + sub, res.StatusCode)
		return false
	} else {
		log.Println("Syncthing is indexing change in " + folder + ": " + sub)
	}
	return true
}

func informChangeAccumulator(interval time.Duration, folder string, folderPath string, dirVsFiles int, input chan string,
			callback func(folder string, sub string) (bool)) func(string) {
	debounce := func(f func(paths []string) (bool)) func(string) {
		subs := make([]string, 0)
		var item = <-input
		for {
			select {
				case item = <-input:
					if len(subs) < maxFiles {
						subs = append(subs, item)
					}
				case <-time.After(interval):
					ok := false
					if len(subs) < maxFiles {
						ok = f(subs)
					} else {
						ok = f([]string{ folderPath })
					}
					if ok {
						subs = make([]string, 0)
					}
			}
		}
	}

	return debounce(func(paths []string) (bool) {
		// Do not inform Syncthing immediately but wait for debounce
		// Therefore, we need to keep track of the paths that were changed
		// This function optimises tracking in two ways:
		//	- If there are more than `dirVsFiles` changes in a directory, we inform Syncthing to scan the entire directory
		//	- Directories with parent directory changes are aggregated. If A/B has 3 changes and A/C has 8, A will have 11 changes and if this is bigger than dirVsFiles we will scan A.
		if (len(paths) == 0) { return false }
		trackedPaths := make(map[string]int) // Map directories to scores; if score == -1 the path is a filename
		sort.Strings(paths) // Make sure parent paths are processed first
		previousPath := "" // Filter duplicates
		for i := range paths {
			path := paths[i]
			if (path == previousPath) {
				continue
			}
			previousPath = path
			dir := filepath.Dir(path)
			score := 1 // File change counts for 1 per directory
			if dir == filepath.Clean(path) {
				score = dirVsFiles // Is directory itself, should definitely inform
			}
			// Search for existing parent directory relations in the map
			for trackedPath, _ := range trackedPaths {
				if strings.HasPrefix(dir, trackedPath) {
					// Increment score of tracked current/parent directory
					trackedPaths[trackedPath] += score
				}
			}
			_, exists := trackedPaths[dir]
			if !exists {
				trackedPaths[dir] = score
			}
			trackedPaths[path] = -1
		}
		var keys []string
		for k := range trackedPaths {
			keys = append(keys, k)
		}
		sort.Strings(keys) // Sort directories before their own files
		previousDone, previousPath := false, ""
		for i := range keys {
			trackedPath := keys[i]
			trackedPathScore, _ := trackedPaths[trackedPath]
			if previousDone && strings.HasPrefix(trackedPath, previousPath) { continue } // Already informed parent directory change
			if trackedPathScore < dirVsFiles && trackedPathScore != -1 { continue } // Not enough files for this directory or it is a file
			previousDone = trackedPathScore != -1
			previousPath = trackedPath
			sub := strings.TrimPrefix(trackedPath, folderPath)
			sub = strings.TrimPrefix(sub, string(os.PathSeparator))
			if !callback(folder, sub) {
				return false
			}
		}
		return true
	})
}

func getHomeDir() string {
	var home string
	switch runtime.GOOS {
		case "windows":
		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
		if home == "" {
			home = os.Getenv("UserProfile")
		}
		default:
			home = os.Getenv("HOME")
	}
	if home == "" {
		log.Fatal("No home path found - set $HOME (or the platform equivalent).")
	}
	return home
}

func expandTilde(p string) string {
	if p == "~" {
		return getHomeDir()
	}
	p = filepath.FromSlash(p)
	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
		return p
	}
	return filepath.Join(getHomeDir(), p[2:])
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"encoding/json"
	"fmt"
	sl "github.com/mcuadros/go-syslog"
	"github.com/stealthly/go-kafka/producer"
	"net"
	"testing"
	"time"
)

var tcpAddr = "0.0.0.0:5140"
var rfc5424message = "<34>1 2003-10-11T22:14:15.003Z localhost.stealth.ly su - ID23 - a simple message"
var rfc3164message = `<34>Jan 12 06:30:00 1.2.3.4 some_server: 1.2.3.4 - - [12/Jan/2011:06:29:59 +0100] "GET /foo/bar.html HTTP/1.1" 301 96 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12 ( .NET CLR 3.5.30729)" PID 18904 Time Taken 0`

//TODO any parser for this? what kind of RFC log/syslog is using at all?
//<PRI>TIMESTAMP HOSTNAME TAG[PID]: MSG
//<132>2014-12-30T11:51:47+02:00 localhost go_kafka_client[13128]: woohoo!

func TestSyslogRFC5424(t *testing.T) {
	testSyslog(t, sl.RFC5424)
}

func TestSyslogRFC3164(t *testing.T) {
	testSyslog(t, sl.RFC3164)
}

func TestSyslogProducer(t *testing.T) {
	Logger = NewDefaultLogger(TraceLevel)
	topic := fmt.Sprintf("syslog-producer-%d", time.Now().Unix())

	consumeMessages := 100
	timeout := 1 * time.Minute
	consumeStatus := make(chan int)

	CreateMultiplePartitionsTopic(localZk, topic, 1)
	EnsureHasLeader(localZk, topic)

	config := NewSyslogProducerConfig()
	config.ProducerConfig = DefaultProducerConfig()
	config.ProducerConfig.BrokerList = []string{localBroker}
	config.ChannelSize = 5
	config.NumProducers = 1
	config.TCPAddr = tcpAddr
	config.Topic = topic
	syslogProducer := NewSyslogProducer(config)
	go syslogProducer.Start()

	time.Sleep(2 * time.Second)
	for i := 0; i < 100; i++ {
		logMessage(sl.RFC5424)
	}

	consumerConfig := testConsumerConfig()
	consumerConfig.Strategy = newCountingStrategy(t, consumeMessages, timeout, consumeStatus)
	consumer := NewConsumer(consumerConfig)
	go consumer.StartStatic(map[string]int{topic: 1})
	if actual := <-consumeStatus; actual != consumeMessages {
		t.Errorf("Failed to consume %d messages within %s. Actual messages = %d", consumeMessages, timeout, actual)
	}
	closeWithin(t, 10*time.Second, consumer)
	syslogProducer.Stop()
}

func testSyslog(t *testing.T, format sl.Format) {
	topic := fmt.Sprintf("syslog-%d", time.Now().Unix())
	numMessages := 1
	consumeTimeout := 10 * time.Second

	CreateMultiplePartitionsTopic(localZk, topic, 1)
	EnsureHasLeader(localZk, topic)

	server, channel := startServer(format)

	kafkaProducer := producer.NewKafkaProducer(topic, []string{localBroker})
	go func() {
		for part := range channel {
			b, err := json.Marshal(part)
			if err != nil {
				t.Fatal(err)
			}
			fmt.Println("sending", string(b))
			kafkaProducer.SendBytesSync(b)
		}
	}()

	logMessage(format)

	consumeStatus := make(chan int)

	config := testConsumerConfig()
	config.Strategy = newCountingStrategy(t, numMessages, consumeTimeout, consumeStatus)
	consumer := NewConsumer(config)
	go consumer.StartStatic(map[string]int{topic: 1})
	if actual := <-consumeStatus; actual != numMessages {
		t.Errorf("Failed to consume %d messages within %s. Actual messages = %d", numMessages, consumeTimeout, actual)
	}
	kafkaProducer.Close()
	server.Kill()
	close(channel)
	closeWithin(t, 10*time.Second, consumer)
}

func startServer(format sl.Format) (*sl.Server, sl.LogPartsChannel) {
	channel := make(sl.LogPartsChannel)
	handler := sl.NewChannelHandler(channel)

	server := sl.NewServer()
	server.SetFormat(format)
	server.SetHandler(handler)
	server.ListenTCP(tcpAddr)
	server.Boot()

	return server, channel
}

//TODO use log/syslog somehow?
func logMessage(format sl.Format) {
	serverAddr, _ := net.ResolveTCPAddr("tcp", tcpAddr)
	con, err := net.DialTCP("tcp", nil, serverAddr)
	if err != nil {
		panic(err)
	}
	if format == sl.RFC5424 {
		con.Write([]byte(rfc5424message))
	} else {
		con.Write([]byte(rfc3164message))
	}
	con.Close()
}
<commit_msg>fixed test for broker list change<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"encoding/json"
	"fmt"
	sl "github.com/mcuadros/go-syslog"
	"github.com/stealthly/go-kafka/producer"
	"net"
	"testing"
	"time"
)

var tcpAddr = "0.0.0.0:5140"
var rfc5424message = "<34>1 2003-10-11T22:14:15.003Z localhost.stealth.ly su - ID23 - a simple message"
var rfc3164message = `<34>Jan 12 06:30:00 1.2.3.4 some_server: 1.2.3.4 - - [12/Jan/2011:06:29:59 +0100] "GET /foo/bar.html HTTP/1.1" 301 96 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12 ( .NET CLR 3.5.30729)" PID 18904 Time Taken 0`

//TODO any parser for this? what kind of RFC log/syslog is using at all?
//<PRI>TIMESTAMP HOSTNAME TAG[PID]: MSG
//<132>2014-12-30T11:51:47+02:00 localhost go_kafka_client[13128]: woohoo!

func TestSyslogRFC5424(t *testing.T) {
	testSyslog(t, sl.RFC5424)
}

func TestSyslogRFC3164(t *testing.T) {
	testSyslog(t, sl.RFC3164)
}

func TestSyslogProducer(t *testing.T) {
	Logger = NewDefaultLogger(TraceLevel)
	topic := fmt.Sprintf("syslog-producer-%d", time.Now().Unix())

	consumeMessages := 100
	timeout := 1 * time.Minute
	consumeStatus := make(chan int)

	CreateMultiplePartitionsTopic(localZk, topic, 1)
	EnsureHasLeader(localZk, topic)

	config := NewSyslogProducerConfig()
	config.ProducerConfig = DefaultProducerConfig()
	config.BrokerList = localBroker
	config.ChannelSize = 5
	config.NumProducers = 1
	config.TCPAddr = tcpAddr
	config.Topic = topic
	syslogProducer := NewSyslogProducer(config)
	go syslogProducer.Start()

	time.Sleep(2 * time.Second)
	for i := 0; i < 100; i++ {
		logMessage(sl.RFC5424)
	}

	consumerConfig := testConsumerConfig()
	consumerConfig.Strategy = newCountingStrategy(t, consumeMessages, timeout, consumeStatus)
	consumer := NewConsumer(consumerConfig)
	go consumer.StartStatic(map[string]int{topic: 1})
	if actual := <-consumeStatus; actual != consumeMessages {
		t.Errorf("Failed to consume %d messages within %s. Actual messages = %d", consumeMessages, timeout, actual)
	}
	closeWithin(t, 10*time.Second, consumer)
	syslogProducer.Stop()
}

func testSyslog(t *testing.T, format sl.Format) {
	topic := fmt.Sprintf("syslog-%d", time.Now().Unix())
	numMessages := 1
	consumeTimeout := 10 * time.Second

	CreateMultiplePartitionsTopic(localZk, topic, 1)
	EnsureHasLeader(localZk, topic)

	server, channel := startServer(format)

	kafkaProducer := producer.NewKafkaProducer(topic, []string{localBroker})
	go func() {
		for part := range channel {
			b, err := json.Marshal(part)
			if err != nil {
				t.Fatal(err)
			}
			fmt.Println("sending", string(b))
			kafkaProducer.SendBytesSync(b)
		}
	}()

	logMessage(format)

	consumeStatus := make(chan int)

	config := testConsumerConfig()
	config.Strategy = newCountingStrategy(t, numMessages, consumeTimeout, consumeStatus)
	consumer := NewConsumer(config)
	go consumer.StartStatic(map[string]int{topic: 1})
	if actual := <-consumeStatus; actual != numMessages {
		t.Errorf("Failed to consume %d messages within %s. Actual messages = %d", numMessages, consumeTimeout, actual)
	}
	kafkaProducer.Close()
	server.Kill()
	close(channel)
	closeWithin(t, 10*time.Second, consumer)
}

func startServer(format sl.Format) (*sl.Server, sl.LogPartsChannel) {
	channel := make(sl.LogPartsChannel)
	handler := sl.NewChannelHandler(channel)

	server := sl.NewServer()
	server.SetFormat(format)
	server.SetHandler(handler)
	server.ListenTCP(tcpAddr)
	server.Boot()

	return server, channel
}

//TODO use log/syslog somehow?
func logMessage(format sl.Format) {
	serverAddr, _ := net.ResolveTCPAddr("tcp", tcpAddr)
	con, err := net.DialTCP("tcp", nil, serverAddr)
	if err != nil {
		panic(err)
	}
	if format == sl.RFC5424 {
		con.Write([]byte(rfc5424message))
	} else {
		con.Write([]byte(rfc3164message))
	}
	con.Close()
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/logline"
	"github.com/google/mtail/tailer/file"
	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher
	fs    afero.Fs // mockable filesystem interface

	handlesMu sync.RWMutex          // protects `handles'
	handles   map[string]*file.File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	pollTicker *time.Ticker

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// PollInterval sets the time interval between polls of the watched log files.
func PollInterval(interval time.Duration) func(*Tailer) error {
	return func(t *Tailer) error {
		if interval > 0 {
			t.pollTicker = time.NewTicker(interval)
		}
		return nil
	}
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, fs afero.Fs, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if fs == nil {
		return nil, errors.New("can't create tailer without FS")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		fs:           fs,
		handles:      make(map[string]*file.File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *file.File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*file.File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) {
	glog.V(2).Infof("AddPattern: %s", pattern)
	t.globPatternsMu.Lock()
	t.globPatterns[pattern] = struct{}{}
	t.globPatternsMu.Unlock()
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	matches, err := afero.Glob(t.fs, pattern)
	if err != nil {
		return err
	}
	t.AddPattern(pattern)
	glog.V(1).Infof("glob matches: %v", matches)
	// TODO(jaq): Error if there are no matches, or do we just assume that it's OK?
	// mtail_test.go assumes that it's ok.  Figure out why.
	// if len(matches) == 0 {
	// 	return errors.Errorf("No matches for pattern %q", pattern)
	// }
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.  TODO(jaq): this
	// seems fallible.
	return t.watchDirname(pattern)
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	logCount.Add(1)
	// TODO(jaq): ex_test/filename.mtail requires we use the original pathname here, not fullpath
	return t.openLogPath(pathname, false, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	err := fd.Follow()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
// TODO(jaq): seenBefore is incorrect for all log creation events received via fsnotify.
func (t *Tailer) openLogPath(pathname string, seenBefore, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v %v", pathname, seenBefore, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := file.New(t.fs, pathname, t.lines, seenBefore, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("AbsPath %q doesn't exist (yet?)", f.Pathname)
			return nil
		}
		return err
	}
	// The file did not exist when
	if f == nil {
		return nil
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("No match for %q", pathname)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// TODO(jaq): avoid code duplication with TailPath here.
		// If this file was just created, read from the start.
		if err := t.w.Add(pathname, t.eventsHandle); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
			continue
		}
		logCount.Add(1)
		// Pretend seenBefore because we want to seek to start.
		if err := t.openLogPath(pathname, false, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("Started tailing %q", pathname)
	}
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)
	defer close(t.lines)

	var ticks <-chan time.Time
	if t.pollTicker != nil {
		ticks = t.pollTicker.C
		defer t.pollTicker.Stop()
	}

	for {
		select {
		case e, ok := <-events:
			if !ok {
				glog.Infof("Shutting down tailer.")
				return
			}

			glog.V(2).Infof("Event type %#v", e)
			t.handleLogEvent(e.Pathname)

		case <-ticks:
			// Something.
			glog.Info("tick")

		}
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*file.File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v string
		m map[string]string
	}{
		{"log_errors_total", data.Errors},
		{"log_rotations_total", data.Rotations},
		{"log_lines_total", data.Lines},
		{"log_truncates_total", data.Truncs},
	} {
		expvar.Get(pair.v).(*expvar.Map).Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}
<commit_msg>Wire up the poll tick to something that does the polling.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"expvar"
	"html/template"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"

	"github.com/google/mtail/logline"
	"github.com/google/mtail/tailer/file"
	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	// logCount records the number of logs that are being tailed
	logCount = expvar.NewInt("log_count")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	lines chan<- *logline.LogLine // Logfile lines being emitted.
	w     watcher.Watcher
	fs    afero.Fs // mockable filesystem interface

	handlesMu sync.RWMutex          // protects `handles'
	handles   map[string]*file.File // File handles for each pathname.

	globPatternsMu sync.RWMutex        // protects `globPatterns'
	globPatterns   map[string]struct{} // glob patterns to match newly created files in dir paths against

	runDone chan struct{} // Signals termination of the run goroutine.

	eventsHandle int // record the handle with which to add new log files to the watcher

	pollTicker *time.Ticker

	oneShot bool
}

// OneShot puts the tailer in one-shot mode.
func OneShot(t *Tailer) error {
	t.oneShot = true
	return nil
}

// PollInterval sets the time interval between polls of the watched log files.
func PollInterval(interval time.Duration) func(*Tailer) error {
	return func(t *Tailer) error {
		if interval > 0 {
			t.pollTicker = time.NewTicker(interval)
		}
		return nil
	}
}

// New creates a new Tailer.
func New(lines chan<- *logline.LogLine, fs afero.Fs, w watcher.Watcher, options ...func(*Tailer) error) (*Tailer, error) {
	if lines == nil {
		return nil, errors.New("can't create tailer without lines channel")
	}
	if fs == nil {
		return nil, errors.New("can't create tailer without FS")
	}
	if w == nil {
		return nil, errors.New("can't create tailer without W")
	}
	t := &Tailer{
		lines:        lines,
		w:            w,
		fs:           fs,
		handles:      make(map[string]*file.File),
		globPatterns: make(map[string]struct{}),
		runDone:      make(chan struct{}),
	}
	if err := t.SetOption(options...); err != nil {
		return nil, err
	}
	handle, eventsChan := t.w.Events()
	t.eventsHandle = handle
	go t.run(eventsChan)
	return t, nil
}

// SetOption takes one or more option functions and applies them in order to Tailer.
func (t *Tailer) SetOption(options ...func(*Tailer) error) error {
	for _, option := range options {
		if err := option(t); err != nil {
			return err
		}
	}
	return nil
}

// setHandle sets a file handle under it's pathname
func (t *Tailer) setHandle(pathname string, f *file.File) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return errors.Wrapf(err, "Failed to lookup abspath of %q", pathname)
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	t.handles[absPath] = f
	return nil
}

// handleForPath retrives a file handle for a pathname.
func (t *Tailer) handleForPath(pathname string) (*file.File, bool) {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", pathname, err)
		return nil, false
	}
	t.handlesMu.Lock()
	defer t.handlesMu.Unlock()
	fd, ok := t.handles[absPath]
	return fd, ok
}

func (t *Tailer) hasHandle(pathname string) bool {
	_, ok := t.handleForPath(pathname)
	return ok
}

// AddPattern adds a pattern to the list of patterns to filter filenames against.
func (t *Tailer) AddPattern(pattern string) {
	glog.V(2).Infof("AddPattern: %s", pattern)
	t.globPatternsMu.Lock()
	t.globPatterns[pattern] = struct{}{}
	t.globPatternsMu.Unlock()
}

// TailPattern registers a pattern to be tailed.  If pattern is a plain
// file then it is watched for updates and opened.  If pattern is a glob, then
// all paths that match the glob are opened and watched, and the directories
// containing those matches, if any, are watched.
func (t *Tailer) TailPattern(pattern string) error {
	matches, err := afero.Glob(t.fs, pattern)
	if err != nil {
		return err
	}
	t.AddPattern(pattern)
	glog.V(1).Infof("glob matches: %v", matches)
	// TODO(jaq): Error if there are no matches, or do we just assume that it's OK?
	// mtail_test.go assumes that it's ok.  Figure out why.
	// if len(matches) == 0 {
	// 	return errors.Errorf("No matches for pattern %q", pattern)
	// }
	for _, pathname := range matches {
		err := t.TailPath(pathname)
		if err != nil {
			return errors.Wrapf(err, "attempting to tail %q", pathname)
		}
	}
	// Add a watch on the containing directory, so we know when a rotation
	// occurs or something shows up that matches this pattern.  TODO(jaq): this
	// seems fallible.
	return t.watchDirname(pattern)
}

// TailPath registers a filesystem pathname to be tailed.
func (t *Tailer) TailPath(pathname string) error {
	if t.hasHandle(pathname) {
		glog.V(2).Infof("already watching %q", pathname)
		return nil
	}
	if err := t.w.Add(pathname, t.eventsHandle); err != nil {
		return err
	}
	logCount.Add(1)
	// TODO(jaq): ex_test/filename.mtail requires we use the original pathname here, not fullpath
	return t.openLogPath(pathname, false, false)
}

// handleLogEvent is dispatched when an Event is received, causing the tailer
// to read all available bytes from an already-opened file and send each log
// line onto lines channel.  Because we handle rotations and truncates when
// reaching EOF in the file reader itself, we don't care what the signal is
// from the filewatcher.
func (t *Tailer) handleLogEvent(pathname string) {
	glog.V(2).Infof("handleLogUpdate %s", pathname)
	fd, ok := t.handleForPath(pathname)
	if !ok {
		glog.V(1).Infof("No file handle found for %q, but is being watched", pathname)
		// We want to open files we have watches on in case the file was
		// unreadable before now; but we have to copmare against the glob to be
		// sure we don't just add all the files in a watched directory as they
		// get modified.
		t.handleCreateGlob(pathname)
		return
	}
	doFollow(fd)
}

// doFollow performs the Follow on an existing file descriptor, logging any errors
func doFollow(fd *file.File) {
	err := fd.Follow()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// pollHandles walks the handles map and polls them all in series.
func (t *Tailer) pollHandles() {
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	for _, fd := range t.handles {
		doFollow(fd)
	}
}

// watchDirname adds the directory containing a path to be watched.
func (t *Tailer) watchDirname(pathname string) error {
	absPath, err := filepath.Abs(pathname)
	if err != nil {
		return err
	}
	d := filepath.Dir(absPath)
	return t.w.Add(d, t.eventsHandle)
}

// openLogPath opens a log file named by pathname.
// TODO(jaq): seenBefore is incorrect for all log creation events received via fsnotify.
func (t *Tailer) openLogPath(pathname string, seenBefore, seekToStart bool) error {
	glog.V(2).Infof("openlogPath %s %v %v", pathname, seenBefore, seekToStart)
	if err := t.watchDirname(pathname); err != nil {
		return err
	}
	f, err := file.New(t.fs, pathname, t.lines, seenBefore, seekToStart || t.oneShot)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			glog.V(1).Infof("AbsPath %q doesn't exist (yet?)", f.Pathname)
			return nil
		}
		return err
	}
	// The file did not exist when
	if f == nil {
		return nil
	}
	glog.V(2).Infof("Adding a file watch on %q", f.Pathname)
	if err := t.w.Add(f.Pathname, t.eventsHandle); err != nil {
		return err
	}
	if err := t.setHandle(pathname, f); err != nil {
		return err
	}
	if err := f.Read(); err != nil && err != io.EOF {
		return err
	}
	glog.Infof("Tailing %s", f.Pathname)
	return nil
}

// handleCreateGlob matches the pathname against the glob patterns and starts tailing the file.
func (t *Tailer) handleCreateGlob(pathname string) {
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()

	for pattern := range t.globPatterns {
		matched, err := filepath.Match(pattern, pathname)
		if err != nil {
			glog.Warningf("Unexpected bad pattern %q not detected earlier", pattern)
			continue
		}
		if !matched {
			glog.V(2).Infof("No match for %q", pathname)
			continue
		}
		glog.V(1).Infof("New file %q matched existing glob %q", pathname, pattern)
		// TODO(jaq): avoid code duplication with TailPath here.
		// If this file was just created, read from the start.
		if err := t.w.Add(pathname, t.eventsHandle); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
			continue
		}
		logCount.Add(1)
		// Pretend seenBefore because we want to seek to start.
		if err := t.openLogPath(pathname, false, true); err != nil {
			glog.Infof("Failed to tail new file %q: %s", pathname, err)
		}
		glog.V(2).Infof("Started tailing %q", pathname)
	}
}

// run the main event loop for the Tailer.  It receives notification of
// log file changes from the watcher channel, and dispatches the log event
// handler.
func (t *Tailer) run(events <-chan watcher.Event) {
	defer close(t.runDone)
	defer close(t.lines)

	var ticks <-chan time.Time
	if t.pollTicker != nil {
		ticks = t.pollTicker.C
		defer t.pollTicker.Stop()
	}

	for {
		select {
		case e, ok := <-events:
			if !ok {
				glog.Infof("Shutting down tailer.")
				return
			}

			glog.V(2).Infof("Event type %#v", e)
			t.handleLogEvent(e.Pathname)

		case <-ticks:
			// Something.
			glog.Info("tick")

			t.pollHandles()

		}
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() error {
	if err := t.w.Close(); err != nil {
		return err
	}
	<-t.runDone
	return nil
}

const tailerTemplate = `

Log Tailer

Patterns

{{range $name, $val := $.Patterns}}

{{$name}}

{{end}}

Log files watched

pathname
errors
rotations
truncations
lines read

{{range $name, $val := $.Handles}}

{{$name}}

{{index $.Errors $name}}
{{index $.Rotations $name}}
{{index $.Truncs $name}}
{{index $.Lines $name}}

{{end}}

`

// WriteStatusHTML emits the Tailer's state in HTML format to the io.Writer w.
func (t *Tailer) WriteStatusHTML(w io.Writer) error {
	tpl, err := template.New("tailer").Parse(tailerTemplate)
	if err != nil {
		return err
	}
	t.handlesMu.RLock()
	defer t.handlesMu.RUnlock()
	t.globPatternsMu.RLock()
	defer t.globPatternsMu.RUnlock()
	data := struct {
		Handles   map[string]*file.File
		Patterns  map[string]struct{}
		Rotations map[string]string
		Lines     map[string]string
		Errors    map[string]string
		Truncs    map[string]string
	}{
		t.handles,
		t.globPatterns,
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
		make(map[string]string),
	}
	for _, pair := range []struct {
		v string
		m map[string]string
	}{
		{"log_errors_total", data.Errors},
		{"log_rotations_total", data.Rotations},
		{"log_lines_total", data.Lines},
		{"log_truncates_total", data.Truncs},
	} {
		expvar.Get(pair.v).(*expvar.Map).Do(func(kv expvar.KeyValue) {
			pair.m[kv.Key] = kv.Value.String()
		})
	}
	return tpl.Execute(w, data)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"

	"net/http"
	"os"
	"time"

	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/logic"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics"
	metrics_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	"k8s.io/client-go/informers"
	kube_client "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	kube_flag "k8s.io/component-base/cli/flag"
	"k8s.io/klog"
)

const (
	defaultResyncPeriod time.Duration = 10 * time.Minute
)

var (
	certsConfiguration = &certsConfig{
		clientCaFile:  flag.String("client-ca-file", "/etc/tls-certs/caCert.pem", "Path to CA PEM file."),
		tlsCertFile:   flag.String("tls-cert-file", "/etc/tls-certs/serverCert.pem", "Path to server certificate PEM file."),
		tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."),
	}

	port           = flag.Int("port", 8000, "The port to listen on.")
	address        = flag.String("address", ":8944", "The address to expose Prometheus metrics.")
	namespace      = os.Getenv("NAMESPACE")
	webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.")
	webhookPort    = flag.String("webhook-port", "", "Server Port for Webhook")
	registerByURL  = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name")
)

func main() {
	klog.InitFlags(nil)
	kube_flag.InitFlags()
	klog.V(1).Infof("Vertical Pod Autoscaler %s Admission Controller", common.VerticalPodAutoscalerVersion)

	healthCheck := metrics.NewHealthCheck(time.Minute, false)
	metrics.Initialize(*address, healthCheck)
	metrics_admission.Register()

	certs := initCerts(*certsConfiguration)

	config, err := rest.InClusterConfig()
	if err != nil {
		klog.Fatal(err)
	}

	vpaClient := vpa_clientset.NewForConfigOrDie(config)
	vpaLister := vpa_api_util.NewAllVpasLister(vpaClient, make(chan struct{}))
	kubeClient := kube_client.NewForConfigOrDie(config)
	factory := informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod)
	targetSelectorFetcher := target.NewVpaTargetSelectorFetcher(config, kubeClient, factory)
	podPreprocessor := logic.NewDefaultPodPreProcessor()
	vpaPreprocessor := logic.NewDefaultVpaPreProcessor()
	var limitRangeCalculator limitrange.LimitRangeCalculator
	limitRangeCalculator, err = limitrange.NewLimitsRangeCalculator(factory)
	if err != nil {
		klog.Errorf("Failed to create limitRangeCalculator, falling back to not checking limits. Error message: %s", err)
		limitRangeCalculator = limitrange.NewNoopLimitsCalculator()
	}
	recommendationProvider := logic.NewRecommendationProvider(limitRangeCalculator, vpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator), targetSelectorFetcher, vpaLister)

	as := logic.NewAdmissionServer(recommendationProvider, podPreprocessor, vpaPreprocessor, limitRangeCalculator)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		as.Serve(w, r)
		healthCheck.UpdateLastActivity()
	})
	clientset := getClient()
	server := &http.Server{
		Addr:      fmt.Sprintf(":%d", *port),
		TLSConfig: configTLS(clientset, certs.serverCert, certs.serverKey),
	}
	url := fmt.Sprintf("%v:%v", webhookAddress, webhookPort)
	go selfRegistration(clientset, certs.caCert, &namespace, url, *registerByURL)
	server.ListenAndServeTLS("", "")
}
<commit_msg>fix the webhook-address and webhook-port reference for vpa-admission-controller<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"

	"net/http"
	"os"
	"time"

	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/logic"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics"
	metrics_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	"k8s.io/client-go/informers"
	kube_client "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	kube_flag "k8s.io/component-base/cli/flag"
	"k8s.io/klog"
)

const (
	defaultResyncPeriod time.Duration = 10 * time.Minute
)

var (
	certsConfiguration = &certsConfig{
		clientCaFile:  flag.String("client-ca-file", "/etc/tls-certs/caCert.pem", "Path to CA PEM file."),
		tlsCertFile:   flag.String("tls-cert-file", "/etc/tls-certs/serverCert.pem", "Path to server certificate PEM file."),
		tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."),
	}

	port           = flag.Int("port", 8000, "The port to listen on.")
	address        = flag.String("address", ":8944", "The address to expose Prometheus metrics.")
	namespace      = os.Getenv("NAMESPACE")
	webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.")
	webhookPort    = flag.String("webhook-port", "", "Server Port for Webhook")
	registerByURL  = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name")
)

func main() {
	klog.InitFlags(nil)
	kube_flag.InitFlags()
	klog.V(1).Infof("Vertical Pod Autoscaler %s Admission Controller", common.VerticalPodAutoscalerVersion)

	healthCheck := metrics.NewHealthCheck(time.Minute, false)
	metrics.Initialize(*address, healthCheck)
	metrics_admission.Register()

	certs := initCerts(*certsConfiguration)

	config, err := rest.InClusterConfig()
	if err != nil {
		klog.Fatal(err)
	}

	vpaClient := vpa_clientset.NewForConfigOrDie(config)
	vpaLister := vpa_api_util.NewAllVpasLister(vpaClient, make(chan struct{}))
	kubeClient := kube_client.NewForConfigOrDie(config)
	factory := informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod)
	targetSelectorFetcher := target.NewVpaTargetSelectorFetcher(config, kubeClient, factory)
	podPreprocessor := logic.NewDefaultPodPreProcessor()
	vpaPreprocessor := logic.NewDefaultVpaPreProcessor()
	var limitRangeCalculator limitrange.LimitRangeCalculator
	limitRangeCalculator, err = limitrange.NewLimitsRangeCalculator(factory)
	if err != nil {
		klog.Errorf("Failed to create limitRangeCalculator, falling back to not checking limits. Error message: %s", err)
		limitRangeCalculator = limitrange.NewNoopLimitsCalculator()
	}
	recommendationProvider := logic.NewRecommendationProvider(limitRangeCalculator, vpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator), targetSelectorFetcher, vpaLister)

	as := logic.NewAdmissionServer(recommendationProvider, podPreprocessor, vpaPreprocessor, limitRangeCalculator)
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		as.Serve(w, r)
		healthCheck.UpdateLastActivity()
	})
	clientset := getClient()
	server := &http.Server{
		Addr:      fmt.Sprintf(":%d", *port),
		TLSConfig: configTLS(clientset, certs.serverCert, certs.serverKey),
	}
	url := fmt.Sprintf("%v:%v", *webhookAddress, *webhookPort)
	go selfRegistration(clientset, certs.caCert, &namespace, url, *registerByURL)
	server.ListenAndServeTLS("", "")
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"errors"
	"syscall"
	"unsafe"
)

// Creates a new *syscall.CertContext representing the leaf certificate in an in-memory
// certificate store containing itself and all of the intermediate certificates specified
// in the opts.Intermediates CertPool.
//
// A pointer to the in-memory store is available in the returned CertContext's Store field.
// The store is automatically freed when the CertContext is freed using
// syscall.CertFreeCertificateContext.
func createStoreContext(leaf *Certificate, opts *VerifyOptions) (*syscall.CertContext, error) {
	var storeCtx *syscall.CertContext

	leafCtx, err := syscall.CertCreateCertificateContext(syscall.X509_ASN_ENCODING|syscall.PKCS_7_ASN_ENCODING, &leaf.Raw[0], uint32(len(leaf.Raw)))
	if err != nil {
		return nil, err
	}
	defer syscall.CertFreeCertificateContext(leafCtx)

	handle, err := syscall.CertOpenStore(syscall.CERT_STORE_PROV_MEMORY, 0, 0, syscall.CERT_STORE_DEFER_CLOSE_UNTIL_LAST_FREE_FLAG, 0)
	if err != nil {
		return nil, err
	}
	defer syscall.CertCloseStore(handle, 0)

	err = syscall.CertAddCertificateContextToStore(handle, leafCtx, syscall.CERT_STORE_ADD_ALWAYS, &storeCtx)
	if err != nil {
		return nil, err
	}

	if opts.Intermediates != nil {
		for _, intermediate := range opts.Intermediates.certs {
			ctx, err := syscall.CertCreateCertificateContext(syscall.X509_ASN_ENCODING|syscall.PKCS_7_ASN_ENCODING, &intermediate.Raw[0], uint32(len(intermediate.Raw)))
			if err != nil {
				return nil, err
			}

			err = syscall.CertAddCertificateContextToStore(handle, ctx, syscall.CERT_STORE_ADD_ALWAYS, nil)
			if err != nil {
				return nil, err
			}

			err = syscall.CertFreeCertificateContext(ctx)
			if err != nil {
				return nil, err
			}
		}
	}

	return storeCtx, nil
}

// extractSimpleChain extracts the final certificate chain from a CertSimpleChain.
func extractSimpleChain(simpleChain **syscall.CertSimpleChain, count int) (chain []*Certificate, err error) {
	if simpleChain == nil || count == 0 {
		return nil, errors.New("x509: invalid simple chain")
	}

	simpleChains := (*[1 << 20]*syscall.CertSimpleChain)(unsafe.Pointer(simpleChain))[:]
	lastChain := simpleChains[count-1]
	elements := (*[1 << 20]*syscall.CertChainElement)(unsafe.Pointer(lastChain.Elements))[:]
	for i := 0; i < int(lastChain.NumElements); i++ {
		// Copy the buf, since ParseCertificate does not create its own copy.
		cert := elements[i].CertContext
		encodedCert := (*[1 << 20]byte)(unsafe.Pointer(cert.EncodedCert))[:]
		buf := make([]byte, cert.Length)
		copy(buf, encodedCert[:])
		parsedCert, err := ParseCertificate(buf)
		if err != nil {
			return nil, err
		}
		chain = append(chain, parsedCert)
	}

	return chain, nil
}

// checkChainTrustStatus checks the trust status of the certificate chain, translating
// any errors it finds into Go errors in the process.
func checkChainTrustStatus(c *Certificate, chainCtx *syscall.CertChainContext) error {
	if chainCtx.TrustStatus.ErrorStatus != syscall.CERT_TRUST_NO_ERROR {
		status := chainCtx.TrustStatus.ErrorStatus
		switch status {
		case syscall.CERT_TRUST_IS_NOT_TIME_VALID:
			return CertificateInvalidError{c, Expired}
		default:
			return UnknownAuthorityError{c}
		}
	}
	return nil
}

// checkChainSSLServerPolicy checks that the certificate chain in chainCtx is valid for
// use as a certificate chain for a SSL/TLS server.
func checkChainSSLServerPolicy(c *Certificate, chainCtx *syscall.CertChainContext, opts *VerifyOptions) error {
	sslPara := &syscall.SSLExtraCertChainPolicyPara{
		AuthType:   syscall.AUTHTYPE_SERVER,
		ServerName: syscall.StringToUTF16Ptr(opts.DNSName),
	}
	sslPara.Size = uint32(unsafe.Sizeof(*sslPara))

	para := &syscall.CertChainPolicyPara{
		ExtraPolicyPara: uintptr(unsafe.Pointer(sslPara)),
	}
	para.Size = uint32(unsafe.Sizeof(*para))

	status := syscall.CertChainPolicyStatus{}
	err := syscall.CertVerifyCertificateChainPolicy(syscall.CERT_CHAIN_POLICY_SSL, chainCtx, para, &status)
	if err != nil {
		return err
	}

	// TODO(mkrautz): use the lChainIndex and lElementIndex fields
	// of the CertChainPolicyStatus to provide proper context, instead
	// using c.
	if status.Error != 0 {
		switch status.Error {
		case syscall.CERT_E_EXPIRED:
			return CertificateInvalidError{c, Expired}
		case syscall.CERT_E_CN_NO_MATCH:
			return HostnameError{c, opts.DNSName}
		case syscall.CERT_E_UNTRUSTEDROOT:
			return UnknownAuthorityError{c}
		default:
			return UnknownAuthorityError{c}
		}
	}

	return nil
}

// systemVerify is like Verify, except that it uses CryptoAPI calls
// to build certificate chains and verify them.
func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
	hasDNSName := opts != nil && len(opts.DNSName) > 0

	storeCtx, err := createStoreContext(c, opts)
	if err != nil {
		return nil, err
	}
	defer syscall.CertFreeCertificateContext(storeCtx)

	para := new(syscall.CertChainPara)
	para.Size = uint32(unsafe.Sizeof(*para))

	// If there's a DNSName set in opts, assume we're verifying
	// a certificate from a TLS server.
	if hasDNSName {
		oids := []*byte{
			&syscall.OID_PKIX_KP_SERVER_AUTH[0],
			// Both IE and Chrome allow certificates with
			// Server Gated Crypto as well. Some certificates
			// in the wild require them.
			&syscall.OID_SERVER_GATED_CRYPTO[0],
			&syscall.OID_SGC_NETSCAPE[0],
		}
		para.RequestedUsage.Type = syscall.USAGE_MATCH_TYPE_OR
		para.RequestedUsage.Usage.Length = uint32(len(oids))
		para.RequestedUsage.Usage.UsageIdentifiers = &oids[0]
	} else {
		para.RequestedUsage.Type = syscall.USAGE_MATCH_TYPE_AND
		para.RequestedUsage.Usage.Length = 0
		para.RequestedUsage.Usage.UsageIdentifiers = nil
	}

	var verifyTime *syscall.Filetime
	if opts != nil && !opts.CurrentTime.IsZero() {
		ft := syscall.NsecToFiletime(opts.CurrentTime.UnixNano())
		verifyTime = &ft
	}

	// CertGetCertificateChain will traverse Windows's root stores
	// in an attempt to build a verified certificate chain.  Once
	// it has found a verified chain, it stops. MSDN docs on
	// CERT_CHAIN_CONTEXT:
	//
	//   When a CERT_CHAIN_CONTEXT is built, the first simple chain
	//   begins with an end certificate and ends with a self-signed
	//   certificate. If that self-signed certificate is not a root
	//   or otherwise trusted certificate, an attempt is made to
	//   build a new chain. CTLs are used to create the new chain
	//   beginning with the self-signed certificate from the original
	//   chain as the end certificate of the new chain. This process
	//   continues building additional simple chains until the first
	//   self-signed certificate is a trusted certificate or until
	//   an additional simple chain cannot be built.
	//
	// The result is that we'll only get a single trusted chain to
	// return to our caller.
	var chainCtx *syscall.CertChainContext
	err = syscall.CertGetCertificateChain(syscall.Handle(0), storeCtx, verifyTime, storeCtx.Store, para, 0, 0, &chainCtx)
	if err != nil {
		return nil, err
	}
	defer syscall.CertFreeCertificateChain(chainCtx)

	err = checkChainTrustStatus(c, chainCtx)
	if err != nil {
		return nil, err
	}

	if hasDNSName {
		err = checkChainSSLServerPolicy(c, chainCtx, opts)
		if err != nil {
			return nil, err
		}
	}

	chain, err := extractSimpleChain(chainCtx.Chains, int(chainCtx.ChainCount))
	if err != nil {
		return nil, err
	}

	chains = append(chains, chain)

	return chains, nil
}

func initSystemRoots() {
	systemRoots = NewCertPool()
}
<commit_msg>crypto/x509: do not forget to free cert context<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"errors"
	"syscall"
	"unsafe"
)

// Creates a new *syscall.CertContext representing the leaf certificate in an in-memory
// certificate store containing itself and all of the intermediate certificates specified
// in the opts.Intermediates CertPool.
//
// A pointer to the in-memory store is available in the returned CertContext's Store field.
// The store is automatically freed when the CertContext is freed using
// syscall.CertFreeCertificateContext.
func createStoreContext(leaf *Certificate, opts *VerifyOptions) (*syscall.CertContext, error) {
	var storeCtx *syscall.CertContext

	leafCtx, err := syscall.CertCreateCertificateContext(syscall.X509_ASN_ENCODING|syscall.PKCS_7_ASN_ENCODING, &leaf.Raw[0], uint32(len(leaf.Raw)))
	if err != nil {
		return nil, err
	}
	defer syscall.CertFreeCertificateContext(leafCtx)

	handle, err := syscall.CertOpenStore(syscall.CERT_STORE_PROV_MEMORY, 0, 0, syscall.CERT_STORE_DEFER_CLOSE_UNTIL_LAST_FREE_FLAG, 0)
	if err != nil {
		return nil, err
	}
	defer syscall.CertCloseStore(handle, 0)

	err = syscall.CertAddCertificateContextToStore(handle, leafCtx, syscall.CERT_STORE_ADD_ALWAYS, &storeCtx)
	if err != nil {
		return nil, err
	}

	if opts.Intermediates != nil {
		for _, intermediate := range opts.Intermediates.certs {
			ctx, err := syscall.CertCreateCertificateContext(syscall.X509_ASN_ENCODING|syscall.PKCS_7_ASN_ENCODING, &intermediate.Raw[0], uint32(len(intermediate.Raw)))
			if err != nil {
				return nil, err
			}

			err = syscall.CertAddCertificateContextToStore(handle, ctx, syscall.CERT_STORE_ADD_ALWAYS, nil)
			syscall.CertFreeCertificateContext(ctx)
			if err != nil {
				return nil, err
			}
		}
	}

	return storeCtx, nil
}

// extractSimpleChain extracts the final certificate chain from a CertSimpleChain.
func extractSimpleChain(simpleChain **syscall.CertSimpleChain, count int) (chain []*Certificate, err error) {
	if simpleChain == nil || count == 0 {
		return nil, errors.New("x509: invalid simple chain")
	}

	simpleChains := (*[1 << 20]*syscall.CertSimpleChain)(unsafe.Pointer(simpleChain))[:]
	lastChain := simpleChains[count-1]
	elements := (*[1 << 20]*syscall.CertChainElement)(unsafe.Pointer(lastChain.Elements))[:]
	for i := 0; i < int(lastChain.NumElements); i++ {
		// Copy the buf, since ParseCertificate does not create its own copy.
		cert := elements[i].CertContext
		encodedCert := (*[1 << 20]byte)(unsafe.Pointer(cert.EncodedCert))[:]
		buf := make([]byte, cert.Length)
		copy(buf, encodedCert[:])
		parsedCert, err := ParseCertificate(buf)
		if err != nil {
			return nil, err
		}
		chain = append(chain, parsedCert)
	}

	return chain, nil
}

// checkChainTrustStatus checks the trust status of the certificate chain, translating
// any errors it finds into Go errors in the process.
func checkChainTrustStatus(c *Certificate, chainCtx *syscall.CertChainContext) error {
	if chainCtx.TrustStatus.ErrorStatus != syscall.CERT_TRUST_NO_ERROR {
		status := chainCtx.TrustStatus.ErrorStatus
		switch status {
		case syscall.CERT_TRUST_IS_NOT_TIME_VALID:
			return CertificateInvalidError{c, Expired}
		default:
			return UnknownAuthorityError{c}
		}
	}
	return nil
}

// checkChainSSLServerPolicy checks that the certificate chain in chainCtx is valid for
// use as a certificate chain for a SSL/TLS server.
func checkChainSSLServerPolicy(c *Certificate, chainCtx *syscall.CertChainContext, opts *VerifyOptions) error {
	sslPara := &syscall.SSLExtraCertChainPolicyPara{
		AuthType:   syscall.AUTHTYPE_SERVER,
		ServerName: syscall.StringToUTF16Ptr(opts.DNSName),
	}
	sslPara.Size = uint32(unsafe.Sizeof(*sslPara))

	para := &syscall.CertChainPolicyPara{
		ExtraPolicyPara: uintptr(unsafe.Pointer(sslPara)),
	}
	para.Size = uint32(unsafe.Sizeof(*para))

	status := syscall.CertChainPolicyStatus{}
	err := syscall.CertVerifyCertificateChainPolicy(syscall.CERT_CHAIN_POLICY_SSL, chainCtx, para, &status)
	if err != nil {
		return err
	}

	// TODO(mkrautz): use the lChainIndex and lElementIndex fields
	// of the CertChainPolicyStatus to provide proper context, instead
	// using c.
	if status.Error != 0 {
		switch status.Error {
		case syscall.CERT_E_EXPIRED:
			return CertificateInvalidError{c, Expired}
		case syscall.CERT_E_CN_NO_MATCH:
			return HostnameError{c, opts.DNSName}
		case syscall.CERT_E_UNTRUSTEDROOT:
			return UnknownAuthorityError{c}
		default:
			return UnknownAuthorityError{c}
		}
	}

	return nil
}

// systemVerify is like Verify, except that it uses CryptoAPI calls
// to build certificate chains and verify them.
func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {
	hasDNSName := opts != nil && len(opts.DNSName) > 0

	storeCtx, err := createStoreContext(c, opts)
	if err != nil {
		return nil, err
	}
	defer syscall.CertFreeCertificateContext(storeCtx)

	para := new(syscall.CertChainPara)
	para.Size = uint32(unsafe.Sizeof(*para))

	// If there's a DNSName set in opts, assume we're verifying
	// a certificate from a TLS server.
	if hasDNSName {
		oids := []*byte{
			&syscall.OID_PKIX_KP_SERVER_AUTH[0],
			// Both IE and Chrome allow certificates with
			// Server Gated Crypto as well. Some certificates
			// in the wild require them.
			&syscall.OID_SERVER_GATED_CRYPTO[0],
			&syscall.OID_SGC_NETSCAPE[0],
		}
		para.RequestedUsage.Type = syscall.USAGE_MATCH_TYPE_OR
		para.RequestedUsage.Usage.Length = uint32(len(oids))
		para.RequestedUsage.Usage.UsageIdentifiers = &oids[0]
	} else {
		para.RequestedUsage.Type = syscall.USAGE_MATCH_TYPE_AND
		para.RequestedUsage.Usage.Length = 0
		para.RequestedUsage.Usage.UsageIdentifiers = nil
	}

	var verifyTime *syscall.Filetime
	if opts != nil && !opts.CurrentTime.IsZero() {
		ft := syscall.NsecToFiletime(opts.CurrentTime.UnixNano())
		verifyTime = &ft
	}

	// CertGetCertificateChain will traverse Windows's root stores
	// in an attempt to build a verified certificate chain.  Once
	// it has found a verified chain, it stops. MSDN docs on
	// CERT_CHAIN_CONTEXT:
	//
	//   When a CERT_CHAIN_CONTEXT is built, the first simple chain
	//   begins with an end certificate and ends with a self-signed
	//   certificate. If that self-signed certificate is not a root
	//   or otherwise trusted certificate, an attempt is made to
	//   build a new chain. CTLs are used to create the new chain
	//   beginning with the self-signed certificate from the original
	//   chain as the end certificate of the new chain. This process
	//   continues building additional simple chains until the first
	//   self-signed certificate is a trusted certificate or until
	//   an additional simple chain cannot be built.
	//
	// The result is that we'll only get a single trusted chain to
	// return to our caller.
	var chainCtx *syscall.CertChainContext
	err = syscall.CertGetCertificateChain(syscall.Handle(0), storeCtx, verifyTime, storeCtx.Store, para, 0, 0, &chainCtx)
	if err != nil {
		return nil, err
	}
	defer syscall.CertFreeCertificateChain(chainCtx)

	err = checkChainTrustStatus(c, chainCtx)
	if err != nil {
		return nil, err
	}

	if hasDNSName {
		err = checkChainSSLServerPolicy(c, chainCtx, opts)
		if err != nil {
			return nil, err
		}
	}

	chain, err := extractSimpleChain(chainCtx.Chains, int(chainCtx.ChainCount))
	if err != nil {
		return nil, err
	}

	chains = append(chains, chain)

	return chains, nil
}

func initSystemRoots() {
	systemRoots = NewCertPool()
}
<|endoftext|>


<commit_before>package dbmaker

import (
	"streamdb/config"
	"streamdb/util"
	"path/filepath"
	"os"
	"errors"
	"log"
)

var(
	sqliteInstance 		*SqliteService
	postgresInstance 	*PostgresService
	gnatsdInstance 		*GnatsdService
	redisInstance 		*RedisService

	ErrNotInitialized = errors.New("Module not yet initialized")
	ErrAlreadyInitialized = errors.New("All subsystems have allready been initialized")
	doneInit = false
)

func initSqlDatabase(config *config.Configuration) error {
	if doneInit {
		return ErrAlreadyInitialized
	}

	sqliteInstance = NewConfigSqliteSerivce(config)
	postgresInstance = NewConfigPostgresService(config)

	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			if err := postgresInstance.Init(); err != nil {
				return err
			}
		case "sqlite":
			if err := sqliteInstance.Init(); err != nil {
				return err
			}
		default:
			return ErrUnrecognizedDatabase
	}

	return nil
}

func Init(config *config.Configuration) error {
	if doneInit {
		return ErrAlreadyInitialized
	}

	log.Printf("Initializing subsystems\n")
	gnatsdInstance = NewConfigGnatsdService(config)
	redisInstance = NewConfigRedisService(config)

	if err := initSqlDatabase(config); err != nil {
		return err
	}

	if err := gnatsdInstance.Init(); err != nil {
		return err
	}

	if err := redisInstance.Init(); err != nil{
		return err
	}
	log.Printf("Finished initializing subsystems\n")

	doneInit = true
	return nil
}

func startSqlDatabase(config *config.Configuration) error {
	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			if err := postgresInstance.Start(); err != nil {
				return err
			}
		case "sqlite":
			if err := sqliteInstance.Start(); err != nil {
				return err
			}
		default:
			return ErrUnrecognizedDatabase
	}

	return nil
}

//Start the necessary servers to run StreamDB
func Start(config *config.Configuration) error {
	log.Printf("Starting subsystems\n")

	os.Chdir(config.StreamdbDirectory)

	if err := startSqlDatabase(config); err != nil {
		return err
	}

	if err := gnatsdInstance.Start(); err != nil {
		return err
	}

	if err := redisInstance.Start(); err != nil{
		return err
	}

	util.Touch(filepath.Join(config.StreamdbDirectory, "connectordb.pid"))

	return nil
}

func stopSqlDatabase(config *config.Configuration) error{
	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			return postgresInstance.Stop()
		case "sqlite":
			return sqliteInstance.Stop()
	}
	return ErrUnrecognizedDatabase
}

//Start the necessary servers to run StreamDB
func Stop(config *config.Configuration) error {
	log.Printf("Stopping subsystems\n")

	var globerr error
	if err := stopSqlDatabase(config); err != nil {
		globerr = err
	}

	if err := gnatsdInstance.Stop(); err != nil {
		globerr = err
	}

	if err := redisInstance.Stop(); err != nil{
		globerr = err
	}

	pidpath := filepath.Join(config.StreamdbDirectory, "connectordb.pid")
	if util.PathExists(pidpath) {
		if err := os.Remove(pidpath); err != nil {
			globerr = err
		}
	}

	return globerr
}

//Start the necessary servers to run StreamDB
func Kill(config *config.Configuration) error {
	log.Printf("Killing subsystems\n")

	var globerr error
	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			if err := postgresInstance.Kill(); err != nil {
				globerr = err
			}
		case "sqlite":
			if err := sqliteInstance.Kill(); err != nil {
				globerr = err
			}
		default:
			globerr = ErrUnrecognizedDatabase
	}

	if err := gnatsdInstance.Kill(); err != nil {
		globerr = err
	}

	if err := redisInstance.Kill(); err != nil{
		globerr = err
	}

	return globerr
}
<commit_msg>removed the chdir in allservices<commit_after>package dbmaker

import (
	"streamdb/config"
	"streamdb/util"
	"path/filepath"
	"os"
	"errors"
	"log"
)

var(
	sqliteInstance 		*SqliteService
	postgresInstance 	*PostgresService
	gnatsdInstance 		*GnatsdService
	redisInstance 		*RedisService

	ErrNotInitialized = errors.New("Module not yet initialized")
	ErrAlreadyInitialized = errors.New("All subsystems have allready been initialized")
	doneInit = false
)

func initSqlDatabase(config *config.Configuration) error {
	if doneInit {
		return ErrAlreadyInitialized
	}

	sqliteInstance = NewConfigSqliteSerivce(config)
	postgresInstance = NewConfigPostgresService(config)

	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			if err := postgresInstance.Init(); err != nil {
				return err
			}
		case "sqlite":
			if err := sqliteInstance.Init(); err != nil {
				return err
			}
		default:
			return ErrUnrecognizedDatabase
	}

	return nil
}

func Init(config *config.Configuration) error {
	if doneInit {
		return ErrAlreadyInitialized
	}

	log.Printf("Initializing subsystems\n")
	gnatsdInstance = NewConfigGnatsdService(config)
	redisInstance = NewConfigRedisService(config)

	if err := initSqlDatabase(config); err != nil {
		return err
	}

	if err := gnatsdInstance.Init(); err != nil {
		return err
	}

	if err := redisInstance.Init(); err != nil{
		return err
	}
	log.Printf("Finished initializing subsystems\n")

	doneInit = true
	return nil
}

func startSqlDatabase(config *config.Configuration) error {
	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			if err := postgresInstance.Start(); err != nil {
				return err
			}
		case "sqlite":
			if err := sqliteInstance.Start(); err != nil {
				return err
			}
		default:
			return ErrUnrecognizedDatabase
	}

	return nil
}

//Start the necessary servers to run StreamDB
func Start(config *config.Configuration) error {
	log.Printf("Starting subsystems\n")

	//os.Chdir(config.StreamdbDirectory)

	if err := startSqlDatabase(config); err != nil {
		return err
	}

	if err := gnatsdInstance.Start(); err != nil {
		return err
	}

	if err := redisInstance.Start(); err != nil{
		return err
	}

	util.Touch(filepath.Join(config.StreamdbDirectory, "connectordb.pid"))

	return nil
}

func stopSqlDatabase(config *config.Configuration) error{
	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			return postgresInstance.Stop()
		case "sqlite":
			return sqliteInstance.Stop()
	}
	return ErrUnrecognizedDatabase
}

//Start the necessary servers to run StreamDB
func Stop(config *config.Configuration) error {
	log.Printf("Stopping subsystems\n")

	var globerr error
	if err := stopSqlDatabase(config); err != nil {
		globerr = err
	}

	if err := gnatsdInstance.Stop(); err != nil {
		globerr = err
	}

	if err := redisInstance.Stop(); err != nil{
		globerr = err
	}

	pidpath := filepath.Join(config.StreamdbDirectory, "connectordb.pid")
	if util.PathExists(pidpath) {
		if err := os.Remove(pidpath); err != nil {
			globerr = err
		}
	}

	return globerr
}

//Start the necessary servers to run StreamDB
func Kill(config *config.Configuration) error {
	log.Printf("Killing subsystems\n")

	var globerr error
	sqlDatabaseType := config.DatabaseType

	switch sqlDatabaseType {
		case "postgres":
			if err := postgresInstance.Kill(); err != nil {
				globerr = err
			}
		case "sqlite":
			if err := sqliteInstance.Kill(); err != nil {
				globerr = err
			}
		default:
			globerr = ErrUnrecognizedDatabase
	}

	if err := gnatsdInstance.Kill(); err != nil {
		globerr = err
	}

	if err := redisInstance.Kill(); err != nil{
		globerr = err
	}

	return globerr
}
<|endoftext|>


<commit_before>// Implementation of Tests for peer.go

package main

import (
  "os/exec"
  "testing"
  "log"
  "strings"
)

type TestMsg struct {
        Message string
        From    Peer
}

type TestPeer struct {
        Name    string
        Address string
}

func TestGetIp(t *testing.T) {

  ipAddr := GetIp()

  out, _ := exec.Command("ping",  ipAddr, "-c 5", "-i 3", "-w 10").Output()
  if strings.Contains(string(out), "Destination Host Unreachable") {
    t.Errorf("GetIP failed, IP address not found  !")
  }

  log.Println("IP Address %s detected.",ipAddr)
}

func TestData(t *testing.T) {
        port := "7500"
        name := "Surematics"
        peer := "127.0.0.1"

        //initialize client control handler
        pdata := Peer{name, peer+":"+port}
        log.Println("pdata.Name detected: ",pdata.Name)

        if pdata.Name != "Surematics" {
           t.Errorf("Access Denied Peer.Name")
	}

        if pdata.Address != "127.0.0.1:7500" {
           t.Errorf("Access Denied Peer.Address")
        }

        testpeercontroller := NewPeerData(pdata)
        testpeercontroller.start()

        if (&testpeercontroller == nil) {
           t.Errorf("Error Initializing server")
        }

        log.Println("server detected: ",&testpeercontroller)
}

<commit_msg>update wrapper<commit_after>//Implementation of a simple peer to peer chat application using
//the net/http package for communication.

package main

import (
	"net/http"
	"reflect"
	"testing"
        "os/exec"
        "log"
        "strings"
)

type TestPeer struct {
        Name    string
        Address string
}

func TestGetIp(t *testing.T) {

  ipAddr := GetIp()

  out, _ := exec.Command("ping",  ipAddr, "-c 5", "-i 3", "-w 10").Output()
  if strings.Contains(string(out), "Destination Host Unreachable") {
    t.Errorf("GetIP failed, IP address not found  !")
  }

  log.Println("IP Address %s detected.",ipAddr)
}

func TestData(t *testing.T) {
        port := "7500"
        name := "Surematics"
        peer := "127.0.0.1"

        //initialize client control handler
        pdata := Peer{name, peer+":"+port}
        log.Println("pdata.Name detected: ",pdata.Name)

        if pdata.Name != "Surematics" {
           t.Errorf("Access Denied Peer.Name")
	}

        if pdata.Address != "127.0.0.1:7500" {
           t.Errorf("Access Denied Peer.Address")
        }

        testpeercontroller := NewPeerData(pdata)
        testpeercontroller.start()

        if (&testpeercontroller == nil) {
           t.Errorf("Error Initializing server")
        }

        log.Println("server detected: ",&testpeercontroller)
}

func TestNewPeerData(t *testing.T) {
	type args struct {
		self Peer
	}
	tests := []struct {
		name string
		args args
		want *PeerData
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := NewPeerData(tt.args.self); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("NewPeerData() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestPeerData_start(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	tests := []struct {
		name   string
		fields fields
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			peercontroller.start()
		})
	}
}

func TestPeerData_cmdLoop(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	tests := []struct {
		name   string
		fields fields
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			peercontroller.cmdLoop()
		})
	}
}

func TestPeerData_existingPeer(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	type args struct {
		peer Peer
	}
	tests := []struct {
		name   string
		fields fields
		args   args
		want   bool
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			if got := peercontroller.existingPeer(tt.args.peer); got != tt.want {
				t.Errorf("PeerData.existingPeer() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestPeerData_sendAdd(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	type args struct {
		peer Peer
	}
	tests := []struct {
		name   string
		fields fields
		args   args
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			peercontroller.sendAdd(tt.args.peer)
		})
	}
}

func TestPeerData_sendTx(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	type args struct {
		peer Peer
		msg  Msg
	}
	tests := []struct {
		name   string
		fields fields
		args   args
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			peercontroller.sendTx(tt.args.peer, tt.args.msg)
		})
	}
}

func TestPeerData_webListener(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	tests := []struct {
		name   string
		fields fields
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			peercontroller.webListener()
		})
	}
}

func Test_createAddHandler(t *testing.T) {
	type args struct {
		peercontroller *PeerData
	}
	tests := []struct {
		name string
		args args
		want func(http.ResponseWriter, *http.Request)
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := createAddHandler(tt.args.peercontroller); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("createAddHandler() = %v, want %v", got, tt.want)
			}
		})
	}
}

func Test_createTxHandler(t *testing.T) {
	type args struct {
		peercontroller *PeerData
	}
	tests := []struct {
		name string
		args args
		want func(http.ResponseWriter, *http.Request)
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := createTxHandler(tt.args.peercontroller); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("createTxHandler() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestPeerData_startStdinListener(t *testing.T) {
	type fields struct {
		Self            Peer
		Peers           Peers
		addPeers        chan (Peer)
		exitPeer        chan (Peer)
		currentPeers    chan (Peers)
		getCurrentPeers chan (bool)
		receivedMsg     chan (Msg)
		userMsg         chan (Msg)
	}
	type args struct {
		sender Peer
	}
	tests := []struct {
		name   string
		fields fields
		args   args
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			peercontroller := &PeerData{
				Self:            tt.fields.Self,
				Peers:           tt.fields.Peers,
				addPeers:        tt.fields.addPeers,
				exitPeer:        tt.fields.exitPeer,
				currentPeers:    tt.fields.currentPeers,
				getCurrentPeers: tt.fields.getCurrentPeers,
				receivedMsg:     tt.fields.receivedMsg,
				userMsg:         tt.fields.userMsg,
			}
			peercontroller.startStdinListener(tt.args.sender)
		})
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type rpcID [8]byte

func (id rpcID) String() string {
	for i := range id {
		if id[i] == 0 {
			id[i] = ' '
		}
	}
	return string(id[:])
}

// handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining
// bytes are 0. A handlerName is specified at the beginning of each network
// call, indicating which function should handle the connection.
func handlerName(name string) (id rpcID) {
	copy(id[:], name)
	return
}

// RPC calls an RPC on the given address. RPC cannot be called on an address
// that the Gateway is not connected to.
func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	id := g.mu.RLock()
	peer, ok := g.peers[addr]
	g.mu.RUnlock(id)
	if !ok {
		return errors.New("can't call RPC on unconnected peer " + string(addr))
	}

	conn, err := peer.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	// write header
	if err := encoding.WriteObject(conn, handlerName(name)); err != nil {
		return err
	}
	// call fn
	return fn(conn)
}

// RegisterRPC registers an RPCFunc as a handler for a given identifier. To
// call an RPC, use gateway.RPC, supplying the same identifier given to
// RegisterRPC. Identifiers should always use PascalCase. The first 8
// characters of an identifier should be unique, as the identifier used
// internally is truncated to 8 bytes.
func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.handlers[handlerName(name)]; ok {
			panic("refusing to overwrite RPC " + name)
		}
	}
	g.handlers[handlerName(name)] = fn
}

// RegisterConnectCall registers a name and RPCFunc to be called on a peer
// upon connecting.
func (g *Gateway) RegisterConnectCall(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.initRPCs[name]; ok {
			panic("refusing to overwrite RPC " + name)
		}
	}
	g.initRPCs[name] = fn
}

// listenPeer listens for new streams on a peer connection and serves them via
// threadedHandleConn.
func (g *Gateway) listenPeer(p *peer) {
	for {
		conn, err := p.accept()
		if err != nil {
			g.log.Println("WARN: lost connection to peer", p.addr)
			break
		}

		// it is the handler's responsibility to close the connection
		go g.threadedHandleConn(conn)
	}
	g.Disconnect(p.addr)
}

// threadedHandleConn reads header data from a connection, then routes it to the
// appropriate handler for further processing.
func (g *Gateway) threadedHandleConn(conn modules.PeerConn) {
	defer conn.Close()
	var id rpcID
	if err := encoding.ReadObject(conn, &id, 8); err != nil {
		return
	}
	// call registered handler for this ID
	lockid := g.mu.RLock()
	fn, ok := g.handlers[id]
	g.mu.RUnlock(lockid)
	if !ok {
		g.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RemoteAddr(), id)
		return
	}

	// call fn
	err := fn(conn)
	// don't log benign errors
	if err == modules.ErrDuplicateTransactionSet || err == modules.ErrBlockKnown {
		err = nil
	}
	if err != nil {
		g.log.Printf("WARN: incoming RPC \"%v\" failed: %v", id, err)
	}
}

// Broadcast calls an RPC on all of the specified peers. The calls are run in
// parallel. Broadcasts are restricted to "one-way" RPCs, which simply write an
// object and disconnect. This is why Broadcast takes an interface{} instead of
// an RPCFunc.
func (g *Gateway) Broadcast(name string, obj interface{}, peers []modules.Peer) {
	g.log.Printf("INFO: broadcasting RPC \"%v\" to %v peers", name, len(peers))

	// only encode obj once, instead of using WriteObject
	enc := encoding.Marshal(obj)
	fn := func(conn modules.PeerConn) error {
		return encoding.WritePrefix(conn, enc)
	}

	var wg sync.WaitGroup
	wg.Add(len(peers))
	for _, p := range peers {
		go func(addr modules.NetAddress) {
			err := g.RPC(addr, name, fn)
			if err != nil {
				// try one more time before giving up
				time.Sleep(10 * time.Second)
				g.RPC(addr, name, fn)
			}
			wg.Done()
		}(p.NetAddress)
	}
	wg.Wait()
}
<commit_msg>Add connection address to log when RPC fails<commit_after>package gateway

import (
	"errors"
	"sync"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

type rpcID [8]byte

func (id rpcID) String() string {
	for i := range id {
		if id[i] == 0 {
			id[i] = ' '
		}
	}
	return string(id[:])
}

// handlerName truncates a string to 8 bytes. If len(name) < 8, the remaining
// bytes are 0. A handlerName is specified at the beginning of each network
// call, indicating which function should handle the connection.
func handlerName(name string) (id rpcID) {
	copy(id[:], name)
	return
}

// RPC calls an RPC on the given address. RPC cannot be called on an address
// that the Gateway is not connected to.
func (g *Gateway) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	id := g.mu.RLock()
	peer, ok := g.peers[addr]
	g.mu.RUnlock(id)
	if !ok {
		return errors.New("can't call RPC on unconnected peer " + string(addr))
	}

	conn, err := peer.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	// write header
	if err := encoding.WriteObject(conn, handlerName(name)); err != nil {
		return err
	}
	// call fn
	return fn(conn)
}

// RegisterRPC registers an RPCFunc as a handler for a given identifier. To
// call an RPC, use gateway.RPC, supplying the same identifier given to
// RegisterRPC. Identifiers should always use PascalCase. The first 8
// characters of an identifier should be unique, as the identifier used
// internally is truncated to 8 bytes.
func (g *Gateway) RegisterRPC(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.handlers[handlerName(name)]; ok {
			panic("refusing to overwrite RPC " + name)
		}
	}
	g.handlers[handlerName(name)] = fn
}

// RegisterConnectCall registers a name and RPCFunc to be called on a peer
// upon connecting.
func (g *Gateway) RegisterConnectCall(name string, fn modules.RPCFunc) {
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if build.DEBUG {
		if _, ok := g.initRPCs[name]; ok {
			panic("refusing to overwrite RPC " + name)
		}
	}
	g.initRPCs[name] = fn
}

// listenPeer listens for new streams on a peer connection and serves them via
// threadedHandleConn.
func (g *Gateway) listenPeer(p *peer) {
	for {
		conn, err := p.accept()
		if err != nil {
			g.log.Println("WARN: lost connection to peer", p.addr)
			break
		}

		// it is the handler's responsibility to close the connection
		go g.threadedHandleConn(conn)
	}
	g.Disconnect(p.addr)
}

// threadedHandleConn reads header data from a connection, then routes it to the
// appropriate handler for further processing.
func (g *Gateway) threadedHandleConn(conn modules.PeerConn) {
	defer conn.Close()
	var id rpcID
	if err := encoding.ReadObject(conn, &id, 8); err != nil {
		return
	}
	// call registered handler for this ID
	lockid := g.mu.RLock()
	fn, ok := g.handlers[id]
	g.mu.RUnlock(lockid)
	if !ok {
		g.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RemoteAddr(), id)
		return
	}

	// call fn
	err := fn(conn)
	// don't log benign errors
	if err == modules.ErrDuplicateTransactionSet || err == modules.ErrBlockKnown {
		err = nil
	}
	if err != nil {
		g.log.Printf("WARN: incoming RPC \"%v\" from conn %v failed: %v", id, conn.RemoteAddr(), err)
	}
}

// Broadcast calls an RPC on all of the specified peers. The calls are run in
// parallel. Broadcasts are restricted to "one-way" RPCs, which simply write an
// object and disconnect. This is why Broadcast takes an interface{} instead of
// an RPCFunc.
func (g *Gateway) Broadcast(name string, obj interface{}, peers []modules.Peer) {
	g.log.Printf("INFO: broadcasting RPC \"%v\" to %v peers", name, len(peers))

	// only encode obj once, instead of using WriteObject
	enc := encoding.Marshal(obj)
	fn := func(conn modules.PeerConn) error {
		return encoding.WritePrefix(conn, enc)
	}

	var wg sync.WaitGroup
	wg.Add(len(peers))
	for _, p := range peers {
		go func(addr modules.NetAddress) {
			err := g.RPC(addr, name, fn)
			if err != nil {
				// try one more time before giving up
				time.Sleep(10 * time.Second)
				g.RPC(addr, name, fn)
			}
			wg.Done()
		}(p.NetAddress)
	}
	wg.Wait()
}
<|endoftext|>