<commit_before>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

/* This file will handle all file related business. It will explore the file
 * system for new files and will watch files for changes. In particular, this
 * file is the gate to the files table in the database. Other components (Parser)
 * should rely on this file to know if a file was explored or not. */

import (
	fsnotify "gopkg.in/fsnotify.v1"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sync"
)

const validCString string = `^[^\.].*\.c$`
const validHString string = `^[^\.].*\.h$`

var files chan string
var wg sync.WaitGroup
var watcher *fsnotify.Watcher
var writer chan *WriterDB

func uptodateFile(file string) bool {
	wr := <-writer
	defer func() { writer <- wr }()

	exist, uptodate, fi, err := wr.UptodateFile(file)

	if err != nil {
		// if there is an error with the dependency, we are going to
		// pretend everything is fine so the parser is not executed
		return true
	}

	if exist && uptodate {
		return true
	} else {
		wr.RemoveFileReferences(file)
		wr.InsertFile(file, fi)
		return false
	}
}

func processFile(parser *Parser) {
	wg.Add(1)
	defer wg.Done()

	// start exploring files
	for {
		file, ok := <-files

		if !ok {
			return
		}

		if !uptodateFile(file) {
			log.Println("parsing", file)
			parser.Parse(file)
		}
	}
}

func traversePath(path string, visitDir func(string), visitC func(string)) {
	filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Println("error opening ", path, " igoring")
			return filepath.SkipDir
		}

		// visit file
		if info.IsDir() {
			if info.Name()[0] == '.' {
				return filepath.SkipDir
			} else {
				visitDir(path)
				return nil
			}
		} else {
			// ignore non-C files
			validC, _ := regexp.MatchString(validCString, path)
			if validC {
				visitC(path)
			}
		}

		return nil
	})
}

func removeFileAndReparseDepends(file string, db *WriterDB) {
	deps := db.RemoveFileDepsReferences(file)
	db.RemoveFileReferences(file)

	for _, d := range deps {
		files <- d
	}
}

func handleFileChange(event fsnotify.Event) {

	validC, _ := regexp.MatchString(validCString, event.Name)
	validH, _ := regexp.MatchString(validHString, event.Name)

	switch {
	case validC:
		files <- event.Name
	case validH:
		db := <-writer
		exist, uptodate, _, err := db.UptodateFile(event.Name)

		if err != nil || (exist && !uptodate) {
			removeFileAndReparseDepends(filepath.Clean(event.Name), db)
		}

		writer <- db
	}
}

func handleDirChange(event fsnotify.Event) {
	switch {
	case event.Op&(fsnotify.Create) != 0:
		// explore the new dir
		visitorDir := func(path string) {
			// add watcher to directory
			watcher.Add(path)
		}
		visitorC := func(path string) {
			// put file in channel
			files <- path
		}
		traversePath(event.Name, visitorDir, visitorC)
	case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
		// remove watcher from dir
		watcher.Remove(event.Name)
	}
}

func isDirectory(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, err
	} else {
		return fi.IsDir(), nil
	}
}

func handleChange(event fsnotify.Event) {

	// ignore if hidden
	if filepath.Base(event.Name)[0] == '.' {
		return
	}

	// first, we need to check if the file is a directory or not
	isDir, err := isDirectory(event.Name)
	if err != nil {
		// ignoring this event
		return
	}

	if isDir {
		handleDirChange(event)
	} else {
		handleFileChange(event)
	}
}

func StartFilesHandler(indexDir []string, nIndexingThreads int, parser *Parser,
	db *DBConnFactory) {

	files = make(chan string, nIndexingThreads)
	writer = make(chan *WriterDB, 1)
	writer <- db.NewWriter()

	// start threads to process files
	for i := 0; i < nIndexingThreads; i++ {
		go processFile(parser)
	}

	// start file watcher
	watcher, _ = fsnotify.NewWatcher()
	go func() {
		wg.Add(1)
		defer wg.Done()
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok {
					return
				}
				handleChange(event)
			case err, ok := <-watcher.Errors:
				if !ok {
					return
				}

				log.Println("watcher error: ", err)
			}
		}
	}()

	// explore all the paths in indexDir and process all files
	rd := db.NewReader()
	removedFilesSet := rd.GetSetFilesInDB()
	rd.Close()
	visitorDir := func(path string) {
		// add watcher to directory
		watcher.Add(path)
	}
	visitorC := func(path string) {
		// update set of removed files
		delete(removedFilesSet, path)
		// put file in channel
		files <- path
	}
	for _, path := range indexDir {
		traversePath(path, visitorDir, visitorC)
	}

	// remove from DB deleted files
	wr := <-writer
	for path := range removedFilesSet {
		wr.RemoveFileReferences(path)
	}
	writer <- wr
}

func UpdateDependency(file, dep string) bool {
	wr := <-writer
	defer func() { writer <- wr }()

	exist, uptodate, fi, err := wr.UptodateFile(dep)

	if err != nil {
		// if there is an error with the dependency, we are going to
		// pretend everything is fine so the parser move forward
		return true
	}

	if !exist {
		wr.InsertFile(dep, fi)
	} else if !uptodate {
		removeFileAndReparseDepends(dep, wr)
		files <- file
		return false
	}

	wr.InsertDependency(file, dep)
	return true
}

func CloseFilesHandler() {
	close(files)

	wr := <-writer
	wr.Close()
	close(writer)

	watcher.Close()

	wg.Wait()
}
<commit_msg>Handling FS event remove and rename<commit_after>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

/* This file will handle all file related business. It will explore the file
 * system for new files and will watch files for changes. In particular, this
 * file is the gate to the files table in the database. Other components (Parser)
 * should rely on this file to know if a file was explored or not. */

import (
	fsnotify "gopkg.in/fsnotify.v1"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sync"
)

const validCString string = `^[^\.].*\.c$`
const validHString string = `^[^\.].*\.h$`

var files chan string
var wg sync.WaitGroup
var watcher *fsnotify.Watcher
var writer chan *WriterDB

func uptodateFile(file string) bool {
	wr := <-writer
	defer func() { writer <- wr }()

	exist, uptodate, fi, err := wr.UptodateFile(file)

	if err != nil {
		// if there is an error with the dependency, we are going to
		// pretend everything is fine so the parser is not executed
		return true
	}

	if exist && uptodate {
		return true
	} else {
		wr.RemoveFileReferences(file)
		wr.InsertFile(file, fi)
		return false
	}
}

func processFile(parser *Parser) {
	wg.Add(1)
	defer wg.Done()

	// start exploring files
	for {
		file, ok := <-files

		if !ok {
			return
		}

		if !uptodateFile(file) {
			log.Println("parsing", file)
			parser.Parse(file)
		}
	}
}

func traversePath(path string, visitDir func(string), visitC func(string)) {
	filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			log.Println("error opening ", path, " igoring")
			return filepath.SkipDir
		}

		// visit file
		if info.IsDir() {
			if info.Name()[0] == '.' {
				return filepath.SkipDir
			} else {
				visitDir(path)
				return nil
			}
		} else {
			// ignore non-C files
			validC, _ := regexp.MatchString(validCString, path)
			if validC {
				visitC(path)
			}
		}

		return nil
	})
}

func removeFileAndReparseDepends(file string, db *WriterDB) {
	deps := db.RemoveFileDepsReferences(file)
	db.RemoveFileReferences(file)

	for _, d := range deps {
		files <- d
	}
}

func handleFileChange(event fsnotify.Event) {

	validC, _ := regexp.MatchString(validCString, event.Name)
	validH, _ := regexp.MatchString(validHString, event.Name)

	db := <-writer
	defer func() { writer <- db }()

	switch {
	case validC:
		switch {
		case event.Op&(fsnotify.Create|fsnotify.Write) != 0:
			files <- event.Name
		case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
			db.RemoveFileReferences(event.Name)
		}
	case validH:
		exist, uptodate, _, err := db.UptodateFile(event.Name)
		switch {
		case err != nil:
			return
		case event.Op&(fsnotify.Write) != 0:
			if exist && !uptodate {
				removeFileAndReparseDepends(filepath.Clean(event.Name), db)
			}
		case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
			if exist {
				removeFileAndReparseDepends(filepath.Clean(event.Name), db)
			}
		}
	}
}

func handleDirChange(event fsnotify.Event) {
	switch {
	case event.Op&(fsnotify.Create) != 0:
		// explore the new dir
		visitorDir := func(path string) {
			// add watcher to directory
			watcher.Add(path)
		}
		visitorC := func(path string) {
			// put file in channel
			files <- path
		}
		traversePath(event.Name, visitorDir, visitorC)
	case event.Op&(fsnotify.Remove|fsnotify.Rename) != 0:
		// remove watcher from dir
		watcher.Remove(event.Name)
	}
}

func isDirectory(path string) (bool, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, err
	} else {
		return fi.IsDir(), nil
	}
}

func handleChange(event fsnotify.Event) {

	// ignore if hidden
	if filepath.Base(event.Name)[0] == '.' {
		return
	}

	// first, we need to check if the file is a directory or not
	isDir, err := isDirectory(event.Name)
	if err != nil {
		// ignoring this event
		return
	}

	if isDir {
		handleDirChange(event)
	} else {
		handleFileChange(event)
	}
}

func StartFilesHandler(indexDir []string, nIndexingThreads int, parser *Parser,
	db *DBConnFactory) {

	files = make(chan string, nIndexingThreads)
	writer = make(chan *WriterDB, 1)
	writer <- db.NewWriter()

	// start threads to process files
	for i := 0; i < nIndexingThreads; i++ {
		go processFile(parser)
	}

	// start file watcher
	watcher, _ = fsnotify.NewWatcher()
	go func() {
		wg.Add(1)
		defer wg.Done()
		for {
			select {
			case event, ok := <-watcher.Events:
				if !ok {
					return
				}
				handleChange(event)
			case err, ok := <-watcher.Errors:
				if !ok {
					return
				}

				log.Println("watcher error: ", err)
			}
		}
	}()

	// explore all the paths in indexDir and process all files
	rd := db.NewReader()
	removedFilesSet := rd.GetSetFilesInDB()
	rd.Close()
	visitorDir := func(path string) {
		// add watcher to directory
		watcher.Add(path)
	}
	visitorC := func(path string) {
		// update set of removed files
		delete(removedFilesSet, path)
		// put file in channel
		files <- path
	}
	for _, path := range indexDir {
		traversePath(path, visitorDir, visitorC)
	}

	// remove from DB deleted files
	wr := <-writer
	for path := range removedFilesSet {
		wr.RemoveFileReferences(path)
	}
	writer <- wr
}

func UpdateDependency(file, dep string) bool {
	wr := <-writer
	defer func() { writer <- wr }()

	exist, uptodate, fi, err := wr.UptodateFile(dep)

	if err != nil {
		// if there is an error with the dependency, we are going to
		// pretend everything is fine so the parser move forward
		return true
	}

	if !exist {
		wr.InsertFile(dep, fi)
	} else if !uptodate {
		removeFileAndReparseDepends(dep, wr)
		files <- file
		return false
	}

	wr.InsertDependency(file, dep)
	return true
}

func CloseFilesHandler() {
	close(files)

	wr := <-writer
	wr.Close()
	close(writer)

	watcher.Close()

	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/googlecloudplatform/gcsfuse/internal/logger"
	mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount"
	"github.com/urfave/cli"
)

// Set up custom help text for gcsfuse; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} [bucket] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func newApp() (app *cli.App) {
	dirModeValue := new(OctalInt)
	*dirModeValue = 0755

	fileModeValue := new(OctalInt)
	*fileModeValue = 0644

	app = &cli.App{
		Name:    "gcsfuse",
		Version: getVersion(),
		Usage:   "Mount a specified GCS bucket or all accessible buckets locally",
		Writer:  os.Stderr,
		Flags: []cli.Flag{

			cli.StringFlag{
				Name:  "app-name",
				Value: "",
				Usage: "The application name of this mount.",
			},

			cli.BoolFlag{
				Name:  "foreground",
				Usage: "Stay in the foreground after mounting.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.GenericFlag{
				Name:  "dir-mode",
				Value: dirModeValue,
				Usage: "Permissions bits for directories, in octal.",
			},

			cli.GenericFlag{
				Name:  "file-mode",
				Value: fileModeValue,
				Usage: "Permission bits for files, in octal.",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: -1,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: -1,
				Usage: "GID owner of all inodes.",
			},

			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See " +
					"docs/semantics.md",
			},

			cli.StringFlag{
				Name:  "only-dir",
				Usage: "Mount only the given directory, relative to the bucket root.",
			},

			cli.IntFlag{
				Name:  "rename-dir-limit",
				Value: 0,
				Usage: "Allow rename a directory containing fewer descendants " +
					"than this limit.",
			},

			/////////////////////////
			// GCS
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "https://storage.googleapis.com:443",
				Usage: "The endpoint to connect to.",
			},

			cli.StringFlag{
				Name:  "billing-project",
				Value: "",
				Usage: "Project to use for billing when accessing requester pays buckets. " +
					"(default: none)",
			},

			cli.StringFlag{
				Name:  "key-file",
				Value: "",
				Usage: "Absolute path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},

			cli.StringFlag{
				Name:  "token-url",
				Value: "",
				Usage: "An url for getting an access token when key-file is absent.",
			},

			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},

			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: -1,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "max-retry-sleep",
				Value: time.Minute,
				Usage: "The maximum duration allowed to sleep in a retry loop with " +
					"exponential backoff for failed requests to GCS backend. Once the " +
					"backoff duration exceeds this limit, the retry stops. The default " +
					"is 1 minute. A value of 0 disables retries.",
			},

			cli.IntFlag{
				Name:  "stat-cache-capacity",
				Value: 4096,
				Usage: "How many entries can the stat cache hold (impacts memory consumption)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			cli.BoolFlag{
				Name:  "experimental-local-file-cache",
				Usage: "Experimental: Cache GCS files on local disk for reads.",
			},

			cli.StringFlag{
				Name:  "temp-dir",
				Value: "",
				Usage: "Absolute path to temporary directory for local GCS object " +
					"copies. (default: system default, likely /tmp)",
			},

			cli.BoolFlag{
				Name: "disable-http2",
				Usage: "Once set, the protocol used for communicating with " +
					"GCS backend would be HTTP/1.1, instead of the default HTTP/2.",
			},

			cli.IntFlag{
				Name:  "max-conns-per-host",
				Value: 10,
				Usage: "The max number of TCP connections allowed per server. " +
					"This is effective when --disable-http2 is set.",
			},

			/////////////////////////
			// Monitoring & Logging
			/////////////////////////

			cli.DurationFlag{
				Name:  "experimental-stackdriver-export-interval",
				Value: 0,
				Usage: "Experimental: Export metrics to stackdriver with this interval. The default value 0 indicates no exporting.",
			},

			cli.StringFlag{
				Name:  "experimental-opentelemetry-collector-address",
				Value: "",
				Usage: "Experimental: Export metrics to the OpenTelemetry collector at this address.",
			},

			cli.StringFlag{
				Name:  "log-file",
				Value: "",
				Usage: "The file for storing logs that can be parsed by " +
					"fluentd. When not provided, plain text logs are printed to " +
					"stdout.",
			},

			cli.StringFlag{
				Name:  "log-format",
				Value: "json",
				Usage: "The format of the log file: 'text' or 'json'.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_fs",
				Usage: "Enable file system debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_gcs",
				Usage: "Print GCS request and timing information.",
			},

			cli.BoolFlag{
				Name:  "debug_http",
				Usage: "Dump HTTP requests and responses to/from GCS.",
			},

			cli.BoolFlag{
				Name:  "debug_invariants",
				Usage: "Panic when internal invariants are violated.",
			},

			cli.BoolFlag{
				Name:  "debug_mutex",
				Usage: "Print debug messages when a mutex is held too long.",
			},
		},
	}

	return
}

type flagStorage struct {
	AppName    string
	Foreground bool

	// File system
	MountOptions   map[string]string
	DirMode        os.FileMode
	FileMode       os.FileMode
	Uid            int64
	Gid            int64
	ImplicitDirs   bool
	OnlyDir        string
	RenameDirLimit int64

	// GCS
	Endpoint                           *url.URL
	BillingProject                     string
	KeyFile                            string
	TokenUrl                           string
	EgressBandwidthLimitBytesPerSecond float64
	OpRateLimitHz                      float64

	// Tuning
	MaxRetrySleep     time.Duration
	StatCacheCapacity int
	StatCacheTTL      time.Duration
	TypeCacheTTL      time.Duration
	LocalFileCache    bool
	TempDir           string
	DisableHTTP2      bool
	MaxConnsPerHost   int

	// Monitoring & Logging
	StackdriverExportInterval time.Duration
	OtelCollectorAddress      string
	LogFile                   string
	LogFormat                 string

	// Debugging
	DebugFuse       bool
	DebugFS         bool
	DebugGCS        bool
	DebugHTTP       bool
	DebugInvariants bool
	DebugMutex      bool
}

const GCSFUSE_PARENT_PROCESS_DIR = "gcsfuse-parent-process-dir"

// 1. Returns the same filepath in case of absolute path or empty filename.
// 2. For relative path with . or .. or nothing, it resolves smartly the path
// in case of child proces it resolves with respect to GCSFUSE_PARENT_PROCESS_DIR
// otherwise with respect to the current working dir.
// 3. For relative path starting with ~, it resolves with respect to home dir.
func getResolvedPath(filePath string) (resolvedPath string, err error) {
	if filePath == "" || path.IsAbs(filePath) {
		resolvedPath = filePath
		return
	}

	// Relative path starting with tilda (~)
	if strings.HasPrefix(filePath, "~/") {
		homeDir, err := os.UserHomeDir()
		if err != nil {
			return "", fmt.Errorf("Fetch home dir: [%w]", err)
		}
		return filepath.Join(homeDir, filePath[2:]), err
	}

	// Means - relative path starting with ., .. or nothing.
	gcsfuseParentProcessDir, _ := os.LookupEnv(GCSFUSE_PARENT_PROCESS_DIR)
	gcsfuseParentProcessDir = strings.TrimSpace(gcsfuseParentProcessDir)
	if gcsfuseParentProcessDir == "" {
		return filepath.Abs(filePath)
	} else {
		return filepath.Join(gcsfuseParentProcessDir, filePath), err
	}
}

// This method resolves path in the context dictionary.
func resolvePathForTheFlagInContext(flagKey string, c *cli.Context) (err error) {
	flagValue := c.String(flagKey)
	resolvedPath, err := getResolvedPath(flagValue)
	if err != nil {
		return
	}
	c.Set(flagKey, resolvedPath)
	logger.Infof("Value of [%s] resolved from [%s] to [%s]\n",
		flagKey, flagValue, resolvedPath)
	return
}

// For parent process: it only resolves the path with respect to home folder.
// For child process (when --foreground flag is disabled): it resolves
// the path relative to both current directory and home directory
func resolvePathForTheRequiredFlagInContext(c *cli.Context) (err error) {
	err = resolvePathForTheFlagInContext("log-file", c)
	if err != nil {
		return fmt.Errorf("resolving for log-file: [%w]", err)
	}

	err = resolvePathForTheFlagInContext("key-file", c)
	if err != nil {
		return fmt.Errorf("resolving for key-file: [%w]", err)
	}

	return
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	endpoint, err := url.Parse(c.String("endpoint"))
	if err != nil {
		fmt.Printf("Could not parse endpoint")
		return nil
	}
	flags = &flagStorage{
		AppName:    c.String("app-name"),
		Foreground: c.Bool("foreground"),

		// File system
		MountOptions:   make(map[string]string),
		DirMode:        os.FileMode(*c.Generic("dir-mode").(*OctalInt)),
		FileMode:       os.FileMode(*c.Generic("file-mode").(*OctalInt)),
		Uid:            int64(c.Int("uid")),
		Gid:            int64(c.Int("gid")),
		ImplicitDirs:   c.Bool("implicit-dirs"),
		OnlyDir:        c.String("only-dir"),
		RenameDirLimit: int64(c.Int("rename-dir-limit")),

		// GCS,
		Endpoint:                           endpoint,
		BillingProject:                     c.String("billing-project"),
		KeyFile:                            c.String("key-file"),
		TokenUrl:                           c.String("token-url"),
		EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"),
		OpRateLimitHz:                      c.Float64("limit-ops-per-sec"),

		// Tuning,
		MaxRetrySleep:     c.Duration("max-retry-sleep"),
		StatCacheCapacity: c.Int("stat-cache-capacity"),
		StatCacheTTL:      c.Duration("stat-cache-ttl"),
		TypeCacheTTL:      c.Duration("type-cache-ttl"),
		LocalFileCache:    c.Bool("experimental-local-file-cache"),
		TempDir:           c.String("temp-dir"),
		DisableHTTP2:      c.Bool("disable-http2"),
		MaxConnsPerHost:   c.Int("max-conns-per-host"),

		// Monitoring & Logging
		StackdriverExportInterval: c.Duration("experimental-stackdriver-export-interval"),
		OtelCollectorAddress:      c.String("experimental-opentelemetry-collector-address"),
		LogFile:                   c.String("log-file"),
		LogFormat:                 c.String("log-format"),

		// Debugging,
		DebugFuse:       c.Bool("debug_fuse"),
		DebugGCS:        c.Bool("debug_gcs"),
		DebugFS:         c.Bool("debug_fs"),
		DebugHTTP:       c.Bool("debug_http"),
		DebugInvariants: c.Bool("debug_invariants"),
		DebugMutex:      c.Bool("debug_mutex"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		mountpkg.ParseOptions(flags.MountOptions, o)
	}

	return
}

// A cli.Generic that can be used with cli.GenericFlag to obtain an int flag
// that is parsed in octal.
type OctalInt int

var _ cli.Generic = (*OctalInt)(nil)

func (oi *OctalInt) Set(value string) (err error) {
	tmp, err := strconv.ParseInt(value, 8, 32)
	if err != nil {
		err = fmt.Errorf("Parsing as octal: %w", err)
		return
	}

	*oi = OctalInt(tmp)
	return
}

func (oi OctalInt) String() string {
	return fmt.Sprintf("%o", oi)
}
<commit_msg>Suppressing the log info when resolve was no-op<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/googlecloudplatform/gcsfuse/internal/logger"
	mountpkg "github.com/googlecloudplatform/gcsfuse/internal/mount"
	"github.com/urfave/cli"
)

// Set up custom help text for gcsfuse; in particular the usage section.
func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} [bucket] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

func newApp() (app *cli.App) {
	dirModeValue := new(OctalInt)
	*dirModeValue = 0755

	fileModeValue := new(OctalInt)
	*fileModeValue = 0644

	app = &cli.App{
		Name:    "gcsfuse",
		Version: getVersion(),
		Usage:   "Mount a specified GCS bucket or all accessible buckets locally",
		Writer:  os.Stderr,
		Flags: []cli.Flag{

			cli.StringFlag{
				Name:  "app-name",
				Value: "",
				Usage: "The application name of this mount.",
			},

			cli.BoolFlag{
				Name:  "foreground",
				Usage: "Stay in the foreground after mounting.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.GenericFlag{
				Name:  "dir-mode",
				Value: dirModeValue,
				Usage: "Permissions bits for directories, in octal.",
			},

			cli.GenericFlag{
				Name:  "file-mode",
				Value: fileModeValue,
				Usage: "Permission bits for files, in octal.",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: -1,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: -1,
				Usage: "GID owner of all inodes.",
			},

			cli.BoolFlag{
				Name: "implicit-dirs",
				Usage: "Implicitly define directories based on content. See " +
					"docs/semantics.md",
			},

			cli.StringFlag{
				Name:  "only-dir",
				Usage: "Mount only the given directory, relative to the bucket root.",
			},

			cli.IntFlag{
				Name:  "rename-dir-limit",
				Value: 0,
				Usage: "Allow rename a directory containing fewer descendants " +
					"than this limit.",
			},

			/////////////////////////
			// GCS
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "https://storage.googleapis.com:443",
				Usage: "The endpoint to connect to.",
			},

			cli.StringFlag{
				Name:  "billing-project",
				Value: "",
				Usage: "Project to use for billing when accessing requester pays buckets. " +
					"(default: none)",
			},

			cli.StringFlag{
				Name:  "key-file",
				Value: "",
				Usage: "Absolute path to JSON key file for use with GCS. " +
					"(default: none, Google application default credentials used)",
			},

			cli.StringFlag{
				Name:  "token-url",
				Value: "",
				Usage: "An url for getting an access token when key-file is absent.",
			},

			cli.Float64Flag{
				Name:  "limit-bytes-per-sec",
				Value: -1,
				Usage: "Bandwidth limit for reading data, measured over a 30-second " +
					"window. (use -1 for no limit)",
			},

			cli.Float64Flag{
				Name:  "limit-ops-per-sec",
				Value: -1,
				Usage: "Operations per second limit, measured over a 30-second window " +
					"(use -1 for no limit)",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.DurationFlag{
				Name:  "max-retry-sleep",
				Value: time.Minute,
				Usage: "The maximum duration allowed to sleep in a retry loop with " +
					"exponential backoff for failed requests to GCS backend. Once the " +
					"backoff duration exceeds this limit, the retry stops. The default " +
					"is 1 minute. A value of 0 disables retries.",
			},

			cli.IntFlag{
				Name:  "stat-cache-capacity",
				Value: 4096,
				Usage: "How many entries can the stat cache hold (impacts memory consumption)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			cli.BoolFlag{
				Name:  "experimental-local-file-cache",
				Usage: "Experimental: Cache GCS files on local disk for reads.",
			},

			cli.StringFlag{
				Name:  "temp-dir",
				Value: "",
				Usage: "Absolute path to temporary directory for local GCS object " +
					"copies. (default: system default, likely /tmp)",
			},

			cli.BoolFlag{
				Name: "disable-http2",
				Usage: "Once set, the protocol used for communicating with " +
					"GCS backend would be HTTP/1.1, instead of the default HTTP/2.",
			},

			cli.IntFlag{
				Name:  "max-conns-per-host",
				Value: 10,
				Usage: "The max number of TCP connections allowed per server. " +
					"This is effective when --disable-http2 is set.",
			},

			/////////////////////////
			// Monitoring & Logging
			/////////////////////////

			cli.DurationFlag{
				Name:  "experimental-stackdriver-export-interval",
				Value: 0,
				Usage: "Experimental: Export metrics to stackdriver with this interval. The default value 0 indicates no exporting.",
			},

			cli.StringFlag{
				Name:  "experimental-opentelemetry-collector-address",
				Value: "",
				Usage: "Experimental: Export metrics to the OpenTelemetry collector at this address.",
			},

			cli.StringFlag{
				Name:  "log-file",
				Value: "",
				Usage: "The file for storing logs that can be parsed by " +
					"fluentd. When not provided, plain text logs are printed to " +
					"stdout.",
			},

			cli.StringFlag{
				Name:  "log-format",
				Value: "json",
				Usage: "The format of the log file: 'text' or 'json'.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_fs",
				Usage: "Enable file system debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_gcs",
				Usage: "Print GCS request and timing information.",
			},

			cli.BoolFlag{
				Name:  "debug_http",
				Usage: "Dump HTTP requests and responses to/from GCS.",
			},

			cli.BoolFlag{
				Name:  "debug_invariants",
				Usage: "Panic when internal invariants are violated.",
			},

			cli.BoolFlag{
				Name:  "debug_mutex",
				Usage: "Print debug messages when a mutex is held too long.",
			},
		},
	}

	return
}

type flagStorage struct {
	AppName    string
	Foreground bool

	// File system
	MountOptions   map[string]string
	DirMode        os.FileMode
	FileMode       os.FileMode
	Uid            int64
	Gid            int64
	ImplicitDirs   bool
	OnlyDir        string
	RenameDirLimit int64

	// GCS
	Endpoint                           *url.URL
	BillingProject                     string
	KeyFile                            string
	TokenUrl                           string
	EgressBandwidthLimitBytesPerSecond float64
	OpRateLimitHz                      float64

	// Tuning
	MaxRetrySleep     time.Duration
	StatCacheCapacity int
	StatCacheTTL      time.Duration
	TypeCacheTTL      time.Duration
	LocalFileCache    bool
	TempDir           string
	DisableHTTP2      bool
	MaxConnsPerHost   int

	// Monitoring & Logging
	StackdriverExportInterval time.Duration
	OtelCollectorAddress      string
	LogFile                   string
	LogFormat                 string

	// Debugging
	DebugFuse       bool
	DebugFS         bool
	DebugGCS        bool
	DebugHTTP       bool
	DebugInvariants bool
	DebugMutex      bool
}

const GCSFUSE_PARENT_PROCESS_DIR = "gcsfuse-parent-process-dir"

// 1. Returns the same filepath in case of absolute path or empty filename.
// 2. For relative path with . or .. or nothing, it resolves smartly the path
// in case of child proces it resolves with respect to GCSFUSE_PARENT_PROCESS_DIR
// otherwise with respect to the current working dir.
// 3. For relative path starting with ~, it resolves with respect to home dir.
func getResolvedPath(filePath string) (resolvedPath string, err error) {
	if filePath == "" || path.IsAbs(filePath) {
		resolvedPath = filePath
		return
	}

	// Relative path starting with tilda (~)
	if strings.HasPrefix(filePath, "~/") {
		homeDir, err := os.UserHomeDir()
		if err != nil {
			return "", fmt.Errorf("Fetch home dir: [%w]", err)
		}
		return filepath.Join(homeDir, filePath[2:]), err
	}

	// Means - relative path starting with ., .. or nothing.
	gcsfuseParentProcessDir, _ := os.LookupEnv(GCSFUSE_PARENT_PROCESS_DIR)
	gcsfuseParentProcessDir = strings.TrimSpace(gcsfuseParentProcessDir)
	if gcsfuseParentProcessDir == "" {
		return filepath.Abs(filePath)
	} else {
		return filepath.Join(gcsfuseParentProcessDir, filePath), err
	}
}

// This method resolves path in the context dictionary.
func resolvePathForTheFlagInContext(flagKey string, c *cli.Context) (err error) {
	flagValue := c.String(flagKey)
	resolvedPath, err := getResolvedPath(flagValue)
	if err != nil {
		return
	}
	c.Set(flagKey, resolvedPath)

	if flagValue != resolvedPath {
		logger.Infof("Value of [%s] resolved from [%s] to [%s]\n",
			flagKey, flagValue, resolvedPath)
	}

	return
}

// For parent process: it only resolves the path with respect to home folder.
// For child process (when --foreground flag is disabled): it resolves
// the path relative to both current directory and home directory
func resolvePathForTheRequiredFlagInContext(c *cli.Context) (err error) {
	err = resolvePathForTheFlagInContext("log-file", c)
	if err != nil {
		return fmt.Errorf("resolving for log-file: [%w]", err)
	}

	err = resolvePathForTheFlagInContext("key-file", c)
	if err != nil {
		return fmt.Errorf("resolving for key-file: [%w]", err)
	}

	return
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func populateFlags(c *cli.Context) (flags *flagStorage) {
	endpoint, err := url.Parse(c.String("endpoint"))
	if err != nil {
		fmt.Printf("Could not parse endpoint")
		return nil
	}
	flags = &flagStorage{
		AppName:    c.String("app-name"),
		Foreground: c.Bool("foreground"),

		// File system
		MountOptions:   make(map[string]string),
		DirMode:        os.FileMode(*c.Generic("dir-mode").(*OctalInt)),
		FileMode:       os.FileMode(*c.Generic("file-mode").(*OctalInt)),
		Uid:            int64(c.Int("uid")),
		Gid:            int64(c.Int("gid")),
		ImplicitDirs:   c.Bool("implicit-dirs"),
		OnlyDir:        c.String("only-dir"),
		RenameDirLimit: int64(c.Int("rename-dir-limit")),

		// GCS,
		Endpoint:                           endpoint,
		BillingProject:                     c.String("billing-project"),
		KeyFile:                            c.String("key-file"),
		TokenUrl:                           c.String("token-url"),
		EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"),
		OpRateLimitHz:                      c.Float64("limit-ops-per-sec"),

		// Tuning,
		MaxRetrySleep:     c.Duration("max-retry-sleep"),
		StatCacheCapacity: c.Int("stat-cache-capacity"),
		StatCacheTTL:      c.Duration("stat-cache-ttl"),
		TypeCacheTTL:      c.Duration("type-cache-ttl"),
		LocalFileCache:    c.Bool("experimental-local-file-cache"),
		TempDir:           c.String("temp-dir"),
		DisableHTTP2:      c.Bool("disable-http2"),
		MaxConnsPerHost:   c.Int("max-conns-per-host"),

		// Monitoring & Logging
		StackdriverExportInterval: c.Duration("experimental-stackdriver-export-interval"),
		OtelCollectorAddress:      c.String("experimental-opentelemetry-collector-address"),
		LogFile:                   c.String("log-file"),
		LogFormat:                 c.String("log-format"),

		// Debugging,
		DebugFuse:       c.Bool("debug_fuse"),
		DebugGCS:        c.Bool("debug_gcs"),
		DebugFS:         c.Bool("debug_fs"),
		DebugHTTP:       c.Bool("debug_http"),
		DebugInvariants: c.Bool("debug_invariants"),
		DebugMutex:      c.Bool("debug_mutex"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		mountpkg.ParseOptions(flags.MountOptions, o)
	}

	return
}

// A cli.Generic that can be used with cli.GenericFlag to obtain an int flag
// that is parsed in octal.
type OctalInt int

var _ cli.Generic = (*OctalInt)(nil)

func (oi *OctalInt) Set(value string) (err error) {
	tmp, err := strconv.ParseInt(value, 8, 32)
	if err != nil {
		err = fmt.Errorf("Parsing as octal: %w", err)
		return
	}

	*oi = OctalInt(tmp)
	return
}

func (oi OctalInt) String() string {
	return fmt.Sprintf("%o", oi)
}
<|endoftext|>


<commit_before>package felixcheck

import (
	"fmt"
	"net"
	"strings"
	"time"

	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/streadway/amqp"
	"github.com/tatsushid/go-fastping"
)

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
)

const (
	sysName     = "1.3.6.1.2.1.1.5.0"
	maxPingTime = 4 * time.Second
)

type CheckFunction func() Event
type MultiCheckFunction func() []Event

func (f CheckFunction) Tags(tags ...string) CheckFunction {
	return func() Event {
		result := f()
		result.Tags = tags
		return result
	}
}

func (f CheckFunction) Attributes(attributes map[string]string) CheckFunction {
	return func() Event {
		result := f()
		result.Attributes = attributes
		return result
	}
}

func (f CheckFunction) Ttl(ttl float32) CheckFunction {
	return func() Event {
		result := f()
		result.Ttl = ttl
		return result
	}
}

func NewPingChecker(host, service, ip string) CheckFunction {
	return func() Event {
		var retRtt time.Duration = 0
		var result Event = Event{Host: host, Service: service, State: "critical"}

		p := fastping.NewPinger()
		p.MaxRTT = maxPingTime
		ra, err := net.ResolveIPAddr("ip4:icmp", ip)
		if err != nil {
			result.Description = err.Error()
		}

		p.AddIPAddr(ra)
		p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
			result.State = "ok"
			result.Metric = float32(retRtt.Nanoseconds() / 1e6)
		}

		err = p.Run()
		if err != nil {
			result.Description = err.Error()
		}
		return result
	}
}

type TcpCheckerConf struct {
	retries   int
	timeout   time.Duration
	retrytime time.Duration
}

var DefaultTcpCheckConf = TcpCheckerConf{
	retries:   3,
	timeout:   2 * time.Second,
	retrytime: 1 * time.Second,
}

func NewTcpPortChecker(host, service, ip string, port int, conf TcpCheckerConf) CheckFunction {
	return func() Event {
		var err error
		var conn net.Conn

		for attempt := 0; attempt < conf.retries; attempt++ {
			conn, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, port), conf.timeout)
			if err == nil {
				conn.Close()
				return Event{Host: host, Service: service, State: "ok"}
			}
			time.Sleep(conf.retrytime)
		}
		return Event{Host: host, Service: service, State: "critical"}
	}
}

type ValidateHttpResponseFunction func(resp *http.Response) (state, description string)

func BodyGreaterThan(minLength int) ValidateHttpResponseFunction {
	return func(httpResp *http.Response) (state, description string) {
		if httpResp.StatusCode != 200 {
			return "critical", fmt.Sprintf("Response %d", httpResp.StatusCode)
		}
		if httpResp.Body == nil {
			return "critical", fmt.Sprintf("Empty body")
		}
		body, err := ioutil.ReadAll(httpResp.Body)
		if err != nil {
			return "critical", fmt.Sprintf("Error geting body")
		}
		if len(body) < minLength {
			return "critical", fmt.Sprintf("Obtained %d bytes, expected more than %d", len(body), minLength)
		} else {
			return "ok", ""
		}
		return "critical", "unknown condition"
	}
}

func NewGenericHttpChecker(host, service, url string, validationFunc ValidateHttpResponseFunction) CheckFunction {
	return func() Event {
		var t1 = time.Now()

		response, err := http.Get(url)
		milliseconds := float32((time.Now().Sub(t1)).Nanoseconds() / 1e6)
		result := Event{Host: host, Service: service, State: "critical", Metric: milliseconds}
		if err != nil {
			result.Description = err.Error()
		} else {
			if response.Body != nil {
				defer response.Body.Close()
			}
			result.State, result.Description = validationFunc(response)
		}
		return result
	}
}

func NewHttpChecker(host, service, url string, expectedStatusCode int) CheckFunction {
	return NewGenericHttpChecker(host, service, url,
		func(httpResp *http.Response) (string, string) {
			if httpResp.StatusCode == expectedStatusCode {
				return "ok", ""
			} else {
				return "critical", fmt.Sprintf("Response %d", httpResp.StatusCode)
			}
		})
}

type SnmpCheckerConf struct {
	retries    int
	timeout    time.Duration
	oidToCheck string
}

var DefaultSnmpCheckConf = SnmpCheckerConf{
	retries:    1,
	timeout:    1 * time.Second,
	oidToCheck: sysName,
}

func NewSnmpChecker(host, service, ip, community string, conf SnmpCheckerConf) CheckFunction {
	return func() Event {

		_, err := snmpGet(ip, community, []string{conf.oidToCheck}, conf.timeout, conf.retries)
		if err == nil {
			return Event{Host: host, Service: service, State: "ok", Description: err.Error()}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func NewC4CMTSTempChecker(host, service, ip, community string, maxAllowedTemp int) CheckFunction {
	return func() Event {

		result, err := snmpWalk(ip, community, "1.3.6.1.4.1.4998.1.1.10.1.4.2.1.29", 2*time.Second, 1)

		if err == nil {
			max := 0
			for _, r := range result {
				if r.Value.(int) != 999 && r.Value.(int) > max {
					max = r.Value.(int)
				}
			}
			var state string = "critical"
			if max < maxAllowedTemp {
				state = "ok"
			}
			return Event{Host: host, Service: service, State: state, Metric: float32(max)}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func getMaxValueFromSnmpWalk(oid, ip, community string) (uint, error) {
	result, err := snmpWalk(ip, community, oid, 2*time.Second, 1)
	if err == nil {
		max := uint(0)
		for _, r := range result {
			if r.Value.(uint) > max {
				max = r.Value.(uint)
			}
		}
		return max, nil
	} else {
		return 0, err
	}
}

func NewJuniperTempChecker(host, service, ip, community string, maxAllowedTemp uint) CheckFunction {
	return func() Event {
		max, err := getMaxValueFromSnmpWalk("1.3.6.1.4.1.2636.3.1.13.1.7", ip, community)
		if err == nil {
			var state string = "critical"
			if max < maxAllowedTemp {
				state = "ok"
			}
			return Event{Host: host, Service: service, State: state, Metric: float32(max)}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func NewJuniperCpuChecker(host, service, ip, community string, maxAllowedTemp uint) CheckFunction {
	return func() Event {
		max, err := getMaxValueFromSnmpWalk("1.3.6.1.4.1.2636.3.1.13.1.8", ip, community)
		if err == nil {
			var state string = "critical"
			if max < maxAllowedTemp {
				state = "ok"
			}
			return Event{Host: host, Service: service, State: state, Metric: float32(max)}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func NewRabbitMQQueueLenCheck(host, service, amqpuri, queue string, max int) CheckFunction {
	return func() Event {
		result := Event{Host: host, Service: service}

		conn, err := amqp.Dial(amqpuri)
		if err != nil {
			result.State = "critical"
			result.Description = err.Error()
			return result
		}

		ch, err := conn.Channel()
		if err != nil {
			result.State = "critical"
			result.Description = err.Error()
			return result
		}
		defer ch.Close()
		defer conn.Close()

		queueInfo, err := ch.QueueInspect(queue)
		if err != nil {
			result.State = "critical"
			result.Description = err.Error()
			return result
		}

		var state string = "critical"
		if queueInfo.Messages <= max {
			state = "ok"
		}
		return Event{Host: host, Service: service, State: state, Metric: float32(queueInfo.Messages)}
	}
}

func NewMysqlConnectionCheck(host, service, mysqluri string) CheckFunction {
	return func() Event {
		u, err := url.Parse(mysqluri)
		if err != nil {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}

		if u.User == nil {
			return Event{Host: host, Service: service, State: "critical", Description: "No user defined"}
		}
		password, hasPassword := u.User.Password()
		if !hasPassword {
			return Event{Host: host, Service: service, State: "critical", Description: "No password defined"}
		}
		hostAndPort := u.Host
		if !strings.Contains(hostAndPort, ":") {
			hostAndPort = hostAndPort + ":3306"
		}
		con, err := sql.Open("mysql", u.User.Username()+":"+password+"@"+"tcp("+hostAndPort+")"+u.Path)
		defer con.Close()
		if err != nil {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
		q := `select CURTIME()`
		row := con.QueryRow(q)
		var date string
		err = row.Scan(&date)
		if err != nil {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
		return Event{Host: host, Service: service, State: "ok"}
	}
}

type ObtainMetricFunction func() float32
type CalculateStateFunction func(float32) string

func NewGenericCheck(host, service string, metricFunc ObtainMetricFunction, stateFunc CalculateStateFunction) CheckFunction {
	return func() Event {
		value := metricFunc()
		var state string = stateFunc(value)
		return Event{Host: host, Service: service, State: state, Metric: value}
	}
}
<commit_msg>Added metric to NewTcpPortChecker<commit_after>package felixcheck

import (
	"fmt"
	"net"
	"strings"
	"time"

	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/streadway/amqp"
	"github.com/tatsushid/go-fastping"
)

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
)

const (
	sysName     = "1.3.6.1.2.1.1.5.0"
	maxPingTime = 4 * time.Second
)

type CheckFunction func() Event
type MultiCheckFunction func() []Event

func (f CheckFunction) Tags(tags ...string) CheckFunction {
	return func() Event {
		result := f()
		result.Tags = tags
		return result
	}
}

func (f CheckFunction) Attributes(attributes map[string]string) CheckFunction {
	return func() Event {
		result := f()
		result.Attributes = attributes
		return result
	}
}

func (f CheckFunction) Ttl(ttl float32) CheckFunction {
	return func() Event {
		result := f()
		result.Ttl = ttl
		return result
	}
}

func NewPingChecker(host, service, ip string) CheckFunction {
	return func() Event {
		var retRtt time.Duration = 0
		var result Event = Event{Host: host, Service: service, State: "critical"}

		p := fastping.NewPinger()
		p.MaxRTT = maxPingTime
		ra, err := net.ResolveIPAddr("ip4:icmp", ip)
		if err != nil {
			result.Description = err.Error()
		}

		p.AddIPAddr(ra)
		p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
			result.State = "ok"
			result.Metric = float32(retRtt.Nanoseconds() / 1e6)
		}

		err = p.Run()
		if err != nil {
			result.Description = err.Error()
		}
		return result
	}
}

type TcpCheckerConf struct {
	retries   int
	timeout   time.Duration
	retrytime time.Duration
}

var DefaultTcpCheckConf = TcpCheckerConf{
	retries:   3,
	timeout:   2 * time.Second,
	retrytime: 1 * time.Second,
}

func NewTcpPortChecker(host, service, ip string, port int, conf TcpCheckerConf) CheckFunction {
	return func() Event {
		var err error
		var conn net.Conn

		for attempt := 0; attempt < conf.retries; attempt++ {
			var t1 = time.Now()
			conn, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, port), conf.timeout)
			if err == nil {
				conn.Close()
				milliseconds := float32((time.Now().Sub(t1)).Nanoseconds() / 1e6)
				return Event{Host: host, Service: service, State: "ok", Metric: milliseconds}
			}
			time.Sleep(conf.retrytime)
		}
		return Event{Host: host, Service: service, State: "critical"}
	}
}

type ValidateHttpResponseFunction func(resp *http.Response) (state, description string)

func BodyGreaterThan(minLength int) ValidateHttpResponseFunction {
	return func(httpResp *http.Response) (state, description string) {
		if httpResp.StatusCode != 200 {
			return "critical", fmt.Sprintf("Response %d", httpResp.StatusCode)
		}
		if httpResp.Body == nil {
			return "critical", fmt.Sprintf("Empty body")
		}
		body, err := ioutil.ReadAll(httpResp.Body)
		if err != nil {
			return "critical", fmt.Sprintf("Error geting body")
		}
		if len(body) < minLength {
			return "critical", fmt.Sprintf("Obtained %d bytes, expected more than %d", len(body), minLength)
		} else {
			return "ok", ""
		}
		return "critical", "unknown condition"
	}
}

func NewGenericHttpChecker(host, service, url string, validationFunc ValidateHttpResponseFunction) CheckFunction {
	return func() Event {
		var t1 = time.Now()

		response, err := http.Get(url)
		milliseconds := float32((time.Now().Sub(t1)).Nanoseconds() / 1e6)
		result := Event{Host: host, Service: service, State: "critical", Metric: milliseconds}
		if err != nil {
			result.Description = err.Error()
		} else {
			if response.Body != nil {
				defer response.Body.Close()
			}
			result.State, result.Description = validationFunc(response)
		}
		return result
	}
}

func NewHttpChecker(host, service, url string, expectedStatusCode int) CheckFunction {
	return NewGenericHttpChecker(host, service, url,
		func(httpResp *http.Response) (string, string) {
			if httpResp.StatusCode == expectedStatusCode {
				return "ok", ""
			} else {
				return "critical", fmt.Sprintf("Response %d", httpResp.StatusCode)
			}
		})
}

type SnmpCheckerConf struct {
	retries    int
	timeout    time.Duration
	oidToCheck string
}

var DefaultSnmpCheckConf = SnmpCheckerConf{
	retries:    1,
	timeout:    1 * time.Second,
	oidToCheck: sysName,
}

func NewSnmpChecker(host, service, ip, community string, conf SnmpCheckerConf) CheckFunction {
	return func() Event {

		_, err := snmpGet(ip, community, []string{conf.oidToCheck}, conf.timeout, conf.retries)
		if err == nil {
			return Event{Host: host, Service: service, State: "ok", Description: err.Error()}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func NewC4CMTSTempChecker(host, service, ip, community string, maxAllowedTemp int) CheckFunction {
	return func() Event {

		result, err := snmpWalk(ip, community, "1.3.6.1.4.1.4998.1.1.10.1.4.2.1.29", 2*time.Second, 1)

		if err == nil {
			max := 0
			for _, r := range result {
				if r.Value.(int) != 999 && r.Value.(int) > max {
					max = r.Value.(int)
				}
			}
			var state string = "critical"
			if max < maxAllowedTemp {
				state = "ok"
			}
			return Event{Host: host, Service: service, State: state, Metric: float32(max)}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func getMaxValueFromSnmpWalk(oid, ip, community string) (uint, error) {
	result, err := snmpWalk(ip, community, oid, 2*time.Second, 1)
	if err == nil {
		max := uint(0)
		for _, r := range result {
			if r.Value.(uint) > max {
				max = r.Value.(uint)
			}
		}
		return max, nil
	} else {
		return 0, err
	}
}

func NewJuniperTempChecker(host, service, ip, community string, maxAllowedTemp uint) CheckFunction {
	return func() Event {
		max, err := getMaxValueFromSnmpWalk("1.3.6.1.4.1.2636.3.1.13.1.7", ip, community)
		if err == nil {
			var state string = "critical"
			if max < maxAllowedTemp {
				state = "ok"
			}
			return Event{Host: host, Service: service, State: state, Metric: float32(max)}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func NewJuniperCpuChecker(host, service, ip, community string, maxAllowedTemp uint) CheckFunction {
	return func() Event {
		max, err := getMaxValueFromSnmpWalk("1.3.6.1.4.1.2636.3.1.13.1.8", ip, community)
		if err == nil {
			var state string = "critical"
			if max < maxAllowedTemp {
				state = "ok"
			}
			return Event{Host: host, Service: service, State: state, Metric: float32(max)}
		} else {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
	}
}

func NewRabbitMQQueueLenCheck(host, service, amqpuri, queue string, max int) CheckFunction {
	return func() Event {
		result := Event{Host: host, Service: service}

		conn, err := amqp.Dial(amqpuri)
		if err != nil {
			result.State = "critical"
			result.Description = err.Error()
			return result
		}

		ch, err := conn.Channel()
		if err != nil {
			result.State = "critical"
			result.Description = err.Error()
			return result
		}
		defer ch.Close()
		defer conn.Close()

		queueInfo, err := ch.QueueInspect(queue)
		if err != nil {
			result.State = "critical"
			result.Description = err.Error()
			return result
		}

		var state string = "critical"
		if queueInfo.Messages <= max {
			state = "ok"
		}
		return Event{Host: host, Service: service, State: state, Metric: float32(queueInfo.Messages)}
	}
}

func NewMysqlConnectionCheck(host, service, mysqluri string) CheckFunction {
	return func() Event {
		u, err := url.Parse(mysqluri)
		if err != nil {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}

		if u.User == nil {
			return Event{Host: host, Service: service, State: "critical", Description: "No user defined"}
		}
		password, hasPassword := u.User.Password()
		if !hasPassword {
			return Event{Host: host, Service: service, State: "critical", Description: "No password defined"}
		}
		hostAndPort := u.Host
		if !strings.Contains(hostAndPort, ":") {
			hostAndPort = hostAndPort + ":3306"
		}
		con, err := sql.Open("mysql", u.User.Username()+":"+password+"@"+"tcp("+hostAndPort+")"+u.Path)
		defer con.Close()
		if err != nil {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
		q := `select CURTIME()`
		row := con.QueryRow(q)
		var date string
		err = row.Scan(&date)
		if err != nil {
			return Event{Host: host, Service: service, State: "critical", Description: err.Error()}
		}
		return Event{Host: host, Service: service, State: "ok"}
	}
}

type ObtainMetricFunction func() float32
type CalculateStateFunction func(float32) string

func NewGenericCheck(host, service string, metricFunc ObtainMetricFunction, stateFunc CalculateStateFunction) CheckFunction {
	return func() Event {
		value := metricFunc()
		var state string = stateFunc(value)
		return Event{Host: host, Service: service, State: state, Metric: value}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/fukata/golang-stats-api-handler"
	"github.com/jessevdk/go-flags"
	"github.com/kazeburo/chocon/proxy"
	"github.com/lestrrat/go-apache-logformat"
	"github.com/lestrrat/go-file-rotatelogs"
	"github.com/lestrrat/go-server-starter-listener"
)

var (
	Version string
)

type cmdOpts struct {
	Listen           string `short:"l" long:"listen" default:"0.0.0.0" description:"address to bind"`
	Port             string `short:"p" long:"port" default:"3000" description:"Port number to bind"`
	LogDir           string `long:"access-log-dir" default:"" description:"directory to store logfiles"`
	LogRotate        int64  `long:"access-log-rotate" default:"30" description:"Number of day before remove logs"`
	Version          bool   `short:"v" long:"version" description:"Show version"`
	KeepaliveConns   int    `short:"c" default:"2" long:"keepalive-conns" description:"maximun keepalive connections for upstream"`
	ReadTimeout      int    `long:"read-timeout" default:"30" description:"timeout of reading request"`
	WriteTimeout     int    `long:"write-timeout" default:"90" description:"timeout of writing response"`
	ProxyReadTimeout int    `long:"proxy-read-timeout" default:"60" description:"timeout of reading response from upstream"`
}

func addStatsHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.Index(r.URL.Path, "/.api/stats") == 0 {
			stats_api.Handler(w, r)
		} else {
			h.ServeHTTP(w, r)
		}
	})
}

func addLogHandler(h http.Handler, log_dir string, log_rotate int64) http.Server {
	apache_log, err := apachelog.New(`%h %l %u %t "%r" %>s %b "%v" %T.%{msec_frac}t %{X-Chocon-Req}i`)
	if err != nil {
		panic(fmt.Sprintf("could not create logger: %v", err))
	}

	if log_dir == "stdout" {
		return http.Server{
			Handler: apache_log.Wrap(h, os.Stdout),
		}
	} else if log_dir == "" {
		return http.Server{
			Handler: apache_log.Wrap(h, os.Stderr),
		}
	} else if log_dir == "none" {
		return http.Server{
			Handler: h,
		}
	}

	log_file := log_dir
	link_name := log_dir
	if !strings.HasSuffix(log_dir, "/") {
		log_file += "/"
		link_name += "/"

	}
	log_file += "access_log.%Y%m%d%H%M"
	link_name += "current"

	rl, err := rotatelogs.New(
		log_file,
		rotatelogs.WithLinkName(link_name),
		rotatelogs.WithMaxAge(time.Duration(log_rotate)*86400*time.Second),
		rotatelogs.WithRotationTime(time.Second*86400),
	)
	if err != nil {
		panic(fmt.Sprintf("rotatelogs.New failed: %v", err))
	}

	return http.Server{
		Handler: apache_log.Wrap(h, rl),
	}
}

func main() {
	opts := cmdOpts{}
	psr := flags.NewParser(&opts, flags.Default)
	_, err := psr.Parse()
	if err != nil {
		os.Exit(1)
	}

	if opts.Version {
		fmt.Printf(`chocon %s
Compiler: %s %s
`,
			Version,
			runtime.Compiler,
			runtime.Version())
		return

	}

	requestConverter := func(r *http.Request, pr *http.Request, ps *proxy_handler.ProxyStatus) {
		if r.Host == "" {
			ps.Status = http.StatusBadRequest
			return
		}
		originalHost := r.Host
		hostSplited := strings.Split(originalHost, ".")
		lastPartIndex := 0
		for i, hostPart := range hostSplited {
			if hostPart == "ccnproxy-ssl" || hostPart == "ccnproxy-secure" || hostPart == "ccnproxy-https" || hostPart == "ccnproxy" {
				lastPartIndex = i
			}
		}
		if lastPartIndex == 0 {
			ps.Status = http.StatusBadRequest
			return
		}

		pr.URL.Host = strings.Join(hostSplited[0:lastPartIndex], ".")
		pr.Host = pr.URL.Host
		if hostSplited[lastPartIndex] == "ccnproxy-https" || hostSplited[lastPartIndex] == "ccnproxy-secure" || hostSplited[lastPartIndex] == "ccnproxy-ssl" {
			pr.URL.Scheme = "https"
		}
	}

	var transport http.RoundTripper = &http.Transport{
		// inherited http.DefaultTransport
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		// self-customized values
		MaxIdleConnsPerHost:   opts.KeepaliveConns,
		ResponseHeaderTimeout: time.Duration(opts.ProxyReadTimeout) * time.Second,
	}

	proxyHandler := addStatsHandler(proxy_handler.NewProxyWithRequestConverter(requestConverter, &transport))

	l, err := ss.NewListener()
	if l == nil || err != nil {
		// Fallback if not running under Server::Starter
		l, err = net.Listen("tcp", fmt.Sprintf("%s:%s", opts.Listen, opts.Port))
		if err != nil {
			panic(fmt.Sprintf("Failed to listen to port %s:%s", opts.Listen, opts.Port))
		}
	}

	server := addLogHandler(proxyHandler, opts.LogDir, opts.LogRotate)
	server.ReadTimeout = time.Duration(opts.ReadTimeout) * time.Second
	server.WriteTimeout = time.Duration(opts.WriteTimeout) * time.Second
	server.Serve(l)
}
<commit_msg>split host and port<commit_after>package main

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/fukata/golang-stats-api-handler"
	"github.com/jessevdk/go-flags"
	"github.com/kazeburo/chocon/proxy"
	"github.com/lestrrat/go-apache-logformat"
	"github.com/lestrrat/go-file-rotatelogs"
	"github.com/lestrrat/go-server-starter-listener"
)

var (
	Version string
)

type cmdOpts struct {
	Listen           string `short:"l" long:"listen" default:"0.0.0.0" description:"address to bind"`
	Port             string `short:"p" long:"port" default:"3000" description:"Port number to bind"`
	LogDir           string `long:"access-log-dir" default:"" description:"directory to store logfiles"`
	LogRotate        int64  `long:"access-log-rotate" default:"30" description:"Number of day before remove logs"`
	Version          bool   `short:"v" long:"version" description:"Show version"`
	KeepaliveConns   int    `short:"c" default:"2" long:"keepalive-conns" description:"maximun keepalive connections for upstream"`
	ReadTimeout      int    `long:"read-timeout" default:"30" description:"timeout of reading request"`
	WriteTimeout     int    `long:"write-timeout" default:"90" description:"timeout of writing response"`
	ProxyReadTimeout int    `long:"proxy-read-timeout" default:"60" description:"timeout of reading response from upstream"`
}

func addStatsHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if strings.Index(r.URL.Path, "/.api/stats") == 0 {
			stats_api.Handler(w, r)
		} else {
			h.ServeHTTP(w, r)
		}
	})
}

func addLogHandler(h http.Handler, log_dir string, log_rotate int64) http.Server {
	apache_log, err := apachelog.New(`%h %l %u %t "%r" %>s %b "%v" %T.%{msec_frac}t %{X-Chocon-Req}i`)
	if err != nil {
		panic(fmt.Sprintf("could not create logger: %v", err))
	}

	if log_dir == "stdout" {
		return http.Server{
			Handler: apache_log.Wrap(h, os.Stdout),
		}
	} else if log_dir == "" {
		return http.Server{
			Handler: apache_log.Wrap(h, os.Stderr),
		}
	} else if log_dir == "none" {
		return http.Server{
			Handler: h,
		}
	}

	log_file := log_dir
	link_name := log_dir
	if !strings.HasSuffix(log_dir, "/") {
		log_file += "/"
		link_name += "/"

	}
	log_file += "access_log.%Y%m%d%H%M"
	link_name += "current"

	rl, err := rotatelogs.New(
		log_file,
		rotatelogs.WithLinkName(link_name),
		rotatelogs.WithMaxAge(time.Duration(log_rotate)*86400*time.Second),
		rotatelogs.WithRotationTime(time.Second*86400),
	)
	if err != nil {
		panic(fmt.Sprintf("rotatelogs.New failed: %v", err))
	}

	return http.Server{
		Handler: apache_log.Wrap(h, rl),
	}
}

func main() {
	opts := cmdOpts{}
	psr := flags.NewParser(&opts, flags.Default)
	_, err := psr.Parse()
	if err != nil {
		os.Exit(1)
	}

	if opts.Version {
		fmt.Printf(`chocon %s
Compiler: %s %s
`,
			Version,
			runtime.Compiler,
			runtime.Version())
		return

	}

	requestConverter := func(r *http.Request, pr *http.Request, ps *proxy_handler.ProxyStatus) {
		if r.Host == "" {
			ps.Status = http.StatusBadRequest
			return
		}
		host, _, err := net.SplitHostPort(r.Host)
		if err != nil {
			ps.Status = http.StatusBadRequest
			return
		}
		hostSplited := strings.Split(host, ".")
		lastPartIndex := 0
		for i, hostPart := range hostSplited {
			if hostPart == "ccnproxy-ssl" || hostPart == "ccnproxy-secure" || hostPart == "ccnproxy-https" || hostPart == "ccnproxy" {
				lastPartIndex = i
			}
		}
		if lastPartIndex == 0 {
			ps.Status = http.StatusBadRequest
			return
		}

		pr.URL.Host = strings.Join(hostSplited[0:lastPartIndex], ".")
		pr.Host = pr.URL.Host
		if hostSplited[lastPartIndex] == "ccnproxy-https" || hostSplited[lastPartIndex] == "ccnproxy-secure" || hostSplited[lastPartIndex] == "ccnproxy-ssl" {
			pr.URL.Scheme = "https"
		}
	}

	var transport http.RoundTripper = &http.Transport{
		// inherited http.DefaultTransport
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		// self-customized values
		MaxIdleConnsPerHost:   opts.KeepaliveConns,
		ResponseHeaderTimeout: time.Duration(opts.ProxyReadTimeout) * time.Second,
	}

	proxyHandler := addStatsHandler(proxy_handler.NewProxyWithRequestConverter(requestConverter, &transport))

	l, err := ss.NewListener()
	if l == nil || err != nil {
		// Fallback if not running under Server::Starter
		l, err = net.Listen("tcp", fmt.Sprintf("%s:%s", opts.Listen, opts.Port))
		if err != nil {
			panic(fmt.Sprintf("Failed to listen to port %s:%s", opts.Listen, opts.Port))
		}
	}

	server := addLogHandler(proxyHandler, opts.LogDir, opts.LogRotate)
	server.ReadTimeout = time.Duration(opts.ReadTimeout) * time.Second
	server.WriteTimeout = time.Duration(opts.WriteTimeout) * time.Second
	server.Serve(l)
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"time"

	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sTunnelTest", func() {

	var kubectl *helpers.Kubectl
	var demoDSPath string
	var initialized bool
	var logger *logrus.Entry

	initialize := func() {
		if initialized == true {
			return
		}
		logger = log.WithFields(logrus.Fields{"testName": "K8sTunnelTest"})
		logger.Info("Starting")

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		demoDSPath = kubectl.ManifestGet("demo_ds.yaml")
		kubectl.Exec("kubectl -n kube-system delete ds cilium")
		// Expect(res.Correct()).Should(BeTrue())

		waitToDeleteCilium(kubectl, logger)
		initialized = true
	}

	BeforeEach(func() {
		initialize()
		kubectl.NodeCleanMetadata()
		kubectl.Apply(demoDSPath)
	}, 600)

	AfterEach(func() {
		kubectl.Delete(demoDSPath)
	})

	It("Check VXLAN mode", func() {
		path := kubectl.ManifestGet("cilium_ds.yaml")
		kubectl.Apply(path)
		_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 5000)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
		Expect(err).Should(BeNil())

		_, err = kubectl.CiliumNodesWait()
		Expect(err).Should(BeNil())
		//Make sure that we delete the ds in case of fail
		defer kubectl.Delete(path)

		By("Checking that BPF tunnels are in place")
		status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
		if tunnels, _ := status.IntOutput(); tunnels != 4 {
			cmds := []string{
				"cilium bpf tunnel list",
			}
			kubectl.CiliumReport(helpers.KubeSystemNamespace, ciliumPod, cmds)
		}

		status.ExpectSuccess()
		Expect(status.IntOutput()).Should(Equal(4))

		By("Checking that BPF tunnels are working correctly")
		tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
		Expect(tunnStatus).Should(BeTrue())
		kubectl.Delete(path)
		waitToDeleteCilium(kubectl, logger)
	}, 600)

	It("Check Geneve mode", func() {
		path := kubectl.ManifestGet("cilium_ds_geneve.yaml")
		kubectl.Apply(path)
		_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 5000)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
		Expect(err).Should(BeNil())

		_, err = kubectl.CiliumNodesWait()
		Expect(err).Should(BeNil())

		//Make sure that we delete the ds in case of fail
		defer kubectl.Delete(path)

		//Check that cilium detects a
		By("Checking that BPF tunnels are in place")
		status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
		status.ExpectSuccess()
		if tunnels, _ := status.IntOutput(); tunnels != 4 {
			cmds := []string{
				"cilium bpf tunnel list",
			}
			kubectl.CiliumReport(helpers.KubeSystemNamespace, ciliumPod, cmds)
		}
		Expect(status.IntOutput()).Should(Equal(4))

		By("Checking that BPF tunnels are working correctly")
		tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
		Expect(tunnStatus).Should(BeTrue())
		//FIXME: Maybe added here a cilium bpf tunnel status?
		kubectl.Delete(path)
		waitToDeleteCilium(kubectl, logger)
	}, 600)
})

func isNodeNetworkingWorking(kubectl *helpers.Kubectl, filter string) bool {
	waitReady, _ := kubectl.WaitforPods(helpers.DefaultNamespace, fmt.Sprintf("-l %s", filter), 3000)
	Expect(waitReady).Should(BeTrue())
	pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, filter)
	Expect(err).Should(BeNil())
	podIP, err := kubectl.Get(
		helpers.DefaultNamespace,
		fmt.Sprintf("pod %s -o json", pods[1])).Filter("{.status.podIP}")
	Expect(err).Should(BeNil())
	_, err = kubectl.ExecPodCmd(helpers.DefaultNamespace, pods[0], helpers.Ping(podIP.String()))
	if err != nil {
		return false
	}
	return true
}

func waitToDeleteCilium(kubectl *helpers.Kubectl, logger *logrus.Entry) {
	status := 1
	for status > 0 {
		pods, err := kubectl.GetCiliumPods(helpers.KubeSystemNamespace)
		status := len(pods)
		logger.Infof("Cilium pods terminating '%d' err='%v' pods='%v'", status, err, pods)
		if status == 0 {
			return
		}
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>Ginkgo: Fix timeout on K8sTunnelTest<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"time"

	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/sirupsen/logrus"
)

var _ = Describe("K8sTunnelTest", func() {

	var kubectl *helpers.Kubectl
	var demoDSPath string
	var initialized bool
	var logger *logrus.Entry

	initialize := func() {
		if initialized == true {
			return
		}
		logger = log.WithFields(logrus.Fields{"testName": "K8sTunnelTest"})
		logger.Info("Starting")

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)
		demoDSPath = kubectl.ManifestGet("demo_ds.yaml")
		kubectl.Exec("kubectl -n kube-system delete ds cilium")
		// Expect(res.Correct()).Should(BeTrue())

		waitToDeleteCilium(kubectl, logger)
		initialized = true
	}

	BeforeEach(func() {
		initialize()
		kubectl.NodeCleanMetadata()
		kubectl.Apply(demoDSPath)
	}, 600)

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			ciliumPod, _ := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
			kubectl.CiliumReport(helpers.KubeSystemNamespace, ciliumPod, []string{
				"cilium bpf tunnel list",
				"cilium endpoint list"})
		}

		kubectl.Delete(demoDSPath)
	})

	It("Check VXLAN mode", func() {
		path := kubectl.ManifestGet("cilium_ds.yaml")
		kubectl.Apply(path)
		_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 500)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
		Expect(err).Should(BeNil())

		_, err = kubectl.CiliumNodesWait()
		Expect(err).Should(BeNil())
		//Make sure that we delete the ds in case of fail
		defer kubectl.Delete(path)

		By("Checking that BPF tunnels are in place")
		status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
		if tunnels, _ := status.IntOutput(); tunnels != 4 {
			cmds := []string{
				"cilium bpf tunnel list",
			}
			kubectl.CiliumReport(helpers.KubeSystemNamespace, ciliumPod, cmds)
		}

		status.ExpectSuccess()
		Expect(status.IntOutput()).Should(Equal(4))

		By("Checking that BPF tunnels are working correctly")
		tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
		Expect(tunnStatus).Should(BeTrue())
		kubectl.Delete(path)
		waitToDeleteCilium(kubectl, logger)
	}, 600)

	It("Check Geneve mode", func() {
		path := kubectl.ManifestGet("cilium_ds_geneve.yaml")
		kubectl.Apply(path)
		_, err := kubectl.WaitforPods(helpers.KubeSystemNamespace, "-l k8s-app=cilium", 500)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode(helpers.KubeSystemNamespace, helpers.K8s1)
		Expect(err).Should(BeNil())

		_, err = kubectl.CiliumNodesWait()
		Expect(err).Should(BeNil())

		//Make sure that we delete the ds in case of fail
		defer kubectl.Delete(path)

		//Check that cilium detects a
		By("Checking that BPF tunnels are in place")
		status := kubectl.CiliumExec(ciliumPod, "cilium bpf tunnel list | wc -l")
		status.ExpectSuccess()
		if tunnels, _ := status.IntOutput(); tunnels != 4 {
			cmds := []string{
				"cilium bpf tunnel list",
			}
			kubectl.CiliumReport(helpers.KubeSystemNamespace, ciliumPod, cmds)
		}
		Expect(status.IntOutput()).Should(Equal(4))

		By("Checking that BPF tunnels are working correctly")
		tunnStatus := isNodeNetworkingWorking(kubectl, "zgroup=testDS")
		Expect(tunnStatus).Should(BeTrue())
		//FIXME: Maybe added here a cilium bpf tunnel status?
		kubectl.Delete(path)
		waitToDeleteCilium(kubectl, logger)
	}, 600)
})

func isNodeNetworkingWorking(kubectl *helpers.Kubectl, filter string) bool {
	waitReady, _ := kubectl.WaitforPods(helpers.DefaultNamespace, fmt.Sprintf("-l %s", filter), 3000)
	Expect(waitReady).Should(BeTrue())
	pods, err := kubectl.GetPodNames(helpers.DefaultNamespace, filter)
	Expect(err).Should(BeNil())
	podIP, err := kubectl.Get(
		helpers.DefaultNamespace,
		fmt.Sprintf("pod %s -o json", pods[1])).Filter("{.status.podIP}")
	Expect(err).Should(BeNil())
	_, err = kubectl.ExecPodCmd(helpers.DefaultNamespace, pods[0], helpers.Ping(podIP.String()))
	if err != nil {
		return false
	}
	return true
}

func waitToDeleteCilium(kubectl *helpers.Kubectl, logger *logrus.Entry) {
	status := 1
	for status > 0 {
		pods, err := kubectl.GetCiliumPods(helpers.KubeSystemNamespace)
		status := len(pods)
		logger.Infof("Cilium pods terminating '%d' err='%v' pods='%v'", status, err, pods)
		if status == 0 {
			return
		}
		time.Sleep(1 * time.Second)
	}
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2014 MessageBird B.V.
// All rights reserved.
//
// Author: Maurice Nonnekes <maurice@messagebird.com>

// Package messagebird is an official library for interacting with MessageBird.com API.
// The MessageBird API connects your website or application to operators around the world. With our API you can integrate SMS, Chat & Voice.
// More documentation you can find on the MessageBird developers portal: https://developers.messagebird.com/
package messagebird

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"time"
)

const (
	// ClientVersion is used in User-Agent request header to provide server with API level.
	ClientVersion = "5.0.0"

	// Endpoint points you to MessageBird REST API.
	Endpoint = "https://rest.messagebird.com"

	// httpClientTimeout is used to limit http.Client waiting time.
	httpClientTimeout = 15 * time.Second
)

var (
	// ErrUnexpectedResponse is used when there was an internal server error and nothing can be done at this point.
	ErrUnexpectedResponse = errors.New("The MessageBird API is currently unavailable")
)

// Client is used to access API with a given key.
// Uses standard lib HTTP client internally, so should be reused instead of created as needed and it is safe for concurrent use.
type Client struct {
	AccessKey  string       // The API access key
	HTTPClient *http.Client // The HTTP client to send requests on
	DebugLog   *log.Logger  // Optional logger for debugging purposes
}

// New creates a new MessageBird client object.
func New(accessKey string) *Client {
	return &Client{
		AccessKey: accessKey,
		HTTPClient: &http.Client{
			Timeout: httpClientTimeout,
		},
	}
}

// Request is for internal use only and unstable.
func (c *Client) Request(v interface{}, method, path string, data interface{}) error {
	if !strings.HasPrefix(path, "https://") && !strings.HasPrefix(path, "http://") {
		path = fmt.Sprintf("%s/%s", Endpoint, path)
	}
	uri, err := url.Parse(path)
	if err != nil {
		return err
	}

	var jsonEncoded []byte
	if data != nil {
		jsonEncoded, err = json.Marshal(data)
		if err != nil {
			return err
		}
	}

	request, err := http.NewRequest(method, uri.String(), bytes.NewBuffer(jsonEncoded))
	if err != nil {
		return err
	}

	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Accept", "application/json")
	request.Header.Set("Authorization", "AccessKey "+c.AccessKey)
	request.Header.Set("User-Agent", "MessageBird/ApiClient/"+ClientVersion+" Go/"+runtime.Version())

	if c.DebugLog != nil {
		if data != nil {
			c.DebugLog.Printf("HTTP REQUEST: %s %s %s", method, uri.String(), jsonEncoded)
		} else {
			c.DebugLog.Printf("HTTP REQUEST: %s %s", method, uri.String())
		}
	}

	response, err := c.HTTPClient.Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	responseBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	if c.DebugLog != nil {
		c.DebugLog.Printf("HTTP RESPONSE: %s", string(responseBody))
	}

	// Status code 500 is a server error and means nothing can be done at this
	// point.
	if response.StatusCode == http.StatusInternalServerError {
		return ErrUnexpectedResponse
	}

	// Status code 204 is returned for successful DELETE requests. Don't try to
	// unmarshal the body: that would return errors.
	if response.StatusCode == http.StatusNoContent && response.ContentLength == 0 {
		return nil
	}

	// Status codes 200 and 201 are indicative of being able to convert the
	// response body to the struct that was specified.
	if response.StatusCode == http.StatusOK || response.StatusCode == http.StatusCreated {
		if err := json.Unmarshal(responseBody, &v); err != nil {
			return fmt.Errorf("could not decode response JSON, %s: %v", string(responseBody), err)
		}
		return nil
	}

	// Anything else than a 200/201/500 should be a JSON error.
	var errorResponse ErrorResponse
	if err := json.Unmarshal(responseBody, &errorResponse); err != nil {
		return err
	}

	return errorResponse
}
<commit_msg>Use switch to determine response errors, in favour of if<commit_after>//
// Copyright (c) 2014 MessageBird B.V.
// All rights reserved.
//
// Author: Maurice Nonnekes <maurice@messagebird.com>

// Package messagebird is an official library for interacting with MessageBird.com API.
// The MessageBird API connects your website or application to operators around the world. With our API you can integrate SMS, Chat & Voice.
// More documentation you can find on the MessageBird developers portal: https://developers.messagebird.com/
package messagebird

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"time"
)

const (
	// ClientVersion is used in User-Agent request header to provide server with API level.
	ClientVersion = "5.0.0"

	// Endpoint points you to MessageBird REST API.
	Endpoint = "https://rest.messagebird.com"

	// httpClientTimeout is used to limit http.Client waiting time.
	httpClientTimeout = 15 * time.Second
)

var (
	// ErrUnexpectedResponse is used when there was an internal server error and nothing can be done at this point.
	ErrUnexpectedResponse = errors.New("The MessageBird API is currently unavailable")
)

// Client is used to access API with a given key.
// Uses standard lib HTTP client internally, so should be reused instead of created as needed and it is safe for concurrent use.
type Client struct {
	AccessKey  string       // The API access key
	HTTPClient *http.Client // The HTTP client to send requests on
	DebugLog   *log.Logger  // Optional logger for debugging purposes
}

// New creates a new MessageBird client object.
func New(accessKey string) *Client {
	return &Client{
		AccessKey: accessKey,
		HTTPClient: &http.Client{
			Timeout: httpClientTimeout,
		},
	}
}

// Request is for internal use only and unstable.
func (c *Client) Request(v interface{}, method, path string, data interface{}) error {
	if !strings.HasPrefix(path, "https://") && !strings.HasPrefix(path, "http://") {
		path = fmt.Sprintf("%s/%s", Endpoint, path)
	}
	uri, err := url.Parse(path)
	if err != nil {
		return err
	}

	var jsonEncoded []byte
	if data != nil {
		jsonEncoded, err = json.Marshal(data)
		if err != nil {
			return err
		}
	}

	request, err := http.NewRequest(method, uri.String(), bytes.NewBuffer(jsonEncoded))
	if err != nil {
		return err
	}

	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Accept", "application/json")
	request.Header.Set("Authorization", "AccessKey "+c.AccessKey)
	request.Header.Set("User-Agent", "MessageBird/ApiClient/"+ClientVersion+" Go/"+runtime.Version())

	if c.DebugLog != nil {
		if data != nil {
			c.DebugLog.Printf("HTTP REQUEST: %s %s %s", method, uri.String(), jsonEncoded)
		} else {
			c.DebugLog.Printf("HTTP REQUEST: %s %s", method, uri.String())
		}
	}

	response, err := c.HTTPClient.Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	responseBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	if c.DebugLog != nil {
		c.DebugLog.Printf("HTTP RESPONSE: %s", string(responseBody))
	}

	switch response.StatusCode {
	case http.StatusOK, http.StatusCreated:
		// Status codes 200 and 201 are indicative of being able to convert the
		// response body to the struct that was specified.
		if err := json.Unmarshal(responseBody, &v); err != nil {
			return fmt.Errorf("could not decode response JSON, %s: %v", string(responseBody), err)
		}
		return nil
	case http.StatusNoContent:
		// Status code 204 is returned for successful DELETE requests. Don't try to
		// unmarshal the body: that would return errors.
		return nil
	case http.StatusInternalServerError:
		// Status code 500 is a server error and means nothing can be done at this
		// point.
		return ErrUnexpectedResponse
	default:
		// Anything else than a 200/201/204/500 should be a JSON error.
		var errorResponse ErrorResponse
		if err := json.Unmarshal(responseBody, &errorResponse); err != nil {
			return err
		}

		return errorResponse
	}
}
<|endoftext|>


<commit_before>package sse

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"
)

type state uint8

const (
	stateConnecting state = 0
	stateOpen       state = 1
	stateClosed     state = 2
)

// Client is main struct that handles connecting/streaming/etc...
type Client struct {
	cl       *http.Client
	req      *http.Request
	r        io.ReadCloser
	last     string
	retry    time.Duration
	stopChan chan struct{}
	event    chan Event

	add       chan listener
	listeners map[string]func(Event)

	sync.RWMutex
	readyState state
}

type listener struct {
	name string
	f    func(Event)
}

// Event is SSE event data represenation
type Event struct {
	ID    string
	Event string
	Data  string
}

// Config is a struct used to define and override default parameters:
//   Client - *http.Client, if non provided, http.DefaultClient will be used.
//   URL    - URL of SSE stream to connect to. Must be provided. No default
//            value.
//   Retry  - time.Duration of how long should SSE client wait before trying
//            to reconnect after disconnection. Default is 2 seconds.
type Config struct {
	Client *http.Client
	URL    string
	Retry  time.Duration
}

// New creates a client based on a passed Config.
func New(cfg *Config) (*Client, error) {
	if cfg.URL == "" {
		return nil, errors.New("sse: URL config option MUST be provided")
	}
	req, err := http.NewRequest("GET", cfg.URL, nil)
	if err != nil {
		return nil, fmt.Errorf("sse: could not make request to %s: %v", cfg.URL, err)
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	retry := 2 * time.Second
	if cfg.Retry != 0 {
		retry = cfg.Retry
	}
	client := &Client{
		readyState: stateConnecting,
		req:        req,
		retry:      retry,
		event:      make(chan Event),
		add:        make(chan listener),
		stopChan:   make(chan struct{}),
		listeners:  make(map[string]func(Event)),
	}
	client.cl = http.DefaultClient
	if cfg.Client != nil {
		client.cl = cfg.Client
	}
	go client.run()
	//go client.read()
	return client, nil
}

// Connect connects to given SSE endpoint and starts reading the stream and
// transmitting events.
func (c *Client) Connect() {
	go c.connect()
}

// AddListener adds a listener for a given event type. A listener is simple
// callback function that passes Event struct.
func (c *Client) AddListener(event string, f func(Event)) {
	c.add <- listener{
		name: event,
		f:    f,
	}
}

// Stop stops the client of accepting any more stream requests.
func (c *Client) Stop() {
	c.stop()
}

func (c *Client) stop() {
	close(c.stopChan)
	c.Lock()
	c.readyState = stateClosed
	c.Unlock()
}

func (c *Client) run() {
	for {
		select {
		case l := <-c.add:
			c.listeners[l.name] = l.f
		case event := <-c.event:
			f, ok := c.listeners[event.Event]
			if !ok {
				continue
			}
			f(event)
		case <-c.stopChan:
			return
		}
	}
}

func (c *Client) read() {
	c.RLock()
	state := c.readyState
	c.RUnlock()
	switch state {
	case stateOpen:
	case stateConnecting:
		return
	case stateClosed:
		return
	default:
		return
	}
	c.decode()
}

func (c *Client) decode() {
	defer c.r.Close()
	dec := bufio.NewReader(c.r)
	_, err := dec.Peek(1)
	if err == io.ErrUnexpectedEOF {
		err = io.EOF
	}
	if err != nil {
		c.fireErrorAndRecover(err)
		return
	}
	for {
		event := new(Event)
		event.Event = "message"
		for {
			line, err := dec.ReadString('\n')
			if err != nil {
				c.fireErrorAndRecover(err)
				return
			}
			if line == "\n" {
				break
			}
			line = strings.TrimSuffix(line, "\n")
			if strings.HasPrefix(line, ":") {
				continue
			}
			sections := strings.SplitN(line, ":", 2)
			field, value := sections[0], ""
			if len(sections) == 2 {
				value = strings.TrimPrefix(sections[1], " ")
			}
			switch field {
			case "event":
				event.Event = value
			case "data":
				event.Data += value + "\n"
			case "id":
				event.ID = value
			}
		}
		event.Data = strings.TrimSuffix(event.Data, "\n")
		c.event <- *event
	}
}

func (c *Client) connect() {
	c.req.Header.Set("Last-Event-ID", c.last)
	resp, err := c.cl.Do(c.req)
	if err != nil {
		go c.connect()
		return
	}
	// TODO: check other status codes
	switch resp.StatusCode {
	case http.StatusOK:
		// TODO: check content-type
	case http.StatusNoContent:
		c.stop()
		return
	case http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
		c.reconnect()
		return
	default:
		c.stop()
		return
	}
	c.r = resp.Body
	c.fireOpen()
}

func (c *Client) reconnect() {
	c.Lock()
	c.readyState = stateConnecting
	c.Unlock()
	time.Sleep(c.retry)
	// TODO: also implement exponential backoff delay
	go c.connect()
}

func (c *Client) fireOpen() {
	c.Lock()
	c.readyState = stateOpen
	c.Unlock()
	go c.read()
	event := new(Event)
	event.Event = "open"
	c.event <- *event
}

func (c *Client) fireErrorAndRecover(err error) {
	c.reconnect()
	event := new(Event)
	event.Event = "error"
	event.Data = err.Error()
	c.event <- *event
}
<commit_msg>Set last event ID before processing event; Other small tweaks<commit_after>package sse

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"
)

type state uint8

const (
	stateConnecting state = 0
	stateOpen       state = 1
	stateClosed     state = 2
)

// Client is main struct that handles connecting/streaming/etc...
type Client struct {
	cl       *http.Client
	req      *http.Request
	r        io.ReadCloser
	last     string
	retry    time.Duration
	stopChan chan struct{}
	event    chan Event

	add       chan listener
	listeners map[string]func(Event)

	sync.RWMutex
	readyState state
}

type listener struct {
	name string
	f    func(Event)
}

// Event is SSE event data represenation
type Event struct {
	ID    string
	Event string
	Data  string
}

// Config is a struct used to define and override default parameters:
//   Client - *http.Client, if non provided, http.DefaultClient will be used.
//   URL    - URL of SSE stream to connect to. Must be provided. No default
//            value.
//   Retry  - time.Duration of how long should SSE client wait before trying
//            to reconnect after disconnection. Default is 2 seconds.
type Config struct {
	Client *http.Client
	URL    string
	Retry  time.Duration
}

// New creates a client based on a passed Config.
func New(cfg *Config) (*Client, error) {
	if cfg.URL == "" {
		return nil, errors.New("sse: URL config option MUST be provided")
	}
	req, err := http.NewRequest("GET", cfg.URL, nil)
	if err != nil {
		return nil, fmt.Errorf("sse: could not make request to %s: %v", cfg.URL, err)
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	retry := 2 * time.Second
	if cfg.Retry != 0 {
		retry = cfg.Retry
	}
	client := &Client{
		readyState: stateConnecting,
		req:        req,
		retry:      retry,
		event:      make(chan Event),
		add:        make(chan listener),
		stopChan:   make(chan struct{}),
		listeners:  make(map[string]func(Event)),
	}
	client.cl = http.DefaultClient
	if cfg.Client != nil {
		client.cl = cfg.Client
	}
	go client.run()
	return client, nil
}

// Connect connects to given SSE endpoint and starts reading the stream and
// transmitting events.
func (c *Client) Connect() {
	go c.connect()
}

// AddListener adds a listener for a given event type. A listener is simple
// callback function that passes Event struct.
func (c *Client) AddListener(event string, f func(Event)) {
	c.add <- listener{
		name: event,
		f:    f,
	}
}

// Stop stops the client of accepting any more stream requests.
func (c *Client) Stop() {
	c.stop()
}

func (c *Client) stop() {
	close(c.stopChan)
	c.Lock()
	c.readyState = stateClosed
	c.Unlock()
}

func (c *Client) run() {
	for {
		select {
		case l := <-c.add:
			c.listeners[l.name] = l.f
		case event := <-c.event:
			f, ok := c.listeners[event.Event]
			if !ok {
				continue
			}
			if event.ID != "" {
				c.last = event.ID
			}
			f(event)
		case <-c.stopChan:
			return
		}
	}
}

func (c *Client) read() {
	c.RLock()
	state := c.readyState
	c.RUnlock()
	switch state {
	case stateOpen:
	case stateConnecting:
		return
	case stateClosed:
		return
	default:
		return
	}
	c.decode()
}

func (c *Client) decode() {
	defer c.r.Close()
	dec := bufio.NewReader(c.r)
	_, err := dec.Peek(1)
	if err == io.ErrUnexpectedEOF {
		err = io.EOF
	}
	if err != nil {
		c.fireErrorAndRecover(err)
		return
	}
	for {
		event := new(Event)
		event.Event = "message"
		for {
			line, err := dec.ReadString('\n')
			if err != nil {
				c.fireErrorAndRecover(err)
				return
			}
			if line == "\n" {
				break
			}
			line = strings.TrimSuffix(line, "\n")
			if strings.HasPrefix(line, ":") {
				continue
			}
			sections := strings.SplitN(line, ":", 2)
			field, value := sections[0], ""
			if len(sections) == 2 {
				value = strings.TrimPrefix(sections[1], " ")
			}
			switch field {
			case "event":
				event.Event = value
			case "data":
				event.Data += value + "\n"
			case "id":
				event.ID = value
			}
		}
		event.Data = strings.TrimSuffix(event.Data, "\n")
		c.event <- *event
	}
}

func (c *Client) connect() {
	c.req.Header.Set("Last-Event-ID", c.last)
	resp, err := c.cl.Do(c.req)
	if err != nil {
		go c.connect()
		return
	}
	// TODO: check other status codes
	switch resp.StatusCode {
	case http.StatusOK:
		// TODO: check content-type
	case http.StatusNoContent:
		c.stop()
		return
	case http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
		c.reconnect()
		return
	default:
		c.stop()
		return
	}
	c.r = resp.Body
	c.fireOpen()
}

func (c *Client) reconnect() {
	c.Lock()
	c.readyState = stateConnecting
	c.Unlock()
	time.Sleep(c.retry)
	// TODO: also implement exponential backoff delay
	go c.connect()
}

func (c *Client) fireOpen() {
	c.Lock()
	c.readyState = stateOpen
	c.Unlock()
	go c.read()
	event := new(Event)
	event.Event = "open"
	c.event <- *event
}

func (c *Client) fireErrorAndRecover(err error) {
	event := new(Event)
	event.Event = "error"
	event.Data = err.Error()
	c.event <- *event
	c.reconnect()
}
<|endoftext|>