<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package config

import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"time"

	"github.com/shirou/gopsutil/disk"

	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/util"
	"github.com/syncthing/syncthing/lib/versioner"
)

var (
	ErrPathNotDirectory = errors.New("folder path not a directory")
	ErrPathMissing      = errors.New("folder path missing")
	ErrMarkerMissing    = errors.New("folder marker missing")
)

const DefaultMarkerName = ".stfolder"

type FolderConfiguration struct {
	ID                      string                      `xml:"id,attr" json:"id"`
	Label                   string                      `xml:"label,attr" json:"label" restart:"false"`
	FilesystemType          fs.FilesystemType           `xml:"filesystemType" json:"filesystemType"`
	Path                    string                      `xml:"path,attr" json:"path"`
	Type                    FolderType                  `xml:"type,attr" json:"type"`
	Devices                 []FolderDeviceConfiguration `xml:"device" json:"devices"`
	RescanIntervalS         int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS" default:"3600"`
	FSWatcherEnabled        bool                        `xml:"fsWatcherEnabled,attr" json:"fsWatcherEnabled" default:"true"`
	FSWatcherDelayS         int                         `xml:"fsWatcherDelayS,attr" json:"fsWatcherDelayS" default:"10"`
	IgnorePerms             bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
	AutoNormalize           bool                        `xml:"autoNormalize,attr" json:"autoNormalize" default:"true"`
	MinDiskFree             Size                        `xml:"minDiskFree" json:"minDiskFree" default:"1%"`
	Versioning              VersioningConfiguration     `xml:"versioning" json:"versioning"`
	Copiers                 int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
	PullerMaxPendingKiB     int                         `xml:"pullerMaxPendingKiB" json:"pullerMaxPendingKiB"`
	Hashers                 int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
	Order                   PullOrder                   `xml:"order" json:"order"`
	IgnoreDelete            bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
	ScanProgressIntervalS   int                         `xml:"scanProgressIntervalS" json:"scanProgressIntervalS"` // Set to a negative value to disable. Value of 0 will get replaced with value of 2 (default value)
	PullerPauseS            int                         `xml:"pullerPauseS" json:"pullerPauseS"`
	MaxConflicts            int                         `xml:"maxConflicts" json:"maxConflicts" default:"-1"`
	DisableSparseFiles      bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
	DisableTempIndexes      bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
	Paused                  bool                        `xml:"paused" json:"paused"`
	WeakHashThresholdPct    int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
	MarkerName              string                      `xml:"markerName" json:"markerName"`
	CopyOwnershipFromParent bool                        `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
	RawModTimeWindowS       int                         `xml:"modTimeWindowS" json:"modTimeWindowS"`

	cachedFilesystem    fs.Filesystem
	cachedModTimeWindow time.Duration

	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
	DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
	DeprecatedPullers        int     `xml:"pullers,omitempty" json:"-"`
}

type FolderDeviceConfiguration struct {
	DeviceID     protocol.DeviceID `xml:"id,attr" json:"deviceID"`
	IntroducedBy protocol.DeviceID `xml:"introducedBy,attr" json:"introducedBy"`
}

func NewFolderConfiguration(myID protocol.DeviceID, id, label string, fsType fs.FilesystemType, path string) FolderConfiguration {
	f := FolderConfiguration{
		ID:             id,
		Label:          label,
		Devices:        []FolderDeviceConfiguration{{DeviceID: myID}},
		FilesystemType: fsType,
		Path:           path,
	}

	util.SetDefaults(&f)

	f.prepare()
	return f
}

func (f FolderConfiguration) Copy() FolderConfiguration {
	c := f
	c.Devices = make([]FolderDeviceConfiguration, len(f.Devices))
	copy(c.Devices, f.Devices)
	c.Versioning = f.Versioning.Copy()
	return c
}

func (f FolderConfiguration) Filesystem() fs.Filesystem {
	// This is intentionally not a pointer method, because things like
	// cfg.Folders["default"].Filesystem() should be valid.
	if f.cachedFilesystem == nil {
		l.Infoln("bug: uncached filesystem call (should only happen in tests)")
		return fs.NewFilesystem(f.FilesystemType, f.Path)
	}
	return f.cachedFilesystem
}

func (f FolderConfiguration) Versioner() versioner.Versioner {
	if f.Versioning.Type == "" {
		return nil
	}
	versionerFactory, ok := versioner.Factories[f.Versioning.Type]
	if !ok {
		panic(fmt.Sprintf("Requested versioning type %q that does not exist", f.Versioning.Type))
	}

	return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
}

func (f FolderConfiguration) ModTimeWindow() time.Duration {
	return f.cachedModTimeWindow
}

func (f *FolderConfiguration) CreateMarker() error {
	if err := f.CheckPath(); err != ErrMarkerMissing {
		return err
	}
	if f.MarkerName != DefaultMarkerName {
		// Folder uses a non-default marker so we shouldn't mess with it.
		// Pretend we created it and let the subsequent health checks sort
		// out the actual situation.
		return nil
	}

	permBits := fs.FileMode(0777)
	if runtime.GOOS == "windows" {
		// Windows has no umask so we must chose a safer set of bits to
		// begin with.
		permBits = 0700
	}
	fs := f.Filesystem()
	err := fs.Mkdir(DefaultMarkerName, permBits)
	if err != nil {
		return err
	}
	if dir, err := fs.Open("."); err != nil {
		l.Debugln("folder marker: open . failed:", err)
	} else if err := dir.Sync(); err != nil {
		l.Debugln("folder marker: fsync . failed:", err)
	}
	fs.Hide(DefaultMarkerName)

	return nil
}

// CheckPath returns nil if the folder root exists and contains the marker file
func (f *FolderConfiguration) CheckPath() error {
	fi, err := f.Filesystem().Stat(".")
	if err != nil {
		if !fs.IsNotExist(err) {
			return err
		}
		return ErrPathMissing
	}

	// Users might have the root directory as a symlink or reparse point.
	// Furthermore, OneDrive bullcrap uses a magic reparse point to the cloudz...
	// Yet it's impossible for this to happen, as filesystem adds a trailing
	// path separator to the root, so even if you point the filesystem at a file
	// Stat ends up calling stat on C:\dir\file\ which, fails with "is not a directory"
	// in the error check above, and we don't even get to here.
	if !fi.IsDir() && !fi.IsSymlink() {
		return ErrPathNotDirectory
	}

	_, err = f.Filesystem().Stat(f.MarkerName)
	if err != nil {
		if !fs.IsNotExist(err) {
			return err
		}
		return ErrMarkerMissing
	}

	return nil
}

func (f *FolderConfiguration) CreateRoot() (err error) {
	// Directory permission bits. Will be filtered down to something
	// sane by umask on Unixes.
	permBits := fs.FileMode(0777)
	if runtime.GOOS == "windows" {
		// Windows has no umask so we must chose a safer set of bits to
		// begin with.
		permBits = 0700
	}

	filesystem := f.Filesystem()

	if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
		err = filesystem.MkdirAll(".", permBits)
	}

	return err
}

func (f FolderConfiguration) Description() string {
	if f.Label == "" {
		return f.ID
	}
	return fmt.Sprintf("%q (%s)", f.Label, f.ID)
}

func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
	deviceIDs := make([]protocol.DeviceID, len(f.Devices))
	for i, n := range f.Devices {
		deviceIDs[i] = n.DeviceID
	}
	return deviceIDs
}

func (f *FolderConfiguration) prepare() {
	f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)

	if f.RescanIntervalS > MaxRescanIntervalS {
		f.RescanIntervalS = MaxRescanIntervalS
	} else if f.RescanIntervalS < 0 {
		f.RescanIntervalS = 0
	}

	if f.FSWatcherDelayS <= 0 {
		f.FSWatcherEnabled = false
		f.FSWatcherDelayS = 10
	}

	if f.Versioning.Params == nil {
		f.Versioning.Params = make(map[string]string)
	}

	if f.WeakHashThresholdPct == 0 {
		f.WeakHashThresholdPct = 25
	}

	if f.MarkerName == "" {
		f.MarkerName = DefaultMarkerName
	}

	switch {
	case f.RawModTimeWindowS > 0:
		f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second
	case runtime.GOOS == "android":
		usage, err := disk.Usage(f.Filesystem().URI())
		if err != nil {
			l.Debugf("Error detecting FS at %v on android, setting mtime window to 2s: %v", f.Path, err)
			f.cachedModTimeWindow = 2 * time.Second
			break
		}
		if strings.Contains(strings.ToLower(usage.Fstype), "fat") {
			l.Debugf("Detecting FS at %v on android, found %v, thus setting mtime window to 2s", f.Path, usage.Fstype)
			f.cachedModTimeWindow = 2 * time.Second
			break
		}
		l.Debugf("Detecting FS at %v on android, found %v, thus leaving mtime window at 0", f.Path, usage.Fstype)
	}
}

// RequiresRestartOnly returns a copy with only the attributes that require
// restart on change.
func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration {
	copy := f

	// Manual handling for things that are not taken care of by the tag
	// copier, yet should not cause a restart.
	copy.cachedFilesystem = nil

	blank := FolderConfiguration{}
	util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {
		if len(v) > 0 && v != "false" {
			panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "false"`, v))
		}
		return v == "false"
	})
	return copy
}

func (f *FolderConfiguration) SharedWith(device protocol.DeviceID) bool {
	for _, dev := range f.Devices {
		if dev.DeviceID == device {
			return true
		}
	}
	return false
}

func (f *FolderConfiguration) CheckAvailableSpace(req int64) error {
	val := f.MinDiskFree.BaseValue()
	if val <= 0 {
		return nil
	}
	fs := f.Filesystem()
	usage, err := fs.Usage(".")
	if err != nil {
		return nil
	}
	usage.Free -= req
	if usage.Free > 0 {
		if err := CheckFreeSpace(f.MinDiskFree, usage); err == nil {
			return nil
		}
	}
	return fmt.Errorf("insufficient space in %v %v", fs.Type(), fs.URI())
}
<commit_msg>lib/config: Handle empty Fstype for mtime-window (#5906)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package config

import (
	"errors"
	"fmt"
	"runtime"
	"strings"
	"time"

	"github.com/shirou/gopsutil/disk"

	"github.com/syncthing/syncthing/lib/fs"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/util"
	"github.com/syncthing/syncthing/lib/versioner"
)

var (
	ErrPathNotDirectory = errors.New("folder path not a directory")
	ErrPathMissing      = errors.New("folder path missing")
	ErrMarkerMissing    = errors.New("folder marker missing")
)

const DefaultMarkerName = ".stfolder"

type FolderConfiguration struct {
	ID                      string                      `xml:"id,attr" json:"id"`
	Label                   string                      `xml:"label,attr" json:"label" restart:"false"`
	FilesystemType          fs.FilesystemType           `xml:"filesystemType" json:"filesystemType"`
	Path                    string                      `xml:"path,attr" json:"path"`
	Type                    FolderType                  `xml:"type,attr" json:"type"`
	Devices                 []FolderDeviceConfiguration `xml:"device" json:"devices"`
	RescanIntervalS         int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS" default:"3600"`
	FSWatcherEnabled        bool                        `xml:"fsWatcherEnabled,attr" json:"fsWatcherEnabled" default:"true"`
	FSWatcherDelayS         int                         `xml:"fsWatcherDelayS,attr" json:"fsWatcherDelayS" default:"10"`
	IgnorePerms             bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
	AutoNormalize           bool                        `xml:"autoNormalize,attr" json:"autoNormalize" default:"true"`
	MinDiskFree             Size                        `xml:"minDiskFree" json:"minDiskFree" default:"1%"`
	Versioning              VersioningConfiguration     `xml:"versioning" json:"versioning"`
	Copiers                 int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
	PullerMaxPendingKiB     int                         `xml:"pullerMaxPendingKiB" json:"pullerMaxPendingKiB"`
	Hashers                 int                         `xml:"hashers" json:"hashers"` // Less than one sets the value to the number of cores. These are CPU bound due to hashing.
	Order                   PullOrder                   `xml:"order" json:"order"`
	IgnoreDelete            bool                        `xml:"ignoreDelete" json:"ignoreDelete"`
	ScanProgressIntervalS   int                         `xml:"scanProgressIntervalS" json:"scanProgressIntervalS"` // Set to a negative value to disable. Value of 0 will get replaced with value of 2 (default value)
	PullerPauseS            int                         `xml:"pullerPauseS" json:"pullerPauseS"`
	MaxConflicts            int                         `xml:"maxConflicts" json:"maxConflicts" default:"-1"`
	DisableSparseFiles      bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
	DisableTempIndexes      bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
	Paused                  bool                        `xml:"paused" json:"paused"`
	WeakHashThresholdPct    int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
	MarkerName              string                      `xml:"markerName" json:"markerName"`
	CopyOwnershipFromParent bool                        `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
	RawModTimeWindowS       int                         `xml:"modTimeWindowS" json:"modTimeWindowS"`

	cachedFilesystem    fs.Filesystem
	cachedModTimeWindow time.Duration

	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
	DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
	DeprecatedPullers        int     `xml:"pullers,omitempty" json:"-"`
}

type FolderDeviceConfiguration struct {
	DeviceID     protocol.DeviceID `xml:"id,attr" json:"deviceID"`
	IntroducedBy protocol.DeviceID `xml:"introducedBy,attr" json:"introducedBy"`
}

func NewFolderConfiguration(myID protocol.DeviceID, id, label string, fsType fs.FilesystemType, path string) FolderConfiguration {
	f := FolderConfiguration{
		ID:             id,
		Label:          label,
		Devices:        []FolderDeviceConfiguration{{DeviceID: myID}},
		FilesystemType: fsType,
		Path:           path,
	}

	util.SetDefaults(&f)

	f.prepare()
	return f
}

func (f FolderConfiguration) Copy() FolderConfiguration {
	c := f
	c.Devices = make([]FolderDeviceConfiguration, len(f.Devices))
	copy(c.Devices, f.Devices)
	c.Versioning = f.Versioning.Copy()
	return c
}

func (f FolderConfiguration) Filesystem() fs.Filesystem {
	// This is intentionally not a pointer method, because things like
	// cfg.Folders["default"].Filesystem() should be valid.
	if f.cachedFilesystem == nil {
		l.Infoln("bug: uncached filesystem call (should only happen in tests)")
		return fs.NewFilesystem(f.FilesystemType, f.Path)
	}
	return f.cachedFilesystem
}

func (f FolderConfiguration) Versioner() versioner.Versioner {
	if f.Versioning.Type == "" {
		return nil
	}
	versionerFactory, ok := versioner.Factories[f.Versioning.Type]
	if !ok {
		panic(fmt.Sprintf("Requested versioning type %q that does not exist", f.Versioning.Type))
	}

	return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
}

func (f FolderConfiguration) ModTimeWindow() time.Duration {
	return f.cachedModTimeWindow
}

func (f *FolderConfiguration) CreateMarker() error {
	if err := f.CheckPath(); err != ErrMarkerMissing {
		return err
	}
	if f.MarkerName != DefaultMarkerName {
		// Folder uses a non-default marker so we shouldn't mess with it.
		// Pretend we created it and let the subsequent health checks sort
		// out the actual situation.
		return nil
	}

	permBits := fs.FileMode(0777)
	if runtime.GOOS == "windows" {
		// Windows has no umask so we must chose a safer set of bits to
		// begin with.
		permBits = 0700
	}
	fs := f.Filesystem()
	err := fs.Mkdir(DefaultMarkerName, permBits)
	if err != nil {
		return err
	}
	if dir, err := fs.Open("."); err != nil {
		l.Debugln("folder marker: open . failed:", err)
	} else if err := dir.Sync(); err != nil {
		l.Debugln("folder marker: fsync . failed:", err)
	}
	fs.Hide(DefaultMarkerName)

	return nil
}

// CheckPath returns nil if the folder root exists and contains the marker file
func (f *FolderConfiguration) CheckPath() error {
	fi, err := f.Filesystem().Stat(".")
	if err != nil {
		if !fs.IsNotExist(err) {
			return err
		}
		return ErrPathMissing
	}

	// Users might have the root directory as a symlink or reparse point.
	// Furthermore, OneDrive bullcrap uses a magic reparse point to the cloudz...
	// Yet it's impossible for this to happen, as filesystem adds a trailing
	// path separator to the root, so even if you point the filesystem at a file
	// Stat ends up calling stat on C:\dir\file\ which, fails with "is not a directory"
	// in the error check above, and we don't even get to here.
	if !fi.IsDir() && !fi.IsSymlink() {
		return ErrPathNotDirectory
	}

	_, err = f.Filesystem().Stat(f.MarkerName)
	if err != nil {
		if !fs.IsNotExist(err) {
			return err
		}
		return ErrMarkerMissing
	}

	return nil
}

func (f *FolderConfiguration) CreateRoot() (err error) {
	// Directory permission bits. Will be filtered down to something
	// sane by umask on Unixes.
	permBits := fs.FileMode(0777)
	if runtime.GOOS == "windows" {
		// Windows has no umask so we must chose a safer set of bits to
		// begin with.
		permBits = 0700
	}

	filesystem := f.Filesystem()

	if _, err = filesystem.Stat("."); fs.IsNotExist(err) {
		err = filesystem.MkdirAll(".", permBits)
	}

	return err
}

func (f FolderConfiguration) Description() string {
	if f.Label == "" {
		return f.ID
	}
	return fmt.Sprintf("%q (%s)", f.Label, f.ID)
}

func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
	deviceIDs := make([]protocol.DeviceID, len(f.Devices))
	for i, n := range f.Devices {
		deviceIDs[i] = n.DeviceID
	}
	return deviceIDs
}

func (f *FolderConfiguration) prepare() {
	f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)

	if f.RescanIntervalS > MaxRescanIntervalS {
		f.RescanIntervalS = MaxRescanIntervalS
	} else if f.RescanIntervalS < 0 {
		f.RescanIntervalS = 0
	}

	if f.FSWatcherDelayS <= 0 {
		f.FSWatcherEnabled = false
		f.FSWatcherDelayS = 10
	}

	if f.Versioning.Params == nil {
		f.Versioning.Params = make(map[string]string)
	}

	if f.WeakHashThresholdPct == 0 {
		f.WeakHashThresholdPct = 25
	}

	if f.MarkerName == "" {
		f.MarkerName = DefaultMarkerName
	}

	switch {
	case f.RawModTimeWindowS > 0:
		f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second
	case runtime.GOOS == "android":
		if usage, err := disk.Usage(f.Filesystem().URI()); err != nil || usage.Fstype == "" || strings.Contains(strings.ToLower(usage.Fstype), "fat") {
			f.cachedModTimeWindow = 2 * time.Second
			l.Debugf(`Detecting FS at %v on android: Setting mtime window to 2s: err == %v, usage.Fstype == "%v"`, f.Path, err, usage.Fstype)
		} else {
			l.Debugf(`Detecting FS at %v on android: Leaving mtime window at 0: usage.Fstype == "%v"`, f.Path, usage.Fstype)
		}
	}
}

// RequiresRestartOnly returns a copy with only the attributes that require
// restart on change.
func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration {
	copy := f

	// Manual handling for things that are not taken care of by the tag
	// copier, yet should not cause a restart.
	copy.cachedFilesystem = nil

	blank := FolderConfiguration{}
	util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {
		if len(v) > 0 && v != "false" {
			panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "false"`, v))
		}
		return v == "false"
	})
	return copy
}

func (f *FolderConfiguration) SharedWith(device protocol.DeviceID) bool {
	for _, dev := range f.Devices {
		if dev.DeviceID == device {
			return true
		}
	}
	return false
}

func (f *FolderConfiguration) CheckAvailableSpace(req int64) error {
	val := f.MinDiskFree.BaseValue()
	if val <= 0 {
		return nil
	}
	fs := f.Filesystem()
	usage, err := fs.Usage(".")
	if err != nil {
		return nil
	}
	usage.Free -= req
	if usage.Free > 0 {
		if err := CheckFreeSpace(f.MinDiskFree, usage); err == nil {
			return nil
		}
	}
	return fmt.Errorf("insufficient space in %v %v", fs.Type(), fs.URI())
}
<|endoftext|>


<commit_before>package driver

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"gopkg.in/mgo.v2/bson"

	"github.com/Gujarats/nearest/util"
	"github.com/Gujarats/receiver"

	"github.com/Gujarats/nearest/model/city/interface"
	driverInterface "github.com/Gujarats/nearest/model/driver/interface"

	driverModel "github.com/Gujarats/nearest/model/driver"

	"github.com/Gujarats/nearest/model/global"
)

// create logger to print error in the console
var logger *log.Logger

func init() {
	logger = log.New(os.Stderr,
		"Controller Driver :: ",
		log.Ldate|log.Ltime|log.Lshortfile)
}

// type for the update driver request
type DriverUpdateRequest struct {
	Id     string  `request:"id,required"`
	Name   string  `request:"name,required"`
	Lat    float64 `request:"latitude,required"`
	Lon    float64 `request:"longitude,required"`
	Status bool    `request:"status,required"`
	City   string  `request:"city,required"`
}

// find specific driver with their ID or name.
// if the desired data didn't exist then insert new data
func UpdateDriver(m *sync.Mutex, driver driverInterface.DriverInterfacce, cityInterface cityInterface.CityInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		//start time for lenght of the process
		startTimer := time.Now()

		w.Header().Set("Access-Control-Allow-Methods", "POST")

		// receive request value to type
		var dr DriverUpdateRequest
		err := receiver.SetData(&dr, r)
		if err != nil {
			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", err.Error())
			return
		}

		// check id for validation format
		ok := bson.IsObjectIdHex(dr.Id)
		if !ok {
			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Invalid Id format")
			return
		}

		// check driver last location which district they were before
		lastDistrict := driver.GetLastDistrict(dr.Id)

		// checks drivers location which district they are now
		// NOTE : getting the nearest district must not null or fail. so we need to repeat the function if we got null.
		// but there is one approach solutions we give more distance value so that we can find the district event it is far.
		district, err := cityInterface.GetNearestDistrict(dr.City, dr.Lat, dr.Lon, 500)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to get nearest district")
			return

		}

		// checking if we got the current district data
		if district.Name == "" {
			w.WriteHeader(http.StatusOK)
			global.SetResponse(w, "Success", "No nearest district found!")
			return

		}

		// check if the location is the same or not, if not then remove the data from the lastdistrict.
		// format current district to meet the last district format
		if lastDistrict != "" {
			// lastDistrict is not empty from redis check if it is the same as current location
			if district.Name+"_district_"+district.Id.Hex() != lastDistrict {
				// remove the driver data in the last district
				// lastDistrict must formatted like collectionKey for district collections
				driver.Remove(dr.Id, lastDistrict)
			}

		}

		// Update driver for given dr / driver data
		driverData := driverModel.DriverData{
			Id:   bson.ObjectIdHex(dr.Id),
			Name: dr.Name, Status: dr.Status,
			Location: driverModel.GeoJson{
				Type: "Point", Coordinates: []float64{dr.Lon, dr.Lat},
			},
		}
		err = driver.Update(district.Name, district.Id.Hex(), driverData)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to update the driver")
			return
		}

		//return succes response
		elpasedTime := time.Since(startTimer).Seconds()
		w.WriteHeader(http.StatusOK)
		global.SetResponseTime(w, "Succes", "Driver Updated", elpasedTime)
		return
	})

}

// getting the value using the tag value
type DriverRequest struct {
	Lat      float64 `request:"latitude,required"`
	Lon      float64 `request:"longitude,required"`
	City     string  `request:"city,required"`
	Distance int64   `request:"distance,required"`
}

func FindDriver(driver driverInterface.DriverInterfacce, cityInterface cityInterface.CityInterfacce) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		//start time for lenght of the process
		startTimer := time.Now()
		var dr DriverRequest
		err := receiver.SetData(&dr, r)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", err.Error())
			return
		}

		// determined which quadran the input locations

		// NOTE : send the driver which locations and its distance from input location
		// 1.  getting all centers location map[int][4]locations
		// 2. determined which quadran is it for inputLocation
		// 3. get nearest marked locations from that quadran
		// 4. get driver from the collections of nearest locations(this is must be array)
		// 5. send the driver
		// 6. if the driver is not found then go to next nearest locations. currentIndex +1. until the last index.
		// 7. if not found find the driver on the next quadran (currentquadran +1) until reach 4
		// 8. if not found find the driver on the next level. repeat this until you found the driver

		// get all district from redis and calculate it
		// calculate nearest location district with given location and city from mongodb
		district, err := cityInterface.GetNearestDistrict(dr.City, dr.Lat, dr.Lon, dr.Distance)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to get nearest district")
			return
		}

		// checking district result from mongodb
		if district.Name == "" {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "No district found")
			return
		}

		//response variable for getting the drivers
		var driverResponse driverModel.DriverData

		// checks drivers int the district from the redis
		drivers := driver.DriversRedis(district.Name, district.Id.Hex())
		if len(drivers) > 0 {
			// get the first index drvier from redis and save it again to redis
			driverResponse = drivers[0]

			driver.SaveLastDistrict(drivers[0].Id.Hex(), district.Name, district.Id.Hex())

			// update the driver's status to unavailable in mongodb
			// Latitude is 1 in the index and Longitude is 0. Rules from mongodb
			drivers[0].Status = false
			err := driver.Update(district.Name, district.Id.Hex(), drivers[0])
			if err != nil {

			}

			// update redis data by removing the first index
			drivers = drivers[1:]
			// save the drivers to redis replacing previous data
			driver.SaveDriversRedis(drivers, district.Name, district.Id.Hex())

		} else {
			// get drivers from mongodb
			drivers = driver.GetAvailableDriver(district.Name, district.Id.Hex())
			if len(drivers) > 0 {
				driverResponse = drivers[0]

				drivers[0].Status = false
				err := driver.Update(district.Name, district.Id.Hex(), drivers[0])
				if err != nil {
					w.WriteHeader(http.StatusInternalServerError)
					global.SetResponse(w, "Failed", "Failed to update ther driver")
					return
				}

				// update redis data by removing the first index
				drivers = drivers[1:]
				// save the drivers to redis replacing previous data
				driver.SaveDriversRedis(drivers, district.Name, district.Id.Hex())
			} else {
				// we could not find any data in redis and mongo
				w.WriteHeader(http.StatusOK)
				global.SetResponse(w, "Success", "We couldn't find any driver")
				return
			}
		}

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data found", Latency: elapsedTime, Data: driverResponse}
		json.NewEncoder(w).Encode(response)
		return

	})
}

func InsertDriver(driver driverInterface.DriverInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		//start time for lenght of the process
		startTimer := time.Now()
		w.Header().Set("Access-Control-Allow-Methods", "POST")

		// getting the parameters
		name := r.FormValue("name")
		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		status := r.FormValue("status")

		isAllExist := util.CheckValue(name, lat, lon, status)
		if !isAllExist {
			logger.Println("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		// convert string to bool
		statusBool, err := strconv.ParseBool(status)
		if err != nil {
			//return Bad response
			logger.Println("Failed to Parse Boolean")
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Parse Boolean Erro")
			return
		}

		// convert string to float64
		convertedFloat, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert float value")
			return
		}
		latFloat := convertedFloat[0]
		lonFloat := convertedFloat[1]

		// insert driver t
		// TODO :  error handler here what happens if process of insertion data fails
		driver.Insert(name, name, latFloat, lonFloat, statusBool)

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data Inserted", Latency: elapsedTime}
		json.NewEncoder(w).Encode(response)
		return
	})
}
<commit_msg>remove logger<commit_after>package driver

import (
	"encoding/json"
	"net/http"
	"strconv"
	"sync"
	"time"

	"gopkg.in/mgo.v2/bson"

	"github.com/Gujarats/nearest/util"
	"github.com/Gujarats/receiver"

	"github.com/Gujarats/nearest/model/city/interface"
	driverInterface "github.com/Gujarats/nearest/model/driver/interface"

	driverModel "github.com/Gujarats/nearest/model/driver"

	"github.com/Gujarats/nearest/model/global"
)

// type for the update driver request
type DriverUpdateRequest struct {
	Id     string  `request:"id,required"`
	Name   string  `request:"name,required"`
	Lat    float64 `request:"latitude,required"`
	Lon    float64 `request:"longitude,required"`
	Status bool    `request:"status,required"`
	City   string  `request:"city,required"`
}

// find specific driver with their ID or name.
// if the desired data didn't exist then insert new data
func UpdateDriver(m *sync.Mutex, driver driverInterface.DriverInterfacce, cityInterface cityInterface.CityInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		//start time for lenght of the process
		startTimer := time.Now()

		w.Header().Set("Access-Control-Allow-Methods", "POST")

		// receive request value to type
		var dr DriverUpdateRequest
		err := receiver.SetData(&dr, r)
		if err != nil {
			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", err.Error())
			return
		}

		// check id for validation format
		ok := bson.IsObjectIdHex(dr.Id)
		if !ok {
			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Invalid Id format")
			return
		}

		// check driver last location which district they were before
		lastDistrict := driver.GetLastDistrict(dr.Id)

		// checks drivers location which district they are now
		// NOTE : getting the nearest district must not null or fail. so we need to repeat the function if we got null.
		// but there is one approach solutions we give more distance value so that we can find the district event it is far.
		district, err := cityInterface.GetNearestDistrict(dr.City, dr.Lat, dr.Lon, 500)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to get nearest district")
			return

		}

		// checking if we got the current district data
		if district.Name == "" {
			w.WriteHeader(http.StatusOK)
			global.SetResponse(w, "Success", "No nearest district found!")
			return

		}

		// check if the location is the same or not, if not then remove the data from the lastdistrict.
		// format current district to meet the last district format
		if lastDistrict != "" {
			// lastDistrict is not empty from redis check if it is the same as current location
			if district.Name+"_district_"+district.Id.Hex() != lastDistrict {
				// remove the driver data in the last district
				// lastDistrict must formatted like collectionKey for district collections
				driver.Remove(dr.Id, lastDistrict)
			}

		}

		// Update driver for given dr / driver data
		driverData := driverModel.DriverData{
			Id:   bson.ObjectIdHex(dr.Id),
			Name: dr.Name, Status: dr.Status,
			Location: driverModel.GeoJson{
				Type: "Point", Coordinates: []float64{dr.Lon, dr.Lat},
			},
		}
		err = driver.Update(district.Name, district.Id.Hex(), driverData)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to update the driver")
			return
		}

		//return succes response
		elpasedTime := time.Since(startTimer).Seconds()
		w.WriteHeader(http.StatusOK)
		global.SetResponseTime(w, "Succes", "Driver Updated", elpasedTime)
		return
	})

}

// getting the value using the tag value
type DriverRequest struct {
	Lat      float64 `request:"latitude,required"`
	Lon      float64 `request:"longitude,required"`
	City     string  `request:"city,required"`
	Distance int64   `request:"distance,required"`
}

func FindDriver(driver driverInterface.DriverInterfacce, cityInterface cityInterface.CityInterfacce) http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		//start time for lenght of the process
		startTimer := time.Now()
		var dr DriverRequest
		err := receiver.SetData(&dr, r)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", err.Error())
			return
		}

		// determined which quadran the input locations
		// NOTE : send the driver which locations and its distance from input location
		// 1.  getting all centers location map[int][4]locations
		// 2. determined which quadran is it for inputLocation
		// 3. get nearest marked locations from that quadran
		// 4. get driver from the collections of nearest locations(this is must be array)
		// 5. send the driver
		// 6. if the driver is not found then go to next nearest locations. currentIndex +1. until the last index.
		// 7. if not found find the driver on the next quadran (currentquadran +1) until reach 4
		// 8. if not found find the driver on the next level. repeat this until you found the driver

		// get all district from redis and calculate it
		// calculate nearest location district with given location and city from mongodb
		district, err := cityInterface.GetNearestDistrict(dr.City, dr.Lat, dr.Lon, dr.Distance)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to get nearest district")
			return
		}

		// checking district result from mongodb
		if district.Name == "" {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "No district found")
			return
		}

		//response variable for getting the drivers
		var driverResponse driverModel.DriverData

		// checks drivers int the district from the redis
		drivers := driver.DriversRedis(district.Name, district.Id.Hex())
		if len(drivers) > 0 {
			// get the first index drvier from redis and save it again to redis
			driverResponse = drivers[0]

			driver.SaveLastDistrict(drivers[0].Id.Hex(), district.Name, district.Id.Hex())

			// update the driver's status to unavailable in mongodb
			// Latitude is 1 in the index and Longitude is 0. Rules from mongodb
			drivers[0].Status = false
			err := driver.Update(district.Name, district.Id.Hex(), drivers[0])
			if err != nil {

			}

			// update redis data by removing the first index
			drivers = drivers[1:]
			// save the drivers to redis replacing previous data
			driver.SaveDriversRedis(drivers, district.Name, district.Id.Hex())

		} else {
			// get drivers from mongodb
			drivers = driver.GetAvailableDriver(district.Name, district.Id.Hex())
			if len(drivers) > 0 {
				driverResponse = drivers[0]

				drivers[0].Status = false
				err := driver.Update(district.Name, district.Id.Hex(), drivers[0])
				if err != nil {
					w.WriteHeader(http.StatusInternalServerError)
					global.SetResponse(w, "Failed", "Failed to update ther driver")
					return
				}

				// update redis data by removing the first index
				drivers = drivers[1:]
				// save the drivers to redis replacing previous data
				driver.SaveDriversRedis(drivers, district.Name, district.Id.Hex())
			} else {
				// we could not find any data in redis and mongo
				w.WriteHeader(http.StatusOK)
				global.SetResponse(w, "Success", "We couldn't find any driver")
				return
			}
		}

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data found", Latency: elapsedTime, Data: driverResponse}
		json.NewEncoder(w).Encode(response)
		return

	})
}

func InsertDriver(driver driverInterface.DriverInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		//start time for lenght of the process
		startTimer := time.Now()
		w.Header().Set("Access-Control-Allow-Methods", "POST")

		// getting the parameters
		name := r.FormValue("name")
		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		status := r.FormValue("status")

		isAllExist := util.CheckValue(name, lat, lon, status)
		if !isAllExist {
			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		// convert string to bool
		statusBool, err := strconv.ParseBool(status)
		if err != nil {
			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Parse Boolean Erro")
			return
		}

		// convert string to float64
		convertedFloat, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert float value")
			return
		}
		latFloat := convertedFloat[0]
		lonFloat := convertedFloat[1]

		// insert driver t
		// TODO :  error handler here what happens if process of insertion data fails
		driver.Insert(name, name, latFloat, lonFloat, statusBool)

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data Inserted", Latency: elapsedTime}
		json.NewEncoder(w).Encode(response)
		return
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

import (
	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/volume"

	"os"

	"fmt"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"k8s.io/apimachinery/pkg/util/errors"
	utilfs "k8s.io/kubernetes/pkg/util/filesystem"
)

type flexVolumeProber struct {
	mutex           sync.Mutex
	pluginDir       string // Flexvolume driver directory
	watcher         utilfs.FSWatcher
	probeNeeded     bool      // Must only read and write this through testAndSetProbeNeeded.
	lastUpdated     time.Time // Last time probeNeeded was updated.
	watchEventCount int
	factory         PluginFactory
	fs              utilfs.Filesystem
}

const (
	// TODO (cxing) Tune these params based on test results.
	// watchEventLimit is the max allowable number of processed watches within watchEventInterval.
	watchEventInterval = 5 * time.Second
	watchEventLimit    = 20
)

func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber {
	return &flexVolumeProber{
		pluginDir: pluginDir,
		watcher:   utilfs.NewFsnotifyWatcher(),
		factory:   pluginFactory{},
		fs:        &utilfs.DefaultFs{},
	}
}

func (prober *flexVolumeProber) Init() error {
	prober.testAndSetProbeNeeded(true)
	prober.lastUpdated = time.Now()

	if err := prober.createPluginDir(); err != nil {
		return err
	}
	if err := prober.initWatcher(); err != nil {
		return err
	}

	return nil
}

// Probes for Flexvolume drivers.
// If a filesystem update has occurred since the last probe, updated = true
// and the list of probed plugins is returned.
// Otherwise, update = false and probedPlugins = nil.
//
// If an error occurs, updated and plugins are set arbitrarily.
func (prober *flexVolumeProber) Probe() (updated bool, plugins []volume.VolumePlugin, err error) {
	probeNeeded := prober.testAndSetProbeNeeded(false)

	if !probeNeeded {
		return false, nil, nil
	}

	files, err := prober.fs.ReadDir(prober.pluginDir)
	if err != nil {
		return false, nil, fmt.Errorf("Error reading the Flexvolume directory: %s", err)
	}

	plugins = []volume.VolumePlugin{}
	allErrs := []error{}
	for _, f := range files {
		// only directories with names that do not begin with '.' are counted as plugins
		// and pluginDir/dirname/dirname should be an executable
		// unless dirname contains '~' for escaping namespace
		// e.g. dirname = vendor~cifs
		// then, executable will be pluginDir/dirname/cifs
		if f.IsDir() && filepath.Base(f.Name())[0] != '.' {
			plugin, pluginErr := prober.factory.NewFlexVolumePlugin(prober.pluginDir, f.Name())
			if pluginErr != nil {
				pluginErr = fmt.Errorf(
					"Error creating Flexvolume plugin from directory %s, skipping. Error: %s",
					f.Name(), pluginErr)
				allErrs = append(allErrs, pluginErr)
				continue
			}

			plugins = append(plugins, plugin)
		}
	}

	return true, plugins, errors.NewAggregate(allErrs)
}

func (prober *flexVolumeProber) handleWatchEvent(event fsnotify.Event) error {
	// event.Name is the watched path.
	if filepath.Base(event.Name)[0] == '.' {
		// Ignore files beginning with '.'
		return nil
	}

	eventPathAbs, err := filepath.Abs(event.Name)
	if err != nil {
		return err
	}

	pluginDirAbs, err := filepath.Abs(prober.pluginDir)
	if err != nil {
		return err
	}

	// If the Flexvolume plugin directory is removed, need to recreate it
	// in order to keep it under watch.
	if eventOpIs(event, fsnotify.Remove) && eventPathAbs == pluginDirAbs {
		if err := prober.createPluginDir(); err != nil {
			return err
		}
		if err := prober.addWatchRecursive(pluginDirAbs); err != nil {
			return err
		}
	} else if eventOpIs(event, fsnotify.Create) {
		if err := prober.addWatchRecursive(eventPathAbs); err != nil {
			return err
		}
	}

	prober.updateProbeNeeded()

	return nil
}

func (prober *flexVolumeProber) updateProbeNeeded() {
	// Within 'watchEventInterval' seconds, a max of 'watchEventLimit' watch events is processed.
	// The watch event will not be registered if the limit is reached.
	// This prevents increased disk usage from Probe() being triggered too frequently (either
	// accidentally or maliciously).
	if time.Since(prober.lastUpdated) > watchEventInterval {
		// Update, then reset the timer and watch count.
		prober.testAndSetProbeNeeded(true)
		prober.lastUpdated = time.Now()
		prober.watchEventCount = 1
	} else if prober.watchEventCount < watchEventLimit {
		prober.testAndSetProbeNeeded(true)
		prober.watchEventCount++
	}
}

// Recursively adds to watch all directories inside and including the file specified by the given filename.
// If the file is a symlink to a directory, it will watch the symlink but not any of the subdirectories.
//
// Each file or directory change triggers two events: one from the watch on itself, another from the watch
// on its parent directory.
func (prober *flexVolumeProber) addWatchRecursive(filename string) error {
	addWatch := func(path string, info os.FileInfo, err error) error {
		if err == nil {
			if info.IsDir() {
				if err := prober.watcher.AddWatch(path); err != nil {
					glog.Errorf("Error recursively adding watch: %v", err)
				}
			}
		}
		return nil
	}
	return prober.fs.Walk(filename, addWatch)
}

// Creates a new filesystem watcher and adds watches for the plugin directory
// and all of its subdirectories.
func (prober *flexVolumeProber) initWatcher() error {
	err := prober.watcher.Init(func(event fsnotify.Event) {
		if err := prober.handleWatchEvent(event); err != nil {
			glog.Errorf("Flexvolume prober watch: %s", err)
		}
	}, func(err error) {
		glog.Errorf("Received an error from watcher: %s", err)
	})
	if err != nil {
		return fmt.Errorf("Error initializing watcher: %s", err)
	}

	if err := prober.addWatchRecursive(prober.pluginDir); err != nil {
		return fmt.Errorf("Error adding watch on Flexvolume directory: %s", err)
	}

	prober.watcher.Run()

	return nil
}

// Creates the plugin directory, if it doesn't already exist.
func (prober *flexVolumeProber) createPluginDir() error {
	if _, err := prober.fs.Stat(prober.pluginDir); os.IsNotExist(err) {
		glog.Warningf("Flexvolume plugin directory at %s does not exist. Recreating.", prober.pluginDir)
		err := prober.fs.MkdirAll(prober.pluginDir, 0755)
		if err != nil {
			return fmt.Errorf("Error (re-)creating driver directory: %s", err)
		}
	}

	return nil
}

func (prober *flexVolumeProber) testAndSetProbeNeeded(newval bool) (oldval bool) {
	prober.mutex.Lock()
	defer prober.mutex.Unlock()
	oldval, prober.probeNeeded = prober.probeNeeded, newval
	return
}

func eventOpIs(event fsnotify.Event, op fsnotify.Op) bool {
	return event.Op&op == op
}
<commit_msg>reduce nesting<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

import (
	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/volume"

	"os"

	"fmt"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"k8s.io/apimachinery/pkg/util/errors"
	utilfs "k8s.io/kubernetes/pkg/util/filesystem"
)

type flexVolumeProber struct {
	mutex           sync.Mutex
	pluginDir       string // Flexvolume driver directory
	watcher         utilfs.FSWatcher
	probeNeeded     bool      // Must only read and write this through testAndSetProbeNeeded.
	lastUpdated     time.Time // Last time probeNeeded was updated.
	watchEventCount int
	factory         PluginFactory
	fs              utilfs.Filesystem
}

const (
	// TODO (cxing) Tune these params based on test results.
	// watchEventLimit is the max allowable number of processed watches within watchEventInterval.
	watchEventInterval = 5 * time.Second
	watchEventLimit    = 20
)

func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber {
	return &flexVolumeProber{
		pluginDir: pluginDir,
		watcher:   utilfs.NewFsnotifyWatcher(),
		factory:   pluginFactory{},
		fs:        &utilfs.DefaultFs{},
	}
}

func (prober *flexVolumeProber) Init() error {
	prober.testAndSetProbeNeeded(true)
	prober.lastUpdated = time.Now()

	if err := prober.createPluginDir(); err != nil {
		return err
	}
	if err := prober.initWatcher(); err != nil {
		return err
	}

	return nil
}

// Probes for Flexvolume drivers.
// If a filesystem update has occurred since the last probe, updated = true
// and the list of probed plugins is returned.
// Otherwise, update = false and probedPlugins = nil.
//
// If an error occurs, updated and plugins are set arbitrarily.
func (prober *flexVolumeProber) Probe() (updated bool, plugins []volume.VolumePlugin, err error) {
	probeNeeded := prober.testAndSetProbeNeeded(false)

	if !probeNeeded {
		return false, nil, nil
	}

	files, err := prober.fs.ReadDir(prober.pluginDir)
	if err != nil {
		return false, nil, fmt.Errorf("Error reading the Flexvolume directory: %s", err)
	}

	plugins = []volume.VolumePlugin{}
	allErrs := []error{}
	for _, f := range files {
		// only directories with names that do not begin with '.' are counted as plugins
		// and pluginDir/dirname/dirname should be an executable
		// unless dirname contains '~' for escaping namespace
		// e.g. dirname = vendor~cifs
		// then, executable will be pluginDir/dirname/cifs
		if f.IsDir() && filepath.Base(f.Name())[0] != '.' {
			plugin, pluginErr := prober.factory.NewFlexVolumePlugin(prober.pluginDir, f.Name())
			if pluginErr != nil {
				pluginErr = fmt.Errorf(
					"Error creating Flexvolume plugin from directory %s, skipping. Error: %s",
					f.Name(), pluginErr)
				allErrs = append(allErrs, pluginErr)
				continue
			}

			plugins = append(plugins, plugin)
		}
	}

	return true, plugins, errors.NewAggregate(allErrs)
}

func (prober *flexVolumeProber) handleWatchEvent(event fsnotify.Event) error {
	// event.Name is the watched path.
	if filepath.Base(event.Name)[0] == '.' {
		// Ignore files beginning with '.'
		return nil
	}

	eventPathAbs, err := filepath.Abs(event.Name)
	if err != nil {
		return err
	}

	pluginDirAbs, err := filepath.Abs(prober.pluginDir)
	if err != nil {
		return err
	}

	// If the Flexvolume plugin directory is removed, need to recreate it
	// in order to keep it under watch.
	if eventOpIs(event, fsnotify.Remove) && eventPathAbs == pluginDirAbs {
		if err := prober.createPluginDir(); err != nil {
			return err
		}
		if err := prober.addWatchRecursive(pluginDirAbs); err != nil {
			return err
		}
	} else if eventOpIs(event, fsnotify.Create) {
		if err := prober.addWatchRecursive(eventPathAbs); err != nil {
			return err
		}
	}

	prober.updateProbeNeeded()

	return nil
}

func (prober *flexVolumeProber) updateProbeNeeded() {
	// Within 'watchEventInterval' seconds, a max of 'watchEventLimit' watch events is processed.
	// The watch event will not be registered if the limit is reached.
	// This prevents increased disk usage from Probe() being triggered too frequently (either
	// accidentally or maliciously).
	if time.Since(prober.lastUpdated) > watchEventInterval {
		// Update, then reset the timer and watch count.
		prober.testAndSetProbeNeeded(true)
		prober.lastUpdated = time.Now()
		prober.watchEventCount = 1
	} else if prober.watchEventCount < watchEventLimit {
		prober.testAndSetProbeNeeded(true)
		prober.watchEventCount++
	}
}

// Recursively adds to watch all directories inside and including the file specified by the given filename.
// If the file is a symlink to a directory, it will watch the symlink but not any of the subdirectories.
//
// Each file or directory change triggers two events: one from the watch on itself, another from the watch
// on its parent directory.
func (prober *flexVolumeProber) addWatchRecursive(filename string) error {
	addWatch := func(path string, info os.FileInfo, err error) error {
		if err == nil && info.IsDir() {
			if err := prober.watcher.AddWatch(path); err != nil {
				glog.Errorf("Error recursively adding watch: %v", err)
			}
		}
		return nil
	}
	return prober.fs.Walk(filename, addWatch)
}

// Creates a new filesystem watcher and adds watches for the plugin directory
// and all of its subdirectories.
func (prober *flexVolumeProber) initWatcher() error {
	err := prober.watcher.Init(func(event fsnotify.Event) {
		if err := prober.handleWatchEvent(event); err != nil {
			glog.Errorf("Flexvolume prober watch: %s", err)
		}
	}, func(err error) {
		glog.Errorf("Received an error from watcher: %s", err)
	})
	if err != nil {
		return fmt.Errorf("Error initializing watcher: %s", err)
	}

	if err := prober.addWatchRecursive(prober.pluginDir); err != nil {
		return fmt.Errorf("Error adding watch on Flexvolume directory: %s", err)
	}

	prober.watcher.Run()

	return nil
}

// Creates the plugin directory, if it doesn't already exist.
func (prober *flexVolumeProber) createPluginDir() error {
	if _, err := prober.fs.Stat(prober.pluginDir); os.IsNotExist(err) {
		glog.Warningf("Flexvolume plugin directory at %s does not exist. Recreating.", prober.pluginDir)
		err := prober.fs.MkdirAll(prober.pluginDir, 0755)
		if err != nil {
			return fmt.Errorf("Error (re-)creating driver directory: %s", err)
		}
	}

	return nil
}

func (prober *flexVolumeProber) testAndSetProbeNeeded(newval bool) (oldval bool) {
	prober.mutex.Lock()
	defer prober.mutex.Unlock()
	oldval, prober.probeNeeded = prober.probeNeeded, newval
	return
}

func eventOpIs(event fsnotify.Event, op fsnotify.Op) bool {
	return event.Op&op == op
}
<|endoftext|>


<commit_before>// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"

	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_driver/go/td"
)

func main() {
	var (
		projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
		taskId    = flag.String("task_id", "", "ID of this task.")
		bot       = flag.String("bot", "", "Name of the task.")
		output    = flag.String("o", "", "Dump JSON step data to the given file, or stdout if -.")
		local     = flag.Bool("local", true, "Running locally (else on the bots)?")

		resources = flag.String("resources", "resources", "Passed to fm -i.")
		script    = flag.String("script", "", "File (or - for stdin) with one job per line.")
	)
	ctx := td.StartRun(projectId, taskId, bot, output, local)
	defer td.EndRun(ctx)

	actualStdout := os.Stdout
	actualStderr := os.Stderr
	verbosity := exec.Info
	if *local {
		// Task Driver echoes every exec.Run() stdout and stderr to the console,
		// which makes it hard to find failures (especially stdout).  Send them to /dev/null.
		devnull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
		if err != nil {
			td.Fatal(ctx, err)
		}
		os.Stdout = devnull
		os.Stderr = devnull
		// Having stifled stderr/stdout, changing Command.Verbose won't have any visible effect,
		// but setting it to Silent will bypass a fair chunk of wasted formatting work.
		verbosity = exec.Silent
	}

	if flag.NArg() < 1 {
		td.Fatalf(ctx, "Please pass an fm binary.")
	}
	fm := flag.Arg(0)

	// Run `fm <flag>` to find the names of all linked GMs or tests.
	query := func(flag string) []string {
		stdout := &bytes.Buffer{}
		cmd := &exec.Command{Name: fm, Stdout: stdout, Verbose: verbosity}
		cmd.Args = append(cmd.Args, "-i", *resources)
		cmd.Args = append(cmd.Args, flag)
		if err := exec.Run(ctx, cmd); err != nil {
			td.Fatal(ctx, err)
		}

		lines := []string{}
		scanner := bufio.NewScanner(stdout)
		for scanner.Scan() {
			lines = append(lines, scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			td.Fatal(ctx, err)
		}
		return lines
	}
	gms := query("--listGMs")
	tests := query("--listTests")

	// Query Gold for all known hashes when running as a bot.
	known := map[string]bool{
		"0832f708a97acc6da385446384647a8f": true, // MD5 of passing unit test.
	}
	if *bot != "" {
		func() {
			url := "https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt"
			resp, err := http.Get(url)
			if err != nil {
				td.Fatal(ctx, err)
			}
			defer resp.Body.Close()

			scanner := bufio.NewScanner(resp.Body)
			for scanner.Scan() {
				known[scanner.Text()] = true
			}
			if err := scanner.Err(); err != nil {
				td.Fatal(ctx, err)
			}

			fmt.Fprintf(actualStdout, "Gold knew %v unique hashes.\n", len(known))
		}()
	}

	// We'll kick off worker goroutines to run batches of work, and on failure,
	// crash, or unknown hash, we'll split that batch into individual reruns to
	// isolate those unusual results.
	var failures int32 = 0
	wg := &sync.WaitGroup{}

	var worker func([]string, []string)
	worker = func(sources []string, flags []string) {
		defer wg.Done()

		stdout := &bytes.Buffer{}
		stderr := &bytes.Buffer{}
		cmd := &exec.Command{Name: fm, Stdout: stdout, Stderr: stderr, Verbose: verbosity}
		cmd.Args = append(cmd.Args, "-i", *resources, "-s")
		cmd.Args = append(cmd.Args, sources...)
		cmd.Args = append(cmd.Args, flags...)

		// Run our FM command.
		err := exec.Run(ctx, cmd)

		// On success, scan stdout for any unknown hashes.
		unknownHash := func() string {
			if err == nil && *bot != "" { // The map of known hashes is only filled when using -bot.
				scanner := bufio.NewScanner(stdout)
				for scanner.Scan() {
					if parts := strings.Fields(scanner.Text()); len(parts) == 3 {
						md5 := parts[1]
						if !known[md5] {
							return md5
						}
					}
				}
				if err := scanner.Err(); err != nil {
					td.Fatal(ctx, err)
				}
			}
			return ""
		}()

		// If a batch failed or produced an unknown hash, isolate with individual runs.
		if len(sources) > 1 && (err != nil || unknownHash != "") {
			wg.Add(len(sources))
			for i := range sources {
				// We could kick off independent goroutines here for more parallelism,
				// but the bots are already parallel enough that they'd exhaust their
				// process limits, and I haven't seen any impact on local runs.
				worker(sources[i:i+1], flags)
			}
			return
		}

		// If an individual run failed, nothing more to do but fail.
		if err != nil {
			atomic.AddInt32(&failures, 1)
			td.FailStep(ctx, err)
			if *local {
				lines := []string{}
				scanner := bufio.NewScanner(stderr)
				for scanner.Scan() {
					lines = append(lines, scanner.Text())
				}
				if err := scanner.Err(); err != nil {
					td.Fatal(ctx, err)
				}
				fmt.Fprintf(actualStderr, "%v %v #failed:\n\t%v\n",
					cmd.Name,
					strings.Join(cmd.Args, " "),
					strings.Join(lines, "\n\t"))
			}
			return
		}

		// If an individual run succeeded but produced an unknown hash, TODO upload .png to Gold.
		if unknownHash != "" {
			fmt.Fprintf(actualStdout, "%v %v #%v\n",
				cmd.Name,
				strings.Join(cmd.Args, " "),
				unknownHash)
		}
	}

	// Start workers that run `FM -s sources... flags...` in small source batches for parallelism.
	kickoff := func(sources, flags []string) {
		if len(sources) == 0 {
			return // A blank or commented job line from -script or the command line.
		}

		// Shuffle the sources randomly as a cheap way to approximate evenly expensive batches.
		// (Intentionally not rand.Seed()'d to stay deterministically reproducible.)
		rand.Shuffle(len(sources), func(i, j int) {
			sources[i], sources[j] = sources[j], sources[i]
		})

		// Round batch sizes up so there's at least one source per batch.
		// Batch size is arbitrary, but nice to scale with the machine like this.
		batches := runtime.NumCPU()
		batch := (len(sources) + batches - 1) / batches
		util.ChunkIter(len(sources), batch, func(start, end int) error {
			wg.Add(1)
			go worker(sources[start:end], flags)
			return nil
		})
	}

	// Parse a job like "gms b=cpu ct=8888" into sources and flags for kickoff().
	parse := func(job []string) (sources, flags []string) {
		for _, token := range job {
			// Everything after # is a comment.
			if strings.HasPrefix(token, "#") {
				break
			}

			// Treat "gm" or "gms" as a shortcut for all known GMs.
			if token == "gm" || token == "gms" {
				sources = append(sources, gms...)
				continue
			}
			// Same for tests.
			if token == "test" || token == "tests" {
				sources = append(sources, tests...)
				continue
			}

			// Is this a flag to pass through to FM?
			if parts := strings.Split(token, "="); len(parts) == 2 {
				f := "-"
				if len(parts[0]) > 1 {
					f += "-"
				}
				f += parts[0]

				flags = append(flags, f, parts[1])
				continue
			}

			// Anything else must be the name of a source for FM to run.
			sources = append(sources, token)
		}
		return
	}

	// Parse one job from the command line, handy for ad hoc local runs.
	kickoff(parse(flag.Args()[1:]))

	// Any number of jobs can come from -script.
	if *script != "" {
		file := os.Stdin
		if *script != "-" {
			file, err := os.Open(*script)
			if err != nil {
				td.Fatal(ctx, err)
			}
			defer file.Close()
		}
		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			kickoff(parse(strings.Fields(scanner.Text())))
		}
		if err := scanner.Err(); err != nil {
			td.Fatal(ctx, err)
		}
	}

	// If we're a bot (or acting as if we are one), add its work too.
	if *bot != "" {
		parts := strings.Split(*bot, "-")
		OS := parts[1]

		// For no reason but as a demo, skip GM aarectmodes and test GoodHash.
		filter := func(in []string, test func(string) bool) (out []string) {
			for _, s := range in {
				if test(s) {
					out = append(out, s)
				}
			}
			return
		}
		if OS == "Debian10" {
			gms = filter(gms, func(s string) bool { return s != "aarectmodes" })
			tests = filter(tests, func(s string) bool { return s != "GoodHash" })
		}

		// You could use parse() here if you like, but it's just as easy to kickoff() directly.
		kickoff(tests, strings.Fields("-b cpu"))
		kickoff(gms, strings.Fields("-b cpu"))
		kickoff(gms, strings.Fields("-b cpu --skvm"))
	}

	wg.Wait()
	if failures > 0 {
		if *local {
			// td.Fatalf() would work fine, but barfs up a panic that we don't need to see.
			fmt.Fprintf(actualStderr, "%v runs of %v failed after retries.\n", failures, fm)
			os.Exit(1)
		} else {
			td.Fatalf(ctx, "%v runs of %v failed after retries.", failures, fm)
		}
	}
}
<commit_msg>Revert fm_driver simplifications<commit_after>// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"

	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_driver/go/td"
)

func main() {
	var (
		projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
		taskId    = flag.String("task_id", "", "ID of this task.")
		bot       = flag.String("bot", "", "Name of the task.")
		output    = flag.String("o", "", "Dump JSON step data to the given file, or stdout if -.")
		local     = flag.Bool("local", true, "Running locally (else on the bots)?")

		resources = flag.String("resources", "resources", "Passed to fm -i.")
		script    = flag.String("script", "", "File (or - for stdin) with one job per line.")
	)
	ctx := td.StartRun(projectId, taskId, bot, output, local)
	defer td.EndRun(ctx)

	actualStdout := os.Stdout
	actualStderr := os.Stderr
	if *local {
		// Task Driver echoes every exec.Run() stdout and stderr to the console,
		// which makes it hard to find failures (especially stdout).  Send them to /dev/null.
		devnull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
		if err != nil {
			td.Fatal(ctx, err)
		}
		os.Stdout = devnull
		os.Stderr = devnull
	}

	if flag.NArg() < 1 {
		td.Fatalf(ctx, "Please pass an fm binary.")
	}
	fm := flag.Arg(0)

	// Run `fm <flag>` to find the names of all linked GMs or tests.
	query := func(flag string) []string {
		stdout := &bytes.Buffer{}
		cmd := &exec.Command{Name: fm, Stdout: stdout}
		cmd.Args = append(cmd.Args, "-i", *resources)
		cmd.Args = append(cmd.Args, flag)
		if err := exec.Run(ctx, cmd); err != nil {
			td.Fatal(ctx, err)
		}

		lines := []string{}
		scanner := bufio.NewScanner(stdout)
		for scanner.Scan() {
			lines = append(lines, scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			td.Fatal(ctx, err)
		}
		return lines
	}
	gms := query("--listGMs")
	tests := query("--listTests")

	// Query Gold for all known hashes when running as a bot.
	known := map[string]bool{
		"0832f708a97acc6da385446384647a8f": true, // MD5 of passing unit test.
	}
	if *bot != "" {
		func() {
			url := "https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt"
			resp, err := http.Get(url)
			if err != nil {
				td.Fatal(ctx, err)
			}
			defer resp.Body.Close()

			scanner := bufio.NewScanner(resp.Body)
			for scanner.Scan() {
				known[scanner.Text()] = true
			}
			if err := scanner.Err(); err != nil {
				td.Fatal(ctx, err)
			}

			fmt.Fprintf(actualStdout, "Gold knew %v unique hashes.\n", len(known))
		}()
	}

	type Work struct {
		Sources []string // Passed to FM -s: names of gms/tests, paths to image files, .skps, etc.
		Flags   []string // Other flags to pass to FM: --ct 565, --msaa 16, etc.
	}
	todo := []Work{}

	// Parse a job like "gms b=cpu ct=8888" into Work{Sources=<all GMs>, Flags={-b,cpu,--ct,8888}}.
	parse := func(job []string) (w Work) {
		for _, token := range job {
			// Everything after # is a comment.
			if strings.HasPrefix(token, "#") {
				break
			}

			// Treat "gm" or "gms" as a shortcut for all known GMs.
			if token == "gm" || token == "gms" {
				w.Sources = append(w.Sources, gms...)
				continue
			}
			// Same for tests.
			if token == "test" || token == "tests" {
				w.Sources = append(w.Sources, tests...)
				continue
			}

			// Is this a flag to pass through to FM?
			if parts := strings.Split(token, "="); len(parts) == 2 {
				f := "-"
				if len(parts[0]) > 1 {
					f += "-"
				}
				f += parts[0]

				w.Flags = append(w.Flags, f, parts[1])
				continue
			}

			// Anything else must be the name of a source for FM to run.
			w.Sources = append(w.Sources, token)
		}
		return
	}

	// Parse one job from the command line, handy for ad hoc local runs.
	todo = append(todo, parse(flag.Args()[1:]))

	// Any number of jobs can come from -script.
	if *script != "" {
		file := os.Stdin
		if *script != "-" {
			file, err := os.Open(*script)
			if err != nil {
				td.Fatal(ctx, err)
			}
			defer file.Close()
		}
		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			todo = append(todo, parse(strings.Fields(scanner.Text())))
		}
		if err := scanner.Err(); err != nil {
			td.Fatal(ctx, err)
		}
	}

	// If we're a bot (or acting as if we are one), add its work too.
	if *bot != "" {
		parts := strings.Split(*bot, "-")
		OS := parts[1]

		// For no reason but as a demo, skip GM aarectmodes and test GoodHash.
		filter := func(in []string, test func(string) bool) (out []string) {
			for _, s := range in {
				if test(s) {
					out = append(out, s)
				}
			}
			return
		}
		if OS == "Debian10" {
			gms = filter(gms, func(s string) bool { return s != "aarectmodes" })
			tests = filter(tests, func(s string) bool { return s != "GoodHash" })
		}

		// You could use parse() here if you like, but it's just as easy to make Work{} directly.
		work := func(sources []string, flags string) {
			todo = append(todo, Work{sources, strings.Fields(flags)})
		}
		work(tests, "-b cpu")
		work(gms, "-b cpu")
		work(gms, "-b cpu --skvm")
	}

	// We'll try to spread our work roughly evenly over a number of worker goroutines.
	// We can't combine Work with different Flags, but we can do the opposite,
	// splitting a single Work into smaller Work units with the same Flags,
	// even all the way down to a single Source.  So we'll optimistically run
	// batches of Sources together, but if a batch fails or crashes, we'll
	// split it up and re-run one at a time to find the precise failures.
	var failures int32 = 0
	wg := &sync.WaitGroup{}
	worker := func(queue chan Work) {
		for w := range queue {
			stdout := &bytes.Buffer{}
			stderr := &bytes.Buffer{}
			cmd := &exec.Command{Name: fm, Stdout: stdout, Stderr: stderr}
			cmd.Args = append(cmd.Args, "-i", *resources)
			cmd.Args = append(cmd.Args, "-s")
			cmd.Args = append(cmd.Args, w.Sources...)
			cmd.Args = append(cmd.Args, w.Flags...)
			// TODO: when len(w.Sources) == 1, add -w ... to cmd.Args to write a .png for upload.

			// On cmd failure or unknown hash, we'll split the Work batch up into individual reruns.
			requeue := func() {
				// Requeuing Work from the workers is what makes sizing the chan buffer tricky:
				// we don't ever want these `queue <-` to block a worker because of a full buffer.
				for _, source := range w.Sources {
					wg.Add(1)
					queue <- Work{[]string{source}, w.Flags}
				}
			}

			if err := exec.Run(ctx, cmd); err != nil {
				if len(w.Sources) == 1 {
					// If a source ran alone and failed, that's just a failure.
					atomic.AddInt32(&failures, 1)
					td.FailStep(ctx, err)
					if *local {
						lines := []string{}
						scanner := bufio.NewScanner(stderr)
						for scanner.Scan() {
							lines = append(lines, scanner.Text())
						}
						if err := scanner.Err(); err != nil {
							td.Fatal(ctx, err)
						}

						fmt.Fprintf(actualStderr, "%v %v #failed:\n\t%v\n",
							cmd.Name,
							strings.Join(cmd.Args, " "),
							strings.Join(lines, "\n\t"))
					}
				} else {
					// If a batch of sources failed, break up the batch to isolate the failures.
					requeue()
				}
			} else {
				// FM completed successfully.  Scan stdout for any unknown hash.
				unknown := func() string {
					if *bot != "" { // The map known[] is only filled when *bot != "".
						scanner := bufio.NewScanner(stdout)
						for scanner.Scan() {
							if parts := strings.Fields(scanner.Text()); len(parts) == 3 {
								md5 := parts[1]
								if !known[md5] {
									return md5
								}
							}
						}
						if err := scanner.Err(); err != nil {
							td.Fatal(ctx, err)
						}
					}
					return ""
				}()

				if unknown != "" {
					if len(w.Sources) == 1 {
						// TODO upload .png with goldctl.
						fmt.Fprintf(actualStdout, "%v %v #%v\n",
							cmd.Name,
							strings.Join(cmd.Args, " "),
							unknown)
					} else {
						// Split the batch to run individually and TODO, write .pngs.
						requeue()
					}
				}
			}
			wg.Done()
		}
	}

	workers := runtime.NumCPU()
	queue := make(chan Work, 1<<20) // Huge buffer to avoid having to be smart about requeuing.
	for i := 0; i < workers; i++ {
		go worker(queue)
	}

	for _, w := range todo {
		if len(w.Sources) == 0 {
			continue // A blank or commented job line from -script or the command line.
		}

		// Shuffle the sources randomly as a cheap way to approximate evenly expensive batches.
		// (Intentionally not rand.Seed()'d to stay deterministically reproducible.)
		rand.Shuffle(len(w.Sources), func(i, j int) {
			w.Sources[i], w.Sources[j] = w.Sources[j], w.Sources[i]
		})

		// Round batch sizes up so there's at least one source per batch.
		batch := (len(w.Sources) + workers - 1) / workers
		util.ChunkIter(len(w.Sources), batch, func(start, end int) error {
			wg.Add(1)
			queue <- Work{w.Sources[start:end], w.Flags}
			return nil
		})
	}
	wg.Wait()

	if failures > 0 {
		if *local {
			// td.Fatalf() would work fine, but barfs up a panic that we don't need to see.
			fmt.Fprintf(actualStderr, "%v runs of %v failed after retries.\n", failures, fm)
			os.Exit(1)
		} else {
			td.Fatalf(ctx, "%v runs of %v failed after retries.", failures, fm)
		}
	}
}
<|endoftext|>


<commit_before>package fasthttp

import (
	"bufio"
	"bytes"
	"fmt"
	"html"
	"net"
	"testing"
	"time"

	"github.com/valyala/bytebufferpool"
)

func TestAppendHTMLEscape(t *testing.T) {
	t.Parallel()

	// Sync with html.EscapeString
	allcases := make([]byte, 256)
	for i := 0; i < 256; i++ {
		allcases[i] = byte(i)
	}
	res := string(AppendHTMLEscape(nil, string(allcases)))
	expect := string(html.EscapeString(string(allcases)))
	if res != expect {
		t.Fatalf("unexpected string %q. Expecting %q.", res, expect)
	}

	testAppendHTMLEscape(t, "", "")
	testAppendHTMLEscape(t, "<", "<")
	testAppendHTMLEscape(t, "a", "a")
	testAppendHTMLEscape(t, `><"''`, "><"''")
	testAppendHTMLEscape(t, "foaxxx", "fo<b x='ss'>a</b>xxx")
}

func testAppendHTMLEscape(t *testing.T, s, expectedS string) {
	buf := AppendHTMLEscapeBytes(nil, []byte(s))
	if string(buf) != expectedS {
		t.Fatalf("unexpected html-escaped string %q. Expecting %q. Original string %q", buf, expectedS, s)
	}
}

func TestParseIPv4(t *testing.T) {
	t.Parallel()

	testParseIPv4(t, "0.0.0.0", true)
	testParseIPv4(t, "255.255.255.255", true)
	testParseIPv4(t, "123.45.67.89", true)

	// ipv6 shouldn't work
	testParseIPv4(t, "2001:4860:0:2001::68", false)

	// invalid ip
	testParseIPv4(t, "foobar", false)
	testParseIPv4(t, "1.2.3", false)
	testParseIPv4(t, "123.456.789.11", false)
}

func testParseIPv4(t *testing.T, ipStr string, isValid bool) {
	ip, err := ParseIPv4(nil, []byte(ipStr))
	if isValid {
		if err != nil {
			t.Fatalf("unexpected error when parsing ip %q: %s", ipStr, err)
		}
		s := string(AppendIPv4(nil, ip))
		if s != ipStr {
			t.Fatalf("unexpected ip parsed %q. Expecting %q", s, ipStr)
		}
	} else {
		if err == nil {
			t.Fatalf("expecting error when parsing ip %q", ipStr)
		}
	}
}

func TestAppendIPv4(t *testing.T) {
	t.Parallel()

	testAppendIPv4(t, "0.0.0.0", true)
	testAppendIPv4(t, "127.0.0.1", true)
	testAppendIPv4(t, "8.8.8.8", true)
	testAppendIPv4(t, "123.45.67.89", true)

	// ipv6 shouldn't work
	testAppendIPv4(t, "2001:4860:0:2001::68", false)
}

func testAppendIPv4(t *testing.T, ipStr string, isValid bool) {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		t.Fatalf("cannot parse ip %q", ipStr)
	}
	s := string(AppendIPv4(nil, ip))
	if isValid {
		if s != ipStr {
			t.Fatalf("unexpected ip %q. Expecting %q", s, ipStr)
		}
	} else {
		ipStr = "non-v4 ip passed to AppendIPv4"
		if s != ipStr {
			t.Fatalf("unexpected ip %q. Expecting %q", s, ipStr)
		}
	}
}

func testAppendUint(t *testing.T, n int) {
	expectedS := fmt.Sprintf("%d", n)
	s := AppendUint(nil, n)
	if string(s) != expectedS {
		t.Fatalf("unexpected uint %q. Expecting %q. n=%d", s, expectedS, n)
	}
}

func testWriteHexInt(t *testing.T, n int, expectedS string) {
	var w bytebufferpool.ByteBuffer
	bw := bufio.NewWriter(&w)
	if err := writeHexInt(bw, n); err != nil {
		t.Fatalf("unexpected error when writing hex %x: %s", n, err)
	}
	if err := bw.Flush(); err != nil {
		t.Fatalf("unexpected error when flushing hex %x: %s", n, err)
	}
	s := string(w.B)
	if s != expectedS {
		t.Fatalf("unexpected hex after writing %q. Expected %q", s, expectedS)
	}
}

func TestReadHexIntError(t *testing.T) {
	t.Parallel()

	testReadHexIntError(t, "")
	testReadHexIntError(t, "ZZZ")
	testReadHexIntError(t, "-123")
	testReadHexIntError(t, "+434")
}

func testReadHexIntError(t *testing.T, s string) {
	r := bytes.NewBufferString(s)
	br := bufio.NewReader(r)
	n, err := readHexInt(br)
	if err == nil {
		t.Fatalf("expecting error when reading hex int %q", s)
	}
	if n >= 0 {
		t.Fatalf("unexpected hex value read %d for hex int %q. must be negative", n, s)
	}
}

func testReadHexIntSuccess(t *testing.T, s string, expectedN int) {
	r := bytes.NewBufferString(s)
	br := bufio.NewReader(r)
	n, err := readHexInt(br)
	if err != nil {
		t.Fatalf("unexpected error: %s. s=%q", err, s)
	}
	if n != expectedN {
		t.Fatalf("unexpected hex int %d. Expected %d. s=%q", n, expectedN, s)
	}
}

func TestAppendHTTPDate(t *testing.T) {
	t.Parallel()

	d := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	s := string(AppendHTTPDate(nil, d))
	expectedS := "Tue, 10 Nov 2009 23:00:00 GMT"
	if s != expectedS {
		t.Fatalf("unexpected date %q. Expecting %q", s, expectedS)
	}

	b := []byte("prefix")
	s = string(AppendHTTPDate(b, d))
	if s[:len(b)] != string(b) {
		t.Fatalf("unexpected prefix %q. Expecting %q", s[:len(b)], b)
	}
	s = s[len(b):]
	if s != expectedS {
		t.Fatalf("unexpected date %q. Expecting %q", s, expectedS)
	}
}

func TestParseUintError(t *testing.T) {
	t.Parallel()

	// empty string
	testParseUintError(t, "")

	// negative value
	testParseUintError(t, "-123")

	// non-num
	testParseUintError(t, "foobar234")

	// non-num chars at the end
	testParseUintError(t, "123w")

	// floating point num
	testParseUintError(t, "1234.545")

	// too big num
	testParseUintError(t, "12345678901234567890")
	testParseUintError(t, "1234567890123456789012")
}

func TestParseUfloatSuccess(t *testing.T) {
	t.Parallel()

	testParseUfloatSuccess(t, "0", 0)
	testParseUfloatSuccess(t, "1.", 1.)
	testParseUfloatSuccess(t, ".1", 0.1)
	testParseUfloatSuccess(t, "123.456", 123.456)
	testParseUfloatSuccess(t, "123", 123)
	testParseUfloatSuccess(t, "1234e2", 1234e2)
	testParseUfloatSuccess(t, "1234E-5", 1234e-5)
	testParseUfloatSuccess(t, "1.234e+3", 1.234e+3)
}

func TestParseUfloatError(t *testing.T) {
	t.Parallel()

	// empty num
	testParseUfloatError(t, "")

	// negative num
	testParseUfloatError(t, "-123.53")

	// non-num chars
	testParseUfloatError(t, "123sdfsd")
	testParseUfloatError(t, "sdsf234")
	testParseUfloatError(t, "sdfdf")

	// non-num chars in exponent
	testParseUfloatError(t, "123e3s")
	testParseUfloatError(t, "12.3e-op")
	testParseUfloatError(t, "123E+SS5")

	// duplicate point
	testParseUfloatError(t, "1.3.4")

	// duplicate exponent
	testParseUfloatError(t, "123e5e6")

	// missing exponent
	testParseUfloatError(t, "123534e")
}

func testParseUfloatError(t *testing.T, s string) {
	n, err := ParseUfloat([]byte(s))
	if err == nil {
		t.Fatalf("Expecting error when parsing %q. obtained %f", s, n)
	}
	if n >= 0 {
		t.Fatalf("Expecting negative num instead of %f when parsing %q", n, s)
	}
}

func testParseUfloatSuccess(t *testing.T, s string, expectedF float64) {
	f, err := ParseUfloat([]byte(s))
	if err != nil {
		t.Fatalf("Unexpected error when parsing %q: %s", s, err)
	}
	delta := f - expectedF
	if delta < 0 {
		delta = -delta
	}
	if delta > expectedF*1e-10 {
		t.Fatalf("Unexpected value when parsing %q: %f. Expected %f", s, f, expectedF)
	}
}

func testParseUintError(t *testing.T, s string) {
	n, err := ParseUint([]byte(s))
	if err == nil {
		t.Fatalf("Expecting error when parsing %q. obtained %d", s, n)
	}
	if n >= 0 {
		t.Fatalf("Unexpected n=%d when parsing %q. Expected negative num", n, s)
	}
}

func testParseUintSuccess(t *testing.T, s string, expectedN int) {
	n, err := ParseUint([]byte(s))
	if err != nil {
		t.Fatalf("Unexpected error when parsing %q: %s", s, err)
	}
	if n != expectedN {
		t.Fatalf("Unexpected value %d. Expected %d. num=%q", n, expectedN, s)
	}
}

func TestAppendUnquotedArg(t *testing.T) {
	t.Parallel()

	testAppendUnquotedArg(t, "", "")
	testAppendUnquotedArg(t, "abc", "abc")
	testAppendUnquotedArg(t, "тест.abc", "тест.abc")
	testAppendUnquotedArg(t, "%D1%82%D0%B5%D1%81%D1%82%20%=&;:", "тест %=&;:")
}

func testAppendUnquotedArg(t *testing.T, s, expectedS string) {
	// test appending to nil
	result := AppendUnquotedArg(nil, []byte(s))
	if string(result) != expectedS {
		t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS)
	}

	// test appending to prefix
	prefix := "prefix"
	dst := []byte(prefix)
	dst = AppendUnquotedArg(dst, []byte(s))
	if !bytes.HasPrefix(dst, []byte(prefix)) {
		t.Fatalf("Unexpected prefix for AppendUnquotedArg(%q)=%q, want %q", s, dst, prefix)
	}
	result = dst[len(prefix):]
	if string(result) != expectedS {
		t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS)
	}

	// test in-place appending
	result = []byte(s)
	result = AppendUnquotedArg(result[:0], result)
	if string(result) != expectedS {
		t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS)
	}

	// verify AppendQuotedArg <-> AppendUnquotedArg conversion
	quotedS := AppendQuotedArg(nil, []byte(s))
	unquotedS := AppendUnquotedArg(nil, quotedS)
	if s != string(unquotedS) {
		t.Fatalf("Unexpected AppendUnquotedArg(AppendQuotedArg(%q))=%q, want %q", s, unquotedS, s)
	}
}
<commit_msg>add a test for AppendQuotedArg (#1255)<commit_after>package fasthttp

import (
	"bufio"
	"bytes"
	"fmt"
	"html"
	"net"
	"net/url"
	"testing"
	"time"

	"github.com/valyala/bytebufferpool"
)

func TestAppendQuotedArg(t *testing.T) {
	t.Parallel()

	// Sync with url.QueryEscape
	allcases := make([]byte, 256)
	for i := 0; i < 256; i++ {
		allcases[i] = byte(i)
	}
	res := string(AppendQuotedArg(nil, allcases))
	expect := url.QueryEscape(string(allcases))
	if res != expect {
		t.Fatalf("unexpected string %q. Expecting %q.", res, expect)
	}
}

func TestAppendHTMLEscape(t *testing.T) {
	t.Parallel()

	// Sync with html.EscapeString
	allcases := make([]byte, 256)
	for i := 0; i < 256; i++ {
		allcases[i] = byte(i)
	}
	res := string(AppendHTMLEscape(nil, string(allcases)))
	expect := string(html.EscapeString(string(allcases)))
	if res != expect {
		t.Fatalf("unexpected string %q. Expecting %q.", res, expect)
	}

	testAppendHTMLEscape(t, "", "")
	testAppendHTMLEscape(t, "<", "<")
	testAppendHTMLEscape(t, "a", "a")
	testAppendHTMLEscape(t, `><"''`, "><"''")
	testAppendHTMLEscape(t, "foaxxx", "fo<b x='ss'>a</b>xxx")
}

func testAppendHTMLEscape(t *testing.T, s, expectedS string) {
	buf := AppendHTMLEscapeBytes(nil, []byte(s))
	if string(buf) != expectedS {
		t.Fatalf("unexpected html-escaped string %q. Expecting %q. Original string %q", buf, expectedS, s)
	}
}

func TestParseIPv4(t *testing.T) {
	t.Parallel()

	testParseIPv4(t, "0.0.0.0", true)
	testParseIPv4(t, "255.255.255.255", true)
	testParseIPv4(t, "123.45.67.89", true)

	// ipv6 shouldn't work
	testParseIPv4(t, "2001:4860:0:2001::68", false)

	// invalid ip
	testParseIPv4(t, "foobar", false)
	testParseIPv4(t, "1.2.3", false)
	testParseIPv4(t, "123.456.789.11", false)
}

func testParseIPv4(t *testing.T, ipStr string, isValid bool) {
	ip, err := ParseIPv4(nil, []byte(ipStr))
	if isValid {
		if err != nil {
			t.Fatalf("unexpected error when parsing ip %q: %s", ipStr, err)
		}
		s := string(AppendIPv4(nil, ip))
		if s != ipStr {
			t.Fatalf("unexpected ip parsed %q. Expecting %q", s, ipStr)
		}
	} else {
		if err == nil {
			t.Fatalf("expecting error when parsing ip %q", ipStr)
		}
	}
}

func TestAppendIPv4(t *testing.T) {
	t.Parallel()

	testAppendIPv4(t, "0.0.0.0", true)
	testAppendIPv4(t, "127.0.0.1", true)
	testAppendIPv4(t, "8.8.8.8", true)
	testAppendIPv4(t, "123.45.67.89", true)

	// ipv6 shouldn't work
	testAppendIPv4(t, "2001:4860:0:2001::68", false)
}

func testAppendIPv4(t *testing.T, ipStr string, isValid bool) {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		t.Fatalf("cannot parse ip %q", ipStr)
	}
	s := string(AppendIPv4(nil, ip))
	if isValid {
		if s != ipStr {
			t.Fatalf("unexpected ip %q. Expecting %q", s, ipStr)
		}
	} else {
		ipStr = "non-v4 ip passed to AppendIPv4"
		if s != ipStr {
			t.Fatalf("unexpected ip %q. Expecting %q", s, ipStr)
		}
	}
}

func testAppendUint(t *testing.T, n int) {
	expectedS := fmt.Sprintf("%d", n)
	s := AppendUint(nil, n)
	if string(s) != expectedS {
		t.Fatalf("unexpected uint %q. Expecting %q. n=%d", s, expectedS, n)
	}
}

func testWriteHexInt(t *testing.T, n int, expectedS string) {
	var w bytebufferpool.ByteBuffer
	bw := bufio.NewWriter(&w)
	if err := writeHexInt(bw, n); err != nil {
		t.Fatalf("unexpected error when writing hex %x: %s", n, err)
	}
	if err := bw.Flush(); err != nil {
		t.Fatalf("unexpected error when flushing hex %x: %s", n, err)
	}
	s := string(w.B)
	if s != expectedS {
		t.Fatalf("unexpected hex after writing %q. Expected %q", s, expectedS)
	}
}

func TestReadHexIntError(t *testing.T) {
	t.Parallel()

	testReadHexIntError(t, "")
	testReadHexIntError(t, "ZZZ")
	testReadHexIntError(t, "-123")
	testReadHexIntError(t, "+434")
}

func testReadHexIntError(t *testing.T, s string) {
	r := bytes.NewBufferString(s)
	br := bufio.NewReader(r)
	n, err := readHexInt(br)
	if err == nil {
		t.Fatalf("expecting error when reading hex int %q", s)
	}
	if n >= 0 {
		t.Fatalf("unexpected hex value read %d for hex int %q. must be negative", n, s)
	}
}

func testReadHexIntSuccess(t *testing.T, s string, expectedN int) {
	r := bytes.NewBufferString(s)
	br := bufio.NewReader(r)
	n, err := readHexInt(br)
	if err != nil {
		t.Fatalf("unexpected error: %s. s=%q", err, s)
	}
	if n != expectedN {
		t.Fatalf("unexpected hex int %d. Expected %d. s=%q", n, expectedN, s)
	}
}

func TestAppendHTTPDate(t *testing.T) {
	t.Parallel()

	d := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	s := string(AppendHTTPDate(nil, d))
	expectedS := "Tue, 10 Nov 2009 23:00:00 GMT"
	if s != expectedS {
		t.Fatalf("unexpected date %q. Expecting %q", s, expectedS)
	}

	b := []byte("prefix")
	s = string(AppendHTTPDate(b, d))
	if s[:len(b)] != string(b) {
		t.Fatalf("unexpected prefix %q. Expecting %q", s[:len(b)], b)
	}
	s = s[len(b):]
	if s != expectedS {
		t.Fatalf("unexpected date %q. Expecting %q", s, expectedS)
	}
}

func TestParseUintError(t *testing.T) {
	t.Parallel()

	// empty string
	testParseUintError(t, "")

	// negative value
	testParseUintError(t, "-123")

	// non-num
	testParseUintError(t, "foobar234")

	// non-num chars at the end
	testParseUintError(t, "123w")

	// floating point num
	testParseUintError(t, "1234.545")

	// too big num
	testParseUintError(t, "12345678901234567890")
	testParseUintError(t, "1234567890123456789012")
}

func TestParseUfloatSuccess(t *testing.T) {
	t.Parallel()

	testParseUfloatSuccess(t, "0", 0)
	testParseUfloatSuccess(t, "1.", 1.)
	testParseUfloatSuccess(t, ".1", 0.1)
	testParseUfloatSuccess(t, "123.456", 123.456)
	testParseUfloatSuccess(t, "123", 123)
	testParseUfloatSuccess(t, "1234e2", 1234e2)
	testParseUfloatSuccess(t, "1234E-5", 1234e-5)
	testParseUfloatSuccess(t, "1.234e+3", 1.234e+3)
}

func TestParseUfloatError(t *testing.T) {
	t.Parallel()

	// empty num
	testParseUfloatError(t, "")

	// negative num
	testParseUfloatError(t, "-123.53")

	// non-num chars
	testParseUfloatError(t, "123sdfsd")
	testParseUfloatError(t, "sdsf234")
	testParseUfloatError(t, "sdfdf")

	// non-num chars in exponent
	testParseUfloatError(t, "123e3s")
	testParseUfloatError(t, "12.3e-op")
	testParseUfloatError(t, "123E+SS5")

	// duplicate point
	testParseUfloatError(t, "1.3.4")

	// duplicate exponent
	testParseUfloatError(t, "123e5e6")

	// missing exponent
	testParseUfloatError(t, "123534e")
}

func testParseUfloatError(t *testing.T, s string) {
	n, err := ParseUfloat([]byte(s))
	if err == nil {
		t.Fatalf("Expecting error when parsing %q. obtained %f", s, n)
	}
	if n >= 0 {
		t.Fatalf("Expecting negative num instead of %f when parsing %q", n, s)
	}
}

func testParseUfloatSuccess(t *testing.T, s string, expectedF float64) {
	f, err := ParseUfloat([]byte(s))
	if err != nil {
		t.Fatalf("Unexpected error when parsing %q: %s", s, err)
	}
	delta := f - expectedF
	if delta < 0 {
		delta = -delta
	}
	if delta > expectedF*1e-10 {
		t.Fatalf("Unexpected value when parsing %q: %f. Expected %f", s, f, expectedF)
	}
}

func testParseUintError(t *testing.T, s string) {
	n, err := ParseUint([]byte(s))
	if err == nil {
		t.Fatalf("Expecting error when parsing %q. obtained %d", s, n)
	}
	if n >= 0 {
		t.Fatalf("Unexpected n=%d when parsing %q. Expected negative num", n, s)
	}
}

func testParseUintSuccess(t *testing.T, s string, expectedN int) {
	n, err := ParseUint([]byte(s))
	if err != nil {
		t.Fatalf("Unexpected error when parsing %q: %s", s, err)
	}
	if n != expectedN {
		t.Fatalf("Unexpected value %d. Expected %d. num=%q", n, expectedN, s)
	}
}

func TestAppendUnquotedArg(t *testing.T) {
	t.Parallel()

	testAppendUnquotedArg(t, "", "")
	testAppendUnquotedArg(t, "abc", "abc")
	testAppendUnquotedArg(t, "тест.abc", "тест.abc")
	testAppendUnquotedArg(t, "%D1%82%D0%B5%D1%81%D1%82%20%=&;:", "тест %=&;:")
}

func testAppendUnquotedArg(t *testing.T, s, expectedS string) {
	// test appending to nil
	result := AppendUnquotedArg(nil, []byte(s))
	if string(result) != expectedS {
		t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS)
	}

	// test appending to prefix
	prefix := "prefix"
	dst := []byte(prefix)
	dst = AppendUnquotedArg(dst, []byte(s))
	if !bytes.HasPrefix(dst, []byte(prefix)) {
		t.Fatalf("Unexpected prefix for AppendUnquotedArg(%q)=%q, want %q", s, dst, prefix)
	}
	result = dst[len(prefix):]
	if string(result) != expectedS {
		t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS)
	}

	// test in-place appending
	result = []byte(s)
	result = AppendUnquotedArg(result[:0], result)
	if string(result) != expectedS {
		t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS)
	}

	// verify AppendQuotedArg <-> AppendUnquotedArg conversion
	quotedS := AppendQuotedArg(nil, []byte(s))
	unquotedS := AppendUnquotedArg(nil, quotedS)
	if s != string(unquotedS) {
		t.Fatalf("Unexpected AppendUnquotedArg(AppendQuotedArg(%q))=%q, want %q", s, unquotedS, s)
	}
}
<|endoftext|>


<commit_before>package compute

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/google/go-querystring/query"
)

// InstanceClient is a client for the Instance functions of the Compute API.
type InstanceClient struct {
	client        *APIClient
	compartmendID string
}

// Instance contains the instance reference from:
// https://docs.us-phoenix-1.oraclecloud.com/api/#/en/iaas/20160918/Instance/
type Instance struct {
	AvailabilityDomain string `json:"availabilityDomain"`
	CompartmentID      string `json:"compartmentId"`
	DisplayName        string `json:"displayName"`
	ExtendedMetadata   struct {
	} `json:"extendedMetadata"`
	ID             string `json:"id"`
	ImageID        string `json:"imageId"`
	IpxeScript     string `json:"ipxeScript"`
	LifecycleState string `json:"lifecycleState"`
	Metadata       struct {
	} `json:"metadata"`
	Region      string `json:"region"`
	Shape       string `json:"shape"`
	TimeCreated string `json:"timeCreated"`
}

// InstancesParameters
type InstancesParameters struct {
	AvailabilityDomain string `url:"availabilityDomain"` //The name of the Availability Domain.
	DisplayName        string `url:"displayName"`        //A user-friendly name. Does not have to be unique, and it's changeable. Avoid entering confidential information.
	Limit              int    `url:"limit"`              //The maximum number of items to return in a paginated "List" call.
	Page               string `url:"page"`               //The value of the opc-next-page response header from the previous "List" call
}

// NewInstanceClient provides a client interface for instance API calls
func (c *APIClient) NewInstanceClient(compartment string) *InstanceClient {
	return &InstanceClient{
		client:        c,
		compartmendID: compartment,
	}
}

// GetInstance returns a struct of an instance request given an instance ID
func (ic *InstanceClient) GetInstance(instanceID string) Instance {
	instance := Instance{}
	resp, err := ic.client.Get("/instances/" + instanceID)
	if err != nil {
		logrus.Error(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}
	if err = json.Unmarshal(body, &instance); err != nil {
		logrus.Fatalf("Unmarshal impossible: %s", err)
	}
	return instance
}

// ListInstances returns a slice struct of all instance
func (ic *InstanceClient) ListInstances(options *InstancesParameters) {
	queryString := url.QueryEscape(ic.compartmendID)
	if options != nil {
		v, _ := query.Values(*options)
		queryString = queryString + "&" + v.Encode()
	}
	resp, err := ic.client.Get(fmt.Sprintf("/instances?compartmentId=%s", queryString))
	if err != nil {
		logrus.Error(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}
	logrus.Info(string(body))
	// if err = json.Unmarshal(body, &instance); err != nil {
	// 	logrus.Fatalf("Unmarshal impossible: %s", err)
	// }
	return
}
<commit_msg>Only pass the querystring that are not empty<commit_after>package compute

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/google/go-querystring/query"
)

// InstanceClient is a client for the Instance functions of the Compute API.
type InstanceClient struct {
	client        *APIClient
	compartmendID string
}

// Instance contains the instance reference from:
// https://docs.us-phoenix-1.oraclecloud.com/api/#/en/iaas/20160918/Instance/
type Instance struct {
	AvailabilityDomain string `json:"availabilityDomain"`
	CompartmentID      string `json:"compartmentId"`
	DisplayName        string `json:"displayName"`
	ExtendedMetadata   struct {
	} `json:"extendedMetadata"`
	ID             string `json:"id"`
	ImageID        string `json:"imageId"`
	IpxeScript     string `json:"ipxeScript"`
	LifecycleState string `json:"lifecycleState"`
	Metadata       struct {
	} `json:"metadata"`
	Region      string `json:"region"`
	Shape       string `json:"shape"`
	TimeCreated string `json:"timeCreated"`
}

// InstancesParameters
type InstancesParameters struct {
	AvailabilityDomain string `url:"availabilityDomain,omitempty"` //The name of the Availability Domain.
	DisplayName        string `url:"displayName,omitempty"`        //A user-friendly name. Does not have to be unique, and it's changeable. Avoid entering confidential information.
	Limit              int    `url:"limit,omitempty"`              //The maximum number of items to return in a paginated "List" call.
	Page               string `url:"page,omitempty"`               //The value of the opc-next-page response header from the previous "List" call
}

// NewInstanceClient provides a client interface for instance API calls
func (c *APIClient) NewInstanceClient(compartment string) *InstanceClient {
	return &InstanceClient{
		client:        c,
		compartmendID: compartment,
	}
}

// GetInstance returns a struct of an instance request given an instance ID
func (ic *InstanceClient) GetInstance(instanceID string) Instance {
	instance := Instance{}
	resp, err := ic.client.Get("/instances/" + instanceID)
	if err != nil {
		logrus.Error(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}
	if err = json.Unmarshal(body, &instance); err != nil {
		logrus.Fatalf("Unmarshal impossible: %s", err)
	}
	return instance
}

// ListInstances returns a slice struct of all instance
func (ic *InstanceClient) ListInstances(options *InstancesParameters) {
	queryString := url.QueryEscape(ic.compartmendID)
	if options != nil {
		v, _ := query.Values(*options)
		queryString = queryString + "&" + v.Encode()
	}
	resp, err := ic.client.Get(fmt.Sprintf("/instances?compartmentId=%s", queryString))
	if err != nil {
		logrus.Error(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}
	logrus.Info(string(body))
	// if err = json.Unmarshal(body, &instance); err != nil {
	// 	logrus.Fatalf("Unmarshal impossible: %s", err)
	// }
	return
}
<|endoftext|>