<commit_before>package main

import (
	"log"
	"net/http"
	"encoding/json"
	"github.com/gorilla/mux"
	"net/http/pprof"
)

// TODO investigate project structure
// TODO investigate dependency management (no go get?)
// TODO investigate error handling and logging
// TODO investigate unit testing
// TODO investigate naming conventions (methods lower?)

// TODO implement runner using Channels : http://guzalexander.com/2013/12/06/golang-channels-tutorial.html

var algorithmRunner *AlgorithmRunner

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/api/cities", ListCities)

	// TODO reconsider api structure when all works
	/* TODO suggest /api/travel
		POST will start traveling
		GET will return current and finished (true|false) until finished
		DELETE will stop
	 */

	router.HandleFunc("/api/currentBest", CurrentBest)
	router.HandleFunc("/api/latestBest", LatestBest)
	router.HandleFunc("/api/stillRunning", StillRunning)
	router.HandleFunc("/api/startAlgorithm", StartAlgorithm)
	router.HandleFunc("/api/stopAlgorithm", StopAlgorithm)

	AttachProfiler(router)

	router.PathPrefix("/").Handler(http.FileServer(http.Dir("../frontend/app")))

	log.Fatal(http.ListenAndServe("localhost:8080", router))

}

func AttachProfiler(router *mux.Router) {
	router.HandleFunc("/debug/pprof/", pprof.Index)
	router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	router.HandleFunc("/debug/pprof/profile", pprof.Profile)
	router.HandleFunc("/debug/pprof/symbol", pprof.Symbol)

	// Manually add support for paths linked to by index page at /debug/pprof/
	router.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
	router.Handle("/debug/pprof/heap", pprof.Handler("heap"))
	router.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
	router.Handle("/debug/pprof/block", pprof.Handler("block"))
}

func ListCities(response http.ResponseWriter, request *http.Request) {
	response.Header().Set("Content-Type", "application/json")
	cities := getAllCities()
	json.NewEncoder(response).Encode(cities)
}

func CurrentBest(response http.ResponseWriter, request *http.Request) {
	json.NewEncoder(response).Encode(algorithmRunner.getCurrentBest())
}

func LatestBest(response http.ResponseWriter, request *http.Request) {
	// TODO: This is just testing REST and FE
	CurrentBest(response, request)
}
func StillRunning(response http.ResponseWriter, request *http.Request) {
	json.NewEncoder(response).Encode(algorithmRunner != nil && algorithmRunner.Running)
}

func StartAlgorithm(response http.ResponseWriter, request *http.Request) {
	if (algorithmRunner != nil && algorithmRunner.Running) {
		log.Println("Trying to start an running Traveler, skipping")
	} else {
		algorithmRunner = startAlgorithmRunner()
	}
}

func StopAlgorithm(response http.ResponseWriter, request *http.Request) {
	if (algorithmRunner == nil || !algorithmRunner.Running) {
		log.Println("Trying to stop an non-running Traveler, skipping")
	} else {
		algorithmRunner.stop()
	}
}
<commit_msg>Removed pprof<commit_after>package main

import (
	"log"
	"net/http"
	"encoding/json"
	"github.com/gorilla/mux"
)

// TODO investigate project structure
// TODO investigate dependency management (no go get?)
// TODO investigate error handling and logging
// TODO investigate unit testing
// TODO investigate naming conventions (methods lower?)

// TODO implement runner using Channels : http://guzalexander.com/2013/12/06/golang-channels-tutorial.html

var algorithmRunner *AlgorithmRunner

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/api/cities", ListCities)

	// TODO reconsider api structure when all works
	/* TODO suggest /api/travel
		POST will start traveling
		GET will return current and finished (true|false) until finished
		DELETE will stop
	 */

	router.HandleFunc("/api/currentBest", CurrentBest)
	router.HandleFunc("/api/latestBest", LatestBest)
	router.HandleFunc("/api/stillRunning", StillRunning)
	router.HandleFunc("/api/startAlgorithm", StartAlgorithm)
	router.HandleFunc("/api/stopAlgorithm", StopAlgorithm)

	router.PathPrefix("/").Handler(http.FileServer(http.Dir("../frontend/app")))

	log.Fatal(http.ListenAndServe("localhost:8080", router))
}

func ListCities(response http.ResponseWriter, request *http.Request) {
	response.Header().Set("Content-Type", "application/json")
	cities := getAllCities()
	json.NewEncoder(response).Encode(cities)
}

func CurrentBest(response http.ResponseWriter, request *http.Request) {
	json.NewEncoder(response).Encode(algorithmRunner.getCurrentBest())
}

func LatestBest(response http.ResponseWriter, request *http.Request) {
	// TODO: This is just testing REST and FE
	CurrentBest(response, request)
}
func StillRunning(response http.ResponseWriter, request *http.Request) {
	json.NewEncoder(response).Encode(algorithmRunner != nil && algorithmRunner.Running)
}

func StartAlgorithm(response http.ResponseWriter, request *http.Request) {
	if (algorithmRunner != nil && algorithmRunner.Running) {
		log.Println("Trying to start an running Traveler, skipping")
	} else {
		algorithmRunner = startAlgorithmRunner()
	}
}

func StopAlgorithm(response http.ResponseWriter, request *http.Request) {
	if (algorithmRunner == nil || !algorithmRunner.Running) {
		log.Println("Trying to stop an non-running Traveler, skipping")
	} else {
		algorithmRunner.stop()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Create a file, truncate it up to a particular size, then measure the
// throughput of repeatedly overwriting its contents without closing it each
// time. This is intended to measure the CPU efficiency of the file system
// rather than GCS throughput.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"
)

var fDir = flag.String("dir", "", "Directory within which to write the file.")
var fDuration = flag.Duration("duration", 10*time.Second, "How long to run.")
var fFileSize = flag.Int64("file_size", 1<<26, "Size of file to use.")
var fWriteSize = flag.Int64("write_size", 1<<20, "Size of each call to write(2).")

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

func run() (err error) {
	if *fDir == "" {
		err = errors.New("You must set --dir.")
		return
	}

	// Create a temporary file.
	log.Printf("Creating a temporary file in %s.", *fDir)

	f, err := ioutil.TempFile(*fDir, "write_locally")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	path := f.Name()

	// Make sure we clean it up later.
	defer func() {
		log.Printf("Deleting %s.", path)
		os.Remove(path)
	}()

	err = errors.New("TODO")
	return
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	flag.Parse()

	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Truncate and close when cleaning up.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Create a file, truncate it up to a particular size, then measure the
// throughput of repeatedly overwriting its contents without closing it each
// time. This is intended to measure the CPU efficiency of the file system
// rather than GCS throughput.
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"
)

var fDir = flag.String("dir", "", "Directory within which to write the file.")
var fDuration = flag.Duration("duration", 10*time.Second, "How long to run.")
var fFileSize = flag.Int64("file_size", 1<<26, "Size of file to use.")
var fWriteSize = flag.Int64("write_size", 1<<20, "Size of each call to write(2).")

////////////////////////////////////////////////////////////////////////
// main logic
////////////////////////////////////////////////////////////////////////

func run() (err error) {
	if *fDir == "" {
		err = errors.New("You must set --dir.")
		return
	}

	// Create a temporary file.
	log.Printf("Creating a temporary file in %s.", *fDir)

	f, err := ioutil.TempFile(*fDir, "write_locally")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	path := f.Name()

	// Make sure we clean it up later.
	defer func() {
		log.Printf("Truncating and closing %s.", path)
		f.Truncate(0)
		f.Close()

		log.Printf("Deleting %s.", path)
		os.Remove(path)
	}()

	err = errors.New("TODO")
	return
}

func main() {
	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
	flag.Parse()

	err := run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"nvim-go/nvim"
	"os"

	"github.com/cweill/gotests/gotests/process"
	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
)

func init() {
	plugin.HandleCommand("GoGenerateTest", &plugin.CommandOptions{NArgs: "*", Complete: "file"}, cmdGenerateTest)
}

func cmdGenerateTest(v *vim.Vim, files []string) {
	go GenerateTest(v, files)
}

// GenerateTest generates the test files based by current buffer or args files
// functions.
func GenerateTest(v *vim.Vim, files []string) error {
	b, err := v.CurrentBuffer()
	if err != nil {
		return nvim.Echoerr(v, "GoGenerateTest: %v", err)
	}

	if len(files) == 0 {
		f, err := v.BufferName(b)
		if err != nil {
			return nvim.Echoerr(v, "GoGenerateTest: %v", err)
		}
		files = append(files, f)
	}

	var opt = process.Options{
		AllFuncs:    true,
		WriteOutput: true,
		PrintInputs: true,
	}

	oldStdout := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	process.Run(w, files, &opt)

	w.Close()
	os.Stdout = oldStdout

	var out string
	scan := bufio.NewScanner(r)
	for scan.Scan() {
		out += scan.Text() + "\n"
	}

	return nvim.EchoRaw(v, out)
}
<commit_msg>GenerateTest: Add context.Build support and open the test dialog<commit_after>package commands

import (
	"bufio"
	"fmt"
	"log"
	"nvim-go/context"
	"nvim-go/nvim"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/cweill/gotests/gotests/process"
	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
)

func init() {
	plugin.HandleCommand("GoGenerateTest", &plugin.CommandOptions{NArgs: "*", Complete: "file", Eval: "expand('%:p:h')"}, cmdGenerateTest)
}

func cmdGenerateTest(v *vim.Vim, files []string, dir string) {
	go GenerateTest(v, files, dir)
}

// GenerateTest generates the test files based by current buffer or args files
// functions.
// TODO(zchee): Currently Support '-all' flag only.
// Needs support -excl, -exported, -i, -only flags.
func GenerateTest(v *vim.Vim, files []string, dir string) error {
	defer nvim.Profile(time.Now(), "GenerateTest")
	var ctxt = context.Build{}
	defer ctxt.SetContext(filepath.Dir(dir))()

	b, err := v.CurrentBuffer()
	if err != nil {
		return nvim.Echoerr(v, "GoGenerateTest: %v", err)
	}

	if len(files) == 0 {
		f, err := v.BufferName(b)
		if err != nil {
			return nvim.Echoerr(v, "GoGenerateTest: %v", err)
		}
		files = append(files, f)
	}

	var opt = process.Options{
		AllFuncs:    true,
		WriteOutput: true,
		PrintInputs: true,
	}

	oldStdout := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	process.Run(w, files, &opt)

	w.Close()
	os.Stdout = oldStdout

	var genFuncs string
	scan := bufio.NewScanner(r)
	for scan.Scan() {
		genFuncs += scan.Text() + "\n"
	}

	// TODO(zchee): More beautiful code
	suffix := "_test.go "
	var ftests, ftestsRel string
	for _, f := range files {
		fnAbs := strings.Split(f, filepath.Ext(f))
		ftests += fnAbs[0] + suffix

		_, fnRel := filepath.Split(fnAbs[0])
		ftestsRel += fnRel + suffix
	}
	log.Println(ftests, ftestsRel)

	ask := fmt.Sprintf("%s\nGoGenerateTest: Generated %s\nGoGenerateTest: Open it? (y, n): ", genFuncs, ftestsRel)
	var answer interface{}
	if err := v.Call("input", &answer, ask); err != nil {
		return err
	}

	// TODO(zchee): Support open the ftests[0] file only.
	// If passes multiple files for 'edit' commands, occur 'E172: Only one file name allowed' errror.
	if answer.(string) != "n" {
		return v.Command(fmt.Sprintf("edit %s", ftests))
	}

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"time"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	f                  FilerOptions
	filerStartS3       *bool
	filerS3Options     S3Options
	filerStartWebDav   *bool
	filerWebDavOptions WebDavOption
	filerStartIam      *bool
	filerIamOptions    IamOptions
)

type FilerOptions struct {
	masters                 map[string]pb.ServerAddress
	mastersString           *string
	ip                      *string
	bindIp                  *string
	port                    *int
	portGrpc                *int
	publicPort              *int
	filerGroup              *string
	collection              *string
	defaultReplicaPlacement *string
	disableDirListing       *bool
	maxMB                   *int
	dirListingLimit         *int
	dataCenter              *string
	rack                    *string
	enableNotification      *bool
	disableHttp             *bool
	cipher                  *bool
	metricsHttpPort         *int
	saveToFilerLimit        *int
	defaultLevelDbDirectory *string
	concurrentUploadLimitMB *int
	debug                   *bool
	debugPort               *int
	localSocket             *string
}

func init() {
	cmdFiler.Run = runFiler // break init cycle
	f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
	f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
	f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
	f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
	f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
	f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")

	// start s3 on filer
	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
	filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
	filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
	filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
	filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")

	// start webdav on filer
	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
	filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port")
	filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files")
	filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files")
	filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
	filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")

	// start iam on filer
	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
}

var cmdFiler = &Command{
	UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
	Short:     "start a file server that points to a master server, or a list of master servers",
	Long: `start a file server which accepts REST operation for any files.

	//create or overwrite the file, the directories /path/to will be automatically created
	POST /path/to/file
	//get the file content
	GET /path/to/file
	//create or overwrite the file, the filename in the multipart request will be used
	POST /path/to/
	//return a json format subdirectory and files listing
	GET /path/to/

	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
	If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".

	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"

`,
}

func runFiler(cmd *Command, args []string) bool {
	if *f.debug {
		go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil)
	}

	util.LoadConfiguration("security", false)

	go stats_collect.StartMetricsServer(*f.metricsHttpPort)

	filerAddress := util.JoinHostPort(*f.ip, *f.port)
	startDelay := time.Duration(2)
	if *filerStartS3 {
		filerS3Options.filer = &filerAddress
		filerS3Options.bindIp = f.bindIp
		filerS3Options.localFilerSocket = f.localSocket
		go func() {
			time.Sleep(startDelay * time.Second)
			filerS3Options.startS3Server()
		}()
		startDelay++
	} else {
		*f.localSocket = ""
	}

	if *filerStartWebDav {
		filerWebDavOptions.filer = &filerAddress
		go func() {
			time.Sleep(startDelay * time.Second)
			filerWebDavOptions.startWebDav()
		}()
		startDelay++
	}

	if *filerStartIam {
		filerIamOptions.filer = &filerAddress
		filerIamOptions.masters = f.mastersString
		go func() {
			time.Sleep(startDelay * time.Second)
			filerIamOptions.startIamServer()
		}()
	}

	f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()

	f.startFiler()

	return true
}

func (fo *FilerOptions) startFiler() {

	defaultMux := http.NewServeMux()
	publicVolumeMux := defaultMux

	if *fo.publicPort != 0 {
		publicVolumeMux = http.NewServeMux()
	}
	if *fo.portGrpc == 0 {
		*fo.portGrpc = 10000 + *fo.port
	}
	if *fo.bindIp == "" {
		*fo.bindIp = *fo.ip
	}

	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")

	filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)

	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
		Masters:               fo.masters,
		FilerGroup:            *fo.filerGroup,
		Collection:            *fo.collection,
		DefaultReplication:    *fo.defaultReplicaPlacement,
		DisableDirListing:     *fo.disableDirListing,
		MaxMB:                 *fo.maxMB,
		DirListingLimit:       *fo.dirListingLimit,
		DataCenter:            *fo.dataCenter,
		Rack:                  *fo.rack,
		DefaultLevelDbDir:     defaultLevelDbDirectory,
		DisableHttp:           *fo.disableHttp,
		Host:                  filerAddress,
		Cipher:                *fo.cipher,
		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
	})
	if nfs_err != nil {
		glog.Fatalf("Filer startup error: %v", nfs_err)
	}

	if *fo.publicPort != 0 {
		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
		publicListener, localPublicListner, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
		if e != nil {
			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
		}
		go func() {
			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
				glog.Fatalf("Volume server fail to serve public: %v", e)
			}
		}()
		if localPublicListner != nil {
			go func() {
				if e := http.Serve(localPublicListner, publicVolumeMux); e != nil {
					glog.Errorf("Volume server fail to serve public: %v", e)
				}
			}()
		}
	}

	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
		*fo.bindIp, *fo.port,
		time.Duration(10)*time.Second,
	)
	if e != nil {
		glog.Fatalf("Filer listener error: %v", e)
	}

	// starting grpc server
	grpcPort := *fo.portGrpc
	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
	reflection.Register(grpcS)
	if grpcLocalL != nil {
		go grpcS.Serve(grpcLocalL)
	}
	go grpcS.Serve(grpcL)

	httpS := &http.Server{Handler: defaultMux}
	if runtime.GOOS != "windows" {
		if *fo.localSocket == "" {
			*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
			if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
				glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
			}
		}
		go func() {
			// start on local unix socket
			filerSocketListener, err := net.Listen("unix", *fo.localSocket)
			if err != nil {
				glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
			}
			httpS.Serve(filerSocketListener)
		}()
	}
	if filerLocalListener != nil {
		go func() {
			if err := httpS.Serve(filerLocalListener); err != nil {
				glog.Errorf("Filer Fail to serve: %v", e)
			}
		}()
	}
	if err := httpS.Serve(filerListener); err != nil {
		glog.Fatalf("Filer Fail to serve: %v", e)
	}

}
<commit_msg>filer: fix customizable local socket file name<commit_after>package command

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"time"

	"google.golang.org/grpc/reflection"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	weed_server "github.com/chrislusf/seaweedfs/weed/server"
	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	f                  FilerOptions
	filerStartS3       *bool
	filerS3Options     S3Options
	filerStartWebDav   *bool
	filerWebDavOptions WebDavOption
	filerStartIam      *bool
	filerIamOptions    IamOptions
)

type FilerOptions struct {
	masters                 map[string]pb.ServerAddress
	mastersString           *string
	ip                      *string
	bindIp                  *string
	port                    *int
	portGrpc                *int
	publicPort              *int
	filerGroup              *string
	collection              *string
	defaultReplicaPlacement *string
	disableDirListing       *bool
	maxMB                   *int
	dirListingLimit         *int
	dataCenter              *string
	rack                    *string
	enableNotification      *bool
	disableHttp             *bool
	cipher                  *bool
	metricsHttpPort         *int
	saveToFilerLimit        *int
	defaultLevelDbDirectory *string
	concurrentUploadLimitMB *int
	debug                   *bool
	debugPort               *int
	localSocket             *string
}

func init() {
	cmdFiler.Run = runFiler // break init cycle
	f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
	f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
	f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
	f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
	f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
	f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")

	// start s3 on filer
	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
	filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
	filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
	filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
	filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")

	// start webdav on filer
	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
	filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port")
	filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files")
	filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files")
	filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
	filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")

	// start iam on filer
	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
}

var cmdFiler = &Command{
	UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
	Short:     "start a file server that points to a master server, or a list of master servers",
	Long: `start a file server which accepts REST operation for any files.

	//create or overwrite the file, the directories /path/to will be automatically created
	POST /path/to/file
	//get the file content
	GET /path/to/file
	//create or overwrite the file, the filename in the multipart request will be used
	POST /path/to/
	//return a json format subdirectory and files listing
	GET /path/to/

	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
	If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".

	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"

`,
}

func runFiler(cmd *Command, args []string) bool {
	if *f.debug {
		go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil)
	}

	util.LoadConfiguration("security", false)

	go stats_collect.StartMetricsServer(*f.metricsHttpPort)

	filerAddress := util.JoinHostPort(*f.ip, *f.port)
	startDelay := time.Duration(2)
	if *filerStartS3 {
		filerS3Options.filer = &filerAddress
		filerS3Options.bindIp = f.bindIp
		filerS3Options.localFilerSocket = f.localSocket
		go func() {
			time.Sleep(startDelay * time.Second)
			filerS3Options.startS3Server()
		}()
		startDelay++
	}

	if *filerStartWebDav {
		filerWebDavOptions.filer = &filerAddress
		go func() {
			time.Sleep(startDelay * time.Second)
			filerWebDavOptions.startWebDav()
		}()
		startDelay++
	}

	if *filerStartIam {
		filerIamOptions.filer = &filerAddress
		filerIamOptions.masters = f.mastersString
		go func() {
			time.Sleep(startDelay * time.Second)
			filerIamOptions.startIamServer()
		}()
	}

	f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()

	f.startFiler()

	return true
}

func (fo *FilerOptions) startFiler() {

	defaultMux := http.NewServeMux()
	publicVolumeMux := defaultMux

	if *fo.publicPort != 0 {
		publicVolumeMux = http.NewServeMux()
	}
	if *fo.portGrpc == 0 {
		*fo.portGrpc = 10000 + *fo.port
	}
	if *fo.bindIp == "" {
		*fo.bindIp = *fo.ip
	}

	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")

	filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)

	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
		Masters:               fo.masters,
		FilerGroup:            *fo.filerGroup,
		Collection:            *fo.collection,
		DefaultReplication:    *fo.defaultReplicaPlacement,
		DisableDirListing:     *fo.disableDirListing,
		MaxMB:                 *fo.maxMB,
		DirListingLimit:       *fo.dirListingLimit,
		DataCenter:            *fo.dataCenter,
		Rack:                  *fo.rack,
		DefaultLevelDbDir:     defaultLevelDbDirectory,
		DisableHttp:           *fo.disableHttp,
		Host:                  filerAddress,
		Cipher:                *fo.cipher,
		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
	})
	if nfs_err != nil {
		glog.Fatalf("Filer startup error: %v", nfs_err)
	}

	if *fo.publicPort != 0 {
		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
		publicListener, localPublicListner, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
		if e != nil {
			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
		}
		go func() {
			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
				glog.Fatalf("Volume server fail to serve public: %v", e)
			}
		}()
		if localPublicListner != nil {
			go func() {
				if e := http.Serve(localPublicListner, publicVolumeMux); e != nil {
					glog.Errorf("Volume server fail to serve public: %v", e)
				}
			}()
		}
	}

	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
		*fo.bindIp, *fo.port,
		time.Duration(10)*time.Second,
	)
	if e != nil {
		glog.Fatalf("Filer listener error: %v", e)
	}

	// starting grpc server
	grpcPort := *fo.portGrpc
	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
	if err != nil {
		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
	}
	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
	reflection.Register(grpcS)
	if grpcLocalL != nil {
		go grpcS.Serve(grpcLocalL)
	}
	go grpcS.Serve(grpcL)

	httpS := &http.Server{Handler: defaultMux}
	if runtime.GOOS != "windows" {
		if *fo.localSocket == "" {
			*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
			if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
				glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
			}
		}
		go func() {
			// start on local unix socket
			filerSocketListener, err := net.Listen("unix", *fo.localSocket)
			if err != nil {
				glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
			}
			httpS.Serve(filerSocketListener)
		}()
	}
	if filerLocalListener != nil {
		go func() {
			if err := httpS.Serve(filerLocalListener); err != nil {
				glog.Errorf("Filer Fail to serve: %v", e)
			}
		}()
	}
	if err := httpS.Serve(filerListener); err != nil {
		glog.Fatalf("Filer Fail to serve: %v", e)
	}

}
<|endoftext|>


<commit_before>package command

import (
	"os"
	"time"
)

type MountOptions struct {
	filer                       *string
	filerMountRootPath          *string
	dir                         *string
	dirAutoCreate               *bool
	collection                  *string
	replication                 *string
	ttlSec                      *int
	chunkSizeLimitMB            *int
	concurrentWriters           *int
	cacheDir                    *string
	cacheSizeMB                 *int64
	dataCenter                  *string
	allowOthers                 *bool
	umaskString                 *string
	nonempty                    *bool
	outsideContainerClusterMode *bool
	uidMap                      *string
	gidMap                      *string
}

var (
	mountOptions       MountOptions
	mountCpuProfile    *string
	mountMemProfile    *string
	mountReadRetryTime *time.Duration
)

func init() {
	cmdMount.Run = runMount // break init cycle
	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
	mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to")
	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 0, "limit concurrent goroutine writers if not 0")
	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
	mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
	mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access volume servers with publicUrl")
	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")

	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
	mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
}

var cmdMount = &Command{
	UsageLine: "mount -filer=localhost:8888 -dir=/some/dir",
	Short:     "mount weed filer to a directory as file system in userspace(FUSE)",
	Long: `mount weed filer to userspace.

  Pre-requisites:
  1) have SeaweedFS master and volume servers running
  2) have a "weed filer" running
  These 2 requirements can be achieved with one command "weed server -filer=true"

  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
  Linux, and OS X.

  On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

  `,
}
<commit_msg>mount: default to 128 concurrent writers<commit_after>package command

import (
	"os"
	"time"
)

type MountOptions struct {
	filer                       *string
	filerMountRootPath          *string
	dir                         *string
	dirAutoCreate               *bool
	collection                  *string
	replication                 *string
	ttlSec                      *int
	chunkSizeLimitMB            *int
	concurrentWriters           *int
	cacheDir                    *string
	cacheSizeMB                 *int64
	dataCenter                  *string
	allowOthers                 *bool
	umaskString                 *string
	nonempty                    *bool
	outsideContainerClusterMode *bool
	uidMap                      *string
	gidMap                      *string
}

var (
	mountOptions       MountOptions
	mountCpuProfile    *string
	mountMemProfile    *string
	mountReadRetryTime *time.Duration
)

func init() {
	cmdMount.Run = runMount // break init cycle
	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
	mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to")
	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files")
	mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 128, "limit concurrent goroutine writers if not 0")
	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)")
	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
	mountOptions.nonempty = cmdMount.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory")
	mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access volume servers with publicUrl")
	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")

	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
	mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
}

var cmdMount = &Command{
	UsageLine: "mount -filer=localhost:8888 -dir=/some/dir",
	Short:     "mount weed filer to a directory as file system in userspace(FUSE)",
	Long: `mount weed filer to userspace.

  Pre-requisites:
  1) have SeaweedFS master and volume servers running
  2) have a "weed filer" running
  These 2 requirements can be achieved with one command "weed server -filer=true"

  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
  Linux, and OS X.

  On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

  `,
}
<|endoftext|>


<commit_before>package command

import (
	"context"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	cmdWatch.Run = runWatch // break init cycle
}

var cmdWatch = &Command{
	UsageLine: "watch [-filer=localhost:8888] [-target=/]",
	Short:     "see recent changes on a filer",
	Long: `See recent changes on a filer.

  `,
}

var (
	watchFiler   = cmdWatch.Flag.String("filer", "localhost:8888", "filer hostname:port")
	watchTarget  = cmdWatch.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer")
	watchStart   = cmdWatch.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
	watchPattern = cmdWatch.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ")
)

func runWatch(cmd *Command, args []string) bool {

	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

	var filterFunc func(dir, fname string) bool
	if *watchPattern != "" {
		if strings.Contains(*watchPattern, "/") {
			println("watch path pattern", *watchPattern)
			filterFunc = func(dir, fname string) bool {
				matched, err := filepath.Match(*watchPattern, dir+"/"+fname)
				if err != nil {
					fmt.Printf("error: %v", err)
				}
				return matched
			}
		} else {
			println("watch file pattern", *watchPattern)
			filterFunc = func(dir, fname string) bool {
				matched, err := filepath.Match(*watchPattern, fname)
				if err != nil {
					fmt.Printf("error: %v", err)
				}
				return matched
			}
		}
	}

	shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool {
		if filterFunc == nil {
			return true
		}
		if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil {
			return false
		}
		if resp.EventNotification.OldEntry != nil && filterFunc(resp.Directory, resp.EventNotification.OldEntry.Name) {
			return true
		}
		if resp.EventNotification.NewEntry != nil && filterFunc(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry.Name) {
			return true
		}
		return false
	}

	watchErr := pb.WithFilerClient(*watchFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
			ClientName: "watch",
			PathPrefix: *watchTarget,
			SinceNs:    time.Now().Add(-*watchStart).UnixNano(),
		})
		if err != nil {
			return fmt.Errorf("listen: %v", err)
		}

		for {
			resp, listenErr := stream.Recv()
			if listenErr == io.EOF {
				return nil
			}
			if listenErr != nil {
				return listenErr
			}
			if !shouldPrint(resp) {
				continue
			}
			fmt.Printf("%+v\n", resp.EventNotification)
		}

	})
	if watchErr != nil {
		fmt.Printf("watch %s: %v\n", *watchFiler, watchErr)
	}

	return true
}
<commit_msg>watch: adjust output format<commit_after>package command

import (
	"context"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	cmdWatch.Run = runWatch // break init cycle
}

var cmdWatch = &Command{
	UsageLine: "watch [-filer=localhost:8888] [-target=/]",
	Short:     "see recent changes on a filer",
	Long: `See recent changes on a filer.

  `,
}

var (
	watchFiler   = cmdWatch.Flag.String("filer", "localhost:8888", "filer hostname:port")
	watchTarget  = cmdWatch.Flag.String("pathPrefix", "/", "path to a folder or file, or common prefix for the folders or files on filer")
	watchStart   = cmdWatch.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
	watchPattern = cmdWatch.Flag.String("pattern", "", "full path or just filename pattern, ex: \"/home/?opher\", \"*.pdf\", see https://golang.org/pkg/path/filepath/#Match ")
)

func runWatch(cmd *Command, args []string) bool {

	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

	var filterFunc func(dir, fname string) bool
	if *watchPattern != "" {
		if strings.Contains(*watchPattern, "/") {
			println("watch path pattern", *watchPattern)
			filterFunc = func(dir, fname string) bool {
				matched, err := filepath.Match(*watchPattern, dir+"/"+fname)
				if err != nil {
					fmt.Printf("error: %v", err)
				}
				return matched
			}
		} else {
			println("watch file pattern", *watchPattern)
			filterFunc = func(dir, fname string) bool {
				matched, err := filepath.Match(*watchPattern, fname)
				if err != nil {
					fmt.Printf("error: %v", err)
				}
				return matched
			}
		}
	}

	shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool {
		if filterFunc == nil {
			return true
		}
		if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil {
			return false
		}
		if resp.EventNotification.OldEntry != nil && filterFunc(resp.Directory, resp.EventNotification.OldEntry.Name) {
			return true
		}
		if resp.EventNotification.NewEntry != nil && filterFunc(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry.Name) {
			return true
		}
		return false
	}

	watchErr := pb.WithFilerClient(*watchFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
			ClientName: "watch",
			PathPrefix: *watchTarget,
			SinceNs:    time.Now().Add(-*watchStart).UnixNano(),
		})
		if err != nil {
			return fmt.Errorf("listen: %v", err)
		}

		for {
			resp, listenErr := stream.Recv()
			if listenErr == io.EOF {
				return nil
			}
			if listenErr != nil {
				return listenErr
			}
			if !shouldPrint(resp) {
				continue
			}
			fmt.Printf("dir:%s %+v\n", resp.Directory, resp.EventNotification)
		}

	})
	if watchErr != nil {
		fmt.Printf("watch %s: %v\n", *watchFiler, watchErr)
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"errors"
	"syscall"
)

// TODO(rsc): Fall back to copy of zoneinfo files.

// BUG(brainman,rsc): On Windows, the operating system does not provide complete
// time zone information.
// The implementation assumes that this year's rules for daylight savings
// time apply to all previous and future years as well.
// Also, time zone abbreviations are unavailable.  The implementation constructs
// them using the capital letters from a longer time zone description.

// abbrev returns the abbreviation to use for the given zone name.
func abbrev(name []uint16) string {
	// name is 'Pacific Standard Time' but we want 'PST'.
	// Extract just capital letters.  It's not perfect but the
	// information we need is not available from the kernel.
	// Because time zone abbreviations are not unique,
	// Windows refuses to expose them.
	//
	// http://social.msdn.microsoft.com/Forums/eu/vclanguage/thread/a87e1d25-fb71-4fe0-ae9c-a9578c9753eb
	// http://stackoverflow.com/questions/4195948/windows-time-zone-abbreviations-in-asp-net
	var short []rune
	for _, c := range name {
		if 'A' <= c && c <= 'Z' {
			short = append(short, rune(c))
		}
	}
	return string(short)
}

// pseudoUnix returns the pseudo-Unix time (seconds since Jan 1 1970 *LOCAL TIME*)
// denoted by the system date+time d in the given year.
// It is up to the caller to convert this local time into a UTC-based time.
func pseudoUnix(year int, d *syscall.Systemtime) int64 {
	// Windows specifies daylight savings information in "day in month" format:
	// d.Month is month number (1-12)
	// d.DayOfWeek is appropriate weekday (Sunday=0 to Saturday=6)
	// d.Day is week within the month (1 to 5, where 5 is last week of the month)
	// d.Hour, d.Minute and d.Second are absolute time
	day := 1
	t := Date(year, Month(d.Month), day, int(d.Hour), int(d.Minute), int(d.Second), 0, UTC)
	i := int(d.DayOfWeek) - int(t.Weekday())
	if i < 0 {
		i += 7
	}
	day += i
	if week := int(d.Day) - 1; week < 4 {
		day += week * 7
	} else {
		// "Last" instance of the day.
		day += 4 * 7
		if day > daysIn(Month(d.Month), year) {
			day -= 7
		}
	}
	return t.sec + int64(day-1)*secondsPerDay
}

func initLocalFromTZI(i *syscall.Timezoneinformation) {
	l := &localLoc

	nzone := 1
	if i.StandardDate.Month > 0 {
		nzone++
	}
	l.zone = make([]zone, nzone)

	std := &l.zone[0]
	std.name = abbrev(i.StandardName[0:])
	if nzone == 1 {
		// No daylight savings.
		std.offset = -int(i.Bias) * 60
		l.cacheStart = -1 << 63
		l.cacheEnd = 1<<63 - 1
		l.cacheZone = std
		return
	}

	// StandardBias must be ignored if StandardDate is not set,
	// so this computation is delayed until after the nzone==1
	// return above.
	std.offset = -int(i.Bias+i.StandardBias) * 60

	dst := &l.zone[1]
	dst.name = abbrev(i.DaylightName[0:])
	dst.offset = -int(i.Bias+i.DaylightBias) * 60
	dst.isDST = true

	// Arrange so that d0 is first transition date, d1 second,
	// i0 is index of zone after first transition, i1 second.
	d0 := &i.StandardDate
	d1 := &i.DaylightDate
	i0 := 0
	i1 := 1
	if d0.Month > d1.Month {
		d0, d1 = d1, d0
		i0, i1 = i1, i0
	}

	// 2 tx per year, 100 years on each side of this year
	l.tx = make([]zoneTrans, 400)

	t := Now().UTC()
	year := t.Year()
	txi := 0
	for y := year - 100; y < year+100; y++ {
		tx := &l.tx[txi]
		tx.when = pseudoUnix(y, d0) - int64(l.zone[i1].offset)
		tx.index = uint8(i0)
		txi++

		tx = &l.tx[txi]
		tx.when = pseudoUnix(y, d1) - int64(l.zone[i0].offset)
		tx.index = uint8(i1)
		txi++
	}
}

var usPacific = syscall.Timezoneinformation{
	Bias: 8 * 60,
	StandardName: [32]uint16{
		'P', 'a', 'c', 'i', 'f', 'i', 'c', ' ', 'S', 't', 'a', 'n', 'd', 'a', 'r', 'd', ' ', 'T', 'i', 'm', 'e',
	},
	StandardDate: syscall.Systemtime{Month: 11, Day: 1, Hour: 2},
	DaylightName: [32]uint16{
		'P', 'a', 'c', 'i', 'f', 'i', 'c', ' ', 'D', 'a', 'y', 'l', 'i', 'g', 'h', 't', ' ', 'T', 'i', 'm', 'e',
	},
	DaylightDate: syscall.Systemtime{Month: 3, Day: 2, Hour: 2},
	DaylightBias: -60,
}

func initTestingZone() {
	initLocalFromTZI(&usPacific)
}

func initLocal() {
	var i syscall.Timezoneinformation
	if _, err := syscall.GetTimeZoneInformation(&i); err != nil {
		localLoc.name = "UTC"
		return
	}
	initLocalFromTZI(&i)
}

// TODO(rsc): Implement.
func loadLocation(name string) (*Location, error) {
	return nil, errors.New("unknown time zone " + name)
}
<commit_msg>time: fix windows build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"errors"
	"syscall"
)

// TODO(rsc): Fall back to copy of zoneinfo files.

// BUG(brainman,rsc): On Windows, the operating system does not provide complete
// time zone information.
// The implementation assumes that this year's rules for daylight savings
// time apply to all previous and future years as well.
// Also, time zone abbreviations are unavailable.  The implementation constructs
// them using the capital letters from a longer time zone description.

// abbrev returns the abbreviation to use for the given zone name.
func abbrev(name []uint16) string {
	// name is 'Pacific Standard Time' but we want 'PST'.
	// Extract just capital letters.  It's not perfect but the
	// information we need is not available from the kernel.
	// Because time zone abbreviations are not unique,
	// Windows refuses to expose them.
	//
	// http://social.msdn.microsoft.com/Forums/eu/vclanguage/thread/a87e1d25-fb71-4fe0-ae9c-a9578c9753eb
	// http://stackoverflow.com/questions/4195948/windows-time-zone-abbreviations-in-asp-net
	var short []rune
	for _, c := range name {
		if 'A' <= c && c <= 'Z' {
			short = append(short, rune(c))
		}
	}
	return string(short)
}

// pseudoUnix returns the pseudo-Unix time (seconds since Jan 1 1970 *LOCAL TIME*)
// denoted by the system date+time d in the given year.
// It is up to the caller to convert this local time into a UTC-based time.
func pseudoUnix(year int, d *syscall.Systemtime) int64 {
	// Windows specifies daylight savings information in "day in month" format:
	// d.Month is month number (1-12)
	// d.DayOfWeek is appropriate weekday (Sunday=0 to Saturday=6)
	// d.Day is week within the month (1 to 5, where 5 is last week of the month)
	// d.Hour, d.Minute and d.Second are absolute time
	day := 1
	t := Date(year, Month(d.Month), day, int(d.Hour), int(d.Minute), int(d.Second), 0, UTC)
	i := int(d.DayOfWeek) - int(t.Weekday())
	if i < 0 {
		i += 7
	}
	day += i
	if week := int(d.Day) - 1; week < 4 {
		day += week * 7
	} else {
		// "Last" instance of the day.
		day += 4 * 7
		if day > daysIn(Month(d.Month), year) {
			day -= 7
		}
	}
	return t.sec + int64(day-1)*secondsPerDay + internalToUnix
}

func initLocalFromTZI(i *syscall.Timezoneinformation) {
	l := &localLoc

	nzone := 1
	if i.StandardDate.Month > 0 {
		nzone++
	}
	l.zone = make([]zone, nzone)

	std := &l.zone[0]
	std.name = abbrev(i.StandardName[0:])
	if nzone == 1 {
		// No daylight savings.
		std.offset = -int(i.Bias) * 60
		l.cacheStart = -1 << 63
		l.cacheEnd = 1<<63 - 1
		l.cacheZone = std
		return
	}

	// StandardBias must be ignored if StandardDate is not set,
	// so this computation is delayed until after the nzone==1
	// return above.
	std.offset = -int(i.Bias+i.StandardBias) * 60

	dst := &l.zone[1]
	dst.name = abbrev(i.DaylightName[0:])
	dst.offset = -int(i.Bias+i.DaylightBias) * 60
	dst.isDST = true

	// Arrange so that d0 is first transition date, d1 second,
	// i0 is index of zone after first transition, i1 second.
	d0 := &i.StandardDate
	d1 := &i.DaylightDate
	i0 := 0
	i1 := 1
	if d0.Month > d1.Month {
		d0, d1 = d1, d0
		i0, i1 = i1, i0
	}

	// 2 tx per year, 100 years on each side of this year
	l.tx = make([]zoneTrans, 400)

	t := Now().UTC()
	year := t.Year()
	txi := 0
	for y := year - 100; y < year+100; y++ {
		tx := &l.tx[txi]
		tx.when = pseudoUnix(y, d0) - int64(l.zone[i1].offset)
		tx.index = uint8(i0)
		txi++

		tx = &l.tx[txi]
		tx.when = pseudoUnix(y, d1) - int64(l.zone[i0].offset)
		tx.index = uint8(i1)
		txi++
	}
}

var usPacific = syscall.Timezoneinformation{
	Bias: 8 * 60,
	StandardName: [32]uint16{
		'P', 'a', 'c', 'i', 'f', 'i', 'c', ' ', 'S', 't', 'a', 'n', 'd', 'a', 'r', 'd', ' ', 'T', 'i', 'm', 'e',
	},
	StandardDate: syscall.Systemtime{Month: 11, Day: 1, Hour: 2},
	DaylightName: [32]uint16{
		'P', 'a', 'c', 'i', 'f', 'i', 'c', ' ', 'D', 'a', 'y', 'l', 'i', 'g', 'h', 't', ' ', 'T', 'i', 'm', 'e',
	},
	DaylightDate: syscall.Systemtime{Month: 3, Day: 2, Hour: 2},
	DaylightBias: -60,
}

func initTestingZone() {
	initLocalFromTZI(&usPacific)
}

func initLocal() {
	var i syscall.Timezoneinformation
	if _, err := syscall.GetTimeZoneInformation(&i); err != nil {
		localLoc.name = "UTC"
		return
	}
	initLocalFromTZI(&i)
}

// TODO(rsc): Implement.
func loadLocation(name string) (*Location, error) {
	return nil, errors.New("unknown time zone " + name)
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/hybridgroup/gobot"
	"github.com/hybridgroup/gobot/platforms/i2c"
	"github.com/hybridgroup/gobot/platforms/intel-iot/edison"
)

func main() {
	gbot := gobot.NewGobot()

	board := edison.NewEdisonAdaptor("edison")
	screen := i2c.NewGroveLcdDriver(board, "screen")

	work := func() {
		screen.Write("hello")

		screen.SetRGB(255, 0, 0)

		gobot.After(5*time.Second, func() {
			screen.Clear()
			screen.Home()
			screen.SetRGB(0, 255, 0)
			screen.Write("goodbye")
		})

		screen.Home()
		<-time.After(1 * time.Second)
		screen.SetRGB(0, 0, 255)
	}

	robot := gobot.NewRobot("screenBot",
		[]gobot.Connection{board},
		[]gobot.Device{screen},
		work,
	)

	gbot.AddRobot(robot)

	gbot.Start()
}
<commit_msg>edison_grove_lcd: add multi-line example<commit_after>package main

import (
	"time"

	"github.com/hybridgroup/gobot"
	"github.com/hybridgroup/gobot/platforms/i2c"
	"github.com/hybridgroup/gobot/platforms/intel-iot/edison"
)

func main() {
	gbot := gobot.NewGobot()

	board := edison.NewEdisonAdaptor("edison")
	screen := i2c.NewGroveLcdDriver(board, "screen")

	work := func() {
		screen.Write("hello")

		screen.SetRGB(255, 0, 0)

		gobot.After(5*time.Second, func() {
			screen.Clear()
			screen.Home()
			screen.SetRGB(0, 255, 0)
			screen.Write("goodbye\nhave a nice day")
		})

		screen.Home()
		<-time.After(1 * time.Second)
		screen.SetRGB(0, 0, 255)
	}

	robot := gobot.NewRobot("screenBot",
		[]gobot.Connection{board},
		[]gobot.Device{screen},
		work,
	)

	gbot.AddRobot(robot)

	gbot.Start()
}
<|endoftext|>


<commit_before>package ccv3_test

import (
	"fmt"
	"net/http"
	"strings"

	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	. "code.cloudfoundry.org/cli/api/cloudcontroller/ccv3"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv3/internal"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/ghttp"
)

var _ = Describe("Info", func() {
	var (
		client          *Client
		rootRespondWith http.HandlerFunc
		v3RespondWith   http.HandlerFunc

		apis       Info
		resources  ResourceLinks
		warnings   Warnings
		executeErr error
	)

	JustBeforeEach(func() {
		client, _ = NewTestClient()

		server.AppendHandlers(
			CombineHandlers(
				VerifyRequest(http.MethodGet, "/"),
				rootRespondWith,
			),
			CombineHandlers(
				VerifyRequest(http.MethodGet, "/v3"),
				v3RespondWith,
			))

		apis, resources, warnings, executeErr = client.GetInfo()
	})

	Describe("when all requests are successful", func() {
		BeforeEach(func() {
			rootResponse := strings.Replace(`{
				"links": {
					"self": {
						"href": "SERVER_URL"
					},
					"cloud_controller_v2": {
						"href": "SERVER_URL/v2",
						"meta": {
							"version": "2.64.0"
						}
					},
					"cloud_controller_v3": {
						"href": "SERVER_URL/v3",
						"meta": {
							"version": "3.0.0-alpha.5"
						}
					},
					"network_policy_v1": {
						"href": "SERVER_URL/networking/v1/external"
					},
					"uaa": {
						"href": "https://uaa.bosh-lite.com"
					},
					"logging": {
						"href": "wss://doppler.bosh-lite.com:443"
					},
					"app_ssh": {
						"href": "ssh.bosh-lite.com:2222",
						"meta": {
							"host_key_fingerprint": "some-fingerprint",
							"oath_client": "some-client"
						}
					}
				}
			}`, "SERVER_URL", server.URL(), -1)

			rootRespondWith = RespondWith(
				http.StatusOK,
				rootResponse,
				http.Header{"X-Cf-Warnings": {"warning 1"}})

			v3Response := strings.Replace(`{
				"links": {
					"self": {
						"href": "SERVER_URL/v3"
					},
					"apps": {
						"href": "SERVER_URL/v3/apps"
					},
					"tasks": {
						"href": "SERVER_URL/v3/tasks"
					}
				}
			}`, "SERVER_URL", server.URL(), -1)

			v3RespondWith = RespondWith(
				http.StatusOK,
				v3Response,
				http.Header{"X-Cf-Warnings": {"warning 2"}})
		})

		It("returns the CC Information", func() {
			Expect(executeErr).NotTo(HaveOccurred())
			Expect(apis.UAA()).To(Equal("https://uaa.bosh-lite.com"))
			Expect(apis.Logging()).To(Equal("wss://doppler.bosh-lite.com:443"))
			Expect(apis.NetworkPolicyV1()).To(Equal(fmt.Sprintf("%s/networking/v1/external", server.URL())))
			Expect(apis.AppSSHHostKeyFingerprint()).To(Equal("some-fingerprint"))
			Expect(apis.AppSSHEndpoint()).To(Equal("ssh.bosh-lite.com:2222"))
			Expect(apis.OAuthClient()).To(Equal("some-client"))
		})

		It("returns back the resource links", func() {
			Expect(executeErr).NotTo(HaveOccurred())
			Expect(resources[internal.AppsResource].HREF).To(Equal(server.URL() + "/v3/apps"))
			Expect(resources[internal.TasksResource].HREF).To(Equal(server.URL() + "/v3/tasks"))
		})

		It("returns all warnings", func() {
			Expect(executeErr).NotTo(HaveOccurred())
			Expect(warnings).To(ConsistOf("warning 1", "warning 2"))
		})
	})

	When("the cloud controller encounters an error", func() {
		When("the root response is invalid", func() {
			BeforeEach(func() {
				rootRespondWith = RespondWith(
					http.StatusNotFound,
					`i am google, bow down`,
					http.Header{"X-Cf-Warnings": {"warning 2"}},
				)
			})

			It("returns an APINotFoundError and no warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.APINotFoundError{URL: server.URL()}))
				Expect(warnings).To(BeNil())
			})
		})

		When("the error occurs making a request to '/'", func() {
			BeforeEach(func() {
				rootRespondWith = RespondWith(
					http.StatusNotFound,
					`{"errors": [{}]}`,
					http.Header{"X-Cf-Warnings": {"this is a warning"}})
			})

			It("returns the same error and all warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.ResourceNotFoundError{}))
				Expect(warnings).To(ConsistOf("this is a warning"))
			})
		})

		When("the error occurs making a request to '/v3'", func() {
			BeforeEach(func() {
				rootResponse := fmt.Sprintf(`{
					"links": {
						"self": {
							"href": "%s"
						},
						"cloud_controller_v2": {
							"href": "%s/v2",
							"meta": {
								"version": "2.64.0"
							}
						},
						"cloud_controller_v3": {
							"href": "%s/v3",
							"meta": {
								"version": "3.0.0-alpha.5"
							}
						},
						"uaa": {
							"href": "https://uaa.bosh-lite.com"
						},
						"logging": {
							"href": "wss://doppler.bosh-lite.com:443"
						}
					}
				}
				`, server.URL(), server.URL(), server.URL())

				rootRespondWith = RespondWith(
					http.StatusOK,
					rootResponse,
					http.Header{"X-Cf-Warnings": {"warning 1"}})
				v3RespondWith = RespondWith(
					http.StatusNotFound,
					`{"errors": [{
							"code": 10010,
							"title": "CF-ResourceNotFound",
							"detail": "Not found, lol"
						}]
					}`,
					http.Header{"X-Cf-Warnings": {"this is a warning"}})
			})

			It("returns the same error and all warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.ResourceNotFoundError{Message: "Not found, lol"}))
				Expect(warnings).To(ConsistOf("warning 1", "this is a warning"))
			})
		})
	})
})
<commit_msg>Always initialize widely-scoped variable in info test<commit_after>package ccv3_test

import (
	"fmt"
	"net/http"
	"strings"

	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	. "code.cloudfoundry.org/cli/api/cloudcontroller/ccv3"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv3/internal"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/ghttp"
)

var _ = Describe("Info", func() {
	var (
		client          *Client
		rootRespondWith http.HandlerFunc
		v3RespondWith   http.HandlerFunc

		apis       Info
		resources  ResourceLinks
		warnings   Warnings
		executeErr error
	)

	BeforeEach(func() {
		rootRespondWith = nil
		v3RespondWith = nil
	})

	JustBeforeEach(func() {
		client, _ = NewTestClient()

		server.AppendHandlers(
			CombineHandlers(
				VerifyRequest(http.MethodGet, "/"),
				rootRespondWith,
			),
			CombineHandlers(
				VerifyRequest(http.MethodGet, "/v3"),
				v3RespondWith,
			))

		apis, resources, warnings, executeErr = client.GetInfo()
	})

	Describe("when all requests are successful", func() {
		BeforeEach(func() {
			rootResponse := strings.Replace(`{
				"links": {
					"self": {
						"href": "SERVER_URL"
					},
					"cloud_controller_v2": {
						"href": "SERVER_URL/v2",
						"meta": {
							"version": "2.64.0"
						}
					},
					"cloud_controller_v3": {
						"href": "SERVER_URL/v3",
						"meta": {
							"version": "3.0.0-alpha.5"
						}
					},
					"network_policy_v1": {
						"href": "SERVER_URL/networking/v1/external"
					},
					"uaa": {
						"href": "https://uaa.bosh-lite.com"
					},
					"logging": {
						"href": "wss://doppler.bosh-lite.com:443"
					},
					"app_ssh": {
						"href": "ssh.bosh-lite.com:2222",
						"meta": {
							"host_key_fingerprint": "some-fingerprint",
							"oath_client": "some-client"
						}
					}
				}
			}`, "SERVER_URL", server.URL(), -1)

			rootRespondWith = RespondWith(
				http.StatusOK,
				rootResponse,
				http.Header{"X-Cf-Warnings": {"warning 1"}})

			v3Response := strings.Replace(`{
				"links": {
					"self": {
						"href": "SERVER_URL/v3"
					},
					"apps": {
						"href": "SERVER_URL/v3/apps"
					},
					"tasks": {
						"href": "SERVER_URL/v3/tasks"
					}
				}
			}`, "SERVER_URL", server.URL(), -1)

			v3RespondWith = RespondWith(
				http.StatusOK,
				v3Response,
				http.Header{"X-Cf-Warnings": {"warning 2"}})
		})

		It("returns the CC Information", func() {
			Expect(executeErr).NotTo(HaveOccurred())
			Expect(apis.UAA()).To(Equal("https://uaa.bosh-lite.com"))
			Expect(apis.Logging()).To(Equal("wss://doppler.bosh-lite.com:443"))
			Expect(apis.NetworkPolicyV1()).To(Equal(fmt.Sprintf("%s/networking/v1/external", server.URL())))
			Expect(apis.AppSSHHostKeyFingerprint()).To(Equal("some-fingerprint"))
			Expect(apis.AppSSHEndpoint()).To(Equal("ssh.bosh-lite.com:2222"))
			Expect(apis.OAuthClient()).To(Equal("some-client"))
		})

		It("returns back the resource links", func() {
			Expect(executeErr).NotTo(HaveOccurred())
			Expect(resources[internal.AppsResource].HREF).To(Equal(server.URL() + "/v3/apps"))
			Expect(resources[internal.TasksResource].HREF).To(Equal(server.URL() + "/v3/tasks"))
		})

		It("returns all warnings", func() {
			Expect(executeErr).NotTo(HaveOccurred())
			Expect(warnings).To(ConsistOf("warning 1", "warning 2"))
		})
	})

	When("the cloud controller encounters an error", func() {
		When("the root response is invalid", func() {
			BeforeEach(func() {
				rootRespondWith = RespondWith(
					http.StatusNotFound,
					`i am google, bow down`,
					http.Header{"X-Cf-Warnings": {"warning 2"}},
				)
			})

			It("returns an APINotFoundError and no warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.APINotFoundError{URL: server.URL()}))
				Expect(warnings).To(BeNil())
			})
		})

		When("the error occurs making a request to '/'", func() {
			BeforeEach(func() {
				rootRespondWith = RespondWith(
					http.StatusNotFound,
					`{"errors": [{}]}`,
					http.Header{"X-Cf-Warnings": {"this is a warning"}})
			})

			It("returns the same error and all warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.ResourceNotFoundError{}))
				Expect(warnings).To(ConsistOf("this is a warning"))
			})
		})

		When("the error occurs making a request to '/v3'", func() {
			BeforeEach(func() {
				rootResponse := fmt.Sprintf(`{
					"links": {
						"self": {
							"href": "%s"
						},
						"cloud_controller_v2": {
							"href": "%s/v2",
							"meta": {
								"version": "2.64.0"
							}
						},
						"cloud_controller_v3": {
							"href": "%s/v3",
							"meta": {
								"version": "3.0.0-alpha.5"
							}
						},
						"uaa": {
							"href": "https://uaa.bosh-lite.com"
						},
						"logging": {
							"href": "wss://doppler.bosh-lite.com:443"
						}
					}
				}
				`, server.URL(), server.URL(), server.URL())

				rootRespondWith = RespondWith(
					http.StatusOK,
					rootResponse,
					http.Header{"X-Cf-Warnings": {"warning 1"}})
				v3RespondWith = RespondWith(
					http.StatusNotFound,
					`{"errors": [{
							"code": 10010,
							"title": "CF-ResourceNotFound",
							"detail": "Not found, lol"
						}]
					}`,
					http.Header{"X-Cf-Warnings": {"this is a warning"}})
			})

			It("returns the same error and all warnings", func() {
				Expect(executeErr).To(MatchError(ccerror.ResourceNotFoundError{Message: "Not found, lol"}))
				Expect(warnings).To(ConsistOf("warning 1", "this is a warning"))
			})
		})
	})
})
<|endoftext|>


<commit_before>/**
 * @file status.go
 * @author Mikhail Klementyev jollheef<AT>riseup.net
 * @license GNU GPLv3
 * @date September, 2015
 * @brief queries for status table
 */

package steward

import "database/sql"

type ServiceState int

const (
	// Service is online, serves the requests, stores and
	// returns flags and behaves as expected
	STATUS_OK ServiceState = iota
	// Service is online, but behaves not as expected, e.g. if HTTP server
	// listens the port, but doesn't respond on request
	STATUS_MUMBLE
	// Service is online, but past flags cannot be retrieved
	STATUS_CORRUPT
	// Service is offline
	STATUS_DOWN
	// Checker error
	STATUS_ERROR
	// Unknown
	STATUS_UNKNOWN
)

type Status struct {
	Round     int
	TeamId    int
	ServiceId int
	State     ServiceState
}

func createStatusTable(db *sql.DB) (err error) {

	_, err = db.Exec(`
	CREATE TABLE IF NOT EXISTS "status" (
		id	SERIAL PRIMARY KEY,
		round	INTEGER NOT NULL,
		team_id	INTEGER NOT NULL,
		service_id	INTEGER NOT NULL,
		state	INTEGER NOT NULL,
		timestamp	TIMESTAMP with time zone DEFAULT now()
	)`)

	return
}

func PutStatus(db *sql.DB, status Status) (err error) {

	stmt, err := db.Prepare("INSERT INTO status (round, team_id, " +
		"service_id, state) VALUES ($1, $2, $3, $4)")
	if err != nil {
		return
	}

	defer stmt.Close()

	_, err = stmt.Exec(status.Round, status.TeamId, status.ServiceId,
		status.State)
	if err != nil {
		return
	}

	return
}

func GetStates(db *sql.DB, halfStatus Status) (states []ServiceState,
	err error) {

	stmt, err := db.Prepare(
		"SELECT state FROM status WHERE round=$1 AND team_id=$2 " +
			"AND service_id=$3")
	if err != nil {
		return
	}

	defer stmt.Close()

	rows, err := stmt.Query(halfStatus.Round, halfStatus.TeamId,
		halfStatus.ServiceId)
	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var state int

		err = rows.Scan(&state)
		if err != nil {
			return
		}

		states = append(states, ServiceState(state))
	}

	return
}

func GetState(db *sql.DB, halfStatus Status) (state ServiceState, err error) {

	stmt, err := db.Prepare(
		"SELECT state FROM status WHERE round=$1 AND team_id=$2 " +
			"AND service_id=$3 " +
			"AND ID = (SELECT MAX(ID) FROM status " +
			"WHERE round=$1 AND team_id=$2 AND service_id=$3)")
	if err != nil {
		return
	}

	defer stmt.Close()

	err = stmt.QueryRow(halfStatus.Round, halfStatus.TeamId,
		halfStatus.ServiceId).Scan(&state)
	if err != nil {
		return
	}

	return
}
<commit_msg>Add string representation for service state<commit_after>/**
 * @file status.go
 * @author Mikhail Klementyev jollheef<AT>riseup.net
 * @license GNU GPLv3
 * @date September, 2015
 * @brief queries for status table
 */

package steward

import "database/sql"

type ServiceState int

const (
	// Service is online, serves the requests, stores and
	// returns flags and behaves as expected
	STATUS_OK ServiceState = iota
	// Service is online, but behaves not as expected, e.g. if HTTP server
	// listens the port, but doesn't respond on request
	STATUS_MUMBLE
	// Service is online, but past flags cannot be retrieved
	STATUS_CORRUPT
	// Service is offline
	STATUS_DOWN
	// Checker error
	STATUS_ERROR
	// Unknown
	STATUS_UNKNOWN
)

func (state ServiceState) String() string {
	switch state {
	case STATUS_OK:
		return "ok"
	case STATUS_MUMBLE:
		return "mumble"
	case STATUS_CORRUPT:
		return "corrupt"
	case STATUS_DOWN:
		return "down"
	case STATUS_ERROR:
		return "error"
	case STATUS_UNKNOWN:
		return "unknown"
	}

	return "undefined"
}

type Status struct {
	Round     int
	TeamId    int
	ServiceId int
	State     ServiceState
}

func createStatusTable(db *sql.DB) (err error) {

	_, err = db.Exec(`
	CREATE TABLE IF NOT EXISTS "status" (
		id	SERIAL PRIMARY KEY,
		round	INTEGER NOT NULL,
		team_id	INTEGER NOT NULL,
		service_id	INTEGER NOT NULL,
		state	INTEGER NOT NULL,
		timestamp	TIMESTAMP with time zone DEFAULT now()
	)`)

	return
}

func PutStatus(db *sql.DB, status Status) (err error) {

	stmt, err := db.Prepare("INSERT INTO status (round, team_id, " +
		"service_id, state) VALUES ($1, $2, $3, $4)")
	if err != nil {
		return
	}

	defer stmt.Close()

	_, err = stmt.Exec(status.Round, status.TeamId, status.ServiceId,
		status.State)
	if err != nil {
		return
	}

	return
}

func GetStates(db *sql.DB, halfStatus Status) (states []ServiceState,
	err error) {

	stmt, err := db.Prepare(
		"SELECT state FROM status WHERE round=$1 AND team_id=$2 " +
			"AND service_id=$3")
	if err != nil {
		return
	}

	defer stmt.Close()

	rows, err := stmt.Query(halfStatus.Round, halfStatus.TeamId,
		halfStatus.ServiceId)
	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var state int

		err = rows.Scan(&state)
		if err != nil {
			return
		}

		states = append(states, ServiceState(state))
	}

	return
}

func GetState(db *sql.DB, halfStatus Status) (state ServiceState, err error) {

	stmt, err := db.Prepare(
		"SELECT state FROM status WHERE round=$1 AND team_id=$2 " +
			"AND service_id=$3 " +
			"AND ID = (SELECT MAX(ID) FROM status " +
			"WHERE round=$1 AND team_id=$2 AND service_id=$3)")
	if err != nil {
		return
	}

	defer stmt.Close()

	err = stmt.QueryRow(halfStatus.Round, halfStatus.TeamId,
		halfStatus.ServiceId).Scan(&state)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "github.com/codegangsta/cli"
  "./task"
)

const (
  APP_VER  = "0.1.0"
  APP_NAME = "qs"
)

func main() {

  var app      = cli.NewApp()
  app.Name     = APP_NAME
  app.Version  = APP_VER
  app.Usage    = "email for mail sending, or web or none for run web interface!"
  app.Commands = []cli.Command{
    task.CmdWeb,
    task.CmdEmail,
  }
	app.Flags = append(app.Flags, []cli.Flag{}...)

  app.Run(os.Args)
}
<commit_msg>Move code into GOPATH/src. Follow import convention other than using local package<commit_after>package main

import (
  "os"
  "github.com/codegangsta/cli"
  "github.com/qSlide/qslide/task"
  //"./task"
)

const (
  APP_VER  = "0.1.0"
  APP_NAME = "qs"
)

func main() {

  var app      = cli.NewApp()
  app.Name     = APP_NAME
  app.Version  = APP_VER
  app.Usage    = "email for mail sending, or web or none for run web interface!"
  app.Commands = []cli.Command{
    task.CmdWeb,
    task.CmdEmail,
  }
	app.Flags = append(app.Flags, []cli.Flag{}...)

  app.Run(os.Args)
}
<|endoftext|>


<commit_before>package s3

// https://github.com/aws/aws-sdk-go
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3.html

// https://github.com/goamz/goamz/blob/master/aws/aws.go
// https://github.com/goamz/goamz/blob/master/s3/s3.go

import (
	"github.com/goamz/goamz/aws"
	aws_s3 "github.com/goamz/goamz/s3"
	"github.com/jeffail/tunny"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	log "github.com/whosonfirst/go-whosonfirst-log"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"
	"strings"
	"time"
)

type Sync struct {
	ACL    aws_s3.ACL
	Bucket aws_s3.Bucket
	Prefix string
	Pool   tunny.WorkPool
	Logger *log.WOFLogger
}

func NewSync(auth aws.Auth, region aws.Region, acl aws_s3.ACL, bucket string, prefix string, procs int, logger *log.WOFLogger) *Sync {

	runtime.GOMAXPROCS(procs)

	pool, _ := tunny.CreatePoolGeneric(procs).Open()

	s := aws_s3.New(auth, region)
	b := s.Bucket(bucket)

	return &Sync{
		ACL:    acl,
		Bucket: *b,
		Prefix: prefix,
		Pool:   *pool,
		Logger: logger,
	}
}

func WOFSync(auth aws.Auth, bucket string, prefix string, procs int, logger *log.WOFLogger) *Sync {

	return NewSync(auth, aws.USEast, aws_s3.PublicRead, bucket, prefix, procs, logger)
}

func (sink Sync) SyncDirectory(root string, debug bool) error {

	defer sink.Pool.Close()

	var files int64
	var failed int64

	t0 := time.Now()

	callback := func(src string, info os.FileInfo) error {

		sink.Logger.Debug("crawling %s", src)

		if info.IsDir() {
			return nil
		}

		files++

		// sudo put all of this in to a function so it can
		// be called from something other than SyncDirectory

		source := src
		dest := source

		dest = strings.Replace(dest, root, "", -1)

		if sink.Prefix != "" {
			dest = path.Join(sink.Prefix, dest)
		}

		// Note: both HasChanged and SyncFile will ioutil.ReadFile(source)
		// which is a potential waste of time and resource. Or maybe we just
		// don't care? (20150930/thisisaaronland)

		sink.Logger.Debug("Looking for changes %s (%s)", dest, sink.Prefix)

		change, ch_err := sink.HasChanged(source, dest)

		if ch_err != nil {
			sink.Logger.Warning("failed to determine whether %s had changed, because '%s'", source, ch_err)
			change = true
		}

		if debug == true {
			sink.Logger.Debug("has %s changed? the answer is %v but does it really matter since debugging is enabled?", source, change)
			return nil
		}

		if !change {
			sink.Logger.Debug("%s has not changed, skipping", source)
			return nil
		}

		s_err := sink.SyncFile(source, dest)

		if s_err != nil {
			sink.Logger.Error("failed to PUT %s, because '%s'", dest, s_err)
			failed++
		}

		return nil
	}

	c := crawl.NewCrawler(root)
	_ = c.Crawl(callback)

	t1 := float64(time.Since(t0)) / 1e9

	sink.Logger.Info("processed %d files (error: %d) in %.3f seconds\n", files, failed, t1)

	return nil
}

func (sink Sync) SyncFile(source string, dest string) error {

	sink.Logger.Debug("prepare %s for syncing", source)

	body, err := ioutil.ReadFile(source)

	if err != nil {
		sink.Logger.Error("Failed to read %s, because %v", source, err)
		return err
	}

	_, err = sink.Pool.SendWork(func() {

		sink.Logger.Debug("PUT %s as %s", dest, sink.ACL)

		o := aws_s3.Options{}

		err := sink.Bucket.Put(dest, body, "text/plain", sink.ACL, o)

		if err != nil {
			sink.Logger.Error("failed to PUT %s, because '%s'", dest, err)
		}

	})

	if err != nil {
		sink.Logger.Error("failed to schedule %s for processing, because '%s'", source, err)
		return err
	}

	sink.Logger.Debug("scheduled %s for processing", source)
	return nil
}

// the following appears to trigger a freak-out-and-die condition... sometimes
// I have no idea why... test under go 1.2.1, 1.4.3 and 1.5.1 / see also:
// https://github.com/whosonfirst/go-mapzen-whosonfirst-s3/issues/2
// (2015/thisisaaronland)

func (sink Sync) HasChanged(source string, dest string) (ch bool, err error) {

	headers := make(http.Header)
	rsp, err := sink.Bucket.Head(dest, headers)

	if err != nil {
		sink.Logger.Error("failed to HEAD %s because %s", dest, err)
		return false, err
	}

	info, err := os.Stat(source)

	if err != nil {
		sink.Logger.Error("failed to stat %s because %s", source, err)
		return false, err
	}

	mtime_local := info.ModTime()

	last_mod := rsp.Header.Get("Last-Modified")
	mtime_remote, err := time.Parse(time.RFC1123, last_mod)

	if err != nil {
		sink.Logger.Error("failed to parse timestamp %s because %s", last_mod, err)
		return false, err
	}

	// Because who remembers this stuff anyway...
	// func (t Time) Before(u Time) bool
	// Before reports whether the time instant t is before u.

	if mtime_local.Before(mtime_remote) {
		sink.Logger.Warning("remote copy of %s has a more recent modification date (local: %s remote: %s)", source, mtime_local, mtime_remote)
		return false, nil
	}

	local_hash, err := utils.HashFile(source)

	if err != nil {
		sink.Logger.Warning("failed to hash %s, because %v", source, err)
		return false, err
	}

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		return false, nil
	}

	return true, nil
}
<commit_msg>perform mtime check (issue #7) last<commit_after>package s3

// https://github.com/aws/aws-sdk-go
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3.html

// https://github.com/goamz/goamz/blob/master/aws/aws.go
// https://github.com/goamz/goamz/blob/master/s3/s3.go

import (
	"github.com/goamz/goamz/aws"
	aws_s3 "github.com/goamz/goamz/s3"
	"github.com/jeffail/tunny"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	log "github.com/whosonfirst/go-whosonfirst-log"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"
	"strings"
	"time"
)

type Sync struct {
	ACL    aws_s3.ACL
	Bucket aws_s3.Bucket
	Prefix string
	Pool   tunny.WorkPool
	Logger *log.WOFLogger
}

func NewSync(auth aws.Auth, region aws.Region, acl aws_s3.ACL, bucket string, prefix string, procs int, logger *log.WOFLogger) *Sync {

	runtime.GOMAXPROCS(procs)

	pool, _ := tunny.CreatePoolGeneric(procs).Open()

	s := aws_s3.New(auth, region)
	b := s.Bucket(bucket)

	return &Sync{
		ACL:    acl,
		Bucket: *b,
		Prefix: prefix,
		Pool:   *pool,
		Logger: logger,
	}
}

func WOFSync(auth aws.Auth, bucket string, prefix string, procs int, logger *log.WOFLogger) *Sync {

	return NewSync(auth, aws.USEast, aws_s3.PublicRead, bucket, prefix, procs, logger)
}

func (sink Sync) SyncDirectory(root string, debug bool) error {

	defer sink.Pool.Close()

	var files int64
	var failed int64

	t0 := time.Now()

	callback := func(src string, info os.FileInfo) error {

		sink.Logger.Debug("crawling %s", src)

		if info.IsDir() {
			return nil
		}

		files++

		// sudo put all of this in to a function so it can
		// be called from something other than SyncDirectory

		source := src
		dest := source

		dest = strings.Replace(dest, root, "", -1)

		if sink.Prefix != "" {
			dest = path.Join(sink.Prefix, dest)
		}

		// Note: both HasChanged and SyncFile will ioutil.ReadFile(source)
		// which is a potential waste of time and resource. Or maybe we just
		// don't care? (20150930/thisisaaronland)

		sink.Logger.Debug("Looking for changes %s (%s)", dest, sink.Prefix)

		change, ch_err := sink.HasChanged(source, dest)

		if ch_err != nil {
			sink.Logger.Warning("failed to determine whether %s had changed, because '%s'", source, ch_err)
			change = true
		}

		if debug == true {
			sink.Logger.Debug("has %s changed? the answer is %v but does it really matter since debugging is enabled?", source, change)
			return nil
		}

		if !change {
			sink.Logger.Debug("%s has not changed, skipping", source)
			return nil
		}

		s_err := sink.SyncFile(source, dest)

		if s_err != nil {
			sink.Logger.Error("failed to PUT %s, because '%s'", dest, s_err)
			failed++
		}

		return nil
	}

	c := crawl.NewCrawler(root)
	_ = c.Crawl(callback)

	t1 := float64(time.Since(t0)) / 1e9

	sink.Logger.Info("processed %d files (error: %d) in %.3f seconds\n", files, failed, t1)

	return nil
}

func (sink Sync) SyncFile(source string, dest string) error {

	sink.Logger.Debug("prepare %s for syncing", source)

	body, err := ioutil.ReadFile(source)

	if err != nil {
		sink.Logger.Error("Failed to read %s, because %v", source, err)
		return err
	}

	_, err = sink.Pool.SendWork(func() {

		sink.Logger.Debug("PUT %s as %s", dest, sink.ACL)

		o := aws_s3.Options{}

		err := sink.Bucket.Put(dest, body, "text/plain", sink.ACL, o)

		if err != nil {
			sink.Logger.Error("failed to PUT %s, because '%s'", dest, err)
		}

	})

	if err != nil {
		sink.Logger.Error("failed to schedule %s for processing, because '%s'", source, err)
		return err
	}

	sink.Logger.Debug("scheduled %s for processing", source)
	return nil
}

// the following appears to trigger a freak-out-and-die condition... sometimes
// I have no idea why... test under go 1.2.1, 1.4.3 and 1.5.1 / see also:
// https://github.com/whosonfirst/go-mapzen-whosonfirst-s3/issues/2
// (2015/thisisaaronland)

func (sink Sync) HasChanged(source string, dest string) (ch bool, err error) {

	headers := make(http.Header)
	rsp, err := sink.Bucket.Head(dest, headers)

	if err != nil {
		sink.Logger.Error("failed to HEAD %s because %s", dest, err)
		return false, err
	}

	local_hash, err := utils.HashFile(source)

	if err != nil {
		sink.Logger.Warning("failed to hash %s, because %v", source, err)
		return false, err
	}

	etag := rsp.Header.Get("Etag")
	remote_hash := strings.Replace(etag, "\"", "", -1)

	if local_hash == remote_hash {
		return false, nil
	}

	// Okay so we think that things have changed but let's just check
	// modification times to be extra sure (20151112/thisisaaronland)

	info, err := os.Stat(source)

	if err != nil {
		sink.Logger.Error("failed to stat %s because %s", source, err)
		return false, err
	}

	mtime_local := info.ModTime()

	last_mod := rsp.Header.Get("Last-Modified")
	mtime_remote, err := time.Parse(time.RFC1123, last_mod)

	if err != nil {
		sink.Logger.Error("failed to parse timestamp %s because %s", last_mod, err)
		return false, err
	}

	// Because who remembers this stuff anyway...
	// func (t Time) Before(u Time) bool
	// Before reports whether the time instant t is before u.

	sink.Logger.Debug("local %s %s", mtime_local, source)
	sink.Logger.Debug("remote %s %s", mtime_remote, dest)

	if mtime_local.Before(mtime_remote) {
		sink.Logger.Warning("remote copy of %s has a more recent modification date (local: %s remote: %s)", source, mtime_local, mtime_remote)
		return false, nil
	}

	return true, nil
}
<|endoftext|>