<commit_before>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

type handle func(http.ResponseWriter, *http.Request)

func main() {
	hostname := flag.String("hostname", ":8080", "Set the hostname used by ys")
	pwd := flag.String("pwd", "", "The password used to protect the content servered by ys")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Println("Please provide file or folder to serve:")
		fmt.Println("ys <dir/file>")
		return
	}

	filename := flag.Arg(0)

	http.HandleFunc("/", rootOnly(logging(auth(*pwd, serve(filename)))))

	log.Println("Starting ys on", *hostname)
	err := http.ListenAndServe(*hostname, nil)
	if err != nil {
		log.Println("Listening error:", err.Error())
	}
}

func rootOnly(h handle) handle {
	return func(res http.ResponseWriter, req *http.Request) {
		if req.RequestURI == "/" {
			h(res, req)
		}
	}
}

func logging(h handle) handle {
	return func(res http.ResponseWriter, req *http.Request) {
		log.Printf("[%s] %s %s\n", req.Method, req.RemoteAddr, req.RequestURI)
		h(res, req)
	}
}

func auth(pwd string, h handle) handle {
	if pwd == "" {
		return h
	} else {
		return func(res http.ResponseWriter, req *http.Request) {
			_, p, _ := req.BasicAuth()
			if pwd == p {
				h(res, req)
				return
			} else {
				res.WriteHeader(http.StatusUnauthorized)
				log.Println("Denied request from %s", req.RemoteAddr)
			}
		}
	}
}

func serve(f string) handle {
	return func(res http.ResponseWriter, req *http.Request) {
		file, err := os.Open(f)
		errPanic(err)

		stat, err := file.Stat()
		errPanic(err)
		if stat.IsDir() {
			res.Header().Add("Content-Disposition",
				fmt.Sprintf("inline; filename=\"%s.zip\"", f))
			zipDir(f, res)
		} else {
			b, err := ioutil.ReadAll(file)
			errPanic(err)
			res.Header().Add("Content-Disposition",
				fmt.Sprintf("inline; filename=\"%s\"", f))
			res.Write(b)
		}
	}
}

func zipDir(file string, w io.Writer) {
	zw := zip.NewWriter(w)
	filepath.Walk(file, func(path string, f os.FileInfo, err error) error {
		if !f.IsDir() {
			errPanic(err)
			w, err := zw.Create(path)
			errPanic(err)
			file, err := os.Open(path)
			errPanic(err)
			_, err = io.Copy(w, file)
			errPanic(err)
		}
		return nil
	})
	err := zw.Close()
	errPanic(err)
}

func errPanic(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>basic auth<commit_after>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

type handle func(http.ResponseWriter, *http.Request)

func main() {
	hostname := flag.String("hostname", ":8080", "Set the hostname used by ys")
	pwd := flag.String("pwd", "", "The password used to protect the content servered by ys")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Println("Please provide file or folder to serve:")
		fmt.Println("ys <dir/file>")
		return
	}

	filename := flag.Arg(0)

	http.HandleFunc("/", rootOnly(logging(auth(*pwd, serve(filename)))))

	log.Println("Starting ys on", *hostname)
	err := http.ListenAndServe(*hostname, nil)
	if err != nil {
		log.Println("Listening error:", err.Error())
	}
}

func rootOnly(h handle) handle {
	return func(res http.ResponseWriter, req *http.Request) {
		if req.RequestURI == "/" {
			h(res, req)
		}
	}
}

func logging(h handle) handle {
	return func(res http.ResponseWriter, req *http.Request) {
		log.Printf("[%s] %s %s\n", req.Method, req.RemoteAddr, req.RequestURI)
		h(res, req)
	}
}

func auth(pwd string, h handle) handle {
	if pwd == "" {
		return h
	} else {
		return func(res http.ResponseWriter, req *http.Request) {
			res.Header().Add("WWW-Authenticate", "Basic realm=\"ys:\"")
			_, p, _ := req.BasicAuth()
			if pwd == p {
				h(res, req)
				return
			} else {
				res.WriteHeader(http.StatusUnauthorized)
				res.Write([]byte("Unauthorized"))
				log.Println("Denied request from %s", req.RemoteAddr)
			}
		}
	}
}

func serve(f string) handle {
	return func(res http.ResponseWriter, req *http.Request) {
		file, err := os.Open(f)
		errPanic(err)

		stat, err := file.Stat()
		errPanic(err)
		if stat.IsDir() {
			res.Header().Add("Content-Disposition",
				fmt.Sprintf("inline; filename=\"%s.zip\"", f))
			zipDir(f, res)
		} else {
			b, err := ioutil.ReadAll(file)
			errPanic(err)
			res.Header().Add("Content-Disposition",
				fmt.Sprintf("inline; filename=\"%s\"", f))
			res.Write(b)
		}
	}
}

func zipDir(file string, w io.Writer) {
	zw := zip.NewWriter(w)
	filepath.Walk(file, func(path string, f os.FileInfo, err error) error {
		if !f.IsDir() {
			errPanic(err)
			w, err := zw.Create(path)
			errPanic(err)
			file, err := os.Open(path)
			errPanic(err)
			_, err = io.Copy(w, file)
			errPanic(err)
		}
		return nil
	})
	err := zw.Close()
	errPanic(err)
}

func errPanic(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/mendersoftware/log"
)

type logOptionsType struct {
	debug      *bool
	info       *bool
	logLevel   *string
	logModules *string
	logFile    *string
	noSyslog   *bool
}

type runOptionsType struct {
	version   *bool
	config    *string
	dataStore *string
	imageFile *string
	commit    *bool
	daemon    *bool
	bootstrap *bool
	httpsClientConfig
}

var (
	errMsgNoArgumentsGiven = errors.New("Must give one of -rootfs, " +
		"-commit, -bootstrap or -daemon arguments")
	errMsgAmbiguousArgumentsGiven = errors.New("Ambiguous parameters given " +
		"- must give exactly one from: -rootfs, -commit, -bootstrap or -daemon")
	errMsgIncompatibleLogOptions = errors.New("One or more " +
		"incompatible log log options specified.")
)

var defaultConfFile string = "/etc/mender/mender.conf"

type Commander interface {
	Command(name string, arg ...string) *exec.Cmd
}

type StatCommander interface {
	Stat(string) (os.FileInfo, error)
	Commander
}

// we need real OS implementation
type osCalls struct {
}

func (osCalls) Command(name string, arg ...string) *exec.Cmd {
	return exec.Command(name, arg...)
}

func (osCalls) Stat(name string) (os.FileInfo, error) {
	return os.Stat(name)
}

func argsParse(args []string) (runOptionsType, error) {
	parsing := flag.NewFlagSet("mender", flag.ContinueOnError)

	// FLAGS ---------------------------------------------------------------

	version := parsing.Bool("version", false, "Show mender agent version and exit.")

	config := parsing.String("config", defaultConfFile,
		"Configuration file location.")

	data := parsing.String("data", defaultDataStore,
		"Mender state data location.")

	commit := parsing.Bool("commit", false, "Commit current update.")

	imageFile := parsing.String("rootfs", "",
		"Root filesystem URI to use for update. Can be either a local "+
			"file or a URL.")

	daemon := parsing.Bool("daemon", false, "Run as a daemon.")

	// add bootstrap related command line options
	certFile := parsing.String("certificate", "", "Client certificate")
	certKey := parsing.String("cert-key", "", "Client certificate's private key")
	serverCert := parsing.String("trusted-certs", "", "Trusted server certificates")
	bootstrap := parsing.Bool("bootstrap", false, "Force bootstrap")

	// add log related command line options
	logFlags := addLogFlags(parsing)

	// PARSING -------------------------------------------------------------

	if err := parsing.Parse(args); err != nil {
		return runOptionsType{}, err
	}

	runOptions := runOptionsType{
		version,
		config,
		data,
		imageFile,
		commit,
		daemon,
		bootstrap,
		httpsClientConfig{
			*certFile,
			*certKey,
			*serverCert,
			false,
		},
	}

	//runOptions.bootstrap = httpsClientConfig{}

	// FLAG LOGIC ----------------------------------------------------------

	// we just want to see the version string, the rest does not
	// matter
	if *version == true {
		return runOptions, nil
	}

	if err := parseLogFlags(logFlags); err != nil {
		return runOptions, err
	}

	if moreThanOneRunOptionSelected(runOptions) {
		return runOptions, errMsgAmbiguousArgumentsGiven
	}

	return runOptions, nil
}

func moreThanOneRunOptionSelected(runOptions runOptionsType) bool {
	// check if more than one command line action is selected
	var runOptionsCount int

	if *runOptions.imageFile != "" {
		runOptionsCount++
	}
	if *runOptions.commit {
		runOptionsCount++
	}
	if *runOptions.daemon {
		runOptionsCount++
	}

	if runOptionsCount > 1 {
		return true
	}
	return false
}

func addLogFlags(f *flag.FlagSet) logOptionsType {

	var logOptions logOptionsType

	logOptions.debug = f.Bool("debug", false, "Debug log level. This is a "+
		"shorthand for '-l debug'.")

	logOptions.info = f.Bool("info", false, "Info log level. This is a "+
		"shorthand for '-l info'.")

	logOptions.logLevel = f.String("log-level", "", "Log level, which can be "+
		"'debug', 'info', 'warning', 'error', 'fatal' or 'panic'. "+
		"Earlier log levels will also log the subsequent levels (so "+
		"'debug' will log everything). The default log level is "+
		"'warning'.")

	logOptions.logModules = f.String("log-modules", "", "Filter logging by "+
		"module. This is a comma separated list of modules to log, "+
		"other modules will be omitted. To see which modules are "+
		"available, take a look at a non-filtered log and select "+
		"the modules appropriate for you.")

	logOptions.noSyslog = f.Bool("no-syslog", false, "Disable logging to "+
		"syslog. Note that debug message are never logged to syslog.")

	logOptions.logFile = f.String("log-file", "", "File to log to.")

	return logOptions

}

func parseLogFlags(args logOptionsType) error {
	var logOptCount int

	if *args.logLevel != "" {
		level, err := log.ParseLevel(*args.logLevel)
		if err != nil {
			return err
		}
		log.SetLevel(level)
		logOptCount++
	}

	if *args.info {
		log.SetLevel(log.InfoLevel)
		logOptCount++
	}

	if *args.debug {
		log.SetLevel(log.DebugLevel)
		logOptCount++
	}

	if logOptCount > 1 {
		return errMsgIncompatibleLogOptions
	} else if logOptCount == 0 {
		// Default log level.
		log.SetLevel(log.WarnLevel)
	}

	if *args.logFile != "" {
		fd, err := os.Create(*args.logFile)
		if err != nil {
			return err
		}
		log.SetOutput(fd)
	}

	if *args.logModules != "" {
		modules := strings.Split(*args.logModules, ",")
		log.SetModuleFilter(modules)
	}

	if !*args.noSyslog {
		if err := log.AddSyslogHook(); err != nil {
			log.Warnf("Could not connect to syslog daemon: %s. "+
				"(use -no-syslog to disable completely)",
				err.Error())
		}
	}

	return nil
}

func ShowVersion() {
	v := fmt.Sprintf("%s\n", VersionString())
	os.Stdout.Write([]byte(v))
}

func doMain(args []string) error {
	runOptions, err := argsParse(args)
	if err != nil {
		return err
	}

	if *runOptions.version {
		ShowVersion()
		return nil
	}

	config, err := LoadConfig(*runOptions.config)
	if err != nil {
		return err
	}

	env := NewEnvironment(new(osCalls))
	device := NewDevice(env, new(osCalls), config.GetDeviceConfig())
	store := NewDirStore(*runOptions.dataStore)

	switch {

	case *runOptions.imageFile != "":
		if err := doRootfs(device, runOptions); err != nil {
			return err
		}

	case *runOptions.commit:
		if err := device.CommitUpdate(); err != nil {
			return err
		}

	case *runOptions.daemon:
		updater, err := NewUpdateClient(config.GetUpdaterConfig())
		if err != nil {
			return errors.New("Cannot initialize daemon. Error instantiating updater. Exiting.")
		}

		controller := NewMender(*config, MenderPieces{
			updater,
			device,
			env,
			store,
		})
		if controller == nil {
			return errors.New("Cannot initialize mender controller")
		}

		if *runOptions.bootstrap {
			controller.ForceBootstrap()
		}

		daemon := NewDaemon(controller)
		return daemon.Run()

	case *runOptions.imageFile == "" && !*runOptions.commit &&
		!*runOptions.daemon:
		return errMsgNoArgumentsGiven
	}

	return nil
}

func main() {
	if err := doMain(os.Args[1:]); err != nil && err != flag.ErrHelp {
		log.Errorln(err.Error())
		os.Exit(1)
	}
}
<commit_msg>main: dir store needs init only in daemon mode<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/mendersoftware/log"
)

type logOptionsType struct {
	debug      *bool
	info       *bool
	logLevel   *string
	logModules *string
	logFile    *string
	noSyslog   *bool
}

type runOptionsType struct {
	version   *bool
	config    *string
	dataStore *string
	imageFile *string
	commit    *bool
	daemon    *bool
	bootstrap *bool
	httpsClientConfig
}

var (
	errMsgNoArgumentsGiven = errors.New("Must give one of -rootfs, " +
		"-commit, -bootstrap or -daemon arguments")
	errMsgAmbiguousArgumentsGiven = errors.New("Ambiguous parameters given " +
		"- must give exactly one from: -rootfs, -commit, -bootstrap or -daemon")
	errMsgIncompatibleLogOptions = errors.New("One or more " +
		"incompatible log log options specified.")
)

var defaultConfFile string = "/etc/mender/mender.conf"

type Commander interface {
	Command(name string, arg ...string) *exec.Cmd
}

type StatCommander interface {
	Stat(string) (os.FileInfo, error)
	Commander
}

// we need real OS implementation
type osCalls struct {
}

func (osCalls) Command(name string, arg ...string) *exec.Cmd {
	return exec.Command(name, arg...)
}

func (osCalls) Stat(name string) (os.FileInfo, error) {
	return os.Stat(name)
}

func argsParse(args []string) (runOptionsType, error) {
	parsing := flag.NewFlagSet("mender", flag.ContinueOnError)

	// FLAGS ---------------------------------------------------------------

	version := parsing.Bool("version", false, "Show mender agent version and exit.")

	config := parsing.String("config", defaultConfFile,
		"Configuration file location.")

	data := parsing.String("data", defaultDataStore,
		"Mender state data location.")

	commit := parsing.Bool("commit", false, "Commit current update.")

	imageFile := parsing.String("rootfs", "",
		"Root filesystem URI to use for update. Can be either a local "+
			"file or a URL.")

	daemon := parsing.Bool("daemon", false, "Run as a daemon.")

	// add bootstrap related command line options
	certFile := parsing.String("certificate", "", "Client certificate")
	certKey := parsing.String("cert-key", "", "Client certificate's private key")
	serverCert := parsing.String("trusted-certs", "", "Trusted server certificates")
	bootstrap := parsing.Bool("bootstrap", false, "Force bootstrap")

	// add log related command line options
	logFlags := addLogFlags(parsing)

	// PARSING -------------------------------------------------------------

	if err := parsing.Parse(args); err != nil {
		return runOptionsType{}, err
	}

	runOptions := runOptionsType{
		version,
		config,
		data,
		imageFile,
		commit,
		daemon,
		bootstrap,
		httpsClientConfig{
			*certFile,
			*certKey,
			*serverCert,
			false,
		},
	}

	//runOptions.bootstrap = httpsClientConfig{}

	// FLAG LOGIC ----------------------------------------------------------

	// we just want to see the version string, the rest does not
	// matter
	if *version == true {
		return runOptions, nil
	}

	if err := parseLogFlags(logFlags); err != nil {
		return runOptions, err
	}

	if moreThanOneRunOptionSelected(runOptions) {
		return runOptions, errMsgAmbiguousArgumentsGiven
	}

	return runOptions, nil
}

func moreThanOneRunOptionSelected(runOptions runOptionsType) bool {
	// check if more than one command line action is selected
	var runOptionsCount int

	if *runOptions.imageFile != "" {
		runOptionsCount++
	}
	if *runOptions.commit {
		runOptionsCount++
	}
	if *runOptions.daemon {
		runOptionsCount++
	}

	if runOptionsCount > 1 {
		return true
	}
	return false
}

func addLogFlags(f *flag.FlagSet) logOptionsType {

	var logOptions logOptionsType

	logOptions.debug = f.Bool("debug", false, "Debug log level. This is a "+
		"shorthand for '-l debug'.")

	logOptions.info = f.Bool("info", false, "Info log level. This is a "+
		"shorthand for '-l info'.")

	logOptions.logLevel = f.String("log-level", "", "Log level, which can be "+
		"'debug', 'info', 'warning', 'error', 'fatal' or 'panic'. "+
		"Earlier log levels will also log the subsequent levels (so "+
		"'debug' will log everything). The default log level is "+
		"'warning'.")

	logOptions.logModules = f.String("log-modules", "", "Filter logging by "+
		"module. This is a comma separated list of modules to log, "+
		"other modules will be omitted. To see which modules are "+
		"available, take a look at a non-filtered log and select "+
		"the modules appropriate for you.")

	logOptions.noSyslog = f.Bool("no-syslog", false, "Disable logging to "+
		"syslog. Note that debug message are never logged to syslog.")

	logOptions.logFile = f.String("log-file", "", "File to log to.")

	return logOptions

}

func parseLogFlags(args logOptionsType) error {
	var logOptCount int

	if *args.logLevel != "" {
		level, err := log.ParseLevel(*args.logLevel)
		if err != nil {
			return err
		}
		log.SetLevel(level)
		logOptCount++
	}

	if *args.info {
		log.SetLevel(log.InfoLevel)
		logOptCount++
	}

	if *args.debug {
		log.SetLevel(log.DebugLevel)
		logOptCount++
	}

	if logOptCount > 1 {
		return errMsgIncompatibleLogOptions
	} else if logOptCount == 0 {
		// Default log level.
		log.SetLevel(log.WarnLevel)
	}

	if *args.logFile != "" {
		fd, err := os.Create(*args.logFile)
		if err != nil {
			return err
		}
		log.SetOutput(fd)
	}

	if *args.logModules != "" {
		modules := strings.Split(*args.logModules, ",")
		log.SetModuleFilter(modules)
	}

	if !*args.noSyslog {
		if err := log.AddSyslogHook(); err != nil {
			log.Warnf("Could not connect to syslog daemon: %s. "+
				"(use -no-syslog to disable completely)",
				err.Error())
		}
	}

	return nil
}

func ShowVersion() {
	v := fmt.Sprintf("%s\n", VersionString())
	os.Stdout.Write([]byte(v))
}

func doMain(args []string) error {
	runOptions, err := argsParse(args)
	if err != nil {
		return err
	}

	if *runOptions.version {
		ShowVersion()
		return nil
	}

	config, err := LoadConfig(*runOptions.config)
	if err != nil {
		return err
	}

	env := NewEnvironment(new(osCalls))
	device := NewDevice(env, new(osCalls), config.GetDeviceConfig())
	switch {

	case *runOptions.imageFile != "":
		if err := doRootfs(device, runOptions); err != nil {
			return err
		}

	case *runOptions.commit:
		if err := device.CommitUpdate(); err != nil {
			return err
		}

	case *runOptions.daemon:
		updater, err := NewUpdateClient(config.GetUpdaterConfig())
		if err != nil {
			return errors.New("Cannot initialize daemon. Error instantiating updater. Exiting.")
		}

		store := NewDirStore(*runOptions.dataStore)

		controller := NewMender(*config, MenderPieces{
			updater,
			device,
			env,
			store,
		})
		if controller == nil {
			return errors.New("Cannot initialize mender controller")
		}

		if *runOptions.bootstrap {
			controller.ForceBootstrap()
		}

		daemon := NewDaemon(controller)
		return daemon.Run()

	case *runOptions.imageFile == "" && !*runOptions.commit &&
		!*runOptions.daemon:
		return errMsgNoArgumentsGiven
	}

	return nil
}

func main() {
	if err := doMain(os.Args[1:]); err != nil && err != flag.ErrHelp {
		log.Errorln(err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 realglobe, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/tls"
	"github.com/realglobe-Inc/edo-access-proxy/api/proxy"
	"github.com/realglobe-Inc/edo-access-proxy/database/session"
	"github.com/realglobe-Inc/edo-auth/database/token"
	keydb "github.com/realglobe-Inc/edo-id-provider/database/key"
	idpdb "github.com/realglobe-Inc/edo-idp-selector/database/idp"
	webdb "github.com/realglobe-Inc/edo-idp-selector/database/web"
	idperr "github.com/realglobe-Inc/edo-idp-selector/error"
	"github.com/realglobe-Inc/edo-lib/driver"
	logutil "github.com/realglobe-Inc/edo-lib/log"
	"github.com/realglobe-Inc/edo-lib/rand"
	"github.com/realglobe-Inc/edo-lib/server"
	"github.com/realglobe-Inc/go-lib/erro"
	"github.com/realglobe-Inc/go-lib/rglog"
	"net"
	"net/http"
	"os"
	"time"
)

func main() {
	var exitCode = 0
	defer func() {
		if exitCode != 0 {
			os.Exit(exitCode)
		}
	}()
	defer rglog.Flush()

	logutil.InitConsole("github.com/realglobe-Inc")

	param, err := parseParameters(os.Args...)
	if err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(erro.Wrap(err))
		exitCode = 1
		return
	}

	logutil.SetupConsole("github.com/realglobe-Inc", param.consLv)
	if err := logutil.Setup("github.com/realglobe-Inc", param.logType, param.logLv, param); err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(erro.Wrap(err))
		exitCode = 1
		return
	}

	if err := serve(param); err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(erro.Wrap(err))
		exitCode = 1
		return
	}

	log.Info("Shut down")
}

func serve(param *parameters) error {

	// バックエンドの準備。

	redPools := driver.NewRedisPoolSet(param.redTimeout, param.redPoolSize, param.redPoolExpIn)
	defer redPools.Close()
	monPools := driver.NewMongoPoolSet(param.monTimeout)
	defer monPools.Close()

	// 鍵。
	var keyDb keydb.Db
	switch param.keyDbType {
	case "file":
		keyDb = keydb.NewFileDb(param.keyDbPath)
		log.Info("Use keys in directory " + param.keyDbPath)
	case "redis":
		keyDb = keydb.NewRedisCache(keydb.NewFileDb(param.keyDbPath), redPools.Get(param.keyDbAddr), param.keyDbTag+"."+param.selfId, param.keyDbExpIn)
		log.Info("Use keys in directory " + param.keyDbPath + " with redis " + param.keyDbAddr + "<" + param.keyDbTag + "." + param.selfId + ">")
	default:
		return erro.New("invalid key DB type " + param.keyDbType)
	}

	// web データ。
	var webDb webdb.Db
	switch param.webDbType {
	case "direct":
		webDb = webdb.NewDirectDb()
		log.Info("Get web data directly")
	case "redis":
		webDb = webdb.NewRedisCache(webdb.NewDirectDb(), redPools.Get(param.webDbAddr), param.webDbTag, param.webDbExpIn)
		log.Info("Get web data with redis " + param.webDbAddr + "<" + param.webDbTag + ">")
	default:
		return erro.New("invalid web data DB type " + param.webDbType)
	}

	// IdP 情報。
	var idpDb idpdb.Db
	switch param.idpDbType {
	case "mongo":
		pool, err := monPools.Get(param.idpDbAddr)
		if err != nil {
			return erro.Wrap(err)
		}
		idpDb = idpdb.NewMongoDb(pool, param.idpDbTag, param.idpDbTag2, webDb)
		log.Info("Use IdP info in mongodb " + param.idpDbAddr + "<" + param.idpDbTag + "." + param.idpDbTag2 + ">")
	default:
		return erro.New("invalid IdP DB type " + param.idpDbType)
	}

	// アクセストークン。
	var tokDb token.Db
	switch param.tokDbType {
	case "memory":
		tokDb = token.NewMemoryDb()
		log.Info("Save access tokens in memory")
	case "redis":
		tokDb = token.NewRedisDb(redPools.Get(param.tokDbAddr), param.tokDbTag)
		log.Info("Save access tokens in redis " + param.tokDbAddr + ": " + param.tokDbTag)
	default:
		return erro.New("invalid access token DB type " + param.tokDbType)
	}

	// セッション。
	var sessDb session.Db
	switch param.sessDbType {
	case "memory":
		sessDb = session.NewMemoryDb()
		log.Info("Save sessions in memory")
	case "redis":
		sessDb = session.NewRedisDb(redPools.Get(param.sessDbAddr), param.sessDbTag)
		log.Info("Save sessions in redis " + param.sessDbAddr + "<" + param.sessDbTag + ">")
	default:
		return erro.New("invalid session DB type " + param.sessDbType)
	}

	idGen := rand.New(time.Minute)

	var tr http.RoundTripper
	if param.noVeri {
		// http.DefaultTransport を参考にした。
		tr = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
		}
	}

	// バックエンドの準備完了。

	if param.debug {
		idperr.Debug = true
	}

	stopper := server.NewStopper()
	defer func() {
		// 処理の終了待ち。
		stopper.Lock()
		defer stopper.Unlock()
		for stopper.Stopped() {
			stopper.Wait()
		}
	}()

	mux := http.NewServeMux()
	routes := map[string]bool{}
	mux.HandleFunc(param.pathOk, idperr.WrapApi(stopper, func(w http.ResponseWriter, r *http.Request) error {
		return nil
	}))
	routes[param.pathOk] = true
	mux.Handle(param.pathProx, proxy.New(
		stopper,
		param.selfId,
		param.sigAlg,
		param.sigKid,
		param.hashAlg,
		param.jtiLen,
		param.jtiExpIn,
		param.fileThres,
		keyDb,
		idpDb,
		tokDb,
		sessDb,
		idGen,
		tr,
		param.debug,
	))
	routes[param.pathProx] = true

	if !routes["/"] {
		mux.HandleFunc("/", idperr.WrapApi(stopper, func(w http.ResponseWriter, r *http.Request) error {
			return erro.Wrap(idperr.New(idperr.Invalid_request, "invalid endpoint", http.StatusNotFound, nil))
		}))
	}
	return server.Serve(param, mux)
}
<commit_msg>処理の順番変更<commit_after>// Copyright 2015 realglobe, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/tls"
	"github.com/realglobe-Inc/edo-access-proxy/api/proxy"
	"github.com/realglobe-Inc/edo-access-proxy/database/session"
	"github.com/realglobe-Inc/edo-auth/database/token"
	keydb "github.com/realglobe-Inc/edo-id-provider/database/key"
	idpdb "github.com/realglobe-Inc/edo-idp-selector/database/idp"
	webdb "github.com/realglobe-Inc/edo-idp-selector/database/web"
	idperr "github.com/realglobe-Inc/edo-idp-selector/error"
	"github.com/realglobe-Inc/edo-lib/driver"
	logutil "github.com/realglobe-Inc/edo-lib/log"
	"github.com/realglobe-Inc/edo-lib/rand"
	"github.com/realglobe-Inc/edo-lib/server"
	"github.com/realglobe-Inc/go-lib/erro"
	"github.com/realglobe-Inc/go-lib/rglog"
	"net"
	"net/http"
	"os"
	"time"
)

func main() {
	var exitCode = 0
	defer func() {
		if exitCode != 0 {
			os.Exit(exitCode)
		}
	}()
	defer rglog.Flush()

	logutil.InitConsole("github.com/realglobe-Inc")

	param, err := parseParameters(os.Args...)
	if err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(erro.Wrap(err))
		exitCode = 1
		return
	}

	logutil.SetupConsole("github.com/realglobe-Inc", param.consLv)
	if err := logutil.Setup("github.com/realglobe-Inc", param.logType, param.logLv, param); err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(erro.Wrap(err))
		exitCode = 1
		return
	}

	if err := serve(param); err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(erro.Wrap(err))
		exitCode = 1
		return
	}

	log.Info("Shut down")
}

func serve(param *parameters) error {

	// バックエンドの準備。

	stopper := server.NewStopper()

	redPools := driver.NewRedisPoolSet(param.redTimeout, param.redPoolSize, param.redPoolExpIn)
	defer redPools.Close()
	monPools := driver.NewMongoPoolSet(param.monTimeout)
	defer monPools.Close()

	// 鍵。
	var keyDb keydb.Db
	switch param.keyDbType {
	case "file":
		keyDb = keydb.NewFileDb(param.keyDbPath)
		log.Info("Use keys in directory " + param.keyDbPath)
	case "redis":
		keyDb = keydb.NewRedisCache(keydb.NewFileDb(param.keyDbPath), redPools.Get(param.keyDbAddr), param.keyDbTag+"."+param.selfId, param.keyDbExpIn)
		log.Info("Use keys in directory " + param.keyDbPath + " with redis " + param.keyDbAddr + "<" + param.keyDbTag + "." + param.selfId + ">")
	default:
		return erro.New("invalid key DB type " + param.keyDbType)
	}

	// web データ。
	var webDb webdb.Db
	switch param.webDbType {
	case "direct":
		webDb = webdb.NewDirectDb()
		log.Info("Get web data directly")
	case "redis":
		webDb = webdb.NewRedisCache(webdb.NewDirectDb(), redPools.Get(param.webDbAddr), param.webDbTag, param.webDbExpIn)
		log.Info("Get web data with redis " + param.webDbAddr + "<" + param.webDbTag + ">")
	default:
		return erro.New("invalid web data DB type " + param.webDbType)
	}

	// IdP 情報。
	var idpDb idpdb.Db
	switch param.idpDbType {
	case "mongo":
		pool, err := monPools.Get(param.idpDbAddr)
		if err != nil {
			return erro.Wrap(err)
		}
		idpDb = idpdb.NewMongoDb(pool, param.idpDbTag, param.idpDbTag2, webDb)
		log.Info("Use IdP info in mongodb " + param.idpDbAddr + "<" + param.idpDbTag + "." + param.idpDbTag2 + ">")
	default:
		return erro.New("invalid IdP DB type " + param.idpDbType)
	}

	// アクセストークン。
	var tokDb token.Db
	switch param.tokDbType {
	case "memory":
		tokDb = token.NewMemoryDb()
		log.Info("Save access tokens in memory")
	case "redis":
		tokDb = token.NewRedisDb(redPools.Get(param.tokDbAddr), param.tokDbTag)
		log.Info("Save access tokens in redis " + param.tokDbAddr + ": " + param.tokDbTag)
	default:
		return erro.New("invalid access token DB type " + param.tokDbType)
	}

	// セッション。
	var sessDb session.Db
	switch param.sessDbType {
	case "memory":
		sessDb = session.NewMemoryDb()
		log.Info("Save sessions in memory")
	case "redis":
		sessDb = session.NewRedisDb(redPools.Get(param.sessDbAddr), param.sessDbTag)
		log.Info("Save sessions in redis " + param.sessDbAddr + "<" + param.sessDbTag + ">")
	default:
		return erro.New("invalid session DB type " + param.sessDbType)
	}

	idGen := rand.New(time.Minute)

	var tr http.RoundTripper
	if param.noVeri {
		// http.DefaultTransport を参考にした。
		tr = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
		}
	}

	// バックエンドの準備完了。

	if param.debug {
		idperr.Debug = true
	}

	mux := http.NewServeMux()
	routes := map[string]bool{}
	mux.HandleFunc(param.pathOk, idperr.WrapApi(stopper, func(w http.ResponseWriter, r *http.Request) error {
		return nil
	}))
	routes[param.pathOk] = true
	mux.Handle(param.pathProx, proxy.New(
		stopper,
		param.selfId,
		param.sigAlg,
		param.sigKid,
		param.hashAlg,
		param.jtiLen,
		param.jtiExpIn,
		param.fileThres,
		keyDb,
		idpDb,
		tokDb,
		sessDb,
		idGen,
		tr,
		param.debug,
	))
	routes[param.pathProx] = true

	if !routes["/"] {
		mux.HandleFunc("/", idperr.WrapApi(stopper, func(w http.ResponseWriter, r *http.Request) error {
			return erro.Wrap(idperr.New(idperr.Invalid_request, "invalid endpoint", http.StatusNotFound, nil))
		}))
	}

	// サーバー設定完了。

	defer func() {
		// 処理の終了待ち。
		stopper.Lock()
		defer stopper.Unlock()
		for stopper.Stopped() {
			stopper.Wait()
		}
	}()
	return server.Serve(param, mux)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/tealeg/xlsx"
)

var xlsxPath = flag.String("f", "", "Path to an XLSX file")
var sheetIndex = flag.Int("i", 0, "Index of sheet to convert, zero based")
var delimiter = flag.String("d", ";", "Delimiter to use between fields")

type outputer func(s string)

func generateCSVFromXLSXFile(excelFileName string, sheetIndex int, outputf outputer) error {
	xlFile, error := xlsx.OpenFile(excelFileName)
	if error != nil {
		return error
	}
	sheetLen := len(xlFile.Sheets)
	switch {
	case sheetLen == 0:
		return errors.New("This XLSX file contains no sheets.")
	case sheetIndex >= sheetLen:
		return fmt.Errorf("No sheet %d available, please select a sheet between 0 and %d\n", sheetIndex, sheetLen-1)
	}
	sheet := xlFile.Sheets[sheetIndex]
	for _, row := range sheet.Rows {
		var vals []string
		if row != nil {
			for _, cell := range row.Cells {
				vals = append(vals, fmt.Sprintf("%q", cell.String()))
			}
			outputf(strings.Join(vals, *delimiter) + "\n")
		}
	}
	return nil
}

func main() {
	flag.Parse()
	var err error
	if len(os.Args) < 3 {
        flag.PrintDefaults()
		return
	}
	flag.Parse()
	err = generateCSVFromXLSXFile(*xlsxPath, *sheetIndex, func(s string) { fmt.Printf("%s", s) })
	if err != nil {
		fmt.Printf(err.Error())
		return
	}
}
<commit_msg>error simplification<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/tealeg/xlsx"
)

var xlsxPath = flag.String("f", "", "Path to an XLSX file")
var sheetIndex = flag.Int("i", 0, "Index of sheet to convert, zero based")
var delimiter = flag.String("d", ";", "Delimiter to use between fields")

type outputer func(s string)

func generateCSVFromXLSXFile(excelFileName string, sheetIndex int, outputf outputer) error {
	xlFile, error := xlsx.OpenFile(excelFileName)
	if error != nil {
		return error
	}
	sheetLen := len(xlFile.Sheets)
	switch {
	case sheetLen == 0:
		return errors.New("This XLSX file contains no sheets.")
	case sheetIndex >= sheetLen:
		return fmt.Errorf("No sheet %d available, please select a sheet between 0 and %d\n", sheetIndex, sheetLen-1)
	}
	sheet := xlFile.Sheets[sheetIndex]
	for _, row := range sheet.Rows {
		var vals []string
		if row != nil {
			for _, cell := range row.Cells {
				vals = append(vals, fmt.Sprintf("%q", cell.String()))
			}
			outputf(strings.Join(vals, *delimiter) + "\n")
		}
	}
	return nil
}

func main() {
	flag.Parse()
	var err error
	if len(os.Args) < 3 {
		flag.PrintDefaults()
		return
	}
	flag.Parse()
	err = generateCSVFromXLSXFile(*xlsxPath, *sheetIndex, func(s string) { fmt.Printf("%s", s) })
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	_ "github.com/joho/godotenv/autoload"
	"github.com/gocms-io/gocms/context"
	"github.com/gocms-io/gocms/controllers"
	"github.com/gocms-io/gocms/database"
	"github.com/gocms-io/gocms/repositories"
	"github.com/gocms-io/gocms/services"
	"log"
	"os"
)

var app *Engine

type Engine struct {
	Gin               *gin.Engine
	ControllersGroup  *controllers.ControllersGroup
	ServicesGroup     *services.ServicesGroup
	RepositoriesGroup *repositories.RepositoriesGroup
	Database          *database.Database
}

//go:generate apidoc -c ./ -i ./models -i ./controllers/ -o ./content/docs/ -f ".*\\.go$" -f ".*\\.js$"
func Default() *Engine {

	// init config environment vars
	context.Init()

	// setup database
	db := database.Default()

	// migrate cms db
	db.MigrateCMSSql()

	// start gin with defaults
	r := gin.Default()
	// setup repositories
	rg := repositories.DefaultRepositoriesGroup(db)

	// setup services
	sg := services.DefaultServicesGroup(rg)

	// setup controllers
	cg := controllers.DefaultControllerGroup(r, sg)

	// create engine
	engine := Engine{
		Gin:               r,
		ControllersGroup:  cg,
		ServicesGroup:     sg,
		RepositoriesGroup: rg,
		Database:          db,
	}
	return &engine
}

func (engine *Engine) Listen(uri string) {

	err := engine.Gin.Run(uri)
	log.Println(err.Error())

}

func main() {

	// startup defaults
	app = Default()

	// start server and listen
	port := context.Config.Port

	// check if env is set and override
	portEnv := os.Getenv("PORT")
	if portEnv != "" {
		port = portEnv
	}

	if port == "" {
		port = "8080"
	}

	app.Listen(":" + port)
}
<commit_msg>add port flag to main.go<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/gocms-io/gocms/context"
	"github.com/gocms-io/gocms/controllers"
	"github.com/gocms-io/gocms/database"
	"github.com/gocms-io/gocms/repositories"
	"github.com/gocms-io/gocms/services"
	_ "github.com/joho/godotenv/autoload"
	"log"
	"os"
)

var app *Engine

type Engine struct {
	Gin               *gin.Engine
	ControllersGroup  *controllers.ControllersGroup
	ServicesGroup     *services.ServicesGroup
	RepositoriesGroup *repositories.RepositoriesGroup
	Database          *database.Database
}

//go:generate apidoc -c ./ -i ./models -i ./controllers/ -o ./content/docs/ -f ".*\\.go$" -f ".*\\.js$"
func Default() *Engine {

	// init config environment vars
	context.Init()

	// setup database
	db := database.Default()

	// migrate cms db
	db.MigrateCMSSql()

	// start gin with defaults
	r := gin.Default()
	// setup repositories
	rg := repositories.DefaultRepositoriesGroup(db)

	// setup services
	sg := services.DefaultServicesGroup(rg)

	// setup controllers
	cg := controllers.DefaultControllerGroup(r, sg)

	// create engine
	engine := Engine{
		Gin:               r,
		ControllersGroup:  cg,
		ServicesGroup:     sg,
		RepositoriesGroup: rg,
		Database:          db,
	}
	return &engine
}

func (engine *Engine) Listen(uri string) {

	err := engine.Gin.Run(uri)
	log.Println(err.Error())

}

func main() {

	// startup defaults
	app = Default()

	// start server and listen
	port := context.Config.Port

	// check if env is set and override
	portEnv := os.Getenv("PORT")
	if portEnv != "" {
		port = portEnv
	}

	// check for port flag and override all
	portFlag := *flag.Int("port", 0, "port to run on. Overrides all.")
	flag.Parse()
	if portFlag != 0 {
		port = fmt.Sprintf("%v", portFlag)
	}

	if port == "" {
		port = "8080"
	}

	app.Listen(":" + port)
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "net/http"
    "log"
    "./lab"
)

var redisServer string
var serverCount string
var dbStartCount string

func sayhelloName(w http.ResponseWriter, r *http.Request) {
    color := "red"
    content := ""
    content += serverCount+" servers currently serving web content.
"
    content += dbStartCount+" server starts have been seen.
"
    content += lab.GetPageCount()+" page loads have happened.
"
    content += ""
    fmt.Fprintf(w, content)
}

func healthCheck(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w,"OK\n")
}

func main() {
    http.HandleFunc("/", sayhelloName) // set router
    http.HandleFunc("/health", healthCheck) // set router

    urlsToRegister := []string{"/"}
    ipOfConsul := "localhost"
    portWeListenOn := "8080"

    result, err := lab.RegisterMe(ipOfConsul, urlsToRegister, portWeListenOn )
    redisServer = lab.GetServiceAddress("redis")
    serverCount = lab.GetServerCount("localhost-")
    dbStartCount = lab.GetDBStartCount()

    if result {
        fmt.Println("Server started and listening on port :"+portWeListenOn)
        err = http.ListenAndServe(":"+portWeListenOn, nil) // set listen port
        if err != nil {
            log.Fatal("ListenAndServe: ", err)
        }
    } else {
        fmt.Println(err)
    }
}
<commit_msg>Path changes<commit_after>package main

import (
    "fmt"
    "net/http"
    "log"
    "github.com/robjporter/CPVT-CloudNativeLab"
)

var redisServer string
var serverCount string
var dbStartCount string

func sayhelloName(w http.ResponseWriter, r *http.Request) {
    color := "red"
    content := ""
    content += serverCount+" servers currently serving web content.
"
    content += dbStartCount+" server starts have been seen.
"
    content += lab.GetPageCount()+" page loads have happened.
"
    content += ""
    fmt.Fprintf(w, content)
}

func healthCheck(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w,"OK\n")
}

func main() {
    http.HandleFunc("/", sayhelloName) // set router
    http.HandleFunc("/health", healthCheck) // set router

    urlsToRegister := []string{"/"}
    ipOfConsul := "localhost"
    portWeListenOn := "8080"

    result, err := lab.RegisterMe(ipOfConsul, urlsToRegister, portWeListenOn )
    redisServer = lab.GetServiceAddress("redis")
    serverCount = lab.GetServerCount("localhost-")
    dbStartCount = lab.GetDBStartCount()

    if result {
        fmt.Println("Server started and listening on port :"+portWeListenOn)
        err = http.ListenAndServe(":"+portWeListenOn, nil) // set listen port
        if err != nil {
            log.Fatal("ListenAndServe: ", err)
        }
    } else {
        fmt.Println(err)
    }
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/metabition/genny/parse"
)

/*

  source | genny gen "KeyType=string,int ValueType=string,int"

*/

const (
	exitcodeInvalidArgs       = 1
	exitcodeInvalidTypeSet    = 2
	exitcodeStdinFailed       = 3
	exitcodeGenFailed         = 4
	exitcodeSourceFileInvalid = 5
)

func main() {
	var (
		filename = flag.String("f", "", "file to parse instead of stdin")
	)
	flag.Parse()
	args := flag.Args()
	if len(args) != 3 {
		usage()
		os.Exit(exitcodeInvalidArgs)
	}
	if strings.ToLower(args[1]) != "gen" {
		usage()
		os.Exit(exitcodeInvalidArgs)
	}

	// parse the typesets
	typeSets, err := parse.TypeSet(args[2])
	if err != nil {
		fatal(exitcodeInvalidTypeSet, err)
	}

	if len(*filename) > 0 {
		file, err := os.Open(*filename)
		if err != nil {
			fatal(exitcodeSourceFileInvalid, err)
		}
		defer file.Close()
		err = gen(*filename, file, typeSets, os.Stdout)
	} else {
		source, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			fatal(exitcodeStdinFailed, err)
		}
		reader := bytes.NewReader(source)
		err = gen("stdin", reader, typeSets, os.Stdout)
	}

	// do the work
	if err != nil {
		fatal(exitcodeGenFailed, err)
	}

}

func usage() {
	fmt.Fprintln(os.Stderr, `usage: genny gen "{types}"

gen - generates type specific code (to stdout) from generic code (via stdin) or the file specified with the -f flag.

{types}  - (required) Specific types for each generic type in the source
{types} format:  {generic}={specific}[,another][ {generic2}={specific2}]

Examples:
  Generic=Specific
  Generic1=Specific1 Generic2=Specific2
  Generic1=Specific1,Specific2 Generic2=Specific3,Specific4

Flags:
`)
	flag.PrintDefaults()
}

func fatal(code int, a ...interface{}) {
	fmt.Println(a...)
	os.Exit(code)
}

// gen performs the generic generation.
func gen(filename string, in io.ReadSeeker, typesets []map[string]string, out io.Writer) error {

	var output []byte
	var err error

	output, err = parse.Generics(filename, in, typesets)
	if err != nil {
		return err
	}

	out.Write(output)
	return nil
}
<commit_msg>small usage tweak<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/metabition/genny/parse"
)

/*

  source | genny gen "KeyType=string,int ValueType=string,int"

*/

const (
	exitcodeInvalidArgs       = 1
	exitcodeInvalidTypeSet    = 2
	exitcodeStdinFailed       = 3
	exitcodeGenFailed         = 4
	exitcodeSourceFileInvalid = 5
)

func main() {
	var (
		filename = flag.String("f", "", "file to parse instead of stdin")
	)
	flag.Parse()
	args := flag.Args()
	if len(args) != 3 {
		usage()
		os.Exit(exitcodeInvalidArgs)
	}
	if strings.ToLower(args[1]) != "gen" {
		usage()
		os.Exit(exitcodeInvalidArgs)
	}

	// parse the typesets
	typeSets, err := parse.TypeSet(args[2])
	if err != nil {
		fatal(exitcodeInvalidTypeSet, err)
	}

	if len(*filename) > 0 {
		file, err := os.Open(*filename)
		if err != nil {
			fatal(exitcodeSourceFileInvalid, err)
		}
		defer file.Close()
		err = gen(*filename, file, typeSets, os.Stdout)
	} else {
		source, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			fatal(exitcodeStdinFailed, err)
		}
		reader := bytes.NewReader(source)
		err = gen("stdin", reader, typeSets, os.Stdout)
	}

	// do the work
	if err != nil {
		fatal(exitcodeGenFailed, err)
	}

}

func usage() {
	fmt.Fprintln(os.Stderr, `usage: genny gen "{types}"

gen - generates type specific code (to stdout) from generic code (via stdin) or the file specified with the -f flag.

{types}  - (required) Specific types for each generic type in the source
{types} format:  {generic}={specific}[,another][ {generic2}={specific2}]

Examples:
  Generic=Specific
  Generic1=Specific1 Generic2=Specific2
  Generic1=Specific1,Specific2 Generic2=Specific3,Specific4

Flags:`)
	flag.PrintDefaults()
}

func fatal(code int, a ...interface{}) {
	fmt.Println(a...)
	os.Exit(code)
}

// gen performs the generic generation.
func gen(filename string, in io.ReadSeeker, typesets []map[string]string, out io.Writer) error {

	var output []byte
	var err error

	output, err = parse.Generics(filename, in, typesets)
	if err != nil {
		return err
	}

	out.Write(output)
	return nil
}
<|endoftext|>


<commit_before>package bookstore

import (
	"fmt"
	"net/http"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"google.golang.org/appengine"
	"google.golang.org/appengine/user"
)

// Match is a result returned from scraping
type Match struct {
	CourseCode string // PHYS-1030-FA
	Synonym    string // 643369
	Title      string // Intro Appl Phys I (Mechanics)
	Instructor string // Dr. Mark C. Gallagher
	Books      string // Link?
	Term       string // Fall
	Department string // Physics
	YearLevel  string // 1
}

func init() {
	http.HandleFunc("/", auth)
}

// HomeHandler handles the home page
func HomeHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello, world!")
}

func auth(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	u := user.Current(c)
	if u == nil {
		url, err := user.LoginURL(c, r.URL.String())
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		w.Header().Set("Location", url)
		w.WriteHeader(http.StatusFound)
		return
	}
	fmt.Fprintf(w, "Hello, %v!", u)
}

// Scrape finds and serializes the data from Lakehead's
// site.
func Scrape(url string) (Matches []*Match, err error) {

	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	root, err := html.Parse(res.Body)
	if err != nil {
		return nil, err
	}

	// Yhat's package expects atomic values for tags, see
	// https://godoc.org/golang.org/x/net/html/atom if you
	// need a different tag.
	data := scrape.FindAll(root, scrape.ByTag(0x10502))
	// matches := make([]*Match, len(data))
	for _, match := range data {
		fmt.Println("Match: ", match)
	}

	return nil, nil
}
<commit_msg>Attempt to figure out appengine logging<commit_after>package bookstore

import (
	"fmt"
	"net/http"

	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"google.golang.org/appengine"
	"google.golang.org/appengine/user"
)

// Match is a result returned from scraping
type Match struct {
	CourseCode string // PHYS-1030-FA
	Synonym    string // 643369
	Title      string // Intro Appl Phys I (Mechanics)
	Instructor string // Dr. Mark C. Gallagher
	Books      string // Link?
	Term       string // Fall
	Department string // Physics
	YearLevel  string // 1
}

func init() {
	http.HandleFunc("/", auth)
	http.HandleFunc("/scrape", HomeHandler)
}

// HomeHandler handles the home page
func HomeHandler(w http.ResponseWriter, r *http.Request) {

	fmt.Fprint(w, "Hello, Benny world!")
	Scrape("http://timetable.lakeheadu.ca/2015FW_UG_TBAY/phys.html", r)
}

func auth(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	u := user.Current(c)
	if u == nil {
		url, err := user.LoginURL(c, r.URL.String())
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		w.Header().Set("Location", url)
		w.WriteHeader(http.StatusFound)
		return
	}
	fmt.Fprintf(w, "Hello, %v!", u)
}

// Scrape finds and serializes the data from Lakehead's
// site.
func Scrape(url string, r *http.Request) (Matches []*Match, err error) {

	// Required for logging
	c := appengine.NewContext(r)

	res, err := http.Get(url)
	if err != nil {
		fmt.Println("Error :", err)
		return nil, err
	}

	root, err := html.Parse(res.Body)
	if err != nil {
		fmt.Println("Error :", err)
		return nil, err
	}

	// Yhat's package expects atomic values for tags, see
	// https://godoc.org/golang.org/x/net/html/atom if you
	// need a different tag.
	data := scrape.FindAll(root, scrape.ByTag(0x10502))
	// matches := make([]*Match, len(data))
	for _, match := range data {
		c.Infof("Match: %v", match)
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"code.google.com/p/go.example/newmath"
)

func main() {
	fmt.Printf("Hello, world. Sqrt(2) = %v\n", newmath.Sqrt(2))
}<commit_msg>Edit package<commit_after>package main

import (
	"fmt"
	"math"
)

func main() {
	fmt.Printf("Hello, world. Sqrt(2) = %v\n", math.Sqrt(2))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"io"

	// "golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/clearsign"

	"github.com/miekg/mmark"
	"github.com/jhillyerd/enmime"
)

// Render a Markdown fragment into a html fragment
func Render(md string) (html string, err error) {
	// not rendering entire page - that will happen using the tmpl html
	page := false
	css := ""
	head := ""

	// set up options
	extensions := 0
	extensions |= mmark.EXTENSION_TABLES
	extensions |= mmark.EXTENSION_FENCED_CODE
	extensions |= mmark.EXTENSION_AUTOLINK
	extensions |= mmark.EXTENSION_SPACE_HEADERS
	extensions |= mmark.EXTENSION_CITATION
	extensions |= mmark.EXTENSION_TITLEBLOCK_TOML
	extensions |= mmark.EXTENSION_HEADER_IDS
	extensions |= mmark.EXTENSION_AUTO_HEADER_IDS
	extensions |= mmark.EXTENSION_UNIQUE_HEADER_IDS
	extensions |= mmark.EXTENSION_FOOTNOTES
	extensions |= mmark.EXTENSION_SHORT_REF
	extensions |= mmark.EXTENSION_INCLUDE
	extensions |= mmark.EXTENSION_PARTS
	extensions |= mmark.EXTENSION_ABBREVIATIONS
	extensions |= mmark.EXTENSION_DEFINITION_LISTS

	var renderer mmark.Renderer
	htmlFlags := 0
	if page {
		htmlFlags |= mmark.HTML_COMPLETE_PAGE
	}
	renderer = mmark.HtmlRenderer(htmlFlags, css, head)

	output := mmark.Parse([]byte(md), renderer, extensions).Bytes()
	return string(output), nil
}

func main() {
	env, err := enmime.ReadEnvelope(os.Stdin)
	if err != nil {
		fmt.Println("[e] ", err);
		return;
	}

	fmt.Fprintf(os.Stderr, "From: %v\n", env.GetHeader("From"));

	for i := range env.Attachments {
		fmt.Println(env.Attachments[i].FileName, env.Attachments[i].ContentType);

		f, _ := os.Create(env.Attachments[i].FileName)
		defer f.Close()
		io.Copy(f, env.Attachments[i]);
	}

	blk, _ := clearsign.Decode([]byte(env.Text));
	if blk == nil {
		fmt.Println("[e] No clearsign message!")
		return
	}

	html, err := Render(string(blk.Plaintext));
	if err != nil {
		fmt.Println("[e] ", err);
		return;
	}
	fmt.Println("md:\n" + html);

	//signer, err = openpgp.CheckArmoredDetachedSignature(keyring, bytes.NewReader(file.content), bytes.NewReader(file.signature))

	//fmt.Println(entity)
}<commit_msg>signature check<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"io"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/clearsign"

	"github.com/miekg/mmark"
	"github.com/jhillyerd/enmime"
)

// Render a Markdown fragment into a html fragment
func Render(md string) (html string, err error) {
	// not rendering entire page - that will happen using the tmpl html
	page := false
	css := ""
	head := ""

	// set up options
	extensions := 0
	extensions |= mmark.EXTENSION_TABLES
	extensions |= mmark.EXTENSION_FENCED_CODE
	extensions |= mmark.EXTENSION_AUTOLINK
	extensions |= mmark.EXTENSION_SPACE_HEADERS
	extensions |= mmark.EXTENSION_CITATION
	extensions |= mmark.EXTENSION_TITLEBLOCK_TOML
	extensions |= mmark.EXTENSION_HEADER_IDS
	extensions |= mmark.EXTENSION_AUTO_HEADER_IDS
	extensions |= mmark.EXTENSION_UNIQUE_HEADER_IDS
	extensions |= mmark.EXTENSION_FOOTNOTES
	extensions |= mmark.EXTENSION_SHORT_REF
	extensions |= mmark.EXTENSION_INCLUDE
	extensions |= mmark.EXTENSION_PARTS
	extensions |= mmark.EXTENSION_ABBREVIATIONS
	extensions |= mmark.EXTENSION_DEFINITION_LISTS

	var renderer mmark.Renderer
	htmlFlags := 0
	if page {
		htmlFlags |= mmark.HTML_COMPLETE_PAGE
	}
	renderer = mmark.HtmlRenderer(htmlFlags, css, head)

	output := mmark.Parse([]byte(md), renderer, extensions).Bytes()
	return string(output), nil
}

func main() {
	env, err := enmime.ReadEnvelope(os.Stdin)
	if err != nil {
		fmt.Println("[e] ", err);
		return;
	}

	keyFile, err := os.Open("pub.key")
	if err != nil {
		fmt.Println("[e] PubKey Error:", err);
		return;
	}

	keyRing, err := openpgp.ReadArmoredKeyRing(keyFile)
	// armoredKey, err := ioutil.ReadFile("pub.key")
	if err != nil {
		fmt.Println("[e] PubKey Error:", err);
		return;
	}

	fmt.Fprintf(os.Stderr, "From: %v\n", env.GetHeader("From"));

	for i := range env.Attachments {
		fmt.Println(env.Attachments[i].FileName, env.Attachments[i].ContentType);

		f, _ := os.Create(env.Attachments[i].FileName)
		defer f.Close()
		io.Copy(f, env.Attachments[i]);
	}

	blk, _ := clearsign.Decode([]byte(env.Text));
	if blk == nil {
		fmt.Println("[e] No clearsign message!")
		return
	}

	signer, err := openpgp.CheckDetachedSignature(keyRing, bytes.NewReader(blk.Bytes), blk.ArmoredSignature.Body)
	if err != nil {
		fmt.Println("[e] Signer:", err);
		return;
	}

	fmt.Println("Signer: ", signer);

	html, err := Render(string(blk.Plaintext));
	if err != nil {
		fmt.Println("[e] ", err);
		return;
	}
	fmt.Println("md:\n" + html);

	//fmt.Println(entity)
}<|endoftext|>


<commit_before>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"github.com/rjeczalik/notify"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"
)

type ReplicatServer struct {
	Name         string
	Address      string
	PublicKey    string
	LastReceived DirTreeMap
	LastSent     DirTreeMap
}

type ServerMap map[string]ReplicatServer

var GlobalServerMap = ServerMap{
	"NodeA": ReplicatServer{
		Address: ":8001",
	},
	"NodeB": ReplicatServer{
		Address: ":8002",
	},
}

type Event struct {
	Source       string
	Name         string
	OriginalName string
	Message      string
	Time         time.Time
	IsDirectory  bool
}

var events = make([]Event, 0, 100)

func main() {
	fmt.Println("replicat initializing....")
	rand.Seed(int64(time.Now().Nanosecond()))

	SetupCli()

	// Update the path that traffic is served from to be the filsystem canonical path. This will allow the event folders that come in to match what we have.
	symPath, err := filepath.EvalSymlinks(globalSettings.Directory)
	if err != nil {
		panic(err)
	}

	if symPath != globalSettings.Directory {
		fmt.Printf("Updating serving directory to: %s\n", symPath)
		globalSettings.Directory = symPath
	}

	listOfFileInfo, err := createListOfFolders()
	if err != nil {
		log.Fatal(err)
	}

	// Make the channel buffered to ensure no event is dropped. Notify will drop
	// an event if the receiver is not able to keep up the sending pace.
	fsEventsChannel := make(chan notify.EventInfo, 1)

	// Set up a watch point listening for events within a directory tree rooted at the specified folder
	err = notify.Watch(globalSettings.Directory+"/...", fsEventsChannel, notify.All)
	if err != nil {
		log.Fatal(err)
	}
	defer notify.Stop(fsEventsChannel)

	fmt.Printf("replicat %s online....\n", globalSettings.Name)
	defer fmt.Println("End of line")

	globalSettings.Address = GlobalServerMap[globalSettings.Name].Address

	totalFiles := 0
	for _, fileInfoList := range listOfFileInfo {
		totalFiles += len(fileInfoList)
	}

	fmt.Printf("Now keeping an eye on %d folders and %d files located under: %s\n", len(listOfFileInfo), totalFiles, globalSettings.Directory)
	go func(c chan notify.EventInfo) {
		for {
			ei := <-c

			// todo add ignore file
			// sendEvent to manager (if it's available)
			fullPath := string(ei.Path())
			directoryLength := len(globalSettings.Directory)

			path := fullPath
			if len(fullPath) >= directoryLength && globalSettings.Directory == fullPath[:directoryLength] {
				// update the path to not have this prefix
				path = fullPath[directoryLength+1:]
			}

			//fmt.Printf("Call to isdir resulted in %v\n", ei.IsDir())

			//// Check if it is a directory based on our directory tree first. Then check the file system
			var isDirectory bool
			//_, exists := listOfFileInfo[fullPath]
			//if exists == true {
			//	isDirectory = true
			//} else {
			//}
			// Get the file info. Useful if we need to restrict our actions to directories
			info, err := os.Stat(fullPath)
			if err == nil {
				isDirectory = info.IsDir()
			} else {
				_, exists := listOfFileInfo[fullPath]
				if exists == true {
					isDirectory = true
				}
			}

			event := Event{Name: ei.Event().String(), Message: path, IsDirectory: isDirectory}
			fmt.Printf("event raw data: %v with path: %v\n", ei.Event(), path)

			// todo copy the source folder on a rename to the event
			sendEvent(&event, globalSettings.ManagerAddress, globalSettings.ManagerCredentials)

			// sendEvent to peers (if any)
			for name, server := range GlobalServerMap {
				if name != globalSettings.Name {
					fmt.Printf("sending to peer %s\n", name)
					sendEvent(&event, server.Address, globalSettings.ManagerCredentials)
				}
			}

			log.Println("Got event:" + ei.Event().String() + ", with Path:" + ei.Path())
		}
	}(fsEventsChannel)

	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))

	fmt.Printf("listening on: %s\n", globalSettings.Address)

	go func() {
		err = http.ListenAndServe(globalSettings.Address, nil)
		if err != nil {
			panic(err)
		}
	}()

	dotCount := 0
	sleepSeconds := time.Duration(25 + rand.Intn(10))
	fmt.Printf("Full Sync time set to: %d seconds\n", sleepSeconds)
	for {
		// Randomize the sync time to decrease oscillation
		time.Sleep(time.Second * sleepSeconds)
		changed, updatedState, newPaths, deletedPaths, matchingPaths := checkForChanges(listOfFileInfo, nil)
		if changed {
			fmt.Println("\nWe have changes, ship it (also updating saved state now that the changes were tracked)")
			fmt.Printf("@Path report: new %d, deleted %d, matching %d, original %d, updated %d\n", len(newPaths), len(deletedPaths), len(matchingPaths), len(listOfFileInfo), len(updatedState))
			fmt.Printf("@New paths: %v\n", newPaths)
			fmt.Printf("@Deleted paths: %v\n", deletedPaths)
			fmt.Println("******************************************************")
			listOfFileInfo = updatedState

			// Post the changes to the other side.
			sendFolderTree(listOfFileInfo)

		} else {
			fmt.Print(".")
			dotCount++
			if dotCount%100 == 0 {
				fmt.Println("")
			}
		}
	}
}

/*
Send the folder tree from this node to another node for comparison
*/
func sendFolderTree(initialTree DirTreeMap) {
	var tree = make(DirTreeMap)
	prefixLength := len(globalSettings.Directory) + 1
	for key, value := range initialTree {
		if key == globalSettings.Directory {
			continue
		}
		fmt.Printf("key is: '%s', length is: %d\n", key, prefixLength)
		key = key[prefixLength:]
		tree[key] = value
	}

	// package up the tree
	jsonStr, err := json.Marshal(tree)
	if err != nil {
		panic(err)
	}
	buffer := bytes.NewBuffer(jsonStr)
	client := &http.Client{}

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)

	for name, server := range GlobalServerMap {
		if name == globalSettings.Name {
			fmt.Printf("skipping %s\n", name)
			continue
		}
		fmt.Printf("sending tree update to: %s\n", name)

		url := "http://" + server.Address + "/tree/"
		fmt.Printf("Posting folder tree to: %s\n", url)

		req, err := http.NewRequest("POST", url, buffer)
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Authorization", "Basic "+authHash)

		resp, err := client.Do(req)
		if err != nil {
			fmt.Printf("we encountered an error!\n%s\n", err)
			//panic(err)
			continue
		}

		fmt.Println("response Status:", resp.Status)
		fmt.Println("response Headers:", resp.Header)
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println("response Body:", string(body))
		resp.Body.Close()
	}
}

func folderTreeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		listOfFileInfo, err := createListOfFolders()
		if err != nil {
			log.Fatal(err)
		}
		json.NewEncoder(w).Encode(listOfFileInfo)
		fmt.Printf("Sending tree of size %d back to client\n", len(listOfFileInfo))
	case "POST":
		//log.Println("Got POST: ", r.Body)
		decoder := json.NewDecoder(r.Body)

		var remoteTreePreTranslate DirTreeMap
		err := decoder.Decode(&remoteTreePreTranslate)
		if err != nil {
			panic("bad json body")
		}

		var remoteTree = make(DirTreeMap)
		for key, value := range remoteTreePreTranslate {
			//fmt.Printf("key is: '%s'\n", key)
			key = fmt.Sprintf("%s/%s", globalSettings.Directory, key)
			remoteTree[key] = value
		}

		fmt.Printf("Received a tree map from: %s\n%s\n", r.RemoteAddr, remoteTree)

		// let's compare to the current one we have.
		_, _, newPaths, deletedPaths, _ := checkForChanges(remoteTree, nil)

		// update the paths to be consistent. Remember, everything is backwards since this is comparing the other side to us instead of two successive states
		// deletion will show up as a new path.

		fmt.Printf("about to check for deletions: %s\n", newPaths)

		// delete folders that were deleted. We delete first, then add to make sure that the old ones will not be in the way
		if len(newPaths) > 0 {
			fmt.Println("Paths were deleted on the other side. Delete them here")
			// Reverse sort the paths so the most specific is first. This allows us to get away without a recursive delete
			sort.Sort(sort.Reverse(sort.StringSlice(newPaths)))
			for _, pathName := range newPaths {
				if pathName == "" || globalSettings.Directory == "" {
					fmt.Sprintf("Trying to delete invalid path: %s\n", pathName)
					panic("Path information is not right. Do not delete")
				} else if pathName == globalSettings.Directory {
					fmt.Printf("We had a request to delete the base path. Skipping: %s\n", pathName)
					continue
				}

				err = os.Remove(pathName)
				if err != nil {
					panic(err)
				}
			}
		}

		// add folders that were created
		fmt.Printf("about to check for new folders: %s\n", deletedPaths)
		if len(deletedPaths) > 0 {
			fmt.Println("Paths were added on the other side. Create them here")
			for _, newPathName := range deletedPaths {
				fmt.Printf("pathname is: %s\n", newPathName)
				//if newPathName == "" {
				//	continue
				//}
				//newFullPath := fmt.Sprintf("%s/%s", globalSettings.Directory, newPathName)
				//fmt.Printf("new full path is: %s\n", newFullPath)
				err = os.Mkdir(newPathName, os.ModeDir+os.ModePerm)
				//todo figure out how to catch a path exists error.
				//if err != nil && err != os.ErrExist {
				//	panic(err)
				//}
			}
		}

	}
}

func sendEvent(event *Event, address string, credentials string) {
	url := "http://" + address + "/event/"
	fmt.Printf("Manager location: %s\n", url)

	// Set the event source (server name)
	event.Source = globalSettings.Name

	jsonStr, _ := json.Marshal(event)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(credentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
	} else {
		defer resp.Body.Close()

		fmt.Println("response Status:", resp.Status)
		fmt.Println("response Headers:", resp.Header)
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println("response Body:", string(body))
	}
}

func eventHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		json.NewEncoder(w).Encode(events)
	case "POST":
		log.Println("Got POST: ", r.Body)
		decoder := json.NewDecoder(r.Body)
		var event Event
		err := decoder.Decode(&event)
		if err != nil {
			panic("bad json body")
		}

		log.Println(event.Name + ", path: " + event.Message)
		log.Printf("Event info: %v\n", event)

		//// Only do stuff for directories at the moment.
		//if event.IsDirectory == false {
		//	fmt.Println("The event is not for a directory, skipping")
		//	return
		//}

		pathName := globalSettings.Directory + "/" + event.Message

		switch event.Name {
		case "notify.Create":
			os.Mkdir(pathName, os.ModeDir+os.ModePerm)
			// todo figure out how to catch a path exists error
			fmt.Printf("create event found. Should be creating: %s\n", pathName)
		case "notify.Remove":
			os.Remove(pathName)
			fmt.Printf("remove attempted on: %s\n", pathName)
		case "notify.Rename":
			os.Remove(pathName)
			fmt.Printf("rename attempted on: %s\n", pathName)
		default:
			fmt.Printf("Unknown event found, doing nothing. Event: %s\n", event.Name)
		}

		events = append([]Event{event}, events...)
	}
}
<commit_msg>Allow self registration of available port to Webcat<commit_after>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"github.com/rjeczalik/notify"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"time"
	"net"
	"strconv"
)

type ReplicatServer struct {
	Name         string
	Address      string
	PublicKey    string
	LastReceived DirTreeMap
	LastSent     DirTreeMap
}

type Node struct {
	Cluster string
	Name    string
	Addr    string
}

var nodes = make(map[string]Node)

type ServerMap map[string]ReplicatServer

var GlobalServerMap = ServerMap{
	"NodeA": ReplicatServer{
		Address: ":8001",
	},
	"NodeB": ReplicatServer{
		Address: ":8002",
	},
}

type Event struct {
	Source       string
	Name         string
	OriginalName string
	Message      string
	Time         time.Time
	IsDirectory  bool
}

var events = make([]Event, 0, 100)

func main() {
	fmt.Println("replicat initializing....")
	rand.Seed(int64(time.Now().Nanosecond()))

	SetupCli()

	// Update the path that traffic is served from to be the filsystem canonical path. This will allow the event folders that come in to match what we have.
	symPath, err := filepath.EvalSymlinks(globalSettings.Directory)
	if err != nil {
		panic(err)
	}

	if symPath != globalSettings.Directory {
		fmt.Printf("Updating serving directory to: %s\n", symPath)
		globalSettings.Directory = symPath
	}

	listOfFileInfo, err := createListOfFolders()
	if err != nil {
		log.Fatal(err)
	}

	// Make the channel buffered to ensure no event is dropped. Notify will drop
	// an event if the receiver is not able to keep up the sending pace.
	fsEventsChannel := make(chan notify.EventInfo, 1)

	// Set up a watch point listening for events within a directory tree rooted at the specified folder
	err = notify.Watch(globalSettings.Directory+"/...", fsEventsChannel, notify.All)
	if err != nil {
		log.Fatal(err)
	}
	defer notify.Stop(fsEventsChannel)

	fmt.Printf("replicat %s online....\n", globalSettings.Name)
	defer fmt.Println("End of line")

	//globalSettings.Address = GlobalServerMap[globalSettings.Name].Address

	totalFiles := 0
	for _, fileInfoList := range listOfFileInfo {
		totalFiles += len(fileInfoList)
	}

	fmt.Printf("Now keeping an eye on %d folders and %d files located under: %s\n", len(listOfFileInfo), totalFiles, globalSettings.Directory)
	go func(c chan notify.EventInfo) {
		for {
			ei := <-c

			// todo add ignore file
			// sendEvent to manager (if it's available)
			fullPath := string(ei.Path())
			directoryLength := len(globalSettings.Directory)

			path := fullPath
			if len(fullPath) >= directoryLength && globalSettings.Directory == fullPath[:directoryLength] {
				// update the path to not have this prefix
				path = fullPath[directoryLength+1:]
			}

			//fmt.Printf("Call to isdir resulted in %v\n", ei.IsDir())

			//// Check if it is a directory based on our directory tree first. Then check the file system
			var isDirectory bool
			//_, exists := listOfFileInfo[fullPath]
			//if exists == true {
			//	isDirectory = true
			//} else {
			//}
			// Get the file info. Useful if we need to restrict our actions to directories
			info, err := os.Stat(fullPath)
			if err == nil {
				isDirectory = info.IsDir()
			} else {
				_, exists := listOfFileInfo[fullPath]
				if exists == true {
					isDirectory = true
				}
			}

			event := Event{Name: ei.Event().String(), Message: path, IsDirectory: isDirectory}
			fmt.Printf("event raw data: %v with path: %v\n", ei.Event(), path)

			// todo copy the source folder on a rename to the event
			sendEvent(&event, globalSettings.ManagerAddress, globalSettings.ManagerCredentials)

			// sendEvent to peers (if any)
			//for name, server := range GlobalServerMap {
			//	if name != globalSettings.Name {
			//		fmt.Printf("sending to peer %s\n", name)
			//		sendEvent(&event, server.Address, globalSettings.ManagerCredentials)
			//	}
			//}
			for k, v := range nodes {
				if k != globalSettings.Name {
					fmt.Printf("sending to peer %s at %s\n", k, v.Addr)
					sendEvent(&event, v.Addr, globalSettings.ManagerCredentials)
				}
			}

			log.Println("Got event:" + ei.Event().String() + ", with Path:" + ei.Path())
		}
	}(fsEventsChannel)

	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))

	//fmt.Printf("listening on: %s\n", globalSettings.Address)

	go func() {
		lsnr, err := net.Listen("tcp4", ":0")
		if err != nil {
			fmt.Println("Error listening:", err)
			os.Exit(1)
		}
		fmt.Println("Listening on:", lsnr.Addr().String())

		go sendConfigToServer(lsnr.Addr())

		//defer resp.Body.Close()
		//if err != nil {
		//	log.Println(err)
		//} else {
		//
		//	fmt.Println("response Status:", resp.Status)
		//	fmt.Println("response Headers:", resp.Header)
		//	body, _ := ioutil.ReadAll(resp.Body)
		//	fmt.Println("response Body:", string(body))
		//}
		//
		err = http.Serve(lsnr, nil)
		if err != nil {
			panic(err)
		}

	}()

	dotCount := 0
	sleepSeconds := time.Duration(25 + rand.Intn(10))
	fmt.Printf("Full Sync time set to: %d seconds\n", sleepSeconds)
	for {
		// Randomize the sync time to decrease oscillation
		time.Sleep(time.Second * sleepSeconds)
		changed, updatedState, newPaths, deletedPaths, matchingPaths := checkForChanges(listOfFileInfo, nil)
		if changed {
			fmt.Println("\nWe have changes, ship it (also updating saved state now that the changes were tracked)")
			fmt.Printf("@Path report: new %d, deleted %d, matching %d, original %d, updated %d\n", len(newPaths), len(deletedPaths), len(matchingPaths), len(listOfFileInfo), len(updatedState))
			fmt.Printf("@New paths: %v\n", newPaths)
			fmt.Printf("@Deleted paths: %v\n", deletedPaths)
			fmt.Println("******************************************************")
			listOfFileInfo = updatedState

			// Post the changes to the other side.
			sendFolderTree(listOfFileInfo)

		} else {
			fmt.Print(".")
			dotCount++
			if dotCount%100 == 0 {
				fmt.Println("")
			}
		}
	}
}

func sendConfigToServer(addr net.Addr) {
	url := "http://" + globalSettings.BootstrapAddress + "/config/"
	fmt.Printf("Manager location: %s\n", url)

	jsonStr, _ := json.Marshal(Node{Name: globalSettings.Name, Addr: "127.0.0.1:" + strconv.Itoa(addr.(*net.TCPAddr).Port)})
	fmt.Printf("jsonStr: %s\n", jsonStr)

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		panic(err)
	}
}

func configHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Printf("configHandler")

	switch r.Method {
	case "POST" :
		decoder := json.NewDecoder(r.Body)
		err := decoder.Decode(&nodes)
		fmt.Printf("nodes: %s", nodes)

		if err != nil {
			fmt.Println(err)
		}
	}
}

/*
Send the folder tree from this node to another node for comparison
*/
func sendFolderTree(initialTree DirTreeMap) {
	var tree = make(DirTreeMap)
	prefixLength := len(globalSettings.Directory) + 1
	for key, value := range initialTree {
		if key == globalSettings.Directory {
			continue
		}
		fmt.Printf("key is: '%s', length is: %d\n", key, prefixLength)
		key = key[prefixLength:]
		tree[key] = value
	}

	// package up the tree
	jsonStr, err := json.Marshal(tree)
	if err != nil {
		panic(err)
	}
	buffer := bytes.NewBuffer(jsonStr)
	client := &http.Client{}

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)

	for name, server := range GlobalServerMap {
		if name == globalSettings.Name {
			fmt.Printf("skipping %s\n", name)
			continue
		}
		fmt.Printf("sending tree update to: %s\n", name)

		url := "http://" + server.Address + "/tree/"
		fmt.Printf("Posting folder tree to: %s\n", url)

		req, err := http.NewRequest("POST", url, buffer)
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Authorization", "Basic "+authHash)

		resp, err := client.Do(req)
		if err != nil {
			fmt.Printf("we encountered an error!\n%s\n", err)
			//panic(err)
			continue
		}

		fmt.Println("response Status:", resp.Status)
		fmt.Println("response Headers:", resp.Header)
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println("response Body:", string(body))
		resp.Body.Close()
	}
}

func folderTreeHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		listOfFileInfo, err := createListOfFolders()
		if err != nil {
			log.Fatal(err)
		}
		json.NewEncoder(w).Encode(listOfFileInfo)
		fmt.Printf("Sending tree of size %d back to client\n", len(listOfFileInfo))
	case "POST":
		//log.Println("Got POST: ", r.Body)
		decoder := json.NewDecoder(r.Body)

		var remoteTreePreTranslate DirTreeMap
		err := decoder.Decode(&remoteTreePreTranslate)
		if err != nil {
			panic("bad json body")
		}

		var remoteTree = make(DirTreeMap)
		for key, value := range remoteTreePreTranslate {
			//fmt.Printf("key is: '%s'\n", key)
			key = fmt.Sprintf("%s/%s", globalSettings.Directory, key)
			remoteTree[key] = value
		}

		fmt.Printf("Received a tree map from: %s\n%s\n", r.RemoteAddr, remoteTree)

		// let's compare to the current one we have.
		_, _, newPaths, deletedPaths, _ := checkForChanges(remoteTree, nil)

		// update the paths to be consistent. Remember, everything is backwards since this is comparing the other side to us instead of two successive states
		// deletion will show up as a new path.

		fmt.Printf("about to check for deletions: %s\n", newPaths)

		// delete folders that were deleted. We delete first, then add to make sure that the old ones will not be in the way
		if len(newPaths) > 0 {
			fmt.Println("Paths were deleted on the other side. Delete them here")
			// Reverse sort the paths so the most specific is first. This allows us to get away without a recursive delete
			sort.Sort(sort.Reverse(sort.StringSlice(newPaths)))
			for _, pathName := range newPaths {
				if pathName == "" || globalSettings.Directory == "" {
					fmt.Sprintf("Trying to delete invalid path: %s\n", pathName)
					panic("Path information is not right. Do not delete")
				} else if pathName == globalSettings.Directory {
					fmt.Printf("We had a request to delete the base path. Skipping: %s\n", pathName)
					continue
				}

				err = os.Remove(pathName)
				if err != nil {
					panic(err)
				}
			}
		}

		// add folders that were created
		fmt.Printf("about to check for new folders: %s\n", deletedPaths)
		if len(deletedPaths) > 0 {
			fmt.Println("Paths were added on the other side. Create them here")
			for _, newPathName := range deletedPaths {
				fmt.Printf("pathname is: %s\n", newPathName)
				//if newPathName == "" {
				//	continue
				//}
				//newFullPath := fmt.Sprintf("%s/%s", globalSettings.Directory, newPathName)
				//fmt.Printf("new full path is: %s\n", newFullPath)
				err = os.Mkdir(newPathName, os.ModeDir+os.ModePerm)
				//todo figure out how to catch a path exists error.
				//if err != nil && err != os.ErrExist {
				//	panic(err)
				//}
			}
		}

	}
}

func sendEvent(event *Event, address string, credentials string) {
	url := "http://" + address + "/event/"
	fmt.Printf("target url: %s\n", url)

	// Set the event source (server name)
	event.Source = globalSettings.Name

	jsonStr, _ := json.Marshal(event)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(credentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err)
	} else {
		defer resp.Body.Close()

		fmt.Println("response Status:", resp.Status)
		fmt.Println("response Headers:", resp.Header)
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println("response Body:", string(body))
	}
}

func eventHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		json.NewEncoder(w).Encode(events)
	case "POST":
		log.Println("Got POST: ", r.Body)
		decoder := json.NewDecoder(r.Body)
		var event Event
		err := decoder.Decode(&event)
		if err != nil {
			panic("bad json body")
		}

		log.Println(event.Name + ", path: " + event.Message)
		log.Printf("Event info: %v\n", event)

		//// Only do stuff for directories at the moment.
		//if event.IsDirectory == false {
		//	fmt.Println("The event is not for a directory, skipping")
		//	return
		//}

		pathName := globalSettings.Directory + "/" + event.Message

		switch event.Name {
		case "notify.Create":
			os.Mkdir(pathName, os.ModeDir+os.ModePerm)
			// todo figure out how to catch a path exists error
			fmt.Printf("create event found. Should be creating: %s\n", pathName)
		case "notify.Remove":
			os.Remove(pathName)
			fmt.Printf("remove attempted on: %s\n", pathName)
		case "notify.Rename":
			os.Remove(pathName)
			fmt.Printf("rename attempted on: %s\n", pathName)
		default:
			fmt.Printf("Unknown event found, doing nothing. Event: %s\n", event.Name)
		}

		events = append([]Event{event}, events...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/antonlindstrom/mesos_stats"
	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	configFile  = flag.String("config.file", "", "Path to config file.")
	metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	cpuLimitVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "cpus_limit",
			Help:      "CPU share limit.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	cpuSysVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "cpus_system_time_secs",
			Help:      "CPU system time for task.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	cpuUsrVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "cpus_user_time_secs",
			Help:      "CPU system time for task.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	memLimitVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "mem_limit_bytes",
			Help:      "Task memory limit in bytes.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	memRssVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "mem_rss_bytes",
			Help:      "Task memory RSS usage in bytes.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
)

type Config struct {
	Port      string      `json:"port"`
	Interval  string      `json:"scrape_interval"`
	Endpoints *[]Endpoint `json:"endpoints"`
}

type Endpoint struct {
	Name       string        `json:"name"`
	Target     string        `json:"target"`
	Interval   string        `json:"scrape_interval,omitempty"`
	TimeoutStr string        `json:"timeout,omitempty"`
	Timeout    time.Duration `json:"-"`
}

func newConfig() (*Config, error) {
	var config Config

	file, err := ioutil.ReadFile(*configFile)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(file, &config)
	return &config, err
}

func request(e Endpoint) error {
	client := http.Client{Timeout: 3 * time.Second}

	resp, err := client.Get(e.Target + "/monitor/statistics.json")
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var stats []mesos_stats.Monitor

	err = json.Unmarshal(body, &stats)
	if err != nil {
		return err
	}

	for _, t := range stats {
		cpuLimitVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(t.Statistics.CpusLimit)
		cpuSysVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(t.Statistics.CpusSystemTimeSecs)
		cpuUsrVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(t.Statistics.CpusUserTimeSecs)
		memLimitVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(float64(t.Statistics.MemLimitBytes))
		memRssVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(float64(t.Statistics.MemRssBytes))
	}

	return err
}

func requestRunner(e Endpoint) {
	for {
		err := request(e)
		if err != nil {
			glog.Warningln(err)
		}

		t, err := time.ParseDuration(e.Interval)
		if err != nil {
			glog.Warningln(err)
			t = 30 * time.Second
		}

		time.Sleep(t)
	}
}

func processRequests(c *Config) {
	for _, endpoint := range *c.Endpoints {
		glog.Infof("Found %s\n", endpoint.Name)

		if len(endpoint.Interval) == 0 {
			endpoint.Interval = c.Interval
		}

		if len(endpoint.TimeoutStr) != 0 {
			t, err := time.ParseDuration(endpoint.TimeoutStr)
			if err != nil {
				glog.Fatalf("Invalid specification of time.Duration: %s\n", err)
			}
			endpoint.Timeout = t
		} else {
			endpoint.Timeout = 3 * time.Second
		}

		go requestRunner(endpoint)
	}
}

func init() {
	prometheus.MustRegister(cpuLimitVal)
	prometheus.MustRegister(cpuSysVal)
	prometheus.MustRegister(cpuUsrVal)
	prometheus.MustRegister(memLimitVal)
	prometheus.MustRegister(memRssVal)
}

func main() {
	flag.Parse()

	if len(*configFile) < 2 {
		fmt.Printf("Error: Flag -config.file is required.\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	config, err := newConfig()
	if err != nil {
		fmt.Printf("Configuration error: %s\n", err)
		os.Exit(2)
	}

	go func() {
		http.Handle(*metricsPath, prometheus.Handler())
		http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, "OK")
		})
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
		})

		processRequests(config)

		glog.Infof("Starting up on %s\n", config.Port)
		err := http.ListenAndServe(":"+config.Port, nil)
		if err != nil {
			glog.Fatal(err)
		}
	}()

	select {}
}
<commit_msg>Separate var blocks for flags and metrics.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/antonlindstrom/mesos_stats"
	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	configFile  = flag.String("config.file", "", "Path to config file.")
	metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
)

var (
	cpuLimitVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "cpus_limit",
			Help:      "CPU share limit.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	cpuSysVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "cpus_system_time_secs",
			Help:      "CPU system time for task.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	cpuUsrVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "cpus_user_time_secs",
			Help:      "CPU system time for task.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	memLimitVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "mem_limit_bytes",
			Help:      "Task memory limit in bytes.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
	memRssVal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: "mesos_task",
			Name:      "mem_rss_bytes",
			Help:      "Task memory RSS usage in bytes.",
		},
		[]string{"service", "mesos_slave", "framework_id"},
	)
)

type Config struct {
	Port      string      `json:"port"`
	Interval  string      `json:"scrape_interval"`
	Endpoints *[]Endpoint `json:"endpoints"`
}

type Endpoint struct {
	Name       string        `json:"name"`
	Target     string        `json:"target"`
	Interval   string        `json:"scrape_interval,omitempty"`
	TimeoutStr string        `json:"timeout,omitempty"`
	Timeout    time.Duration `json:"-"`
}

func newConfig() (*Config, error) {
	var config Config

	file, err := ioutil.ReadFile(*configFile)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(file, &config)
	return &config, err
}

func request(e Endpoint) error {
	client := http.Client{Timeout: 3 * time.Second}

	resp, err := client.Get(e.Target + "/monitor/statistics.json")
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	var stats []mesos_stats.Monitor

	err = json.Unmarshal(body, &stats)
	if err != nil {
		return err
	}

	for _, t := range stats {
		cpuLimitVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(t.Statistics.CpusLimit)
		cpuSysVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(t.Statistics.CpusSystemTimeSecs)
		cpuUsrVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(t.Statistics.CpusUserTimeSecs)
		memLimitVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(float64(t.Statistics.MemLimitBytes))
		memRssVal.WithLabelValues(t.Source, e.Name, t.FrameworkId).Set(float64(t.Statistics.MemRssBytes))
	}

	return err
}

func requestRunner(e Endpoint) {
	for {
		err := request(e)
		if err != nil {
			glog.Warningln(err)
		}

		t, err := time.ParseDuration(e.Interval)
		if err != nil {
			glog.Warningln(err)
			t = 30 * time.Second
		}

		time.Sleep(t)
	}
}

func processRequests(c *Config) {
	for _, endpoint := range *c.Endpoints {
		glog.Infof("Found %s\n", endpoint.Name)

		if len(endpoint.Interval) == 0 {
			endpoint.Interval = c.Interval
		}

		if len(endpoint.TimeoutStr) != 0 {
			t, err := time.ParseDuration(endpoint.TimeoutStr)
			if err != nil {
				glog.Fatalf("Invalid specification of time.Duration: %s\n", err)
			}
			endpoint.Timeout = t
		} else {
			endpoint.Timeout = 3 * time.Second
		}

		go requestRunner(endpoint)
	}
}

func init() {
	prometheus.MustRegister(cpuLimitVal)
	prometheus.MustRegister(cpuSysVal)
	prometheus.MustRegister(cpuUsrVal)
	prometheus.MustRegister(memLimitVal)
	prometheus.MustRegister(memRssVal)
}

func main() {
	flag.Parse()

	if len(*configFile) < 2 {
		fmt.Printf("Error: Flag -config.file is required.\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	config, err := newConfig()
	if err != nil {
		fmt.Printf("Configuration error: %s\n", err)
		os.Exit(2)
	}

	go func() {
		http.Handle(*metricsPath, prometheus.Handler())
		http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, "OK")
		})
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
		})

		processRequests(config)

		glog.Infof("Starting up on %s\n", config.Port)
		err := http.ListenAndServe(":"+config.Port, nil)
		if err != nil {
			glog.Fatal(err)
		}
	}()

	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"github.com/bogem/id3v2"
	"github.com/mmcdole/gofeed"
	"gopkg.in/cheggaaa/pb.v1"
)

func main() {
	// Parse CLI args
	downloadDirectory := flag.String("dir", "musicforprogramming", "Directory into which the tracks will be downloaded")
	flag.Parse()
	if *downloadDirectory == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	createDir(*downloadDirectory)
	getCover(*downloadDirectory)
	tracks := getTracks()
	for trackIndex, URL := range tracks.URL {
		getTrack(*downloadDirectory, tracks.Title[trackIndex], URL)
	}
}

type tracks struct {
	Title []string
	URL   []string
}

// Borrowed from https://stackoverflow.com/questions/34816489/reverse-slice-of-strings
func reverseStringSlice(ss []string) []string {
	last := len(ss) - 1
	for i := 0; i < len(ss)/2; i++ {
		ss[i], ss[last-i] = ss[last-i], ss[i]
	}
	return ss
}

// getTracks gets titles and URLs to tracks from RSS feed
func getTracks() (tracks tracks) {
	log.Println("Downloading and parsing tracks index...")
	feedParser := gofeed.NewParser()
	feed, err := feedParser.ParseURL("https://musicforprogramming.net/rss.php")
	if err != nil {
		panic(err)
	}
	// Read and append track title and URL to tracks
	for _, item := range feed.Items {
		tracks.Title = append(tracks.Title, item.Title)
		for _, enclosure := range item.Enclosures {
			tracks.URL = append(tracks.URL, enclosure.URL)
		}
	}
	// Invert both slices
	// Having index 0 be the first track eases testing and makes more sense to the user
	tracks.Title = reverseStringSlice(tracks.Title)
	tracks.URL = reverseStringSlice(tracks.URL)
	return tracks
}

// getCover downloads folder.jpg (album cover image) if it doesn't exist
func getCover(destDirectory string) {
	coverPath := filepath.Join(destDirectory, "folder.jpg")
	if _, err := os.Stat(coverPath); os.IsNotExist(err) {
		log.Printf("Album cover does not exist, downloading...")

		file, err := os.Create(coverPath)
		if err != nil {
			panic(err)
		}
		defer file.Close()

		resp, err := http.Get("https://musicforprogramming.net/img/folder.jpg")
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		// See https://stackoverflow.com/questions/11692860/how-can-i-efficiently-download-a-large-file-using-go
		// Stream HTTP response into file
		n, err := io.Copy(file, resp.Body)
		if err != nil {
			panic(err)
		}
		// We don't use n
		_ = n
		log.Println("Album cover downloaded!")
	}
}

// createDir checks whether the directory exists and creates it if it doesn't
func createDir(destDirectory string) {
	if _, err := os.Stat(destDirectory); os.IsNotExist(err) {
		log.Printf("Directory " + destDirectory + " does not exist, creating...")
		err := os.MkdirAll(destDirectory, 0775)
		if err != nil {
			panic(err)
		}
		log.Printf("Directory " + destDirectory + " created!")
	}
}

// Add album metadata to the track supplied.
// This is needed because the downloaded files don't contain this data.
func setAlbum(file string) {
	mp3File, err := id3v2.Open(file, id3v2.Options{Parse: true})
	if err != nil {
		panic(err)
	}
	defer mp3File.Close()
	mp3File.SetAlbum("Music For Programming")
}

// Download supplied track to supplied directory if it doesn't exist on disk
func getTrack(destDirectory string, title string, URL string) {
	trackPath := filepath.Join(destDirectory, title+".mp3")
	if _, err := os.Stat(trackPath); os.IsNotExist(err) {
		log.Printf("Track '" + title + "' does not exist on disk, downloading... ")
		file, err := os.Create(trackPath)
		if err != nil {
			panic(err)
		}
		defer file.Close()

		resp, err := http.Get(URL)
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		// Prepare to show a progress bar
		// Start by getting the size of the file we'll be downloading
		size, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
		// Set up our progress bar
		// See https://github.com/cheggaaa/pb/#progress-bar-for-io-operations
		bar := pb.New(size).SetUnits(pb.U_BYTES)
		bar.Start()
		writer := io.MultiWriter(file, bar)
		// See https://stackoverflow.com/questions/11692860/how-can-i-efficiently-download-a-large-file-using-go
		// Stream HTTP response into file
		n, err := io.Copy(writer, resp.Body)
		if err != nil {
			panic(err)
		}
		bar.Finish()
		// We don't use n
		_ = n
		setAlbum(trackPath)
		log.Printf("Track '" + title + "' downloaded!")
	}
}
<commit_msg>Embed album art; redownload files only when needed<commit_after>package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"github.com/bogem/id3v2"
	"github.com/mmcdole/gofeed"
	"gopkg.in/cheggaaa/pb.v1"
)

func main() {
	// Parse CLI args
	downloadDirectory := flag.String("dir", "musicforprogramming", "Directory into which the tracks will be downloaded")
	flag.Parse()
	if *downloadDirectory == "" {
		flag.PrintDefaults()
		os.Exit(1)
	}

	createDir(*downloadDirectory)
	getCover(*downloadDirectory)
	tracks := getTracks()
	for trackIndex, URL := range tracks.URL {
		getTrack(*downloadDirectory, tracks.Title[trackIndex], URL)
	}
}

type tracks struct {
	Title []string
	URL   []string
}

// Borrowed from https://stackoverflow.com/questions/34816489/reverse-slice-of-strings
func reverseStringSlice(ss []string) []string {
	last := len(ss) - 1
	for i := 0; i < len(ss)/2; i++ {
		ss[i], ss[last-i] = ss[last-i], ss[i]
	}
	return ss
}

// getTracks gets titles and URLs to tracks from RSS feed
func getTracks() (tracks tracks) {
	log.Println("Downloading and parsing tracks index...")
	feedParser := gofeed.NewParser()
	feed, err := feedParser.ParseURL("https://musicforprogramming.net/rss.php")
	if err != nil {
		panic(err)
	}
	// Read and append track title and URL to tracks
	for _, item := range feed.Items {
		tracks.Title = append(tracks.Title, item.Title)
		for _, enclosure := range item.Enclosures {
			tracks.URL = append(tracks.URL, enclosure.URL)
		}
	}
	// Invert both slices
	// Having index 0 be the first track eases testing and makes more sense to the user
	tracks.Title = reverseStringSlice(tracks.Title)
	tracks.URL = reverseStringSlice(tracks.URL)
	return tracks
}

// getCover downloads folder.jpg (album cover image) if it doesn't exist
func getCover(destDirectory string) {
	coverPath := filepath.Join(destDirectory, "folder.jpg")
	if _, err := os.Stat(coverPath); os.IsNotExist(err) {
		log.Printf("Album cover does not exist, downloading...")

		file, err := os.Create(coverPath)
		if err != nil {
			panic(err)
		}
		defer file.Close()

		resp, err := http.Get("https://musicforprogramming.net/img/folder.jpg")
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		// See https://stackoverflow.com/questions/11692860/how-can-i-efficiently-download-a-large-file-using-go
		// Stream HTTP response into file
		n, err := io.Copy(file, resp.Body)
		if err != nil {
			panic(err)
		}
		// We don't use n
		_ = n
		log.Println("Album cover downloaded!")
	}
}

// createDir checks whether the directory exists and creates it if it doesn't
func createDir(destDirectory string) {
	if _, err := os.Stat(destDirectory); os.IsNotExist(err) {
		log.Printf("Directory " + destDirectory + " does not exist, creating...")
		err := os.MkdirAll(destDirectory, 0775)
		if err != nil {
			panic(err)
		}
		log.Printf("Directory " + destDirectory + " created!")
	}
}

// setMetadata adds album metadata and artwork to the track supplied.
func setMetadata(file string) {
	mp3File, err := id3v2.Open(file, id3v2.Options{Parse: true})
	if err != nil {
		panic(err)
	}
	defer mp3File.Close()
	mp3File.SetAlbum("Music For Programming")

	dir := filepath.Dir(file)
	// Ensure cover is present in DL dir
	getCover(dir)
	coverPath := filepath.Join(dir, "folder.jpg")
	cover, err := ioutil.ReadFile(coverPath)
	if err != nil {
		log.Fatal("Error while reading album artwork file", err)
	}
	pic := id3v2.PictureFrame{
		Encoding:    id3v2.EncodingUTF8,
		MimeType:    "image/jpeg",
		PictureType: id3v2.PTFrontCover,
		Description: "Front cover",
		Picture:     cover,
	}
	mp3File.AddAttachedPicture(pic)
}

// isTrackComplete checks whether a track is present and fully downloaded.
func isTrackComplete(dir string, title string, URL string) (bool, error) {
	trackPath := filepath.Join(dir, title+".mp3")
	stat, err := os.Stat(trackPath)
	// Check whether track exists at all.
	if os.IsNotExist(err) {
		return false, nil
	}
	// Unknown stat error, return it.
	if err != nil {
		return false, err
	}
	// Check if length matches.
	// If not it's a sign of an incomplete download/data corruption.
	req, err := http.Head(URL)
	if err != nil {
		return false, err
	}
	if stat.Size() != req.ContentLength {
		log.Printf("Track %v is of length %d on disk, %d expected. Redownloading...\n", title, stat.Size(), req.ContentLength)
		return false, nil
	}
	return true, nil
}

// Download supplied track to supplied directory if it doesn't exist on disk
func getTrack(destDirectory string, title string, URL string) {
	trackPath := filepath.Join(destDirectory, title+".mp3")
	complete, err := isTrackComplete(destDirectory, title, URL)
	if err != nil {
		log.Printf("Encountered error while trying to check whether track %v has already been downloaded: %v\n", title, err.Error())
	}
	if !complete {
		log.Printf("Track %v does not exist on disk, downloading...\n", title)
		file, err := os.Create(trackPath)
		if err != nil {
			panic(err)
		}
		defer file.Close()

		resp, err := http.Get(URL)
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		// Prepare to show a progress bar
		// Start by getting the size of the file we'll be downloading
		size, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
		// Set up our progress bar
		// See https://github.com/cheggaaa/pb/#progress-bar-for-io-operations
		bar := pb.New(size).SetUnits(pb.U_BYTES)
		bar.Start()
		writer := io.MultiWriter(file, bar)
		// See https://stackoverflow.com/questions/11692860/how-can-i-efficiently-download-a-large-file-using-go
		// Stream HTTP response into file
		n, err := io.Copy(writer, resp.Body)
		if err != nil {
			panic(err)
		}
		bar.Finish()
		// We don't use n
		_ = n

		log.Printf("Track %v downloaded!\n", title)
		log.Printf("Setting metadata for track %v\n", title)
		setMetadata(trackPath)
		log.Println("Metadata set!")
	}
}
<|endoftext|>