<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/openebs/openebs/api"
	apiserver "github.com/openebs/openebs/api/server"
	"github.com/openebs/openebs/api/server/middleware"
	"github.com/openebs/openebs/api/server/router"
	systemrouter "github.com/openebs/openebs/api/server/router/system"
	vsmrouter "github.com/openebs/openebs/api/server/router/vsm"
	cliflags "github.com/openebs/openebs/cli/flags"
	"github.com/openebs/openebs/daemon"
	"github.com/openebs/openebs/opts"
	"github.com/openebs/openebs/pkg/jsonlog"
	"github.com/openebs/openebs/pkg/listeners"
	flag "github.com/openebs/openebs/pkg/mflag"
	"github.com/openebs/openebs/pkg/pidfile"
	"github.com/openebs/openebs/pkg/signal"
	"github.com/openebs/openebs/utils"
	"github.com/openebs/openebs/version"
)

const (
	daemonConfigFileFlag = "-config-file"
)

// DaemonCli represents the daemon CLI.
type DaemonCli struct {
	*daemon.Config
	commonFlags *cliflags.CommonFlags
	configFile  *string

	api *apiserver.Server
	d   *daemon.Daemon
}

func presentInHelp(usage string) string { return usage }
func absentFromHelp(string) string      { return "" }

// NewDaemonCli returns a pre-configured daemon CLI
func NewDaemonCli() *DaemonCli {
	// TODO(tiborvass): remove InstallFlags?
	daemonConfig := new(daemon.Config)
	// We want openebsd to run on TLS
	daemonConfig.TLS = true

	daemonConfig.LogConfig.Config = make(map[string]string)

	daemonConfig.InstallFlags(flag.CommandLine, presentInHelp)
	configFile := flag.CommandLine.String([]string{daemonConfigFileFlag}, defaultDaemonConfigFile, "Daemon configuration file")
	flag.CommandLine.Require(flag.Exact, 0)

	return &DaemonCli{
		Config:      daemonConfig,
		commonFlags: cliflags.InitCommonFlags(),
		configFile:  configFile,
	}
}

func (cli *DaemonCli) start() (err error) {
	stopc := make(chan bool)
	defer close(stopc)

	flags := flag.CommandLine
	cli.commonFlags.PostParse()

	cliConfig, err := loadDaemonCliConfig(cli.Config, flags, cli.commonFlags, *cli.configFile)
	if err != nil {
		return err
	}
	cli.Config = cliConfig

	if cli.Config.Debug {
		utils.EnableDebug()
	}

	logrus.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: jsonlog.RFC3339NanoFixed,
		DisableColors:   cli.Config.RawLogs,
	})

	if err := setDefaultUmask(); err != nil {
		return fmt.Errorf("Failed to set umask: %v", err)
	}

	//TODO : Kiran
	//if len(cli.LogConfig.Config) > 0 {
	//	if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
	//		return fmt.Errorf("Failed to set log opts: %v", err)
	//	}
	//}

	if cli.Pidfile != "" {
		pf, err := pidfile.New(cli.Pidfile)
		if err != nil {
			return fmt.Errorf("Error starting daemon: %v", err)
		}
		defer func() {
			if err := pf.Remove(); err != nil {
				logrus.Error(err)
			}
		}()
	}

	serverConfig := &apiserver.Config{
		Logging:     true,
		SocketGroup: cli.Config.SocketGroup,
		Version:     version.Version,
	}

	if len(cli.Config.Hosts) == 0 {
		cli.Config.Hosts = make([]string, 1)
	}

	api := apiserver.New(serverConfig)
	cli.api = api

	for i := 0; i < len(cli.Config.Hosts); i++ {
		var err error
		if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
			return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
		}

		protoAddr := cli.Config.Hosts[i]
		protoAddrParts := strings.SplitN(protoAddr, "://", 2)
		if len(protoAddrParts) != 2 {
			return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
		}

		proto := protoAddrParts[0]
		addr := protoAddrParts[1]

		// It's a bad idea to bind to TCP without tlsverify.
		if proto == "tcp" {
			logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
		}
		ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
		if err != nil {
			return err
		}
		ls = wrapListeners(proto, ls)
		// If we're binding to a TCP port, make sure that a container doesn't try to use it.
		if proto == "tcp" {
			if err := allocateDaemonPort(addr); err != nil {
				return err
			}
		}
		logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1])
		api.Accept(protoAddrParts[1], ls...)
	}

	signal.Trap(func() {
		cli.stop()
		<-stopc // wait for daemonCli.start() to return
	})

	d, err := daemon.NewDaemon(cli.Config)
	if err != nil {
		return fmt.Errorf("Error starting daemon: %v", err)
	}

	//name, _ := os.Hostname()

	logrus.Info("Daemon has completed initialization")

	logrus.WithFields(logrus.Fields{
		"version": version.Version,
		"commit":  version.GitCommit,
	}).Info("OpenEBS daemon")

	cli.initMiddlewares(api, serverConfig)
	initRouter(api, d)

	cli.d = d
	cli.setupConfigReloadTrap()

	// The serve API routine never exits unless an error occurs
	// We need to start it as a goroutine and wait on it so
	// daemon doesn't exit
	serveAPIWait := make(chan error)
	go api.Wait(serveAPIWait)

	// after the daemon is done setting up we can notify systemd api
	notifySystem()

	// Daemon is fully initialized and handling API traffic
	// Wait for serve API to complete
	errAPI := <-serveAPIWait
	shutdownDaemon(d, 15)
	if errAPI != nil {
		return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
	}

	return nil
}

func (cli *DaemonCli) reloadConfig() {
	reload := func(config *daemon.Config) {
		if err := cli.d.Reload(config); err != nil {
			logrus.Errorf("Error reconfiguring the daemon: %v", err)
			return
		}
		if config.IsValueSet("debug") {
			debugEnabled := utils.IsDebugEnabled()
			switch {
			case debugEnabled && !config.Debug: // disable debug
				utils.DisableDebug()
				cli.api.DisableProfiler()
			case config.Debug && !debugEnabled: // enable debug
				utils.EnableDebug()
				cli.api.EnableProfiler()
			}

		}
	}

	if err := daemon.ReloadConfiguration(*cli.configFile, flag.CommandLine, reload); err != nil {
		logrus.Error(err)
	}
}

func (cli *DaemonCli) stop() {
	cli.api.Close()
}

// shutdownDaemon just wraps daemon.Shutdown() to handle a timeout in case
// d.Shutdown() is waiting too long to kill container or worst it's
// blocked there
func shutdownDaemon(d *daemon.Daemon, timeout time.Duration) {
	ch := make(chan struct{})
	go func() {
		d.Shutdown()
		close(ch)
	}()
	select {
	case <-ch:
		logrus.Debug("Clean shutdown succeeded")
	case <-time.After(timeout * time.Second):
		logrus.Error("Force shutdown daemon")
	}
}

func loadDaemonCliConfig(config *daemon.Config, flags *flag.FlagSet, commonConfig *cliflags.CommonFlags, configFile string) (*daemon.Config, error) {
	config.Debug = commonConfig.Debug
	config.Hosts = commonConfig.Hosts
	config.LogLevel = commonConfig.LogLevel

	if configFile != "" {
		c, err := daemon.MergeDaemonConfigurations(config, flags, configFile)
		if err != nil {
			if flags.IsSet(daemonConfigFileFlag) || !os.IsNotExist(err) {
				return nil, fmt.Errorf("unable to configure the OpenEBS daemon with file %s: %v\n", configFile, err)
			}
		}
		// the merged configuration can be nil if the config file didn't exist.
		// leave the current configuration as it is if when that happens.
		if c != nil {
			config = c
		}
	}

	if err := daemon.ValidateConfiguration(config); err != nil {
		return nil, err
	}

	// ensure that the log level is the one set after merging configurations
	cliflags.SetDaemonLogLevel(config.LogLevel)

	return config, nil
}

func initRouter(s *apiserver.Server, d *daemon.Daemon) {

	routers := []router.Router{
		systemrouter.NewRouter(d),
		vsmrouter.NewRouter(d),
	}
	s.InitRouter(utils.IsDebugEnabled(), routers...)
}

func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, cfg *apiserver.Config) {
	v := cfg.Version

	vm := middleware.NewVersionMiddleware(v, api.DefaultVersion, api.MinVersion)
	s.UseMiddleware(vm)

	u := middleware.NewUserAgentMiddleware(v)
	s.UseMiddleware(u)

}
<commit_msg>Removed options required to setup TCP in code or config, good to set at runtime, e.g. openebsd -H tcp://0.0.0.0:8000<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/openebs/openebs/api"
	apiserver "github.com/openebs/openebs/api/server"
	"github.com/openebs/openebs/api/server/middleware"
	"github.com/openebs/openebs/api/server/router"
	systemrouter "github.com/openebs/openebs/api/server/router/system"
	vsmrouter "github.com/openebs/openebs/api/server/router/vsm"
	cliflags "github.com/openebs/openebs/cli/flags"
	"github.com/openebs/openebs/daemon"
	"github.com/openebs/openebs/opts"
	"github.com/openebs/openebs/pkg/jsonlog"
	"github.com/openebs/openebs/pkg/listeners"
	flag "github.com/openebs/openebs/pkg/mflag"
	"github.com/openebs/openebs/pkg/pidfile"
	"github.com/openebs/openebs/pkg/signal"
	"github.com/openebs/openebs/utils"
	"github.com/openebs/openebs/version"
)

const (
	daemonConfigFileFlag = "-config-file"
)

// DaemonCli represents the daemon CLI.
type DaemonCli struct {
	*daemon.Config
	commonFlags *cliflags.CommonFlags
	configFile  *string

	api *apiserver.Server
	d   *daemon.Daemon
}

func presentInHelp(usage string) string { return usage }
func absentFromHelp(string) string      { return "" }

// NewDaemonCli returns a pre-configured daemon CLI
func NewDaemonCli() *DaemonCli {
	// TODO(tiborvass): remove InstallFlags?
	daemonConfig := new(daemon.Config)

	daemonConfig.LogConfig.Config = make(map[string]string)

	daemonConfig.InstallFlags(flag.CommandLine, presentInHelp)
	configFile := flag.CommandLine.String([]string{daemonConfigFileFlag}, defaultDaemonConfigFile, "Daemon configuration file")
	flag.CommandLine.Require(flag.Exact, 0)

	return &DaemonCli{
		Config:      daemonConfig,
		commonFlags: cliflags.InitCommonFlags(),
		configFile:  configFile,
	}
}

func (cli *DaemonCli) start() (err error) {
	stopc := make(chan bool)
	defer close(stopc)

	flags := flag.CommandLine
	cli.commonFlags.PostParse()

	cliConfig, err := loadDaemonCliConfig(cli.Config, flags, cli.commonFlags, *cli.configFile)
	if err != nil {
		return err
	}
	cli.Config = cliConfig

	if cli.Config.Debug {
		utils.EnableDebug()
	}

	logrus.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: jsonlog.RFC3339NanoFixed,
		DisableColors:   cli.Config.RawLogs,
	})

	if err := setDefaultUmask(); err != nil {
		return fmt.Errorf("Failed to set umask: %v", err)
	}

	//TODO : Kiran
	//if len(cli.LogConfig.Config) > 0 {
	//	if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
	//		return fmt.Errorf("Failed to set log opts: %v", err)
	//	}
	//}

	if cli.Pidfile != "" {
		pf, err := pidfile.New(cli.Pidfile)
		if err != nil {
			return fmt.Errorf("Error starting daemon: %v", err)
		}
		defer func() {
			if err := pf.Remove(); err != nil {
				logrus.Error(err)
			}
		}()
	}

	serverConfig := &apiserver.Config{
		Logging:     true,
		SocketGroup: cli.Config.SocketGroup,
		Version:     version.Version,
	}

	if len(cli.Config.Hosts) == 0 {
		cli.Config.Hosts = make([]string, 1)
	}

	api := apiserver.New(serverConfig)
	cli.api = api

	for i := 0; i < len(cli.Config.Hosts); i++ {
		var err error
		if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
			return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
		}

		protoAddr := cli.Config.Hosts[i]
		protoAddrParts := strings.SplitN(protoAddr, "://", 2)
		if len(protoAddrParts) != 2 {
			return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
		}

		proto := protoAddrParts[0]
		addr := protoAddrParts[1]

		// It's a bad idea to bind to TCP without tlsverify.
		if proto == "tcp" {
			logrus.Warn("[!] DON'T BIND ON ANY IP ADDRESS WITHOUT setting -tlsverify IF YOU DON'T KNOW WHAT YOU'RE DOING [!]")
		}
		ls, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
		if err != nil {
			return err
		}
		ls = wrapListeners(proto, ls)
		// If we're binding to a TCP port, make sure that a container doesn't try to use it.
		if proto == "tcp" {
			if err := allocateDaemonPort(addr); err != nil {
				return err
			}
		}
		logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1])
		api.Accept(protoAddrParts[1], ls...)
	}

	signal.Trap(func() {
		cli.stop()
		<-stopc // wait for daemonCli.start() to return
	})

	d, err := daemon.NewDaemon(cli.Config)
	if err != nil {
		return fmt.Errorf("Error starting daemon: %v", err)
	}

	//name, _ := os.Hostname()

	logrus.Info("Daemon has completed initialization")

	logrus.WithFields(logrus.Fields{
		"version": version.Version,
		"commit":  version.GitCommit,
	}).Info("OpenEBS daemon")

	cli.initMiddlewares(api, serverConfig)
	initRouter(api, d)

	cli.d = d
	cli.setupConfigReloadTrap()

	// The serve API routine never exits unless an error occurs
	// We need to start it as a goroutine and wait on it so
	// daemon doesn't exit
	serveAPIWait := make(chan error)
	go api.Wait(serveAPIWait)

	// after the daemon is done setting up we can notify systemd api
	notifySystem()

	// Daemon is fully initialized and handling API traffic
	// Wait for serve API to complete
	errAPI := <-serveAPIWait
	shutdownDaemon(d, 15)
	if errAPI != nil {
		return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
	}

	return nil
}

func (cli *DaemonCli) reloadConfig() {
	reload := func(config *daemon.Config) {
		if err := cli.d.Reload(config); err != nil {
			logrus.Errorf("Error reconfiguring the daemon: %v", err)
			return
		}
		if config.IsValueSet("debug") {
			debugEnabled := utils.IsDebugEnabled()
			switch {
			case debugEnabled && !config.Debug: // disable debug
				utils.DisableDebug()
				cli.api.DisableProfiler()
			case config.Debug && !debugEnabled: // enable debug
				utils.EnableDebug()
				cli.api.EnableProfiler()
			}

		}
	}

	if err := daemon.ReloadConfiguration(*cli.configFile, flag.CommandLine, reload); err != nil {
		logrus.Error(err)
	}
}

func (cli *DaemonCli) stop() {
	cli.api.Close()
}

// shutdownDaemon just wraps daemon.Shutdown() to handle a timeout in case
// d.Shutdown() is waiting too long to kill container or worst it's
// blocked there
func shutdownDaemon(d *daemon.Daemon, timeout time.Duration) {
	ch := make(chan struct{})
	go func() {
		d.Shutdown()
		close(ch)
	}()
	select {
	case <-ch:
		logrus.Debug("Clean shutdown succeeded")
	case <-time.After(timeout * time.Second):
		logrus.Error("Force shutdown daemon")
	}
}

func loadDaemonCliConfig(config *daemon.Config, flags *flag.FlagSet, commonConfig *cliflags.CommonFlags, configFile string) (*daemon.Config, error) {
	config.Debug = commonConfig.Debug
	config.Hosts = commonConfig.Hosts
	config.LogLevel = commonConfig.LogLevel

	if configFile != "" {
		c, err := daemon.MergeDaemonConfigurations(config, flags, configFile)
		if err != nil {
			if flags.IsSet(daemonConfigFileFlag) || !os.IsNotExist(err) {
				return nil, fmt.Errorf("unable to configure the OpenEBS daemon with file %s: %v\n", configFile, err)
			}
		}
		// the merged configuration can be nil if the config file didn't exist.
		// leave the current configuration as it is if when that happens.
		if c != nil {
			config = c
		}
	}

	if err := daemon.ValidateConfiguration(config); err != nil {
		return nil, err
	}

	// ensure that the log level is the one set after merging configurations
	cliflags.SetDaemonLogLevel(config.LogLevel)

	return config, nil
}

func initRouter(s *apiserver.Server, d *daemon.Daemon) {

	routers := []router.Router{
		systemrouter.NewRouter(d),
		vsmrouter.NewRouter(d),
	}
	s.InitRouter(utils.IsDebugEnabled(), routers...)
}

func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, cfg *apiserver.Config) {
	v := cfg.Version

	vm := middleware.NewVersionMiddleware(v, api.DefaultVersion, api.MinVersion)
	s.UseMiddleware(vm)

	u := middleware.NewUserAgentMiddleware(v)
	s.UseMiddleware(u)

}
<|endoftext|>


<commit_before>/*
 * Minio Client (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"time"

	"github.com/minio/cli"
	"github.com/minio/minio/pkg/probe"
)

var (
	shareListFlags = []cli.Flag{}
)

// Share documents via URL.
var shareList = cli.Command{
	Name:   "list",
	Usage:  "List previously shared objects and folders.",
	Action: mainShareList,
	Before: setGlobalsFromContext,
	Flags:  append(shareListFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
   {{.HelpName}} COMMAND - {{.Usage}}

COMMAND:
   upload:   list previously shared access to uploads.
   download: list previously shared access to downloads.

USAGE:
   {{.HelpName}}

EXAMPLES:
   1. List previously shared downloads, that haven't expired yet.
       $ {{.HelpName}} download
   2. List previously shared uploads, that haven't expired yet.
       $ {{.HelpName}} upload

`,
}

// validate command-line args.
func checkShareListSyntax(ctx *cli.Context) {
	args := ctx.Args()
	if !args.Present() || (args.First() != "upload" && args.First() != "download") {
		cli.ShowCommandHelpAndExit(ctx, "list", 1) // last argument is exit code.
	}
}

// doShareList list shared url's.
func doShareList(cmd string) *probe.Error {
	if cmd != "upload" && cmd != "download" {
		return probe.NewError(fmt.Errorf("Unknown argument `%s` passed", cmd))
	}

	// Fetch defaults.
	uploadsFile := getShareUploadsFile()
	downloadsFile := getShareDownloadsFile()

	// Load previously saved upload-shares.
	shareDB := newShareDBV1()

	// if upload - read uploads file.
	if cmd == "upload" {
		if err := shareDB.Load(uploadsFile); err != nil {
			return err.Trace(uploadsFile)
		}
	}

	// if download - read downloads file.
	if cmd == "download" {
		if err := shareDB.Load(downloadsFile); err != nil {
			return err.Trace(downloadsFile)
		}
	}

	// Print previously shared entries.
	for shareURL, share := range shareDB.Shares {
		printMsg(shareMesssage{
			ObjectURL:   share.URL,
			ShareURL:    shareURL,
			TimeLeft:    share.Expiry - time.Since(share.Date),
			ContentType: share.ContentType,
		})
	}
	return nil
}

// main entry point for share list.
func mainShareList(ctx *cli.Context) error {

	// validate command-line args.
	checkShareListSyntax(ctx)

	// Additional command speific theme customization.
	shareSetColor()

	// Initialize share config folder.
	initShareConfig()

	// List shares.
	fatalIf(doShareList(ctx.Args().First()).Trace(), "Unable to list previously shared URLs.")
	return nil
}
<commit_msg>Fix for issue #2093 NAME: (#2100)<commit_after>/*
 * Minio Client (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"time"

	"github.com/minio/cli"
	"github.com/minio/minio/pkg/probe"
)

var (
	shareListFlags = []cli.Flag{}
)

// Share documents via URL.
var shareList = cli.Command{
	Name:   "list",
	Usage:  "List previously shared objects and folders.",
	Action: mainShareList,
	Before: setGlobalsFromContext,
	Flags:  append(shareListFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
   {{.HelpName}} COMMAND - {{.Usage}}

COMMAND:
   upload:   list previously shared access to uploads.
   download: list previously shared access to downloads.

USAGE:
   {{.HelpName}} COMMAND

EXAMPLES:
   1. List previously shared downloads, that haven't expired yet.
       $ {{.HelpName}} download
   2. List previously shared uploads, that haven't expired yet.
       $ {{.HelpName}} upload

`,
}

// validate command-line args.
func checkShareListSyntax(ctx *cli.Context) {
	args := ctx.Args()
	if !args.Present() || (args.First() != "upload" && args.First() != "download") {
		cli.ShowCommandHelpAndExit(ctx, "list", 1) // last argument is exit code.
	}
}

// doShareList list shared url's.
func doShareList(cmd string) *probe.Error {
	if cmd != "upload" && cmd != "download" {
		return probe.NewError(fmt.Errorf("Unknown argument `%s` passed", cmd))
	}

	// Fetch defaults.
	uploadsFile := getShareUploadsFile()
	downloadsFile := getShareDownloadsFile()

	// Load previously saved upload-shares.
	shareDB := newShareDBV1()

	// if upload - read uploads file.
	if cmd == "upload" {
		if err := shareDB.Load(uploadsFile); err != nil {
			return err.Trace(uploadsFile)
		}
	}

	// if download - read downloads file.
	if cmd == "download" {
		if err := shareDB.Load(downloadsFile); err != nil {
			return err.Trace(downloadsFile)
		}
	}

	// Print previously shared entries.
	for shareURL, share := range shareDB.Shares {
		printMsg(shareMesssage{
			ObjectURL:   share.URL,
			ShareURL:    shareURL,
			TimeLeft:    share.Expiry - time.Since(share.Date),
			ContentType: share.ContentType,
		})
	}
	return nil
}

// main entry point for share list.
func mainShareList(ctx *cli.Context) error {

	// validate command-line args.
	checkShareListSyntax(ctx)

	// Additional command speific theme customization.
	shareSetColor()

	// Initialize share config folder.
	initShareConfig()

	// List shares.
	fatalIf(doShareList(ctx.Args().First()).Trace(), "Unable to list previously shared URLs.")
	return nil
}
<|endoftext|>


<commit_before>package main

// Much of this code originates from https://github.com/AaronO/go-git-http
// Licensed under Apache-2.0

import (
	"compress/flate"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gitpods/gitpods/cmd"
	"github.com/go-chi/chi"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

type GitHTTP struct {
	root   string
	git    string
	Logger log.Logger
}

func NewGitHTTP(root string) *GitHTTP {
	return &GitHTTP{
		root:   root,
		git:    "/usr/bin/git",
		Logger: log.NewNopLogger(),
	}
}

func (gh *GitHTTP) Handler() *chi.Mux {
	r := chi.NewRouter()
	r.Use(cmd.NewRequestLogger(gh.Logger))

	//dings := githttp.New(gh.root)
	//r.Mount("/", dings)
	//return r

	r.Get("/{owner}/{name}/HEAD", NoCaching(gh.headHandler))
	r.Get("/{owner}/{name}/info/refs", NoCaching(gh.infoRefsHandler))
	r.Get("/{owner}/{name}/objects/{folder:[0-9a-f]{2}}/{file:[0-9a-f]{38}}", CacheForever(gh.looseObjectHandler))
	r.Get("/{owner}/{name}/objects/info/{thing:[^/]*}", NoCaching(gh.infoHandler)) // TODO
	r.Get("/{owner}/{name}/objects/info/alternates", NoCaching(gh.alternatesHandler))
	r.Get("/{owner}/{name}/objects/info/http-alternates", NoCaching(gh.httpAlternatesHandler))
	r.Get("/{owner}/{name}/objects/info/packs", implementHandler)
	r.Get("/{owner}/{name}/objects/pack/pack-[0-9a-f]{40}\\.idx", implementHandler)
	r.Get("/{owner}/{name}/objects/pack/pack-[0-9a-f]{40}\\.pack", implementHandler)
	r.Post("/{owner}/{name}/git-{service}", gh.serviceHandler)

	r.NotFound(func(w http.ResponseWriter, r *http.Request) {
		level.Debug(gh.Logger).Log(
			"msg", "not found",
			"path", r.URL.String(),
		)
	})

	return r
}

func implementHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "not implemented", http.StatusInternalServerError)
}

func NoCaching(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT")
		w.Header().Set("Pragma", "no-cache")
		w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate")
		next.ServeHTTP(w, r)
	}
}

func CacheForever(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		now := time.Now()
		expires := now.AddDate(1, 0, 0)
		w.Header().Set("Date", fmt.Sprintf("%d", now.Unix()))
		w.Header().Set("Expires", fmt.Sprintf("%d", expires.Unix()))
		w.Header().Set("Cache-Control", "public, max-age=31536000")
		next.ServeHTTP(w, r)
	}
}

func (gh *GitHTTP) headHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "HEAD")

	h := gh.textFileHandler(path, "text/plain")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) alternatesHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "/objects/info/alternates")

	h := gh.textFileHandler(path, "text/plain")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) httpAlternatesHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "/objects/info/http-alternates")

	h := gh.textFileHandler(path, "text/plain")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) textFileHandler(path string, contentType string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		f, err := os.Stat(path)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", contentType)
		w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Size()))
		w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat))
		http.ServeFile(w, r, path)
	}
}

func (gh *GitHTTP) serviceHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	service := chi.URLParam(r, "service")
	logger := log.With(gh.Logger,
		"owner", owner,
		"name", name,
		"service", service,
	)

	defer r.Body.Close()

	var body io.Reader
	var err error
	switch r.Header.Get("content-encoding") {
	case "gzip":
		body, err = gzip.NewReader(r.Body)
	case "deflate":
		body = flate.NewReader(r.Body)
	default:
		body = r.Body
	}

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to create body reader", "err", err)
		return
	}

	args := []string{service, "--stateless-rpc", "."}
	cmd := exec.CommandContext(context.Background(), gh.git, args...)
	cmd.Dir = filepath.Join(gh.root, owner, name)

	stdin, err := cmd.StdinPipe()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to create pipe to git's stdin", "err", err)
		return
	}
	defer stdin.Close()

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to create pipe to git's stdout", "err", err)
		return
	}
	defer stdout.Close()

	if err := cmd.Start(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to start git", "err", err)
		return
	}

	if _, err := io.Copy(stdin, body); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to copy request to git's stdin", "err", err)
		return
	}
	stdin.Close()

	if _, err := io.Copy(w, stdout); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to copy git's stdout to response", "err", err)
		return
	}

	if err := cmd.Wait(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to wait for git", "err", err)
		return
	}

	// TODO: Fire events to channel

	w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-result", service))

}

func (gh *GitHTTP) infoRefsHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	service := serviceQuery(r)
	logger := log.With(gh.Logger,
		"owner", owner,
		"name", name,
		"service", service,
	)

	ctx, cancel := context.WithTimeout(r.Context(), time.Second)
	defer cancel()

	args := []string{service, "--stateless-rpc", "--advertise-refs", "."}
	cmd := exec.CommandContext(ctx, gh.git, args...)
	cmd.Dir = filepath.Join(gh.root, owner, name)

	refs, err := cmd.Output()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to get refs", "err", err)
		return
	}

	w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-advertisement", service))
	w.Write(packetWrite(fmt.Sprintf("# service=git-%s\n", service)))
	w.Write(packetFlush())
	w.Write(refs)
}

func (gh *GitHTTP) looseObjectHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	folder := chi.URLParam(r, "folder")
	file := chi.URLParam(r, "file")
	path := filepath.Join(gh.root, owner, name, folder, file)

	gh.textFileHandler(path, "application/x-git-loose-object")
}

func (gh *GitHTTP) infoHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	thing := chi.URLParam(r, "thing")
	path := filepath.Join(gh.root, owner, name, "objects", "info", thing)

	fmt.Println(r.URL.String())
	gh.textFileHandler(path, "text/plain")

}

func ownerName(r *http.Request) (string, string) {
	return chi.URLParam(r, "owner"), chi.URLParam(r, "name")
}

func serviceQuery(r *http.Request) string {
	return strings.TrimPrefix(r.URL.Query().Get("service"), "git-")
}

func packetFlush() []byte {
	return []byte("0000")
}

func packetWrite(str string) []byte {
	s := strconv.FormatInt(int64(len(str)+4), 16)

	if len(s)%4 != 0 {
		s = strings.Repeat("0", 4-len(s)%4) + s
	}

	return []byte(s + str)
}
<commit_msg>Implement last missing idx & pack handlers<commit_after>package main

// Much of this code originates from https://github.com/AaronO/go-git-http
// Licensed under Apache-2.0

import (
	"compress/flate"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gitpods/gitpods/cmd"
	"github.com/go-chi/chi"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

type GitHTTP struct {
	root   string
	git    string
	Logger log.Logger
}

func NewGitHTTP(root string) *GitHTTP {
	return &GitHTTP{
		root:   root,
		git:    "/usr/bin/git",
		Logger: log.NewNopLogger(),
	}
}

func (gh *GitHTTP) Handler() *chi.Mux {
	r := chi.NewRouter()
	r.Use(cmd.NewRequestLogger(gh.Logger))

	//dings := githttp.New(gh.root)
	//r.Mount("/", dings)
	//return r

	r.Get("/{owner}/{name}/HEAD", NoCaching(gh.headHandler))
	r.Get("/{owner}/{name}/info/refs", NoCaching(gh.infoRefsHandler))
	r.Get("/{owner}/{name}/objects/{folder:[0-9a-f]{2}}/{file:[0-9a-f]{38}}", CacheForever(gh.looseObjectHandler))
	r.Get("/{owner}/{name}/objects/info/{thing:[^/]*}", NoCaching(gh.infoHandler)) // TODO
	r.Get("/{owner}/{name}/objects/info/alternates", NoCaching(gh.alternatesHandler))
	r.Get("/{owner}/{name}/objects/info/http-alternates", NoCaching(gh.httpAlternatesHandler))
	r.Get("/{owner}/{name}/objects/info/packs", CacheForever(gh.infoPacksHandler))
	r.Get("/{owner}/{name}/objects/pack/pack-{hash:[0-9a-f]{40}}.idx", CacheForever(gh.idxHandler))
	r.Get("/{owner}/{name}/objects/pack/pack-{hash:[0-9a-f]{40}}.pack", CacheForever(gh.packHandler))
	r.Post("/{owner}/{name}/git-{service}", gh.serviceHandler)

	r.NotFound(func(w http.ResponseWriter, r *http.Request) {
		level.Debug(gh.Logger).Log(
			"msg", "not found",
			"path", r.URL.String(),
		)
	})

	return r
}

func NoCaching(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT")
		w.Header().Set("Pragma", "no-cache")
		w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate")
		next.ServeHTTP(w, r)
	}
}

func CacheForever(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		now := time.Now()
		expires := now.AddDate(1, 0, 0)
		w.Header().Set("Date", fmt.Sprintf("%d", now.Unix()))
		w.Header().Set("Expires", fmt.Sprintf("%d", expires.Unix()))
		w.Header().Set("Cache-Control", "public, max-age=31536000")
		next.ServeHTTP(w, r)
	}
}

func (gh *GitHTTP) headHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "HEAD")

	h := gh.textFileHandler(path, "text/plain")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) alternatesHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "/objects/info/alternates")

	h := gh.textFileHandler(path, "text/plain")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) httpAlternatesHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "/objects/info/http-alternates")

	h := gh.textFileHandler(path, "text/plain")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) infoPacksHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	path := filepath.Join(gh.root, owner, name, "/objects/info/packs")

	h := gh.textFileHandler(path, "text/plain; charset=utf-8")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) idxHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	hash := chi.URLParam(r, "hash")
	path := filepath.Join(gh.root, owner, name, fmt.Sprintf("objects/pack/pack-%s.idx", hash))

	h := gh.textFileHandler(path, "application/x-git-packed-objects-toc")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) packHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	hash := chi.URLParam(r, "hash")
	path := filepath.Join(gh.root, owner, name, fmt.Sprintf("objects/pack/pack-%s.pack", hash))

	h := gh.textFileHandler(path, "application/x-git-packed-objects")
	h.ServeHTTP(w, r)
}

func (gh *GitHTTP) textFileHandler(path string, contentType string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		f, err := os.Stat(path)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", contentType)
		w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Size()))
		w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat))
		http.ServeFile(w, r, path)
	}
}

func (gh *GitHTTP) serviceHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	service := chi.URLParam(r, "service")
	logger := log.With(gh.Logger,
		"owner", owner,
		"name", name,
		"service", service,
	)

	defer r.Body.Close()

	var body io.Reader
	var err error
	switch r.Header.Get("content-encoding") {
	case "gzip":
		body, err = gzip.NewReader(r.Body)
	case "deflate":
		body = flate.NewReader(r.Body)
	default:
		body = r.Body
	}

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to create body reader", "err", err)
		return
	}

	args := []string{service, "--stateless-rpc", "."}
	cmd := exec.CommandContext(context.Background(), gh.git, args...)
	cmd.Dir = filepath.Join(gh.root, owner, name)

	stdin, err := cmd.StdinPipe()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to create pipe to git's stdin", "err", err)
		return
	}
	defer stdin.Close()

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to create pipe to git's stdout", "err", err)
		return
	}
	defer stdout.Close()

	if err := cmd.Start(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to start git", "err", err)
		return
	}

	if _, err := io.Copy(stdin, body); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to copy request to git's stdin", "err", err)
		return
	}
	stdin.Close()

	if _, err := io.Copy(w, stdout); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to copy git's stdout to response", "err", err)
		return
	}

	if err := cmd.Wait(); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to wait for git", "err", err)
		return
	}

	// TODO: Fire events to channel

	w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-result", service))

}

func (gh *GitHTTP) infoRefsHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	service := serviceQuery(r)
	logger := log.With(gh.Logger,
		"owner", owner,
		"name", name,
		"service", service,
	)

	ctx, cancel := context.WithTimeout(r.Context(), time.Second)
	defer cancel()

	args := []string{service, "--stateless-rpc", "--advertise-refs", "."}
	cmd := exec.CommandContext(ctx, gh.git, args...)
	cmd.Dir = filepath.Join(gh.root, owner, name)

	refs, err := cmd.Output()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		level.Warn(logger).Log("msg", "failed to get refs", "err", err)
		return
	}

	w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-advertisement", service))
	w.Write(packetWrite(fmt.Sprintf("# service=git-%s\n", service)))
	w.Write(packetFlush())
	w.Write(refs)
}

func (gh *GitHTTP) looseObjectHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	folder := chi.URLParam(r, "folder")
	file := chi.URLParam(r, "file")
	path := filepath.Join(gh.root, owner, name, folder, file)

	gh.textFileHandler(path, "application/x-git-loose-object")
}

func (gh *GitHTTP) infoHandler(w http.ResponseWriter, r *http.Request) {
	owner, name := ownerName(r)
	thing := chi.URLParam(r, "thing")
	path := filepath.Join(gh.root, owner, name, "objects", "info", thing)

	fmt.Println(r.URL.String())
	gh.textFileHandler(path, "text/plain")

}

func ownerName(r *http.Request) (string, string) {
	return chi.URLParam(r, "owner"), chi.URLParam(r, "name")
}

func serviceQuery(r *http.Request) string {
	return strings.TrimPrefix(r.URL.Query().Get("service"), "git-")
}

func packetFlush() []byte {
	return []byte("0000")
}

func packetWrite(str string) []byte {
	s := strconv.FormatInt(int64(len(str)+4), 16)

	if len(s)%4 != 0 {
		s = strings.Repeat("0", 4-len(s)%4) + s
	}

	return []byte(s + str)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/anz-bank/sysl/pkg/cmdutils"
	"github.com/anz-bank/sysl/pkg/syslwrapper"

	"github.com/anz-bank/sysl/pkg/exporter"
	sysl "github.com/anz-bank/sysl/pkg/sysl"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"gopkg.in/alecthomas/kingpin.v2"
)

type exportCmd struct {
	appName string
	out     string
	mode    string
	format  string
}

const (
	swaggerMode  = "swagger"
	openapi2Mode = "openapi2"
	openapi3Mode = "openapi3"
	jsonMode     = "json"
	yamlMode     = "yaml"
)

func (p *exportCmd) Name() string       { return "export" }
func (p *exportCmd) MaxSyslModule() int { return 1 }

func (p *exportCmd) Configure(app *kingpin.Application) *kingpin.CmdClause {
	cmd := app.Command(p.Name(), "Export sysl to external types. Supported types: Swagger,openapi2,openapi3")
	cmd.Flag("app-name", "name of the sysl App defined in the sysl model."+
		" if there are multiple Apps defined in the sysl model,"+
		" swagger will be generated only for the given app").Short('a').StringVar(&p.appName)
	cmd.Flag(
		"format",
		"format of export, supported options; (swagger | openapi2 | openapi3)",
	).Default("swagger").Short('f').StringVar(&p.mode)
	cmd.Flag("output", "output filepath.format(yaml | json) (default: %(appname).yaml)").Default(
		"%(appname).yaml").Short('o').StringVar(&p.out)
	EnsureFlagsNonEmpty(cmd)
	return cmd
}

func (p *exportCmd) writeSwaggerForApp(
	fs afero.Fs,
	filename string,
	syslApp *sysl.Application,
	logger *logrus.Logger,
) error {
	var output []byte
	switch p.mode {
	case openapi2Mode, swaggerMode:
		swaggerExporter := exporter.MakeSwaggerExporter(syslApp, logger)
		err := swaggerExporter.GenerateSwagger()
		if err != nil {
			logger.Warnf("Error generating Swagger/Openapi2 for the application %s", err)
			return err
		}
		output, err = swaggerExporter.SerializeOutput(p.format)
		if err != nil {
			return err
		}
	case openapi3Mode:
		mod := &sysl.Module{
			Apps: map[string]*sysl.Application{
				syslApp.Name.Part[0]: syslApp,
			},
		}
		mapper := syslwrapper.MakeAppMapper(mod)
		mapper.IndexTypes()
		simpleApps, err := mapper.Map()
		if err != nil {
			return err
		}
		openapi3Exporter := exporter.MakeOpenAPI3Exporter(simpleApps, logger)
		err = openapi3Exporter.Export()
		if err != nil {
			logger.Warnf("Error generating Openapi3 for the application %s", err)
			return err
		}
		output, err = openapi3Exporter.SerializeOutput(syslApp.Name.Part[0], p.format)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unsupported export format")
	}

	dir, err := filepath.Abs(filepath.Dir(filename))
	if err != nil {
		return err
	}
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		logger.Debugf("Target folder does not exist %s", err)
		if err = os.Mkdir(dir, 0755); err != nil {
			logger.Errorf("Error creating target folder; check permission")
			return err
		}
	}
	err = afero.WriteFile(fs, filename, output, os.ModePerm)
	if err != nil {
		return err
	}
	return nil
}

func (p *exportCmd) Execute(args cmdutils.ExecuteArgs) error {
	err := p.determineOperationMode(p.out)
	if err != nil {
		return err
	}

	if strings.Contains(p.out, "%(appname)") {
		for appName, syslApp := range args.Modules[0].GetApps() {
			outputFileName := cmdutils.MakeFormatParser(p.out).LabelApp(appName, "", syslApp.GetAttrs())
			err := p.writeSwaggerForApp(args.Filesystem, outputFileName, syslApp, args.Logger)
			if err != nil {
				return err
			}
		}
		return nil
	} else if syslApp, syslAppFound := args.Modules[0].GetApps()[p.appName]; syslAppFound {
		return p.writeSwaggerForApp(args.Filesystem, p.out, syslApp, args.Logger)
	}
	return fmt.Errorf("app not found in the Sysl file")
}

func (p *exportCmd) determineOperationMode(filename string) error {
	fileExtn := strings.TrimLeft(filepath.Ext(filepath.Base(filename)), ".")
	switch fileExtn {
	case jsonMode:
		p.format = jsonMode
	case yamlMode:
		p.format = yamlMode
	default:
		return fmt.Errorf("invalid output file format %s", fileExtn)
	}
	return nil
}
<commit_msg>Fix the `sysl export` command line to be more user friendly (#881)<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/anz-bank/sysl/pkg/cmdutils"
	"github.com/anz-bank/sysl/pkg/syslwrapper"

	"github.com/anz-bank/sysl/pkg/exporter"
	sysl "github.com/anz-bank/sysl/pkg/sysl"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"gopkg.in/alecthomas/kingpin.v2"
)

type exportCmd struct {
	appName string
	out     string
	mode    string
	format  string
}

const (
	swaggerMode  = "swagger"
	openapi2Mode = "openapi2"
	openapi3Mode = "openapi3"
	jsonMode     = "json"
	yamlMode     = "yaml"
)

func (p *exportCmd) Name() string       { return "export" }
func (p *exportCmd) MaxSyslModule() int { return 1 }

func (p *exportCmd) Configure(app *kingpin.Application) *kingpin.CmdClause {
	cmd := app.Command(p.Name(), "Export sysl to external types. Supported types: Swagger,openapi2,openapi3")
	cmd.Flag("app-name", "name of the sysl App defined in the sysl model."+
		" if there are multiple Apps defined in the sysl model,"+
		" swagger will be generated only for the given app").Short('a').StringVar(&p.appName)
	cmd.Flag(
		"format",
		"format of export, supported options; (swagger | openapi2 | openapi3)",
	).Default("swagger").Short('f').StringVar(&p.mode)
	cmd.Flag("output", "output filepath.format(yaml | json) (default: %(appname).yaml)").Default(
		"%(appname).yaml").Short('o').StringVar(&p.out)
	EnsureFlagsNonEmpty(cmd)
	return cmd
}

func (p *exportCmd) writeSwaggerForApp(
	fs afero.Fs,
	filename string,
	syslApp *sysl.Application,
	logger *logrus.Logger,
) error {
	var output []byte
	switch p.mode {
	case openapi2Mode, swaggerMode:
		swaggerExporter := exporter.MakeSwaggerExporter(syslApp, logger)
		err := swaggerExporter.GenerateSwagger()
		if err != nil {
			logger.Warnf("Error generating Swagger/Openapi2 for the application %s", err)
			return err
		}
		output, err = swaggerExporter.SerializeOutput(p.format)
		if err != nil {
			return err
		}
	case openapi3Mode:
		mod := &sysl.Module{
			Apps: map[string]*sysl.Application{
				syslApp.Name.Part[0]: syslApp,
			},
		}
		mapper := syslwrapper.MakeAppMapper(mod)
		mapper.IndexTypes()
		simpleApps, err := mapper.Map()
		if err != nil {
			return err
		}
		openapi3Exporter := exporter.MakeOpenAPI3Exporter(simpleApps, logger)
		err = openapi3Exporter.Export()
		if err != nil {
			logger.Warnf("Error generating Openapi3 for the application %s", err)
			return err
		}
		output, err = openapi3Exporter.SerializeOutput(syslApp.Name.Part[0], p.format)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unsupported export format")
	}

	dir, err := filepath.Abs(filepath.Dir(filename))
	if err != nil {
		return err
	}
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		logger.Debugf("Target folder does not exist %s", err)
		if err = os.Mkdir(dir, 0755); err != nil {
			logger.Errorf("Error creating target folder; check permission")
			return err
		}
	}
	err = afero.WriteFile(fs, filename, output, os.ModePerm)
	if err != nil {
		return err
	}
	return nil
}

func (p *exportCmd) Execute(args cmdutils.ExecuteArgs) error {
	err := p.determineOperationMode(p.out)
	if err != nil {
		return err
	}

	var writeCount int
	for appName, syslApp := range args.Modules[0].GetApps() {
		if appName == p.appName || p.appName == "" {
			outputFileName := cmdutils.MakeFormatParser(p.out).LabelApp(appName, "", syslApp.GetAttrs())
			if err := args.Filesystem.MkdirAll(filepath.Dir(outputFileName), os.ModePerm); err != nil {
				return err
			}
			if p.appName == "" && outputFileName == p.out {
				ext := filepath.Ext(outputFileName)
				// convert out.yaml something like out.Fooapp.yaml
				outputFileName = fmt.Sprintf("%s.%s%s", strings.TrimSuffix(outputFileName, ext), appName, ext)
			}
			args.Logger.Infof("Exporting app `%s` -> %s\n", appName, outputFileName)
			err := p.writeSwaggerForApp(args.Filesystem, outputFileName, syslApp, args.Logger)
			if err != nil {
				return err
			}
			writeCount++
		}
	}
	if writeCount == 0 {
		return fmt.Errorf("app not found in the Sysl file")
	}
	return nil
}

func (p *exportCmd) determineOperationMode(filename string) error {
	fileExtn := strings.TrimLeft(filepath.Ext(filepath.Base(filename)), ".")
	switch fileExtn {
	case jsonMode:
		p.format = jsonMode
	case yamlMode:
		p.format = yamlMode
	default:
		return fmt.Errorf("invalid output file format %s", fileExtn)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	fmtlog "log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenk/backoff"
	"github.com/containous/flaeg"
	"github.com/containous/staert"
	"github.com/containous/traefik/acme"
	"github.com/containous/traefik/collector"
	"github.com/containous/traefik/configuration"
	"github.com/containous/traefik/job"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider/ecs"
	"github.com/containous/traefik/provider/kubernetes"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/server"
	"github.com/containous/traefik/server/uuid"
	traefikTls "github.com/containous/traefik/tls"
	"github.com/containous/traefik/types"
	"github.com/containous/traefik/version"
	"github.com/coreos/go-systemd/daemon"
)

func main() {
	//traefik config inits
	traefikConfiguration := NewTraefikConfiguration()
	traefikPointersConfiguration := NewTraefikDefaultPointersConfiguration()
	//traefik Command init
	traefikCmd := &flaeg.Command{
		Name: "traefik",
		Description: `traefik is a modern HTTP reverse proxy and load balancer made to deploy microservices with ease.
Complete documentation is available at https://traefik.io`,
		Config:                traefikConfiguration,
		DefaultPointersConfig: traefikPointersConfiguration,
		Run: func() error {
			run(&traefikConfiguration.GlobalConfiguration, traefikConfiguration.ConfigFile)
			return nil
		},
	}

	//storeconfig Command init
	storeConfigCmd := newStoreConfigCmd(traefikConfiguration, traefikPointersConfiguration)

	//init flaeg source
	f := flaeg.New(traefikCmd, os.Args[1:])
	//add custom parsers
	f.AddParser(reflect.TypeOf(configuration.EntryPoints{}), &configuration.EntryPoints{})
	f.AddParser(reflect.TypeOf(configuration.DefaultEntryPoints{}), &configuration.DefaultEntryPoints{})
	f.AddParser(reflect.TypeOf(traefikTls.RootCAs{}), &traefikTls.RootCAs{})
	f.AddParser(reflect.TypeOf(types.Constraints{}), &types.Constraints{})
	f.AddParser(reflect.TypeOf(kubernetes.Namespaces{}), &kubernetes.Namespaces{})
	f.AddParser(reflect.TypeOf(ecs.Clusters{}), &ecs.Clusters{})
	f.AddParser(reflect.TypeOf([]acme.Domain{}), &acme.Domains{})
	f.AddParser(reflect.TypeOf(types.Buckets{}), &types.Buckets{})

	//add commands
	f.AddCommand(newVersionCmd())
	f.AddCommand(newBugCmd(traefikConfiguration, traefikPointersConfiguration))
	f.AddCommand(storeConfigCmd)
	f.AddCommand(newHealthCheckCmd(traefikConfiguration, traefikPointersConfiguration))

	usedCmd, err := f.GetCommand()
	if err != nil {
		fmtlog.Println(err)
		os.Exit(-1)
	}

	if _, err := f.Parse(usedCmd); err != nil {
		fmtlog.Printf("Error parsing command: %s\n", err)
		os.Exit(-1)
	}

	//staert init
	s := staert.NewStaert(traefikCmd)
	//init toml source
	toml := staert.NewTomlSource("traefik", []string{traefikConfiguration.ConfigFile, "/etc/traefik/", "$HOME/.traefik/", "."})

	//add sources to staert
	s.AddSource(toml)
	s.AddSource(f)
	if _, err := s.LoadConfig(); err != nil {
		fmtlog.Printf("Error reading TOML config file %s : %s\n", toml.ConfigFileUsed(), err)
		os.Exit(-1)
	}

	traefikConfiguration.ConfigFile = toml.ConfigFileUsed()

	kv, err := createKvSource(traefikConfiguration)
	if err != nil {
		fmtlog.Printf("Error creating kv store: %s\n", err)
		os.Exit(-1)
	}
	storeConfigCmd.Run = runStoreConfig(kv, traefikConfiguration)

	// IF a KV Store is enable and no sub-command called in args
	if kv != nil && usedCmd == traefikCmd {
		if traefikConfiguration.Cluster == nil {
			traefikConfiguration.Cluster = &types.Cluster{Node: uuid.Get()}
		}
		if traefikConfiguration.Cluster.Store == nil {
			traefikConfiguration.Cluster.Store = &types.Store{Prefix: kv.Prefix, Store: kv.Store}
		}
		s.AddSource(kv)
		operation := func() error {
			_, err := s.LoadConfig()
			return err
		}
		notify := func(err error, time time.Duration) {
			log.Errorf("Load config error: %+v, retrying in %s", err, time)
		}
		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
		if err != nil {
			fmtlog.Printf("Error loading configuration: %s\n", err)
			os.Exit(-1)
		}
	}

	if err := s.Run(); err != nil {
		fmtlog.Printf("Error running traefik: %s\n", err)
		os.Exit(-1)
	}

	os.Exit(0)
}

func run(globalConfiguration *configuration.GlobalConfiguration, configFile string) {
	configureLogging(globalConfiguration)

	if len(configFile) > 0 {
		log.Infof("Using TOML configuration file %s", configFile)
	}

	http.DefaultTransport.(*http.Transport).Proxy = http.ProxyFromEnvironment

	globalConfiguration.SetEffectiveConfiguration(configFile)
	globalConfiguration.ValidateConfiguration()

	jsonConf, _ := json.Marshal(globalConfiguration)
	log.Infof("Traefik version %s built on %s", version.Version, version.BuildDate)

	if globalConfiguration.CheckNewVersion {
		checkNewVersion()
	}

	stats(globalConfiguration)

	log.Debugf("Global configuration loaded %s", string(jsonConf))
	svr := server.NewServer(*globalConfiguration)
	svr.Start()
	defer svr.Close()

	sent, err := daemon.SdNotify(false, "READY=1")
	if !sent && err != nil {
		log.Error("Fail to notify", err)
	}

	t, err := daemon.SdWatchdogEnabled(false)
	if err != nil {
		log.Error("Problem with watchdog", err)
	} else if t != 0 {
		// Send a ping each half time given
		t = t / 2
		log.Info("Watchdog activated with timer each ", t)
		safe.Go(func() {
			tick := time.Tick(t)
			for range tick {
				_, errHealthCheck := healthCheck(*globalConfiguration)
				if globalConfiguration.Ping == nil || errHealthCheck == nil {
					if ok, _ := daemon.SdNotify(false, "WATCHDOG=1"); !ok {
						log.Error("Fail to tick watchdog")
					}
				} else {
					log.Error(errHealthCheck)
				}
			}
		})
	}

	svr.Wait()
	log.Info("Shutting down")
	logrus.Exit(0)
}

func configureLogging(globalConfiguration *configuration.GlobalConfiguration) {
	// configure default log flags
	fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)

	if globalConfiguration.Debug {
		globalConfiguration.LogLevel = "DEBUG"
	}

	// configure log level
	level, err := logrus.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
	if err != nil {
		log.Error("Error getting level", err)
	}
	log.SetLevel(level)

	// configure log output file
	logFile := globalConfiguration.TraefikLogsFile
	if len(logFile) > 0 {
		log.Warn("top-level traefikLogsFile has been deprecated -- please use traefiklog.filepath")
	}
	if globalConfiguration.TraefikLog != nil && len(globalConfiguration.TraefikLog.FilePath) > 0 {
		logFile = globalConfiguration.TraefikLog.FilePath
	}

	// configure log format
	var formatter logrus.Formatter
	if globalConfiguration.TraefikLog != nil && globalConfiguration.TraefikLog.Format == "json" {
		formatter = &logrus.JSONFormatter{}
	} else {
		disableColors := false
		if len(logFile) > 0 {
			disableColors = true
		}
		formatter = &logrus.TextFormatter{DisableColors: disableColors, FullTimestamp: true, DisableSorting: true}
	}
	log.SetFormatter(formatter)

	if len(logFile) > 0 {
		dir := filepath.Dir(logFile)

		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.Errorf("Failed to create log path %s: %s", dir, err)
		}

		err = log.OpenFile(logFile)
		logrus.RegisterExitHandler(func() {
			if err := log.CloseFile(); err != nil {
				log.Error("Error closing log", err)
			}
		})
		if err != nil {
			log.Error("Error opening file", err)
		}
	}
}

func checkNewVersion() {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			version.CheckNewVersion()
		}
	})
}

func stats(globalConfiguration *configuration.GlobalConfiguration) {
	if globalConfiguration.SendAnonymousUsage {
		log.Info(`
Stats collection is enabled.
Many thanks for contributing to Traefik's improvement by allowing us to receive anonymous information from your configuration.
Help us improve Traefik by leaving this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
		collect(globalConfiguration)
	} else {
		log.Info(`
Stats collection is disabled.
Help us improve Traefik by turning this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
	}
}

func collect(globalConfiguration *configuration.GlobalConfiguration) {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			if err := collector.Collect(globalConfiguration); err != nil {
				log.Debug(err)
			}
		}
	})
}
<commit_msg>Hide the pflag error when displaying help.<commit_after>package main

import (
	"encoding/json"
	fmtlog "log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenk/backoff"
	"github.com/containous/flaeg"
	"github.com/containous/staert"
	"github.com/containous/traefik/acme"
	"github.com/containous/traefik/collector"
	"github.com/containous/traefik/configuration"
	"github.com/containous/traefik/job"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider/ecs"
	"github.com/containous/traefik/provider/kubernetes"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/server"
	"github.com/containous/traefik/server/uuid"
	traefikTls "github.com/containous/traefik/tls"
	"github.com/containous/traefik/types"
	"github.com/containous/traefik/version"
	"github.com/coreos/go-systemd/daemon"
	"github.com/ogier/pflag"
)

func main() {
	//traefik config inits
	traefikConfiguration := NewTraefikConfiguration()
	traefikPointersConfiguration := NewTraefikDefaultPointersConfiguration()
	//traefik Command init
	traefikCmd := &flaeg.Command{
		Name: "traefik",
		Description: `traefik is a modern HTTP reverse proxy and load balancer made to deploy microservices with ease.
Complete documentation is available at https://traefik.io`,
		Config:                traefikConfiguration,
		DefaultPointersConfig: traefikPointersConfiguration,
		Run: func() error {
			run(&traefikConfiguration.GlobalConfiguration, traefikConfiguration.ConfigFile)
			return nil
		},
	}

	//storeconfig Command init
	storeConfigCmd := newStoreConfigCmd(traefikConfiguration, traefikPointersConfiguration)

	//init flaeg source
	f := flaeg.New(traefikCmd, os.Args[1:])
	//add custom parsers
	f.AddParser(reflect.TypeOf(configuration.EntryPoints{}), &configuration.EntryPoints{})
	f.AddParser(reflect.TypeOf(configuration.DefaultEntryPoints{}), &configuration.DefaultEntryPoints{})
	f.AddParser(reflect.TypeOf(traefikTls.RootCAs{}), &traefikTls.RootCAs{})
	f.AddParser(reflect.TypeOf(types.Constraints{}), &types.Constraints{})
	f.AddParser(reflect.TypeOf(kubernetes.Namespaces{}), &kubernetes.Namespaces{})
	f.AddParser(reflect.TypeOf(ecs.Clusters{}), &ecs.Clusters{})
	f.AddParser(reflect.TypeOf([]acme.Domain{}), &acme.Domains{})
	f.AddParser(reflect.TypeOf(types.Buckets{}), &types.Buckets{})

	//add commands
	f.AddCommand(newVersionCmd())
	f.AddCommand(newBugCmd(traefikConfiguration, traefikPointersConfiguration))
	f.AddCommand(storeConfigCmd)
	f.AddCommand(newHealthCheckCmd(traefikConfiguration, traefikPointersConfiguration))

	usedCmd, err := f.GetCommand()
	if err != nil {
		fmtlog.Println(err)
		os.Exit(-1)
	}

	if _, err := f.Parse(usedCmd); err != nil {
		if err == pflag.ErrHelp {
			os.Exit(0)
		}
		fmtlog.Printf("Error parsing command: %s\n", err)
		os.Exit(-1)
	}

	//staert init
	s := staert.NewStaert(traefikCmd)
	//init toml source
	toml := staert.NewTomlSource("traefik", []string{traefikConfiguration.ConfigFile, "/etc/traefik/", "$HOME/.traefik/", "."})

	//add sources to staert
	s.AddSource(toml)
	s.AddSource(f)
	if _, err := s.LoadConfig(); err != nil {
		fmtlog.Printf("Error reading TOML config file %s : %s\n", toml.ConfigFileUsed(), err)
		os.Exit(-1)
	}

	traefikConfiguration.ConfigFile = toml.ConfigFileUsed()

	kv, err := createKvSource(traefikConfiguration)
	if err != nil {
		fmtlog.Printf("Error creating kv store: %s\n", err)
		os.Exit(-1)
	}
	storeConfigCmd.Run = runStoreConfig(kv, traefikConfiguration)

	// IF a KV Store is enable and no sub-command called in args
	if kv != nil && usedCmd == traefikCmd {
		if traefikConfiguration.Cluster == nil {
			traefikConfiguration.Cluster = &types.Cluster{Node: uuid.Get()}
		}
		if traefikConfiguration.Cluster.Store == nil {
			traefikConfiguration.Cluster.Store = &types.Store{Prefix: kv.Prefix, Store: kv.Store}
		}
		s.AddSource(kv)
		operation := func() error {
			_, err := s.LoadConfig()
			return err
		}
		notify := func(err error, time time.Duration) {
			log.Errorf("Load config error: %+v, retrying in %s", err, time)
		}
		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
		if err != nil {
			fmtlog.Printf("Error loading configuration: %s\n", err)
			os.Exit(-1)
		}
	}

	if err := s.Run(); err != nil {
		fmtlog.Printf("Error running traefik: %s\n", err)
		os.Exit(-1)
	}

	os.Exit(0)
}

func run(globalConfiguration *configuration.GlobalConfiguration, configFile string) {
	configureLogging(globalConfiguration)

	if len(configFile) > 0 {
		log.Infof("Using TOML configuration file %s", configFile)
	}

	http.DefaultTransport.(*http.Transport).Proxy = http.ProxyFromEnvironment

	globalConfiguration.SetEffectiveConfiguration(configFile)
	globalConfiguration.ValidateConfiguration()

	jsonConf, _ := json.Marshal(globalConfiguration)
	log.Infof("Traefik version %s built on %s", version.Version, version.BuildDate)

	if globalConfiguration.CheckNewVersion {
		checkNewVersion()
	}

	stats(globalConfiguration)

	log.Debugf("Global configuration loaded %s", string(jsonConf))
	svr := server.NewServer(*globalConfiguration)
	svr.Start()
	defer svr.Close()

	sent, err := daemon.SdNotify(false, "READY=1")
	if !sent && err != nil {
		log.Error("Fail to notify", err)
	}

	t, err := daemon.SdWatchdogEnabled(false)
	if err != nil {
		log.Error("Problem with watchdog", err)
	} else if t != 0 {
		// Send a ping each half time given
		t = t / 2
		log.Info("Watchdog activated with timer each ", t)
		safe.Go(func() {
			tick := time.Tick(t)
			for range tick {
				_, errHealthCheck := healthCheck(*globalConfiguration)
				if globalConfiguration.Ping == nil || errHealthCheck == nil {
					if ok, _ := daemon.SdNotify(false, "WATCHDOG=1"); !ok {
						log.Error("Fail to tick watchdog")
					}
				} else {
					log.Error(errHealthCheck)
				}
			}
		})
	}

	svr.Wait()
	log.Info("Shutting down")
	logrus.Exit(0)
}

func configureLogging(globalConfiguration *configuration.GlobalConfiguration) {
	// configure default log flags
	fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)

	if globalConfiguration.Debug {
		globalConfiguration.LogLevel = "DEBUG"
	}

	// configure log level
	level, err := logrus.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
	if err != nil {
		log.Error("Error getting level", err)
	}
	log.SetLevel(level)

	// configure log output file
	logFile := globalConfiguration.TraefikLogsFile
	if len(logFile) > 0 {
		log.Warn("top-level traefikLogsFile has been deprecated -- please use traefiklog.filepath")
	}
	if globalConfiguration.TraefikLog != nil && len(globalConfiguration.TraefikLog.FilePath) > 0 {
		logFile = globalConfiguration.TraefikLog.FilePath
	}

	// configure log format
	var formatter logrus.Formatter
	if globalConfiguration.TraefikLog != nil && globalConfiguration.TraefikLog.Format == "json" {
		formatter = &logrus.JSONFormatter{}
	} else {
		disableColors := false
		if len(logFile) > 0 {
			disableColors = true
		}
		formatter = &logrus.TextFormatter{DisableColors: disableColors, FullTimestamp: true, DisableSorting: true}
	}
	log.SetFormatter(formatter)

	if len(logFile) > 0 {
		dir := filepath.Dir(logFile)

		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.Errorf("Failed to create log path %s: %s", dir, err)
		}

		err = log.OpenFile(logFile)
		logrus.RegisterExitHandler(func() {
			if err := log.CloseFile(); err != nil {
				log.Error("Error closing log", err)
			}
		})
		if err != nil {
			log.Error("Error opening file", err)
		}
	}
}

func checkNewVersion() {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			version.CheckNewVersion()
		}
	})
}

func stats(globalConfiguration *configuration.GlobalConfiguration) {
	if globalConfiguration.SendAnonymousUsage {
		log.Info(`
Stats collection is enabled.
Many thanks for contributing to Traefik's improvement by allowing us to receive anonymous information from your configuration.
Help us improve Traefik by leaving this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
		collect(globalConfiguration)
	} else {
		log.Info(`
Stats collection is disabled.
Help us improve Traefik by turning this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
	}
}

func collect(globalConfiguration *configuration.GlobalConfiguration) {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			if err := collector.Collect(globalConfiguration); err != nil {
				log.Debug(err)
			}
		}
	})
}
<|endoftext|>


<commit_before>// Package twse - Fetch stock data from TWSE, OTC
// 擷取台灣股市上市、上櫃股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

// NewTWSE 建立一個 TWSE 上市股票
func NewTWSE(No string, Date time.Time) *Data {
	data := new(Data)
	data.No = No
	data.Date = Date
	data.exchange = "tse"
	return data
}

// NewOTC 建立一個 OTC 上櫃股票
func NewOTC(No string, Date time.Time) *Data {
	data := new(Data)
	data.No = No
	data.Date = Date
	data.exchange = "otc"
	return data
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, utils.RandInt())
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No, utils.RandInt())
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, d.Date.Location())
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

// GetByTimeMap return a map by key of time.Time
func (d Data) GetByTimeMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.Get()
	for _, v := range dailyData {
		data[utils.ParseDate(v[0])] = v
	}
	return data
}

func (d Data) getColsList(colsNo int) []string {
	var result []string
	result = make([]string, len(d.RawData))
	for i, value := range d.RawData {
		result[i] = value[colsNo]
	}
	return result
}

func (d Data) getColsListFloat64(colsNo int) []float64 {
	var result []float64
	result = make([]float64, len(d.RawData))
	for i, v := range d.getColsList(colsNo) {
		result[i], _ = strconv.ParseFloat(v, 64)
	}
	return result
}

// GetVolumeList 取得 成交股數 序列
func (d *Data) GetVolumeList() []uint64 {
	if d.volumeList == nil {
		var result []uint64
		result = make([]uint64, len(d.RawData))
		for i, v := range d.getColsList(1) {
			result[i], _ = strconv.ParseUint(strings.Replace(v, ",", "", -1), 10, 64)
		}
		d.volumeList = result
	}
	return d.volumeList
}

// GetOpenList 取得 開盤價 序列
func (d *Data) GetOpenList() []float64 {
	if d.openList == nil {
		d.openList = d.getColsListFloat64(3)
	}
	return d.openList
}

// GetPriceList 取得 收盤價 序列
func (d *Data) GetPriceList() []float64 {
	if d.priceList == nil {
		d.priceList = d.getColsListFloat64(6)
	}
	return d.priceList
}

// GetRangeList 取得 漲跌價差 序列
func (d *Data) GetRangeList() []float64 {
	if d.rangeList == nil {
		d.rangeList = d.getColsListFloat64(7)
	}
	return d.rangeList
}

// MA 計算 收盤價 的移動平均
func (d Data) MA(days int) []float64 {
	var result []float64
	var priceList = d.GetPriceList()
	result = make([]float64, len(priceList)-days+1)
	for i := range priceList[days-1:] {
		result[i] = utils.AvgFlast64(priceList[i : i+days])
	}
	return result
}

// MAV 計算 成交股數 的移動平均
func (d Data) MAV(days int) []uint64 {
	var result []uint64
	var volumeList = d.GetVolumeList()
	result = make([]uint64, len(volumeList)-days+1)
	for i := range volumeList[days-1:] {
		result[i] = utils.AvgUint64(volumeList[i : i+days])
	}
	return result
}

// IsRed 計算是否收紅 K
func (d Data) IsRed() bool {
	var rangeList = d.GetRangeList()
	return rangeList[len(rangeList)-1] > 0
}

// FmtData is struct for daily data format.
type FmtData struct {
	Date       time.Time
	Volume     uint64  //成交股數
	TotalPrice uint64  //成交金額
	Open       float64 //開盤價
	High       float64 //最高價
	Low        float64 //最低價
	Price      float64 //收盤價
	Range      float64 //漲跌價差
	Totalsale  uint64  //成交筆數
}

// FormatData is format daily data.
func (d Data) FormatData() []FmtData {
	result := make([]FmtData, len(d.RawData))
	var loopd FmtData
	for i, v := range d.RawData {
		loopd.Date = utils.ParseDate(v[0])
		loopd.Volume, _ = strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		loopd.TotalPrice, _ = strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		loopd.Open, _ = strconv.ParseFloat(v[3], 64)
		loopd.High, _ = strconv.ParseFloat(v[4], 64)
		loopd.Low, _ = strconv.ParseFloat(v[5], 64)
		loopd.Price, _ = strconv.ParseFloat(v[6], 64)
		loopd.Range, _ = strconv.ParseFloat(v[7], 64)
		loopd.Totalsale, _ = strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		result[i] = loopd
	}
	return result
}
<commit_msg>Rewrite new struct.<commit_after>// Package twse - Fetch stock data from TWSE, OTC
// 擷取台灣股市上市、上櫃股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

// NewTWSE 建立一個 TWSE 上市股票
func NewTWSE(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "tse",
	}
}

// NewOTC 建立一個 OTC 上櫃股票
func NewOTC(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "otc",
	}
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No, utils.RandInt())
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No, utils.RandInt())
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, d.Date.Location())
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		csvFiles, err := http.Get(d.URL())
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		defer csvFiles.Body.Close()
		data, _ := ioutil.ReadAll(csvFiles.Body)
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

// GetByTimeMap return a map by key of time.Time
func (d Data) GetByTimeMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.Get()
	for _, v := range dailyData {
		data[utils.ParseDate(v[0])] = v
	}
	return data
}

func (d Data) getColsList(colsNo int) []string {
	var result []string
	result = make([]string, len(d.RawData))
	for i, value := range d.RawData {
		result[i] = value[colsNo]
	}
	return result
}

func (d Data) getColsListFloat64(colsNo int) []float64 {
	var result []float64
	result = make([]float64, len(d.RawData))
	for i, v := range d.getColsList(colsNo) {
		result[i], _ = strconv.ParseFloat(v, 64)
	}
	return result
}

// GetVolumeList 取得 成交股數 序列
func (d *Data) GetVolumeList() []uint64 {
	if d.volumeList == nil {
		var result []uint64
		result = make([]uint64, len(d.RawData))
		for i, v := range d.getColsList(1) {
			result[i], _ = strconv.ParseUint(strings.Replace(v, ",", "", -1), 10, 64)
		}
		d.volumeList = result
	}
	return d.volumeList
}

// GetOpenList 取得 開盤價 序列
func (d *Data) GetOpenList() []float64 {
	if d.openList == nil {
		d.openList = d.getColsListFloat64(3)
	}
	return d.openList
}

// GetPriceList 取得 收盤價 序列
func (d *Data) GetPriceList() []float64 {
	if d.priceList == nil {
		d.priceList = d.getColsListFloat64(6)
	}
	return d.priceList
}

// GetRangeList 取得 漲跌價差 序列
func (d *Data) GetRangeList() []float64 {
	if d.rangeList == nil {
		d.rangeList = d.getColsListFloat64(7)
	}
	return d.rangeList
}

// MA 計算 收盤價 的移動平均
func (d Data) MA(days int) []float64 {
	var result []float64
	var priceList = d.GetPriceList()
	result = make([]float64, len(priceList)-days+1)
	for i := range priceList[days-1:] {
		result[i] = utils.AvgFlast64(priceList[i : i+days])
	}
	return result
}

// MAV 計算 成交股數 的移動平均
func (d Data) MAV(days int) []uint64 {
	var result []uint64
	var volumeList = d.GetVolumeList()
	result = make([]uint64, len(volumeList)-days+1)
	for i := range volumeList[days-1:] {
		result[i] = utils.AvgUint64(volumeList[i : i+days])
	}
	return result
}

// IsRed 計算是否收紅 K
func (d Data) IsRed() bool {
	var rangeList = d.GetRangeList()
	return rangeList[len(rangeList)-1] > 0
}

// FmtData is struct for daily data format.
type FmtData struct {
	Date       time.Time
	Volume     uint64  //成交股數
	TotalPrice uint64  //成交金額
	Open       float64 //開盤價
	High       float64 //最高價
	Low        float64 //最低價
	Price      float64 //收盤價
	Range      float64 //漲跌價差
	Totalsale  uint64  //成交筆數
}

// FormatData is format daily data.
func (d Data) FormatData() []FmtData {
	result := make([]FmtData, len(d.RawData))
	var loopd FmtData
	for i, v := range d.RawData {
		loopd.Date = utils.ParseDate(v[0])
		loopd.Volume, _ = strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		loopd.TotalPrice, _ = strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		loopd.Open, _ = strconv.ParseFloat(v[3], 64)
		loopd.High, _ = strconv.ParseFloat(v[4], 64)
		loopd.Low, _ = strconv.ParseFloat(v[5], 64)
		loopd.Price, _ = strconv.ParseFloat(v[6], 64)
		loopd.Range, _ = strconv.ParseFloat(v[7], 64)
		loopd.Totalsale, _ = strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		result[i] = loopd
	}
	return result
}
<|endoftext|>


<commit_before>// Package twse - Fetch stock data from TWSE, OTC
// 擷取台灣股市上市、上櫃股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	iconv "github.com/djimenez/iconv-go"
	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Name        string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

// NewTWSE 建立一個 TWSE 上市股票
func NewTWSE(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "tse",
	}
}

// NewOTC 建立一個 OTC 上櫃股票
func NewOTC(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "otc",
	}
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No)
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No)
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, d.Date.Location())
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		data, err := hCache.Get(d.URL(), true)
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				if d.Name == "" {
					parseTitle, _ := iconv.ConvertString(csvArrayContent[0], "cp950", "utf-8")
					d.Name = strings.Split(parseTitle, " ")[2]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				if d.Name == "" {
					parseTitle, _ := iconv.ConvertString(csvArrayContent[2], "cp950", "utf-8")
					d.Name = strings.Split(parseTitle, ":")[1]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

// GetByTimeMap return a map by key of time.Time
func (d Data) GetByTimeMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.Get()
	for _, v := range dailyData {
		data[utils.ParseDate(v[0])] = v
	}
	return data
}

func (d Data) getColsList(colsNo int) []string {
	var result []string
	result = make([]string, len(d.RawData))
	for i, value := range d.RawData {
		result[i] = value[colsNo]
	}
	return result
}

func (d Data) getColsListFloat64(colsNo int) []float64 {
	var result []float64
	result = make([]float64, len(d.RawData))
	for i, v := range d.getColsList(colsNo) {
		result[i], _ = strconv.ParseFloat(strings.Replace(v, ",", "", -1), 64)
	}
	return result
}

// GetVolumeList 取得 成交股數 序列
func (d *Data) GetVolumeList() []uint64 {
	if d.volumeList == nil {
		var result []uint64
		result = make([]uint64, len(d.RawData))
		for i, v := range d.getColsList(1) {
			result[i], _ = strconv.ParseUint(strings.Replace(v, ",", "", -1), 10, 64)
		}
		d.volumeList = result
	}
	return d.volumeList
}

// GetOpenList 取得 開盤價 序列
func (d *Data) GetOpenList() []float64 {
	if d.openList == nil {
		d.openList = d.getColsListFloat64(3)
	}
	return d.openList
}

// GetPriceList 取得 收盤價 序列
func (d *Data) GetPriceList() []float64 {
	if d.priceList == nil {
		d.priceList = d.getColsListFloat64(6)
	}
	return d.priceList
}

// GetRangeList 取得 漲跌價差 序列
func (d *Data) GetRangeList() []float64 {
	if d.rangeList == nil {
		d.rangeList = d.getColsListFloat64(7)
	}
	return d.rangeList
}

// MA 計算 收盤價 的移動平均
func (d Data) MA(days int) []float64 {
	var result []float64
	var priceList = d.GetPriceList()
	result = make([]float64, len(priceList)-days+1)
	for i := range priceList[days-1:] {
		result[i] = utils.AvgFloat64(priceList[i : i+days])
	}
	return result
}

// MABR 計算 收盤價移動平均 的乖離
func (d Data) MABR(days1, days2 int) []float64 {
	return utils.CalDiffFloat64(d.MA(days1), d.MA(days2))
}

// MAV 計算 成交股數 的移動平均
func (d Data) MAV(days int) []uint64 {
	var result []uint64
	var volumeList = d.GetVolumeList()
	result = make([]uint64, len(volumeList)-days+1)
	for i := range volumeList[days-1:] {
		result[i] = utils.AvgUint64(volumeList[i : i+days])
	}
	return result
}

// MAVBR 計算 成交股數移動平均 的乖離
func (d Data) MAVBR(days1, days2 int) []int64 {
	MAV1 := d.MAV(days1)
	MAV2 := d.MAV(days2)
	var length int
	if len(MAV1) <= len(MAV2) {
		length = len(MAV1)
	} else {
		length = len(MAV2)
	}
	result1 := make([]int64, length)
	result2 := make([]int64, length)
	for i := 1; i <= length; i++ {
		result1[length-i] = int64(MAV1[len(MAV1)-i])
		result2[length-i] = int64(MAV2[len(MAV2)-i])
	}

	return utils.CalDiffInt64(result1, result2)
}

// IsRed 計算是否收紅 K
func (d Data) IsRed() bool {
	var openList = d.GetOpenList()
	var priceList = d.GetPriceList()
	return priceList[len(priceList)-1] > openList[len(openList)-1]
}

// IsThanYesterday 計算漲跌與昨日收盤相比
func (d Data) IsThanYesterday() bool {
	var rangeList = d.GetRangeList()
	return rangeList[len(rangeList)-1] > 0
}

// Len 返回資料序列長度
func (d Data) Len() int {
	return len(d.RawData)
}

// FmtData is struct for daily data format.
type FmtData struct {
	Date       time.Time
	Volume     uint64  //成交股數
	TotalPrice uint64  //成交金額
	Open       float64 //開盤價
	High       float64 //最高價
	Low        float64 //最低價
	Price      float64 //收盤價
	Range      float64 //漲跌價差
	Totalsale  uint64  //成交筆數
}

// FormatData is format daily data.
func (d Data) FormatData() []FmtData {
	result := make([]FmtData, len(d.RawData))
	var loopd FmtData
	for i, v := range d.RawData {
		loopd.Date = utils.ParseDate(v[0])
		loopd.Volume, _ = strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		loopd.TotalPrice, _ = strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		loopd.Open, _ = strconv.ParseFloat(v[3], 64)
		loopd.High, _ = strconv.ParseFloat(v[4], 64)
		loopd.Low, _ = strconv.ParseFloat(v[5], 64)
		loopd.Price, _ = strconv.ParseFloat(v[6], 64)
		loopd.Range, _ = strconv.ParseFloat(v[7], 64)
		loopd.Totalsale, _ = strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		result[i] = loopd
	}
	return result
}

var hCache *utils.HTTPCache

func init() {
	hCache = utils.NewHTTPCache("/Volumes/RamDisk/.gogrs")
}
<commit_msg>Add runtime.GOOS check.<commit_after>// Package twse - Fetch stock data from TWSE, OTC
// 擷取台灣股市上市、上櫃股票資訊
//
package twse

import (
	"encoding/csv"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	iconv "github.com/djimenez/iconv-go"
	"github.com/toomore/gogrs/utils"
)

type unixMapData map[int64][][]string

// Data start with stock no, date.
type Data struct {
	No          string
	Name        string
	Date        time.Time
	RawData     [][]string
	UnixMapData unixMapData
	exchange    string
	openList    []float64
	priceList   []float64
	rangeList   []float64
	volumeList  []uint64
}

// NewTWSE 建立一個 TWSE 上市股票
func NewTWSE(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "tse",
	}
}

// NewOTC 建立一個 OTC 上櫃股票
func NewOTC(No string, Date time.Time) *Data {
	return &Data{
		No:       No,
		Date:     Date,
		exchange: "otc",
	}
}

// URL return stock csv url path.
func (d Data) URL() string {
	var path string
	var host string

	if d.exchange == "tse" {
		path = fmt.Sprintf(utils.TWSECSV, d.Date.Year(), d.Date.Month(), d.Date.Year(), d.Date.Month(), d.No)
		host = utils.TWSEHOST
	} else if d.exchange == "otc" {
		path = fmt.Sprintf(utils.OTCCSV, d.Date.Year()-1911, d.Date.Month(), d.No)
		host = utils.OTCHOST
	}

	return fmt.Sprintf("%s%s", host, path)
}

// Round will do sub one month.
func (d *Data) Round() {
	year, month, _ := d.Date.Date()
	d.Date = time.Date(year, month-1, 1, 0, 0, 0, 0, d.Date.Location())
}

// PlusData will do Round() and Get().
func (d *Data) PlusData() {
	d.Round()
	d.Get()
}

func (d *Data) clearCache() {
	d.rangeList = nil
	d.openList = nil
	d.priceList = nil
	d.volumeList = nil
}

// Get return csv data in array.
func (d *Data) Get() ([][]string, error) {
	if d.UnixMapData == nil {
		d.UnixMapData = make(unixMapData)
	}
	if len(d.UnixMapData[d.Date.Unix()]) == 0 {
		data, err := hCache.Get(d.URL(), true)
		if err != nil {
			return nil, fmt.Errorf("Network fail: %s", err)
		}
		csvArrayContent := strings.Split(string(data), "\n")
		for i := range csvArrayContent {
			csvArrayContent[i] = strings.TrimSpace(csvArrayContent[i])
		}
		var csvReader *csv.Reader
		if (d.exchange == "tse" && len(csvArrayContent) > 2) || (d.exchange == "otc" && len(csvArrayContent) > 5) {
			if d.exchange == "tse" {
				if d.Name == "" {
					parseTitle, _ := iconv.ConvertString(csvArrayContent[0], "cp950", "utf-8")
					d.Name = strings.Split(parseTitle, " ")[2]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[2:], "\n")))
			} else if d.exchange == "otc" {
				if d.Name == "" {
					parseTitle, _ := iconv.ConvertString(csvArrayContent[2], "cp950", "utf-8")
					d.Name = strings.Split(parseTitle, ":")[1]
				}
				csvReader = csv.NewReader(strings.NewReader(strings.Join(csvArrayContent[5:len(csvArrayContent)-1], "\n")))
			}
			allData, err := csvReader.ReadAll()
			d.RawData = append(allData, d.RawData...)
			d.UnixMapData[d.Date.Unix()] = allData
			d.clearCache()
			return allData, err
		}
		return nil, errors.New("Not enough data.")
	}
	return d.UnixMapData[d.Date.Unix()], nil
}

// GetByTimeMap return a map by key of time.Time
func (d Data) GetByTimeMap() map[time.Time]interface{} {
	data := make(map[time.Time]interface{})
	dailyData, _ := d.Get()
	for _, v := range dailyData {
		data[utils.ParseDate(v[0])] = v
	}
	return data
}

func (d Data) getColsList(colsNo int) []string {
	var result []string
	result = make([]string, len(d.RawData))
	for i, value := range d.RawData {
		result[i] = value[colsNo]
	}
	return result
}

func (d Data) getColsListFloat64(colsNo int) []float64 {
	var result []float64
	result = make([]float64, len(d.RawData))
	for i, v := range d.getColsList(colsNo) {
		result[i], _ = strconv.ParseFloat(strings.Replace(v, ",", "", -1), 64)
	}
	return result
}

// GetVolumeList 取得 成交股數 序列
func (d *Data) GetVolumeList() []uint64 {
	if d.volumeList == nil {
		var result []uint64
		result = make([]uint64, len(d.RawData))
		for i, v := range d.getColsList(1) {
			result[i], _ = strconv.ParseUint(strings.Replace(v, ",", "", -1), 10, 64)
		}
		d.volumeList = result
	}
	return d.volumeList
}

// GetOpenList 取得 開盤價 序列
func (d *Data) GetOpenList() []float64 {
	if d.openList == nil {
		d.openList = d.getColsListFloat64(3)
	}
	return d.openList
}

// GetPriceList 取得 收盤價 序列
func (d *Data) GetPriceList() []float64 {
	if d.priceList == nil {
		d.priceList = d.getColsListFloat64(6)
	}
	return d.priceList
}

// GetRangeList 取得 漲跌價差 序列
func (d *Data) GetRangeList() []float64 {
	if d.rangeList == nil {
		d.rangeList = d.getColsListFloat64(7)
	}
	return d.rangeList
}

// MA 計算 收盤價 的移動平均
func (d Data) MA(days int) []float64 {
	var result []float64
	var priceList = d.GetPriceList()
	result = make([]float64, len(priceList)-days+1)
	for i := range priceList[days-1:] {
		result[i] = utils.AvgFloat64(priceList[i : i+days])
	}
	return result
}

// MABR 計算 收盤價移動平均 的乖離
func (d Data) MABR(days1, days2 int) []float64 {
	return utils.CalDiffFloat64(d.MA(days1), d.MA(days2))
}

// MAV 計算 成交股數 的移動平均
func (d Data) MAV(days int) []uint64 {
	var result []uint64
	var volumeList = d.GetVolumeList()
	result = make([]uint64, len(volumeList)-days+1)
	for i := range volumeList[days-1:] {
		result[i] = utils.AvgUint64(volumeList[i : i+days])
	}
	return result
}

// MAVBR 計算 成交股數移動平均 的乖離
func (d Data) MAVBR(days1, days2 int) []int64 {
	MAV1 := d.MAV(days1)
	MAV2 := d.MAV(days2)
	var length int
	if len(MAV1) <= len(MAV2) {
		length = len(MAV1)
	} else {
		length = len(MAV2)
	}
	result1 := make([]int64, length)
	result2 := make([]int64, length)
	for i := 1; i <= length; i++ {
		result1[length-i] = int64(MAV1[len(MAV1)-i])
		result2[length-i] = int64(MAV2[len(MAV2)-i])
	}

	return utils.CalDiffInt64(result1, result2)
}

// IsRed 計算是否收紅 K
func (d Data) IsRed() bool {
	var openList = d.GetOpenList()
	var priceList = d.GetPriceList()
	return priceList[len(priceList)-1] > openList[len(openList)-1]
}

// IsThanYesterday 計算漲跌與昨日收盤相比
func (d Data) IsThanYesterday() bool {
	var rangeList = d.GetRangeList()
	return rangeList[len(rangeList)-1] > 0
}

// Len 返回資料序列長度
func (d Data) Len() int {
	return len(d.RawData)
}

// FmtData is struct for daily data format.
type FmtData struct {
	Date       time.Time
	Volume     uint64  //成交股數
	TotalPrice uint64  //成交金額
	Open       float64 //開盤價
	High       float64 //最高價
	Low        float64 //最低價
	Price      float64 //收盤價
	Range      float64 //漲跌價差
	Totalsale  uint64  //成交筆數
}

// FormatData is format daily data.
func (d Data) FormatData() []FmtData {
	result := make([]FmtData, len(d.RawData))
	var loopd FmtData
	for i, v := range d.RawData {
		loopd.Date = utils.ParseDate(v[0])
		loopd.Volume, _ = strconv.ParseUint(strings.Replace(v[1], ",", "", -1), 10, 32)
		loopd.TotalPrice, _ = strconv.ParseUint(strings.Replace(v[2], ",", "", -1), 10, 32)
		loopd.Open, _ = strconv.ParseFloat(v[3], 64)
		loopd.High, _ = strconv.ParseFloat(v[4], 64)
		loopd.Low, _ = strconv.ParseFloat(v[5], 64)
		loopd.Price, _ = strconv.ParseFloat(v[6], 64)
		loopd.Range, _ = strconv.ParseFloat(v[7], 64)
		loopd.Totalsale, _ = strconv.ParseUint(strings.Replace(v[8], ",", "", -1), 10, 64)
		result[i] = loopd
	}
	return result
}

var hCache *utils.HTTPCache

func init() {
	var dir string
	switch runtime.GOOS {
	case "darwin":
		dir = "/Volumes/RamDisk/"
	case "linux":
		dir = "/run/shm/"
	default:
		dir = os.TempDir()
	}
	hCache = utils.NewHTTPCache(filepath.Join(dir, ".gogrs"))
}
<|endoftext|>


<commit_before>package sshutils

// This package includes utilities implemented by Golang.
// @author Harrison Feng <feng.harrison@gmail.com

import (
	"bytes"
	"fmt"
	"golang.org/x/crypto/ssh"
	"log"
)

func GetSshConfig(user string, password string) *ssh.ClientConfig {

	config := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{
			ssh.KeyboardInteractive(func(user, instruction string, questions []string, echos []bool) ([]string, error) {
				// Just send the password back for all questions
				answers := make([]string, len(questions))
				for i, _ := range answers {
					answers[i] = password
				}
				return answers, nil
			}),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}

	return config

}

func ExecSshCmd(cmd string, hostname string, port string, config *ssh.ClientConfig) (string, error) {

	var b bytes.Buffer

	client, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", hostname, port), config)
	if err != nil {
		log.Fatal("Failed to dail: ", err)
	}

	session, err := client.NewSession()
	if err != nil {
		log.Fatal("Failed to create session: ", err)
	}
	defer session.Close()

	session.Stdout = &b

	if err := session.Run(cmd); err != nil {
		// log.Fatal("Failed to run: " + err.Error())
		log.Fatal("Errors: " + b.String())
		return b.String(), err
	}

	o := b.String()

	return o, nil
}

func Service(params map[string]string, name string, action string, su bool) error {

	var cmd string

	conf := GetSshConfig(params["user"], params["password"])

	if su == true {
		// This requires the user has the same password as root.
		cmd = fmt.Sprintf("echo '%s' | su -c 'service %s %s' root",
			params["password"],
			name,
			action)
	} else {
		cmd = fmt.Sprintf("service %s %s", name, action)
	}

	_, err := ExecSshCmd(cmd, params["hostname"], "22", conf)
	if err != nil {
		return err
	}

	return nil

}

func DownloadWithCurl(params map[string]string, url string, targetDir string, su bool) error {

	var cmd string

	if su == true {
		// This requires the user has the same password as root.
		cmd = fmt.Sprint("echo '%s' | su -c 'cd %s && curl -O %s' root",
			params["password"],
			targetDir,
			url)
	} else {
		cmd = fmt.Sprint("cd %s && curl -O %s", targetDir, url)
	}

	conf := GetSshConfig(params["user"], params["password"])

	_, err := ExecSshCmd(cmd, params["hostname"], "22", conf)
	if err != nil {
		log.Fatal(err)
		return err
	}

	return nil

}
<commit_msg>correct func call<commit_after>package sshutils

// This package includes utilities implemented by Golang.
// @author Harrison Feng <feng.harrison@gmail.com

import (
	"bytes"
	"fmt"
	"golang.org/x/crypto/ssh"
	"log"
)

func GetSshConfig(user string, password string) *ssh.ClientConfig {

	config := &ssh.ClientConfig{
		User: user,
		Auth: []ssh.AuthMethod{
			ssh.KeyboardInteractive(func(user, instruction string, questions []string, echos []bool) ([]string, error) {
				// Just send the password back for all questions
				answers := make([]string, len(questions))
				for i, _ := range answers {
					answers[i] = password
				}
				return answers, nil
			}),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}

	return config

}

func ExecSshCmd(cmd string, hostname string, port string, config *ssh.ClientConfig) (string, error) {

	var b bytes.Buffer

	client, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", hostname, port), config)
	if err != nil {
		log.Fatal("Failed to dail: ", err)
	}

	session, err := client.NewSession()
	if err != nil {
		log.Fatal("Failed to create session: ", err)
	}
	defer session.Close()

	session.Stdout = &b

	if err := session.Run(cmd); err != nil {
		log.Fatal("Failed to run: " + err.Error())
		return b.String(), err
	}

	o := b.String()

	return o, nil
}

func Service(params map[string]string, name string, action string, su bool) error {

	var cmd string

	conf := GetSshConfig(params["user"], params["password"])

	if su == true {
		// This requires the user has the same password as root.
		cmd = fmt.Sprintf("echo '%s' | su -c 'service %s %s' root",
			params["password"],
			name,
			action)
	} else {
		cmd = fmt.Sprintf("service %s %s", name, action)
	}

	_, err := ExecSshCmd(cmd, params["hostname"], "22", conf)
	if err != nil {
		return err
	}

	return nil

}

func DownloadWithCurl(params map[string]string, url string, targetDir string, su bool) error {

	var cmd string

	if su == true {
		// This requires the user has the same password as root.
		cmd = fmt.Sprintf("echo '%s' | su -c 'cd %s && curl -O %s' root",
			params["password"],
			targetDir,
			url)
	} else {
		cmd = fmt.Sprint("cd %s && curl -O %s", targetDir, url)
	}

	conf := GetSshConfig(params["user"], params["password"])

	_, err := ExecSshCmd(cmd, params["hostname"], "22", conf)
	if err != nil {
		log.Fatal(err)
		return err
	}

	return nil

}
<|endoftext|>


<commit_before>package migrations

import "github.com/BurntSushi/migration"

func AddTTLToContainers(tx migration.LimitedTx) error {
	_, err := tx.Exec(`
		ALTER TABLE containers ADD COLUMN ttl text;
	`)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>make ttl not-nullable; default to 0<commit_after>package migrations

import "github.com/BurntSushi/migration"

func AddTTLToContainers(tx migration.LimitedTx) error {
	_, err := tx.Exec(`
		ALTER TABLE containers ADD COLUMN ttl text NOT NULL DEFAULT 0;
	`)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"net/http"

	"github.com/gin-gonic/gin"
)

// Version of IronFunctions
var Version = "0.1.25"

func handleVersion(c *gin.Context) {
	c.JSON(http.StatusOK, gin.H{"version": Version})
}
<commit_msg>functions: 0.1.26 release [skip ci]<commit_after>package server

import (
	"net/http"

	"github.com/gin-gonic/gin"
)

// Version of IronFunctions
var Version = "0.1.26"

func handleVersion(c *gin.Context) {
	c.JSON(http.StatusOK, gin.H{"version": Version})
}
<|endoftext|>