<commit_before>package cmd

import (
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/go-errors/errors"
	"github.com/mitchellh/mapstructure"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/server"
	"github.com/privacybydesign/irmago/server/requestorserver"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cast"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var logger = server.NewLogger(0, false, false)
var conf *requestorserver.Configuration

var RootCommand = &cobra.Command{
	Use:   "irmad",
	Short: "IRMA server for verifying and issuing attributes",
	Run: func(command *cobra.Command, args []string) {
		if err := configure(command); err != nil {
			die(errors.WrapPrefix(err, "Failed to read configuration", 0))
		}
		serv, err := requestorserver.New(conf)
		if err != nil {
			die(errors.WrapPrefix(err, "Failed to configure server", 0))
		}

		stopped := make(chan struct{})
		interrupt := make(chan os.Signal, 1)
		signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)

		go func() {
			if err := serv.Start(conf); err != nil {
				die(errors.WrapPrefix(err, "Failed to start server", 0))
			}
			conf.Logger.Debug("Server stopped")
			stopped <- struct{}{}
		}()

		for {
			select {
			case <-interrupt:
				conf.Logger.Debug("Caught interrupt")
				serv.Stop() // causes serv.Start() above to return
				conf.Logger.Debug("Sent stop signal to server")
			case <-stopped:
				conf.Logger.Info("Exiting")
				close(stopped)
				close(interrupt)
				return
			}
		}
	},
}

func init() {
	if err := setFlags(RootCommand, productionMode()); err != nil {
		die(errors.WrapPrefix(err, "Failed to attach flags to "+RootCommand.Name()+" command", 0))
	}
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the RootCommand.
func Execute() {
	if err := RootCommand.Execute(); err != nil {
		die(errors.Wrap(err, 0))
	}
}

func die(err *errors.Error) {
	msg := err.Error()
	if logger.IsLevelEnabled(logrus.DebugLevel) {
		msg += "\nStack trace:\n" + string(err.Stack())
	}
	logger.Fatal(msg)
}

func setFlags(cmd *cobra.Command, production bool) error {
	flags := cmd.Flags()
	flags.SortFlags = false

	var defaulturl string
	var err error
	if !production {
		defaulturl, err = server.LocalIP()
		if err != nil {
			logger.Warn("Could not determine local IP address: ", err.Error())
		} else {
			defaulturl = "http://" + defaulturl + ":port"
		}
	}

	schemespath := server.DefaultSchemesPath()

	flags.StringP("config", "c", "", "path to configuration file")
	flags.StringP("schemes-path", "s", schemespath, "path to irma_configuration")
	flags.String("schemes-assets-path", "", "if specified, copy schemes from here into --schemes-path")
	flags.Int("schemes-update", 60, "update IRMA schemes every x minutes (0 to disable)")
	flags.Bool("disable-schemes-update", false, "disable IRMA scheme updating")
	flags.StringP("privkeys", "k", "", "path to IRMA private keys")
	flags.String("static-path", "", "Host files under this path as static files (leave empty to disable)")
	flags.String("static-prefix", "/", "Host static files under this URL prefix")
	flags.StringP("url", "u", defaulturl, "external URL to server to which the IRMA client connects")
	flags.Bool("sse", false, "Enable server sent for status updates (experimental)")

	flags.IntP("port", "p", 8088, "port at which to listen")
	flags.StringP("listen-addr", "l", "", "address at which to listen (default 0.0.0.0)")
	flags.Int("client-port", 0, "if specified, start a separate server for the IRMA app at this port")
	flags.String("client-listen-addr", "", "address at which server for IRMA app listens")
	flags.Lookup("port").Header = `Server address and port to listen on`

	flags.Bool("no-auth", !production, "whether or not to authenticate requestors (and reject all authenticated requests)")
	flags.String("requestors", "", "requestor configuration (in JSON)")
	flags.StringSlice("disclose-perms", nil, "list of attributes that all requestors may verify (default *)")
	flags.StringSlice("sign-perms", nil, "list of attributes that all requestors may request in signatures (default *)")
	issHelp := "list of attributes that all requestors may issue"
	if !production {
		issHelp += " (default *)"
	}
	flags.StringSlice("issue-perms", nil, issHelp)
	flags.String("static-sessions", "", "preconfigured static sessions (in JSON)")
	flags.Lookup("no-auth").Header = `Requestor authentication and default requestor permissions`

	flags.StringP("jwt-issuer", "j", "irmaserver", "JWT issuer")
	flags.String("jwt-privkey", "", "JWT private key")
	flags.String("jwt-privkey-file", "", "path to JWT private key")
	flags.Int("max-request-age", 300, "max age in seconds of a session request JWT")
	flags.Lookup("jwt-issuer").Header = `JWT configuration`

	flags.String("tls-cert", "", "TLS certificate (chain)")
	flags.String("tls-cert-file", "", "path to TLS certificate (chain)")
	flags.String("tls-privkey", "", "TLS private key")
	flags.String("tls-privkey-file", "", "path to TLS private key")
	flags.String("client-tls-cert", "", "TLS certificate (chain) for IRMA app server")
	flags.String("client-tls-cert-file", "", "path to TLS certificate (chain) for IRMA app server")
	flags.String("client-tls-privkey", "", "TLS private key for IRMA app server")
	flags.String("client-tls-privkey-file", "", "path to TLS private key for IRMA app server")
	flags.Bool("no-tls", false, "Disable TLS")
	flags.Lookup("tls-cert").Header = "TLS configuration (leave empty to disable TLS)"

	flags.StringP("email", "e", "", "Email address of server admin, for incidental notifications such as breaking API changes")
	flags.Bool("no-email", !production, "Opt out of prodiding an email address with --email")
	flags.Lookup("email").Header = "Email address (see README for more info)"

	flags.CountP("verbose", "v", "verbose (repeatable)")
	flags.BoolP("quiet", "q", false, "quiet")
	flags.Bool("log-json", false, "Log in JSON format")
	flags.Bool("production", false, "Production mode")
	flags.Lookup("verbose").Header = `Other options`

	return nil
}

func configure(cmd *cobra.Command) error {
	dashReplacer := strings.NewReplacer("-", "_")
	viper.SetEnvKeyReplacer(dashReplacer)
	viper.SetFileKeyReplacer(dashReplacer)
	viper.SetEnvPrefix("IRMASERVER")
	viper.AutomaticEnv()
	if err := viper.BindPFlags(cmd.Flags()); err != nil {
		return err
	}

	// Locate and read configuration file
	confpath := viper.GetString("config")
	if confpath != "" {
		dir, file := filepath.Dir(confpath), filepath.Base(confpath)
		viper.SetConfigName(strings.TrimSuffix(file, filepath.Ext(file)))
		viper.AddConfigPath(dir)
	} else {
		viper.SetConfigName("irmaserver")
		viper.AddConfigPath(".")
		viper.AddConfigPath("/etc/irmaserver/")
		viper.AddConfigPath("$HOME/.irmaserver")
	}
	err := viper.ReadInConfig() // Hold error checking until we know how much of it to log

	// Create our logger instance
	logger = server.NewLogger(viper.GetInt("verbose"), viper.GetBool("quiet"), viper.GetBool("log-json"))

	// First log output: hello, development or production mode, log level
	mode := "development"
	if viper.GetBool("production") {
		mode = "production"
	}
	logger.WithFields(logrus.Fields{
		"version":   irma.Version,
		"mode":      mode,
		"verbosity": server.Verbosity(viper.GetInt("verbose")),
	}).Info("irma server running")

	// Now we finally examine and log any error from viper.ReadInConfig()
	if err != nil {
		if _, notfound := err.(viper.ConfigFileNotFoundError); notfound {
			logger.Info("No configuration file found")
		} else {
			die(errors.WrapPrefix(err, "Failed to unmarshal configuration file at "+viper.ConfigFileUsed(), 0))
		}
	} else {
		logger.Info("Config file: ", viper.ConfigFileUsed())
	}

	// Read configuration from flags and/or environmental variables
	conf = &requestorserver.Configuration{
		Configuration: &server.Configuration{
			SchemesPath:           viper.GetString("schemes-path"),
			SchemesAssetsPath:     viper.GetString("schemes-assets-path"),
			SchemesUpdateInterval: viper.GetInt("schemes-update"),
			DisableSchemesUpdate:  viper.GetBool("disable-schemes-update") || viper.GetInt("schemes-update") == 0,
			IssuerPrivateKeysPath: viper.GetString("privkeys"),
			URL:        viper.GetString("url"),
			DisableTLS: viper.GetBool("no-tls"),
			Email:      viper.GetString("email"),
			EnableSSE:  viper.GetBool("sse"),
			Verbose:    viper.GetInt("verbose"),
			Quiet:      viper.GetBool("quiet"),
			LogJSON:    viper.GetBool("log-json"),
			Logger:     logger,
			Production: viper.GetBool("production"),
		},
		Permissions: requestorserver.Permissions{
			Disclosing: handlePermission("disclose-perms"),
			Signing:    handlePermission("sign-perms"),
			Issuing:    handlePermission("issue-perms"),
		},
		ListenAddress:                  viper.GetString("listen-addr"),
		Port:                           viper.GetInt("port"),
		ClientListenAddress:            viper.GetString("client-listen-addr"),
		ClientPort:                     viper.GetInt("client-port"),
		DisableRequestorAuthentication: viper.GetBool("no-auth"),
		Requestors:                     make(map[string]requestorserver.Requestor),
		JwtIssuer:                      viper.GetString("jwt-issuer"),
		JwtPrivateKey:                  viper.GetString("jwt-privkey"),
		JwtPrivateKeyFile:              viper.GetString("jwt-privkey-file"),
		MaxRequestAge:                  viper.GetInt("max-request-age"),
		StaticPath:                     viper.GetString("static-path"),
		StaticPrefix:                   viper.GetString("static-prefix"),

		TlsCertificate:           viper.GetString("tls-cert"),
		TlsCertificateFile:       viper.GetString("tls-cert-file"),
		TlsPrivateKey:            viper.GetString("tls-privkey"),
		TlsPrivateKeyFile:        viper.GetString("tls-privkey-file"),
		ClientTlsCertificate:     viper.GetString("client-tls-cert"),
		ClientTlsCertificateFile: viper.GetString("client-tls-cert-file"),
		ClientTlsPrivateKey:      viper.GetString("client-tls-privkey"),
		ClientTlsPrivateKeyFile:  viper.GetString("client-tls-privkey-file"),
	}

	if conf.Production {
		if !viper.GetBool("no-email") && conf.Email == "" {
			return errors.New("In production mode it is required to specify either an email address with the --email flag, or explicitly opting out with --no-email. See help or README for more info.")
		}
		if viper.GetBool("no-email") && conf.Email != "" {
			return errors.New("--no-email cannot be combined with --email")
		}
	}

	// Handle requestors
	var requestors map[string]interface{}
	if val, flagOrEnv := viper.Get("requestors").(string); !flagOrEnv || val != "" {
		if requestors, err = cast.ToStringMapE(viper.Get("requestors")); err != nil {
			return errors.WrapPrefix(err, "Failed to unmarshal requestors from flag or env var", 0)
		}
	}
	if len(requestors) > 0 {
		if err := mapstructure.Decode(requestors, &conf.Requestors); err != nil {
			return errors.WrapPrefix(err, "Failed to unmarshal requestors from config file", 0)
		}
	}

	if err = handleMapOrString("static-sessions", &conf.StaticSessions); err != nil {
		return err
	}

	logger.Debug("Done configuring")

	return nil
}

func handleMapOrString(key string, dest interface{}) error {
	var m map[string]interface{}
	var err error
	if val, flagOrEnv := viper.Get(key).(string); !flagOrEnv || val != "" {
		if m, err = cast.ToStringMapE(viper.Get(key)); err != nil {
			return errors.WrapPrefix(err, "Failed to unmarshal "+key+" from flag or env var", 0)
		}
	}
	if len(m) == 0 {
		return nil
	}
	if err := mapstructure.Decode(m, dest); err != nil {
		return errors.WrapPrefix(err, "Failed to unmarshal "+key+" from config file", 0)
	}
	return nil
}

func handlePermission(typ string) []string {
	if !viper.IsSet(typ) && (!viper.GetBool("production") || typ != "issue-perms") {
		return []string{"*"}
	}
	perms := viper.GetStringSlice(typ)
	if perms == nil {
		return []string{}
	}
	return perms
}

// productionMode examines the arguments passed to the executably to see if --production is enabled.
// (This should really be done using viper, but when the help message is printed, viper is not yet
// initialized.)
func productionMode() bool {
	for i, arg := range os.Args {
		if arg == "--production" {
			if len(os.Args) == i+1 || strings.HasPrefix(os.Args[i+1], "--") {
				return true
			}
			if checkConfVal(os.Args[i+1]) {
				return true
			}
		}
	}

	return checkConfVal(os.Getenv("IRMASERVER_PRODUCTION"))
}
func checkConfVal(val string) bool {
	lc := strings.ToLower(val)
	return lc == "1" || lc == "true" || lc == "yes" || lc == "t"
}
<commit_msg>fix: bug in server configuration defaults when enabling production mode through config file<commit_after>package cmd

import (
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/go-errors/errors"
	"github.com/mitchellh/mapstructure"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/server"
	"github.com/privacybydesign/irmago/server/requestorserver"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cast"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var logger = server.NewLogger(0, false, false)
var conf *requestorserver.Configuration

var RootCommand = &cobra.Command{
	Use:   "irmad",
	Short: "IRMA server for verifying and issuing attributes",
	Run: func(command *cobra.Command, args []string) {
		if err := configure(command); err != nil {
			die(errors.WrapPrefix(err, "Failed to read configuration", 0))
		}
		serv, err := requestorserver.New(conf)
		if err != nil {
			die(errors.WrapPrefix(err, "Failed to configure server", 0))
		}

		stopped := make(chan struct{})
		interrupt := make(chan os.Signal, 1)
		signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)

		go func() {
			if err := serv.Start(conf); err != nil {
				die(errors.WrapPrefix(err, "Failed to start server", 0))
			}
			conf.Logger.Debug("Server stopped")
			stopped <- struct{}{}
		}()

		for {
			select {
			case <-interrupt:
				conf.Logger.Debug("Caught interrupt")
				serv.Stop() // causes serv.Start() above to return
				conf.Logger.Debug("Sent stop signal to server")
			case <-stopped:
				conf.Logger.Info("Exiting")
				close(stopped)
				close(interrupt)
				return
			}
		}
	},
}

func init() {
	if err := setFlags(RootCommand, productionMode()); err != nil {
		die(errors.WrapPrefix(err, "Failed to attach flags to "+RootCommand.Name()+" command", 0))
	}
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the RootCommand.
func Execute() {
	if err := RootCommand.Execute(); err != nil {
		die(errors.Wrap(err, 0))
	}
}

func die(err *errors.Error) {
	msg := err.Error()
	if logger.IsLevelEnabled(logrus.DebugLevel) {
		msg += "\nStack trace:\n" + string(err.Stack())
	}
	logger.Fatal(msg)
}

func setFlags(cmd *cobra.Command, production bool) error {
	flags := cmd.Flags()
	flags.SortFlags = false

	var defaulturl string
	var err error
	if !production {
		defaulturl, err = server.LocalIP()
		if err != nil {
			logger.Warn("Could not determine local IP address: ", err.Error())
		} else {
			defaulturl = "http://" + defaulturl + ":port"
		}
	}

	schemespath := server.DefaultSchemesPath()

	flags.StringP("config", "c", "", "path to configuration file")
	flags.StringP("schemes-path", "s", schemespath, "path to irma_configuration")
	flags.String("schemes-assets-path", "", "if specified, copy schemes from here into --schemes-path")
	flags.Int("schemes-update", 60, "update IRMA schemes every x minutes (0 to disable)")
	flags.Bool("disable-schemes-update", false, "disable IRMA scheme updating")
	flags.StringP("privkeys", "k", "", "path to IRMA private keys")
	flags.String("static-path", "", "Host files under this path as static files (leave empty to disable)")
	flags.String("static-prefix", "/", "Host static files under this URL prefix")
	flags.StringP("url", "u", defaulturl, "external URL to server to which the IRMA client connects")
	flags.Bool("sse", false, "Enable server sent for status updates (experimental)")

	flags.IntP("port", "p", 8088, "port at which to listen")
	flags.StringP("listen-addr", "l", "", "address at which to listen (default 0.0.0.0)")
	flags.Int("client-port", 0, "if specified, start a separate server for the IRMA app at this port")
	flags.String("client-listen-addr", "", "address at which server for IRMA app listens")
	flags.Lookup("port").Header = `Server address and port to listen on`

	flags.Bool("no-auth", !production, "whether or not to authenticate requestors (and reject all authenticated requests)")
	flags.String("requestors", "", "requestor configuration (in JSON)")
	flags.StringSlice("disclose-perms", nil, "list of attributes that all requestors may verify (default *)")
	flags.StringSlice("sign-perms", nil, "list of attributes that all requestors may request in signatures (default *)")
	issHelp := "list of attributes that all requestors may issue"
	if !production {
		issHelp += " (default *)"
	}
	flags.StringSlice("issue-perms", nil, issHelp)
	flags.String("static-sessions", "", "preconfigured static sessions (in JSON)")
	flags.Lookup("no-auth").Header = `Requestor authentication and default requestor permissions`

	flags.StringP("jwt-issuer", "j", "irmaserver", "JWT issuer")
	flags.String("jwt-privkey", "", "JWT private key")
	flags.String("jwt-privkey-file", "", "path to JWT private key")
	flags.Int("max-request-age", 300, "max age in seconds of a session request JWT")
	flags.Lookup("jwt-issuer").Header = `JWT configuration`

	flags.String("tls-cert", "", "TLS certificate (chain)")
	flags.String("tls-cert-file", "", "path to TLS certificate (chain)")
	flags.String("tls-privkey", "", "TLS private key")
	flags.String("tls-privkey-file", "", "path to TLS private key")
	flags.String("client-tls-cert", "", "TLS certificate (chain) for IRMA app server")
	flags.String("client-tls-cert-file", "", "path to TLS certificate (chain) for IRMA app server")
	flags.String("client-tls-privkey", "", "TLS private key for IRMA app server")
	flags.String("client-tls-privkey-file", "", "path to TLS private key for IRMA app server")
	flags.Bool("no-tls", false, "Disable TLS")
	flags.Lookup("tls-cert").Header = "TLS configuration (leave empty to disable TLS)"

	flags.StringP("email", "e", "", "Email address of server admin, for incidental notifications such as breaking API changes")
	flags.Bool("no-email", !production, "Opt out of prodiding an email address with --email")
	flags.Lookup("email").Header = "Email address (see README for more info)"

	flags.CountP("verbose", "v", "verbose (repeatable)")
	flags.BoolP("quiet", "q", false, "quiet")
	flags.Bool("log-json", false, "Log in JSON format")
	flags.Bool("production", false, "Production mode")
	flags.Lookup("verbose").Header = `Other options`

	return nil
}

func configure(cmd *cobra.Command) error {
	dashReplacer := strings.NewReplacer("-", "_")
	viper.SetEnvKeyReplacer(dashReplacer)
	viper.SetFileKeyReplacer(dashReplacer)
	viper.SetEnvPrefix("IRMASERVER")
	viper.AutomaticEnv()
	if err := viper.BindPFlags(cmd.Flags()); err != nil {
		return err
	}

	// Locate and read configuration file
	confpath := viper.GetString("config")
	if confpath != "" {
		dir, file := filepath.Dir(confpath), filepath.Base(confpath)
		viper.SetConfigName(strings.TrimSuffix(file, filepath.Ext(file)))
		viper.AddConfigPath(dir)
	} else {
		viper.SetConfigName("irmaserver")
		viper.AddConfigPath(".")
		viper.AddConfigPath("/etc/irmaserver/")
		viper.AddConfigPath("$HOME/.irmaserver")
	}
	err := viper.ReadInConfig() // Hold error checking until we know how much of it to log

	// Create our logger instance
	logger = server.NewLogger(viper.GetInt("verbose"), viper.GetBool("quiet"), viper.GetBool("log-json"))

	// First log output: hello, development or production mode, log level
	mode := "development"
	if viper.GetBool("production") {
		mode = "production"
		viper.SetDefault("no-auth", false)
		viper.SetDefault("no-email", false)
		viper.SetDefault("url", "")
	}
	logger.WithFields(logrus.Fields{
		"version":   irma.Version,
		"mode":      mode,
		"verbosity": server.Verbosity(viper.GetInt("verbose")),
	}).Info("irma server running")

	// Now we finally examine and log any error from viper.ReadInConfig()
	if err != nil {
		if _, notfound := err.(viper.ConfigFileNotFoundError); notfound {
			logger.Info("No configuration file found")
		} else {
			die(errors.WrapPrefix(err, "Failed to unmarshal configuration file at "+viper.ConfigFileUsed(), 0))
		}
	} else {
		logger.Info("Config file: ", viper.ConfigFileUsed())
	}

	// Read configuration from flags and/or environmental variables
	conf = &requestorserver.Configuration{
		Configuration: &server.Configuration{
			SchemesPath:           viper.GetString("schemes-path"),
			SchemesAssetsPath:     viper.GetString("schemes-assets-path"),
			SchemesUpdateInterval: viper.GetInt("schemes-update"),
			DisableSchemesUpdate:  viper.GetBool("disable-schemes-update") || viper.GetInt("schemes-update") == 0,
			IssuerPrivateKeysPath: viper.GetString("privkeys"),
			URL:        viper.GetString("url"),
			DisableTLS: viper.GetBool("no-tls"),
			Email:      viper.GetString("email"),
			EnableSSE:  viper.GetBool("sse"),
			Verbose:    viper.GetInt("verbose"),
			Quiet:      viper.GetBool("quiet"),
			LogJSON:    viper.GetBool("log-json"),
			Logger:     logger,
			Production: viper.GetBool("production"),
		},
		Permissions: requestorserver.Permissions{
			Disclosing: handlePermission("disclose-perms"),
			Signing:    handlePermission("sign-perms"),
			Issuing:    handlePermission("issue-perms"),
		},
		ListenAddress:                  viper.GetString("listen-addr"),
		Port:                           viper.GetInt("port"),
		ClientListenAddress:            viper.GetString("client-listen-addr"),
		ClientPort:                     viper.GetInt("client-port"),
		DisableRequestorAuthentication: viper.GetBool("no-auth"),
		Requestors:                     make(map[string]requestorserver.Requestor),
		JwtIssuer:                      viper.GetString("jwt-issuer"),
		JwtPrivateKey:                  viper.GetString("jwt-privkey"),
		JwtPrivateKeyFile:              viper.GetString("jwt-privkey-file"),
		MaxRequestAge:                  viper.GetInt("max-request-age"),
		StaticPath:                     viper.GetString("static-path"),
		StaticPrefix:                   viper.GetString("static-prefix"),

		TlsCertificate:           viper.GetString("tls-cert"),
		TlsCertificateFile:       viper.GetString("tls-cert-file"),
		TlsPrivateKey:            viper.GetString("tls-privkey"),
		TlsPrivateKeyFile:        viper.GetString("tls-privkey-file"),
		ClientTlsCertificate:     viper.GetString("client-tls-cert"),
		ClientTlsCertificateFile: viper.GetString("client-tls-cert-file"),
		ClientTlsPrivateKey:      viper.GetString("client-tls-privkey"),
		ClientTlsPrivateKeyFile:  viper.GetString("client-tls-privkey-file"),
	}

	if conf.Production {
		if !viper.GetBool("no-email") && conf.Email == "" {
			return errors.New("In production mode it is required to specify either an email address with the --email flag, or explicitly opting out with --no-email. See help or README for more info.")
		}
		if viper.GetBool("no-email") && conf.Email != "" {
			return errors.New("--no-email cannot be combined with --email")
		}
	}

	// Handle requestors
	var requestors map[string]interface{}
	if val, flagOrEnv := viper.Get("requestors").(string); !flagOrEnv || val != "" {
		if requestors, err = cast.ToStringMapE(viper.Get("requestors")); err != nil {
			return errors.WrapPrefix(err, "Failed to unmarshal requestors from flag or env var", 0)
		}
	}
	if len(requestors) > 0 {
		if err := mapstructure.Decode(requestors, &conf.Requestors); err != nil {
			return errors.WrapPrefix(err, "Failed to unmarshal requestors from config file", 0)
		}
	}

	if err = handleMapOrString("static-sessions", &conf.StaticSessions); err != nil {
		return err
	}

	logger.Debug("Done configuring")

	return nil
}

func handleMapOrString(key string, dest interface{}) error {
	var m map[string]interface{}
	var err error
	if val, flagOrEnv := viper.Get(key).(string); !flagOrEnv || val != "" {
		if m, err = cast.ToStringMapE(viper.Get(key)); err != nil {
			return errors.WrapPrefix(err, "Failed to unmarshal "+key+" from flag or env var", 0)
		}
	}
	if len(m) == 0 {
		return nil
	}
	if err := mapstructure.Decode(m, dest); err != nil {
		return errors.WrapPrefix(err, "Failed to unmarshal "+key+" from config file", 0)
	}
	return nil
}

func handlePermission(typ string) []string {
	if !viper.IsSet(typ) && (!viper.GetBool("production") || typ != "issue-perms") {
		return []string{"*"}
	}
	perms := viper.GetStringSlice(typ)
	if perms == nil {
		return []string{}
	}
	return perms
}

// productionMode examines the arguments passed to the executably to see if --production is enabled.
// (This should really be done using viper, but when the help message is printed, viper is not yet
// initialized.)
func productionMode() bool {
	for i, arg := range os.Args {
		if arg == "--production" {
			if len(os.Args) == i+1 || strings.HasPrefix(os.Args[i+1], "-") {
				return true
			}
			if checkConfVal(os.Args[i+1]) {
				return true
			}
		}
	}

	return checkConfVal(os.Getenv("IRMASERVER_PRODUCTION"))
}
func checkConfVal(val string) bool {
	lc := strings.ToLower(val)
	return lc == "1" || lc == "true" || lc == "yes" || lc == "t"
}
<|endoftext|>


<commit_before>package conn_test

import (
	"github.com/jordwest/imap-server/conn"
	. "github.com/onsi/ginkgo"
)

var _ = Describe("LOGOUT Command", func() {
	Context("When logged in", func() {
		BeforeEach(func() {
			tConn.SetState(conn.StateAuthenticated)
			tConn.User = mStore.User
		})

		It("should log the user out", func() {
			SendLine("abcd.123 LOGOUT")
			ExpectResponse("* BYE IMAP4rev1 server logging out")
			ExpectResponse("abcd.123 OK LOGOUT completed")
		})
	})

	Context("When not logged in", func() {
		BeforeEach(func() {
			tConn.SetState(conn.StateNotAuthenticated)
		})

		It("should give an error", func() {
			SendLine("abcd.123 LOGOUT")
			ExpectResponse("abcd.123 NO not logged in")
		})
	})
})
<commit_msg>LOGOUT command should still work when not authenticated<commit_after>package conn_test

import (
	"github.com/jordwest/imap-server/conn"
	. "github.com/onsi/ginkgo"
)

var _ = Describe("LOGOUT Command", func() {
	Context("When logged in", func() {
		BeforeEach(func() {
			tConn.SetState(conn.StateAuthenticated)
			tConn.User = mStore.User
		})

		It("should log the user out", func() {
			SendLine("abcd.123 LOGOUT")
			ExpectResponse("* BYE IMAP4rev1 server logging out")
			ExpectResponse("abcd.123 OK LOGOUT completed")
		})
	})

	Context("When not logged in", func() {
		BeforeEach(func() {
			tConn.SetState(conn.StateNotAuthenticated)
		})

		It("should give an error", func() {
			SendLine("abcd.123 LOGOUT")
			ExpectResponse("* BYE IMAP4rev1 server logging out")
			ExpectResponse("abcd.123 OK LOGOUT completed")
		})
	})
})
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	fmt.Println("Hello whycc")
}<commit_msg>Ok first simple parsing is working.<commit_after>package main

import (
	"fmt"
	"encoding/csv"
	"os"
	"bufio"
	"io"
	"log"
)

func main() {
	f,_ := os.Open("./testdata/Umsatzanzeige_1234567890_20160410.csv")
	r := csv.NewReader(bufio.NewReader(f))
	r.Comma =';'
	r.FieldsPerRecord = -1
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		if len(record) != 9 || record[0] == "Buchung" {
			continue
		}

		fmt.Println(record)
	}
}

<|endoftext|>


<commit_before>package config_service

import (
	"strings"
	cfgsvc "github.com/Flipkart/config-service/client-go"
	"github.com/kelseyhightower/confd/log"
	"errors"
	"reflect"
	"github.com/pquerna/ffjson/ffjson"
	"strconv"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *cfgsvc.ConfigServiceClient
}

type BucketListener struct{
	watchResp chan *watchResponse
	currentIndex uint64
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (this *BucketListener) Connected(bucketName string) {
	log.Info("Connected! " + bucketName)
}

func (this *BucketListener) Disconnected(bucketName string, err error) {
	log.Info("Disconnected! " + bucketName)
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex, err: err}
}

func (this *BucketListener) Deleted(bucketName string) {
	log.Info("deleted " + bucketName)
	this.watchResp <- &watchResponse{waitIndex: 0, err: errors.New(bucketName + " was deleted")}
}

func (this *BucketListener) Updated(oldBucket *cfgsvc.Bucket, newBucket *cfgsvc.Bucket) {
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex+1, err: nil}
}

func NewConfigClient(machines []string) (*Client, error) {
	c, err := cfgsvc.NewConfigServiceClient(machines[0], 50) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		bucketsKey := strings.Split(strings.TrimPrefix(v, "/"), "/")
		buckets := strings.Split(bucketsKey[0], ",")
		key := bucketsKey[1]

		dynamicBuckets, err := c.getDynamicBuckets(buckets)
		if err != nil {
			return vars, err
		}

		for _, dynamicBucket := range dynamicBuckets {
			val := dynamicBucket.GetKeys()[key]
			valType := reflect.TypeOf(val).Kind()
			if valType == reflect.Slice {
				data, err := ffjson.Marshal(val)
				if err != nil {
				    log.Error("Failed decoding from JSON")
				} else {
					vars[key] = string(data[:])
				}
			} else {
				switch val.(type) {
					case int,int64:
					vars[key] = strconv.FormatInt(val.(int64), 64)
					case string:
					vars[key] = val.(string)
					case bool:
					vars[key] = strconv.FormatBool(val.(bool))
					case float32,float64:
					vars[key] = strconv.FormatFloat(val.(float64), 'f', -1, 64)
				}
			}
		}

	}
	return vars, nil
}

func (c *Client) getDynamicBuckets(buckets []string) ([]*cfgsvc.DynamicBucket, error) {
	var dynamicBuckets []*cfgsvc.DynamicBucket
	for _, bucket := range buckets {
		bucketName := strings.TrimSpace(bucket)
		dynamicBucket, err := c.client.GetDynamicBucket(bucketName)
		if err != nil {
			return dynamicBuckets, err
		}
		dynamicBuckets = append(dynamicBuckets, dynamicBucket)
	}
	return dynamicBuckets, nil
}

func setupDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.AddListeners(bucketListener)
	}
}

func removeDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.RemoveListeners(bucketListener)
	}
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	prefix = strings.TrimPrefix(prefix, "/")
	prefixes := strings.Split(prefix, ",")
	dynamicBuckets, err := c.getDynamicBuckets(prefixes)
	if err != nil {
		return waitIndex, err
	}

	if waitIndex == 0 {
		return waitIndex+1, nil
	}  else {
		watchResp := make(chan *watchResponse)
		bucketListener := &BucketListener{watchResp: watchResp, currentIndex: waitIndex}
		setupDynamicBucketListeners(dynamicBuckets, bucketListener)
		select {
			case watchResp := <- watchResp:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
		 		return watchResp.waitIndex, watchResp.err
		    case <-stopChan:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
				return 0, nil
		}
	}
}

<commit_msg>Skip the key if its nil<commit_after>package config_service

import (
	"strings"
	cfgsvc "github.com/Flipkart/config-service/client-go"
	"github.com/kelseyhightower/confd/log"
	"errors"
	"reflect"
	"github.com/pquerna/ffjson/ffjson"
	"strconv"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *cfgsvc.ConfigServiceClient
}

type BucketListener struct{
	watchResp chan *watchResponse
	currentIndex uint64
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (this *BucketListener) Connected(bucketName string) {
	log.Info("Connected! " + bucketName)
}

func (this *BucketListener) Disconnected(bucketName string, err error) {
	log.Info("Disconnected! " + bucketName)
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex, err: err}
}

func (this *BucketListener) Deleted(bucketName string) {
	log.Info("deleted " + bucketName)
	this.watchResp <- &watchResponse{waitIndex: 0, err: errors.New(bucketName + " was deleted")}
}

func (this *BucketListener) Updated(oldBucket *cfgsvc.Bucket, newBucket *cfgsvc.Bucket) {
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex+1, err: nil}
}

func NewConfigClient(machines []string) (*Client, error) {
	c, err := cfgsvc.NewConfigServiceClient(machines[0], 50) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		bucketsKey := strings.Split(strings.TrimPrefix(v, "/"), "/")
		buckets := strings.Split(bucketsKey[0], ",")
		key := bucketsKey[1]

		dynamicBuckets, err := c.getDynamicBuckets(buckets)
		if err != nil {
			return vars, err
		}

		for _, dynamicBucket := range dynamicBuckets {
			val := dynamicBucket.GetKeys()[key]
			if val == nil {
				continue;
			}
			valType := reflect.TypeOf(val).Kind()
			if valType == reflect.Slice {
				data, err := ffjson.Marshal(val)
				if err != nil {
				    log.Error("Failed decoding from JSON")
				} else {
					vars[key] = string(data[:])
				}
			} else {
				switch val.(type) {
					case int,int64:
					vars[key] = strconv.FormatInt(val.(int64), 64)
					case string:
					vars[key] = val.(string)
					case bool:
					vars[key] = strconv.FormatBool(val.(bool))
					case float32,float64:
					vars[key] = strconv.FormatFloat(val.(float64), 'f', -1, 64)
				}
			}
		}

	}
	return vars, nil
}

func (c *Client) getDynamicBuckets(buckets []string) ([]*cfgsvc.DynamicBucket, error) {
	var dynamicBuckets []*cfgsvc.DynamicBucket
	for _, bucket := range buckets {
		bucketName := strings.TrimSpace(bucket)
		dynamicBucket, err := c.client.GetDynamicBucket(bucketName)
		if err != nil {
			return dynamicBuckets, err
		}
		dynamicBuckets = append(dynamicBuckets, dynamicBucket)
	}
	return dynamicBuckets, nil
}

func setupDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.AddListeners(bucketListener)
	}
}

func removeDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.RemoveListeners(bucketListener)
	}
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	prefix = strings.TrimPrefix(prefix, "/")
	prefixes := strings.Split(prefix, ",")
	dynamicBuckets, err := c.getDynamicBuckets(prefixes)
	if err != nil {
		return waitIndex, err
	}

	if waitIndex == 0 {
		return waitIndex+1, nil
	}  else {
		watchResp := make(chan *watchResponse)
		bucketListener := &BucketListener{watchResp: watchResp, currentIndex: waitIndex}
		setupDynamicBucketListeners(dynamicBuckets, bucketListener)
		select {
			case watchResp := <- watchResp:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
		 		return watchResp.waitIndex, watchResp.err
		    case <-stopChan:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
				return 0, nil
		}
	}
}

<|endoftext|>


<commit_before>package endly

import "github.com/viant/toolbox/url"

//ProcessStartRequest represents a start request
type ProcessStartRequest struct {
	Name          string
	Target        *url.Resource
	Options       *ExecutionOptions
	SystemService bool
	Directory     string
	Command       string
	Arguments     []string
}

//ProcessStartResponse represents a start response
type ProcessStartResponse struct {
	Command string
	Info    []*ProcessInfo
}
<commit_msg>udpated<commit_after>package endly

import "github.com/viant/toolbox/url"

//ProcessStartRequest represents a start request
type ProcessStartRequest struct {
	Name          string
	Target        *url.Resource
	Options       *ExecutionOptions
	Directory     string
	Arguments     []string
	ImmuneToHangups bool //start process as nohup
}

//ProcessStartResponse represents a start response
type ProcessStartResponse struct {
	Command string
	Info    []*ProcessInfo
}
<|endoftext|>


<commit_before>// (C) 2016 by Ricardo Branco
//
// MIT License
//
// v0.4
//
// TODO:
// + Read filenames from file
// + Support -c option like md5sum(1)
// + Use getopt
// + Use different output formats for display

package main

import (
	"crypto"
	"crypto/hmac"
	/*
		_ "crypto/md5"
		_ "crypto/sha1"
		_ "crypto/sha256"
		_ "crypto/sha512"
	*/
	"encoding/hex"
	"flag"
	"fmt"
	"github.com/ricardobranco777/dgst/blake2b"
	"github.com/ricardobranco777/dgst/blake2s"
	_ "github.com/ricardobranco777/dgst/md4"
	_ "github.com/ricardobranco777/dgst/md5"
	_ "github.com/ricardobranco777/dgst/ripemd160"
	_ "github.com/ricardobranco777/dgst/sha1"
	_ "github.com/ricardobranco777/dgst/sha256"
	_ "github.com/ricardobranco777/dgst/sha512"
	/*
		"golang.org/x/crypto/blake2b"
		"golang.org/x/crypto/blake2s"
		_ "golang.org/x/crypto/md4"
		_ "golang.org/x/crypto/ripemd160"
	*/
	_ "golang.org/x/crypto/sha3"
	"hash"
	"io"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

const (
	BLAKE2b256 = 100 + iota
	BLAKE2b384
	BLAKE2b512
	BLAKE2s256
)

var hashes = []*struct {
	hash crypto.Hash
	name string
	sum  string
	hash.Hash
	size int
}{
	{name: "BLAKE2b256",
		hash: BLAKE2b256,
		size: 32},
	{name: "BLAKE2b384",
		hash: BLAKE2b384,
		size: 48},
	{name: "BLAKE2b512",
		hash: BLAKE2b512,
		size: 64},
	{name: "BLAKE2s256",
		hash: BLAKE2s256,
		size: 32},
	{name: "MD4",
		hash: crypto.MD4,
		size: 16},
	{name: "MD5",
		hash: crypto.MD5,
		size: 16},
	{name: "RIPEMD160",
		hash: crypto.RIPEMD160,
		size: 20},
	{name: "SHA1",
		hash: crypto.SHA1,
		size: 20},
	{name: "SHA224",
		hash: crypto.SHA224,
		size: 28},
	{name: "SHA256",
		hash: crypto.SHA256,
		size: 32},
	{name: "SHA384",
		hash: crypto.SHA384,
		size: 48},
	{name: "SHA512",
		hash: crypto.SHA512,
		size: 64},
	{name: "SHA512-224",
		hash: crypto.SHA512_224,
		size: 28},
	{name: "SHA512-256",
		hash: crypto.SHA512_256,
		size: 32},
	{name: "SHA3-224",
		hash: crypto.SHA3_224,
		size: 28},
	{name: "SHA3-256",
		hash: crypto.SHA3_256,
		size: 32},
	{name: "SHA3-384",
		hash: crypto.SHA3_384,
		size: 48},
	{name: "SHA3-512",
		hash: crypto.SHA3_512,
		size: 64},
}

var progname string

var done chan error

func main() {
	progname = path.Base(os.Args[0])

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] [-s STRING...]|[FILE... DIRECTORY...]\n\n", progname)
		flag.PrintDefaults()
	}

	chosen := make(map[crypto.Hash]*bool)
	for i := range hashes {
		name := strings.ToLower(hashes[i].name)
		chosen[hashes[i].hash] = flag.Bool(name, false, fmt.Sprintf("%s algorithm", name))
	}

	var all, isString *bool
	all = flag.Bool("all", false, "all algorithms")
	isString = flag.Bool("s", false, "treat arguments as strings")

	var mac_key []byte
	var hexkey strFlag
	flag.Var(&hexkey, "key", "key for HMAC (in hexadecimal)")

	var size_hashes = map[int]*struct {
		hashes []crypto.Hash
		set    *bool
	}{
		128: {}, 160: {}, 224: {}, 256: {}, 384: {}, 512: {},
	}

	for h := range hashes {
		if hashes[h].hash.Available() {
			size_hashes[hashes[h].size*8].hashes = append(size_hashes[hashes[h].size*8].hashes, hashes[h].hash)
		}
	}

	for size := range size_hashes {
		sizeStr := strconv.Itoa(size)
		if len(size_hashes[size].hashes) > 0 {
			size_hashes[size].set = flag.Bool(sizeStr, false, "all "+sizeStr+" bits algorithms")
		}
	}

	flag.Parse()

	if hexkey.value != nil {
		key, err := hex.DecodeString(*hexkey.value)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %s: Invalid hexadecimal key: %v\n", progname, *hexkey.value)
			os.Exit(1)
		}
		mac_key = key
	}

	for size := range size_hashes {
		if !*(size_hashes[size].set) {
			continue
		}
		for _, h := range size_hashes[size].hashes {
			*chosen[h] = true
		}
	}

	if !(*all || choices(chosen)) {
		*all = true
	}

	for h := range chosen {
		if (*all && *chosen[h]) || !(*all || *chosen[h]) {
			removeHash(h)
		}
		delete(chosen, h)
	}

	for i := 0; ; i++ {
		if i >= len(hashes) {
			break
		}
		switch hashes[i].hash {
		case BLAKE2b256:
			hashes[i].Hash = blake2_(blake2b.New256, mac_key)
		case BLAKE2b384:
			hashes[i].Hash = blake2_(blake2b.New384, mac_key)
		case BLAKE2b512:
			hashes[i].Hash = blake2_(blake2b.New512, mac_key)
		case BLAKE2s256:
			hashes[i].Hash = blake2_(blake2s.New256, mac_key)
		default:
			if !hashes[i].hash.Available() {
				removeHash(hashes[i].hash)
				i--
				continue
			}
			if mac_key != nil {
				hashes[i].Hash = hmac.New(hashes[i].hash.New, mac_key)
			} else {
				hashes[i].Hash = hashes[i].hash.New()
			}
		}
	}

	if len(hashes) == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if flag.NArg() == 0 {
		hash_stdin()
		os.Exit(0)
	}

	var errors bool

	if *isString {
		for _, s := range flag.Args() {
			hash_string(s)
		}
	} else {
		done = make(chan error, len(hashes))
		defer close(done)

		for _, pathname := range flag.Args() {
			info, err := os.Stat(pathname)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
				errors = true
			} else if info.IsDir() {
				errors = hash_dir(pathname)
			} else {
				errors = hash_file(pathname)
			}
		}
	}

	if errors {
		os.Exit(1)
	}
}

type strFlag struct {
	value *string
}

func (f *strFlag) Set(value string) error {
	f.value = &value
	return nil
}

func (f *strFlag) String() string {
	if f.value != nil {
		return *f.value
	}
	return ""
}

func removeHash(h crypto.Hash) {
	i := -1
	for i = range hashes {
		if hashes[i].hash == h {
			break
		}
	}
	copy(hashes[i:], hashes[i+1:])
	hashes[len(hashes)-1] = nil // item will be garbage-collected
	hashes = hashes[:len(hashes)-1]
}

// Wrapper for the Blake2 New() methods that needs an optional for MAC
func blake2_(f func([]byte) (hash.Hash, error), key []byte) hash.Hash {
	h, err := f(key)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s: %v", progname, err)
		os.Exit(1)
	}
	return h
}

// Returns true if at least some algorithm was specified on the command line
func choices(chosen map[crypto.Hash]*bool) bool {
	for h := range chosen {
		if *chosen[h] {
			return true
		}
	}
	return false
}

func display() {
	for i := range hashes {
		fmt.Println(hashes[i].sum)
	}
}

func hash_string(str string) {
	var wg sync.WaitGroup
	wg.Add(len(hashes))
	for h := range hashes {
		go func(h int) {
			defer wg.Done()
			hashes[h].Write([]byte(str))
			hashes[h].sum = fmt.Sprintf("%s(\"%s\") = %x", hashes[h].name, str, hashes[h].Sum(nil))
			hashes[h].Reset()
		}(h)
	}
	wg.Wait()
	display()
}

func hash_file(filename string) (errors bool) {
	for h := range hashes {
		go func(h int) {
			f, err := os.Open(filename)
			if err != nil {
				done <- err
				return
			}
			defer f.Close()

			if _, err := io.Copy(hashes[h], f); err != nil {
				done <- err
				return
			}
			hashes[h].sum = fmt.Sprintf("%s(%s) = %x", hashes[h].name, filename, hashes[h].Sum(nil))
			hashes[h].Reset()
			done <- nil
		}(h)
	}
	for range hashes {
		err := <-done
		if err != nil {
			if !errors {
				fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
			}
			errors = true
		}
	}
	if !errors {
		display()
	}
	return
}

func visit(path string, f os.FileInfo, err error) error {
	if err != nil && !os.IsPermission(err) {
		return err
	}
	if f.Mode().IsRegular() {
		hash_file(path)
	}
	return nil
}

func hash_dir(dir string) bool {
	err := filepath.Walk(dir, visit)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
		return false
	}
	return true
}

func hash_stdin() (errors bool) {
	for h := range hashes {
		go func(h int) {
			if _, err := io.Copy(hashes[h], os.Stdin); err != nil {
				done <- err
				return
			}
			hashes[h].sum = fmt.Sprintf("%s() = %x", hashes[h].name, hashes[h].Sum(nil))
			hashes[h].Reset()
			done <- nil
		}(h)
	}
	for range hashes {
		err := <-done
		if err != nil {
			if !errors {
				fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
			}
			errors = true
		}
	}
	if !errors {
		display()
	}
	return
}
<commit_msg>Get pathnames from file (-i option). Added -0 option to handle nul-terminated lines<commit_after>// (C) 2016 by Ricardo Branco
//
// MIT License
//
// v0.5
//
// TODO:
// + Support -c option like md5sum(1)
// + Use different output formats for display

package main

import (
	"bufio"
	"crypto"
	"crypto/hmac"
	/*
		_ "crypto/md5"
		_ "crypto/sha1"
		_ "crypto/sha256"
		_ "crypto/sha512"
	*/
	"encoding/hex"
	"flag"
	"fmt"
	"github.com/ricardobranco777/dgst/blake2b"
	"github.com/ricardobranco777/dgst/blake2s"
	_ "github.com/ricardobranco777/dgst/md4"
	_ "github.com/ricardobranco777/dgst/md5"
	_ "github.com/ricardobranco777/dgst/ripemd160"
	_ "github.com/ricardobranco777/dgst/sha1"
	_ "github.com/ricardobranco777/dgst/sha256"
	_ "github.com/ricardobranco777/dgst/sha512"
	/*
		"golang.org/x/crypto/blake2b"
		"golang.org/x/crypto/blake2s"
		_ "golang.org/x/crypto/md4"
		_ "golang.org/x/crypto/ripemd160"
	*/
	_ "golang.org/x/crypto/sha3"
	"hash"
	"io"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
)

const (
	BLAKE2b256 = 100 + iota
	BLAKE2b384
	BLAKE2b512
	BLAKE2s256
)

var hashes = []*struct {
	hash crypto.Hash
	name string
	sum  string
	hash.Hash
	size int
}{
	{name: "BLAKE2b256",
		hash: BLAKE2b256,
		size: 32},
	{name: "BLAKE2b384",
		hash: BLAKE2b384,
		size: 48},
	{name: "BLAKE2b512",
		hash: BLAKE2b512,
		size: 64},
	{name: "BLAKE2s256",
		hash: BLAKE2s256,
		size: 32},
	{name: "MD4",
		hash: crypto.MD4,
		size: 16},
	{name: "MD5",
		hash: crypto.MD5,
		size: 16},
	{name: "RIPEMD160",
		hash: crypto.RIPEMD160,
		size: 20},
	{name: "SHA1",
		hash: crypto.SHA1,
		size: 20},
	{name: "SHA224",
		hash: crypto.SHA224,
		size: 28},
	{name: "SHA256",
		hash: crypto.SHA256,
		size: 32},
	{name: "SHA384",
		hash: crypto.SHA384,
		size: 48},
	{name: "SHA512",
		hash: crypto.SHA512,
		size: 64},
	{name: "SHA512-224",
		hash: crypto.SHA512_224,
		size: 28},
	{name: "SHA512-256",
		hash: crypto.SHA512_256,
		size: 32},
	{name: "SHA3-224",
		hash: crypto.SHA3_224,
		size: 28},
	{name: "SHA3-256",
		hash: crypto.SHA3_256,
		size: 32},
	{name: "SHA3-384",
		hash: crypto.SHA3_384,
		size: 48},
	{name: "SHA3-512",
		hash: crypto.SHA3_512,
		size: 64},
}

var progname string

var done chan error

var zeroFlag *bool

func main() {
	progname = path.Base(os.Args[0])

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] [-s STRING...]|[FILE... DIRECTORY...]\n\n", progname)
		flag.PrintDefaults()
	}

	chosen := make(map[crypto.Hash]*bool)
	for i := range hashes {
		name := strings.ToLower(hashes[i].name)
		chosen[hashes[i].hash] = flag.Bool(name, false, fmt.Sprintf("%s algorithm", name))
	}

	var all, isString *bool
	all = flag.Bool("all", false, "all algorithms")
	isString = flag.Bool("s", false, "treat arguments as strings")
	zeroFlag = flag.Bool("0", false, "lines are terminated by a null character by using the -c or -i options")

	var files strFlag
	flag.Var(&files, "i", "read pathnames from file")

	var mac_key []byte
	var hexkey strFlag
	flag.Var(&hexkey, "key", "key for HMAC (in hexadecimal)")

	var size_hashes = map[int]*struct {
		hashes []crypto.Hash
		set    *bool
	}{
		128: {}, 160: {}, 224: {}, 256: {}, 384: {}, 512: {},
	}

	for h := range hashes {
		if hashes[h].hash.Available() {
			size_hashes[hashes[h].size*8].hashes = append(size_hashes[hashes[h].size*8].hashes, hashes[h].hash)
		}
	}

	for size := range size_hashes {
		sizeStr := strconv.Itoa(size)
		if len(size_hashes[size].hashes) > 0 {
			size_hashes[size].set = flag.Bool(sizeStr, false, "all "+sizeStr+" bits algorithms")
		}
	}

	flag.Parse()

	if hexkey.value != nil {
		key, err := hex.DecodeString(*hexkey.value)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %s: Invalid hexadecimal key: %v\n", progname, *hexkey.value)
			os.Exit(1)
		}
		mac_key = key
	}

	for size := range size_hashes {
		if !*(size_hashes[size].set) {
			continue
		}
		for _, h := range size_hashes[size].hashes {
			*chosen[h] = true
		}
	}

	if !(*all || choices(chosen)) {
		*all = true
	}

	for h := range chosen {
		if (*all && *chosen[h]) || !(*all || *chosen[h]) {
			removeHash(h)
		}
		delete(chosen, h)
	}

	for i := 0; ; i++ {
		if i >= len(hashes) {
			break
		}
		switch hashes[i].hash {
		case BLAKE2b256:
			hashes[i].Hash = blake2_(blake2b.New256, mac_key)
		case BLAKE2b384:
			hashes[i].Hash = blake2_(blake2b.New384, mac_key)
		case BLAKE2b512:
			hashes[i].Hash = blake2_(blake2b.New512, mac_key)
		case BLAKE2s256:
			hashes[i].Hash = blake2_(blake2s.New256, mac_key)
		default:
			if !hashes[i].hash.Available() {
				removeHash(hashes[i].hash)
				i--
				continue
			}
			if mac_key != nil {
				hashes[i].Hash = hmac.New(hashes[i].hash.New, mac_key)
			} else {
				hashes[i].Hash = hashes[i].hash.New()
			}
		}
	}

	if len(hashes) == 0 {
		flag.Usage()
		os.Exit(1)
	}

	var errors bool

	if files.value != nil {
		done = make(chan error, len(hashes))
		defer close(done)

		errors = hash_fromFile(*files.value)
	} else if flag.NArg() == 0 {
		hash_stdin()
		os.Exit(0)
	}

	if *isString {
		for _, s := range flag.Args() {
			hash_string(s)
		}
	} else {
		done = make(chan error, len(hashes))
		defer close(done)

		for _, pathname := range flag.Args() {
			errors = hash_pathname(pathname)
		}
	}

	if errors {
		os.Exit(1)
	}
}

type strFlag struct {
	value *string
}

func (f *strFlag) Set(value string) error {
	f.value = &value
	return nil
}

func (f *strFlag) String() string {
	if f.value != nil {
		return *f.value
	}
	return ""
}

func removeHash(h crypto.Hash) {
	i := -1
	for i = range hashes {
		if hashes[i].hash == h {
			break
		}
	}
	copy(hashes[i:], hashes[i+1:])
	hashes[len(hashes)-1] = nil // item will be garbage-collected
	hashes = hashes[:len(hashes)-1]
}

// Wrapper for the Blake2 New() methods that needs an optional for MAC
func blake2_(f func([]byte) (hash.Hash, error), key []byte) hash.Hash {
	h, err := f(key)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s: %v", progname, err)
		os.Exit(1)
	}
	return h
}

// Returns true if at least some algorithm was specified on the command line
func choices(chosen map[crypto.Hash]*bool) bool {
	for h := range chosen {
		if *chosen[h] {
			return true
		}
	}
	return false
}

func display() {
	for i := range hashes {
		fmt.Println(hashes[i].sum)
	}
}

func hash_string(str string) {
	var wg sync.WaitGroup
	wg.Add(len(hashes))
	for h := range hashes {
		go func(h int) {
			defer wg.Done()
			hashes[h].Write([]byte(str))
			hashes[h].sum = fmt.Sprintf("%s(\"%s\") = %x", hashes[h].name, str, hashes[h].Sum(nil))
			hashes[h].Reset()
		}(h)
	}
	wg.Wait()
	display()
}

func hash_fromFile(file string) (errors bool) {
	var terminator string = "\n"
	if *zeroFlag {
		terminator = "\x00"
	}

	f, err := os.Open(file)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v", progname, err)
	}
	defer f.Close()

	inputReader := bufio.NewReader(f)
	for {
		pathname, err := inputReader.ReadString(terminator[0])
		if err == io.EOF {
			return
		}
		pathname = strings.TrimRight(pathname, terminator)
		errors = hash_pathname(pathname)
	}

	return
}

func hash_pathname(pathname string) (errors bool) {
	info, err := os.Stat(pathname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
		errors = true
	} else if info.IsDir() {
		errors = hash_dir(pathname)
	} else {
		errors = hash_file(pathname)
	}
	return
}

func hash_file(filename string) (errors bool) {
	for h := range hashes {
		go func(h int) {
			f, err := os.Open(filename)
			if err != nil {
				done <- err
				return
			}
			defer f.Close()

			if _, err := io.Copy(hashes[h], f); err != nil {
				done <- err
				return
			}
			hashes[h].sum = fmt.Sprintf("%s(%s) = %x", hashes[h].name, filename, hashes[h].Sum(nil))
			hashes[h].Reset()
			done <- nil
		}(h)
	}
	for range hashes {
		err := <-done
		if err != nil {
			if !errors {
				fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
			}
			errors = true
		}
	}
	if !errors {
		display()
	}
	return
}

func visit(path string, f os.FileInfo, err error) error {
	if err != nil && !os.IsPermission(err) {
		return err
	}
	if f.Mode().IsRegular() {
		hash_file(path)
	}
	return nil
}

func hash_dir(dir string) bool {
	err := filepath.Walk(dir, visit)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
		return false
	}
	return true
}

func hash_stdin() (errors bool) {
	for h := range hashes {
		go func(h int) {
			if _, err := io.Copy(hashes[h], os.Stdin); err != nil {
				done <- err
				return
			}
			hashes[h].sum = fmt.Sprintf("%s() = %x", hashes[h].name, hashes[h].Sum(nil))
			hashes[h].Reset()
			done <- nil
		}(h)
	}
	for range hashes {
		err := <-done
		if err != nil {
			if !errors {
				fmt.Fprintf(os.Stderr, "%s: %v\n", progname, err)
			}
			errors = true
		}
	}
	if !errors {
		display()
	}
	return
}
<|endoftext|>


<commit_before>package consensus

import (
	"github.com/golang/glog"
	"github.com/heidi-ann/ios/msgs"
	"time"
)

var noop = msgs.ClientRequest{-1, -1, true, "noop"}

// RunMaster implements the Master mode
func RunMaster(view int, commit_index int, initial bool, io *msgs.Io, config Config) {
	// setup
	glog.Info("Starting up master in view ", view)
	glog.Info("Master is configured to delegate replication to ",config.DelegateReplication)
	majority := Majority(config.N)

	// determine next safe index
	index := -1
	if !initial {
		// dispatch new view requests
		req := msgs.NewViewRequest{config.ID, view}
		(*io).OutgoingBroadcast.Requests.NewView <- req

		// collect responses
		glog.Info("Waiting for ", majority, " new view responses")
		min_index := commit_index
		// TODO: FEATURE add option to wait longer

		for i := 0; i < majority; {
			msg := <-(*io).Incoming.Responses.NewView
			// check msg replies to the msg we just sent
			if msg.Request == req {
				res := msg.Response
				glog.Info("Received ", res)
				if res.Index > index {
					index = res.Index
				} else if res.Index < min_index {
					min_index = res.Index
				}
				i++
				// TODO: BUG need to check view
				glog.Info("Successful new view received, waiting for ", majority-i, " more")
			}

		}
		glog.Info("Index is ", index)

		// recover entries
		for curr_index := commit_index + 1; curr_index <= index; curr_index++ {
			RunRecoveryCoordinator(view, curr_index, io, config)
		}

	}
	// store the first coordinator to ask
	coordinator := config.ID
	if config.DelegateReplication > 0 {
		coordinator += 1
	}
	window_start := index

	if config.BatchInterval == 0 {
		glog.Info("Ready to handle requests. No batching enabled")
		// handle client requests (1 at a time)
		for {

			// wait for request
			req := <-io.IncomingRequests
			glog.Info("Request received: ", req)

			// if possible, handle request without replication
			if !req.Replicate {
				io.OutgoingRequests <- req
				glog.Info("Read-only request was handled without replication: ", req)
			} else {

				//wait for window slot
				for index > window_start + config.WindowSize {
					time.Sleep(100 * time.Millisecond)
				}
				index++
				glog.Info("Request assigned index: ", index)

				// ok := RunCoordinator(view, index, []msgs.ClientRequest{req}, io, config, true)
				entry := msgs.Entry{view, false, []msgs.ClientRequest{req}}
				coord := msgs.CoordinateRequest{config.ID, view, index, true, entry}
				io.OutgoingUnicast[coordinator].Requests.Coordinate <- coord
				// TODO: BUG: need to handle coordinator failure
				go func() {
					reply := <-(*io).Incoming.Responses.Coordinate
					// TODO: check msg replies to the msg we just sent
					if !reply.Response.Success {
						glog.Warning("Commit unsuccessful")
					}
					glog.Info("Finished replicating request: ", req)
					if reply.Request.Index==window_start+1{
						window_start += 1
					} else {
						// TODO: BUG: handle out-of-order commitment
						glog.Fatal("STUB: to implement")
					}
				}()

				// rotate coordinator is nessacary
				if config.DelegateReplication > 1 {
					coordinator += 1
					if coordinator>config.ID + config.DelegateReplication {
						coordinator = config.ID + 1
					}
				}
			}

		}
	} else {
		glog.Info("Ready to handle request. Batch every ", config.BatchInterval, " milliseconds")
		for {
			// setup for holding requests
			reqs := make([]msgs.ClientRequest, config.MaxBatch)
			reqs_num := 0

			// start collecting requests
			timeout := make(chan bool, 1)
			go func() {
				<-time.After(time.Millisecond * time.Duration(config.BatchInterval))
				timeout <- true
			}()

			exit := false
			for exit == false {
				select {
				case req := <-io.IncomingRequests:
					if !req.Replicate {
						io.OutgoingRequests <- req
						glog.Info("Request handled without replication: ", req)
					} else {
						reqs[reqs_num] = req
						glog.Info("Request ", reqs_num, " is : ", req)
						reqs_num = reqs_num + 1
						if reqs_num == config.MaxBatch {
							exit = true
							break
						}
					}
				case <-timeout:
					exit = true
					break
				}
			}

			// assign requests to coordinators
			if reqs_num > 0 {
				glog.Info("Starting to replicate ", reqs_num, " requests")
				reqs_small := reqs[:reqs_num]
				index++
				// ok := RunCoordinator(view, index, reqs_small, io, config, true)
				entry := msgs.Entry{view, false, reqs_small}
				coord := msgs.CoordinateRequest{config.ID, view, index, true, entry}
				io.OutgoingUnicast[coordinator].Requests.Coordinate <- coord
				go func() {
					reply := <- io.Incoming.Responses.Coordinate
					// TODO: check msg replies to the msg we just sent
					if !reply.Response.Success {
						glog.Warning("Commit unsuccessful")
					}
					glog.Info("Finished replicating requests: ", reqs_small)
				} ()
				// rotate coordinator is nessacary
				if config.DelegateReplication > 1 {
					coordinator += 1
					if coordinator>config.ID + config.DelegateReplication {
						coordinator = config.ID + 1
					}
				}
			}
		}

	}

	glog.Warning("Master stepping down")

}
<commit_msg>removing support for non-replciated reads<commit_after>package consensus

import (
	"github.com/golang/glog"
	"github.com/heidi-ann/ios/msgs"
	"time"
)

var noop = msgs.ClientRequest{-1, -1, true, "noop"}

// RunMaster implements the Master mode
func RunMaster(view int, commit_index int, initial bool, io *msgs.Io, config Config) {
	// setup
	glog.Info("Starting up master in view ", view)
	glog.Info("Master is configured to delegate replication to ",config.DelegateReplication)
	majority := Majority(config.N)

	// determine next safe index
	index := -1
	if !initial {
		// dispatch new view requests
		req := msgs.NewViewRequest{config.ID, view}
		(*io).OutgoingBroadcast.Requests.NewView <- req

		// collect responses
		glog.Info("Waiting for ", majority, " new view responses")
		min_index := commit_index
		// TODO: FEATURE add option to wait longer

		for i := 0; i < majority; {
			msg := <-(*io).Incoming.Responses.NewView
			// check msg replies to the msg we just sent
			if msg.Request == req {
				res := msg.Response
				glog.Info("Received ", res)
				if res.Index > index {
					index = res.Index
				} else if res.Index < min_index {
					min_index = res.Index
				}
				i++
				// TODO: BUG need to check view
				glog.Info("Successful new view received, waiting for ", majority-i, " more")
			}

		}
		glog.Info("Index is ", index)

		// recover entries
		for curr_index := commit_index + 1; curr_index <= index; curr_index++ {
			RunRecoveryCoordinator(view, curr_index, io, config)
		}

	}
	// store the first coordinator to ask
	coordinator := config.ID
	if config.DelegateReplication > 0 {
		coordinator += 1
	}
	window_start := index

	for {
		glog.Info("Ready to handle request")
		req1 := <-io.IncomingRequests
		glog.Info("Request received: ", req1)
		var reqs []msgs.ClientRequest

		//wait for window slot
		//TOOD: replace with better mechanism then polling
		for index > window_start + config.WindowSize {
			time.Sleep(10 * time.Millisecond)
		}

		if config.BatchInterval == 0 || config.MaxBatch == 1 {
			glog.Info("No batching enabled")
			// handle client requests (1 at a time)
			reqs = []msgs.ClientRequest{req1}
		} else {
			glog.Info("Ready to handle more requests. Batch every ", config.BatchInterval, " milliseconds")
			// setup for holding requests
			reqs_all := make([]msgs.ClientRequest, config.MaxBatch)
			reqs_num := 1
			reqs_all[0] = req1

			exit := false
			for exit == false {
				select {
				case req := <-io.IncomingRequests:
						reqs_all[reqs_num] = req
						glog.Info("Request ", reqs_num, " is : ", req)
						reqs_num = reqs_num + 1
						if reqs_num == config.MaxBatch {
							exit = true
							break
						}
				case <-time.After(time.Millisecond * time.Duration(config.BatchInterval)):
					exit = true
					break
				}
			}
			// this batch is ready
			glog.Info("Starting to replicate ", reqs_num, " requests")
			reqs = reqs_all[:reqs_num]
		}

		index++
		glog.Info("Request assigned index: ", index)

		// ok := RunCoordinator(view, index, []msgs.ClientRequest{req}, io, config, true)
		entry := msgs.Entry{view, false, reqs}
		coord := msgs.CoordinateRequest{config.ID, view, index, true, entry}
		io.OutgoingUnicast[coordinator].Requests.Coordinate <- coord
		// TODO: BUG: need to handle coordinator failure
		go func() {
			reply := <-(*io).Incoming.Responses.Coordinate
			// TODO: check msg replies to the msg we just sent
			if !reply.Response.Success {
				glog.Warning("Commit unsuccessful")
			}
			glog.Info("Finished replicating request: ", reqs)
			if reply.Request.Index==window_start+1{
				window_start += 1
			} else {
				// TODO: BUG: handle out-of-order commitment
				glog.Fatal("STUB: to implement")
			}
		}()

		// rotate coordinator is nessacary
		if config.DelegateReplication > 1 {
			coordinator += 1
			if coordinator>config.ID + config.DelegateReplication {
				coordinator = config.ID + 1
			}
		}
	}
	glog.Warning("Master stepping down")

}
<|endoftext|>


<commit_before>package consensus

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"
	"time"

	abci "github.com/tendermint/abci/types"
	auto "github.com/tendermint/go-autofile"
	. "github.com/tendermint/go-common"
	cfg "github.com/tendermint/go-config"
	"github.com/tendermint/go-wire"

	"github.com/tendermint/tendermint/proxy"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

// Functionality to replay blocks and messages on recovery from a crash.
// There are two general failure scenarios: failure during consensus, and failure while applying the block.
// The former is handled by the WAL, the latter by the proxyApp Handshake on restart,
// which ultimately hands off the work to the WAL.

//-----------------------------------------
// recover from failure during consensus
// by replaying messages from the WAL

// Unmarshal and apply a single message to the consensus state
// as if it were received in receiveRoutine
// Lines that start with "#" are ignored.
// NOTE: receiveRoutine should not be running
func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan interface{}) error {
	// Skip over empty and meta lines
	if len(msgBytes) == 0 || msgBytes[0] == '#' {
		return nil
	}
	var err error
	var msg TimedWALMessage
	wire.ReadJSON(&msg, msgBytes, &err)
	if err != nil {
		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
		return fmt.Errorf("Error reading json data: %v", err)
	}

	// for logging
	switch m := msg.Msg.(type) {
	case types.EventDataRoundState:
		log.Notice("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
		// these are playback checks
		ticker := time.After(time.Second * 2)
		if newStepCh != nil {
			select {
			case mi := <-newStepCh:
				m2 := mi.(types.EventDataRoundState)
				if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
					return fmt.Errorf("RoundState mismatch. Got %v; Expected %v", m2, m)
				}
			case <-ticker:
				return fmt.Errorf("Failed to read off newStepCh")
			}
		}
	case msgInfo:
		peerKey := m.PeerKey
		if peerKey == "" {
			peerKey = "local"
		}
		switch msg := m.Msg.(type) {
		case *ProposalMessage:
			p := msg.Proposal
			log.Notice("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
				p.BlockPartsHeader, "pol", p.POLRound, "peer", peerKey)
		case *BlockPartMessage:
			log.Notice("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerKey)
		case *VoteMessage:
			v := msg.Vote
			log.Notice("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
				"blockID", v.BlockID, "peer", peerKey)
		}

		cs.handleMsg(m, cs.RoundState)
	case timeoutInfo:
		log.Notice("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
		cs.handleTimeout(m, cs.RoundState)
	default:
		return fmt.Errorf("Replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg))
	}
	return nil
}

// replay only those messages since the last block.
// timeoutRoutine should run concurrently to read off tickChan
func (cs *ConsensusState) catchupReplay(csHeight int) error {

	// set replayMode
	cs.replayMode = true
	defer func() { cs.replayMode = false }()

	// Ensure that ENDHEIGHT for this height doesn't exist
	gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight))
	if found {
		return errors.New(Fmt("WAL should not contain height %d.", csHeight))
	}
	if gr != nil {
		gr.Close()
	}

	// Search for last height marker
	gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1))
	if err == io.EOF {
		log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight-1)
		return nil
	} else if err != nil {
		return err
	}
	if !found {
		return errors.New(Fmt("WAL does not contain height %d.", csHeight))
	}
	defer gr.Close()

	log.Notice("Catchup by replaying consensus messages", "height", csHeight)

	for {
		line, err := gr.ReadLine()
		if err != nil {
			if err == io.EOF {
				break
			} else {
				return err
			}
		}
		// NOTE: since the priv key is set when the msgs are received
		// it will attempt to eg double sign but we can just ignore it
		// since the votes will be replayed and we'll get to the next step
		if err := cs.readReplayMessage([]byte(line), nil); err != nil {
			return err
		}
	}
	log.Notice("Replay: Done")
	return nil
}

//--------------------------------------------------------------------------------

// Parses marker lines of the form:
// #ENDHEIGHT: 12345
func makeHeightSearchFunc(height int) auto.SearchFunc {
	return func(line string) (int, error) {
		line = strings.TrimRight(line, "\n")
		parts := strings.Split(line, " ")
		if len(parts) != 2 {
			return -1, errors.New("Line did not have 2 parts")
		}
		i, err := strconv.Atoi(parts[1])
		if err != nil {
			return -1, errors.New("Failed to parse INFO: " + err.Error())
		}
		if height < i {
			return 1, nil
		} else if height == i {
			return 0, nil
		} else {
			return -1, nil
		}
	}
}

//----------------------------------------------
// Recover from failure during block processing
// by handshaking with the app to figure out where
// we were last and using the WAL to recover there

type Handshaker struct {
	config cfg.Config
	state  *sm.State
	store  types.BlockStore

	nBlocks int // number of blocks applied to the state
}

func NewHandshaker(config cfg.Config, state *sm.State, store types.BlockStore) *Handshaker {
	return &Handshaker{config, state, store, 0}
}

func (h *Handshaker) NBlocks() int {
	return h.nBlocks
}

var ErrReplayLastBlockTimeout = errors.New("Timed out waiting for last block to be replayed")

// TODO: retry the handshake/replay if it fails ?
func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
	// handshake is done via info request on the query conn
	res, err := proxyApp.Query().InfoSync()
	if err != nil {
		return errors.New(Fmt("Error calling Info: %v", err))
	}

	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
	appHash := res.LastBlockAppHash

	log.Notice("ABCI Handshake", "appHeight", blockHeight, "appHash", appHash)

	// TODO: check version

	// replay blocks up to the latest in the blockstore
	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
	if err == ErrReplayLastBlockTimeout {
		log.Warn("Failed to sync via handshake. Trying other means. If they fail, please increase the timeout_handshake parameter")
		return nil

	} else if err != nil {
		return errors.New(Fmt("Error on replay: %v", err))
	}

	log.Notice("Completed ABCI Handshake - Tendermint and App are synced", "appHeight", blockHeight, "appHash", appHash)

	// TODO: (on restart) replay mempool

	return nil
}

// Replay all blocks since appBlockHeight and ensure the result matches the current state.
// Returns the final AppHash or an error
func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {

	storeBlockHeight := h.store.Height()
	stateBlockHeight := h.state.LastBlockHeight
	log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)

	// First handle edge cases and constraints on the storeBlockHeight
	if storeBlockHeight == 0 {
		return appHash, h.checkAppHash(appHash)

	} else if storeBlockHeight < appBlockHeight {
		// the app should never be ahead of the store (but this is under app's control)
		return appHash, sm.ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}

	} else if storeBlockHeight < stateBlockHeight {
		// the state should never be ahead of the store (this is under tendermint's control)
		PanicSanity(Fmt("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))

	} else if storeBlockHeight > stateBlockHeight+1 {
		// store should be at most one ahead of the state (this is under tendermint's control)
		PanicSanity(Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
	}

	// Now either store is equal to state, or one ahead.
	// For each, consider all cases of where the app could be, given app <= store
	if storeBlockHeight == stateBlockHeight {
		// Tendermint ran Commit and saved the state.
		// Either the app is asking for replay, or we're all synced up.
		if appBlockHeight < storeBlockHeight {
			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)

		} else if appBlockHeight == storeBlockHeight {
			// We're good!
			return appHash, h.checkAppHash(appHash)
		}

	} else if storeBlockHeight == stateBlockHeight+1 {
		// We saved the block in the store but haven't updated the state,
		// so we'll need to replay a block using the WAL.
		if appBlockHeight < stateBlockHeight {
			// the app is further behind than it should be, so replay blocks
			// but leave the last block to go through the WAL
			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)

		} else if appBlockHeight == stateBlockHeight {
			// We haven't run Commit (both the state and app are one block behind),
			// so run through consensus with the real app
			log.Info("Replay last block using real app")
			return h.replayLastBlock(proxyApp.Consensus())

		} else if appBlockHeight == storeBlockHeight {
			// We ran Commit, but didn't save the state, so run through consensus with mock app
			mockApp := newMockProxyApp(appHash)
			log.Info("Replay last block using mock app")
			return h.replayLastBlock(mockApp)
		}

	}

	PanicSanity("Should never happen")
	return nil, nil
}

func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) {
	// App is further behind than it should be, so we need to replay blocks.
	// We replay all blocks from appBlockHeight+1.
	// If useReplayFunc == true, stop short of the last block
	// so it can be replayed using the WAL in ReplayBlocks.
	// Note that we don't have an old version of the state,
	// so we by-pass state validation using sm.ApplyBlock.

	var appHash []byte
	var err error
	finalBlock := storeBlockHeight
	if useReplayFunc {
		finalBlock -= 1
	}
	for i := appBlockHeight + 1; i <= finalBlock; i++ {
		log.Info("Applying block", "height", i)
		block := h.store.LoadBlock(i)
		appHash, err = sm.ApplyBlock(proxyApp.Consensus(), block)
		if err != nil {
			return nil, err
		}

		h.nBlocks += 1
	}

	if useReplayFunc {
		// sync the final block
		return h.ReplayBlocks(appHash, finalBlock, proxyApp)
	}

	return appHash, h.checkAppHash(appHash)
}

// Replay the last block through the consensus and return the AppHash from after Commit.
func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, error) {
	mempool := types.MockMempool{}
	cs := NewConsensusState(h.config, h.state, proxyApp, h.store, mempool)

	evsw := types.NewEventSwitch()
	evsw.Start()
	defer evsw.Stop()
	cs.SetEventSwitch(evsw)
	newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1)

	// run through the WAL, commit new block, stop
	if _, err := cs.Start(); err != nil {
		return nil, err
	}
	defer cs.Stop()

	timeout := h.config.GetInt("timeout_handshake")
	timer := time.NewTimer(time.Duration(timeout) * time.Millisecond)
	log.Notice("Attempting to replay last block", "height", h.store.Height(), "timeout", timeout)

	select {
	case <-newBlockCh:
	case <-timer.C:
		return nil, ErrReplayLastBlockTimeout
	}

	h.nBlocks += 1

	return cs.state.AppHash, nil
}

func (h *Handshaker) checkAppHash(appHash []byte) error {
	if !bytes.Equal(h.state.AppHash, appHash) {
		panic(errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)).Error())
		return nil
	}
	return nil
}

//--------------------------------------------------------------------------------

func newMockProxyApp(appHash []byte) proxy.AppConnConsensus {
	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{appHash: appHash})
	cli, _ := clientCreator.NewABCIClient()
	return proxy.NewAppConnConsensus(cli)
}

type mockProxyApp struct {
	abci.BaseApplication

	appHash []byte
}

func (mock *mockProxyApp) Commit() abci.Result {
	return abci.NewResultOK(mock.appHash, "")
}
<commit_msg>consensus/replay: remove timeout<commit_after>package consensus

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"
	"time"

	abci "github.com/tendermint/abci/types"
	auto "github.com/tendermint/go-autofile"
	. "github.com/tendermint/go-common"
	cfg "github.com/tendermint/go-config"
	"github.com/tendermint/go-wire"

	"github.com/tendermint/tendermint/proxy"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
)

// Functionality to replay blocks and messages on recovery from a crash.
// There are two general failure scenarios: failure during consensus, and failure while applying the block.
// The former is handled by the WAL, the latter by the proxyApp Handshake on restart,
// which ultimately hands off the work to the WAL.

//-----------------------------------------
// recover from failure during consensus
// by replaying messages from the WAL

// Unmarshal and apply a single message to the consensus state
// as if it were received in receiveRoutine
// Lines that start with "#" are ignored.
// NOTE: receiveRoutine should not be running
func (cs *ConsensusState) readReplayMessage(msgBytes []byte, newStepCh chan interface{}) error {
	// Skip over empty and meta lines
	if len(msgBytes) == 0 || msgBytes[0] == '#' {
		return nil
	}
	var err error
	var msg TimedWALMessage
	wire.ReadJSON(&msg, msgBytes, &err)
	if err != nil {
		fmt.Println("MsgBytes:", msgBytes, string(msgBytes))
		return fmt.Errorf("Error reading json data: %v", err)
	}

	// for logging
	switch m := msg.Msg.(type) {
	case types.EventDataRoundState:
		log.Notice("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
		// these are playback checks
		ticker := time.After(time.Second * 2)
		if newStepCh != nil {
			select {
			case mi := <-newStepCh:
				m2 := mi.(types.EventDataRoundState)
				if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
					return fmt.Errorf("RoundState mismatch. Got %v; Expected %v", m2, m)
				}
			case <-ticker:
				return fmt.Errorf("Failed to read off newStepCh")
			}
		}
	case msgInfo:
		peerKey := m.PeerKey
		if peerKey == "" {
			peerKey = "local"
		}
		switch msg := m.Msg.(type) {
		case *ProposalMessage:
			p := msg.Proposal
			log.Notice("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
				p.BlockPartsHeader, "pol", p.POLRound, "peer", peerKey)
		case *BlockPartMessage:
			log.Notice("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerKey)
		case *VoteMessage:
			v := msg.Vote
			log.Notice("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
				"blockID", v.BlockID, "peer", peerKey)
		}

		cs.handleMsg(m, cs.RoundState)
	case timeoutInfo:
		log.Notice("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
		cs.handleTimeout(m, cs.RoundState)
	default:
		return fmt.Errorf("Replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg))
	}
	return nil
}

// replay only those messages since the last block.
// timeoutRoutine should run concurrently to read off tickChan
func (cs *ConsensusState) catchupReplay(csHeight int) error {

	// set replayMode
	cs.replayMode = true
	defer func() { cs.replayMode = false }()

	// Ensure that ENDHEIGHT for this height doesn't exist
	gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight))
	if found {
		return errors.New(Fmt("WAL should not contain height %d.", csHeight))
	}
	if gr != nil {
		gr.Close()
	}

	// Search for last height marker
	gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1))
	if err == io.EOF {
		log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight-1)
		return nil
	} else if err != nil {
		return err
	}
	if !found {
		return errors.New(Fmt("WAL does not contain height %d.", csHeight))
	}
	defer gr.Close()

	log.Notice("Catchup by replaying consensus messages", "height", csHeight)

	for {
		line, err := gr.ReadLine()
		if err != nil {
			if err == io.EOF {
				break
			} else {
				return err
			}
		}
		// NOTE: since the priv key is set when the msgs are received
		// it will attempt to eg double sign but we can just ignore it
		// since the votes will be replayed and we'll get to the next step
		if err := cs.readReplayMessage([]byte(line), nil); err != nil {
			return err
		}
	}
	log.Notice("Replay: Done")
	return nil
}

//--------------------------------------------------------------------------------

// Parses marker lines of the form:
// #ENDHEIGHT: 12345
func makeHeightSearchFunc(height int) auto.SearchFunc {
	return func(line string) (int, error) {
		line = strings.TrimRight(line, "\n")
		parts := strings.Split(line, " ")
		if len(parts) != 2 {
			return -1, errors.New("Line did not have 2 parts")
		}
		i, err := strconv.Atoi(parts[1])
		if err != nil {
			return -1, errors.New("Failed to parse INFO: " + err.Error())
		}
		if height < i {
			return 1, nil
		} else if height == i {
			return 0, nil
		} else {
			return -1, nil
		}
	}
}

//----------------------------------------------
// Recover from failure during block processing
// by handshaking with the app to figure out where
// we were last and using the WAL to recover there

type Handshaker struct {
	config cfg.Config
	state  *sm.State
	store  types.BlockStore

	nBlocks int // number of blocks applied to the state
}

func NewHandshaker(config cfg.Config, state *sm.State, store types.BlockStore) *Handshaker {
	return &Handshaker{config, state, store, 0}
}

func (h *Handshaker) NBlocks() int {
	return h.nBlocks
}

var ErrReplayLastBlockTimeout = errors.New("Timed out waiting for last block to be replayed")

// TODO: retry the handshake/replay if it fails ?
func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
	// handshake is done via info request on the query conn
	res, err := proxyApp.Query().InfoSync()
	if err != nil {
		return errors.New(Fmt("Error calling Info: %v", err))
	}

	blockHeight := int(res.LastBlockHeight) // XXX: beware overflow
	appHash := res.LastBlockAppHash

	log.Notice("ABCI Handshake", "appHeight", blockHeight, "appHash", appHash)

	// TODO: check version

	// replay blocks up to the latest in the blockstore
	_, err = h.ReplayBlocks(appHash, blockHeight, proxyApp)
	if err == ErrReplayLastBlockTimeout {
		log.Warn("Failed to sync via handshake. Trying other means. If they fail, please increase the timeout_handshake parameter")
		return nil

	} else if err != nil {
		return errors.New(Fmt("Error on replay: %v", err))
	}

	log.Notice("Completed ABCI Handshake - Tendermint and App are synced", "appHeight", blockHeight, "appHash", appHash)

	// TODO: (on restart) replay mempool

	return nil
}

// Replay all blocks since appBlockHeight and ensure the result matches the current state.
// Returns the final AppHash or an error
func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp proxy.AppConns) ([]byte, error) {

	storeBlockHeight := h.store.Height()
	stateBlockHeight := h.state.LastBlockHeight
	log.Notice("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight)

	// First handle edge cases and constraints on the storeBlockHeight
	if storeBlockHeight == 0 {
		return appHash, h.checkAppHash(appHash)

	} else if storeBlockHeight < appBlockHeight {
		// the app should never be ahead of the store (but this is under app's control)
		return appHash, sm.ErrAppBlockHeightTooHigh{storeBlockHeight, appBlockHeight}

	} else if storeBlockHeight < stateBlockHeight {
		// the state should never be ahead of the store (this is under tendermint's control)
		PanicSanity(Fmt("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))

	} else if storeBlockHeight > stateBlockHeight+1 {
		// store should be at most one ahead of the state (this is under tendermint's control)
		PanicSanity(Fmt("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
	}

	// Now either store is equal to state, or one ahead.
	// For each, consider all cases of where the app could be, given app <= store
	if storeBlockHeight == stateBlockHeight {
		// Tendermint ran Commit and saved the state.
		// Either the app is asking for replay, or we're all synced up.
		if appBlockHeight < storeBlockHeight {
			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, false)

		} else if appBlockHeight == storeBlockHeight {
			// We're good!
			return appHash, h.checkAppHash(appHash)
		}

	} else if storeBlockHeight == stateBlockHeight+1 {
		// We saved the block in the store but haven't updated the state,
		// so we'll need to replay a block using the WAL.
		if appBlockHeight < stateBlockHeight {
			// the app is further behind than it should be, so replay blocks
			// but leave the last block to go through the WAL
			return h.replayBlocks(proxyApp, appBlockHeight, storeBlockHeight, true)

		} else if appBlockHeight == stateBlockHeight {
			// We haven't run Commit (both the state and app are one block behind),
			// so run through consensus with the real app
			log.Info("Replay last block using real app")
			return h.replayLastBlock(proxyApp.Consensus())

		} else if appBlockHeight == storeBlockHeight {
			// We ran Commit, but didn't save the state, so run through consensus with mock app
			mockApp := newMockProxyApp(appHash)
			log.Info("Replay last block using mock app")
			return h.replayLastBlock(mockApp)
		}

	}

	PanicSanity("Should never happen")
	return nil, nil
}

func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) {
	// App is further behind than it should be, so we need to replay blocks.
	// We replay all blocks from appBlockHeight+1.
	// If useReplayFunc == true, stop short of the last block
	// so it can be replayed using the WAL in ReplayBlocks.
	// Note that we don't have an old version of the state,
	// so we by-pass state validation using sm.ApplyBlock.

	var appHash []byte
	var err error
	finalBlock := storeBlockHeight
	if useReplayFunc {
		finalBlock -= 1
	}
	for i := appBlockHeight + 1; i <= finalBlock; i++ {
		log.Info("Applying block", "height", i)
		block := h.store.LoadBlock(i)
		appHash, err = sm.ApplyBlock(proxyApp.Consensus(), block)
		if err != nil {
			return nil, err
		}

		h.nBlocks += 1
	}

	if useReplayFunc {
		// sync the final block
		return h.ReplayBlocks(appHash, finalBlock, proxyApp)
	}

	return appHash, h.checkAppHash(appHash)
}

// Replay the last block through the consensus and return the AppHash from after Commit.
func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, error) {
	mempool := types.MockMempool{}
	cs := NewConsensusState(h.config, h.state, proxyApp, h.store, mempool)

	evsw := types.NewEventSwitch()
	evsw.Start()
	defer evsw.Stop()
	cs.SetEventSwitch(evsw)

	log.Notice("Attempting to replay last block", "height", h.store.Height())
	// run through the WAL, commit new block, stop
	if _, err := cs.Start(); err != nil {
		return nil, err
	}
	cs.Stop()

	h.nBlocks += 1

	return cs.state.AppHash, nil
}

func (h *Handshaker) checkAppHash(appHash []byte) error {
	if !bytes.Equal(h.state.AppHash, appHash) {
		panic(errors.New(Fmt("Tendermint state.AppHash does not match AppHash after replay. Got %X, expected %X", appHash, h.state.AppHash)).Error())
		return nil
	}
	return nil
}

//--------------------------------------------------------------------------------

func newMockProxyApp(appHash []byte) proxy.AppConnConsensus {
	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{appHash: appHash})
	cli, _ := clientCreator.NewABCIClient()
	return proxy.NewAppConnConsensus(cli)
}

type mockProxyApp struct {
	abci.BaseApplication

	appHash []byte
}

func (mock *mockProxyApp) Commit() abci.Result {
	return abci.NewResultOK(mock.appHash, "")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"log"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func search(msg string) (string, error) {
	uri := fmt.Sprintf("%s/search?type=content&q=%s", cfg.Zhihu.Host, url.QueryEscape(msg))
	doc, err := goquery.NewDocument(uri)
	if err != nil {
		log.Println(err)
		return "", err
	}

	msg = ""
	doc.Find("ul.list li").Each(func(i int, s *goquery.Selection) {
		title := s.Find(".title").Text()
		smy := s.Find(".content .summary")
		smy.Find("a.toggle-expand").Remove()
		summary := smy.Text()
		// content := s.Find(".visible-expanded .content").Text()

		questionLink, _ := s.Find("a").Attr("href")
		answerLink, _ := s.Find(".entry-body .entry-content").Attr("data-entry-url")

		msg = fmt.Sprintf(`%s<a href="%s/%s">%s</a><br>%s <a href="%s/%s">...显示全部</a><br><br>`,
			msg, cfg.Zhihu.Host, questionLink, title, html.EscapeString(summary), cfg.Zhihu.Host, answerLink)
	})

	msg = format(msg)
	return msg, nil
}

var (
	Warp = `
	`
	ReplaceHTML = map[string]string{
		"
":       Warp,
		"<br>": Warp,
	}
)

func format(msg string) string {
	for k, v := range ReplaceHTML {
		msg = strings.Replace(msg, k, v, -1)
	}
	return msg
}
<commit_msg>update<commit_after>package main

import (
	"fmt"
	"html"
	"log"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func search(msg string) (string, error) {
	uri := fmt.Sprintf("%s/search?type=content&q=%s", cfg.Zhihu.Host, url.QueryEscape(msg))
	doc, err := goquery.NewDocument(uri)
	if err != nil {
		log.Println(err)
		return "", err
	}

	msg = ""
	doc.Find("ul.list li").Each(func(i int, s *goquery.Selection) {
		title := s.Find(".title").Text()
		smy := s.Find(".content .summary")
		smy.Find("a.toggle-expand").Remove()
		summary := smy.Text()
		// content := s.Find(".visible-expanded .content").Text()

		questionLink, _ := s.Find("a").Attr("href")
		answerLink, _ := s.Find(".entry-body .entry-content").Attr("data-entry-url")
		if title == "" {
			return
		}

		msg = fmt.Sprintf(`%s<a href="%s/%s">%s</a><br>%s <a href="%s/%s">...显示全部</a><br><br>`,
			msg, cfg.Zhihu.Host, questionLink, title, html.EscapeString(summary), cfg.Zhihu.Host, answerLink)
	})

	msg = format(msg)
	return msg, nil
}

var (
	Warp = `
	`
	ReplaceHTML = map[string]string{
		"
":       Warp,
		"<br>": Warp,
	}
)

func format(msg string) string {
	for k, v := range ReplaceHTML {
		msg = strings.Replace(msg, k, v, -1)
	}
	return msg
}
<|endoftext|>


<commit_before>package libstns

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"

	stns_settings "github.com/STNS/STNS/settings"
	"github.com/STNS/STNS/stns"
	"github.com/STNS/libnss_stns/cache"
	"github.com/STNS/libnss_stns/settings"
)

type Request struct {
	ApiPath string
	Config  *Config
}

func NewRequest(config *Config, paths ...string) (*Request, error) {
	r := Request{
		ApiPath: path.Clean(strings.Join(paths, "/")),
		Config:  config,
	}
	return &r, nil
}

// only use wrapper command
func (r *Request) GetRawData() ([]byte, error) {

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	rch := make(chan []byte, len(r.Config.ApiEndPoint))
	ech := make(chan error, len(r.Config.ApiEndPoint))

	for _, e := range r.Config.ApiEndPoint {
		go func(endPoint string) {
			if cache.IsLockEndPoint(endPoint) {
				ech <- fmt.Errorf("endpoint %s is locked", endPoint)
				return
			}

			u := strings.TrimRight(endPoint, "/") + "/" + strings.TrimLeft(r.ApiPath, "/")
			req, err := http.NewRequest("GET", u, nil)
			if err != nil {
				ech <- err
				return
			}

			if r.Config.User != "" && r.Config.Password != "" {
				req.SetBasicAuth(r.Config.User, r.Config.Password)
			}

			r.httpDo(
				ctx,
				req,
				func(res *http.Response, err error) {
					if err != nil {
						if _, ok := err.(*url.Error); ok {
							cache.LockEndPoint(endPoint)
						}
						ech <- err
						return
					}
					defer res.Body.Close()
					body, err := ioutil.ReadAll(res.Body)
					switch res.StatusCode {
					case http.StatusOK, http.StatusNotFound:
						reg := regexp.MustCompile(`/v2[/]?$`)
						switch {
						// version1
						case !reg.MatchString(endPoint):
							buffer, err := r.migrateV2Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						default:
							rch <- body
							return
						}
					case http.StatusUnauthorized:
						ech <- fmt.Errorf("authenticate error: %s", u)
						return
					default:
						ech <- fmt.Errorf("error: %s", u)
						return
					}
				},
			)
		}(e)
	}

	var cnt int
	for {
		select {
		case r := <-rch:
			return r, nil
		case e := <-ech:
			cnt++
			if cnt == len(r.Config.ApiEndPoint) {
				return nil, e
			}
		}
	}
}

func (r *Request) httpDo(
	ctx context.Context,
	req *http.Request,
	f func(*http.Response, error),
) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: !r.Config.SslVerify},
		Dial: (&net.Dialer{
			Timeout:   time.Duration(r.Config.RequestTimeOut) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	client := &http.Client{Transport: tr}

	go func() { f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		return
	}
}

func (r *Request) migrateV2Format(body []byte) ([]byte, error) {
	var attr stns.Attributes
	err := json.Unmarshal(body, &attr)

	if err != nil {
		return nil, err
	}

	if attr == nil {
		return nil, errors.New(settings.V2_FORMAT_ERROR)
	}

	mig := stns.ResponseFormat{
		&stns.MetaData{
			1.0,
			false,
			0,
			"sha256",
			stns_settings.SUCCESS,
			0,
		},
		&attr,
	}

	j, err := json.Marshal(mig)
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (r *Request) GetByWrapperCmd() (stns.ResponseFormat, error) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(r.Config.WrapperCommand, r.ApiPath)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()

	if err != nil {
		return stns.ResponseFormat{}, err
	}

	if len(stderr.Bytes()) > 0 {
		return stns.ResponseFormat{}, fmt.Errorf("command error:%s", stderr.String())
	}

	var res stns.ResponseFormat
	err = json.Unmarshal(stdout.Bytes(), &res)
	if err != nil {
		return stns.ResponseFormat{}, err
	}
	return res, nil
}
<commit_msg>Respect proxy env on create http.Transport<commit_after>package libstns

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"golang.org/x/net/context"

	stns_settings "github.com/STNS/STNS/settings"
	"github.com/STNS/STNS/stns"
	"github.com/STNS/libnss_stns/cache"
	"github.com/STNS/libnss_stns/settings"
)

type Request struct {
	ApiPath string
	Config  *Config
}

func NewRequest(config *Config, paths ...string) (*Request, error) {
	r := Request{
		ApiPath: path.Clean(strings.Join(paths, "/")),
		Config:  config,
	}
	return &r, nil
}

// only use wrapper command
func (r *Request) GetRawData() ([]byte, error) {

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	rch := make(chan []byte, len(r.Config.ApiEndPoint))
	ech := make(chan error, len(r.Config.ApiEndPoint))

	for _, e := range r.Config.ApiEndPoint {
		go func(endPoint string) {
			if cache.IsLockEndPoint(endPoint) {
				ech <- fmt.Errorf("endpoint %s is locked", endPoint)
				return
			}

			u := strings.TrimRight(endPoint, "/") + "/" + strings.TrimLeft(r.ApiPath, "/")
			req, err := http.NewRequest("GET", u, nil)
			if err != nil {
				ech <- err
				return
			}

			if r.Config.User != "" && r.Config.Password != "" {
				req.SetBasicAuth(r.Config.User, r.Config.Password)
			}

			r.httpDo(
				ctx,
				req,
				func(res *http.Response, err error) {
					if err != nil {
						if _, ok := err.(*url.Error); ok {
							cache.LockEndPoint(endPoint)
						}
						ech <- err
						return
					}
					defer res.Body.Close()
					body, err := ioutil.ReadAll(res.Body)
					switch res.StatusCode {
					case http.StatusOK, http.StatusNotFound:
						reg := regexp.MustCompile(`/v2[/]?$`)
						switch {
						// version1
						case !reg.MatchString(endPoint):
							buffer, err := r.migrateV2Format(body)
							if err != nil {
								ech <- err
								return
							}
							rch <- buffer
							return
						default:
							rch <- body
							return
						}
					case http.StatusUnauthorized:
						ech <- fmt.Errorf("authenticate error: %s", u)
						return
					default:
						ech <- fmt.Errorf("error: %s", u)
						return
					}
				},
			)
		}(e)
	}

	var cnt int
	for {
		select {
		case r := <-rch:
			return r, nil
		case e := <-ech:
			cnt++
			if cnt == len(r.Config.ApiEndPoint) {
				return nil, e
			}
		}
	}
}

func (r *Request) httpDo(
	ctx context.Context,
	req *http.Request,
	f func(*http.Response, error),
) {
	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: !r.Config.SslVerify},
		Dial: (&net.Dialer{
			Timeout:   time.Duration(r.Config.RequestTimeOut) * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
	}

	client := &http.Client{Transport: tr}

	go func() { f(client.Do(req)) }()
	select {
	case <-ctx.Done():
		tr.CancelRequest(req)
		return
	}
}

func (r *Request) migrateV2Format(body []byte) ([]byte, error) {
	var attr stns.Attributes
	err := json.Unmarshal(body, &attr)

	if err != nil {
		return nil, err
	}

	if attr == nil {
		return nil, errors.New(settings.V2_FORMAT_ERROR)
	}

	mig := stns.ResponseFormat{
		&stns.MetaData{
			1.0,
			false,
			0,
			"sha256",
			stns_settings.SUCCESS,
			0,
		},
		&attr,
	}

	j, err := json.Marshal(mig)
	if err != nil {
		return nil, err
	}

	return j, nil
}

func (r *Request) GetByWrapperCmd() (stns.ResponseFormat, error) {
	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(r.Config.WrapperCommand, r.ApiPath)
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()

	if err != nil {
		return stns.ResponseFormat{}, err
	}

	if len(stderr.Bytes()) > 0 {
		return stns.ResponseFormat{}, fmt.Errorf("command error:%s", stderr.String())
	}

	var res stns.ResponseFormat
	err = json.Unmarshal(stdout.Bytes(), &res)
	if err != nil {
		return stns.ResponseFormat{}, err
	}
	return res, nil
}
<|endoftext|>