<commit_before>package jwt

import (
	// "fmt"
	"github.com/pkg/errors"
	"log"
	// jose "gopkg.in/square/go-jose.v2"
	"net/http"
	"time"
	// jwt "gopkg.in/square/go-jose.v2/jwt"
)

// Auth is a struct that provides jwt based authentication.
type Auth struct {
	authStore    *jwtStore
	refreshStore *jwtStore
	csrfStore    *mixStore

	options Options

	// Handlers for when an error occurs
	errorHandler        http.Handler
	unauthorizedHandler http.Handler

	// funcs for verifiing and revoking tokens
	revokeTokenByID    TokenRevoker
	checkTokenId       TokenIdChecker
	getTokenId         TokenIdGetter
	verifyAuthToken    func(r *http.Request) error
	verifyRefreshToken func(r *http.Request) error

	// CsrfEncrypter aead
	// csrfEncrypter jose.Encrypter
}

const (
	defaultRefreshTokenValidTime  = 72 * time.Hour
	defaultAuthTokenValidTime     = 10 * time.Minute
	defaultBearerAuthTokenName    = "X-Auth-Token"
	defaultBearerRefreshTokenName = "X-Refresh-Token"
	defaultCSRFTokenName          = "X-CSRF-Token"
	defaultCookieAuthTokenName    = "AuthToken"
	defaultCookieRefreshTokenName = "RefreshToken"
)

// CSRF token length in bytes.
const tokenLength = 32

const (
	AuthToken    = 0
	RefreshToken = 1
)

var (
	UnauthorizedRequest = errors.New("Unauthorized Request")
)

func defaultTokenRevoker(tokenId string) error {
	return nil
}

// TokenRevoker : a type to revoke tokens
type TokenRevoker func(tokenId string) error

func defaultCheckTokenId(tokenId string) bool {
	// return true if the token id is valid (has not been revoked). False otherwise
	return true
}

// TokenIdChecker : a type to check tokens
type TokenIdChecker func(tokenId string) bool

func defaultGetTokenId() string {
	// return empty string
	return ""
}

// TokenIdGetter : a type to get token ids
type TokenIdGetter func() string

func defaultErrorHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Internal Server Error", 500)
	return
}

func defaultUnauthorizedHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Unauthorized", 401)
	return
}

// func defaultValidator(a *Auth, r *http.Request) error {
// 	u := r.URL
// 	err := c.Claims.Validate(jwt.Expected{
// 		Issuer:  a.options.Issuer,
// 		Subject: from(r),
// 		Time:    time.Now().UTC(),
// 	})
// 	return err
// }

// New constructs a new Auth instance with supplied options.
func New(o ...Options) (*Auth, error) {
	var opts Options
	var err error
	if len(o) == 0 {
		opts = Options{}
		err = DevelOpts(&opts)
		if err != nil {
			return nil, errors.Wrap(err, "Error init development options")
		}
	} else {
		opts = (o[0])
		err = DefOpts(&opts)
		if err != nil {
			return nil, errors.Wrap(err, "Error init auth options")
		}
	}
	auth := &Auth{}
	err = auth.setOptions(&opts)
	if err != nil {
		return nil, errors.Wrap(err, "Error setting auth options")
	}
	return auth, nil
}

func (a *Auth) setOptions(o *Options) error {
	aus, err := NewJWTStore(o, o.AuthTokenName, AuthToken, false)
	if err != nil {
		return errors.Wrap(err, "Error creating auth store")
	}
	a.authStore = aus
	rs, err := NewJWTStore(o, o.RefreshTokenName, RefreshToken, true)
	if err != nil {
		return errors.Wrap(err, "Error creating refresh store")
	}
	a.refreshStore = rs
	a.csrfStore = &mixStore{o.CSRFTokenName}

	a.options = *o

	a.errorHandler = http.HandlerFunc(defaultErrorHandler)
	a.unauthorizedHandler = http.HandlerFunc(defaultUnauthorizedHandler)
	a.revokeTokenByID = defaultTokenRevoker
	a.checkTokenId = defaultCheckTokenId
	a.getTokenId = defaultGetTokenId
	// a.verifyAuthToken = defaultValidator
	// a.verifyRefreshToken = defaultValidator
	return nil
}

// SetErrorHandler : add methods to allow the changing of default functions
func (a *Auth) SetErrorHandler(handler http.Handler) {
	a.errorHandler = handler
}

// SetUnauthorizedHandler : set the 401 handler
func (a *Auth) SetUnauthorizedHandler(handler http.Handler) {
	a.unauthorizedHandler = handler
}

// SetRevokeTokenFunction : set the function which revokes a token
func (a *Auth) SetRevokeTokenFunction(revoker TokenRevoker) {
	a.revokeTokenByID = revoker
}

// SetCheckTokenIdFunction : set the function which checks token id's
func (a *Auth) SetCheckTokenIdFunction(checker TokenIdChecker) {
	a.checkTokenId = checker
}

func (a *Auth) SetVerifyAuthFunction(fn func(r *http.Request) error) {
	a.verifyAuthToken = fn
}

func (a *Auth) SetVerifyRefreshFunction(fn func(r *http.Request) error) {
	a.verifyRefreshToken = fn
}

func (a *Auth) SetBearerTokens(bt bool) error {
	if a.authStore == nil || a.refreshStore == nil {
		return errors.New("Auth.SetBearerTokens error: token store is not initialized")
	}
	a.options.BearerTokens = bt
	a.authStore.bearerTokens = bt
	a.refreshStore.bearerTokens = bt
	var authName, refreshName string
	if bt {
		authName = defaultBearerAuthTokenName
		refreshName = defaultBearerRefreshTokenName
	} else {
		authName = defaultCookieAuthTokenName
		refreshName = defaultCookieRefreshTokenName
	}
	a.options.AuthTokenName = authName
	a.options.RefreshTokenName = refreshName
	a.authStore.tokenName = authName
	a.authStore.cookieStore.name = authName
	a.refreshStore.tokenName = refreshName
	a.refreshStore.cookieStore.name = refreshName
	return nil
}

// Handler implements the http.HandlerFunc for integration with the standard net/http lib.
func (a *Auth) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Process the request. If it returns an error,
		// that indicates the request should not continue.
		id, err := a.Process(w, r)

		// If there was an error, do not continue.
		if err != nil {
			a.NullifyTokens(id, w)
			if err == UnauthorizedRequest {
				a.pkgLog("Unauthorized processing\n")
				a.unauthorizedHandler.ServeHTTP(w, r)
				return
			}
			a.pkgLog("Error processing\n")
			a.pkgLog("%#v\n", err)
			a.errorHandler.ServeHTTP(w, r)
			return
		}

		h.ServeHTTP(w, r)
	})
}

// HandlerFunc works identically to Handler, but takes a HandlerFunc instead of a Handler.
func (a *Auth) HandlerFunc(fn http.HandlerFunc) http.Handler {
	if fn == nil {
		return a.Handler(nil)
	}
	return a.Handler(fn)
}

// HandlerFuncWithNext is a special implementation for Negroni, but could be used elsewhere.
func (a *Auth) HandlerFuncWithNext(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	id, err := a.Process(w, r)

	// If there was an error, do not call next.
	if err == nil && next != nil {
		next(w, r)
	} else {
		_ = a.NullifyTokens(id, w)
		if err == UnauthorizedRequest {
			a.unauthorizedHandler.ServeHTTP(w, r)
		} else {
			a.errorHandler.ServeHTTP(w, r)
		}
	}
}

// Process runs the actual checks and returns an error if the middleware chain should stop.
func (a *Auth) Process(w http.ResponseWriter, r *http.Request) (string, error) {
	// cookies aren't included with options, so simply pass through
	if r.Method == "OPTIONS" {
		return "", nil
	}

	// grab the credentials from the request
	var c credentials
	if err := a.getCredentials(r, &c); err != nil {
		return "", errors.Wrap(err, "Auth.Process: Error getting auth credentials")
	}
	a.pkgLog("%#v\n", c.AuthToken)

	// // check the credential's validity; updating expiry's if necessary and/or allowed
	if err := c.Validate(r); err != nil {
		if err == AuthTokenExpired {
			a.pkgLog("Auth token is expired. Renew Auth token\n")
			err = c.RenewAuthToken(r)
			if err != nil {
				return c.AuthToken.ID, errors.Wrap(err, "Invalid credentials")
			}
		}
		return c.AuthToken.ID, errors.Wrap(err, "Invalid credentials")
	}
	a.pkgLog("Auth token is not expired. Process...\n")

	// // if we've made it this far, everything is valid!
	// // And tokens have been refreshed if need-be
	if !a.options.VerifyOnlyServer {
		if err := a.setCredentials(w, &c); err != nil {
			return c.AuthToken.ID, errors.Wrap(err, "Error setting credentials")
		}
	}

	return c.AuthToken.ID, nil
}

// IssueNewTokens : and also modify create refresh and auth token functions!
func (a *Auth) IssueNewTokens(w http.ResponseWriter, claims *ClaimsType) error {
	if a.options.VerifyOnlyServer {
		return errors.New("Auth.IssueNewTokens: Server is not authorized to issue new tokens")

	}

	var c credentials
	err := a.newCredentials(&c, claims)
	if err != nil {
		return errors.Wrap(err, "Error creating new credentials")
	}
	// fmt.Printf("%#v\n", c.AuthToken)
	// fmt.Printf("%#v\n", c.RefreshToken)

	err = a.setCredentials(w, &c)
	if err != nil {
		return errors.Wrap(err, "Error setting credentials")
	}

	return nil
}

// NullifyTokens : invalidate tokens
// note @adam-hanna: what if there are no credentials in the request?
func (a *Auth) NullifyTokens(tokenID string, w http.ResponseWriter) error {
	a.authStore.Revoke(w)
	a.refreshStore.Revoke(w)
	a.csrfStore.Save("", w)

	err := a.revokeTokenByID(tokenID)
	if err != nil {
		return errors.Wrap(err, "Auth.NullifyTokens: Error revoking token")
	}

	return nil
}

// GrabTokenClaims : extract the claims from the request
// note: we always grab from the authToken
func (a *Auth) GrabTokenClaims(r *http.Request) (*ClaimsType, error) {
	ca, err := a.authStore.Get(r)
	if err != nil {
		return nil, errors.Wrap(err, "Auth.GrabTokenClaims: Error getting auth claims")
	}

	return ca, nil
}

func (c *Auth) pkgLog(format string, v ...interface{}) {
	if c.options.Debug {
		log.Printf(format, v...)
	}
}
<commit_msg>Add UnauthorizedRequest error to Process func<commit_after>package jwt

import (
	// "fmt"
	"github.com/pkg/errors"
	"log"
	// jose "gopkg.in/square/go-jose.v2"
	"net/http"
	"time"
	// jwt "gopkg.in/square/go-jose.v2/jwt"
)

// Auth is a struct that provides jwt based authentication.
type Auth struct {
	authStore    *jwtStore
	refreshStore *jwtStore
	csrfStore    *mixStore

	options Options

	// Handlers for when an error occurs
	errorHandler        http.Handler
	unauthorizedHandler http.Handler

	// funcs for verifiing and revoking tokens
	revokeTokenByID    TokenRevoker
	checkTokenId       TokenIdChecker
	getTokenId         TokenIdGetter
	verifyAuthToken    func(r *http.Request) error
	verifyRefreshToken func(r *http.Request) error

	// CsrfEncrypter aead
	// csrfEncrypter jose.Encrypter
}

const (
	defaultRefreshTokenValidTime  = 72 * time.Hour
	defaultAuthTokenValidTime     = 10 * time.Minute
	defaultBearerAuthTokenName    = "X-Auth-Token"
	defaultBearerRefreshTokenName = "X-Refresh-Token"
	defaultCSRFTokenName          = "X-CSRF-Token"
	defaultCookieAuthTokenName    = "AuthToken"
	defaultCookieRefreshTokenName = "RefreshToken"
)

// CSRF token length in bytes.
const tokenLength = 32

const (
	AuthToken    = 0
	RefreshToken = 1
)

var (
	UnauthorizedRequest = errors.New("Unauthorized Request")
)

func defaultTokenRevoker(tokenId string) error {
	return nil
}

// TokenRevoker : a type to revoke tokens
type TokenRevoker func(tokenId string) error

func defaultCheckTokenId(tokenId string) bool {
	// return true if the token id is valid (has not been revoked). False otherwise
	return true
}

// TokenIdChecker : a type to check tokens
type TokenIdChecker func(tokenId string) bool

func defaultGetTokenId() string {
	// return empty string
	return ""
}

// TokenIdGetter : a type to get token ids
type TokenIdGetter func() string

func defaultErrorHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Internal Server Error", 500)
	return
}

func defaultUnauthorizedHandler(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Unauthorized", 401)
	return
}

// func defaultValidator(a *Auth, r *http.Request) error {
// 	u := r.URL
// 	err := c.Claims.Validate(jwt.Expected{
// 		Issuer:  a.options.Issuer,
// 		Subject: from(r),
// 		Time:    time.Now().UTC(),
// 	})
// 	return err
// }

// New constructs a new Auth instance with supplied options.
func New(o ...Options) (*Auth, error) {
	var opts Options
	var err error
	if len(o) == 0 {
		opts = Options{}
		err = DevelOpts(&opts)
		if err != nil {
			return nil, errors.Wrap(err, "Error init development options")
		}
	} else {
		opts = (o[0])
		err = DefOpts(&opts)
		if err != nil {
			return nil, errors.Wrap(err, "Error init auth options")
		}
	}
	auth := &Auth{}
	err = auth.setOptions(&opts)
	if err != nil {
		return nil, errors.Wrap(err, "Error setting auth options")
	}
	return auth, nil
}

func (a *Auth) setOptions(o *Options) error {
	aus, err := NewJWTStore(o, o.AuthTokenName, AuthToken, false)
	if err != nil {
		return errors.Wrap(err, "Error creating auth store")
	}
	a.authStore = aus
	rs, err := NewJWTStore(o, o.RefreshTokenName, RefreshToken, true)
	if err != nil {
		return errors.Wrap(err, "Error creating refresh store")
	}
	a.refreshStore = rs
	a.csrfStore = &mixStore{o.CSRFTokenName}

	a.options = *o

	a.errorHandler = http.HandlerFunc(defaultErrorHandler)
	a.unauthorizedHandler = http.HandlerFunc(defaultUnauthorizedHandler)
	a.revokeTokenByID = defaultTokenRevoker
	a.checkTokenId = defaultCheckTokenId
	a.getTokenId = defaultGetTokenId
	// a.verifyAuthToken = defaultValidator
	// a.verifyRefreshToken = defaultValidator
	return nil
}

// SetErrorHandler : add methods to allow the changing of default functions
func (a *Auth) SetErrorHandler(handler http.Handler) {
	a.errorHandler = handler
}

// SetUnauthorizedHandler : set the 401 handler
func (a *Auth) SetUnauthorizedHandler(handler http.Handler) {
	a.unauthorizedHandler = handler
}

// SetRevokeTokenFunction : set the function which revokes a token
func (a *Auth) SetRevokeTokenFunction(revoker TokenRevoker) {
	a.revokeTokenByID = revoker
}

// SetCheckTokenIdFunction : set the function which checks token id's
func (a *Auth) SetCheckTokenIdFunction(checker TokenIdChecker) {
	a.checkTokenId = checker
}

func (a *Auth) SetVerifyAuthFunction(fn func(r *http.Request) error) {
	a.verifyAuthToken = fn
}

func (a *Auth) SetVerifyRefreshFunction(fn func(r *http.Request) error) {
	a.verifyRefreshToken = fn
}

func (a *Auth) SetBearerTokens(bt bool) error {
	if a.authStore == nil || a.refreshStore == nil {
		return errors.New("Auth.SetBearerTokens error: token store is not initialized")
	}
	a.options.BearerTokens = bt
	a.authStore.bearerTokens = bt
	a.refreshStore.bearerTokens = bt
	var authName, refreshName string
	if bt {
		authName = defaultBearerAuthTokenName
		refreshName = defaultBearerRefreshTokenName
	} else {
		authName = defaultCookieAuthTokenName
		refreshName = defaultCookieRefreshTokenName
	}
	a.options.AuthTokenName = authName
	a.options.RefreshTokenName = refreshName
	a.authStore.tokenName = authName
	a.authStore.cookieStore.name = authName
	a.refreshStore.tokenName = refreshName
	a.refreshStore.cookieStore.name = refreshName
	return nil
}

// Handler implements the http.HandlerFunc for integration with the standard net/http lib.
func (a *Auth) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Process the request. If it returns an error,
		// that indicates the request should not continue.
		id, err := a.Process(w, r)

		// If there was an error, do not continue.
		if err != nil {
			a.NullifyTokens(id, w)
			if err == UnauthorizedRequest {
				a.pkgLog("Unauthorized processing\n")
				a.unauthorizedHandler.ServeHTTP(w, r)
				return
			}
			a.pkgLog("Error processing\n")
			a.pkgLog("%#v\n", err)
			a.errorHandler.ServeHTTP(w, r)
			return
		}

		h.ServeHTTP(w, r)
	})
}

// HandlerFunc works identically to Handler, but takes a HandlerFunc instead of a Handler.
func (a *Auth) HandlerFunc(fn http.HandlerFunc) http.Handler {
	if fn == nil {
		return a.Handler(nil)
	}
	return a.Handler(fn)
}

// HandlerFuncWithNext is a special implementation for Negroni, but could be used elsewhere.
func (a *Auth) HandlerFuncWithNext(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	id, err := a.Process(w, r)

	// If there was an error, do not call next.
	if err == nil && next != nil {
		next(w, r)
	} else {
		_ = a.NullifyTokens(id, w)
		if err == UnauthorizedRequest {
			a.unauthorizedHandler.ServeHTTP(w, r)
		} else {
			a.errorHandler.ServeHTTP(w, r)
		}
	}
}

// Process runs the actual checks and returns an error if the middleware chain should stop.
func (a *Auth) Process(w http.ResponseWriter, r *http.Request) (string, error) {
	// cookies aren't included with options, so simply pass through
	if r.Method == "OPTIONS" {
		return "", nil
	}

	// grab the credentials from the request
	var c credentials
	if err := a.getCredentials(r, &c); err != nil {
		return "", UnauthorizedRequest
	}
	a.pkgLog("%#v\n", c.AuthToken)

	// // check the credential's validity; updating expiry's if necessary and/or allowed
	if err := c.Validate(r); err != nil {
		if err == AuthTokenExpired {
			a.pkgLog("Auth token is expired. Renew Auth token\n")
			err = c.RenewAuthToken(r)
			if err != nil {
				return c.AuthToken.ID, UnauthorizedRequest
			}
		}
		return c.AuthToken.ID, UnauthorizedRequest
	}
	a.pkgLog("Auth token is not expired. Process...\n")

	// // if we've made it this far, everything is valid!
	// // And tokens have been refreshed if need-be
	if !a.options.VerifyOnlyServer {
		if err := a.setCredentials(w, &c); err != nil {
			return c.AuthToken.ID, errors.Wrap(err, "Error setting credentials")
		}
	}

	return c.AuthToken.ID, nil
}

// IssueNewTokens : and also modify create refresh and auth token functions!
func (a *Auth) IssueNewTokens(w http.ResponseWriter, claims *ClaimsType) error {
	if a.options.VerifyOnlyServer {
		return errors.New("Auth.IssueNewTokens: Server is not authorized to issue new tokens")

	}

	var c credentials
	err := a.newCredentials(&c, claims)
	if err != nil {
		return errors.Wrap(err, "Error creating new credentials")
	}
	// fmt.Printf("%#v\n", c.AuthToken)
	// fmt.Printf("%#v\n", c.RefreshToken)

	err = a.setCredentials(w, &c)
	if err != nil {
		return errors.Wrap(err, "Error setting credentials")
	}

	return nil
}

// NullifyTokens : invalidate tokens
// note @adam-hanna: what if there are no credentials in the request?
func (a *Auth) NullifyTokens(tokenID string, w http.ResponseWriter) error {
	a.authStore.Revoke(w)
	a.refreshStore.Revoke(w)
	a.csrfStore.Save("", w)

	err := a.revokeTokenByID(tokenID)
	if err != nil {
		return errors.Wrap(err, "Auth.NullifyTokens: Error revoking token")
	}

	return nil
}

// GrabTokenClaims : extract the claims from the request
// note: we always grab from the authToken
func (a *Auth) GrabTokenClaims(r *http.Request) (*ClaimsType, error) {
	ca, err := a.authStore.Get(r)
	if err != nil {
		return nil, errors.Wrap(err, "Auth.GrabTokenClaims: Error getting auth claims")
	}

	return ca, nil
}

func (c *Auth) pkgLog(format string, v ...interface{}) {
	if c.options.Debug {
		log.Printf(format, v...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"
)

// https://play.golang.org/p/N8A5cXa3RM
func wordCountWithoutLinks(data_string string) int {
	// from https://regex101.com/r/tZ6yK9/3
	data := data_string
	re := regexp.MustCompile("\\[([^\\]]+)\\]\\(([^\\)]+)\\)")
	matches := re.FindAllString(data, -1)
	newdata := data
	for _, match := range matches {
		if match[0:2] == "[!" {
			match = match[2:]
		}
		newdata = strings.Replace(newdata, match, "", -1)
	}
	data = newdata
	matches = re.FindAllString(data, -1)
	newdata = data
	for _, match := range matches {
		if match[0:3] == "[![" {
			match = match[2:]
		}
		newdata = strings.Replace(newdata, match, "", -1)
	}
	return len(strings.Split(strings.TrimSpace(newdata), " "))
}

func timeTrack(start time.Time, name string) {
	elapsed := time.Since(start)
	log.Printf("%s took %s", name, elapsed)
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func downloadUrl(url string) string {
	defer timeTrack(time.Now(), "downloadUrl")
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("%s", err)
		os.Exit(1)
	} else {
		defer response.Body.Close()
		contents, err := ioutil.ReadAll(response.Body)
		if err != nil {
			fmt.Printf("%s", err)
			os.Exit(1)
		}
		fmt.Printf("%s\n", string(contents))
		f, err := os.Create("test.html")
		check(err)
		defer f.Close()
		_, err = f.Write(contents)
		check(err)

	}
	return "test.html"
}

func parseURL(url string) string {
	defer timeTrack(time.Now(), "wget")
	var (
		cmdOut []byte
		err    error
	)
	cmdName := "pandoc"
	cmdArgs := []string{"--columns=70000", "-t", "markdown", "-s", downloadUrl(url)}
	if cmdOut, err = exec.Command(cmdName, cmdArgs...).Output(); err != nil {
		fmt.Fprintln(os.Stderr, "There was an error running git rev-parse command: ", err)
		os.Exit(1)
	}
	sha := string(cmdOut)
	defer timeTrack(time.Now(), "parsing")
	result := strings.Split(sha, "\n")

	var w bytes.Buffer

	lastSentenceGood := false
	for i := range result {
		if i < 3 {
			if strings.Contains(result[i],"title:") {
				w.WriteString("# ")
				w.WriteString(strings.Split(strings.TrimSpace(result[i]), "title:")[1])
				w.WriteString("\n\n")
			}
		} else {
			strLen := len(strings.Split(strings.TrimSpace(result[i]), " "))
			strLen2 := wordCountWithoutLinks(result[i])

			fmt.Printf("%d) %d/%d: %s\n", i, strLen, strLen2, result[i])
			if strLen2 > 10 {
				w.WriteString(result[i])
				w.WriteString("\n\n")
				lastSentenceGood = true
			} else if lastSentenceGood == true && strLen2 > 3 {
				w.WriteString(result[i])
				w.WriteString("\n\n")
				lastSentenceGood = false
			} else if strLen > 1 {
				lastSentenceGood = false
			}
		}
	}

	fmt.Println("WEB RESULT:\n\n")
	output := blackfriday.MarkdownCommon(w.Bytes())
	return string(output)
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	defer timeTrack(time.Now(), "indexHandler")
	url := r.URL.Query()["url"]
	if len(url) > 0 {
		result := parseURL(url[0])
		fmt.Fprintln(w, result)
	}
	fmt.Fprintln(w, "No url")
}

func startServer() {
	runtime.GOMAXPROCS(runtime.NumCPU() - 1) // one core for wrk

	http.HandleFunc("/", indexHandler)
	http.ListenAndServe(":4000", nil)
	fmt.Println("Fin Bench running on Port 4000")
}
func main() {
	startServer()
}
<commit_msg>Works pretty well now<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"
)

// https://play.golang.org/p/N8A5cXa3RM
func wordCountWithoutLinks(data_string string) int {
	// from https://regex101.com/r/tZ6yK9/3
	data := data_string
	re := regexp.MustCompile("\\[(.*?)\\]\\((.+?)\\)")
	matches := re.FindAllString(data, -1)
	newdata := data
	for _, match := range matches {
		if match[0:2] == "[!" {
			match = match[2:]
		}
		newdata = strings.Replace(newdata, match, "", -1)
	}
	data = newdata
	matches = re.FindAllString(data, -1)
	newdata = data
	for _, match := range matches {
		if match[0:3] == "[![" {
			match = match[2:]
		}
		newdata = strings.Replace(newdata, match, "", -1)
	}
	return len(strings.Split(strings.TrimSpace(newdata), " "))
}

func processString(data string) string {
	re := regexp.MustCompile("\\{([^}]+)\\}")
	newdata := data
	for {

		matches := re.FindAllString(newdata, -1)
		if len(matches) == 0 {
			break
		}

		for _, match := range matches {
			newdata = strings.Replace(newdata, match, "", -1)
		}
	}
	return newdata
}

func timeTrack(start time.Time, name string) {
	elapsed := time.Since(start)
	log.Printf("%s took %s", name, elapsed)
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func downloadUrl(url string) string {
	defer timeTrack(time.Now(), "downloadUrl")
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("%s", err)
		os.Exit(1)
	} else {
		defer response.Body.Close()
		contents, err := ioutil.ReadAll(response.Body)
		if err != nil {
			fmt.Printf("%s", err)
			os.Exit(1)
		}
		fmt.Printf("%s\n", string(contents))
		f, err := os.Create("test.html")
		check(err)
		defer f.Close()
		_, err = f.Write(contents)
		check(err)

	}
	return "test.html"
}

func parseURL(url string) string {
	defer timeTrack(time.Now(), "wget")
	var (
		cmdOut []byte
		err    error
	)
	cmdName := "pandoc"
	cmdArgs := []string{"--columns=70000", "-t", "markdown", "-s", downloadUrl(url)}
	if cmdOut, err = exec.Command(cmdName, cmdArgs...).Output(); err != nil {
		fmt.Fprintln(os.Stderr, "There was an error running git rev-parse command: ", err)
		os.Exit(1)
	}
	sha := string(cmdOut)
	defer timeTrack(time.Now(), "parsing")
	result := strings.Split(sha, "\n")

	var w bytes.Buffer

	lastSentenceGood := false
	for i := range result {
		result[i] = strings.TrimSpace(processString(result[i]))
		if i < 3 {
			if strings.Contains(result[i], "title:") {
				w.WriteString("# ")
				w.WriteString(strings.Split(result[i], "title:")[1])
				w.WriteString("\n\n")
			}
		} else {
			strLen := len(strings.Split(result[i], " "))
			strLen2 := wordCountWithoutLinks(result[i])

			fmt.Printf("%d) %d/%d: %s\n", i, strLen, strLen2, result[i])
			if strLen2 > 20 {
				w.WriteString(result[i])
				w.WriteString("\n\n")
				lastSentenceGood = true
			} else if lastSentenceGood == true && strLen2 > 5 {
				w.WriteString(result[i])
				w.WriteString("\n\n")
				lastSentenceGood = false
			} else if strLen > 1 {
				lastSentenceGood = false
			}
		}
	}

	fmt.Println("WEB RESULT:\n\n")
	output := blackfriday.MarkdownCommon(w.Bytes())

	front := `<!DOCTYPE html>

    <meta charset="UTF-8">
         <link rel="shortcut icon" sizes="16x16 24x24 32x32 48x48 64x64" href="/static/img/favicon.ico" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <title>Go Just Read</title>

← Another    Original.

`

	back := `

← Another

`
	s := []string{front, string(output),back};
	s2 := strings.Replace(string(strings.Join(s, " ")),"ORIGINAL_URL",url,-1)
	return s2
}

func indexHandler(w http.ResponseWriter, r *http.Request) {

	indexPage := `
<!DOCTYPE html>

    <meta charset=utf-8 />
    <title>Just Read</title>
         <link rel="shortcut icon" sizes="16x16 24x24 32x32 48x48 64x64" href="/static/img/favicon.ico" />
    <meta name='viewport' content='initial-scale=1,maximum-scale=1,user-scalable=no' />

    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet" integrity="sha256-7s5uDGW3AHqw6xtJmNNtr+OBRJUlgkNJEo78P4b0yRw= sha512-nNo+yCHEyn0smMxSswnf/OnX6/KwJuZTlNZBjauKhTK0c+zT+q5JOCx0UFhXQ6rJR9jg6Es8gPuD2uZcYDLqSw==" crossorigin="anonymous">

  <meta name="author" content="Zack Scholl">
  <meta name="description" content=""/>

Just Read.

Copy and paste URL to get just the text content of that site.

               <form action='/' method='POST'>
                <input type='text' class="form-control input-lg" name='group' id='group' placeholder='Type a URL' autofocus></input>
               </form>

...or use /?url=X in the browser.

`

	defer timeTrack(time.Now(), "indexHandler")
	if r.Method == "GET" {
		url := r.URL.Query()["url"]
		if len(url) > 0 {
			fmt.Fprintln(w, parseURL(url[0]))
		} else {
			fmt.Fprintln(w, indexPage)
		}
	} else {
		r.ParseForm()
		url := r.Form["group"][0]
		fmt.Fprintln(w, parseURL(url))
	}
}

func startServer() {
	fmt.Printf(processString("The Middle Class Is No Longer The Majority in the U.S. {.article-title itemprop=“headline”}"))
	runtime.GOMAXPROCS(runtime.NumCPU() - 1) // one core for wrk
	http.HandleFunc("/", indexHandler)
	http.ListenAndServe(":4000", nil)
	fmt.Println("Fin Bench running on Port 4000")
}
func main() {
startServer()
result := parseURL("http://feeds.reuters.com/~r/reuters/topNews/~3/7lhCFz6edso/story01.htm")
fmt.Println(result)

}
<|endoftext|>


<commit_before>package redisc

import "fmt"

type StringsCache interface {
	MGet(ns string, keys []string) (result []string, err error)
	MSet(ns string, kv []*KeyValueString) error
}

type Set interface {
	// Add
	SAdd(key string, items [][]byte) (int, error)
	// Remove
	SRem(key string, items [][]byte) (int, error)

	SRandMember(key string, count int) ([][]byte, error)

	SMembers(listKey string) ([][]byte, error)
}

type SortedSet interface {
	ZAdd(key string, members []*ScoredMember) (int, error)

	// ZRevRange returns a subset ordered in descending order
	ZRevRange(key string, start, stop int) ([]*ScoredMember, error)

	ZIncrBy(key string, amount int, member []byte) (int, error)

	// ZCard returns the Cardinality (i.e. count) of the set
	ZCard(key string) (int, error)
}

type Client interface {
	StringsCache
	SortedSet
}

type ScoredMember struct {
	Score  int
	Member string
}

func getKey(ns, key string) string {
	useKey := fmt.Sprintf("%s:%s", ns, key)
	return useKey
}

func getKeys(ns string, keys []string) []interface{} {
	result := make([]interface{}, len(keys))
	for i, k := range keys {
		result[i] = getKey(ns, k)
	}
	return result
}
<commit_msg>redis interface<commit_after>package redisc

import "fmt"

type StringsCache interface {
	MGet(ns string, keys []string) (result []string, err error)
	MSet(ns string, kv []*KeyValueString) error
}

type Set interface {
	// Add
	SAdd(key string, items [][]byte) (int, error)
	// Remove
	SRem(key string, items [][]byte) (int, error)

	SRandMember(key string, count int) ([][]byte, error)

	SMembers(listKey string) ([][]byte, error)
}

type SortedSet interface {
	ZAdd(key string, members []*ScoredMember) (int, error)

	// ZRevRange returns a subset ordered in descending order
	ZRevRange(key string, start, stop int) ([]*ScoredMember, error)

	ZIncrBy(key string, amount int, member interface{}) (int, error)

	// ZCard returns the Cardinality (i.e. count) of the set
	ZCard(key string) (int, error)
}

type Client interface {
	StringsCache
	SortedSet
}

type ScoredMember struct {
	Score  int
	Member string
}

func getKey(ns, key string) string {
	useKey := fmt.Sprintf("%s:%s", ns, key)
	return useKey
}

func getKeys(ns string, keys []string) []interface{} {
	result := make([]interface{}, len(keys))
	for i, k := range keys {
		result[i] = getKey(ns, k)
	}
	return result
}
<|endoftext|>


<commit_before>package redlot

import (
	"strconv"

	"github.com/syndtr/goleveldb/leveldb/util"
)

func encodeHashKey(name, key []byte) (buf []byte) {
	buf = append(buf, typeHASH)
	buf = append(buf, uint32ToBytes(uint32(len(name)))...)
	buf = append(buf, name...)
	buf = append(buf, uint32ToBytes(uint32(len(key)))...)
	buf = append(buf, key...)
	return
}

func decodeHashKey(b []byte) (name, key []byte) {
	nameLen := bytesToUint32(b[1:5])
	name = b[5 : 5+nameLen]
	key = b[9+nameLen:]
	return
}

func encodeHsizeKey(name []byte) (buf []byte) {
	buf = append(buf, typeHSIZE)
	buf = append(buf, uint32ToBytes(uint32(len(name)))...)
	buf = append(buf, name...)
	return
}

func decodeHsizeKey(b []byte) (key []byte) {
	return b[5:]
}

func hashSizeIncr(name []byte, incr int) {
	hsize := encodeHsizeKey(name)

	var size uint32
	if b, err := db.Get(hsize, nil); err == nil {
		size = bytesToUint32(b)
	}

	if incr > 0 {
		size += uint32(incr)
	} else {
		size -= uint32(incr)
	}
	db.Put(hsize, uint32ToBytes(size), nil)
}

// Hset will set a hashmap value by the key.
// Args: name string, key string, value any
func Hset(args [][]byte) (r interface{}, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}

	hashSizeIncr(args[0], 1)
	err = db.Put(encodeHashKey(args[0], args[1]), args[2], nil)
	if err != nil {
		hashSizeIncr(args[0], -1)
		return nil, err
	}

	return
}

// Hset will return a hashmap value by the key.
// Args: name string, key string
func Hget(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}
	var b []byte
	b, err = db.Get(encodeHashKey(args[0], args[1]), nil)
	return string(b), err
}

// Hdel will delete a hashmap value by the key.
// Args: name string, key string
func Hdel(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}

func hincr(name, key []byte, increment int) (r int64, err error) {
	hash := encodeHashKey(name, key)
	v, _ := db.Get(hash, nil)
	var number int
	var exists bool
	if len(v) != 0 {
		number, err = strconv.Atoi(string(v))
		exists = true
		if err != nil {
			return -1, errNotInt
		}
	}
	number += increment
	r = int64(number)
	err = db.Put(hash, []byte(strconv.Itoa(number)), nil)
	if err == nil && !exists {
		hashSizeIncr(name, 1)
	}
	return
}

// Hincr will incr a hashmap value by the key.
// Args: name string, key string
func Hincr(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return hincr(args[0], args[1], 1)
}

// Hincrby will incr number a hashmap value by the key.
// Args: name string, key string, value int
func Hincrby(args [][]byte) (r interface{}, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}
	i, e := strconv.Atoi(string(args[2]))
	if e != nil {
		return -1, errNotInt
	}

	return hincr(args[0], args[1], i)
}

// Hexists will check the hashmap key is exists.
// Args: name string, key string
func Hexists(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}

// Hsize will return the hashmap size.
// Args: name string
func Hsize(args [][]byte) (r interface{}, err error) {
	if len(args) < 1 {
		return nil, errNosArgs
	}

	var size int64
	if b, e := db.Get(encodeHsizeKey(args[0]), nil); e != nil {
		size = -1
	} else {
		size = int64(bytesToUint32(b))
	}

	return size, nil
}

// Hlist will list all hashmap in the range.
// Args: start string, end string, limit int
func Hlist(args [][]byte) (r []string, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}

	return
}

// Hrlist will reverse list all hashmap in the range.
// Args: start string, end string, limit int
func Hrlist(args [][]byte) (r []string, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}

	return
}

// Hkeys will list the hashmap keys in the range.
// Args: name string, start string, end string, limit int
func Hkeys(args [][]byte) (r []string, err error) {
	if len(args) < 4 {
		return nil, errNosArgs
	}

	return
}

// Hgetall will list all keys/value in the hashmap.
// Args: name string
func Hgetall(args [][]byte) (r []string, err error) {
	if len(args) < 1 {
		return nil, errNosArgs
	}

	if _, err = db.Get(encodeHsizeKey(args[0]), nil); err != nil {
		return
	}

	var buf []byte
	buf = append(buf, typeHASH)
	buf = append(buf, uint32ToBytes(uint32(len(args[0])))...)
	buf = append(buf, args[0]...)
	ke := append(buf, []byte{0xff}...)

	iter := db.NewIterator(&util.Range{Start: buf, Limit: ke}, nil)
	for iter.Next() {
		_, key := decodeHashKey(iter.Key())
		r = append(r, string(key))
		r = append(r, string(iter.Value()))
	}
	iter.Release()
	err = iter.Error()
	return
}

// Hscan will list keys/value of the hashmap in the range.
// Args: name string, start string, end string, limit int
func Hscan(args [][]byte) (r []string, err error) {
	if len(args) < 4 {
		return nil, errNosArgs
	}

	return
}

// Hrscan will reverse list keys/value of the hashmap in the range.
// Args: name string, start string, end string, limit int
func Hrscan(args [][]byte) (r []string, err error) {
	if len(args) < 4 {
		return nil, errNosArgs
	}

	return
}

// Hclear will remove all value in the hashmap.
// Args: name string
func Hclear(args [][]byte) (r interface{}, err error) {
	if len(args) < 1 {
		return nil, errNosArgs
	}

	return
}

// MultiHget will return multi hashmap value by keys.
func MultiHget(args [][]byte) (r []string, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}

// MultiHset will set multi hashmap value by keys.
func MultiHset(args [][]byte) (r interface{}, err error) {
	if len(args) < 3 && len(args)%2 == 0 {
		return nil, errNosArgs
	}

	return
}

// MultiHdel will delete multi hashmap value by keys.
func MultiHdel(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}
<commit_msg>Modify hash key struct.<commit_after>package redlot

import (
	"strconv"

	"github.com/syndtr/goleveldb/leveldb/util"
)

func encodeHashKey(name, key []byte) (buf []byte) {
	buf = append(buf, typeHASH)
	buf = append(buf, uint32ToBytes(uint32(len(name)))...)
	buf = append(buf, name...)
	buf = append(buf, key...)
	return
}

func decodeHashKey(b []byte) (name, key []byte) {
	nameLen := bytesToUint32(b[1:5])
	name = b[5 : 5+nameLen]
	key = b[5+nameLen:]
	return
}

func encodeHsizeKey(name []byte) (buf []byte) {
	buf = append(buf, typeHSIZE)
	buf = append(buf, uint32ToBytes(uint32(len(name)))...)
	buf = append(buf, name...)
	return
}

func decodeHsizeKey(b []byte) (key []byte) {
	return b[5:]
}

func hashSizeIncr(name []byte, incr int) {
	hsize := encodeHsizeKey(name)

	var size uint32
	if b, err := db.Get(hsize, nil); err == nil {
		size = bytesToUint32(b)
	}

	if incr > 0 {
		size += uint32(incr)
	} else {
		size -= uint32(incr)
	}
	db.Put(hsize, uint32ToBytes(size), nil)
}

// Hset will set a hashmap value by the key.
// Args: name string, key string, value any
func Hset(args [][]byte) (r interface{}, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}

	hashSizeIncr(args[0], 1)
	err = db.Put(encodeHashKey(args[0], args[1]), args[2], nil)
	if err != nil {
		hashSizeIncr(args[0], -1)
		return nil, err
	}

	return
}

// Hset will return a hashmap value by the key.
// Args: name string, key string
func Hget(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}
	var b []byte
	b, err = db.Get(encodeHashKey(args[0], args[1]), nil)
	return string(b), err
}

// Hdel will delete a hashmap value by the key.
// Args: name string, key string
func Hdel(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}

func hincr(name, key []byte, increment int) (r int64, err error) {
	hash := encodeHashKey(name, key)
	v, _ := db.Get(hash, nil)
	var number int
	var exists bool
	if len(v) != 0 {
		number, err = strconv.Atoi(string(v))
		exists = true
		if err != nil {
			return -1, errNotInt
		}
	}
	number += increment
	r = int64(number)
	err = db.Put(hash, []byte(strconv.Itoa(number)), nil)
	if err == nil && !exists {
		hashSizeIncr(name, 1)
	}
	return
}

// Hincr will incr a hashmap value by the key.
// Args: name string, key string
func Hincr(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return hincr(args[0], args[1], 1)
}

// Hincrby will incr number a hashmap value by the key.
// Args: name string, key string, value int
func Hincrby(args [][]byte) (r interface{}, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}
	i, e := strconv.Atoi(string(args[2]))
	if e != nil {
		return -1, errNotInt
	}

	return hincr(args[0], args[1], i)
}

// Hexists will check the hashmap key is exists.
// Args: name string, key string
func Hexists(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}

// Hsize will return the hashmap size.
// Args: name string
func Hsize(args [][]byte) (r interface{}, err error) {
	if len(args) < 1 {
		return nil, errNosArgs
	}

	var size int64
	if b, e := db.Get(encodeHsizeKey(args[0]), nil); e != nil {
		size = -1
	} else {
		size = int64(bytesToUint32(b))
	}

	return size, nil
}

// Hlist will list all hashmap in the range.
// Args: start string, end string, limit int
func Hlist(args [][]byte) (r []string, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}

	return
}

// Hrlist will reverse list all hashmap in the range.
// Args: start string, end string, limit int
func Hrlist(args [][]byte) (r []string, err error) {
	if len(args) < 3 {
		return nil, errNosArgs
	}

	return
}

// Hkeys will list the hashmap keys in the range.
// Args: name string, start string, end string, limit int
func Hkeys(args [][]byte) (r []string, err error) {
	if len(args) < 4 {
		return nil, errNosArgs
	}

	return
}

// Hgetall will list all keys/value in the hashmap.
// Args: name string
func Hgetall(args [][]byte) (r []string, err error) {
	if len(args) < 1 {
		return nil, errNosArgs
	}

	if _, err = db.Get(encodeHsizeKey(args[0]), nil); err != nil {
		return
	}

	var buf []byte
	buf = append(buf, typeHASH)
	buf = append(buf, uint32ToBytes(uint32(len(args[0])))...)
	buf = append(buf, args[0]...)
	ke := append(buf, []byte{0xff}...)

	iter := db.NewIterator(&util.Range{Start: buf, Limit: ke}, nil)
	for iter.Next() {
		_, key := decodeHashKey(iter.Key())
		r = append(r, string(key))
		r = append(r, string(iter.Value()))
	}
	iter.Release()
	err = iter.Error()
	return
}

// Hscan will list keys/value of the hashmap in the range.
// Args: name string, start string, end string, limit int
func Hscan(args [][]byte) (r []string, err error) {
	if len(args) < 4 {
		return nil, errNosArgs
	}

	return
}

// Hrscan will reverse list keys/value of the hashmap in the range.
// Args: name string, start string, end string, limit int
func Hrscan(args [][]byte) (r []string, err error) {
	if len(args) < 4 {
		return nil, errNosArgs
	}

	return
}

// Hclear will remove all value in the hashmap.
// Args: name string
func Hclear(args [][]byte) (r interface{}, err error) {
	if len(args) < 1 {
		return nil, errNosArgs
	}

	return
}

// MultiHget will return multi hashmap value by keys.
func MultiHget(args [][]byte) (r []string, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}

// MultiHset will set multi hashmap value by keys.
func MultiHset(args [][]byte) (r interface{}, err error) {
	if len(args) < 3 && len(args)%2 == 0 {
		return nil, errNosArgs
	}

	return
}

// MultiHdel will delete multi hashmap value by keys.
func MultiHdel(args [][]byte) (r interface{}, err error) {
	if len(args) < 2 {
		return nil, errNosArgs
	}

	return
}
<|endoftext|>


<commit_before>package otp

import (
	"testing"
)

func TestInvalidMetho(t *testing.T) {
	key := Key{
		Method: "crypto!",
	}
	v, _ := key.IsValid()
	if v == true {
		t.Fail()
	}
}

func TestTotpString(t *testing.T) {
	key, _ := NewTotp(
		"label",
		"MFRGGZDFMZTWQ2LK",
		"issuer",
		"sha1",
		6,
		30,
	)

	uri := key.String()
	if uri != "otpauth://totp/label?Secret=MFRGGZDFMZTWQ2LK&Issuer=issuer&Algo=SHA1&Digits=6&Period=30" {
		t.Error(uri)
	}
}
<commit_msg>add test for invalid label<commit_after>package otp

import (
	"testing"
)

func TestInvalidMethod(t *testing.T) {
	key := Key{
		Method: "crypto!",
	}
	v, _ := key.IsValid()
	if v == true {
		t.Fail()
	}
}

func TestInvalidLabel(t *testing.T) {
	key := Key{
		Method: "totp",
		Label:  "t/w",
	}
	v, _ := key.IsValid()
	if v == true {
		t.Fail()
	}
}

func TestTotpString(t *testing.T) {
	key, _ := NewTotp(
		"label",
		"MFRGGZDFMZTWQ2LK",
		"issuer",
		"sha1",
		6,
		30,
	)

	uri := key.String()
	if uri != "otpauth://totp/label?Secret=MFRGGZDFMZTWQ2LK&Issuer=issuer&Algo=SHA1&Digits=6&Period=30" {
		t.Error(uri)
	}
}
<|endoftext|>


<commit_before>package dbf

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
)

type StructCrack struct {
	Id            string `json:"id"`
	Type          string `json:"type"`
	Generator     string `json:"generator"`
	Cracker       string `json:"cracker"`
	Cmd_generator string `json:"cmd_generator"`
	Cmd_cracker   string `json:"cmd_cracker"`
	Target        string `json:"target"`
}

func processCrack(task *StructCrackTask, crackInfoPath *string) bool {
	var vendorPath, cmdJsonStr string
	var cmdArg []string
	var resultStatus int

	resultStatus = -1
	taskPath := getPath(_PATH_TASK) + task.Platform + PATH_SEPARATOR

	defer func() {
		resultByte, err := ioutil.ReadFile(taskPath + "result")
		if err != nil {
			Log.Printf("%s\n", err)
			resultByte = nil
			resultStatus = -2
		}

		fmt.Printf("Sending result of crack #%s...\n", task.Crack_id)

		if sendResult(`[{"crack_id":"`+task.Crack_id+`","start":"`+task.Start+`","offset":"`+task.Offset+`","result":"`+string(resultByte)+`","status":"`+strconv.Itoa(resultStatus)+`"}]`) == true {
			fmt.Printf("Removing task info of crack #%s (%s)...\n", task.Crack_id, task.Platform)
			err = os.RemoveAll(taskPath)
			if err != nil {
				Log.Printf("%s\n", err)
			}
		}
	}()

	crackJson, err := ioutil.ReadFile(*crackInfoPath)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -3
		return false
	}

	var crack StructCrack
	err = json.Unmarshal(crackJson, &crack)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -4
		return false
	}

	/* Process crack */
	// Check cracker
	if crack.Cracker != "" {
		vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_CRACKER + PATH_SEPARATOR + crack.Cracker + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR
		err := checkDir(vendorPath)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -5
			return false
		}
		vendorPath += _VENDOR_TYPE_CRACKER + extExecutable
		if checkVendor(_VENDOR_TYPE_CRACKER, &crack.Cracker, &task.Platform, &vendorPath) == false {
			resultStatus = -6
			return false
		}
	}

	// Check hashfile
	*crackInfoPath = filepath.Dir(*crackInfoPath) + PATH_SEPARATOR + "hashfile"
	if _, err := os.Stat(*crackInfoPath); err != nil {
		if os.IsNotExist(err) { // Does not exist, so create it
			err = ioutil.WriteFile(*crackInfoPath, []byte(crack.Target), 0664)
			if err != nil {
				Log.Printf("%s\n", err) // Error in creating
				resultStatus = -7
				return false
			}
		} else {
			Log.Printf("%s\n", err) // Error in accessing
			resultStatus = -8
			return false
		}
	}

	generatorReplacer := strings.NewReplacer("START", task.Start, "OFFSET", task.Offset, "IN_FILE", taskPath+"file.fifo")
	crackerReplacer := strings.NewReplacer("HASH_FILE", *crackInfoPath, "OUT_FILE", taskPath+"result", "IN_FILE", taskPath+"file.fifo")

	if crack.Type == "embed" { // Embeded
		cmdJsonStr = generatorReplacer.Replace(crack.Cmd_cracker)
		cmdJsonStr = crackerReplacer.Replace(cmdJsonStr)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -9
			return false
		}

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		err = exec.Command(vendorPath, cmdArg...).Run()
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -10
			return false
		} else {
			resultStatus = 0
			return true
		}
	} else { // Not embeded
		// Prepare cracker
		cmdJsonStr = crackerReplacer.Replace(crack.Cmd_cracker)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -11
			return false
		}
		execCracker := exec.Command(vendorPath, cmdArg...)

		// Check generator
		vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_GENERATOR + PATH_SEPARATOR + crack.Generator + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR
		err := checkDir(vendorPath)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -12
			return false
		}
		vendorPath += _VENDOR_TYPE_GENERATOR + extExecutable
		if checkVendor(_VENDOR_TYPE_GENERATOR, &crack.Generator, &task.Platform, &vendorPath) == false {
			resultStatus = -13
			return false
		}

		// Prepare generator
		cmdJsonStr = generatorReplacer.Replace(crack.Cmd_generator)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -14
			return false
		}
		execGenerator := exec.Command(vendorPath, cmdArg...)

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		if crack.Type == "infile" {
			err = execGenerator.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -15
				return false
			}

			err = execCracker.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -16
				return false
			}

			errG := execGenerator.Wait()
			errC := execCracker.Wait()
			if (errG != nil) || (errC != nil) {

				resultStatus = -16

				if errG != nil {
					Log.Printf("%s\n", errG)
					resultStatus += -1
				} else if errC != nil {
					Log.Printf("%s\n", errC)
					resultStatus += -2
				}

				// Max resultStatus: -19

				return false
			} else {
				resultStatus = 0
				return true
			}
		} else { // Stdin
			r, w := io.Pipe()
			execGenerator.Stdout = w
			execCracker.Stdin = r

			err = execGenerator.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -20
				return false
			}

			err = execCracker.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -21
				return false
			}

			errG := execGenerator.Wait()
			errW := w.Close()
			errC := execCracker.Wait()
			if (errG != nil) || (errW != nil) || (errC != nil) {
				resultStatus = -21

				if errG != nil {
					Log.Printf("%s\n", errG)
					resultStatus += -1
				} else if errW != nil {
					Log.Printf("%s\n", errW)
					resultStatus += -2
				} else if errC != nil {
					Log.Printf("%s\n", errC)
					resultStatus += -4
				}

				// Max resultStatus: -28

				return false
			} else {
				resultStatus = 0
				return true
			}
		}
	}

	return true
}
<commit_msg>Create fifo file before infile mode<commit_after>package dbf

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
)

type StructCrack struct {
	Id            string `json:"id"`
	Type          string `json:"type"`
	Generator     string `json:"generator"`
	Cracker       string `json:"cracker"`
	Cmd_generator string `json:"cmd_generator"`
	Cmd_cracker   string `json:"cmd_cracker"`
	Target        string `json:"target"`
}

func processCrack(task *StructCrackTask, crackInfoPath *string) bool {
	var vendorPath, cmdJsonStr string
	var cmdArg []string
	var resultStatus int

	resultStatus = -1
	taskPath := getPath(_PATH_TASK) + task.Platform + PATH_SEPARATOR

	defer func() {
		resultByte, err := ioutil.ReadFile(taskPath + "result")
		if err != nil {
			Log.Printf("%s\n", err)
			resultByte = nil
			resultStatus = -2
		}

		fmt.Printf("Sending result of crack #%s...\n", task.Crack_id)

		if sendResult(`[{"crack_id":"`+task.Crack_id+`","start":"`+task.Start+`","offset":"`+task.Offset+`","result":"`+string(resultByte)+`","status":"`+strconv.Itoa(resultStatus)+`"}]`) == true {
			fmt.Printf("Removing task info of crack #%s (%s)...\n", task.Crack_id, task.Platform)
			err = os.RemoveAll(taskPath)
			if err != nil {
				Log.Printf("%s\n", err)
			}
		}
	}()

	crackJson, err := ioutil.ReadFile(*crackInfoPath)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -3
		return false
	}

	var crack StructCrack
	err = json.Unmarshal(crackJson, &crack)
	if err != nil {
		Log.Printf("%s\n", err)
		resultStatus = -4
		return false
	}

	/* Process crack */
	// Check cracker
	if crack.Cracker != "" {
		vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_CRACKER + PATH_SEPARATOR + crack.Cracker + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR
		err := checkDir(vendorPath)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -5
			return false
		}
		vendorPath += _VENDOR_TYPE_CRACKER + extExecutable
		if checkVendor(_VENDOR_TYPE_CRACKER, &crack.Cracker, &task.Platform, &vendorPath) == false {
			resultStatus = -6
			return false
		}
	}

	// Check hashfile
	*crackInfoPath = filepath.Dir(*crackInfoPath) + PATH_SEPARATOR + "hashfile"
	if _, err := os.Stat(*crackInfoPath); err != nil {
		if os.IsNotExist(err) { // Does not exist, so create it
			err = ioutil.WriteFile(*crackInfoPath, []byte(crack.Target), 0664)
			if err != nil {
				Log.Printf("%s\n", err) // Error in creating
				resultStatus = -7
				return false
			}
		} else {
			Log.Printf("%s\n", err) // Error in accessing
			resultStatus = -8
			return false
		}
	}

	generatorReplacer := strings.NewReplacer("START", task.Start, "OFFSET", task.Offset, "IN_FILE", taskPath+"file.fifo")
	crackerReplacer := strings.NewReplacer("HASH_FILE", *crackInfoPath, "OUT_FILE", taskPath+"result", "IN_FILE", taskPath+"file.fifo")

	if crack.Type == "embed" { // Embeded
		cmdJsonStr = generatorReplacer.Replace(crack.Cmd_cracker)
		cmdJsonStr = crackerReplacer.Replace(cmdJsonStr)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -9
			return false
		}

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		err = exec.Command(vendorPath, cmdArg...).Run()
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -10
			return false
		} else {
			resultStatus = 0
			return true
		}
	} else { // Not embeded
		// Prepare cracker
		cmdJsonStr = crackerReplacer.Replace(crack.Cmd_cracker)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -11
			return false
		}
		execCracker := exec.Command(vendorPath, cmdArg...)

		// Check generator
		vendorPath = getPath(_PATH_VENDOR) + _VENDOR_TYPE_GENERATOR + PATH_SEPARATOR + crack.Generator + PATH_SEPARATOR + task.Platform + PATH_SEPARATOR
		err := checkDir(vendorPath)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -12
			return false
		}
		vendorPath += _VENDOR_TYPE_GENERATOR + extExecutable
		if checkVendor(_VENDOR_TYPE_GENERATOR, &crack.Generator, &task.Platform, &vendorPath) == false {
			resultStatus = -13
			return false
		}

		// Prepare generator
		cmdJsonStr = generatorReplacer.Replace(crack.Cmd_generator)
		err = json.Unmarshal([]byte(cmdJsonStr), &cmdArg)
		if err != nil {
			Log.Printf("%s\n", err)
			resultStatus = -14
			return false
		}
		execGenerator := exec.Command(vendorPath, cmdArg...)

		fmt.Printf("Performing crack #%s...\n", task.Crack_id)

		if crack.Type == "infile" {
			err = exec.Command("mkfifo", taskPath+"file.fifo").Run()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -15
				return false
			}

			err = execGenerator.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -16
				return false
			}

			err = execCracker.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -17
				return false
			}

			errG := execGenerator.Wait()
			errC := execCracker.Wait()
			if (errG != nil) || (errC != nil) {

				resultStatus = -17

				if errG != nil {
					Log.Printf("%s\n", errG)
					resultStatus += -1
				} else if errC != nil {
					Log.Printf("%s\n", errC)
					resultStatus += -2
				}

				// Max resultStatus: -20

				return false
			} else {
				resultStatus = 0
				return true
			}
		} else { // Stdin
			r, w := io.Pipe()
			execGenerator.Stdout = w
			execCracker.Stdin = r

			err = execGenerator.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -21
				return false
			}

			err = execCracker.Start()
			if err != nil {
				Log.Printf("%s\n", err)
				resultStatus = -22
				return false
			}

			errG := execGenerator.Wait()
			errW := w.Close()
			errC := execCracker.Wait()
			if (errG != nil) || (errW != nil) || (errC != nil) {
				resultStatus = -22

				if errG != nil {
					Log.Printf("%s\n", errG)
					resultStatus += -1
				} else if errW != nil {
					Log.Printf("%s\n", errW)
					resultStatus += -2
				} else if errC != nil {
					Log.Printf("%s\n", errC)
					resultStatus += -4
				}

				// Max resultStatus: -29

				return false
			} else {
				resultStatus = 0
				return true
			}
		}
	}

	return true
}
<|endoftext|>


<commit_before>package domain

import (
	"time"
	"github.com/pborman/uuid"
	"github.com/galahade/bus_staff_managment/util"

	"database/sql/driver"
	"strings"
	"errors"
	"database/sql"
	"fmt"
)

type PhoneNumbers []string
type JobType byte

type Staff struct {
	Domain
	Name                     string
	StaffId                  string
	JobType                  JobType
	OnboardTime              time.Time
	PersonalID               string
	DriverType               string
	IsInternship             bool
	IsMultiTimeHired         bool
	FirstOnboardTime         time.Time
	Phone                    PhoneNumbers
	Department               string
	EmergencyContact         string
	EmergencyContactPhone    PhoneNumbers
	EmergencyContactRelation string
}

func (staff Staff) InsertString() string {
	return "INSERT STAFF SET ID=?, NAME=?, STAFF_ID=?, JOB_TYPE=?, ONBOARD_TIME=?, PERSONAL_ID=?, DRIVER_TYPE=?, " +
		"IS_INTERNSHIP=?, IS_MULTITIME_HIRED=?, FIRST_ONBOARD_TIME=?, PHONE=?, DEPARTMENT=?, EMERGENCY_CONTACT=?, " +
		"EMERGENCY_CONTACT_PHONE=?, EMERGENCY_CONTACT_RELATION=?"
}
func (staff Staff) QueryAllString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM STAFF"
}
func (staff Staff) QueryByIDString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM STAFF WHERE ID = ?"
}
func (staff Staff) QueryByStaffIDString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM STAFF WHERE STAFF_ID = ?"
}
func (staff Staff)  QueryByJobTypeString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM STAFF WHERE JOB_TYPE = ?"
}

func (staff Staff) DeleteByIdString() string {
	return "DELETE FROM STAFF WHERE ID = ?"
}
// map go type to db type
func (phoneNumbers PhoneNumbers) Value() (driver.Value, error) {
	var phones string
	for _, phone := range phoneNumbers {
		if phones == "" {
			phones = phone
		} else {
			phones = phones + "," + phone
		}
	}
	return phones, nil
}
func (jobType JobType) Value() (driver.Value, error) {
	var jobTypeDB []byte
	jobTypeDB = append(jobTypeDB, byte(jobType))
	return jobTypeDB, nil
}

func (phoneNumbers *PhoneNumbers) Scan(value interface{}) error {
	if value == nil {
		*phoneNumbers = make([]string, 0)
		return nil
	}
	if phones, err := driver.String.ConvertValue(value); err == nil {
		if v, ok := phones.(string); ok {
			results := strings.Split(v, ",")
			*phoneNumbers = results
		} else if v, ok := phones.([]byte); ok {
			temp := string(v)
			results := strings.Split(temp, ",")
			*phoneNumbers = results
		}
		return nil
	}
	return errors.New("failed to scan PhoneNumbers.")
}
func (jobType *JobType) Scan(value interface{}) error {
	if value == nil {
		*jobType = 0
		return nil
	}
	if v, ok := value.([]byte); ok {
		*jobType = JobType(v[0])
		return nil
	}
	return errors.New("failed to scan JobType.")
}

func (staff *Staff) Insert() {

	stmtP, err := db.Prepare(staff.InsertString())
	util.CheckErr(err)

	res, err := stmtP.Exec(uuid.NewUUID(), staff.Name, staff.StaffId, staff.JobType, staff.OnboardTime,
		staff.PersonalID, staff.DriverType, staff.IsInternship, staff.IsMultiTimeHired, staff.FirstOnboardTime,
		staff.Phone, staff.Department, staff.EmergencyContact, staff.EmergencyContactPhone, staff.EmergencyContactRelation)
	util.CheckErr(err)

	if affected, _ := res.RowsAffected(); affected == 0 {
		fmt.Errorf("Fail to insert staff data into db.\n")
	}
}

func (staff *Staff) QueryByID() {
	stmtP, err := db.Prepare(staff.QueryByIDString())
	util.CheckErr(err)

	err = stmtP.QueryRow(staff.ID).Scan(&staff.ID, &staff.Name, &staff.StaffId, &staff.JobType, &staff.OnboardTime, &staff.PersonalID, &staff.DriverType,
		&staff.IsInternship, &staff.IsMultiTimeHired, &staff.FirstOnboardTime, &staff.Phone, &staff.Department, &staff.EmergencyContact,
		&staff.EmergencyContactPhone, &staff.EmergencyContactRelation)
	util.CheckErr(err)

}

func (staff *Staff) QueryByStaffID() error {
	stmtP, err := db.Prepare(staff.QueryByStaffIDString())
	util.CheckErr(err)

	err = stmtP.QueryRow(staff.StaffId).Scan(&staff.ID, &staff.Name, &staff.StaffId, &staff.JobType, &staff.OnboardTime, &staff.PersonalID, &staff.DriverType,
		&staff.IsInternship, &staff.IsMultiTimeHired, &staff.FirstOnboardTime, &staff.Phone, &staff.Department, &staff.EmergencyContact,
		&staff.EmergencyContactPhone, &staff.EmergencyContactRelation)
	if(err == sql.ErrNoRows) {
		return err
	} else {
		util.CheckErr(err)
	}
	return nil
}

func (staff *Staff) QueryAll() []Staff {
	rows, err := db.Query(staff.QueryAllString())
	util.CheckErr(err)
	return scanQueryResult(rows)

}

func (staff Staff) QueryByJobType(jobType byte) []Staff {
	stmtP, err := db.Prepare(staff.QueryByJobTypeString())
	util.CheckErr(err)

	rows, err := stmtP.Query(jobType)

	return scanQueryResult(rows)
}

func (staff *Staff) DeleteById() {
	stmtP, err := db.Prepare(staff.DeleteByIdString())
	util.CheckErr(err)

	res, err := stmtP.Exec(staff.ID)
	checkChangeDBFailed(res, err, "")
}

func (staff Staff) String() string {
	return fmt.Sprintf("Staff data are : \n id : %s,\n name : %s,\n staffId : %s,\n jobType : %d,\n onboardTime : %s,\n PersonalId : %s,\n " +
		"DriverType : %s,\n IsInternship : %t,\n isMultiTimeHired : %t,\n firstOnboardTime : %s,\n phone : %s,\n department : %s,\n " +
		"emergencyContact : %s,\n emergencyContactPhone : %s,\n emergencyContactRelation : %s,\n",
		staff.ID, staff.Name, staff.StaffId, staff.JobType, staff.OnboardTime, staff.PersonalID, staff.DriverType, staff.IsInternship,
		staff.IsMultiTimeHired, staff.FirstOnboardTime, staff.Phone, staff.Department, staff.EmergencyContact, staff.EmergencyContactPhone,
		staff.EmergencyContactRelation)
}

func (staff Staff) IsQualified() (result bool) {
	result = false
	if staff.IsInternship {
		return
	}
	if strings.Contains(staff.DriverType, "A1") || strings.Contains(staff.DriverType, "A3") {
		result = true
	}
	return
}

func (staff Staff) NeedUpgrade() (result bool) {
	result = false
	if !staff.IsQualified() && !staff.IsInternship && staff.Department == "运营" {
		result = true
	}
	return
}

func scanQueryResult(rows *sql.Rows) []Staff {

	var staffs []Staff
	for rows.Next() {
		staffP := new(Staff)

		err = rows.Scan(&staffP.ID, &staffP.Name, &staffP.StaffId, &staffP.JobType, &staffP.OnboardTime, &staffP.PersonalID,
			&staffP.DriverType, &staffP.IsInternship, &staffP.IsMultiTimeHired, &staffP.FirstOnboardTime,
			&staffP.Phone, &staffP.Department, &staffP.EmergencyContact, &staffP.EmergencyContactPhone, &staffP.EmergencyContactRelation)
		util.CheckErr(err)

		staffs = append(staffs, *staffP)
	}

	return staffs
}<commit_msg>change staff sql capital<commit_after>package domain

import (
	"time"
	"github.com/pborman/uuid"
	"github.com/galahade/bus_staff_managment/util"

	"database/sql/driver"
	"strings"
	"errors"
	"database/sql"
	"fmt"
)

type PhoneNumbers []string
type JobType byte

type Staff struct {
	Domain
	Name                     string
	StaffId                  string
	JobType                  JobType
	OnboardTime              time.Time
	PersonalID               string
	DriverType               string
	IsInternship             bool
	IsMultiTimeHired         bool
	FirstOnboardTime         time.Time
	Phone                    PhoneNumbers
	Department               string
	EmergencyContact         string
	EmergencyContactPhone    PhoneNumbers
	EmergencyContactRelation string
}

func (staff Staff) InsertString() string {
	return "INSERT staff SET ID=?, NAME=?, STAFF_ID=?, JOB_TYPE=?, ONBOARD_TIME=?, PERSONAL_ID=?, DRIVER_TYPE=?, " +
		"IS_INTERNSHIP=?, IS_MULTITIME_HIRED=?, FIRST_ONBOARD_TIME=?, PHONE=?, DEPARTMENT=?, EMERGENCY_CONTACT=?, " +
		"EMERGENCY_CONTACT_PHONE=?, EMERGENCY_CONTACT_RELATION=?"
}
func (staff Staff) QueryAllString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM staff"
}
func (staff Staff) QueryByIDString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM staff WHERE ID = ?"
}
func (staff Staff) QueryByStaffIDString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM staff WHERE STAFF_ID = ?"
}
func (staff Staff)  QueryByJobTypeString() string {
	return "SELECT ID, NAME, STAFF_ID, JOB_TYPE, ONBOARD_TIME, PERSONAL_ID, DRIVER_TYPE, " +
		"IS_INTERNSHIP, IS_MULTITIME_HIRED, FIRST_ONBOARD_TIME, PHONE, DEPARTMENT, EMERGENCY_CONTACT, " +
		"EMERGENCY_CONTACT_PHONE, EMERGENCY_CONTACT_RELATION FROM staff WHERE JOB_TYPE = ?"
}

func (staff Staff) DeleteByIdString() string {
	return "DELETE FROM staff WHERE ID = ?"
}
// map go type to db type
func (phoneNumbers PhoneNumbers) Value() (driver.Value, error) {
	var phones string
	for _, phone := range phoneNumbers {
		if phones == "" {
			phones = phone
		} else {
			phones = phones + "," + phone
		}
	}
	return phones, nil
}
func (jobType JobType) Value() (driver.Value, error) {
	var jobTypeDB []byte
	jobTypeDB = append(jobTypeDB, byte(jobType))
	return jobTypeDB, nil
}

func (phoneNumbers *PhoneNumbers) Scan(value interface{}) error {
	if value == nil {
		*phoneNumbers = make([]string, 0)
		return nil
	}
	if phones, err := driver.String.ConvertValue(value); err == nil {
		if v, ok := phones.(string); ok {
			results := strings.Split(v, ",")
			*phoneNumbers = results
		} else if v, ok := phones.([]byte); ok {
			temp := string(v)
			results := strings.Split(temp, ",")
			*phoneNumbers = results
		}
		return nil
	}
	return errors.New("failed to scan PhoneNumbers.")
}
func (jobType *JobType) Scan(value interface{}) error {
	if value == nil {
		*jobType = 0
		return nil
	}
	if v, ok := value.([]byte); ok {
		*jobType = JobType(v[0])
		return nil
	}
	return errors.New("failed to scan JobType.")
}

func (staff *Staff) Insert() {

	stmtP, err := db.Prepare(staff.InsertString())
	util.CheckErr(err)

	res, err := stmtP.Exec(uuid.NewUUID(), staff.Name, staff.StaffId, staff.JobType, staff.OnboardTime,
		staff.PersonalID, staff.DriverType, staff.IsInternship, staff.IsMultiTimeHired, staff.FirstOnboardTime,
		staff.Phone, staff.Department, staff.EmergencyContact, staff.EmergencyContactPhone, staff.EmergencyContactRelation)
	util.CheckErr(err)

	if affected, _ := res.RowsAffected(); affected == 0 {
		fmt.Errorf("Fail to insert staff data into db.\n")
	}
}

func (staff *Staff) QueryByID() {
	stmtP, err := db.Prepare(staff.QueryByIDString())
	util.CheckErr(err)

	err = stmtP.QueryRow(staff.ID).Scan(&staff.ID, &staff.Name, &staff.StaffId, &staff.JobType, &staff.OnboardTime, &staff.PersonalID, &staff.DriverType,
		&staff.IsInternship, &staff.IsMultiTimeHired, &staff.FirstOnboardTime, &staff.Phone, &staff.Department, &staff.EmergencyContact,
		&staff.EmergencyContactPhone, &staff.EmergencyContactRelation)
	util.CheckErr(err)

}

func (staff *Staff) QueryByStaffID() error {
	stmtP, err := db.Prepare(staff.QueryByStaffIDString())
	util.CheckErr(err)

	err = stmtP.QueryRow(staff.StaffId).Scan(&staff.ID, &staff.Name, &staff.StaffId, &staff.JobType, &staff.OnboardTime, &staff.PersonalID, &staff.DriverType,
		&staff.IsInternship, &staff.IsMultiTimeHired, &staff.FirstOnboardTime, &staff.Phone, &staff.Department, &staff.EmergencyContact,
		&staff.EmergencyContactPhone, &staff.EmergencyContactRelation)
	if(err == sql.ErrNoRows) {
		return err
	} else {
		util.CheckErr(err)
	}
	return nil
}

func (staff *Staff) QueryAll() []Staff {
	rows, err := db.Query(staff.QueryAllString())
	util.CheckErr(err)
	return scanQueryResult(rows)

}

func (staff Staff) QueryByJobType(jobType byte) []Staff {
	stmtP, err := db.Prepare(staff.QueryByJobTypeString())
	util.CheckErr(err)

	rows, err := stmtP.Query(jobType)

	return scanQueryResult(rows)
}

func (staff *Staff) DeleteById() {
	stmtP, err := db.Prepare(staff.DeleteByIdString())
	util.CheckErr(err)

	res, err := stmtP.Exec(staff.ID)
	checkChangeDBFailed(res, err, "")
}

func (staff Staff) String() string {
	return fmt.Sprintf("Staff data are : \n id : %s,\n name : %s,\n staffId : %s,\n jobType : %d,\n onboardTime : %s,\n PersonalId : %s,\n " +
		"DriverType : %s,\n IsInternship : %t,\n isMultiTimeHired : %t,\n firstOnboardTime : %s,\n phone : %s,\n department : %s,\n " +
		"emergencyContact : %s,\n emergencyContactPhone : %s,\n emergencyContactRelation : %s,\n",
		staff.ID, staff.Name, staff.StaffId, staff.JobType, staff.OnboardTime, staff.PersonalID, staff.DriverType, staff.IsInternship,
		staff.IsMultiTimeHired, staff.FirstOnboardTime, staff.Phone, staff.Department, staff.EmergencyContact, staff.EmergencyContactPhone,
		staff.EmergencyContactRelation)
}

func (staff Staff) IsQualified() (result bool) {
	result = false
	if staff.IsInternship {
		return
	}
	if strings.Contains(staff.DriverType, "A1") || strings.Contains(staff.DriverType, "A3") {
		result = true
	}
	return
}

func (staff Staff) NeedUpgrade() (result bool) {
	result = false
	if !staff.IsQualified() && !staff.IsInternship && staff.Department == "运营" {
		result = true
	}
	return
}

func scanQueryResult(rows *sql.Rows) []Staff {

	var staffs []Staff
	for rows.Next() {
		staffP := new(Staff)

		err = rows.Scan(&staffP.ID, &staffP.Name, &staffP.StaffId, &staffP.JobType, &staffP.OnboardTime, &staffP.PersonalID,
			&staffP.DriverType, &staffP.IsInternship, &staffP.IsMultiTimeHired, &staffP.FirstOnboardTime,
			&staffP.Phone, &staffP.Department, &staffP.EmergencyContact, &staffP.EmergencyContactPhone, &staffP.EmergencyContactRelation)
		util.CheckErr(err)

		staffs = append(staffs, *staffP)
	}

	return staffs
}<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/asadovsky/gosh"

	"github.com/asadovsky/cdb/server/hub"
)

var (
	loopback  = flag.Bool("loopback", true, "")
	port      = flag.Int("port", 4000, "")
	peerAddrs = flag.String("peer-addrs", "", "comma-separated peer addrs")
)

var serve = gosh.RegisterFunc("serve", hub.Serve)

func ok(err error) {
	if err != nil {
		panic(err)
	}
}

func ip() (string, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return "", err
	}
	for _, a := range addrs {
		if x, ok := a.(*net.IPNet); ok && !x.IP.IsLoopback() && x.IP.To4() != nil {
			return x.IP.String(), nil
		}
	}
	return "", errors.New("not found")
}

func main() {
	gosh.InitMain()
	flag.Parse()
	sh := gosh.NewShell(nil)
	defer sh.Cleanup()
	cwd, err := os.Getwd()
	ok(err)
	hostname := "localhost"
	if !*loopback {
		hostname, err = ip()
		ok(err)
	}
	addr := fmt.Sprintf("%s:%d", hostname, *port)
	httpAddr := fmt.Sprintf("%s:%d", hostname, *port+100)
	c := sh.FuncCmd(serve, addr, strings.Split(*peerAddrs, ","))
	c.AddStderrWriter(os.Stderr)
	c.Start()
	c.AwaitVars("ready")
	// Note, the "open" command doesn't support query strings in file urls.
	fmt.Printf("http://%s/demo/index.html?addr=%s\n", httpAddr, url.QueryEscape(addr))
	ok(http.ListenAndServe(httpAddr, http.FileServer(http.Dir(filepath.Join(cwd)))))
	c.Wait()
}
<commit_msg>remove extraneous filepath.Join<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/asadovsky/gosh"

	"github.com/asadovsky/cdb/server/hub"
)

var (
	loopback  = flag.Bool("loopback", true, "")
	port      = flag.Int("port", 4000, "")
	peerAddrs = flag.String("peer-addrs", "", "comma-separated peer addrs")
)

var serve = gosh.RegisterFunc("serve", hub.Serve)

func ok(err error) {
	if err != nil {
		panic(err)
	}
}

func ip() (string, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return "", err
	}
	for _, a := range addrs {
		if x, ok := a.(*net.IPNet); ok && !x.IP.IsLoopback() && x.IP.To4() != nil {
			return x.IP.String(), nil
		}
	}
	return "", errors.New("not found")
}

func main() {
	gosh.InitMain()
	flag.Parse()
	sh := gosh.NewShell(nil)
	defer sh.Cleanup()
	cwd, err := os.Getwd()
	ok(err)
	hostname := "localhost"
	if !*loopback {
		hostname, err = ip()
		ok(err)
	}
	addr := fmt.Sprintf("%s:%d", hostname, *port)
	httpAddr := fmt.Sprintf("%s:%d", hostname, *port+100)
	c := sh.FuncCmd(serve, addr, strings.Split(*peerAddrs, ","))
	c.AddStderrWriter(os.Stderr)
	c.Start()
	c.AwaitVars("ready")
	// Note, the "open" command doesn't support query strings in file urls.
	fmt.Printf("http://%s/demo/index.html?addr=%s\n", httpAddr, url.QueryEscape(addr))
	ok(http.ListenAndServe(httpAddr, http.FileServer(http.Dir(cwd))))
	c.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package client is a go client library for socketservice.Service.
package client

import (
	"os"
	"time"

	log "github.com/golang/glog"
	"github.com/golang/protobuf/ptypes"
	"github.com/google/fleetspeak/fleetspeak/src/client/channel"

	fcpb "github.com/google/fleetspeak/fleetspeak/src/client/channel/proto/fleetspeak_channel"
	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
)

// Back-off constants for retrying channel connections.
// Current values will result in the following sequence of
// delays (seconds, rounded off): [1, 1.5, 2.3, 3.4, 5.0, 7.6, 11.4, 15, 15, 15, ...]
const (
	maxChannelRetryDelay      = 15 * time.Second
	channelRetryBackoffFactor = 1.5
)

func backOffChannelRetryDelay(currentDelay time.Duration) time.Duration {
	newDelay := time.Duration(float64(currentDelay) * channelRetryBackoffFactor)
	if newDelay > maxChannelRetryDelay {
		return maxChannelRetryDelay
	} else {
		return newDelay
	}
}

// OpenChannel creates a channel.RelentlessChannel to a fleetspeak client
// through an agreed upon unix domain socket.
func OpenChannel(socketPath string, version string) *channel.RelentlessChannel {
	return channel.NewRelentlessChannel(
		func() (*channel.Channel, func()) {
			sd, err := ptypes.MarshalAny(&fcpb.StartupData{Pid: int64(os.Getpid()), Version: version})
			if err != nil {
				log.Fatalf("unable to marshal StartupData: %v", err)
			}
			m := &fspb.Message{
				MessageType: "StartupData",
				Destination: &fspb.Address{ServiceName: "system"},
				Data:        sd,
			}
		L:
			for {
				ch, fin := buildChannel(socketPath)
				if ch == nil {
					return ch, fin
				}
				select {
				case e := <-ch.Err:
					log.Errorf("Channel failed with error: %v", e)
					continue L
				case ch.Out <- m:
					return ch, fin
				}
			}
		})
}
<commit_msg>Tear down the freshly opened channel (#192)<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package client is a go client library for socketservice.Service.
package client

import (
	"os"
	"time"

	log "github.com/golang/glog"
	"github.com/golang/protobuf/ptypes"
	"github.com/google/fleetspeak/fleetspeak/src/client/channel"

	fcpb "github.com/google/fleetspeak/fleetspeak/src/client/channel/proto/fleetspeak_channel"
	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
)

// Back-off constants for retrying channel connections.
// Current values will result in the following sequence of
// delays (seconds, rounded off): [1, 1.5, 2.3, 3.4, 5.0, 7.6, 11.4, 15, 15, 15, ...]
const (
	maxChannelRetryDelay      = 15 * time.Second
	channelRetryBackoffFactor = 1.5
)

func backOffChannelRetryDelay(currentDelay time.Duration) time.Duration {
	newDelay := time.Duration(float64(currentDelay) * channelRetryBackoffFactor)
	if newDelay > maxChannelRetryDelay {
		return maxChannelRetryDelay
	} else {
		return newDelay
	}
}

// OpenChannel creates a channel.RelentlessChannel to a fleetspeak client
// through an agreed upon unix domain socket.
func OpenChannel(socketPath string, version string) *channel.RelentlessChannel {
	return channel.NewRelentlessChannel(
		func() (*channel.Channel, func()) {
			sd, err := ptypes.MarshalAny(&fcpb.StartupData{Pid: int64(os.Getpid()), Version: version})
			if err != nil {
				log.Fatalf("unable to marshal StartupData: %v", err)
			}
			m := &fspb.Message{
				MessageType: "StartupData",
				Destination: &fspb.Address{ServiceName: "system"},
				Data:        sd,
			}
		L:
			for {
				ch, fin := buildChannel(socketPath)
				if ch == nil {
					return ch, fin
				}
				select {
				case e := <-ch.Err:
					log.Errorf("Channel failed with error: %v", e)
					fin()
					continue L
				case ch.Out <- m:
					return ch, fin
				}
			}
		})
}
<|endoftext|>


<commit_before>package xing

import (
	"github.com/ghts/ghts/lib"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"
	"time"
)

func main() {
	lib.F확인(xing.F초기화(xt.P서버_모의투자))
	defer xing.F종료()

	lib.F확인(xing.F주문_응답_실시간_정보_구독())
	defer xing.F주문_응답_실시간_정보_해지()

	시각, 에러 := (<-xing.TrT0167_시각_조회()).G값()
	lib.F확인(에러)

	차이 := time.Now().Sub(시각)

	if 차이 > 0 {
		차이 = 차이 * -1
	}

	lib.F조건부_패닉(차이 > lib.P1시간, "차이가 너무 큽니다. '%v'", 차이)
}
<commit_msg>미완성 샘플 삭제<commit_after><|endoftext|>


<commit_before>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package main

import (
  "os"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instances"
  "time"
  "flag"
  "sort"
  "encoding/json"
  "io/ioutil"
)

type MembersByOnlineNodes []instances.Instance
var debug = false

var InfoLog = log.New(os.Stderr,
              "INFO: ",
              log.Ldate|log.Ltime|log.Lshortfile)

var DebugLog = log.New(os.Stderr,
               "DEBUG: ",
               log.Ldate|log.Ltime|log.Lshortfile)

func main(){
  var seed_host string
  var seed_port string
  var mysql_user string
  var mysql_pass string
  var mysql_auth_file string
  type json_mysql_auth struct {
    User string
    Password string
  }

  flag.StringVar( &seed_host, "seed_host", "", "IP/Hostname of the seed node used to start monitoring the Group Replication cluster" )
  flag.StringVar( &seed_port, "seed_port", "3306", "Port of the seed node used to start monitoring the Group Replication cluster" )
  flag.BoolVar( &debug, "debug", false, "Execute in debug mode with all debug logging enabled" )
  flag.StringVar( &mysql_user, "mysql_user", "root", "The mysql user account to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_pass, "mysql_password", "", "The mysql user account password to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_auth_file, "mysql_auth_file", "", "The JSON encoded file containining user and password entities for the mysql account to be used when connecting to any node in the cluster" )

  flag.Parse()

  // ToDo: I need to handle the password on the command-line more securely
  //       I need to do some data masking for the processlist

  // A host is required, the default port of 3306 will then be attempted
  if( seed_host == "" ){
    log.Fatal( "myarbitratord usage: myarbitratord -seed_host=<seed_host> [-seed_port=<seed_port>] [-mysql_user=<mysql_user>] [-mysql_password=<mysql_pass>] [-mysql_auth_file=<path to json file>] [-debug=true]" )
  }

  if( debug ){
    instances.Debug = true
  }

  if( mysql_auth_file != "" && mysql_pass == "" ){
    if( debug ){
      DebugLog.Printf( "Reading MySQL credentials from file: %s\n", mysql_auth_file )
    }

    jsonfile, err := ioutil.ReadFile( mysql_auth_file )

    if( err != nil ){
      log.Fatal( "Could not read mysql credentials from specified file: " + mysql_auth_file )
    }

    var jsonauth json_mysql_auth
    json.Unmarshal( jsonfile, &jsonauth )

    if( debug ){
      DebugLog.Printf( "Unmarshaled mysql auth file contents: %+v\n", jsonauth )
    }

    mysql_user = jsonauth.User
    mysql_pass = jsonauth.Password

    if( mysql_user == "" || mysql_pass == "" ){
      errstr := "Failed to read user and password from " + mysql_auth_file + ". Ensure that the file contents are in the required format: \n{\n  \"user\": \"myser\",\n  \"password\": \"mypass\"\n}"
      log.Fatal( errstr )
    }

    if( debug ){
      DebugLog.Printf( "Read mysql auth info from file. user: %s, password: %s\n", mysql_user, mysql_pass )
    }
  }

  InfoLog.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  InfoLog.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instances.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instances.Instance ) error {
  loop := true
  var err error
  last_view := []instances.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err = seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      InfoLog.Println( "Attempting to get a new seed node..." )

      for _, member := range last_view {
        err = member.Connect()
        if( err == nil && member.Member_state == "ONLINE" ){
          seed_node = &member
          InfoLog.Printf( "Updated seed node! New seed node is: '%s:%s'\n", seed_node.Mysql_host, seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members[:] )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    if( debug ){
      DebugLog.Printf( "Seed node details: %+v", seed_node )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          InfoLog.Printf( "Shutting down non-healthy node: '%s:%s'\n", member.Mysql_host, member.Mysql_port )

          err = member.Connect()

          if( err != nil ){
            InfoLog.Printf( "Could not connect to '%s:%s' in order to shut it down\n", member.Mysql_host, member.Mysql_port )
          }

          err = member.Shutdown()

          if( err != nil ){
            InfoLog.Printf( "Could not shutdown instance: '%s:%s'\n", member.Mysql_host, member.Mysql_port )
          }
        }
      }
    } else {
      // handling other network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one. We'll then need to contact 1 node in the new primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"'. Finally we'll need to try
      // and connect to the nodes on the losing side(s) of the partition and attempt to shutdown the mysqlds

      InfoLog.Println( "Network partition detected! Attempting to handle... " )

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false

      for _, member := range last_view {
        member.Connect()
        quorum, err := member.HasQuorum()
        if( err == nil && quorum ){
          seed_node = &member
          primary_partition = true
          break
        }
      }

      // If no one in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members. The participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster
      // ToDo: should we consider GTID_EXECUTED sets when choosing the primary partition???
      if( primary_partition == false ){
        InfoLog.Println( "No primary partition found! Attempting to choose and force a new one ... " )

        sort.Sort( MembersByOnlineNodes(last_view) )
        // now the last element in the array is the one to use as it's coordinating with the most nodes
        seed_node = &last_view[len(last_view)-1]

        err = seed_node.Connect()

        if( err != nil ){
          // let's just loop again
          continue
        }

        if( debug ){
          DebugLog.Printf( "Member view sorted by number of online nodes: %+v\n", last_view )
        }

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()

        force_member_string := ""

        for i, member := range *members {
          err = member.Connect()

          if( err == nil && member.Member_state == "ONLINE" ){
            if( i != 0 ){
              force_member_string = force_member_string + ","
            }
            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          }
        }

        if( force_member_string != "" ){
          InfoLog.Printf( "Forcing group membership to form new primary partition! Using: '%s'\n", force_member_string )
          err = seed_node.ForceMembers( force_member_string )
        } else {
          InfoLog.Println( "No valid group membership to force!" )
        }
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<commit_msg>Added experimental warning note<commit_after>/*
Copyright 2017 Matthew Lord (mattalord@gmail.com)

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

// WARNING: This is experimental and for demonstration purposes only!

package main

import (
  "os"
  "log"
  "github.com/mattlord/myarbitratord/group_replication/instances"
  "time"
  "flag"
  "sort"
  "encoding/json"
  "io/ioutil"
)

type MembersByOnlineNodes []instances.Instance
var debug = false

var InfoLog = log.New(os.Stderr,
              "INFO: ",
              log.Ldate|log.Ltime|log.Lshortfile)

var DebugLog = log.New(os.Stderr,
               "DEBUG: ",
               log.Ldate|log.Ltime|log.Lshortfile)

func main(){
  var seed_host string
  var seed_port string
  var mysql_user string
  var mysql_pass string
  var mysql_auth_file string
  type json_mysql_auth struct {
    User string
    Password string
  }

  flag.StringVar( &seed_host, "seed_host", "", "IP/Hostname of the seed node used to start monitoring the Group Replication cluster" )
  flag.StringVar( &seed_port, "seed_port", "3306", "Port of the seed node used to start monitoring the Group Replication cluster" )
  flag.BoolVar( &debug, "debug", false, "Execute in debug mode with all debug logging enabled" )
  flag.StringVar( &mysql_user, "mysql_user", "root", "The mysql user account to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_pass, "mysql_password", "", "The mysql user account password to be used when connecting to any node in the cluster" )
  flag.StringVar( &mysql_auth_file, "mysql_auth_file", "", "The JSON encoded file containining user and password entities for the mysql account to be used when connecting to any node in the cluster" )

  flag.Parse()

  // ToDo: I need to handle the password on the command-line more securely
  //       I need to do some data masking for the processlist

  // A host is required, the default port of 3306 will then be attempted
  if( seed_host == "" ){
    log.Fatal( "myarbitratord usage: myarbitratord -seed_host=<seed_host> [-seed_port=<seed_port>] [-mysql_user=<mysql_user>] [-mysql_password=<mysql_pass>] [-mysql_auth_file=<path to json file>] [-debug=true]" )
  }

  if( debug ){
    instances.Debug = true
  }

  if( mysql_auth_file != "" && mysql_pass == "" ){
    if( debug ){
      DebugLog.Printf( "Reading MySQL credentials from file: %s\n", mysql_auth_file )
    }

    jsonfile, err := ioutil.ReadFile( mysql_auth_file )

    if( err != nil ){
      log.Fatal( "Could not read mysql credentials from specified file: " + mysql_auth_file )
    }

    var jsonauth json_mysql_auth
    json.Unmarshal( jsonfile, &jsonauth )

    if( debug ){
      DebugLog.Printf( "Unmarshaled mysql auth file contents: %+v\n", jsonauth )
    }

    mysql_user = jsonauth.User
    mysql_pass = jsonauth.Password

    if( mysql_user == "" || mysql_pass == "" ){
      errstr := "Failed to read user and password from " + mysql_auth_file + ". Ensure that the file contents are in the required format: \n{\n  \"user\": \"myser\",\n  \"password\": \"mypass\"\n}"
      log.Fatal( errstr )
    }

    if( debug ){
      DebugLog.Printf( "Read mysql auth info from file. user: %s, password: %s\n", mysql_user, mysql_pass )
    }
  }

  InfoLog.Println( "Welcome to the MySQL Group Replication Arbitrator!" )

  InfoLog.Printf( "Starting operations from seed node: '%s:%s'\n", seed_host, seed_port )

  seed_node := instances.New( seed_host, seed_port, mysql_user, mysql_pass )
  err := MonitorCluster( seed_node )

  if( err != nil ){
    log.Fatal( err )
    os.Exit( 100 )
  } else {
    os.Exit( 0 )
  }
}

func MonitorCluster( seed_node *instances.Instance ) error {
  loop := true
  var err error
  last_view := []instances.Instance{}

  for( loop == true ){
    // let's check the status of the current seed node
    // if the seed node
    err = seed_node.Connect()

    if( err != nil || seed_node.Member_state != "ONLINE" ){
      // if we couldn't connect to the current seed node or it's no longer part of the group
      // let's try and get a new seed node from the last known membership view
      InfoLog.Println( "Attempting to get a new seed node..." )

      for _, member := range last_view {
        err = member.Connect()
        if( err == nil && member.Member_state == "ONLINE" ){
          seed_node = &member
          InfoLog.Printf( "Updated seed node! New seed node is: '%s:%s'\n", seed_node.Mysql_host, seed_node.Mysql_port )
          break
        }
      }
    }

    members, err := seed_node.GetMembers()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    // save this view in case the seed node is no longer valid next time
    //last_view = copy( last_view, members[:] )
    last_view = *members

    quorum, err := seed_node.HasQuorum()

    if( err != nil ){
      // something is up with our current seed node, let's loop again
      continue
    }

    if( debug ){
      DebugLog.Printf( "Seed node details: %+v", seed_node )
    }

    if( quorum ){
      // Let's try and shutdown the nodes NOT in the primary partition if we can reach them from the arbitrator

      for _, member := range *members {
        if( member.Member_state == "ERROR" || member.Member_state == "UNREACHABLE" ){
          InfoLog.Printf( "Shutting down non-healthy node: '%s:%s'\n", member.Mysql_host, member.Mysql_port )

          err = member.Connect()

          if( err != nil ){
            InfoLog.Printf( "Could not connect to '%s:%s' in order to shut it down\n", member.Mysql_host, member.Mysql_port )
          }

          err = member.Shutdown()

          if( err != nil ){
            InfoLog.Printf( "Could not shutdown instance: '%s:%s'\n", member.Mysql_host, member.Mysql_port )
          }
        }
      }
    } else {
      // handling other network partitions and split brain scenarios will be much trickier... I'll need to try and
      // contact each member in the last seen view and try to determine which partition should become the
      // primary one. We'll then need to contact 1 node in the new primary partition and explicitly set the new
      // membership with 'set global group_replication_force_members="<node_list>"'. Finally we'll need to try
      // and connect to the nodes on the losing side(s) of the partition and attempt to shutdown the mysqlds

      InfoLog.Println( "Network partition detected! Attempting to handle... " )

      // does anyone have a quorum? Let's double check before forcing the membership
      primary_partition := false

      for _, member := range last_view {
        member.Connect()
        quorum, err := member.HasQuorum()
        if( err == nil && quorum ){
          seed_node = &member
          primary_partition = true
          break
        }
      }

      // If no one in fact has a quorum, then let's see which partition has the most
      // online/participating/communicating members. The participants in that partition
      // will then be the ones that we use to force the new membership and unlock the cluster
      // ToDo: should we consider GTID_EXECUTED sets when choosing the primary partition???
      if( primary_partition == false ){
        InfoLog.Println( "No primary partition found! Attempting to choose and force a new one ... " )

        sort.Sort( MembersByOnlineNodes(last_view) )
        // now the last element in the array is the one to use as it's coordinating with the most nodes
        seed_node = &last_view[len(last_view)-1]

        err = seed_node.Connect()

        if( err != nil ){
          // let's just loop again
          continue
        }

        if( debug ){
          DebugLog.Printf( "Member view sorted by number of online nodes: %+v\n", last_view )
        }

        // let's build a string of '<host>:<port>' combinations that we want to use for the new membership view
        members, _ := seed_node.GetMembers()

        force_member_string := ""

        for i, member := range *members {
          err = member.Connect()

          if( err == nil && member.Member_state == "ONLINE" ){
            if( i != 0 ){
              force_member_string = force_member_string + ","
            }
            force_member_string = force_member_string + member.Mysql_host + ":" + member.Mysql_port
          }
        }

        if( force_member_string != "" ){
          InfoLog.Printf( "Forcing group membership to form new primary partition! Using: '%s'\n", force_member_string )
          err = seed_node.ForceMembers( force_member_string )
        } else {
          InfoLog.Println( "No valid group membership to force!" )
        }
      }
    }

    if( err != nil ){
      loop = false
    } else {
      time.Sleep( time.Millisecond * 2000 )
    }
  }

  return err
}

// The remaining functions are used to sort our membership slice
func (a MembersByOnlineNodes) Len() int {
  return len(a)
}

func (a MembersByOnlineNodes) Swap( i, j int ) {
  a[i], a[j] = a[j], a[i]
}

func (a MembersByOnlineNodes) Less( i, j int ) bool {
  return a[i].Online_participants < a[j].Online_participants
}
<|endoftext|>