<commit_before>package etcd

import (
	"time"

	jose "gopkg.in/square/go-jose.v2"

	"github.com/dexidp/dex/storage"
)

// AuthCode is a mirrored struct from storage with JSON struct tags
type AuthCode struct {
	ID          string   `json:"ID"`
	ClientID    string   `json:"clientID"`
	RedirectURI string   `json:"redirectURI"`
	Nonce       string   `json:"nonce,omitempty"`
	Scopes      []string `json:"scopes,omitempty"`

	ConnectorID string `json:"connectorID,omitempty"`
	Claims      Claims `json:"claims,omitempty"`

	Expiry time.Time `json:"expiry"`
}

func fromStorageAuthCode(a storage.AuthCode) AuthCode {
	return AuthCode{
		ID:          a.ID,
		ClientID:    a.ClientID,
		RedirectURI: a.RedirectURI,
		ConnectorID: a.ConnectorID,
		Nonce:       a.Nonce,
		Scopes:      a.Scopes,
		Claims:      fromStorageClaims(a.Claims),
		Expiry:      a.Expiry,
	}
}

// AuthRequest is a mirrored struct from storage with JSON struct tags
type AuthRequest struct {
	ID       string `json:"id"`
	ClientID string `json:"client_id"`

	ResponseTypes []string `json:"response_types"`
	Scopes        []string `json:"scopes"`
	RedirectURI   string   `json:"redirect_uri"`
	Nonce         string   `json:"nonce"`
	State         string   `json:"state"`

	ForceApprovalPrompt bool `json:"force_approval_prompt"`

	Expiry time.Time `json:"expiry"`

	LoggedIn bool `json:"logged_in"`

	Claims Claims `json:"claims"`

	ConnectorID   string `json:"connector_id"`
	ConnectorData []byte `json:"connector_data"`
}

func fromStorageAuthRequest(a storage.AuthRequest) AuthRequest {
	return AuthRequest{
		ID:                  a.ID,
		ClientID:            a.ClientID,
		ResponseTypes:       a.ResponseTypes,
		Scopes:              a.Scopes,
		RedirectURI:         a.RedirectURI,
		Nonce:               a.Nonce,
		State:               a.State,
		ForceApprovalPrompt: a.ForceApprovalPrompt,
		Expiry:              a.Expiry,
		LoggedIn:            a.LoggedIn,
		Claims:              fromStorageClaims(a.Claims),
		ConnectorID:         a.ConnectorID,
	}
}

func toStorageAuthRequest(a AuthRequest) storage.AuthRequest {
	return storage.AuthRequest{
		ID:                  a.ID,
		ClientID:            a.ClientID,
		ResponseTypes:       a.ResponseTypes,
		Scopes:              a.Scopes,
		RedirectURI:         a.RedirectURI,
		Nonce:               a.Nonce,
		State:               a.State,
		ForceApprovalPrompt: a.ForceApprovalPrompt,
		LoggedIn:            a.LoggedIn,
		ConnectorID:         a.ConnectorID,
		Expiry:              a.Expiry,
		Claims:              toStorageClaims(a.Claims),
	}
}

// RefreshToken is a mirrored struct from storage with JSON struct tags
type RefreshToken struct {
	ID string `json:"id"`

	Token string `json:"token"`

	CreatedAt time.Time `json:"created_at"`
	LastUsed  time.Time `json:"last_used"`

	ClientID string `json:"client_id"`

	ConnectorID   string `json:"connector_id"`
	ConnectorData []byte `json:"connector_data"`
	Claims        Claims `json:"claims"`

	Scopes []string `json:"scopes"`

	Nonce string `json:"nonce"`
}

func toStorageRefreshToken(r RefreshToken) storage.RefreshToken {
	return storage.RefreshToken{
		ID:          r.ID,
		Token:       r.Token,
		CreatedAt:   r.CreatedAt,
		LastUsed:    r.LastUsed,
		ClientID:    r.ClientID,
		ConnectorID: r.ConnectorID,
		Scopes:      r.Scopes,
		Nonce:       r.Nonce,
		Claims:      toStorageClaims(r.Claims),
	}
}

func fromStorageRefreshToken(r storage.RefreshToken) RefreshToken {
	return RefreshToken{
		ID:          r.ID,
		Token:       r.Token,
		CreatedAt:   r.CreatedAt,
		LastUsed:    r.LastUsed,
		ClientID:    r.ClientID,
		ConnectorID: r.ConnectorID,
		Scopes:      r.Scopes,
		Nonce:       r.Nonce,
		Claims:      fromStorageClaims(r.Claims),
	}
}

// Claims is a mirrored struct from storage with JSON struct tags.
type Claims struct {
	UserID            string   `json:"userID"`
	Username          string   `json:"username"`
	PreferredUsername string   `json:"preferredUsername"`
	Email             string   `json:"email"`
	EmailVerified     bool     `json:"emailVerified"`
	Groups            []string `json:"groups,omitempty"`
}

func fromStorageClaims(i storage.Claims) Claims {
	return Claims{
		UserID:            i.UserID,
		Username:          i.Username,
		PreferredUsername: i.PreferredUsername,
		Email:             i.Email,
		EmailVerified:     i.EmailVerified,
		Groups:            i.Groups,
	}
}

func toStorageClaims(i Claims) storage.Claims {
	return storage.Claims{
		UserID:            i.UserID,
		Username:          i.Username,
		PreferredUsername: i.PreferredUsername,
		Email:             i.Email,
		EmailVerified:     i.EmailVerified,
		Groups:            i.Groups,
	}
}

// Keys is a mirrored struct from storage with JSON struct tags
type Keys struct {
	SigningKey       *jose.JSONWebKey          `json:"signing_key,omitempty"`
	SigningKeyPub    *jose.JSONWebKey          `json:"signing_key_pub,omitempty"`
	VerificationKeys []storage.VerificationKey `json:"verification_keys"`
	NextRotation     time.Time                 `json:"next_rotation"`
}

// OfflineSessions is a mirrored struct from storage with JSON struct tags
type OfflineSessions struct {
	UserID        string                              `json:"user_id,omitempty"`
	ConnID        string                              `json:"conn_id,omitempty"`
	Refresh       map[string]*storage.RefreshTokenRef `json:"refresh,omitempty"`
	ConnectorData []byte                              `json:"connectorData,omitempty"`
}

func fromStorageOfflineSessions(o storage.OfflineSessions) OfflineSessions {
	return OfflineSessions{
		UserID:        o.UserID,
		ConnID:        o.ConnID,
		Refresh:       o.Refresh,
		ConnectorData: o.ConnectorData,
	}
}

func toStorageOfflineSessions(o OfflineSessions) storage.OfflineSessions {
	s := storage.OfflineSessions{
		UserID:        o.UserID,
		ConnID:        o.ConnID,
		Refresh:       o.Refresh,
		ConnectorData: o.ConnectorData,
	}
	if s.Refresh == nil {
		// Server code assumes this will be non-nil.
		s.Refresh = make(map[string]*storage.RefreshTokenRef)
	}
	return s
}
<commit_msg>Revert "Fix ETCD storage backend"<commit_after>package etcd

import (
	"time"

	jose "gopkg.in/square/go-jose.v2"

	"github.com/dexidp/dex/storage"
)

// AuthCode is a mirrored struct from storage with JSON struct tags
type AuthCode struct {
	ID          string   `json:"ID"`
	ClientID    string   `json:"clientID"`
	RedirectURI string   `json:"redirectURI"`
	Nonce       string   `json:"nonce,omitempty"`
	Scopes      []string `json:"scopes,omitempty"`

	ConnectorID   string `json:"connectorID,omitempty"`
	ConnectorData []byte `json:"connectorData,omitempty"`
	Claims        Claims `json:"claims,omitempty"`

	Expiry time.Time `json:"expiry"`
}

func fromStorageAuthCode(a storage.AuthCode) AuthCode {
	return AuthCode{
		ID:            a.ID,
		ClientID:      a.ClientID,
		RedirectURI:   a.RedirectURI,
		ConnectorID:   a.ConnectorID,
		ConnectorData: a.ConnectorData,
		Nonce:         a.Nonce,
		Scopes:        a.Scopes,
		Claims:        fromStorageClaims(a.Claims),
		Expiry:        a.Expiry,
	}
}

// AuthRequest is a mirrored struct from storage with JSON struct tags
type AuthRequest struct {
	ID       string `json:"id"`
	ClientID string `json:"client_id"`

	ResponseTypes []string `json:"response_types"`
	Scopes        []string `json:"scopes"`
	RedirectURI   string   `json:"redirect_uri"`
	Nonce         string   `json:"nonce"`
	State         string   `json:"state"`

	ForceApprovalPrompt bool `json:"force_approval_prompt"`

	Expiry time.Time `json:"expiry"`

	LoggedIn bool `json:"logged_in"`

	Claims Claims `json:"claims"`

	ConnectorID   string `json:"connector_id"`
	ConnectorData []byte `json:"connector_data"`
}

func fromStorageAuthRequest(a storage.AuthRequest) AuthRequest {
	return AuthRequest{
		ID:                  a.ID,
		ClientID:            a.ClientID,
		ResponseTypes:       a.ResponseTypes,
		Scopes:              a.Scopes,
		RedirectURI:         a.RedirectURI,
		Nonce:               a.Nonce,
		State:               a.State,
		ForceApprovalPrompt: a.ForceApprovalPrompt,
		Expiry:              a.Expiry,
		LoggedIn:            a.LoggedIn,
		Claims:              fromStorageClaims(a.Claims),
		ConnectorID:         a.ConnectorID,
		ConnectorData:       a.ConnectorData,
	}
}

func toStorageAuthRequest(a AuthRequest) storage.AuthRequest {
	return storage.AuthRequest{
		ID:                  a.ID,
		ClientID:            a.ClientID,
		ResponseTypes:       a.ResponseTypes,
		Scopes:              a.Scopes,
		RedirectURI:         a.RedirectURI,
		Nonce:               a.Nonce,
		State:               a.State,
		ForceApprovalPrompt: a.ForceApprovalPrompt,
		LoggedIn:            a.LoggedIn,
		ConnectorID:         a.ConnectorID,
		ConnectorData:       a.ConnectorData,
		Expiry:              a.Expiry,
		Claims:              toStorageClaims(a.Claims),
	}
}

// RefreshToken is a mirrored struct from storage with JSON struct tags
type RefreshToken struct {
	ID string `json:"id"`

	Token string `json:"token"`

	CreatedAt time.Time `json:"created_at"`
	LastUsed  time.Time `json:"last_used"`

	ClientID string `json:"client_id"`

	ConnectorID   string `json:"connector_id"`
	ConnectorData []byte `json:"connector_data"`
	Claims        Claims `json:"claims"`

	Scopes []string `json:"scopes"`

	Nonce string `json:"nonce"`
}

func toStorageRefreshToken(r RefreshToken) storage.RefreshToken {
	return storage.RefreshToken{
		ID:            r.ID,
		Token:         r.Token,
		CreatedAt:     r.CreatedAt,
		LastUsed:      r.LastUsed,
		ClientID:      r.ClientID,
		ConnectorID:   r.ConnectorID,
		ConnectorData: r.ConnectorData,
		Scopes:        r.Scopes,
		Nonce:         r.Nonce,
		Claims:        toStorageClaims(r.Claims),
	}
}

func fromStorageRefreshToken(r storage.RefreshToken) RefreshToken {
	return RefreshToken{
		ID:            r.ID,
		Token:         r.Token,
		CreatedAt:     r.CreatedAt,
		LastUsed:      r.LastUsed,
		ClientID:      r.ClientID,
		ConnectorID:   r.ConnectorID,
		ConnectorData: r.ConnectorData,
		Scopes:        r.Scopes,
		Nonce:         r.Nonce,
		Claims:        fromStorageClaims(r.Claims),
	}
}

// Claims is a mirrored struct from storage with JSON struct tags.
type Claims struct {
	UserID            string   `json:"userID"`
	Username          string   `json:"username"`
	PreferredUsername string   `json:"preferredUsername"`
	Email             string   `json:"email"`
	EmailVerified     bool     `json:"emailVerified"`
	Groups            []string `json:"groups,omitempty"`
}

func fromStorageClaims(i storage.Claims) Claims {
	return Claims{
		UserID:            i.UserID,
		Username:          i.Username,
		PreferredUsername: i.PreferredUsername,
		Email:             i.Email,
		EmailVerified:     i.EmailVerified,
		Groups:            i.Groups,
	}
}

func toStorageClaims(i Claims) storage.Claims {
	return storage.Claims{
		UserID:            i.UserID,
		Username:          i.Username,
		PreferredUsername: i.PreferredUsername,
		Email:             i.Email,
		EmailVerified:     i.EmailVerified,
		Groups:            i.Groups,
	}
}

// Keys is a mirrored struct from storage with JSON struct tags
type Keys struct {
	SigningKey       *jose.JSONWebKey          `json:"signing_key,omitempty"`
	SigningKeyPub    *jose.JSONWebKey          `json:"signing_key_pub,omitempty"`
	VerificationKeys []storage.VerificationKey `json:"verification_keys"`
	NextRotation     time.Time                 `json:"next_rotation"`
}

// OfflineSessions is a mirrored struct from storage with JSON struct tags
type OfflineSessions struct {
	UserID        string                              `json:"user_id,omitempty"`
	ConnID        string                              `json:"conn_id,omitempty"`
	Refresh       map[string]*storage.RefreshTokenRef `json:"refresh,omitempty"`
	ConnectorData []byte                              `json:"connectorData,omitempty"`
}

func fromStorageOfflineSessions(o storage.OfflineSessions) OfflineSessions {
	return OfflineSessions{
		UserID:        o.UserID,
		ConnID:        o.ConnID,
		Refresh:       o.Refresh,
		ConnectorData: o.ConnectorData,
	}
}

func toStorageOfflineSessions(o OfflineSessions) storage.OfflineSessions {
	s := storage.OfflineSessions{
		UserID:        o.UserID,
		ConnID:        o.ConnID,
		Refresh:       o.Refresh,
		ConnectorData: o.ConnectorData,
	}
	if s.Refresh == nil {
		// Server code assumes this will be non-nil.
		s.Refresh = make(map[string]*storage.RefreshTokenRef)
	}
	return s
}
<|endoftext|>


<commit_before>package util

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/Masterminds/vcs"
)

func init() {
	// Precompile the regular expressions used to check VCS locations.
	for _, v := range vcsList {
		v.regex = regexp.MustCompile(v.pattern)
	}
}

// GetRootFromPackage retrives the top level package from a name.
//
// From a package name find the root repo. For example,
// the package github.com/Masterminds/cookoo/io has a root repo
// at github.com/Masterminds/cookoo
func GetRootFromPackage(pkg string) string {
	for _, v := range vcsList {
		m := v.regex.FindStringSubmatch(pkg)
		if m == nil {
			continue
		}

		if m[1] != "" {
			return m[1]
		}
	}

	// There are cases where a package uses the special go get magic for
	// redirects. If we've not discovered the location already try that.
	pkg = getRootFromGoGet(pkg)

	return pkg
}

// Pages like https://golang.org/x/net provide an html document with
// meta tags containing a location to work with. The go tool uses
// a meta tag with the name go-import which is what we use here.
// godoc.org also has one call go-source that we do not need to use.
// The value of go-import is in the form "prefix vcs repo". The prefix
// should match the vcsURL and the repo is a location that can be
// checked out. Note, to get the html document you you need to add
// ?go-get=1 to the url.
func getRootFromGoGet(pkg string) string {

	p, found := checkRemotePackageCache(pkg)
	if found {
		return p
	}

	vcsURL := "https://" + pkg
	u, err := url.Parse(vcsURL)
	if err != nil {
		return pkg
	}
	if u.RawQuery == "" {
		u.RawQuery = "go-get=1"
	} else {
		u.RawQuery = u.RawQuery + "+go-get=1"
	}
	checkURL := u.String()
	resp, err := http.Get(checkURL)
	if err != nil {
		addToRemotePackageCache(pkg)
		return pkg
	}
	defer resp.Body.Close()

	nu, err := parseImportFromBody(u, resp.Body)
	if err != nil {
		addToRemotePackageCache(pkg)
		return pkg
	} else if nu == "" {
		addToRemotePackageCache(pkg)
		return pkg
	}

	addToRemotePackageCache(pkg)
	return nu
}

// The caching is not concurrency safe but should be made to be that way.
// This implementation is far too much of a hack... rewrite needed.
var remotePackageCache = make(map[string]bool)

func checkRemotePackageCache(pkg string) (string, bool) {
	for k := range remotePackageCache {
		if strings.HasPrefix(pkg, k) {
			return k, true
		}
	}

	return pkg, false
}

func addToRemotePackageCache(pkg string) {
	remotePackageCache[pkg] = true
}

func parseImportFromBody(ur *url.URL, r io.ReadCloser) (u string, err error) {
	d := xml.NewDecoder(r)
	d.CharsetReader = charsetReader
	d.Strict = false
	var t xml.Token
	for {
		t, err = d.Token()
		if err != nil {
			if err == io.EOF {
				// If we hit the end of the markup and don't have anything
				// we return an error.
				err = vcs.ErrCannotDetectVCS
			}
			return
		}
		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
			return
		}
		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
			return
		}
		e, ok := t.(xml.StartElement)
		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
			continue
		}
		if attrValue(e.Attr, "name") != "go-import" {
			continue
		}
		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {

			// If the prefix supplied by the remote system isn't a prefix to the
			// url we're fetching return continue looking for more go-imports.
			// This will work for exact matches and prefixes. For example,
			// golang.org/x/net as a prefix will match for golang.org/x/net and
			// golang.org/x/net/context.
			vcsURL := ur.Host + ur.Path
			if !strings.HasPrefix(vcsURL, f[0]) {
				continue
			} else {
				u = f[0]
				return
			}

		}
	}
}

func charsetReader(charset string, input io.Reader) (io.Reader, error) {
	switch strings.ToLower(charset) {
	case "ascii":
		return input, nil
	default:
		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
	}
}

func attrValue(attrs []xml.Attr, name string) string {
	for _, a := range attrs {
		if strings.EqualFold(a.Name.Local, name) {
			return a.Value
		}
	}
	return ""
}

type vcsInfo struct {
	host    string
	pattern string
	regex   *regexp.Regexp
}

var vcsList = []*vcsInfo{
	{
		host:    "github.com",
		pattern: `^(?P<rootpkg>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "bitbucket.org",
		pattern: `^(?P<rootpkg>bitbucket\.org/([A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "launchpad.net",
		pattern: `^(?P<rootpkg>launchpad\.net/(([A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "git.launchpad.net",
		pattern: `^(?P<rootpkg>git\.launchpad\.net/(([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))$`,
	},
	{
		host:    "go.googlesource.com",
		pattern: `^(?P<rootpkg>go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`,
	},
	// TODO: Once Google Code becomes fully deprecated this can be removed.
	{
		host:    "code.google.com",
		pattern: `^(?P<rootpkg>code\.google\.com/[pr]/([a-z0-9\-]+)(\.([a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`,
	},
	// Alternative Google setup for SVN. This is the previous structure but it still works... until Google Code goes away.
	{
		pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/svn(/.*)?)$`,
	},
	// Alternative Google setup. This is the previous structure but it still works... until Google Code goes away.
	{
		pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/(git|hg))(/.*)?$`,
	},
	// If none of the previous detect the type they will fall to this looking for the type in a generic sense
	// by the extension to the path.
	{
		pattern: `^(?P<rootpkg>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`,
	},
}
<commit_msg>Fixed a caching bug.<commit_after>package util

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/Masterminds/vcs"
)

func init() {
	// Precompile the regular expressions used to check VCS locations.
	for _, v := range vcsList {
		v.regex = regexp.MustCompile(v.pattern)
	}
}

// GetRootFromPackage retrives the top level package from a name.
//
// From a package name find the root repo. For example,
// the package github.com/Masterminds/cookoo/io has a root repo
// at github.com/Masterminds/cookoo
func GetRootFromPackage(pkg string) string {
	for _, v := range vcsList {
		m := v.regex.FindStringSubmatch(pkg)
		if m == nil {
			continue
		}

		if m[1] != "" {
			return m[1]
		}
	}

	// There are cases where a package uses the special go get magic for
	// redirects. If we've not discovered the location already try that.
	pkg = getRootFromGoGet(pkg)

	return pkg
}

// Pages like https://golang.org/x/net provide an html document with
// meta tags containing a location to work with. The go tool uses
// a meta tag with the name go-import which is what we use here.
// godoc.org also has one call go-source that we do not need to use.
// The value of go-import is in the form "prefix vcs repo". The prefix
// should match the vcsURL and the repo is a location that can be
// checked out. Note, to get the html document you you need to add
// ?go-get=1 to the url.
func getRootFromGoGet(pkg string) string {

	p, found := checkRemotePackageCache(pkg)
	if found {
		return p
	}

	vcsURL := "https://" + pkg
	u, err := url.Parse(vcsURL)
	if err != nil {
		return pkg
	}
	if u.RawQuery == "" {
		u.RawQuery = "go-get=1"
	} else {
		u.RawQuery = u.RawQuery + "+go-get=1"
	}
	checkURL := u.String()
	resp, err := http.Get(checkURL)
	if err != nil {
		addToRemotePackageCache(pkg)
		return pkg
	}
	defer resp.Body.Close()

	nu, err := parseImportFromBody(u, resp.Body)
	if err != nil {
		addToRemotePackageCache(pkg)
		return pkg
	} else if nu == "" {
		addToRemotePackageCache(pkg)
		return pkg
	}

	addToRemotePackageCache(nu)
	return nu
}

// The caching is not concurrency safe but should be made to be that way.
// This implementation is far too much of a hack... rewrite needed.
var remotePackageCache = make(map[string]bool)

func checkRemotePackageCache(pkg string) (string, bool) {
	for k := range remotePackageCache {
		if strings.HasPrefix(pkg, k) {
			return k, true
		}
	}

	return pkg, false
}

func addToRemotePackageCache(pkg string) {
	remotePackageCache[pkg] = true
}

func parseImportFromBody(ur *url.URL, r io.ReadCloser) (u string, err error) {
	d := xml.NewDecoder(r)
	d.CharsetReader = charsetReader
	d.Strict = false
	var t xml.Token
	for {
		t, err = d.Token()
		if err != nil {
			if err == io.EOF {
				// If we hit the end of the markup and don't have anything
				// we return an error.
				err = vcs.ErrCannotDetectVCS
			}
			return
		}
		if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") {
			return
		}
		if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") {
			return
		}
		e, ok := t.(xml.StartElement)
		if !ok || !strings.EqualFold(e.Name.Local, "meta") {
			continue
		}
		if attrValue(e.Attr, "name") != "go-import" {
			continue
		}
		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {

			// If the prefix supplied by the remote system isn't a prefix to the
			// url we're fetching return continue looking for more go-imports.
			// This will work for exact matches and prefixes. For example,
			// golang.org/x/net as a prefix will match for golang.org/x/net and
			// golang.org/x/net/context.
			vcsURL := ur.Host + ur.Path
			if !strings.HasPrefix(vcsURL, f[0]) {
				continue
			} else {
				u = f[0]
				return
			}

		}
	}
}

func charsetReader(charset string, input io.Reader) (io.Reader, error) {
	switch strings.ToLower(charset) {
	case "ascii":
		return input, nil
	default:
		return nil, fmt.Errorf("can't decode XML document using charset %q", charset)
	}
}

func attrValue(attrs []xml.Attr, name string) string {
	for _, a := range attrs {
		if strings.EqualFold(a.Name.Local, name) {
			return a.Value
		}
	}
	return ""
}

type vcsInfo struct {
	host    string
	pattern string
	regex   *regexp.Regexp
}

var vcsList = []*vcsInfo{
	{
		host:    "github.com",
		pattern: `^(?P<rootpkg>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "bitbucket.org",
		pattern: `^(?P<rootpkg>bitbucket\.org/([A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "launchpad.net",
		pattern: `^(?P<rootpkg>launchpad\.net/(([A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
	},
	{
		host:    "git.launchpad.net",
		pattern: `^(?P<rootpkg>git\.launchpad\.net/(([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))$`,
	},
	{
		host:    "go.googlesource.com",
		pattern: `^(?P<rootpkg>go\.googlesource\.com/[A-Za-z0-9_.\-]+/?)$`,
	},
	// TODO: Once Google Code becomes fully deprecated this can be removed.
	{
		host:    "code.google.com",
		pattern: `^(?P<rootpkg>code\.google\.com/[pr]/([a-z0-9\-]+)(\.([a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`,
	},
	// Alternative Google setup for SVN. This is the previous structure but it still works... until Google Code goes away.
	{
		pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/svn(/.*)?)$`,
	},
	// Alternative Google setup. This is the previous structure but it still works... until Google Code goes away.
	{
		pattern: `^(?P<rootpkg>[a-z0-9_\-.]+\.googlecode\.com/(git|hg))(/.*)?$`,
	},
	// If none of the previous detect the type they will fall to this looking for the type in a generic sense
	// by the extension to the path.
	{
		pattern: `^(?P<rootpkg>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`,
	},
}
<|endoftext|>


<commit_before>package util

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"reflect"
	"strings"

	"code.google.com/p/go-uuid/uuid"
)

// MapToStruct is a way to deserialize a dictionary of strings into a struct
//
func MapHeaderToStruct(m map[string][]string, val interface{}) error {
	simple := make(map[string]string, len(m))
	for k, v := range m {
		simple[k] = v[0]
	}
	return MapToStruct(simple, &val)
}

// MapToStruct is a way to deserialize a dictionary of strings into a struct
//
func MapToStruct(m map[string]string, val interface{}) error {
	b := new(bytes.Buffer)
	e := json.NewEncoder(b)
	e.Encode(m)
	d := json.NewDecoder(b)
	err := d.Decode(&val)
	return err
}

//      Invoke(YourT2{}, "MethodFoo", 10, "abc")
//      Invoke(YourT1{}, "MethodBar")
func Invoke(any interface{}, name string, args ...interface{}) {
	inputs := make([]reflect.Value, len(args))
	for i, _ := range args {
		inputs[i] = reflect.ValueOf(args[i])
	}
	reflect.ValueOf(any).MethodByName(name).Call(inputs)
}

func NewUid() string {
	uid := uuid.NewRandom()
	return Base64Safe(uid)
}

func NoQuotes(target string) string {
	return strings.Replace(target, "\"", "", -1)
}

// Base64Safe - Makes safe for file paths, removes the forward slash '/' and equals '='
func Base64Safe(bts []byte) string {
	safe := strings.Replace(base64.StdEncoding.EncodeToString(bts), "/", "-", -1)
	return strings.Replace(safe, "=", "", -1)
}

func NotEmpty(values ...string) string {
	for _, v := range values {
		if v != "" {
			return v
		}
	}
	return ""
}

func NotNil(values ...interface{}) interface{} {
	for _, v := range values {
		if v != nil {
			return v
		}
	}
	return nil

}
<commit_msg>safe path<commit_after>package util

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"reflect"
	"strings"
	"unicode"

	"code.google.com/p/go-uuid/uuid"
)

// MapToStruct is a way to deserialize a dictionary of strings into a struct
//
func MapHeaderToStruct(m map[string][]string, val interface{}) error {
	simple := make(map[string]string, len(m))
	for k, v := range m {
		simple[k] = v[0]
	}
	return MapToStruct(simple, &val)
}

// MapToStruct is a way to deserialize a dictionary of strings into a struct
//
func MapToStruct(m map[string]string, val interface{}) error {
	b := new(bytes.Buffer)
	e := json.NewEncoder(b)
	e.Encode(m)
	d := json.NewDecoder(b)
	err := d.Decode(&val)
	return err
}

//      Invoke(YourT2{}, "MethodFoo", 10, "abc")
//      Invoke(YourT1{}, "MethodBar")
func Invoke(any interface{}, name string, args ...interface{}) {
	inputs := make([]reflect.Value, len(args))
	for i, _ := range args {
		inputs[i] = reflect.ValueOf(args[i])
	}
	reflect.ValueOf(any).MethodByName(name).Call(inputs)
}

func NewUid() string {
	uid := uuid.NewRandom()
	return Base64Safe(uid)
}

func NoQuotes(target string) string {
	return strings.Replace(target, "\"", "", -1)
}

// Base64Safe - Makes safe for file paths, removes the forward slash '/' and equals '='
func Base64Safe(bts []byte) string {
	safe := strings.Replace(base64.StdEncoding.EncodeToString(bts), "/", "-", -1)
	safe = strings.Replace(safe, "+", "_", -1)
	return strings.Replace(safe, "=", "", -1)
}

func NotEmpty(values ...string) string {
	for _, v := range values {
		if v != "" {
			return v
		}
	}
	return ""
}

func NotNil(values ...interface{}) interface{} {
	for _, v := range values {
		if v != nil {
			return v
		}
	}
	return nil

}

func IsPathSafe(v string) (bool, int) {
	if v == "" {
		return true, -1
	}

	for i, r := range v {
		safe := unicode.IsNumber(r) || unicode.IsLetter(r) || r == '-' || r == '_' || r == '.' || r == '~'
		if !safe {
			return false, i
		}
	}

	return true, -1

}
<|endoftext|>


<commit_before>package util

import (
	//	"appengine"
	"github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/gorilla/mux"
	"net/http"
	"time"
)

//WARNING: This will not work in local SDK
//unless you add tzdata into google_appengine/goroot/lib/time
var Tz, _ = time.LoadLocation("Europe/Bratislava")

//Context is the type used for passing data to handlers
type Context struct {
	Ac   miniprofiler_gae.Context
	W    http.ResponseWriter
	R    *http.Request
	Vars map[string]string
}

//Handler maps standard net/http handlers to handlers accepting Context
func Handler(hand func(Context) error) http.Handler {
	return miniprofiler_gae.NewHandler(func(c miniprofiler_gae.Context, w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		err := hand(Context{Ac: c, W: w, R: r, Vars: vars})
		if err != nil {
			c.Errorf("Error 500. %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	})
}

//NormalizeDate strips the time part from time.Date leaving only
//year, month and day.
func NormalizeDate(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, Tz)
}
<commit_msg>Changed util to uss appstats instead of miniprofiler<commit_after>package util

import (
	"appengine"
	"github.com/gorilla/mux"
	"github.com/mjibson/appstats"
	"net/http"
	"time"
)

//WARNING: This will not work in local SDK
//unless you add tzdata into google_appengine/goroot/lib/time
var Tz, _ = time.LoadLocation("Europe/Bratislava")

//Context is the type used for passing data to handlers
type Context struct {
	Ac   appengine.Context
	W    http.ResponseWriter
	R    *http.Request
	Vars map[string]string
}

//Handler maps standard net/http handlers to handlers accepting Context
func Handler(hand func(Context) error) http.Handler {
	return appstats.NewHandler(func(c appengine.Context, w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		err := hand(Context{Ac: c, W: w, R: r, Vars: vars})
		if err != nil {
			c.Errorf("Error 500. %v", err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	})
}

//NormalizeDate strips the time part from time.Date leaving only
//year, month and day.
func NormalizeDate(t time.Time) time.Time {
	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, Tz)
}
<|endoftext|>


<commit_before><commit_msg>go fmt<commit_after><|endoftext|>


<commit_before>package opentsdbclient_test

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"

	"../opentsdbclient"

	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gogo/protobuf/proto"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var bodyChan chan []byte

var _ = Describe("OpentsdbClient", func() {

	var ts *httptest.Server

	BeforeEach(func() {
		bodyChan = make(chan []byte, 1)
		ts = httptest.NewServer(http.HandlerFunc(handlePost))
	})

	It("posts ValueMetrics in JSON format", func() {
		c := opentsdbclient.New(ts.URL, "")

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(5),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		})

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(2000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(76),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		})

		err := c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())
		Eventually(bodyChan).Should(Receive(MatchJSON(`[
          {
            "metric": "origin.metricName",
            "value": 5,
            "timestamp": 1,
            "tags": [
              "deployment:deployment-name",
              "job:doppler"
            ]
          },
          {
            "metric": "origin.metricName",
            "value": 76,
            "timestamp": 2,
            "tags": [
              "deployment:deployment-name",
              "job:doppler"
            ]
          }
        ]`)))
	})

	It("registers metrics with the same name but different tags as different", func() {
		c := opentsdbclient.New(ts.URL, "")

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(5),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		})

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(2000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(76),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("gorouter"),
		})

		err := c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())

		var receivedBytes []byte
		Eventually(bodyChan).Should(Receive(&receivedBytes))

		Expect(receivedBytes).To(ContainSubstring(`["deployment:deployment-name","job:doppler"]`))
		Expect(receivedBytes).To(ContainSubstring(`["deployment:deployment-name","job:gorouter"]`))
	})

	It("posts CounterEvents in JSON format and empties map after post", func() {
		c := opentsdbclient.New(ts.URL, "")

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_CounterEvent.Enum(),
			CounterEvent: &events.CounterEvent{
				Name:  proto.String("counterName"),
				Delta: proto.Uint64(1),
				Total: proto.Uint64(5),
			},
		})

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(2000000000),
			EventType: events.Envelope_CounterEvent.Enum(),
			CounterEvent: &events.CounterEvent{
				Name:  proto.String("counterName"),
				Delta: proto.Uint64(6),
				Total: proto.Uint64(11),
			},
		})

		err := c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())
		Eventually(bodyChan).Should(Receive(MatchJSON(`[
			{
        "metric": "origin.counterName",
        "value": 5,
        "timestamp": 1
      },
      {
        "metric": "origin.counterName",
        "value": 11,
        "timestamp": 2
      }
		]`)))

		err = c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())
		Eventually(bodyChan).Should(Receive(MatchJSON(`[]`)))
	})

})

func handlePost(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		panic("No body!")
	}

	bodyChan <- body
}
<commit_msg>fix tests after adopt new json format<commit_after>package opentsdbclient_test

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"

	"../opentsdbclient"

	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gogo/protobuf/proto"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var bodyChan chan []byte

var _ = Describe("OpentsdbClient", func() {

	var ts *httptest.Server

	BeforeEach(func() {
		bodyChan = make(chan []byte, 1)
		ts = httptest.NewServer(http.HandlerFunc(handlePost))
	})

	It("posts ValueMetrics in JSON format", func() {
		c := opentsdbclient.New(ts.URL, "")

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(5),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		})

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(2000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(76),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		})

		err := c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())
		Eventually(bodyChan).Should(Receive(MatchJSON(`[
        {
          "metric": "origin.metricName",
          "value": 5,
          "timestamp": 1,
          "tags": {
            "deployment": "deployment-name",
            "job": "doppler",
            "index": 0,
            "ip": ""
          }
        },
        {
          "metric": "origin.metricName",
          "value": 76,
          "timestamp": 2,
          "tags": {
            "deployment": "deployment-name",
            "job": "doppler",
            "index": 0,
            "ip": ""
          }
        }
    ]`)))
	})

	It("registers metrics with the same name but different tags as different", func() {
		c := opentsdbclient.New(ts.URL, "")

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(5),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("doppler"),
		})

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(2000000000),
			EventType: events.Envelope_ValueMetric.Enum(),
			ValueMetric: &events.ValueMetric{
				Name:  proto.String("metricName"),
				Value: proto.Float64(76),
			},
			Deployment: proto.String("deployment-name"),
			Job:        proto.String("gorouter"),
		})

		err := c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())

		var receivedBytes []byte
		Eventually(bodyChan).Should(Receive(&receivedBytes))

		Expect(receivedBytes).To(ContainSubstring(`"deployment":"deployment-name","job":"doppler"`))
		Expect(receivedBytes).To(ContainSubstring(`"deployment":"deployment-name","job":"gorouter"`))
	})

	It("posts CounterEvents in JSON format and empties map after post", func() {
		c := opentsdbclient.New(ts.URL, "")

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(1000000000),
			EventType: events.Envelope_CounterEvent.Enum(),
			CounterEvent: &events.CounterEvent{
				Name:  proto.String("counterName"),
				Delta: proto.Uint64(1),
				Total: proto.Uint64(5),
			},
		})

		c.AddMetric(&events.Envelope{
			Origin:    proto.String("origin"),
			Timestamp: proto.Int64(2000000000),
			EventType: events.Envelope_CounterEvent.Enum(),
			CounterEvent: &events.CounterEvent{
				Name:  proto.String("counterName"),
				Delta: proto.Uint64(6),
				Total: proto.Uint64(11),
			},
		})

		err := c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())
		Eventually(bodyChan).Should(Receive(MatchJSON(`[
        {
          "metric": "origin.counterName",
          "value": 5,
          "timestamp": 1,
          "tags": {
            "deployment": "",
            "job": "",
            "index": 0,
            "ip": ""
          }
        },
        {
          "metric": "origin.counterName",
          "value": 11,
          "timestamp": 2,
          "tags": {
            "deployment": "",
            "job": "",
            "index": 0,
            "ip": ""
          }
        }
		]`)))

		err = c.PostMetrics()
		Expect(err).ToNot(HaveOccurred())
		Eventually(bodyChan).Should(Receive(MatchJSON(`[]`)))
	})

})

func handlePost(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		panic("No body!")
	}

	bodyChan <- body
}
<|endoftext|>


<commit_before>// 文件监控类
// create by gloomy 2017-5-3 11:40:32
package gutil

import (
	"errors"
	"fmt"
	"github.com/howeyc/fsnotify"
	"os"
	"sync"
	"time"
)

var (
	autoMatedTaskLock sync.RWMutex
	autoMatedTaskFile map[string]int = make(map[string]int)
)

// 文件监控
// create by gloomy 2017-5-3 11:42:09
func WatchFile(ch chan struct{}, filePathStr string, deleteFileCallBack, modifyFileCallBack, renameFileCallBack, createFileCallBack func(string)) (*fsnotify.Watcher, error) {
	fi, err := os.Stat(filePathStr)
	if err != nil {
		return nil, err
	}
	if !fi.IsDir() {
		return nil, errors.New("WatchFile filePathStr isn't dir!")
	}
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if ev == nil {
					continue
				}
				if ev.IsDelete() && deleteFileCallBack != nil {
					deleteFileCallBack(ev.Name)
				}
				if ev.IsRename() && renameFileCallBack != nil {
					renameFileCallBack(ev.Name)
				}
				if ev.IsCreate() && createFileCallBack != nil {
					createFileCallBack(ev.Name)
				}
				if ev.IsModify() && modifyFileCallBack != nil {
					autoMatedTaskLock.RLock()
					_, ok := autoMatedTaskFile[ev.Name]
					autoMatedTaskLock.RUnlock()
					if !ok {
						autoMatedTaskLock.Lock()
						autoMatedTaskFile[ev.Name] = 0
						autoMatedTaskLock.Unlock()
						go watchFileAutoMated(ev.Name, modifyFileCallBack)
					}
				}
			case err := <-watcher.Error:
				if err == nil {
					continue
				}
				fmt.Printf("WatchFile fsnotify watcher is error! err: %s \n", err.Error())
			}
		}
	}()
	err = watcher.WatchFlags(filePathStr, fsnotify.FSN_ALL)
	if err != nil {
		return nil, err
	}
	return watcher, err
}

// 自动化创建任务 需要监控的文件,判断文件是否上传完毕
// 创建人:邵炜
// 创建时间:2016年9月5日14:58:13
// 输入参数: 文件路劲
func watchFileAutoMated(filePath string, callBack func(string)) {
	tmrIntal := 30 * time.Second
	fileSaveTmr := time.NewTimer(tmrIntal)
	fileState, err := os.Stat(filePath)
	if err != nil {
		fmt.Sprintf("watchFileAutoMated can't load file! path: %s err: %s \n", filePath, err.Error())
		return
	}
	var (
		size   = fileState.Size()
		number int64
	)
	defer func() {
		fileSaveTmr.Stop()
		autoMatedTaskLock.Lock()
		delete(autoMatedTaskFile, filePath)
		autoMatedTaskLock.Unlock()
	}()
	<-fileSaveTmr.C
	for {
		fileState, err = os.Stat(filePath)
		if err != nil {
			fmt.Sprintf("watchFileAutoMated can't load file! path: %s err: %s \n", filePath, err.Error())
			return
		}
		number = fileState.Size()
		if size == number {
			go callBack(filePath)
			return
		}
		size = number
		fileSaveTmr.Reset(tmrIntal)
		<-fileSaveTmr.C
	}
}
<commit_msg>删除无效参数<commit_after>// 文件监控类
// create by gloomy 2017-5-3 11:40:32
package gutil

import (
	"errors"
	"fmt"
	"github.com/howeyc/fsnotify"
	"os"
	"sync"
	"time"
)

var (
	autoMatedTaskLock sync.RWMutex
	autoMatedTaskFile map[string]int = make(map[string]int)
)

// 文件监控
// create by gloomy 2017-5-3 11:42:09
func WatchFile(filePathStr string, deleteFileCallBack, modifyFileCallBack, renameFileCallBack, createFileCallBack func(string)) (*fsnotify.Watcher, error) {
	fi, err := os.Stat(filePathStr)
	if err != nil {
		return nil, err
	}
	if !fi.IsDir() {
		return nil, errors.New("WatchFile filePathStr isn't dir!")
	}
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if ev == nil {
					continue
				}
				if ev.IsDelete() && deleteFileCallBack != nil {
					deleteFileCallBack(ev.Name)
				}
				if ev.IsRename() && renameFileCallBack != nil {
					renameFileCallBack(ev.Name)
				}
				if ev.IsCreate() && createFileCallBack != nil {
					createFileCallBack(ev.Name)
				}
				if ev.IsModify() && modifyFileCallBack != nil {
					autoMatedTaskLock.RLock()
					_, ok := autoMatedTaskFile[ev.Name]
					autoMatedTaskLock.RUnlock()
					if !ok {
						autoMatedTaskLock.Lock()
						autoMatedTaskFile[ev.Name] = 0
						autoMatedTaskLock.Unlock()
						go watchFileAutoMated(ev.Name, modifyFileCallBack)
					}
				}
			case err := <-watcher.Error:
				if err == nil {
					continue
				}
				fmt.Printf("WatchFile fsnotify watcher is error! err: %s \n", err.Error())
			}
		}
	}()
	err = watcher.WatchFlags(filePathStr, fsnotify.FSN_ALL)
	if err != nil {
		return nil, err
	}
	return watcher, err
}

// 自动化创建任务 需要监控的文件,判断文件是否上传完毕
// 创建人:邵炜
// 创建时间:2016年9月5日14:58:13
// 输入参数: 文件路劲
func watchFileAutoMated(filePath string, callBack func(string)) {
	tmrIntal := 30 * time.Second
	fileSaveTmr := time.NewTimer(tmrIntal)
	fileState, err := os.Stat(filePath)
	if err != nil {
		fmt.Sprintf("watchFileAutoMated can't load file! path: %s err: %s \n", filePath, err.Error())
		return
	}
	var (
		size   = fileState.Size()
		number int64
	)
	defer func() {
		fileSaveTmr.Stop()
		autoMatedTaskLock.Lock()
		delete(autoMatedTaskFile, filePath)
		autoMatedTaskLock.Unlock()
	}()
	<-fileSaveTmr.C
	for {
		fileState, err = os.Stat(filePath)
		if err != nil {
			fmt.Sprintf("watchFileAutoMated can't load file! path: %s err: %s \n", filePath, err.Error())
			return
		}
		number = fileState.Size()
		if size == number {
			go callBack(filePath)
			return
		}
		size = number
		fileSaveTmr.Reset(tmrIntal)
		<-fileSaveTmr.C
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

// OsEnv implements interface to wrap os.Getenv
type OsEnv struct {
}

// Getenv wraps os.Getenv
func (OsEnv) Getenv(key string) string {
	return os.Getenv(key)
}

func makeRequestHandler(config *WatchdogConfig) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			parts := strings.Split(config.faasProcess, " ")

			targetCmd := exec.Command(parts[0], parts[1:]...)
			writer, _ := targetCmd.StdinPipe()

			res, _ := ioutil.ReadAll(r.Body)

			writer.Write(res)
			writer.Close()

			out, err := targetCmd.CombinedOutput()

			if err != nil {
				if config.writeDebug == true {
					log.Println(targetCmd, err)
				}

				w.WriteHeader(500)
				response := bytes.NewBufferString(err.Error())
				w.Write(response.Bytes())
				return
			}
			if config.writeDebug == true {
				os.Stdout.Write(out)
			}

			w.WriteHeader(200)
			w.Write(out)
		} else {
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	}
}

func main() {
	osEnv := OsEnv{}
	readConfig := ReadConfig{}
	config := readConfig.Read(osEnv)

	if len(config.faasProcess) == 0 {
		log.Panicln("Provide a valid process via fprocess environmental variable.")
		return
	}

	readTimeout := time.Duration(config.readTimeout) * time.Second
	writeTimeout := time.Duration(config.writeTimeout) * time.Second

	s := &http.Server{
		Addr:           ":8080",
		ReadTimeout:    readTimeout,
		WriteTimeout:   writeTimeout,
		MaxHeaderBytes: 1 << 20,
	}

	http.HandleFunc("/", makeRequestHandler(&config))

	log.Fatal(s.ListenAndServe())
}
<commit_msg>Fix attempt for reported connection buildup <commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

// OsEnv implements interface to wrap os.Getenv
type OsEnv struct {
}

// Getenv wraps os.Getenv
func (OsEnv) Getenv(key string) string {
	return os.Getenv(key)
}

func makeRequestHandler(config *WatchdogConfig) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			parts := strings.Split(config.faasProcess, " ")

			targetCmd := exec.Command(parts[0], parts[1:]...)
			writer, _ := targetCmd.StdinPipe()

			res, _ := ioutil.ReadAll(r.Body)
			defer r.Body.Close()

			writer.Write(res)
			writer.Close()

			out, err := targetCmd.CombinedOutput()

			if err != nil {
				if config.writeDebug == true {
					log.Println(targetCmd, err)
				}

				w.WriteHeader(500)
				response := bytes.NewBufferString(err.Error())
				w.Write(response.Bytes())
				return
			}
			if config.writeDebug == true {
				os.Stdout.Write(out)
			}

			w.WriteHeader(200)
			w.Write(out)
		} else {
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	}
}

func main() {
	osEnv := OsEnv{}
	readConfig := ReadConfig{}
	config := readConfig.Read(osEnv)

	if len(config.faasProcess) == 0 {
		log.Panicln("Provide a valid process via fprocess environmental variable.")
		return
	}

	readTimeout := time.Duration(config.readTimeout) * time.Second
	writeTimeout := time.Duration(config.writeTimeout) * time.Second

	s := &http.Server{
		Addr:           ":8080",
		ReadTimeout:    readTimeout,
		WriteTimeout:   writeTimeout,
		MaxHeaderBytes: 1 << 20,
	}

	http.HandleFunc("/", makeRequestHandler(&config))

	log.Fatal(s.ListenAndServe())
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

package gdb

import (
    "fmt"
    "errors"
    "strings"
    "database/sql"
    _ "github.com/lib/pq"
    _ "github.com/go-sql-driver/mysql"
    "gitee.com/johng/gf/g/util/gconv"
    "reflect"
)

// 数据库事务对象
type Tx struct {
    db *Db
    tx *sql.Tx
}

// 事务操作，提交
func (tx *Tx) Commit() error {
    return tx.tx.Commit()
}

// 事务操作，回滚
func (tx *Tx) Rollback() error {
    return tx.tx.Rollback()
}

// (事务)数据库sql查询操作，主要执行查询
func (tx *Tx) Query(query string, args ...interface{}) (*sql.Rows, error) {
    p         := tx.db.link.handleSqlBeforeExec(&query)
    rows, err := tx.tx.Query(*p, args ...)
    err        = tx.db.formatError(err, p, args...)
    if err == nil {
        return rows, nil
    }
    return nil, err
}

// (事务)执行一条sql，并返回执行情况，主要用于非查询操作
func (tx *Tx) Exec(query string, args ...interface{}) (sql.Result, error) {
    p      := tx.db.link.handleSqlBeforeExec(&query)
    r, err := tx.tx.Exec(*p, args ...)
    err     = tx.db.formatError(err, p, args...)
    return r, err
}

// 数据库查询，获取查询结果集，以列表结构返回
func (tx *Tx) GetAll(query string, args ...interface{}) (Result, error) {
    // 执行sql
    rows, err := tx.Query(query, args ...)
    if err != nil || rows == nil {
        return nil, err
    }
    // 列名称列表
    columns, err := rows.Columns()
    if err != nil {
        return nil, err
    }
    // 返回结构组装
    values   := make([]sql.RawBytes, len(columns))
    scanArgs := make([]interface{}, len(values))
    records  := make(Result, 0)
    for i := range values {
        scanArgs[i] = &values[i]
    }
    for rows.Next() {
        err = rows.Scan(scanArgs...)
        if err != nil {
            return records, err
        }
        row := make(Record)
        // 注意col字段是一个[]byte类型(slice类型本身是一个指针)，多个记录循环时该变量指向的是同一个内存地址
        for i, col := range values {
            k := columns[i]
            v := make([]byte, len(col))
            copy(v, col)
            row[k] = v
        }
        //fmt.Printf("%p\n", row["typeid"])
        records = append(records, row)
    }
    return records, nil
}

// 数据库查询，获取查询结果记录，以关联数组结构返回
func (tx *Tx) GetOne(query string, args ...interface{}) (Record, error) {
    list, err := tx.GetAll(query, args ...)
    if err != nil {
        return nil, err
    }
    if len(list) > 0 {
        return list[0], nil
    }
    return nil, nil
}

// 数据库查询，获取查询结果记录，自动映射数据到给定的struct对象中
func (tx *Tx) GetStruct(obj interface{}, query string, args ...interface{}) error {
    one, err := tx.GetOne(query, args...)
    if err != nil {
        return err
    }
    return one.ToStruct(obj)
}

// 数据库查询，获取查询字段值
func (tx *Tx) GetValue(query string, args ...interface{}) (Value, error) {
    one, err := tx.GetOne(query, args ...)
    if err != nil {
        return nil, err
    }
    for _, v := range one {
        return v, nil
    }
    return nil, nil
}

// 数据库查询，获取查询数量
func (tx *Tx) GetCount(query string, args ...interface{}) (int, error) {
    val, err := tx.GetValue(query, args ...)
    if err != nil {
        return 0, err
    }
    return gconv.Int(val), nil
}

// 数据表查询，其中tables可以是多个联表查询语句，这种查询方式较复杂，建议使用链式操作
func (tx *Tx) Select(tables, fields string, condition interface{}, groupBy, orderBy string, first, limit int, args ... interface{}) (Result, error) {
    s := fmt.Sprintf("SELECT %s FROM %s ", fields, tables)
    if condition != nil {
        s += fmt.Sprintf("WHERE %s ", tx.db.formatCondition(condition))
    }
    if len(groupBy) > 0 {
        s += fmt.Sprintf("GROUP BY %s ", groupBy)
    }
    if len(orderBy) > 0 {
        s += fmt.Sprintf("ORDER BY %s ", orderBy)
    }
    if limit > 0 {
        s += fmt.Sprintf("LIMIT %d,%d ", first, limit)
    }
    return tx.GetAll(s, args ... )
}

// sql预处理，执行完成后调用返回值sql.Stmt.Exec完成sql操作
// 记得调用sql.Stmt.Close关闭操作对象
func (tx *Tx) Prepare(query string) (*sql.Stmt, error) {
    return tx.tx.Prepare(query)
}

// insert、replace, save， ignore操作
// 0: insert:  仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
// 1: replace: 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
// 2: save:    如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
// 3: ignore:  如果数据存在(主键或者唯一索引)，那么什么也不做
func (tx *Tx) insert(table string, data Map, option uint8) (sql.Result, error) {
    var keys   []string
    var values []string
    var params []interface{}
    for k, v := range data {
        keys   = append(keys,   tx.db.charl + k + tx.db.charr)
        values = append(values, "?")
        params = append(params, v)
    }
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for k, _ := range data {
            updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    return tx.Exec(
        fmt.Sprintf("%s INTO %s%s%s(%s) VALUES(%s) %s",
            operation, tx.db.charl, table, tx.db.charr, strings.Join(keys, ","), strings.Join(values, ","), updatestr), params...
    )
}

// CURD操作:单条数据写入, 仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
func (tx *Tx) Insert(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_INSERT)
}

// CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) Replace(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_REPLACE)
}

// CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) Save(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_SAVE)
}

// 批量写入数据
func (tx *Tx) batchInsert(table string, list List, batch int, option uint8) (sql.Result, error) {
    var keys    []string
    var values  []string
    var bvalues []string
    var params  []interface{}
    var result  sql.Result
    var size = len(list)
    // 判断长度
    if size < 1 {
        return result, errors.New("empty data list")
    }
    // 首先获取字段名称及记录长度
    for k, _ := range list[0] {
        keys   = append(keys,   k)
        values = append(values, "?")
    }
    keyStr         := tx.db.charl + strings.Join(keys, tx.db.charl + "," + tx.db.charr) + tx.db.charr
    valueHolderStr := "(" + strings.Join(values, ",") + ")"
    // 操作判断
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for _, k := range keys {
            updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    // 构造批量写入数据格式(注意map的遍历是无序的)
    for i := 0; i < size; i++ {
        for _, k := range keys {
            params = append(params, list[i][k])
        }
        bvalues = append(bvalues, valueHolderStr)
        if len(bvalues) == batch {
            r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
                operation, tx.db.charl, table, tx.db.charr, keyStr, strings.Join(bvalues, ","),
                updatestr),
                params...)
            if err != nil {
                return result, err
            }
            result  = r
            params  = params[:0]
            bvalues = bvalues[:0]
        }
    }
    // 处理最后不构成指定批量的数据
    if len(bvalues) > 0 {
        r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
            operation, tx.db.charl, table, tx.db.charr, kstr, strings.Join(bvalues, ","),
            updatestr),
            params...)
        if err != nil {
            return result, err
        }
        result = r
    }
    return result, nil
}

// CURD操作:批量数据指定批次量写入
func (tx *Tx) BatchInsert(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_INSERT)
}

// CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) BatchReplace(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_REPLACE)
}

// CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) BatchSave(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_SAVE)
}

// CURD操作:数据更新，统一采用sql预处理
// data参数支持字符串或者关联数组类型，内部会自行做判断处理
func (tx *Tx) Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) {
    var params  []interface{}
    var updates string
    refValue := reflect.ValueOf(data)
    if refValue.Kind() == reflect.Map {
        var fields []string
        keys := refValue.MapKeys()
        for _, k := range keys {
            fields = append(fields, fmt.Sprintf("%s%s%s=?", tx.db.charl, k, tx.db.charr))
            params = append(params, gconv.String(refValue.MapIndex(k).Interface()))
            updates = strings.Join(fields,   ",")
        }
    } else {
        updates = gconv.String(data)
    }
    for _, v := range args {
        params = append(params, gconv.String(v))
    }
    return tx.Exec(fmt.Sprintf("UPDATE %s%s%s SET %s WHERE %s", tx.db.charl, table, tx.db.charr, updates, tx.db.formatCondition(condition)), params...)
}

// CURD操作:删除数据
func (tx *Tx) Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) {
    return tx.Exec(fmt.Sprintf("DELETE FROM %s%s%s WHERE %s", tx.db.charl, table, tx.db.charr, tx.db.formatCondition(condition)), args...)
}

<commit_msg>修复gdb批量数据Save错误<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

package gdb

import (
    "fmt"
    "errors"
    "strings"
    "database/sql"
    _ "github.com/lib/pq"
    _ "github.com/go-sql-driver/mysql"
    "gitee.com/johng/gf/g/util/gconv"
    "reflect"
)

// 数据库事务对象
type Tx struct {
    db *Db
    tx *sql.Tx
}

// 事务操作，提交
func (tx *Tx) Commit() error {
    return tx.tx.Commit()
}

// 事务操作，回滚
func (tx *Tx) Rollback() error {
    return tx.tx.Rollback()
}

// (事务)数据库sql查询操作，主要执行查询
func (tx *Tx) Query(query string, args ...interface{}) (*sql.Rows, error) {
    p         := tx.db.link.handleSqlBeforeExec(&query)
    rows, err := tx.tx.Query(*p, args ...)
    err        = tx.db.formatError(err, p, args...)
    if err == nil {
        return rows, nil
    }
    return nil, err
}

// (事务)执行一条sql，并返回执行情况，主要用于非查询操作
func (tx *Tx) Exec(query string, args ...interface{}) (sql.Result, error) {
    p      := tx.db.link.handleSqlBeforeExec(&query)
    r, err := tx.tx.Exec(*p, args ...)
    err     = tx.db.formatError(err, p, args...)
    return r, err
}

// 数据库查询，获取查询结果集，以列表结构返回
func (tx *Tx) GetAll(query string, args ...interface{}) (Result, error) {
    // 执行sql
    rows, err := tx.Query(query, args ...)
    if err != nil || rows == nil {
        return nil, err
    }
    // 列名称列表
    columns, err := rows.Columns()
    if err != nil {
        return nil, err
    }
    // 返回结构组装
    values   := make([]sql.RawBytes, len(columns))
    scanArgs := make([]interface{}, len(values))
    records  := make(Result, 0)
    for i := range values {
        scanArgs[i] = &values[i]
    }
    for rows.Next() {
        err = rows.Scan(scanArgs...)
        if err != nil {
            return records, err
        }
        row := make(Record)
        // 注意col字段是一个[]byte类型(slice类型本身是一个指针)，多个记录循环时该变量指向的是同一个内存地址
        for i, col := range values {
            k := columns[i]
            v := make([]byte, len(col))
            copy(v, col)
            row[k] = v
        }
        //fmt.Printf("%p\n", row["typeid"])
        records = append(records, row)
    }
    return records, nil
}

// 数据库查询，获取查询结果记录，以关联数组结构返回
func (tx *Tx) GetOne(query string, args ...interface{}) (Record, error) {
    list, err := tx.GetAll(query, args ...)
    if err != nil {
        return nil, err
    }
    if len(list) > 0 {
        return list[0], nil
    }
    return nil, nil
}

// 数据库查询，获取查询结果记录，自动映射数据到给定的struct对象中
func (tx *Tx) GetStruct(obj interface{}, query string, args ...interface{}) error {
    one, err := tx.GetOne(query, args...)
    if err != nil {
        return err
    }
    return one.ToStruct(obj)
}

// 数据库查询，获取查询字段值
func (tx *Tx) GetValue(query string, args ...interface{}) (Value, error) {
    one, err := tx.GetOne(query, args ...)
    if err != nil {
        return nil, err
    }
    for _, v := range one {
        return v, nil
    }
    return nil, nil
}

// 数据库查询，获取查询数量
func (tx *Tx) GetCount(query string, args ...interface{}) (int, error) {
    val, err := tx.GetValue(query, args ...)
    if err != nil {
        return 0, err
    }
    return gconv.Int(val), nil
}

// 数据表查询，其中tables可以是多个联表查询语句，这种查询方式较复杂，建议使用链式操作
func (tx *Tx) Select(tables, fields string, condition interface{}, groupBy, orderBy string, first, limit int, args ... interface{}) (Result, error) {
    s := fmt.Sprintf("SELECT %s FROM %s ", fields, tables)
    if condition != nil {
        s += fmt.Sprintf("WHERE %s ", tx.db.formatCondition(condition))
    }
    if len(groupBy) > 0 {
        s += fmt.Sprintf("GROUP BY %s ", groupBy)
    }
    if len(orderBy) > 0 {
        s += fmt.Sprintf("ORDER BY %s ", orderBy)
    }
    if limit > 0 {
        s += fmt.Sprintf("LIMIT %d,%d ", first, limit)
    }
    return tx.GetAll(s, args ... )
}

// sql预处理，执行完成后调用返回值sql.Stmt.Exec完成sql操作
// 记得调用sql.Stmt.Close关闭操作对象
func (tx *Tx) Prepare(query string) (*sql.Stmt, error) {
    return tx.tx.Prepare(query)
}

// insert、replace, save， ignore操作
// 0: insert:  仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
// 1: replace: 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
// 2: save:    如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
// 3: ignore:  如果数据存在(主键或者唯一索引)，那么什么也不做
func (tx *Tx) insert(table string, data Map, option uint8) (sql.Result, error) {
    var keys   []string
    var values []string
    var params []interface{}
    for k, v := range data {
        keys   = append(keys,   tx.db.charl + k + tx.db.charr)
        values = append(values, "?")
        params = append(params, v)
    }
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for k, _ := range data {
            updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    return tx.Exec(
        fmt.Sprintf("%s INTO %s%s%s(%s) VALUES(%s) %s",
            operation, tx.db.charl, table, tx.db.charr, strings.Join(keys, ","), strings.Join(values, ","), updatestr), params...
    )
}

// CURD操作:单条数据写入, 仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
func (tx *Tx) Insert(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_INSERT)
}

// CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) Replace(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_REPLACE)
}

// CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) Save(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_SAVE)
}

// 批量写入数据
func (tx *Tx) batchInsert(table string, list List, batch int, option uint8) (sql.Result, error) {
    var keys    []string
    var values  []string
    var bvalues []string
    var params  []interface{}
    var result  sql.Result
    var size = len(list)
    // 判断长度
    if size < 1 {
        return result, errors.New("empty data list")
    }
    // 首先获取字段名称及记录长度
    for k, _ := range list[0] {
        keys   = append(keys,   k)
        values = append(values, "?")
    }
    keyStr         := tx.db.charl + strings.Join(keys, tx.db.charl + "," + tx.db.charr) + tx.db.charr
    valueHolderStr := "(" + strings.Join(values, ",") + ")"
    // 操作判断
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for _, k := range keys {
            updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    // 构造批量写入数据格式(注意map的遍历是无序的)
    for i := 0; i < size; i++ {
        for _, k := range keys {
            params = append(params, list[i][k])
        }
        bvalues = append(bvalues, valueHolderStr)
        if len(bvalues) == batch {
            r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
                operation, tx.db.charl, table, tx.db.charr, keyStr, strings.Join(bvalues, ","),
                updatestr),
                params...)
            if err != nil {
                return result, err
            }
            result  = r
            params  = params[:0]
            bvalues = bvalues[:0]
        }
    }
    // 处理最后不构成指定批量的数据
    if len(bvalues) > 0 {
        r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
            operation, tx.db.charl, table, tx.db.charr, keyStr, strings.Join(bvalues, ","),
            updatestr),
            params...)
        if err != nil {
            return result, err
        }
        result = r
    }
    return result, nil
}

// CURD操作:批量数据指定批次量写入
func (tx *Tx) BatchInsert(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_INSERT)
}

// CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) BatchReplace(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_REPLACE)
}

// CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) BatchSave(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_SAVE)
}

// CURD操作:数据更新，统一采用sql预处理
// data参数支持字符串或者关联数组类型，内部会自行做判断处理
func (tx *Tx) Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) {
    var params  []interface{}
    var updates string
    refValue := reflect.ValueOf(data)
    if refValue.Kind() == reflect.Map {
        var fields []string
        keys := refValue.MapKeys()
        for _, k := range keys {
            fields = append(fields, fmt.Sprintf("%s%s%s=?", tx.db.charl, k, tx.db.charr))
            params = append(params, gconv.String(refValue.MapIndex(k).Interface()))
            updates = strings.Join(fields,   ",")
        }
    } else {
        updates = gconv.String(data)
    }
    for _, v := range args {
        params = append(params, gconv.String(v))
    }
    return tx.Exec(fmt.Sprintf("UPDATE %s%s%s SET %s WHERE %s", tx.db.charl, table, tx.db.charr, updates, tx.db.formatCondition(condition)), params...)
}

// CURD操作:删除数据
func (tx *Tx) Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) {
    return tx.Exec(fmt.Sprintf("DELETE FROM %s%s%s WHERE %s", tx.db.charl, table, tx.db.charr, tx.db.formatCondition(condition)), args...)
}

<|endoftext|>


<commit_before>// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.
// 路由控制.

package ghttp

import (
    "errors"
    "strings"
    "container/list"
    "gitee.com/johng/gf/g/util/gregx"
)

// handler缓存项，根据URL.Path进行缓存，因此对象中带有缓存参数
type handlerCacheItem struct {
    item    *HandlerItem        // 准确的执行方法内存地址
    values   map[string][]string // GET解析参数
}

// 查询请求处理方法
// 这里有个锁机制，可以并发读，但是不能并发写
func (s *Server) getHandler(r *Request) *HandlerItem {
    s.hmcmu.RLock()
    defer s.hmcmu.RUnlock()

    var handlerItem *handlerCacheItem
    if v := s.handlerCache.Get(r.URL.Path); v == nil {
        handlerItem = s.searchHandler(r)
        if handlerItem != nil {
            s.handlerCache.Set(r.URL.Path, handlerItem, 0)
        }
    } else {
        handlerItem = v.(*handlerCacheItem)
    }
    if handlerItem != nil {
        for k, v := range handlerItem.values {
            r.values[k] = v
        }
        return handlerItem.item
    }
    return nil
}

// 解析pattern
func (s *Server)parsePattern(pattern string) (domain, method, uri string, err error) {
    uri    = pattern
    domain = gDEFAULT_DOMAIN
    method = gDEFAULT_METHOD
    if array, err := gregx.MatchString(`([a-zA-Z]+):(.+)`, pattern); len(array) > 1 && err == nil {
        method = array[1]
        uri    = array[2]
    }
    if array, err := gregx.MatchString(`(.+)@([\w\.\-]+)`, uri); len(array) > 1 && err == nil {
        uri     = array[1]
        domain  = array[2]
    }
    if uri == "" {
        err = errors.New("invalid pattern")
    }
    // 去掉末尾的"/"符号，与路由匹配时处理一直
    if uri != "/" {
        uri = strings.TrimRight(uri, "/")
    }
    return
}

// 注册服务处理方法
func (s *Server) setHandler(pattern string, item *HandlerItem) error {
    domain, method, uri, err := s.parsePattern(pattern)
    if err != nil {
        return errors.New("invalid pattern")
    }
    item.uri    = uri
    item.domain = domain
    item.method = method
    // 静态注册
    s.hmmu.Lock()
    defer s.hmmu.Unlock()
    defer s.clearHandlerCache()
    if method == gDEFAULT_METHOD {
        for v, _ := range s.methodsMap {
            s.handlerMap[s.handlerKey(domain, v, uri)] = item
        }
    } else {
        s.handlerMap[s.handlerKey(domain, method, uri)] = item
    }

    // 动态注册，首先需要判断是否是动态注册，如果不是那么就没必要添加到动态注册记录变量中
    // 非叶节点为哈希表检索节点，按照URI注册的层级进行高效检索，直至到叶子链表节点；
    // 叶子节点是链表，按照优先级进行排序，优先级高的排前面，按照遍历检索，按照哈希表层级检索后的叶子链表一般数据量不大，所以效率比较高；
    if s.isUriHasRule(uri) {
        if _, ok := s.handlerTree[domain]; !ok {
            s.handlerTree[domain] = make(map[string]interface{})
        }
        p            := s.handlerTree[domain]
        array        := strings.Split(uri[1:], "/")
        item.priority = len(array)
        for _, v := range array {
            if len(v) == 0 {
                continue
            }
            switch v[0] {
                case ':':
                    fallthrough
                case '*':
                    v = "/"
                    fallthrough
                default:
                    if _, ok := p.(map[string]interface{})[v]; !ok {
                        p.(map[string]interface{})[v] = make(map[string]interface{})
                    }
                    p = p.(map[string]interface{})[v]

            }
        }
        // 到达叶子节点
        var l *list.List
        if v, ok := p.(map[string]interface{})["*list"]; !ok {
            l = list.New()
            p.(map[string]interface{})["*list"] = l
        } else {
            l = v.(*list.List)
        }
        //b,_ := gjson.New(s.handlerTree).ToJsonIndent()
        //fmt.Println(string(b))
        // 从头开始遍历链表，优先级高的放在前面
        for e := l.Front(); e != nil; e = e.Next() {
            if s.compareHandlerItemPriority(item, e.Value.(*HandlerItem)) {
                l.InsertBefore(item, e)
                return nil
            }
        }
        l.PushBack(item)
    }
    return nil
}

// 对比两个HandlerItem的优先级，需要非常注意的是，注意新老对比项的参数先后顺序
func (s *Server) compareHandlerItemPriority(newItem, oldItem *HandlerItem) bool {
    if newItem.priority > oldItem.priority {
        return true
    }
    if newItem.priority < oldItem.priority {
        return false
    }
    if strings.Count(newItem.uri, "/:") > strings.Count(oldItem.uri, "/:") {
        return true
    }
    return false
}

// 服务方法检索
func (s *Server) searchHandler(r *Request) *handlerCacheItem {
    item := s.searchHandlerStatic(r)
    if item == nil {
        item = s.searchHandlerDynamic(r)
    }
    return item
}

// 检索静态路由规则
func (s *Server) searchHandlerStatic(r *Request) *handlerCacheItem {
    s.hmmu.RLock()
    defer s.hmmu.RUnlock()
    domains := []string{gDEFAULT_DOMAIN, strings.Split(r.Host, ":")[0]}
    // 首先进行静态匹配
    for _, domain := range domains {
        if f, ok := s.handlerMap[s.handlerKey(domain, r.Method, r.URL.Path)]; ok {
            return &handlerCacheItem{f, nil}
        }
    }
    return nil
}

// 检索动态路由规则
func (s *Server) searchHandlerDynamic(r *Request) *handlerCacheItem {
    s.hmmu.RLock()
    defer s.hmmu.RUnlock()
    domains := []string{gDEFAULT_DOMAIN, strings.Split(r.Host, ":")[0]}
    array   := strings.Split(r.URL.Path[1:], "/")
    for _, domain := range domains {
        p, ok := s.handlerTree[domain]
        if !ok {
            continue
        }
        // 多层链表的目的是当叶子节点未有任何规则匹配时，让父级模糊匹配规则继续处理
        lists := make([]*list.List, 0)
        for k, v := range array {
            if _, ok := p.(map[string]interface{})["*list"]; ok {
                lists = append(lists, p.(map[string]interface{})["*list"].(*list.List))
            }
            if _, ok := p.(map[string]interface{})[v]; !ok {
                if _, ok := p.(map[string]interface{})["/"]; ok {
                    p = p.(map[string]interface{})["/"]
                    if k == len(array) - 1 {
                        if _, ok := p.(map[string]interface{})["*list"]; ok {
                            lists = append(lists, p.(map[string]interface{})["*list"].(*list.List))
                        }
                    }
                } else {
                    break
                }
            } else {
                p = p.(map[string]interface{})[v]
                if k == len(array) - 1 {
                    if _, ok := p.(map[string]interface{})["*list"]; ok {
                        lists = append(lists, p.(map[string]interface{})["*list"].(*list.List))
                    }
                }
            }
        }

        // 多层链表遍历检索，从数组末尾的链表开始遍历，末尾的深度高优先级也高
        for i := len(lists) - 1; i >= 0; i-- {
            for e := lists[i].Front(); e != nil; e = e.Next() {
                item := e.Value.(*HandlerItem)
                if strings.EqualFold(item.method, gDEFAULT_METHOD) || strings.EqualFold(item.method, r.Method) {
                    regrule, names := s.patternToRegRule(item.uri)
                    if gregx.IsMatchString(regrule, r.URL.Path) {
                        handlerItem := &handlerCacheItem{item, nil}
                        // 如果需要query匹配，那么需要重新解析URL
                        if len(names) > 0 {
                            if match, err := gregx.MatchString(regrule, r.URL.Path); err == nil {
                                array := strings.Split(names, ",")
                                if len(match) > len(array) {
                                    handlerItem.values = make(map[string][]string)
                                    for index, name := range array {
                                        handlerItem.values[name] = []string{match[index + 1]}
                                    }
                                }
                            }
                        }
                        return handlerItem
                    }
                }
            }
        }
    }
    return nil
}

// 将pattern（不带method和domain）解析成正则表达式匹配以及对应的query字符串
func (s *Server) patternToRegRule(rule string) (regrule string, names string) {
    if len(rule) < 2 {
        return rule, ""
    }
    regrule = "^"
    array  := strings.Split(rule[1:], "/")
    for _, v := range array {
        if len(v) == 0 {
            continue
        }
        switch v[0] {
            case ':':
                regrule += `/([\w\.\-]+)`
                if len(names) > 0 {
                    names += ","
                }
                names += v[1:]
            case '*':
                regrule += `/(.*)`
                if len(names) > 0 {
                    names += ","
                }
                names += v[1:]
            default:
                regrule += "/" + v
        }
    }
    regrule += `$`
    return
}

// 判断URI中是否包含动态注册规则
func (s *Server) isUriHasRule(uri string) bool {
    if len(uri) > 1 && (strings.Index(uri, "/:") != -1 || strings.Index(uri, "/*") != -1) {
        return true
    }
    return false
}

// 生成回调方法查询的Key
func (s *Server) handlerKey(domain, method, uri string) string {
    return strings.ToUpper(method) + ":" + uri + "@" + strings.ToLower(domain)
}

<commit_msg>修复模糊匹配路由规则<commit_after>// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.
// 路由控制.

package ghttp

import (
    "errors"
    "strings"
    "container/list"
    "gitee.com/johng/gf/g/util/gregx"
)

// handler缓存项，根据URL.Path进行缓存，因此对象中带有缓存参数
type handlerCacheItem struct {
    item    *HandlerItem        // 准确的执行方法内存地址
    values   map[string][]string // GET解析参数
}

// 查询请求处理方法
// 这里有个锁机制，可以并发读，但是不能并发写
func (s *Server) getHandler(r *Request) *HandlerItem {
    s.hmcmu.RLock()
    defer s.hmcmu.RUnlock()

    var handlerItem *handlerCacheItem
    if v := s.handlerCache.Get(r.URL.Path); v == nil {
        handlerItem = s.searchHandler(r)
        if handlerItem != nil {
            s.handlerCache.Set(r.URL.Path, handlerItem, 0)
        }
    } else {
        handlerItem = v.(*handlerCacheItem)
    }
    if handlerItem != nil {
        for k, v := range handlerItem.values {
            r.values[k] = v
        }
        return handlerItem.item
    }
    return nil
}

// 解析pattern
func (s *Server)parsePattern(pattern string) (domain, method, uri string, err error) {
    uri    = pattern
    domain = gDEFAULT_DOMAIN
    method = gDEFAULT_METHOD
    if array, err := gregx.MatchString(`([a-zA-Z]+):(.+)`, pattern); len(array) > 1 && err == nil {
        method = array[1]
        uri    = array[2]
    }
    if array, err := gregx.MatchString(`(.+)@([\w\.\-]+)`, uri); len(array) > 1 && err == nil {
        uri     = array[1]
        domain  = array[2]
    }
    if uri == "" {
        err = errors.New("invalid pattern")
    }
    // 去掉末尾的"/"符号，与路由匹配时处理一直
    if uri != "/" {
        uri = strings.TrimRight(uri, "/")
    }
    return
}

// 注册服务处理方法
func (s *Server) setHandler(pattern string, item *HandlerItem) error {
    domain, method, uri, err := s.parsePattern(pattern)
    if err != nil {
        return errors.New("invalid pattern")
    }
    item.uri    = uri
    item.domain = domain
    item.method = method
    // 静态注册
    s.hmmu.Lock()
    defer s.hmmu.Unlock()
    defer s.clearHandlerCache()
    if method == gDEFAULT_METHOD {
        for v, _ := range s.methodsMap {
            s.handlerMap[s.handlerKey(domain, v, uri)] = item
        }
    } else {
        s.handlerMap[s.handlerKey(domain, method, uri)] = item
    }

    // 动态注册，首先需要判断是否是动态注册，如果不是那么就没必要添加到动态注册记录变量中
    // 非叶节点为哈希表检索节点，按照URI注册的层级进行高效检索，直至到叶子链表节点；
    // 叶子节点是链表，按照优先级进行排序，优先级高的排前面，按照遍历检索，按照哈希表层级检索后的叶子链表一般数据量不大，所以效率比较高；
    if s.isUriHasRule(uri) {
        if _, ok := s.handlerTree[domain]; !ok {
            s.handlerTree[domain] = make(map[string]interface{})
        }
        p            := s.handlerTree[domain]
        array        := strings.Split(uri[1:], "/")
        item.priority = len(array)
        for _, v := range array {
            if len(v) == 0 {
                continue
            }
            switch v[0] {
                case ':':
                    fallthrough
                case '*':
                    v = "/"
                    fallthrough
                default:
                    if _, ok := p.(map[string]interface{})[v]; !ok {
                        p.(map[string]interface{})[v] = make(map[string]interface{})
                    }
                    p = p.(map[string]interface{})[v]

            }
        }
        // 到达叶子节点
        var l *list.List
        if v, ok := p.(map[string]interface{})["*list"]; !ok {
            l = list.New()
            p.(map[string]interface{})["*list"] = l
        } else {
            l = v.(*list.List)
        }
        //b,_ := gjson.New(s.handlerTree).ToJsonIndent()
        //fmt.Println(string(b))
        // 从头开始遍历链表，优先级高的放在前面
        for e := l.Front(); e != nil; e = e.Next() {
            if s.compareHandlerItemPriority(item, e.Value.(*HandlerItem)) {
                l.InsertBefore(item, e)
                return nil
            }
        }
        l.PushBack(item)
    }
    return nil
}

// 对比两个HandlerItem的优先级，需要非常注意的是，注意新老对比项的参数先后顺序
func (s *Server) compareHandlerItemPriority(newItem, oldItem *HandlerItem) bool {
    if newItem.priority > oldItem.priority {
        return true
    }
    if newItem.priority < oldItem.priority {
        return false
    }
    if strings.Count(newItem.uri, "/:") > strings.Count(oldItem.uri, "/:") {
        return true
    }
    return false
}

// 服务方法检索
func (s *Server) searchHandler(r *Request) *handlerCacheItem {
    item := s.searchHandlerStatic(r)
    if item == nil {
        item = s.searchHandlerDynamic(r)
    }
    return item
}

// 检索静态路由规则
func (s *Server) searchHandlerStatic(r *Request) *handlerCacheItem {
    s.hmmu.RLock()
    defer s.hmmu.RUnlock()
    domains := []string{gDEFAULT_DOMAIN, strings.Split(r.Host, ":")[0]}
    // 首先进行静态匹配
    for _, domain := range domains {
        if f, ok := s.handlerMap[s.handlerKey(domain, r.Method, r.URL.Path)]; ok {
            return &handlerCacheItem{f, nil}
        }
    }
    return nil
}

// 检索动态路由规则
func (s *Server) searchHandlerDynamic(r *Request) *handlerCacheItem {
    s.hmmu.RLock()
    defer s.hmmu.RUnlock()
    domains := []string{gDEFAULT_DOMAIN, strings.Split(r.Host, ":")[0]}
    array   := strings.Split(r.URL.Path[1:], "/")
    for _, domain := range domains {
        p, ok := s.handlerTree[domain]
        if !ok {
            continue
        }
        // 多层链表的目的是当叶子节点未有任何规则匹配时，让父级模糊匹配规则继续处理
        lists := make([]*list.List, 0)
        for k, v := range array {
            if _, ok := p.(map[string]interface{})["*list"]; ok {
                lists = append(lists, p.(map[string]interface{})["*list"].(*list.List))
            }
            if _, ok := p.(map[string]interface{})[v]; ok {
                p = p.(map[string]interface{})[v]
                if k == len(array) - 1 {
                    if _, ok := p.(map[string]interface{})["*list"]; ok {
                        lists = append(lists, p.(map[string]interface{})["*list"].(*list.List))
                    }
                }
            }
            // 如果是叶子节点，同时判断当前层级的"/"键名，解决例如：/user/*action 匹配 /user 的规则
            if k == len(array) - 1 {
                if _, ok := p.(map[string]interface{})["/"]; ok {
                    p = p.(map[string]interface{})["/"]
                    if _, ok := p.(map[string]interface{})["*list"]; ok {
                        lists = append(lists, p.(map[string]interface{})["*list"].(*list.List))
                    }
                }
            }
        }

        // 多层链表遍历检索，从数组末尾的链表开始遍历，末尾的深度高优先级也高
        for i := len(lists) - 1; i >= 0; i-- {
            for e := lists[i].Front(); e != nil; e = e.Next() {
                item := e.Value.(*HandlerItem)
                if strings.EqualFold(item.method, gDEFAULT_METHOD) || strings.EqualFold(item.method, r.Method) {
                    regrule, names := s.patternToRegRule(item.uri)
                    if gregx.IsMatchString(regrule, r.URL.Path) {
                        handlerItem := &handlerCacheItem{item, nil}
                        // 如果需要query匹配，那么需要重新解析URL
                        if len(names) > 0 {
                            if match, err := gregx.MatchString(regrule, r.URL.Path); err == nil {
                                array := strings.Split(names, ",")
                                if len(match) > len(array) {
                                    handlerItem.values = make(map[string][]string)
                                    for index, name := range array {
                                        handlerItem.values[name] = []string{match[index + 1]}
                                    }
                                }
                            }
                        }
                        return handlerItem
                    }
                }
            }
        }
    }
    return nil
}

// 将pattern（不带method和domain）解析成正则表达式匹配以及对应的query字符串
func (s *Server) patternToRegRule(rule string) (regrule string, names string) {
    if len(rule) < 2 {
        return rule, ""
    }
    regrule = "^"
    array  := strings.Split(rule[1:], "/")
    for _, v := range array {
        if len(v) == 0 {
            continue
        }
        switch v[0] {
            case ':':
                regrule += `/([\w\.\-]+)`
                if len(names) > 0 {
                    names += ","
                }
                names += v[1:]
            case '*':
                regrule += `/{0,1}(.*)`
                if len(names) > 0 {
                    names += ","
                }
                names += v[1:]
            default:
                regrule += "/" + v
        }
    }
    regrule += `$`
    return
}

// 判断URI中是否包含动态注册规则
func (s *Server) isUriHasRule(uri string) bool {
    if len(uri) > 1 && (strings.Index(uri, "/:") != -1 || strings.Index(uri, "/*") != -1) {
        return true
    }
    return false
}

// 生成回调方法查询的Key
func (s *Server) handlerKey(domain, method, uri string) string {
    return strings.ToUpper(method) + ":" + uri + "@" + strings.ToLower(domain)
}

<|endoftext|>


<commit_before>package hamt64

import (
	"fmt"
	"log"
	"strings"
)

type fullTable struct {
	hashPath uint64 // depth*Nbits of hash to get to this location in the Trie
	numEnts  uint
	nodes    [TableCapacity]nodeI
}

func createRootFullTable(leaf leafI) tableI {
	var idx = index(leaf.Hash60(), 0)

	var ft = new(fullTable)
	//ft.hashPath = 0
	ft.numEnts = 1
	ft.nodes[idx] = leaf

	return ft
}

func createFullTable(depth uint, leaf1 leafI, leaf2 flatLeaf) tableI {
	var retTable = new(fullTable)
	retTable.hashPath = leaf1.Hash60() & hashPathMask(depth)

	var curTable = retTable
	var hashPath = retTable.hashPath
	var d uint
	for d = depth; d < MaxDepth; d++ {
		var idx1 = index(leaf1.Hash60(), d)
		var idx2 = index(leaf2.Hash60(), d)

		if idx1 != idx2 {
			curTable.nodes[idx1] = leaf1
			curTable.nodes[idx2] = leaf2

			curTable.numEnts = 2

			break
		}
		// idx1 == idx2 && continue

		hashPath = buildHashPath(hashPath, idx1, d)

		var newTable = new(fullTable)
		newTable.hashPath = hashPath

		curTable.numEnts = 1
		curTable.nodes[idx1] = newTable

		curTable = newTable
	}
	// We either BREAK out of the loop,
	// OR we hit d == MaxDepth.
	if d == MaxDepth {
		var idx1 = index(leaf1.Hash60(), d)
		var idx2 = index(leaf2.Hash60(), d)

		if idx1 != idx2 {
			curTable.nodes[idx1] = leaf1
			curTable.nodes[idx2] = leaf2

			curTable.numEnts = 2

			return retTable
		}
		// idx1 == idx2

		// NOTE: This condition should never result. The condition is
		// leaf1.Hash60() == leaf2.Hash60() all the way to MaxDepth;
		// because Hamt.newTable() is called only once, and after a
		// leaf1.Hash60() == leaf2.Hash60() check. It is here for completeness.
		log.Printf("full_table.go:newFullTable: SHOULD NOT BE CALLED")
		if leaf1.Hash60() != leaf2.Hash60() {
			log.Printf("madDepth=%d; d=%d; idx1=%d; idx2=%d", MaxDepth, d, idx1, idx2)
			log.Panicf("newFullTable: %s != %s", hash60String(leaf1.Hash60()), hash60String(leaf2.Hash60()))
		}
		var newLeaf, _ = leaf1.put(leaf2.key, leaf2.val)
		curTable.insert(idx1, newLeaf)
	}

	return retTable
}

func upgradeToFullTable(hashPath uint64, tabEnts []tableEntry) tableI {
	var ft = new(fullTable)
	ft.hashPath = hashPath
	ft.numEnts = uint(len(tabEnts))

	for _, ent := range tabEnts {
		ft.nodes[ent.idx] = ent.node
	}

	return ft
}

// Hash60() is required for nodeI
func (t fullTable) Hash60() uint64 {
	return t.hashPath
}

// copy() is required for nodeI
func (t fullTable) copy() *fullTable {
	var nt = new(fullTable)
	nt.hashPath = t.hashPath
	nt.numEnts = t.numEnts
	//for i := 0; i < len(t.nodes); i++ {
	//	nt.nodes[i] = t.nodes[i]
	//}
	nt.nodes = t.nodes

	return nt
}

// String() is required for nodeI
func (t fullTable) String() string {
	// fullTable{hashPath:/%d/%d/%d/%d/%d/%d/%d/%d/%d/%d, nentries:%d,}
	return fmt.Sprintf("fullTable{hashPath:%s, nentries()=%d}", hash60String(t.hashPath), t.nentries())
}

// LongString() is required for tableI
func (t fullTable) LongString(indent string, depth uint) string {
	var strs = make([]string, 2+len(t.nodes))

	strs[0] = indent + fmt.Sprintf("fullTable{hashPath:%s, nentries()=%d,", hashPathString(t.hashPath, depth), t.nentries())

	for i, n := range t.nodes {
		if t.nodes[i] == nil {
			strs[1+i] = indent + fmt.Sprintf("\tt.nodes[%d]: nil", i)
		} else {
			if t, ok := t.nodes[i].(tableI); ok {
				strs[1+i] = indent + fmt.Sprintf("\tt.nodes[%d]:\n%s", i, t.LongString(indent+"\t", depth+1))
			} else {
				strs[1+i] = indent + fmt.Sprintf("\tt.nodes[%d]: %s", i, n)
			}
		}
	}

	strs[len(strs)-1] = indent + "}"

	return strings.Join(strs, "\n")
}

// nentries() is required for tableI
func (t fullTable) nentries() uint {
	return t.numEnts
}

// This function MUST return the slice of tableEntry structs from lowest
// tableEntry.idx to highest tableEntry.idx .
func (t fullTable) entries() []tableEntry {
	var n = t.nentries()
	var ents = make([]tableEntry, n)
	for i, j := uint(0), 0; i < TableCapacity; i++ {
		if t.nodes[i] != nil {
			//The difference with compressedTable is t.nodes[i] vs. t.nodes[j]
			ents[j] = tableEntry{i, t.nodes[i]}
			j++
		}
	}
	return ents
}

// get(uint64) is required for tableI
func (t fullTable) get(idx uint) nodeI {
	return t.nodes[idx]
}

func (t fullTable) insert(idx uint, entry nodeI) tableI {
	// t.nodes[idx] == nil
	var nt = t.copy()
	nt.nodes[idx] = entry
	nt.numEnts++
	return nt
}

func (t fullTable) replace(idx uint, entry nodeI) tableI {
	// t.nodes[idx] != nil
	var nt = t.copy()
	nt.nodes[idx] = entry
	return nt
}

func (t fullTable) remove(idx uint) tableI {
	// t.nodes[idx] != nil
	var nt = t.copy()
	nt.nodes[idx] = nil
	nt.numEnts--

	if GradeTables && nt.numEnts < DowngradeThreshold {
		return downgradeToCompressedTable(nt.hashPath, nt.entries())
	}

	if nt.numEnts == 0 {
		return nil
	}

	return nt
}
<commit_msg>moved String() and LongString() to end of file<commit_after>package hamt64

import (
	"fmt"
	"log"
	"strings"
)

type fullTable struct {
	hashPath uint64 // depth*Nbits of hash to get to this location in the Trie
	numEnts  uint
	nodes    [TableCapacity]nodeI
}

func createRootFullTable(leaf leafI) tableI {
	var idx = index(leaf.Hash60(), 0)

	var ft = new(fullTable)
	//ft.hashPath = 0
	ft.numEnts = 1
	ft.nodes[idx] = leaf

	return ft
}

func createFullTable(depth uint, leaf1 leafI, leaf2 flatLeaf) tableI {
	var retTable = new(fullTable)
	retTable.hashPath = leaf1.Hash60() & hashPathMask(depth)

	var curTable = retTable
	var hashPath = retTable.hashPath
	var d uint
	for d = depth; d < MaxDepth; d++ {
		var idx1 = index(leaf1.Hash60(), d)
		var idx2 = index(leaf2.Hash60(), d)

		if idx1 != idx2 {
			curTable.nodes[idx1] = leaf1
			curTable.nodes[idx2] = leaf2

			curTable.numEnts = 2

			break
		}
		// idx1 == idx2 && continue

		hashPath = buildHashPath(hashPath, idx1, d)

		var newTable = new(fullTable)
		newTable.hashPath = hashPath

		curTable.numEnts = 1
		curTable.nodes[idx1] = newTable

		curTable = newTable
	}
	// We either BREAK out of the loop,
	// OR we hit d == MaxDepth.
	if d == MaxDepth {
		var idx1 = index(leaf1.Hash60(), d)
		var idx2 = index(leaf2.Hash60(), d)

		if idx1 != idx2 {
			curTable.nodes[idx1] = leaf1
			curTable.nodes[idx2] = leaf2

			curTable.numEnts = 2

			return retTable
		}
		// idx1 == idx2

		// NOTE: This condition should never result. The condition is
		// leaf1.Hash60() == leaf2.Hash60() all the way to MaxDepth;
		// because Hamt.newTable() is called only once, and after a
		// leaf1.Hash60() == leaf2.Hash60() check. It is here for completeness.
		log.Printf("full_table.go:newFullTable: SHOULD NOT BE CALLED")
		if leaf1.Hash60() != leaf2.Hash60() {
			log.Printf("madDepth=%d; d=%d; idx1=%d; idx2=%d", MaxDepth, d, idx1, idx2)
			log.Panicf("newFullTable: %s != %s", hash60String(leaf1.Hash60()), hash60String(leaf2.Hash60()))
		}
		var newLeaf, _ = leaf1.put(leaf2.key, leaf2.val)
		curTable.insert(idx1, newLeaf)
	}

	return retTable
}

func upgradeToFullTable(hashPath uint64, tabEnts []tableEntry) tableI {
	var ft = new(fullTable)
	ft.hashPath = hashPath
	ft.numEnts = uint(len(tabEnts))

	for _, ent := range tabEnts {
		ft.nodes[ent.idx] = ent.node
	}

	return ft
}

// Hash60() is required for nodeI
func (t fullTable) Hash60() uint64 {
	return t.hashPath
}

// copy() is required for nodeI
func (t fullTable) copy() *fullTable {
	var nt = new(fullTable)
	nt.hashPath = t.hashPath
	nt.numEnts = t.numEnts
	//for i := 0; i < len(t.nodes); i++ {
	//	nt.nodes[i] = t.nodes[i]
	//}
	nt.nodes = t.nodes

	return nt
}

// nentries() is required for tableI
func (t fullTable) nentries() uint {
	return t.numEnts
}

// This function MUST return the slice of tableEntry structs from lowest
// tableEntry.idx to highest tableEntry.idx .
func (t fullTable) entries() []tableEntry {
	var n = t.nentries()
	var ents = make([]tableEntry, n)
	for i, j := uint(0), 0; i < TableCapacity; i++ {
		if t.nodes[i] != nil {
			//The difference with compressedTable is t.nodes[i] vs. t.nodes[j]
			ents[j] = tableEntry{i, t.nodes[i]}
			j++
		}
	}
	return ents
}

// get(uint64) is required for tableI
func (t fullTable) get(idx uint) nodeI {
	return t.nodes[idx]
}

func (t fullTable) insert(idx uint, entry nodeI) tableI {
	// t.nodes[idx] == nil
	var nt = t.copy()
	nt.nodes[idx] = entry
	nt.numEnts++
	return nt
}

func (t fullTable) replace(idx uint, entry nodeI) tableI {
	// t.nodes[idx] != nil
	var nt = t.copy()
	nt.nodes[idx] = entry
	return nt
}

func (t fullTable) remove(idx uint) tableI {
	// t.nodes[idx] != nil
	var nt = t.copy()
	nt.nodes[idx] = nil
	nt.numEnts--

	if GradeTables && nt.numEnts < DowngradeThreshold {
		return downgradeToCompressedTable(nt.hashPath, nt.entries())
	}

	if nt.numEnts == 0 {
		return nil
	}

	return nt
}

// String() is required for nodeI
func (t fullTable) String() string {
	// fullTable{hashPath:/%d/%d/%d/%d/%d/%d/%d/%d/%d/%d, nentries:%d,}
	return fmt.Sprintf("fullTable{hashPath:%s, nentries()=%d}", hash60String(t.hashPath), t.nentries())
}

// LongString() is required for tableI
func (t fullTable) LongString(indent string, depth uint) string {
	var strs = make([]string, 2+len(t.nodes))

	strs[0] = indent + fmt.Sprintf("fullTable{hashPath:%s, nentries()=%d,", hashPathString(t.hashPath, depth), t.nentries())

	for i, n := range t.nodes {
		if t.nodes[i] == nil {
			strs[1+i] = indent + fmt.Sprintf("\tt.nodes[%d]: nil", i)
		} else {
			if t, ok := t.nodes[i].(tableI); ok {
				strs[1+i] = indent + fmt.Sprintf("\tt.nodes[%d]:\n%s", i, t.LongString(indent+"\t", depth+1))
			} else {
				strs[1+i] = indent + fmt.Sprintf("\tt.nodes[%d]: %s", i, n)
			}
		}
	}

	strs[len(strs)-1] = indent + "}"

	return strings.Join(strs, "\n")
}
<|endoftext|>