<commit_before>package hstspreload

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"golang.org/x/net/publicsuffix"
)

const (
	// dialTimeout specifies the amount of time that TCP or TLS connections
	// can take to complete.
	dialTimeout = 10 * time.Second

	// The maximum number of redirects when you visit the root path of the
	// domain over HTTP or HTTPS.
	maxRedirects = 3
	httpsScheme  = "https"
)

// dialer is a global net.Dialer that's used whenever making TLS connections in
// order to enforce dialTimeout.
var dialer = net.Dialer{
	Timeout: dialTimeout,
}

var clientWithTimeout = http.Client{
	Timeout: dialTimeout,
}

// PreloadableDomain checks whether the domain passes HSTS preload
// requirements for Chromium. This includes:
//
// - Serving a single HSTS header that passes header requirements.
//
// - Using TLS settings that will not cause new problems for
// Chromium/Chrome users. (Example of a new problem: a missing intermediate certificate
// will turn an error page from overrideable to non-overridable on
// some mobile devices.)
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func PreloadableDomain(domain string) (header *string, issues Issues) {
	// Check domain format issues first, since we can report something
	// useful even if the other checks fail.
	issues = combineIssues(issues, checkDomainFormat(domain))
	if len(issues.Errors) > 0 {
		return header, issues
	}

	// We don't currently allow automatic submissions of subdomains.
	levelIssues := preloadableDomainLevel(domain)
	issues = combineIssues(issues, levelIssues)

	// Start with an initial probe, and don't do the follow-up checks if
	// we can't connect.
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		issues = combineIssues(issues, checkSHA1(certChain(*resp.TLS)))

		preloadableResponse := make(chan Issues)
		httpRedirects := make(chan Issues)
		httpFirstRedirectsHSTS := make(chan Issues)
		httpsRedirects := make(chan Issues)
		www := make(chan Issues)

		// PreloadableResponse
		go func() {
			var preloadableIssues Issues
			header, preloadableIssues = PreloadableResponse(resp)
			preloadableResponse <- preloadableIssues
		}()

		// checkHTTPRedirects
		go func() {
			mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirects(domain)
			httpRedirects <- mainIssues
			httpFirstRedirectsHSTS <- firstRedirectHSTSIssues
		}()

		// checkHTTPSRedirects
		go func() {
			httpsRedirects <- preloadableHTTPSRedirects(domain)
		}()

		// checkWWW
		go func() {
			// Skip the WWW check if the domain is not eTLD+1.
			if len(levelIssues.Errors) == 0 {
				www <- checkWWW(domain)
			} else {
				www <- Issues{}
			}
		}()

		// Combine the issues in deterministic order.
		preloadableResponseIssues := <-preloadableResponse
		issues = combineIssues(issues, preloadableResponseIssues)
		issues = combineIssues(issues, <-httpRedirects)
		// If there are issues with the HSTS header in the main
		// PreloadableResponse() check, it is redundant to report
		// them in the response after redirecting from HTTP.
		if len(preloadableResponseIssues.Errors) == 0 {
			issues = combineIssues(issues, <-httpFirstRedirectsHSTS)
		}
		issues = combineIssues(issues, <-httpsRedirects)
		issues = combineIssues(issues, <-www)
	}

	return header, issues
}

// RemovableDomain checks whether the domain satisfies the requirements
// for being removed from the Chromium preload list:
//
// - Serving a single valid HSTS header.
//
// - The header must not contain the `preload` directive..
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func RemovableDomain(domain string) (header *string, issues Issues) {
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		var removableIssues Issues
		header, removableIssues = RemovableResponse(resp)
		issues = combineIssues(issues, removableIssues)
	}

	return header, issues
}

func getResponse(domain string) (*http.Response, Issues) {
	issues := Issues{}

	redirectPrevented := errors.New("REDIRECT_PREVENTED")

	client := http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return redirectPrevented
		},
		Timeout: dialTimeout,
	}

	resp, err := client.Get("https://" + domain)
	if err != nil {
		if urlError, ok := err.(*url.Error); !ok || urlError.Err != redirectPrevented {
			return resp, issues.addErrorf(
				IssueCode("domain.tls.cannot_connect"),
				"Cannot connect using TLS",
				"We cannot connect to https://%s using TLS (%q). This "+
					"might be caused by an incomplete certificate chain, which causes "+
					"issues on mobile devices. Check out your site at "+
					"https://www.ssllabs.com/ssltest/",
				domain,
				err,
			)
		}
	}

	return resp, issues
}

func checkDomainFormat(domain string) Issues {
	issues := Issues{}

	if strings.HasPrefix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.begins_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.HasSuffix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.ends_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.Index(domain, "..") != -1 {
		return issues.addErrorf(
			IssueCode("domain.format.contains_double_dot"),
			"Invalid domain name",
			"Please provide a domain that does not contain `..`")
	}
	if strings.Count(domain, ".") < 1 {
		return issues.addErrorf(
			IssueCode("domain.format.only_one_label"),
			"Invalid domain name",
			"Please provide a domain with least two labels "+
				"(e.g. `example.com` rather than `example` or `com`).")
	}

	domain = strings.ToLower(domain)
	for _, r := range domain {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '.' {
			continue
		}

		return issues.addErrorf("domain.format.invalid_characters", "Invalid domain name", "Please provide a domain using valid characters (letters, numbers, dashes, dots).")
	}

	return issues
}

func preloadableDomainLevel(domain string) Issues {
	issues := Issues{}

	canon, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		return issues.addErrorf("internal.domain.name.cannot_compute_etld1", "Internal Error", "Could not compute eTLD+1.")
	}
	if canon != domain {
		return issues.addErrorf(
			IssueCode("domain.is_subdomain"),
			"Subdomain",
			"`%s` is a subdomain. Please preload `%s` instead. "+
				"(Due to the size of the preload list and the behaviour of "+
				"cookies across subdomains, we only accept automated preload list "+
				"submissions of whole registered domains.)",
			domain,
			canon,
		)
	}

	return issues
}

func checkSHA1(chain []*x509.Certificate) Issues {
	issues := Issues{}

	if firstSHA1, found := findPropertyInChain(isSHA1, chain); found {
		return issues.addErrorf(
			IssueCode("domain.tls.sha1"),
			"SHA-1 Certificate",
			"One or more of the certificates in your certificate chain "+
				"is signed using SHA-1. This needs to be replaced. "+
				"See https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html. "+
				"(The first SHA-1 certificate found has a common-name of %q.)",
			firstSHA1.Subject.CommonName,
		)
	}

	return issues
}

func checkWWW(host string) Issues {
	issues := Issues{}

	hasWWW := false
	if conn, err := net.DialTimeout("tcp", "www."+host+":443", dialTimeout); err == nil {
		hasWWW = true
		conn.Close()
	}

	if hasWWW {
		wwwConn, err := tls.DialWithDialer(&dialer, "tcp", "www."+host+":443", nil)
		if err != nil {
			return issues.addErrorf(
				IssueCode("domain.www.no_tls"),
				"www subdomain does not support HTTPS",
				"Domain error: The www subdomain exists, but we couldn't connect to it using HTTPS (%q). "+
					"Since many people type this by habit, HSTS preloading would likely "+
					"cause issues for your site.",
				err,
			)
		}
		wwwConn.Close()
	}

	return issues
}

func certChain(connState tls.ConnectionState) []*x509.Certificate {
	chain := connState.VerifiedChains[0]
	return chain[:len(chain)-1]
}

func findPropertyInChain(pred func(*x509.Certificate) bool, chain []*x509.Certificate) (*x509.Certificate, bool) {
	for _, cert := range chain {
		if pred(cert) {
			return cert, true
		}
	}

	return nil, false
}

func isSHA1(cert *x509.Certificate) bool {
	switch cert.SignatureAlgorithm {
	case x509.SHA1WithRSA, x509.ECDSAWithSHA1:
		return true
	default:
		return false
	}
}
<commit_msg>Clean up checkSHA1() helpers into a single function and move the cert chain computation into checkChain().<commit_after>package hstspreload

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"golang.org/x/net/publicsuffix"
)

const (
	// dialTimeout specifies the amount of time that TCP or TLS connections
	// can take to complete.
	dialTimeout = 10 * time.Second

	// The maximum number of redirects when you visit the root path of the
	// domain over HTTP or HTTPS.
	maxRedirects = 3
	httpsScheme  = "https"
)

// dialer is a global net.Dialer that's used whenever making TLS connections in
// order to enforce dialTimeout.
var dialer = net.Dialer{
	Timeout: dialTimeout,
}

var clientWithTimeout = http.Client{
	Timeout: dialTimeout,
}

// PreloadableDomain checks whether the domain passes HSTS preload
// requirements for Chromium. This includes:
//
// - Serving a single HSTS header that passes header requirements.
//
// - Using TLS settings that will not cause new problems for
// Chromium/Chrome users. (Example of a new problem: a missing intermediate certificate
// will turn an error page from overrideable to non-overridable on
// some mobile devices.)
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func PreloadableDomain(domain string) (header *string, issues Issues) {
	// Check domain format issues first, since we can report something
	// useful even if the other checks fail.
	issues = combineIssues(issues, checkDomainFormat(domain))
	if len(issues.Errors) > 0 {
		return header, issues
	}

	// We don't currently allow automatic submissions of subdomains.
	levelIssues := preloadableDomainLevel(domain)
	issues = combineIssues(issues, levelIssues)

	// Start with an initial probe, and don't do the follow-up checks if
	// we can't connect.
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		issues = combineIssues(issues, checkChain(*resp.TLS))

		preloadableResponse := make(chan Issues)
		httpRedirects := make(chan Issues)
		httpFirstRedirectsHSTS := make(chan Issues)
		httpsRedirects := make(chan Issues)
		www := make(chan Issues)

		// PreloadableResponse
		go func() {
			var preloadableIssues Issues
			header, preloadableIssues = PreloadableResponse(resp)
			preloadableResponse <- preloadableIssues
		}()

		// checkHTTPRedirects
		go func() {
			mainIssues, firstRedirectHSTSIssues := preloadableHTTPRedirects(domain)
			httpRedirects <- mainIssues
			httpFirstRedirectsHSTS <- firstRedirectHSTSIssues
		}()

		// checkHTTPSRedirects
		go func() {
			httpsRedirects <- preloadableHTTPSRedirects(domain)
		}()

		// checkWWW
		go func() {
			// Skip the WWW check if the domain is not eTLD+1.
			if len(levelIssues.Errors) == 0 {
				www <- checkWWW(domain)
			} else {
				www <- Issues{}
			}
		}()

		// Combine the issues in deterministic order.
		preloadableResponseIssues := <-preloadableResponse
		issues = combineIssues(issues, preloadableResponseIssues)
		issues = combineIssues(issues, <-httpRedirects)
		// If there are issues with the HSTS header in the main
		// PreloadableResponse() check, it is redundant to report
		// them in the response after redirecting from HTTP.
		if len(preloadableResponseIssues.Errors) == 0 {
			issues = combineIssues(issues, <-httpFirstRedirectsHSTS)
		}
		issues = combineIssues(issues, <-httpsRedirects)
		issues = combineIssues(issues, <-www)
	}

	return header, issues
}

// RemovableDomain checks whether the domain satisfies the requirements
// for being removed from the Chromium preload list:
//
// - Serving a single valid HSTS header.
//
// - The header must not contain the `preload` directive..
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func RemovableDomain(domain string) (header *string, issues Issues) {
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		var removableIssues Issues
		header, removableIssues = RemovableResponse(resp)
		issues = combineIssues(issues, removableIssues)
	}

	return header, issues
}

func getResponse(domain string) (*http.Response, Issues) {
	issues := Issues{}

	redirectPrevented := errors.New("REDIRECT_PREVENTED")

	client := http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return redirectPrevented
		},
		Timeout: dialTimeout,
	}

	resp, err := client.Get("https://" + domain)
	if err != nil {
		if urlError, ok := err.(*url.Error); !ok || urlError.Err != redirectPrevented {
			return resp, issues.addErrorf(
				IssueCode("domain.tls.cannot_connect"),
				"Cannot connect using TLS",
				"We cannot connect to https://%s using TLS (%q). This "+
					"might be caused by an incomplete certificate chain, which causes "+
					"issues on mobile devices. Check out your site at "+
					"https://www.ssllabs.com/ssltest/",
				domain,
				err,
			)
		}
	}

	return resp, issues
}

func checkDomainFormat(domain string) Issues {
	issues := Issues{}

	if strings.HasPrefix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.begins_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.HasSuffix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.ends_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.Index(domain, "..") != -1 {
		return issues.addErrorf(
			IssueCode("domain.format.contains_double_dot"),
			"Invalid domain name",
			"Please provide a domain that does not contain `..`")
	}
	if strings.Count(domain, ".") < 1 {
		return issues.addErrorf(
			IssueCode("domain.format.only_one_label"),
			"Invalid domain name",
			"Please provide a domain with least two labels "+
				"(e.g. `example.com` rather than `example` or `com`).")
	}

	domain = strings.ToLower(domain)
	for _, r := range domain {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '.' {
			continue
		}

		return issues.addErrorf("domain.format.invalid_characters", "Invalid domain name", "Please provide a domain using valid characters (letters, numbers, dashes, dots).")
	}

	return issues
}

func preloadableDomainLevel(domain string) Issues {
	issues := Issues{}

	canon, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		return issues.addErrorf("internal.domain.name.cannot_compute_etld1", "Internal Error", "Could not compute eTLD+1.")
	}
	if canon != domain {
		return issues.addErrorf(
			IssueCode("domain.is_subdomain"),
			"Subdomain",
			"`%s` is a subdomain. Please preload `%s` instead. "+
				"(Due to the size of the preload list and the behaviour of "+
				"cookies across subdomains, we only accept automated preload list "+
				"submissions of whole registered domains.)",
			domain,
			canon,
		)
	}

	return issues
}

func checkChain(connState tls.ConnectionState) Issues {
	fullChain := connState.VerifiedChains[0]
	chain := fullChain[:len(fullChain)-1] // Ignore the root CA
	return checkSHA1(chain)
}

func checkSHA1(chain []*x509.Certificate) Issues {
	issues := Issues{}

	for _, cert := range chain {
		if cert.SignatureAlgorithm == x509.SHA1WithRSA || cert.SignatureAlgorithm == x509.ECDSAWithSHA1 {
			return issues.addErrorf(
				IssueCode("domain.tls.sha1"),
				"SHA-1 Certificate",
				"One or more of the certificates in your certificate chain "+
					"is signed using SHA-1. This needs to be replaced. "+
					"See https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html. "+
					"(The first SHA-1 certificate found has a common-name of %q.)",
				cert.Subject.CommonName,
			)
		}
	}

	return issues
}

func checkWWW(host string) Issues {
	issues := Issues{}

	hasWWW := false
	if conn, err := net.DialTimeout("tcp", "www."+host+":443", dialTimeout); err == nil {
		hasWWW = true
		conn.Close()
	}

	if hasWWW {
		wwwConn, err := tls.DialWithDialer(&dialer, "tcp", "www."+host+":443", nil)
		if err != nil {
			return issues.addErrorf(
				IssueCode("domain.www.no_tls"),
				"www subdomain does not support HTTPS",
				"Domain error: The www subdomain exists, but we couldn't connect to it using HTTPS (%q). "+
					"Since many people type this by habit, HSTS preloading would likely "+
					"cause issues for your site.",
				err,
			)
		}
		wwwConn.Close()
	}

	return issues
}
<|endoftext|>


<commit_before>package nimbusec

import (
	"bytes"
	"fmt"
	"strconv"
	"time"
)

// Domain represents a nimbusec monitored domain.
type Domain struct {
	Id        int      `json:"id,omitempty"` // Unique identification of domain
	Bundle    string   `json:"bundle"`       // ID of assigned bundle
	Name      string   `json:"name"`         // Name of domain (usually DNS name)
	Scheme    string   `json:"scheme"`       // Flag whether the domain uses http or https
	DeepScan  string   `json:"deepScan"`     // Starting point for the domain deep scan
	FastScans []string `json:"fastScans"`    // Landing pages of the domain scanned
}

type DomainEvent struct {
	Time    Timestamp `json:"time"`
	Event   string    `json:"event"`
	Human   string    `json:"human"`
	Machine string    `json:"machine"`
}

type Timestamp struct {
	time.Time
}

func (t Timestamp) MarshalJSON() ([]byte, error) {
	ts := t.Unix()
	stamp := strconv.FormatInt(ts*1000, 10)
	return []byte(stamp), nil
}

func (t *Timestamp) UnmarshalJSON(b []byte) error {
	if bytes.Equal(b, []byte("null")) {
		return nil
	}

	ts, err := strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		return err
	}

	*t = Timestamp{time.Unix(ts/1000, 0)}
	return nil
}

// CreateDomain issues the API to create the given domain.
func (a *API) CreateDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain")
	err := a.post(url, params{}, domain, dst)
	return dst, err
}

// CreateOrUpdateDomain issues the nimbusec API to create the given domain. Instead
// of failing when attempting to create a duplicate domain, this method will update
// the remote domain instead.
func (a *API) CreateOrUpdateDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain")
	err := a.post(url, params{"upsert": "true"}, domain, dst)
	return dst, err
}

// CreateOrGetDomain issues the nimbusec API to create the given domain. Instead
// of failing when attempting to create a duplicate domain, this method will fetch
// the remote domain instead.
func (a *API) CreateOrGetDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain")
	err := a.post(url, params{"upsert": "false"}, domain, dst)
	return dst, err
}

// GetDomain retrieves a domain from the API by its ID.
func (a *API) GetDomain(domain int) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain/%d", domain)
	err := a.get(url, params{}, dst)
	return dst, err
}

// GetDomainByName fetches an domain by its name.
func (a *API) GetDomainByName(name string) (*Domain, error) {
	domains, err := a.FindDomains(fmt.Sprintf("name eq \"%s\"", name))
	if err != nil {
		return nil, err
	}

	if len(domains) == 0 {
		return nil, ErrNotFound
	}

	if len(domains) > 1 {
		return nil, fmt.Errorf("name %q matched too many domains. please contact nimbusec.", name)
	}

	return &domains[0], nil
}

// FindDomains searches for domains that match the given filter criteria.
func (a *API) FindDomains(filter string) ([]Domain, error) {
	params := params{}
	if filter != EmptyFilter {
		params["q"] = filter
	}

	dst := make([]Domain, 0)
	url := a.geturl("/v2/domain")
	err := a.get(url, params, &dst)
	return dst, err
}

// UpdateDOmain issues the nimbusec API to update a domain.
func (a *API) UpdateDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain/%d", domain.Id)
	err := a.put(url, params{}, domain, dst)
	return dst, err
}

// DeleteDomain issues the API to delete a domain. When clean=false, the domain and
// all assiciated data will only be marked as deleted, whereas with clean=true the data
// will also be removed from the nimbusec system.
func (a *API) DeleteDomain(d *Domain, clean bool) error {
	url := a.geturl("/v2/domain/%d", d.Id)
	return a.delete(url, params{
		"pleaseremovealldata": fmt.Sprintf("%t", clean),
	})
}

// FindInfected searches for domains that have pending Results that match the
// given filter criteria.
func (a *API) FindInfected(filter string) ([]Domain, error) {
	params := make(map[string]string)
	if filter != EmptyFilter {
		params["q"] = filter
	}

	dst := make([]Domain, 0)
	url := a.geturl("/v2/infected")
	err := a.get(url, params, &dst)
	return dst, err
}

// ListDomainConfigs fetches the list of all available configuration keys for the
// given domain.
func (a *API) ListDomainConfigs(domain int) ([]string, error) {
	dst := make([]string, 0)
	url := a.geturl("/v2/domain/%d/config", domain)
	err := a.get(url, params{}, &dst)
	return dst, err
}

// GetDomainConfig fetches the requested domain configuration.
func (a *API) GetDomainConfig(domain int, key string) (string, error) {
	url := a.geturl("/v2/domain/%d/config/%s/", domain, key)
	return a.getTextPlain(url, params{})
}

// SetDomainConfig sets the domain configuration `key` to the requested value.
// This method will create the domain configuration if it does not exist yet.
func (a *API) SetDomainConfig(domain int, key string, value string) (string, error) {
	url := a.geturl("/v2/domain/%d/config/%s/", domain, key)
	return a.putTextPlain(url, params{}, value)
}

// DeleteDomainConfig issues the API to delete the domain configuration with
// the provided key.
func (a *API) DeleteDomainConfig(domain int, key string) error {
	url := a.geturl("/v2/domain/%d/config/%s/", domain, key)
	return a.delete(url, params{})
}

func (a *API) GetDomainEvent(domain int, filter string, limit int) ([]DomainEvent, error) {
	params := params{
		"limit": strconv.Itoa(limit),
	}
	if filter != EmptyFilter {
		params["q"] = filter
	}

	dst := make([]DomainEvent, 0)
	url := a.geturl("/v2/domain/%d/events", domain)
	err := a.get(url, params, &dst)
	return dst, err
}

func (a *API) CreateDomainEvent(domain int, log *DomainEvent) error {
	url := a.geturl("/v2/domain/%d/events", domain)
	return a.post(url, params{}, log, nil)
}
<commit_msg>added cms api<commit_after>package nimbusec

import (
	"bytes"
	"fmt"
	"strconv"
	"time"
)

// Domain represents a nimbusec monitored domain.
type Domain struct {
	Id        int      `json:"id,omitempty"` // Unique identification of domain
	Bundle    string   `json:"bundle"`       // ID of assigned bundle
	Name      string   `json:"name"`         // Name of domain (usually DNS name)
	Scheme    string   `json:"scheme"`       // Flag whether the domain uses http or https
	DeepScan  string   `json:"deepScan"`     // Starting point for the domain deep scan
	FastScans []string `json:"fastScans"`    // Landing pages of the domain scanned
}

type DomainEvent struct {
	Time    Timestamp `json:"time"`
	Event   string    `json:"event"`
	Human   string    `json:"human"`
	Machine string    `json:"machine"`
}

type Timestamp struct {
	time.Time
}

type DomainMetadata struct {
	LastDeepScan Timestamp `json:"lastDeepScan"` // timestamp (in ms) of last external scan of the whole site
	NextDeepScan Timestamp `json:"nextDeepScan"` // timestamp (in ms) for next external scan of the whole site
	LastFastScan Timestamp `json:"lastFastScan"` // timestamp (in ms) of last external scan of the landing pages
	NextFastScan Timestamp `json:"nextFastScan"` // timestamp (in ms) for next external scan of the landing pages
	Agent        Timestamp `json:"agent"`        // status of server agent for the given domain
	Cms          string    `json:"cms"`          // detected CMS vendor and version
	Httpd        string    `json:"httpd"`        // detected HTTP server vendor and version
	Php          string    `json:"php"`          // detected PHP version
	Files        int       `json:"files"`        // number of downloaded files/URLs for last deep scan
	Size         int       `json:"size"`         // size of downloaded files for last deep scan (in byte)}
}

type CMSView struct {
	CMS          string `json:"cpeId"`
	LatestStable string `json:"latestStable"`
}

func (t Timestamp) MarshalJSON() ([]byte, error) {
	ts := t.Unix()
	stamp := strconv.FormatInt(ts*1000, 10)
	return []byte(stamp), nil
}

func (t *Timestamp) UnmarshalJSON(b []byte) error {
	if bytes.Equal(b, []byte("null")) {
		return nil
	}

	ts, err := strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		return err
	}

	*t = Timestamp{time.Unix(ts/1000, 0)}
	return nil
}

// CreateDomain issues the API to create the given domain.
func (a *API) CreateDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain")
	err := a.post(url, params{}, domain, dst)
	return dst, err
}

// CreateOrUpdateDomain issues the nimbusec API to create the given domain. Instead
// of failing when attempting to create a duplicate domain, this method will update
// the remote domain instead.
func (a *API) CreateOrUpdateDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain")
	err := a.post(url, params{"upsert": "true"}, domain, dst)
	return dst, err
}

// CreateOrGetDomain issues the nimbusec API to create the given domain. Instead
// of failing when attempting to create a duplicate domain, this method will fetch
// the remote domain instead.
func (a *API) CreateOrGetDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain")
	err := a.post(url, params{"upsert": "false"}, domain, dst)
	return dst, err
}

// GetDomain retrieves a domain from the API by its ID.
func (a *API) GetDomain(domain int) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain/%d", domain)
	err := a.get(url, params{}, dst)
	return dst, err
}

// GetDomainByName fetches an domain by its name.
func (a *API) GetDomainByName(name string) (*Domain, error) {
	domains, err := a.FindDomains(fmt.Sprintf("name eq \"%s\"", name))
	if err != nil {
		return nil, err
	}

	if len(domains) == 0 {
		return nil, ErrNotFound
	}

	if len(domains) > 1 {
		return nil, fmt.Errorf("name %q matched too many domains. please contact nimbusec.", name)
	}

	return &domains[0], nil
}

// FindDomains searches for domains that match the given filter criteria.
func (a *API) FindDomains(filter string) ([]Domain, error) {
	params := params{}
	if filter != EmptyFilter {
		params["q"] = filter
	}

	dst := make([]Domain, 0)
	url := a.geturl("/v2/domain")
	err := a.get(url, params, &dst)
	return dst, err
}

// UpdateDOmain issues the nimbusec API to update a domain.
func (a *API) UpdateDomain(domain *Domain) (*Domain, error) {
	dst := new(Domain)
	url := a.geturl("/v2/domain/%d", domain.Id)
	err := a.put(url, params{}, domain, dst)
	return dst, err
}

// DeleteDomain issues the API to delete a domain. When clean=false, the domain and
// all assiciated data will only be marked as deleted, whereas with clean=true the data
// will also be removed from the nimbusec system.
func (a *API) DeleteDomain(d *Domain, clean bool) error {
	url := a.geturl("/v2/domain/%d", d.Id)
	return a.delete(url, params{
		"pleaseremovealldata": fmt.Sprintf("%t", clean),
	})
}

// FindInfected searches for domains that have pending Results that match the
// given filter criteria.
func (a *API) FindInfected(filter string) ([]Domain, error) {
	params := make(map[string]string)
	if filter != EmptyFilter {
		params["q"] = filter
	}

	dst := make([]Domain, 0)
	url := a.geturl("/v2/infected")
	err := a.get(url, params, &dst)
	return dst, err
}

// ListDomainConfigs fetches the list of all available configuration keys for the
// given domain.
func (a *API) ListDomainConfigs(domain int) ([]string, error) {
	dst := make([]string, 0)
	url := a.geturl("/v2/domain/%d/config", domain)
	err := a.get(url, params{}, &dst)
	return dst, err
}

// GetDomainConfig fetches the requested domain configuration.
func (a *API) GetDomainConfig(domain int, key string) (string, error) {
	url := a.geturl("/v2/domain/%d/config/%s/", domain, key)
	return a.getTextPlain(url, params{})
}

// SetDomainConfig sets the domain configuration `key` to the requested value.
// This method will create the domain configuration if it does not exist yet.
func (a *API) SetDomainConfig(domain int, key string, value string) (string, error) {
	url := a.geturl("/v2/domain/%d/config/%s/", domain, key)
	return a.putTextPlain(url, params{}, value)
}

// DeleteDomainConfig issues the API to delete the domain configuration with
// the provided key.
func (a *API) DeleteDomainConfig(domain int, key string) error {
	url := a.geturl("/v2/domain/%d/config/%s/", domain, key)
	return a.delete(url, params{})
}

func (a *API) GetDomainEvent(domain int, filter string, limit int) ([]DomainEvent, error) {
	params := params{
		"limit": strconv.Itoa(limit),
	}
	if filter != EmptyFilter {
		params["q"] = filter
	}

	dst := make([]DomainEvent, 0)
	url := a.geturl("/v2/domain/%d/events", domain)
	err := a.get(url, params, &dst)
	return dst, err
}

func (a *API) CreateDomainEvent(domain int, log *DomainEvent) error {
	url := a.geturl("/v2/domain/%d/events", domain)
	return a.post(url, params{}, log, nil)
}

func (a *API) GetDomainMetadata(domain int) (*DomainMetadata, error) {
	dst := new(DomainMetadata)
	url := a.geturl("/v2/domain/%d/metadata", domain)
	err := a.get(url, params{}, &dst)
	return dst, err
}

func (a *API) GetDomainCMS(domain int) ([]CMSView, error) {
	dst := make([]CMSView, 0)
	url := a.geturl("/v2/domain/%d/cms", domain)
	err := a.get(url, params{}, &dst)
	return dst, err
}

func (a *API) GetCMSName(cpeID string) (string, error) {
	params := params{
		"cpeid": cpeID,
	}
	url := a.geturl("/v2/cms/name")
	return a.getTextPlain(url, params)
}
<|endoftext|>


<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/google/subcommands"
	"github.com/stratumn/go/generator"
	"github.com/stratumn/go/generator/repo"
	"golang.org/x/net/context"
)

// Generate is a command to generate projects.
type Generate struct {
	repo      string
	generator string
	owner     string
}

// Name implements github.com/google/subcommands.Command.Name().
func (*Generate) Name() string {
	return "generate"
}

// Synopsis implements github.com/google/subcommands.Command.Synopsis().
func (*Generate) Synopsis() string {
	return "generate a project"
}

// Usage implements github.com/google/subcommands.Command.Usage().
func (*Generate) Usage() string {
	return `generate [flags] [out]:
  Generate a project.
`
}

// SetFlags implements github.com/google/subcommands.Command.SetFlags().
func (cmd *Generate) SetFlags(f *flag.FlagSet) {
	f.StringVar(&cmd.owner, "owner", "", "Github owner")
	f.StringVar(&cmd.repo, "repo", "", "Github repository")
	f.StringVar(&cmd.generator, "generator", "", "generator name")
}

// Execute implements github.com/google/subcommands.Command.Execute().
func (cmd *Generate) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	args := f.Args()

	if len(args) != 1 {
		fmt.Println(cmd.Usage())
		return subcommands.ExitUsageError
	}

	out := args[0]

	if cmd.owner == "" {
		cmd.owner = DefaultGeneratorsOwner
	}
	if cmd.repo == "" {
		cmd.repo = DefaultGeneratorsRepo
	}

	path, err := generatorPath(cmd.owner, cmd.repo)
	if err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}
	repo := repo.New(path, cmd.owner, cmd.repo)
	if err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}

	name := cmd.generator
	if name == "" {
		list, err := repo.List()
		if err != nil {
			fmt.Println(err)
			return subcommands.ExitFailure
		}

		in := generator.StringSelect{
			InputShared: generator.InputShared{
				Prompt: "What would you like to generate?",
			},
			Options: []generator.StringSelectOption{},
		}
		for i, desc := range list {
			in.Options = append(in.Options, generator.StringSelectOption{
				Input: strconv.Itoa(i + 1),
				Value: desc.Name,
				Text:  desc.Description,
			})
		}

		fmt.Print(in.Msg())
		reader := bufio.NewReader(os.Stdin)

		for {
			fmt.Print("? ")
			str, err := reader.ReadString('\n')
			if err != nil {
				fmt.Println(err)
				return subcommands.ExitFailure
			}
			str = strings.TrimSpace(str)
			if err := in.Set(str); err != nil {
				fmt.Println(err)
				continue
			}
			name = in.Get().(string)
			break
		}
	}

	varsPath, err := varsPath()
	if err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}
	varsFile, err := os.Open(varsPath)

	vars := map[string]interface{}{}

	if err != nil {
		if !os.IsNotExist(err) {
			fmt.Println(err)
			return subcommands.ExitFailure
		}
	} else {
		dec := json.NewDecoder(varsFile)
		if err := dec.Decode(&vars); err != nil {
			fmt.Println(err)
			return subcommands.ExitFailure
		}
	}

	vars["dir"] = out

	opts := generator.Options{
		DefVars:  vars,
		TmplVars: vars,
	}

	if err := repo.Generate(name, out, &opts); err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}

	fmt.Println("Done!")

	return subcommands.ExitSuccess
}
<commit_msg>Fix dir variable in generator (#37)<commit_after>// Copyright 2016 Stratumn SAS. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/google/subcommands"
	"github.com/stratumn/go/generator"
	"github.com/stratumn/go/generator/repo"
	"golang.org/x/net/context"
)

// Generate is a command to generate projects.
type Generate struct {
	repo      string
	generator string
	owner     string
}

// Name implements github.com/google/subcommands.Command.Name().
func (*Generate) Name() string {
	return "generate"
}

// Synopsis implements github.com/google/subcommands.Command.Synopsis().
func (*Generate) Synopsis() string {
	return "generate a project"
}

// Usage implements github.com/google/subcommands.Command.Usage().
func (*Generate) Usage() string {
	return `generate [flags] [out]:
  Generate a project.
`
}

// SetFlags implements github.com/google/subcommands.Command.SetFlags().
func (cmd *Generate) SetFlags(f *flag.FlagSet) {
	f.StringVar(&cmd.owner, "owner", "", "Github owner")
	f.StringVar(&cmd.repo, "repo", "", "Github repository")
	f.StringVar(&cmd.generator, "generator", "", "generator name")
}

// Execute implements github.com/google/subcommands.Command.Execute().
func (cmd *Generate) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	args := f.Args()

	if len(args) != 1 {
		fmt.Println(cmd.Usage())
		return subcommands.ExitUsageError
	}

	out := args[0]

	if cmd.owner == "" {
		cmd.owner = DefaultGeneratorsOwner
	}
	if cmd.repo == "" {
		cmd.repo = DefaultGeneratorsRepo
	}

	path, err := generatorPath(cmd.owner, cmd.repo)
	if err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}
	repo := repo.New(path, cmd.owner, cmd.repo)
	if err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}

	name := cmd.generator
	if name == "" {
		list, err := repo.List()
		if err != nil {
			fmt.Println(err)
			return subcommands.ExitFailure
		}

		in := generator.StringSelect{
			InputShared: generator.InputShared{
				Prompt: "What would you like to generate?",
			},
			Options: []generator.StringSelectOption{},
		}
		for i, desc := range list {
			in.Options = append(in.Options, generator.StringSelectOption{
				Input: strconv.Itoa(i + 1),
				Value: desc.Name,
				Text:  desc.Description,
			})
		}

		fmt.Print(in.Msg())
		reader := bufio.NewReader(os.Stdin)

		for {
			fmt.Print("? ")
			str, err := reader.ReadString('\n')
			if err != nil {
				fmt.Println(err)
				return subcommands.ExitFailure
			}
			str = strings.TrimSpace(str)
			if err := in.Set(str); err != nil {
				fmt.Println(err)
				continue
			}
			name = in.Get().(string)
			break
		}
	}

	varsPath, err := varsPath()
	if err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}
	varsFile, err := os.Open(varsPath)

	vars := map[string]interface{}{}

	if err != nil {
		if !os.IsNotExist(err) {
			fmt.Println(err)
			return subcommands.ExitFailure
		}
	} else {
		dec := json.NewDecoder(varsFile)
		if err := dec.Decode(&vars); err != nil {
			fmt.Println(err)
			return subcommands.ExitFailure
		}
	}

	vars["dir"] = filepath.Base(out)

	opts := generator.Options{
		DefVars:  vars,
		TmplVars: vars,
	}

	if err := repo.Generate(name, out, &opts); err != nil {
		fmt.Println(err)
		return subcommands.ExitFailure
	}

	fmt.Println("Done!")

	return subcommands.ExitSuccess
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ardielle/ardielle-go/rdl"
	"testing"
	"io/ioutil"
	"os"
	"encoding/json"
	"github.com/yahoo/parsec-rdl-gen/utils"
)

func TestParseRegex(test *testing.T) {
	data, err := ioutil.ReadFile("../../testdata/rdl.json")
	if err != nil {
		test.Error("can not read sample file ")
		os.Exit(1)
	}
	var schema rdl.Schema
	err = json.Unmarshal(data, &schema)
	if err != nil {
		test.Error("unmarshal sample data fail")
		os.Exit(1)
	}
	pathInfos := extractPathInfo(&schema, "/api")
	pathInfoJson, err := json.Marshal(pathInfos)
	if err != nil {
		test.Errorf("marshal json error: %v", err)
		os.Exit(1)
	}
	expectedPathInfoJson, err := ioutil.ReadFile("../../testdata/expectedRdlPathInfo.json")
	if err != nil {
		test.Error("read expected data fail")
		os.Exit(1)
	}
	if string(pathInfoJson) != string(expectedPathInfoJson) {
		test.Error("result not as expected")
		os.Exit(1)
	}
}

func TestPathRegexGenerator(test *testing.T) {
	type pathInfos struct {
		method string
		path string
	}
	uriPaths := []pathInfos {
		{"POST", "/passcodes"},
		{"GET", "/passcodes"},
		{"GET", "/passcodes/{id}"},
		{"GET", "/passcodes/{id}/bbb/{id2}"},
		{"GET", "/transactions?offset={offset}&count={count}"},
	}
	expectedPathRegex := []string {
		`/passcodes/?$`,
		`/passcodes(/?\?|/?$)`,
		`/passcodes/[^/]+(/?\?|/?$)`,
		`/passcodes/[^/]+/bbb/[^/]+(/?\?|/?$)`,
		`/transactions(/?\?|/?$)`,
	}
	for idx, pathInfo := range uriPaths {
		if pathRegex := genUriRegex(pathInfo.path, pathInfo.method);pathRegex != expectedPathRegex[idx] {
			test.Errorf("pathRegex generated not as expected, path: %v, actulPathRegex: %v",
			pathInfo.path, pathRegex)
		}
	}
}

<commit_msg>remove unused import<commit_after>package main

import (
	"github.com/ardielle/ardielle-go/rdl"
	"testing"
	"io/ioutil"
	"os"
	"encoding/json"
)

func TestParseRegex(test *testing.T) {
	data, err := ioutil.ReadFile("../../testdata/rdl.json")
	if err != nil {
		test.Error("can not read sample file ")
		os.Exit(1)
	}
	var schema rdl.Schema
	err = json.Unmarshal(data, &schema)
	if err != nil {
		test.Error("unmarshal sample data fail")
		os.Exit(1)
	}
	pathInfos := extractPathInfo(&schema, "/api")
	pathInfoJson, err := json.Marshal(pathInfos)
	if err != nil {
		test.Errorf("marshal json error: %v", err)
		os.Exit(1)
	}
	expectedPathInfoJson, err := ioutil.ReadFile("../../testdata/expectedRdlPathInfo.json")
	if err != nil {
		test.Error("read expected data fail")
		os.Exit(1)
	}
	if string(pathInfoJson) != string(expectedPathInfoJson) {
		test.Error("result not as expected")
		os.Exit(1)
	}
}

func TestPathRegexGenerator(test *testing.T) {
	type pathInfos struct {
		method string
		path string
	}
	uriPaths := []pathInfos {
		{"POST", "/passcodes"},
		{"GET", "/passcodes"},
		{"GET", "/passcodes/{id}"},
		{"GET", "/passcodes/{id}/bbb/{id2}"},
		{"GET", "/transactions?offset={offset}&count={count}"},
	}
	expectedPathRegex := []string {
		`/passcodes/?$`,
		`/passcodes(/?\?|/?$)`,
		`/passcodes/[^/]+(/?\?|/?$)`,
		`/passcodes/[^/]+/bbb/[^/]+(/?\?|/?$)`,
		`/transactions(/?\?|/?$)`,
	}
	for idx, pathInfo := range uriPaths {
		if pathRegex := genUriRegex(pathInfo.path, pathInfo.method);pathRegex != expectedPathRegex[idx] {
			test.Errorf("pathRegex generated not as expected, path: %v, actulPathRegex: %v",
			pathInfo.path, pathRegex)
		}
	}
}

<|endoftext|>


<commit_before>package test

import "path/filepath"
import "tritium/whale"
import "testing"
//import "log4go"
//import "runtime/debug"
import "runtime"
import "fmt"
import "tritium/spec"
import tp "tritium/proto"
import "tritium/packager"
import "golog"
import "time"

func RunTest(path string) (result *spec.Result) {
	result = spec.NewResult()

	logger := golog.NewLogger("tritium")
	logger.AddProcessor("info", golog.NewConsoleProcessor(golog.LOG_INFO, true))

  /*** TODO(SJ) : Reintegrate w new log system. We need to catch errors when running tests
	defer func() {
		if x := recover(); x != nil {
			err, ok := x.(error)
			if ok {
				logger.Error(path + " === " + err.Error() + "\n\n" + string(debug.Stack()))
			} else {
				logger.Error(path + " === " + x.(string) + "\n\n" + string(debug.Stack()))
			}
		}
		for _, rec := range logWriter.Logs {
			error := log4go.FormatLogRecord("[%D %T] [%L] (%S) %M", rec)
			result.Error(path, error)
		}
	}()
	*/

	spec, err := spec.LoadSpec(path, pkg)

	if err != nil {
		result.Error(path, fmt.Sprintf("Error loading test spec:\n%v\n", err.Error()))
		return
	}

	eng := whale.NewEngine(logger)
	d, _ := time.ParseDuration("1m")
	result.Merge(spec.Compare(eng.Run(spec.Script, nil, spec.Input, spec.Vars, time.Now().Add(d))))

	return
}

func GatherTests(directory string) (tests []string) {
	matches, err := filepath.Glob(filepath.Join(directory, "main.ts"))

	if err == nil && matches != nil{
		tests = append(tests, directory)
	}

	subdirs, _ := filepath.Glob(filepath.Join(directory, "*"))

	for _, subdir := range subdirs {
		tests = append(tests, GatherTests(subdir)...)
	}

	return
}

func relativeDirectory(directoryFromRoot string) (directory string, ok bool) {
	_, file, _, ok := runtime.Caller(0)

	if !ok {
		return
	}

	directory = filepath.Join(file, "../../", directoryFromRoot)

	return
}

var pkg *tp.Package

func initializePackage() {
	packagesPath, ok := relativeDirectory("packages")

	if !ok {
		panic("Can't find root tritium directory to build default package")
	}

	tpkg := packager.LoadDefaultPackage(&packagesPath)
	pkg = tpkg.Package
}

func RunTestSuite(directoryFromRoot string, t *testing.T) {
	directory, ok := relativeDirectory(directoryFromRoot)
	globalResult := spec.NewResult()
	initializePackage()

	if !ok {
		t.Error("Couldn't resolve root directory")
		t.FailNow()
	}

	testPaths := GatherTests(directory)

	for _, testPath := range testPaths {
		testResult := RunTest(testPath)
		print(testResult.CharStatus())
		globalResult.Merge(testResult)
	}

	for _, error := range globalResult.Errors {
		t.Fail()
		println("\n=========================================", error.Location, "\n")
		if error.Panic {
			fmt.Printf(error.Message)
		} else {
			fmt.Printf("\n==========\n%v :: %v \n\n Got \n----------\n%v\n\n Expected \n----------\n%v\n", error.Name, error.Message, error.Got, error.Expected)
		}
	}
	fmt.Printf("\n+++ Finished test suite(%v) +++\n\n", directoryFromRoot)

}
<commit_msg>better display<commit_after>package test

import "path/filepath"
import "tritium/whale"
import "testing"
//import "log4go"
//import "runtime/debug"
import "runtime"
import "fmt"
import "tritium/spec"
import tp "tritium/proto"
import "tritium/packager"
import "golog"
import "time"

func RunTest(path string) (result *spec.Result) {
	result = spec.NewResult()

	logger := golog.NewLogger("tritium")
	logger.AddProcessor("info", golog.NewConsoleProcessor(golog.LOG_INFO, true))

  /*** TODO(SJ) : Reintegrate w new log system. We need to catch errors when running tests
	defer func() {
		if x := recover(); x != nil {
			err, ok := x.(error)
			if ok {
				logger.Error(path + " === " + err.Error() + "\n\n" + string(debug.Stack()))
			} else {
				logger.Error(path + " === " + x.(string) + "\n\n" + string(debug.Stack()))
			}
		}
		for _, rec := range logWriter.Logs {
			error := log4go.FormatLogRecord("[%D %T] [%L] (%S) %M", rec)
			result.Error(path, error)
		}
	}()
	*/

	spec, err := spec.LoadSpec(path, pkg)

	if err != nil {
		result.Error(path, fmt.Sprintf("Error loading test spec:\n%v\n", err.Error()))
		return
	}

	eng := whale.NewEngine(logger)
	d, _ := time.ParseDuration("1m")
	result.Merge(spec.Compare(eng.Run(spec.Script, nil, spec.Input, spec.Vars, time.Now().Add(d))))

	return
}

func GatherTests(directory string) (tests []string) {
	matches, err := filepath.Glob(filepath.Join(directory, "main.ts"))

	if err == nil && matches != nil{
		tests = append(tests, directory)
	}

	subdirs, _ := filepath.Glob(filepath.Join(directory, "*"))

	for _, subdir := range subdirs {
		tests = append(tests, GatherTests(subdir)...)
	}

	return
}

func relativeDirectory(directoryFromRoot string) (directory string, ok bool) {
	_, file, _, ok := runtime.Caller(0)

	if !ok {
		return
	}

	directory = filepath.Join(file, "../../", directoryFromRoot)

	return
}

var pkg *tp.Package

func initializePackage() {
	packagesPath, ok := relativeDirectory("packages")

	if !ok {
		panic("Can't find root tritium directory to build default package")
	}

	tpkg := packager.LoadDefaultPackage(&packagesPath)
	pkg = tpkg.Package
}

func RunTestSuite(directoryFromRoot string, t *testing.T) {
	directory, ok := relativeDirectory(directoryFromRoot)
	globalResult := spec.NewResult()
	initializePackage()

	if !ok {
		t.Error("Couldn't resolve root directory")
		t.FailNow()
	}

	testPaths := GatherTests(directory)

	for _, testPath := range testPaths {
		testResult := RunTest(testPath)
		print(testResult.CharStatus())
		globalResult.Merge(testResult)
	}

	for _, error := range globalResult.Errors {
		t.Fail()
		println("\n=========================================", error.Location, "\n")
		if error.Panic {
			fmt.Printf(error.Message)
		} else {
			println(fmt.Sprintf("\n==========\n%v :: %v \n\n Got \n----------\n[%v]\n\n Expected \n----------\n[%v]\n", error.Name, error.Message, error.Got, error.Expected))
		}
	}
	fmt.Printf("\n+++ Finished test suite(%v) +++\n\n", directoryFromRoot)

}
<|endoftext|>


<commit_before>package aws_client

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/service/ec2"

	"github.com/xingzhou/go_service_broker/utils"
)

const (
	AMI_ID                = "ami-dc5e75b4" //"ami-ecb68a84"
	SECURITY_GROUP_ID     = "sg-b23aead6"
	SUBNET_ID             = "subnet-0c75a427"
	KEYPAIR_NAME          = "broker_keypair"
	INSTANCE_TYPE         = "t2.micro"
	LINUX_USER            = "ubuntu"
	KEYPAIR_DIR_NAME      = ".gsb"
	PIRVATE_KEY_FILE_NAME = "broker_id_rsa"
	PUBLIC_KEY_FILE_NAME  = "broker_id_rsa.pub"
)

type Client interface {
	CreateInstance(parameters interface{}) (string, error)
	GetInstanceState(instanceId string) (string, error)
	InjectKeyPair(instanceId string) (string, error)
	DeleteInstance(instanceId string) error
	RevokeKeyPair(instanceId string, privateKey string) error
}

type AWSClient struct {
	EC2Client *ec2.EC2
}

func NewClient(region string) *AWSClient {
	return &AWSClient{
		EC2Client: ec2.New(&aws.Config{Region: region}),
	}
}

func (c *AWSClient) CreateInstance(parameters interface{}) (string, error) {
	var amiId string

	switch parameters.(type) {
	case map[string]interface{}:
		param := parameters.(map[string]interface{})
		amiId = param["ami_id"].(string)
	default:
		amiId = AMI_ID
	}

	return c.createInstance(amiId)
}

func (c *AWSClient) GetInstanceState(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	state, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].State.Name))
	return state, nil
}

func (c *AWSClient) setupKeyPair() error {
	private_key_file := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME)

	if !utils.Exists(private_key_file) {
		keypairInput := &ec2.CreateKeyPairInput{
			KeyName: aws.String(KEYPAIR_NAME),
		}

		keypairOutput, err := c.EC2Client.CreateKeyPair(keypairInput)
		if err != nil {
			return err
		}

		key_dir := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME)
		if !utils.MkDir(key_dir) {
			return errors.New("failed to create local keypair directory")
		}

		key_data, _ := strconv.Unquote(awsutil.StringValue(keypairOutput.KeyMaterial))
		err = utils.WriteFile(private_key_file, []byte(key_data))
		if err != nil {
			return errors.New("failed to save private key file")
		}
	}

	return nil
}

func (c *AWSClient) InjectKeyPair(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	ip, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].PublicIPAddress))
	pemBytes, err := utils.ReadFile(path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME))
	if err != nil {
		return "", err
	}

	awsSShClient, err := utils.GetSshClient(LINUX_USER, pemBytes, ip)
	if err != nil {
		return "", err
	}

	command := `rm -f ./broker_id_rsa ./broker_id_rsa.pub
		ssh-keygen -q -t rsa -N ""  -f ./broker_id_rsa
		cat ./broker_id_rsa.pub >> .ssh/authorized_keys
		cat ./broker_id_rsa`

	privateKey, err := awsSShClient.ExecCommand(command)
	if err != nil {
		return "", err
	}

	return privateKey, nil
}

func (c *AWSClient) createInstance(imageId string) (string, error) {
	err := c.setupKeyPair()
	if err != nil {
		return "", err
	}

	instanceInput := &ec2.RunInstancesInput{
		ImageID:  aws.String(imageId), // Required
		MaxCount: aws.Long(1),         // Required
		MinCount: aws.Long(1),         // Required
		// AdditionalInfo: aws.String("String"),
		// BlockDeviceMappings: []*ec2.BlockDeviceMapping{
		// 	&ec2.BlockDeviceMapping{ // Required
		// 		DeviceName: aws.String("String"),
		// 		EBS: &ec2.EBSBlockDevice{
		// 			DeleteOnTermination: aws.Boolean(true),
		// 			Encrypted:           aws.Boolean(true),
		// 			IOPS:                aws.Long(1),
		// 			SnapshotID:          aws.String("String"),
		// 			VolumeSize:          aws.Long(1),
		// 			VolumeType:          aws.String("VolumeType"),
		// 		},
		// 		NoDevice:    aws.String("String"),
		// 		VirtualName: aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// ClientToken: aws.String("String"),
		// DisableAPITermination: aws.Boolean(true),
		// DryRun:                aws.Boolean(true),
		// EBSOptimized:          aws.Boolean(true),
		// IAMInstanceProfile: &ec2.IAMInstanceProfileSpecification{
		// 	ARN:  aws.String("String"),
		// 	Name: aws.String("String"),
		// },
		// InstanceInitiatedShutdownBehavior: aws.String("ShutdownBehavior"),
		InstanceType: aws.String(INSTANCE_TYPE),
		// KernelID:                          aws.String("String"),
		KeyName: aws.String(KEYPAIR_NAME),
		// Monitoring: &ec2.RunInstancesMonitoringEnabled{
		// 	Enabled: aws.Boolean(true), // Required
		// },
		// NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
		// 	&ec2.InstanceNetworkInterfaceSpecification{ // Required
		// 		AssociatePublicIPAddress: aws.Boolean(true),
		// 		DeleteOnTermination:      aws.Boolean(true),
		// 		Description:              aws.String("String"),
		// 		DeviceIndex:              aws.Long(1),
		// 		Groups: []*string{
		// 			aws.String("String"), // Required
		// 			// More values...
		// 		},
		// 		NetworkInterfaceID: aws.String("String"),
		// 		PrivateIPAddress:   aws.String("String"),
		// 		PrivateIPAddresses: []*ec2.PrivateIPAddressSpecification{
		// 			&ec2.PrivateIPAddressSpecification{ // Required
		// 				PrivateIPAddress: aws.String("String"), // Required
		// 				Primary:          aws.Boolean(true),
		// 			},
		// 			// More values...
		// 		},
		// 		SecondaryPrivateIPAddressCount: aws.Long(1),
		// 		SubnetID:                       aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// Placement: &ec2.Placement{
		// 	AvailabilityZone: aws.String("String"),
		// 	GroupName:        aws.String("String"),
		// 	Tenancy:          aws.String("Tenancy"),
		// },
		// PrivateIPAddress: aws.String("String"),
		// RAMDiskID:        aws.String("String"),
		SecurityGroupIDs: []*string{
			aws.String(SECURITY_GROUP_ID), // Required
			// More values...
		},
		SubnetID: aws.String(SUBNET_ID),
	}

	instanceOutput, err := c.EC2Client.RunInstances(instanceInput)
	if err != nil {
		return "", err
	}

	fmt.Println(awsutil.StringValue(instanceOutput))
	instanceId, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Instances[0].InstanceID))

	return instanceId, nil
}

func (c *AWSClient) DeleteInstance(instanceId string) error {
	terminateInstanceInput := &ec2.TerminateInstancesInput{
		// One or more instance IDs.
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	_, err := c.EC2Client.TerminateInstances(terminateInstanceInput)
	if err != nil {
		return err
	}

	return nil
}

func (c *AWSClient) RevokeKeyPair(instanceId string, privateKey string) error {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId),
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return err
	}

	ip, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].PublicIPAddress))
	pemBytes, err := utils.ReadFile(path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME))
	if err != nil {
		return err
	}

	awsSShClient, err := utils.GetSshClient(LINUX_USER, pemBytes, ip)
	if err != nil {
		return err
	}

	publicKey, err := utils.GeneratePublicKey([]byte(privateKey))
	if err != nil {
		return err
	}

	escapedPublicKey := strings.Replace(publicKey, "/", "\\/", -1)
	command := fmt.Sprintf("sed '/%s/d' -i ~/.ssh/authorized_keys && echo 'revoked the public key: %s'", escapedPublicKey, publicKey)

	result, err := awsSShClient.ExecCommand(command)
	if err != nil {
		return err
	}
	fmt.Println(result)

	return nil
}
<commit_msg>fix one bug for service parameter<commit_after>package aws_client

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/service/ec2"

	"github.com/xingzhou/go_service_broker/utils"
)

const (
	AMI_ID                = "ami-dc5e75b4" //"ami-ecb68a84"
	SECURITY_GROUP_ID     = "sg-b23aead6"
	SUBNET_ID             = "subnet-0c75a427"
	KEYPAIR_NAME          = "broker_keypair"
	INSTANCE_TYPE         = "t2.micro"
	LINUX_USER            = "ubuntu"
	KEYPAIR_DIR_NAME      = ".gsb"
	PIRVATE_KEY_FILE_NAME = "broker_id_rsa"
	PUBLIC_KEY_FILE_NAME  = "broker_id_rsa.pub"
)

type Client interface {
	CreateInstance(parameters interface{}) (string, error)
	GetInstanceState(instanceId string) (string, error)
	InjectKeyPair(instanceId string) (string, error)
	DeleteInstance(instanceId string) error
	RevokeKeyPair(instanceId string, privateKey string) error
}

type AWSClient struct {
	EC2Client *ec2.EC2
}

func NewClient(region string) *AWSClient {
	return &AWSClient{
		EC2Client: ec2.New(&aws.Config{Region: region}),
	}
}

func (c *AWSClient) CreateInstance(parameters interface{}) (string, error) {
	var amiId string

	switch parameters.(type) {
	case map[string]interface{}:
		param := parameters.(map[string]interface{})
		if param["ami_id"] != nil {
			amiId = param["ami_id"].(string)
		} else {
			amiId = AMI_ID
		}

	default:
		amiId = AMI_ID
	}

	return c.createInstance(amiId)
}

func (c *AWSClient) GetInstanceState(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	state, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].State.Name))
	return state, nil
}

func (c *AWSClient) setupKeyPair() error {
	private_key_file := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME)

	if !utils.Exists(private_key_file) {
		keypairInput := &ec2.CreateKeyPairInput{
			KeyName: aws.String(KEYPAIR_NAME),
		}

		keypairOutput, err := c.EC2Client.CreateKeyPair(keypairInput)
		if err != nil {
			return err
		}

		key_dir := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME)
		if !utils.MkDir(key_dir) {
			return errors.New("failed to create local keypair directory")
		}

		key_data, _ := strconv.Unquote(awsutil.StringValue(keypairOutput.KeyMaterial))
		err = utils.WriteFile(private_key_file, []byte(key_data))
		if err != nil {
			return errors.New("failed to save private key file")
		}
	}

	return nil
}

func (c *AWSClient) InjectKeyPair(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	ip, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].PublicIPAddress))
	pemBytes, err := utils.ReadFile(path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME))
	if err != nil {
		return "", err
	}

	awsSShClient, err := utils.GetSshClient(LINUX_USER, pemBytes, ip)
	if err != nil {
		return "", err
	}

	command := `rm -f ./broker_id_rsa ./broker_id_rsa.pub
		ssh-keygen -q -t rsa -N ""  -f ./broker_id_rsa
		cat ./broker_id_rsa.pub >> .ssh/authorized_keys
		cat ./broker_id_rsa`

	privateKey, err := awsSShClient.ExecCommand(command)
	if err != nil {
		return "", err
	}

	return privateKey, nil
}

func (c *AWSClient) createInstance(imageId string) (string, error) {
	err := c.setupKeyPair()
	if err != nil {
		return "", err
	}

	instanceInput := &ec2.RunInstancesInput{
		ImageID:  aws.String(imageId), // Required
		MaxCount: aws.Long(1),         // Required
		MinCount: aws.Long(1),         // Required
		// AdditionalInfo: aws.String("String"),
		// BlockDeviceMappings: []*ec2.BlockDeviceMapping{
		// 	&ec2.BlockDeviceMapping{ // Required
		// 		DeviceName: aws.String("String"),
		// 		EBS: &ec2.EBSBlockDevice{
		// 			DeleteOnTermination: aws.Boolean(true),
		// 			Encrypted:           aws.Boolean(true),
		// 			IOPS:                aws.Long(1),
		// 			SnapshotID:          aws.String("String"),
		// 			VolumeSize:          aws.Long(1),
		// 			VolumeType:          aws.String("VolumeType"),
		// 		},
		// 		NoDevice:    aws.String("String"),
		// 		VirtualName: aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// ClientToken: aws.String("String"),
		// DisableAPITermination: aws.Boolean(true),
		// DryRun:                aws.Boolean(true),
		// EBSOptimized:          aws.Boolean(true),
		// IAMInstanceProfile: &ec2.IAMInstanceProfileSpecification{
		// 	ARN:  aws.String("String"),
		// 	Name: aws.String("String"),
		// },
		// InstanceInitiatedShutdownBehavior: aws.String("ShutdownBehavior"),
		InstanceType: aws.String(INSTANCE_TYPE),
		// KernelID:                          aws.String("String"),
		KeyName: aws.String(KEYPAIR_NAME),
		// Monitoring: &ec2.RunInstancesMonitoringEnabled{
		// 	Enabled: aws.Boolean(true), // Required
		// },
		// NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
		// 	&ec2.InstanceNetworkInterfaceSpecification{ // Required
		// 		AssociatePublicIPAddress: aws.Boolean(true),
		// 		DeleteOnTermination:      aws.Boolean(true),
		// 		Description:              aws.String("String"),
		// 		DeviceIndex:              aws.Long(1),
		// 		Groups: []*string{
		// 			aws.String("String"), // Required
		// 			// More values...
		// 		},
		// 		NetworkInterfaceID: aws.String("String"),
		// 		PrivateIPAddress:   aws.String("String"),
		// 		PrivateIPAddresses: []*ec2.PrivateIPAddressSpecification{
		// 			&ec2.PrivateIPAddressSpecification{ // Required
		// 				PrivateIPAddress: aws.String("String"), // Required
		// 				Primary:          aws.Boolean(true),
		// 			},
		// 			// More values...
		// 		},
		// 		SecondaryPrivateIPAddressCount: aws.Long(1),
		// 		SubnetID:                       aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// Placement: &ec2.Placement{
		// 	AvailabilityZone: aws.String("String"),
		// 	GroupName:        aws.String("String"),
		// 	Tenancy:          aws.String("Tenancy"),
		// },
		// PrivateIPAddress: aws.String("String"),
		// RAMDiskID:        aws.String("String"),
		SecurityGroupIDs: []*string{
			aws.String(SECURITY_GROUP_ID), // Required
			// More values...
		},
		SubnetID: aws.String(SUBNET_ID),
	}

	instanceOutput, err := c.EC2Client.RunInstances(instanceInput)
	if err != nil {
		return "", err
	}

	fmt.Println(awsutil.StringValue(instanceOutput))
	instanceId, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Instances[0].InstanceID))

	return instanceId, nil
}

func (c *AWSClient) DeleteInstance(instanceId string) error {
	terminateInstanceInput := &ec2.TerminateInstancesInput{
		// One or more instance IDs.
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	_, err := c.EC2Client.TerminateInstances(terminateInstanceInput)
	if err != nil {
		return err
	}

	return nil
}

func (c *AWSClient) RevokeKeyPair(instanceId string, privateKey string) error {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId),
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return err
	}

	ip, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].PublicIPAddress))
	pemBytes, err := utils.ReadFile(path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME))
	if err != nil {
		return err
	}

	awsSShClient, err := utils.GetSshClient(LINUX_USER, pemBytes, ip)
	if err != nil {
		return err
	}

	publicKey, err := utils.GeneratePublicKey([]byte(privateKey))
	if err != nil {
		return err
	}

	escapedPublicKey := strings.Replace(publicKey, "/", "\\/", -1)
	command := fmt.Sprintf("sed '/%s/d' -i ~/.ssh/authorized_keys && echo 'revoked the public key: %s'", escapedPublicKey, publicKey)

	result, err := awsSShClient.ExecCommand(command)
	if err != nil {
		return err
	}
	fmt.Println(result)

	return nil
}
<|endoftext|>


<commit_before>package topic

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"path/filepath"
	"sync/atomic"
	"time"

	"github.com/tylertreat/BoomFilters"

	"github.com/celrenheit/sandflake"

	"github.com/celrenheit/sandglass-grpc/go/sgproto"
	"github.com/celrenheit/sandglass/sgutils"
	"github.com/celrenheit/sandglass/storage"
	"github.com/celrenheit/sandglass/storage/badger"
	"github.com/celrenheit/sandglass/storage/rocksdb"
	"github.com/celrenheit/sandglass/storage/scommons"
	"github.com/gogo/protobuf/proto"
	"github.com/willf/bloom"
)

var (
	ErrNoKeySet = errors.New("ErrNoKeySet")
)

type Partition struct {
	db       storage.Storage
	Id       string
	Replicas []string
	idgen    sandflake.Generator
	basepath string
	topic    *Topic

	bf  *bloom.BloomFilter // TODO: persist bloom filter
	ibf boom.Filter

	lastIndex  *uint64
	pendingKey []byte
}

func (t *Partition) InitStore(basePath string) error {
	t.bf = bloom.NewWithEstimates(1e3, 1e-2)
	t.ibf = boom.NewInverseBloomFilter(1e3)
	msgdir := filepath.Join(basePath, t.Id)
	if err := sgutils.MkdirIfNotExist(msgdir); err != nil {
		return err
	}

	t.pendingKey = scommons.PrependPrefix(scommons.PendingPrefix, []byte{0})

	mo := &storage.MergeOperator{
		Key:       t.pendingKey,
		MergeFunc: mergeFunc,
	}

	var err error
	switch t.topic.StorageDriver {
	case sgproto.StorageDriver_Badger:
		t.db, err = badger.NewStorage(msgdir, mo)
	case sgproto.StorageDriver_RocksDB:
		t.db, err = rocksdb.NewStorage(msgdir, mo)
	default:
		return fmt.Errorf("unknown storage driver: %v for topic: %v", t.topic.StorageDriver, t.topic.Name)
	}
	if err != nil {
		return err
	}

	t.basepath = basePath

	var index uint64
	msg, err := t.LastMessage()
	if err != nil {
		return fmt.Errorf("unable to fetch last message for init partition: %v", err)
	}

	if msg != nil {
		index = msg.Index
	}
	t.lastIndex = &index
	return nil
}

func mergeFunc(existing, value []byte) ([]byte, bool) {
	var operation sgproto.MergeOperation
	err := proto.Unmarshal(value, &operation)
	if err != nil {
		return nil, false
	}

	var state sgproto.MergeState
	err = proto.Unmarshal(existing, &state)
	if err != nil {
		return nil, false
	}

	switch operation.Operation {
	case sgproto.MergeOperation_APPEND:
		state.Messages = append(state.Messages, operation.Messages...)
	case sgproto.MergeOperation_CUT:
		if len(operation.Messages) <= int(operation.N) {
			state.Messages = nil
		} else {
			state.Messages = state.Messages[operation.N:]
		}
	}

	newState, err := proto.Marshal(&state)
	if err != nil {
		return nil, false
	}

	return newState, true
}

func (p *Partition) applyPendingToWal() error {
	fn := func(val []byte) ([]*storage.Entry, []byte, error) {
		var state sgproto.MergeState
		err := proto.Unmarshal(val, &state)
		if err != nil {
			return nil, nil, err
		}

		entries := []*storage.Entry{}
		for _, msg := range state.Messages {
			storagekey := p.getStorageKey(msg)
			val, err := proto.Marshal(msg)
			if err != nil {
				return nil, nil, err
			}
			//    t.bf.Add(storagekey)
			//    t.ibf.Add(storagekey)
			// entries = append(entries, &storage.Entry{
			// 	Key:   storagekey,
			// 	Value: val,
			// })
			entries = append(entries, &storage.Entry{
				Key:   p.newWALKey(storagekey, msg.Index),
				Value: val,
			})
		}

		operation, err := proto.Marshal(&sgproto.MergeOperation{
			Operation: sgproto.MergeOperation_CUT,
			N:         int32(len(state.Messages)),
		})
		if err != nil {
			return nil, nil, err
		}

		return entries, operation, nil
	}

	return p.db.ProcessMergedKey(p.pendingKey, fn)
}

func (p *Partition) WalToView(start, end uint64) error {
	entries := []*storage.Entry{}
	err := p.db.ForRangeWAL(start, end, func(msg *sgproto.Message) error {
		storagekey := p.getStorageKey(msg)

		b, err := proto.Marshal(msg)
		if err != nil {
			return err
		}

		entries = append(entries, &storage.Entry{
			Key:   storagekey,
			Value: b,
		})

		return nil
	})
	if err != nil {
		return err
	}

	return p.db.BatchPut(entries)
}

func (t *Partition) String() string {
	return t.Id
}

func (t *Partition) getStorageKey(msg *sgproto.Message) []byte {
	var storekey []byte
	switch t.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		storekey = msg.Offset[:]
	case sgproto.TopicKind_KVKind:
		storekey = msg.Key
		if len(msg.ClusteringKey) > 0 {
			storekey = joinKeys(msg.Key, msg.ClusteringKey)
		}
	default:
		panic("INVALID STORAGE KIND: " + t.topic.Kind.String())
	}
	return scommons.PrependPrefix(scommons.ViewPrefix, storekey)
}

func (s *Partition) GetMessage(offset sgproto.Offset, k, suffix []byte) (*sgproto.Message, error) {
	switch s.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		val, err := s.db.Get(scommons.PrependPrefix(scommons.ViewPrefix, offset[:]))
		if err != nil {
			return nil, err
		}

		var msg sgproto.Message
		err = proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	case sgproto.TopicKind_KVKind:
		val := s.db.LastKVForPrefix(scommons.PrependPrefix(scommons.ViewPrefix, k), suffix)
		if val == nil {
			return nil, nil
		}

		var msg sgproto.Message
		err := proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	default:
		return nil, fmt.Errorf("invalid storage kind: %s", s.topic.Kind.String())
	}
}

func (t *Partition) HasKey(key, clusterKey []byte) (bool, error) {
	switch t.topic.Kind {
	case sgproto.TopicKind_KVKind:
	default:
		return false, errors.New("HasKey should be used only with a KV topic")
	}

	pk := joinKeys(key, clusterKey)
	existKey := scommons.PrependPrefix(scommons.ViewPrefix, pk)

	if t.ibf.Test(existKey) {
		return true, nil
	}

	if !t.bf.Test(existKey) {
		return false, nil
	}

	msg, err := t.GetMessage(sgproto.Nil, pk, nil)
	if err != nil {
		return false, err
	}

	if msg == nil {
		return false, nil
	}

	return msg.Offset != sgproto.Nil, nil
}

func (t *Partition) newWALKey(prefix []byte, index uint64) []byte {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, index)
	return bytes.Join([][]byte{scommons.WalPrefix, prefix, b}, storage.Separator)
}

func (t *Partition) PutMessage(msg *sgproto.Message) error {
	return t.BatchPutMessages([]*sgproto.Message{msg})
}

func (t *Partition) BatchPutMessages(msgs []*sgproto.Message) error {
	if len(msgs) == 0 {
		return nil
	}

	now := time.Now().UTC()

	var mo sgproto.MergeOperation
	mo.Operation = sgproto.MergeOperation_APPEND

	for _, msg := range msgs {
		if msg.Index == 0 {
			msg.Index = t.NextIndex()
		}
		if msg.Offset == sgproto.Nil {
			msg.Offset = sgproto.NewOffset(msg.Index, now.Add(msg.ConsumeIn))
		}
		msg.ProducedAt = now
	}
	mo.Messages = msgs

	b, err := proto.Marshal(&mo)
	if err != nil {
		return err
	}

	return t.db.Merge(t.pendingKey, b)
}

func (t *Partition) NextIndex() uint64 {
	return atomic.AddUint64(t.lastIndex, 1)
}

func (p *Partition) ForRange(min, max sgproto.Offset, fn func(msg *sgproto.Message) error) error {
	var lastKey []byte
	switch p.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		return p.db.ForRange(min, max, func(msg *sgproto.Message) error {
			err := fn(msg)
			return err
		})
	case sgproto.TopicKind_KVKind:
		it := scommons.NewMessageIterator(p.db, &storage.IterOptions{
			FetchValues: true,
			Reverse:     true,
		})
		defer it.Close()
		for m := it.Rewind(); it.Valid(); m = it.Next() {
			if lastKey == nil || !bytes.Equal(m.Key, lastKey) {
				err := fn(m)
				if err != nil {
					return err
				}
				lastKey = m.Key
			}
		}
	default:
		panic("unknown topic kind: " + p.topic.Kind.String())
	}
	return nil
}

func (p *Partition) Close() error {
	return p.db.Close()
}

func (p *Partition) Iter() storage.MessageIterator {
	return scommons.NewMessageIterator(p.db, &storage.IterOptions{
		FetchValues: true,
		Reverse:     false,
	})
}

func (p *Partition) RangeFromWAL(min []byte, fn func(*sgproto.Message) error) error {
	return p.db.ForEachWALEntry(min, fn)
}

func (p *Partition) LastWALEntry() []byte {
	return p.db.LastKeyForPrefix(scommons.WalPrefix)
}

func (p *Partition) LastMessage() (*sgproto.Message, error) {
	value := p.db.LastKVForPrefix(scommons.WalPrefix, nil)
	if len(value) == 0 {
		return nil, nil
	}

	var msg sgproto.Message
	if err := proto.Unmarshal(value, &msg); err != nil {
		return nil, err
	}

	return &msg, nil
}

func (p *Partition) getMessageByStorageKey(k []byte) (*sgproto.Message, error) {
	b, err := p.db.Get(k)
	if err != nil {
		return nil, err
	}

	if b == nil {
		panic("should not happend, key in wal should always be also in msgs")
	}

	var msg sgproto.Message
	err = proto.Unmarshal(b, &msg)
	if err != nil {
		return nil, err
	}

	return &msg, nil
}

func joinKeys(key, clusterKey []byte) []byte {
	return bytes.Join([][]byte{
		key,
		clusterKey,
	}, storage.Separator)
}
<commit_msg>launch pending to wal loop<commit_after>package topic

import (
	"bytes"
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/tylertreat/BoomFilters"

	"github.com/celrenheit/sandflake"

	"github.com/celrenheit/sandglass-grpc/go/sgproto"
	"github.com/celrenheit/sandglass/sgutils"
	"github.com/celrenheit/sandglass/storage"
	"github.com/celrenheit/sandglass/storage/badger"
	"github.com/celrenheit/sandglass/storage/rocksdb"
	"github.com/celrenheit/sandglass/storage/scommons"
	"github.com/gogo/protobuf/proto"
	"github.com/willf/bloom"
)

var (
	ErrNoKeySet = errors.New("ErrNoKeySet")
)

type Partition struct {
	db       storage.Storage
	Id       string
	Replicas []string
	idgen    sandflake.Generator
	basepath string
	topic    *Topic

	bf  *bloom.BloomFilter // TODO: persist bloom filter
	ibf boom.Filter

	lastIndex  *uint64
	pendingKey []byte

	ctxPending    context.Context
	cancelPending context.CancelFunc
	wg            sync.WaitGroup
}

func (t *Partition) InitStore(basePath string) error {
	t.bf = bloom.NewWithEstimates(1e3, 1e-2)
	t.ibf = boom.NewInverseBloomFilter(1e3)
	msgdir := filepath.Join(basePath, t.Id)
	if err := sgutils.MkdirIfNotExist(msgdir); err != nil {
		return err
	}

	t.pendingKey = scommons.PrependPrefix(scommons.PendingPrefix, []byte{0})

	mo := &storage.MergeOperator{
		Key:       t.pendingKey,
		MergeFunc: mergeFunc,
	}

	var err error
	switch t.topic.StorageDriver {
	case sgproto.StorageDriver_Badger:
		t.db, err = badger.NewStorage(msgdir, mo)
	case sgproto.StorageDriver_RocksDB:
		t.db, err = rocksdb.NewStorage(msgdir, mo)
	default:
		return fmt.Errorf("unknown storage driver: %v for topic: %v", t.topic.StorageDriver, t.topic.Name)
	}
	if err != nil {
		return err
	}

	t.basepath = basePath

	var index uint64
	msg, err := t.LastMessage()
	if err != nil {
		return fmt.Errorf("unable to fetch last message for init partition: %v", err)
	}

	if msg != nil {
		index = msg.Index
	}
	t.lastIndex = &index

	t.LaunchPendingLoop()
	return nil
}

func mergeFunc(existing, value []byte) ([]byte, bool) {
	var operation sgproto.MergeOperation
	err := proto.Unmarshal(value, &operation)
	if err != nil {
		return nil, false
	}

	var state sgproto.MergeState
	err = proto.Unmarshal(existing, &state)
	if err != nil {
		return nil, false
	}

	switch operation.Operation {
	case sgproto.MergeOperation_APPEND:
		state.Messages = append(state.Messages, operation.Messages...)
	case sgproto.MergeOperation_CUT:
		if len(operation.Messages) <= int(operation.N) {
			state.Messages = nil
		} else {
			state.Messages = state.Messages[operation.N:]
		}
	}

	newState, err := proto.Marshal(&state)
	if err != nil {
		return nil, false
	}

	return newState, true
}

func (p *Partition) LaunchPendingLoop() {
	p.ctxPending, p.cancelPending = context.WithCancel(context.Background())
	p.wg.Add(1)
	go func() {
		defer p.wg.Done()
		for {
			select {
			case <-p.ctxPending.Done():
				return
			case <-time.After(100 * time.Millisecond):
				if err := p.applyPendingToWal(); err != nil {
					logrus.WithError(err).Printf("apply pending loop")
				}
			}
		}
	}()
}

func (p *Partition) applyPendingToWal() error {
	fn := func(val []byte) ([]*storage.Entry, []byte, error) {
		var state sgproto.MergeState
		err := proto.Unmarshal(val, &state)
		if err != nil {
			return nil, nil, err
		}

		entries := []*storage.Entry{}
		for _, msg := range state.Messages {
			storagekey := p.getStorageKey(msg)
			val, err := proto.Marshal(msg)
			if err != nil {
				return nil, nil, err
			}
			//    t.bf.Add(storagekey)
			//    t.ibf.Add(storagekey)
			// entries = append(entries, &storage.Entry{
			// 	Key:   storagekey,
			// 	Value: val,
			// })
			entries = append(entries, &storage.Entry{
				Key:   p.newWALKey(storagekey, msg.Index),
				Value: val,
			})
		}

		operation, err := proto.Marshal(&sgproto.MergeOperation{
			Operation: sgproto.MergeOperation_CUT,
			N:         int32(len(state.Messages)),
		})
		if err != nil {
			return nil, nil, err
		}

		return entries, operation, nil
	}

	return p.db.ProcessMergedKey(p.pendingKey, fn)
}

func (p *Partition) WalToView(start, end uint64) error {
	entries := []*storage.Entry{}
	err := p.db.ForRangeWAL(start, end, func(msg *sgproto.Message) error {
		storagekey := p.getStorageKey(msg)

		b, err := proto.Marshal(msg)
		if err != nil {
			return err
		}

		entries = append(entries, &storage.Entry{
			Key:   storagekey,
			Value: b,
		})

		return nil
	})
	if err != nil {
		return err
	}

	return p.db.BatchPut(entries)
}

func (t *Partition) String() string {
	return t.Id
}

func (t *Partition) getStorageKey(msg *sgproto.Message) []byte {
	var storekey []byte
	switch t.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		storekey = msg.Offset[:]
	case sgproto.TopicKind_KVKind:
		storekey = msg.Key
		if len(msg.ClusteringKey) > 0 {
			storekey = joinKeys(msg.Key, msg.ClusteringKey)
		}
	default:
		panic("INVALID STORAGE KIND: " + t.topic.Kind.String())
	}
	return scommons.PrependPrefix(scommons.ViewPrefix, storekey)
}

func (s *Partition) GetMessage(offset sgproto.Offset, k, suffix []byte) (*sgproto.Message, error) {
	switch s.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		val, err := s.db.Get(scommons.PrependPrefix(scommons.ViewPrefix, offset[:]))
		if err != nil {
			return nil, err
		}

		var msg sgproto.Message
		err = proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	case sgproto.TopicKind_KVKind:
		val := s.db.LastKVForPrefix(scommons.PrependPrefix(scommons.ViewPrefix, k), suffix)
		if val == nil {
			return nil, nil
		}

		var msg sgproto.Message
		err := proto.Unmarshal(val, &msg)
		if err != nil {
			return nil, err
		}

		return &msg, nil
	default:
		return nil, fmt.Errorf("invalid storage kind: %s", s.topic.Kind.String())
	}
}

func (t *Partition) HasKey(key, clusterKey []byte) (bool, error) {
	switch t.topic.Kind {
	case sgproto.TopicKind_KVKind:
	default:
		return false, errors.New("HasKey should be used only with a KV topic")
	}

	pk := joinKeys(key, clusterKey)
	existKey := scommons.PrependPrefix(scommons.ViewPrefix, pk)

	if t.ibf.Test(existKey) {
		return true, nil
	}

	if !t.bf.Test(existKey) {
		return false, nil
	}

	msg, err := t.GetMessage(sgproto.Nil, pk, nil)
	if err != nil {
		return false, err
	}

	if msg == nil {
		return false, nil
	}

	return msg.Offset != sgproto.Nil, nil
}

func (t *Partition) newWALKey(prefix []byte, index uint64) []byte {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, index)
	return bytes.Join([][]byte{scommons.WalPrefix, prefix, b}, storage.Separator)
}

func (t *Partition) PutMessage(msg *sgproto.Message) error {
	return t.BatchPutMessages([]*sgproto.Message{msg})
}

func (t *Partition) BatchPutMessages(msgs []*sgproto.Message) error {
	if len(msgs) == 0 {
		return nil
	}

	now := time.Now().UTC()

	var mo sgproto.MergeOperation
	mo.Operation = sgproto.MergeOperation_APPEND

	for _, msg := range msgs {
		if msg.Index == 0 {
			msg.Index = t.NextIndex()
		}
		if msg.Offset == sgproto.Nil {
			msg.Offset = sgproto.NewOffset(msg.Index, now.Add(msg.ConsumeIn))
		}
		msg.ProducedAt = now
	}
	mo.Messages = msgs

	b, err := proto.Marshal(&mo)
	if err != nil {
		return err
	}

	return t.db.Merge(t.pendingKey, b)
}

func (t *Partition) NextIndex() uint64 {
	return atomic.AddUint64(t.lastIndex, 1)
}

func (p *Partition) ForRange(min, max sgproto.Offset, fn func(msg *sgproto.Message) error) error {
	var lastKey []byte
	switch p.topic.Kind {
	case sgproto.TopicKind_TimerKind:
		return p.db.ForRange(min, max, func(msg *sgproto.Message) error {
			err := fn(msg)
			return err
		})
	case sgproto.TopicKind_KVKind:
		it := scommons.NewMessageIterator(p.db, &storage.IterOptions{
			FetchValues: true,
			Reverse:     true,
		})
		defer it.Close()
		for m := it.Rewind(); it.Valid(); m = it.Next() {
			if lastKey == nil || !bytes.Equal(m.Key, lastKey) {
				err := fn(m)
				if err != nil {
					return err
				}
				lastKey = m.Key
			}
		}
	default:
		panic("unknown topic kind: " + p.topic.Kind.String())
	}
	return nil
}

func (p *Partition) Close() error {
	if p.cancelPending != nil {
		p.cancelPending()
	}
	p.wg.Wait()
	return p.db.Close()
}

func (p *Partition) Iter() storage.MessageIterator {
	return scommons.NewMessageIterator(p.db, &storage.IterOptions{
		FetchValues: true,
		Reverse:     false,
	})
}

func (p *Partition) RangeFromWAL(min []byte, fn func(*sgproto.Message) error) error {
	return p.db.ForEachWALEntry(min, fn)
}

func (p *Partition) LastWALEntry() []byte {
	return p.db.LastKeyForPrefix(scommons.WalPrefix)
}

func (p *Partition) LastMessage() (*sgproto.Message, error) {
	value := p.db.LastKVForPrefix(scommons.WalPrefix, nil)
	if len(value) == 0 {
		return nil, nil
	}

	var msg sgproto.Message
	if err := proto.Unmarshal(value, &msg); err != nil {
		return nil, err
	}

	return &msg, nil
}

func (p *Partition) getMessageByStorageKey(k []byte) (*sgproto.Message, error) {
	b, err := p.db.Get(k)
	if err != nil {
		return nil, err
	}

	if b == nil {
		panic("should not happend, key in wal should always be also in msgs")
	}

	var msg sgproto.Message
	err = proto.Unmarshal(b, &msg)
	if err != nil {
		return nil, err
	}

	return &msg, nil
}

func joinKeys(key, clusterKey []byte) []byte {
	return bytes.Join([][]byte{
		key,
		clusterKey,
	}, storage.Separator)
}
<|endoftext|>


<commit_before>package mail

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"
	"text/template"
)

var (
	boundaryRe = regexp.MustCompile("Gondola\\-Boundary\\-\\w+")
	tmpl       = template.Must(template.New("tmpl").Parse("{{ .foo }}"))
)

func testCredentials(t *testing.T, addr, server, username, password string, cram bool) {
	cr, user, passwd, host := parseServer(addr)
	if cr != cram || server != host || user != username || password != passwd {
		t.Errorf("Expecting %v, %v, %v, %v, got %v, %v, %v, %v",
			server, username, password, cram, host, user, passwd, cr)
	}
}

func TestCredentials(t *testing.T) {
	testCredentials(t, "smtp.example.com", "smtp.example.com", "", "", false)
	testCredentials(t, "pepe:lotas@smtp.example.com", "smtp.example.com", "pepe", "lotas", false)
	testCredentials(t, "cram?pepe:lotas@smtp.example.com", "smtp.example.com", "pepe", "lotas", true)
	testCredentials(t, "invalid?pepe:lotas@smtp.example.com", "smtp.example.com", "invalid?pepe", "lotas", false)
	testCredentials(t, "pepe@lotas.com:mayonesa@smtp.example.com", "smtp.example.com", "pepe@lotas.com", "mayonesa", false)
}

type Validation struct {
	Address    string
	Email      string
	UseNetwork bool
	Valid      bool
}

func TestValidation(t *testing.T) {
	cases := []Validation{
		{"pepe  @gmail.com", "", true, false},
		{"pepe@lotas@gmail.com", "", true, false},
		{"pepe", "", true, false},
		{"pepe@", "", true, false},
		{"@gmail.com", "", true, false},
		{"pepe@gmail.com", "", true, true},
		{"Pepe <pepe@gmail.com>", "pepe@gmail.com", true, true},
		{"fiam@abra.rm-fr.net", "", true, true},
		{"pepe@gmaildoesnotexistwolololhopefullynooneregistersthisdomainandbreaksthistest.com", "", false, true},
		{"pepe@gmaildoesnotexistwolololhopefullynooneregistersthisdomainandbreaksthistest.com", "", true, false},
	}
	for _, v := range cases {
		email, err := Validate(v.Address, v.UseNetwork)
		t.Logf("Validated address %q (net %v), error: %v", v.Address, v.UseNetwork, err)
		valid := err == nil
		if valid != v.Valid {
			e := "valid"
			if valid {
				e = "invalid"
			}
			t.Errorf("Error validating %q (net %v), expecting %s address", v.Address, v.UseNetwork, e)
		}
		if v.Email != "" && v.Email != email {
			t.Errorf("invalid email %q from %q, want %q", email, v.Address, v.Email)
		}
	}
}

type EmailTest struct {
	Message *Message
	Expect  string
}

func makeAttachments(file string, contentId string) []*Attachment {
	f, err := os.Open(filepath.Join("testdata", file))
	if err != nil {
		panic(err)
	}
	att, err := NewAttachment(file, f)
	if err != nil {
		panic(err)
	}
	att.ContentID = contentId
	return []*Attachment{att}
}

var (
	sendTests = []*Message{
		&Message{
			TextBody: "foo",
		},
		&Message{
			TextBody:    "foo",
			Attachments: makeAttachments("lenna.jpg", ""),
		},
		&Message{
			TextBody:    "This is lenna",
			HTMLBody:    "THIS IS LENNA",
			Attachments: makeAttachments("lenna.jpg", ""),
		},
		&Message{
			TextBody:    "This is lenna",
			HTMLBody:    "LENNA

EMBEDDED",
			Attachments: makeAttachments("lenna.jpg", "LENNA"),
		},
	}
)

func replaceBoundary(s string) string {
	return boundaryRe.ReplaceAllString(s, "Gondola-Boundary-A")
}

func TestSendEmail(t *testing.T) {
	p := printer
	defer func() {
		printer = p
	}()
	var res string
	count := 0
	printer = func(format string, args ...interface{}) (int, error) {
		res = fmt.Sprintf(format, args...)
		// This is useful when adding new tests
		ioutil.WriteFile(filepath.Join("testdata", fmt.Sprintf("out.%d.eml", count)), []byte(res), 0644)
		count++
		return len(res), nil
	}
	for ii, v := range sendTests {
		if v.To == nil {
			v.To = []string{"receiver@example.com"}
		}
		if v.From == "" {
			v.From = "sender@example.com"
		}
		v.Server = "echo"
		if err := Send(v); err != nil {
			t.Error(err)
			continue
		}
		// boundary is random, so we need to change it
		res = replaceBoundary(res)
		path := filepath.Join("testdata", fmt.Sprintf("expect.%d.eml", ii))
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Error(err)
			continue
		}
		if res != string(data) {
			t.Errorf("message %v expecting email %q, got %q instead", v, string(data), res)
		}
	}
}
<commit_msg>Replace the boundary when writing test output files<commit_after>package mail

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"
	"text/template"
)

var (
	boundaryRe = regexp.MustCompile("Gondola\\-Boundary\\-\\w+")
	tmpl       = template.Must(template.New("tmpl").Parse("{{ .foo }}"))
)

func testCredentials(t *testing.T, addr, server, username, password string, cram bool) {
	cr, user, passwd, host := parseServer(addr)
	if cr != cram || server != host || user != username || password != passwd {
		t.Errorf("Expecting %v, %v, %v, %v, got %v, %v, %v, %v",
			server, username, password, cram, host, user, passwd, cr)
	}
}

func TestCredentials(t *testing.T) {
	testCredentials(t, "smtp.example.com", "smtp.example.com", "", "", false)
	testCredentials(t, "pepe:lotas@smtp.example.com", "smtp.example.com", "pepe", "lotas", false)
	testCredentials(t, "cram?pepe:lotas@smtp.example.com", "smtp.example.com", "pepe", "lotas", true)
	testCredentials(t, "invalid?pepe:lotas@smtp.example.com", "smtp.example.com", "invalid?pepe", "lotas", false)
	testCredentials(t, "pepe@lotas.com:mayonesa@smtp.example.com", "smtp.example.com", "pepe@lotas.com", "mayonesa", false)
}

type Validation struct {
	Address    string
	Email      string
	UseNetwork bool
	Valid      bool
}

func TestValidation(t *testing.T) {
	cases := []Validation{
		{"pepe  @gmail.com", "", true, false},
		{"pepe@lotas@gmail.com", "", true, false},
		{"pepe", "", true, false},
		{"pepe@", "", true, false},
		{"@gmail.com", "", true, false},
		{"pepe@gmail.com", "", true, true},
		{"Pepe <pepe@gmail.com>", "pepe@gmail.com", true, true},
		{"fiam@abra.rm-fr.net", "", true, true},
		{"pepe@gmaildoesnotexistwolololhopefullynooneregistersthisdomainandbreaksthistest.com", "", false, true},
		{"pepe@gmaildoesnotexistwolololhopefullynooneregistersthisdomainandbreaksthistest.com", "", true, false},
	}
	for _, v := range cases {
		email, err := Validate(v.Address, v.UseNetwork)
		t.Logf("Validated address %q (net %v), error: %v", v.Address, v.UseNetwork, err)
		valid := err == nil
		if valid != v.Valid {
			e := "valid"
			if valid {
				e = "invalid"
			}
			t.Errorf("Error validating %q (net %v), expecting %s address", v.Address, v.UseNetwork, e)
		}
		if v.Email != "" && v.Email != email {
			t.Errorf("invalid email %q from %q, want %q", email, v.Address, v.Email)
		}
	}
}

type EmailTest struct {
	Message *Message
	Expect  string
}

func makeAttachments(file string, contentId string) []*Attachment {
	f, err := os.Open(filepath.Join("testdata", file))
	if err != nil {
		panic(err)
	}
	att, err := NewAttachment(file, f)
	if err != nil {
		panic(err)
	}
	att.ContentID = contentId
	return []*Attachment{att}
}

var (
	sendTests = []*Message{
		&Message{
			TextBody: "foo",
		},
		&Message{
			TextBody:    "foo",
			Attachments: makeAttachments("lenna.jpg", ""),
		},
		&Message{
			TextBody:    "This is lenna",
			HTMLBody:    "THIS IS LENNA",
			Attachments: makeAttachments("lenna.jpg", ""),
		},
		&Message{
			TextBody:    "This is lenna",
			HTMLBody:    "LENNA

EMBEDDED",
			Attachments: makeAttachments("lenna.jpg", "LENNA"),
		},
	}
)

func replaceBoundary(s string) string {
	return boundaryRe.ReplaceAllString(s, "Gondola-Boundary-A")
}

func TestSendEmail(t *testing.T) {
	p := printer
	defer func() {
		printer = p
	}()
	var res string
	count := 0
	printer = func(format string, args ...interface{}) (int, error) {
		res = fmt.Sprintf(format, args...)
		res = replaceBoundary(res)
		// This is useful when adding new tests
		ioutil.WriteFile(filepath.Join("testdata", fmt.Sprintf("out.%d.eml", count)), []byte(res), 0644)
		count++
		return len(res), nil
	}
	for ii, v := range sendTests {
		if v.To == nil {
			v.To = []string{"receiver@example.com"}
		}
		if v.From == "" {
			v.From = "sender@example.com"
		}
		v.Server = "echo"
		if err := Send(v); err != nil {
			t.Error(err)
			continue
		}
		// boundary is random, so we need to change it
		res = replaceBoundary(res)
		path := filepath.Join("testdata", fmt.Sprintf("expect.%d.eml", ii))
		data, err := ioutil.ReadFile(path)
		if err != nil {
			t.Error(err)
			continue
		}
		if res != string(data) {
			t.Errorf("message %v expecting email %q, got %q instead", v, string(data), res)
		}
	}
}
<|endoftext|>


<commit_before>// Package azurestoragecache provides an implementation of httpcache.Cache that
// stores and retrieves data using Azure Storage.
package azurestoragecache // import "github.com/PaulARoy/azurestoragecache"

import (
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"bytes"

	vendorstorage "github.com/Azure/azure-sdk-for-go/storage"
)

// Cache objects store and retrieve data using Azure Storage
type Cache struct {
	// Our configuration for Azure Storage
	Config Config

	// The Azure Blob Storage Client
	Client vendorstorage.BlobStorageClient
}

type Config struct {
	// Account configuration for Azure Storage
	AccountName string
	AccountKey string

	// Container name to use to store blob
	ContainerName string
}

var noLogErrors, _ = strconv.ParseBool(os.Getenv("NO_LOG_AZUREBSCACHE_ERRORS"))

func (c *Cache) Get(key string) (resp []byte, ok bool) {
	rdr, err := c.Client.GetBlob(c.Config.ContainerName, key)
	if err != nil {
		return []byte{}, false
	}
	rdr.Close()

	resp, err = ioutil.ReadAll(rdr)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Get failed: %s", err)
		}
	}
	return resp, err == nil
}

func (c *Cache) Set(key string, block []byte) {
	err := c.Client.CreateBlockBlobFromReader(c.Config.ContainerName,
												key,
												uint64(len(block)),
												bytes.NewReader(block),
												nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Set failed: %s", err)
		}
		return
	}
}

func (c *Cache) Delete(key string) {
	res, err := c.Client.DeleteBlobIfExists(c.Config.ContainerName, key, nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Delete failed: %s", err)
		}
		return
	}
}

// New returns a new Cache with underlying client for Azure Storage
//
// containerName is the container name for azure blob service
//
// The environment variables AZURESTORAGE_ACCOUNT_NAME and AZURESTORAGE_ACCESS_KEY
// are used as credentials. To use different credentials, construct a Cache object
// manually.
func New(accountName string, accountKey string, containerName string) *Cache {
	cache := Cache{
		Config: Config{
			AccountName: accountName, // || os.Getenv("AZURESTORAGE_ACCOUNT_NAME"),
			AccountKey: accountKey, // || os.Getenv("AZURESTORAGE_ACCESS_KEY"),
			ContainerName: containerName,
		},
	}

	api, err := vendorstorage.NewBasicClient(cache.Config.AccountName, cache.Config.AccountKey)
	cache.Client = api.GetBlobService()
	cache.Client.CreateContainerIfNotExists(cache.Config.ContainerName,
											vendorstorage.ContainerAccessTypeBlob)
	return &cache
}<commit_msg>avoid unused variables<commit_after>// Package azurestoragecache provides an implementation of httpcache.Cache that
// stores and retrieves data using Azure Storage.
package azurestoragecache // import "github.com/PaulARoy/azurestoragecache"

import (
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"bytes"

	vendorstorage "github.com/Azure/azure-sdk-for-go/storage"
)

// Cache objects store and retrieve data using Azure Storage
type Cache struct {
	// Our configuration for Azure Storage
	Config Config

	// The Azure Blob Storage Client
	Client vendorstorage.BlobStorageClient
}

type Config struct {
	// Account configuration for Azure Storage
	AccountName string
	AccountKey string

	// Container name to use to store blob
	ContainerName string
}

var noLogErrors, _ = strconv.ParseBool(os.Getenv("NO_LOG_AZUREBSCACHE_ERRORS"))

func (c *Cache) Get(key string) (resp []byte, ok bool) {
	rdr, err := c.Client.GetBlob(c.Config.ContainerName, key)
	if err != nil {
		return []byte{}, false
	}
	rdr.Close()

	resp, err = ioutil.ReadAll(rdr)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Get failed: %s", err)
		}
	}
	return resp, err == nil
}

func (c *Cache) Set(key string, block []byte) {
	err := c.Client.CreateBlockBlobFromReader(c.Config.ContainerName,
												key,
												uint64(len(block)),
												bytes.NewReader(block),
												nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Set failed: %s", err)
		}
		return
	}
}

func (c *Cache) Delete(key string) bool {
	res, err := c.Client.DeleteBlobIfExists(c.Config.ContainerName, key, nil)
	if err != nil {
		if !noLogErrors {
			log.Printf("azurestoragecache.Delete failed: %s", err)
		}
		return false
	}
	return res
}

// New returns a new Cache with underlying client for Azure Storage
//
// containerName is the container name for azure blob service
//
// The environment variables AZURESTORAGE_ACCOUNT_NAME and AZURESTORAGE_ACCESS_KEY
// are used as credentials. To use different credentials, construct a Cache object
// manually.
func New(accountName string, accountKey string, containerName string) *Cache {
	cache := Cache{
		Config: Config{
			AccountName: accountName, // || os.Getenv("AZURESTORAGE_ACCOUNT_NAME"),
			AccountKey: accountKey, // || os.Getenv("AZURESTORAGE_ACCESS_KEY"),
			ContainerName: containerName,
		},
	}

	api, err := vendorstorage.NewBasicClient(cache.Config.AccountName, cache.Config.AccountKey)
	if err != nil {
		return nil
	}

	cache.Client = api.GetBlobService()
	cache.Client.CreateContainerIfNotExists(cache.Config.ContainerName,
											vendorstorage.ContainerAccessTypeBlob)
	return &cache
}<|endoftext|>


<commit_before>// Copyright (C) 2017. See AUTHORS.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !openssl_static

package openssl

// #cgo linux windows pkg-config: libssl libcrypto
// #cgo linux CFLAGS: -Wno-deprecated-declarations
// #cgo darwin CFLAGS: -I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/openssl/include -Wno-deprecated-declarations
// #cgo darwin LDFLAGS: -w -L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/openssl/lib -lssl -lcrypto
// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN
import "C"
<commit_msg>remove unsupported build flags on go1.9.4+<commit_after>// Copyright (C) 2017. See AUTHORS.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !openssl_static

package openssl

// #cgo linux windows pkg-config: libssl libcrypto
// #cgo darwin CFLAGS: -I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/openssl/include
// #cgo darwin LDFLAGS: -L/usr/local/opt/openssl@1.1/lib -L/usr/local/opt/openssl/lib -lssl -lcrypto
// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN
import "C"
<|endoftext|>