<commit_before>// Package sp provides tools for buildin an SP such as serving metadata,
// authenticating an assertion and building assertions for IdPs.
package saml

import (
	"bytes"
	"compress/flate"
	"context"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/goware/saml/xmlsec"
	"github.com/pkg/errors"
)

var idAttrs = []string{
	"urn:oasis:names:tc:SAML:2.0:protocol:Response",
	"urn:oasis:names:tc:SAML:2.0:assertion:Assertion",
}

func parseFormAndKeepBody(r *http.Request) error {
	var buf bytes.Buffer

	// Fill buf while reading r.Body
	r.Body = ioutil.NopCloser(io.TeeReader(r.Body, &buf))

	// ParseForm reads all data from r.Body and empties it because it's a buffer.
	if err := r.ParseForm(); err != nil {
		return err
	}

	// Restore body so it can be read again.
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// RequestAuthHandler creates an authentication assert and makes the user send it
// to the IdP (via redirection).
func (sp *ServiceProvider) RequestAuthHandler(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	destination, err := sp.GetIdPAuthResource()
	if err != nil {
		internalErr(w, errors.Errorf("GetIdPAuthResource: %v", err))
		return
	}

	authnRequest, err := sp.MakeAuthenticationRequest(destination)
	if err != nil {
		internalErr(w, errors.Errorf("Failed to make auth request to %v: %v", destination, err))
		return
	}

	buf, err := xml.MarshalIndent(authnRequest, "", "\t")
	if err != nil {
		internalErr(w, errors.Errorf("Failed to marshal auth request %v", err))
		return
	}

	// RelayState is an opaque string that can be used to keep track of this
	// session on our side.
	var relayState string
	token := ctx.Value("saml.RelayState")
	if token != nil {
		relayState, _ = token.(string)
	}

	fbuf := bytes.NewBuffer(nil)
	fwri, err := flate.NewWriter(fbuf, flate.DefaultCompression)
	if err != nil {
		internalErr(w, errors.Errorf("Failed to build buffer %v", err))
		return
	}

	_, err = fwri.Write(buf)
	if err != nil {
		internalErr(w, errors.Errorf("Failed to write to buffer %v", err))
		return
	}
	fwri.Close()
	message := base64.StdEncoding.EncodeToString(fbuf.Bytes())

	redirectURL := destination + fmt.Sprintf(`?RelayState=%s&SAMLRequest=%s`, url.QueryEscape(relayState), url.QueryEscape(message))

	w.Header().Add("Location", redirectURL)
	w.WriteHeader(http.StatusFound)
	return
}

// MetadataHandler creates and serves a metadata XML file.
func (sp *ServiceProvider) MetadataHandler(w http.ResponseWriter, r *http.Request) {
	metadata, err := sp.Metadata()
	if err != nil {
		internalErr(w, errors.Errorf("Failed to build metadata %v", err))
		return
	}
	out, err := xml.MarshalIndent(metadata, "", "\t")
	if err != nil {
		internalErr(w, errors.Errorf("Failed to format metadata %v", err))
		return
	}
	w.Header().Set("Content-Type", "application/xml; charset=utf8")
	w.Write([]byte("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"))
	w.Write(out)
}

func (sp *ServiceProvider) possibleResponseIDs() []string {
	responseIDs := []string{}
	if sp.AllowIdpInitiated {
		responseIDs = append(responseIDs, "")
	}
	return responseIDs
}

func (sp *ServiceProvider) verifySignature(plaintextMessage []byte) error {
	idpCertFile, err := sp.GetIdPCertFile()
	if err != nil {
		return err
	}

	var validationErr error
	for _, idAttr := range idAttrs {
		err := xmlsec.Verify(plaintextMessage, idpCertFile, idAttr)
		if err == nil {
			// No error, this message is OK
			return nil
		}

		// We got an error...
		if !IsSecurityException(err, &sp.SecurityOpts) {
			// ...but it was not a security exception, so we ignore it and accept
			// the verification.
			return nil
		}

		// We had an error, let's try with the next ID.
		validationErr = err
	}
	if validationErr != nil {
		return validationErr
	}

	return errors.New("could not find validation node ID")
}

// AssertionConsumer creates an HTTP handler that can be used to authenticate and
// validate an assertion. If the assertion is valid the flow it passed to the
// given grantFn function.
func (sp *ServiceProvider) AssertionConsumer(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		now := Now()

		if err := parseFormAndKeepBody(r); err != nil {
			clientErr(w, r, errors.Wrap(err, "Unable to read POST data"))
		}

		samlResponse := r.Form.Get("SAMLResponse")

		// This RelayState (if any) needs to be been validated by the invoker.
		relayState := r.Form.Get("RelayState")

		// TODO: Remove this when we're stable enough.
		Logf("SAMLResponse -> %v", samlResponse)
		Logf("relayState -> %v", relayState)

		_ = relayState // Don't know what to do with this yet.

		samlResponseXML, err := base64.StdEncoding.DecodeString(samlResponse)
		if err != nil {
			err = errors.Wrapf(err, "could not decode base64 payload: %s", samlResponse)
			clientErr(w, r, errors.Wrap(err, "Malformed payload"))
			return
		}

		Logf("SAMLResponse (XML) -> %v", string(samlResponseXML))

		var res Response
		err = xml.Unmarshal(samlResponseXML, &res)
		if err != nil {
			err = errors.Wrapf(err, "could not unmarshal XML document: %s", string(samlResponseXML))
			clientErr(w, r, errors.Wrap(err, "Malformed XML"))
			return
		}

		_, err = sp.GetIdPMetadata()
		if err != nil {
			clientErr(w, r, errors.Wrap(err, "unable to retrieve IdP metadata"))
			return
		}

		// Validate message.

		if res.Destination != sp.AcsURL {
			// Note: OneLogin triggers this error when the Recipient field
			// is left blank (or when not set to the correct ACS endpoint)
			// in the OneLogin SAML configuration page. OneLogin returns
			// Destination="{recipient}" in the SAML reponse in this case.
			err := errors.Errorf("Wrong ACS destination, expecting %q, got %q", sp.AcsURL, res.Destination)
			clientErr(w, r, errors.Wrap(err, "Wrong ACS destination"))
			return
		}

		if sp.IdPMetadata.EntityID != "" {
			if res.Issuer.Value != sp.IdPMetadata.EntityID {
				err := errors.Errorf("Issuer %q does not match expected entity ID %q", res.Issuer.Value, sp.IdPMetadata.EntityID)
				clientErr(w, r, errors.Wrap(err, "Issuer does not match expected entity ID"))
				return
			}
		}

		if res.Status.StatusCode.Value != "urn:oasis:names:tc:SAML:2.0:status:Success" {
			err := errors.Errorf("Unexpected status code: %v", res.Status.StatusCode.Value)
			clientErr(w, r, errors.Wrap(err, "Unexpected status code"))
			return
		}

		expectedResponse := false
		responseIDs := sp.possibleResponseIDs()
		for i := range responseIDs {
			if responseIDs[i] == res.InResponseTo {
				expectedResponse = true
			}
		}
		if len(responseIDs) == 1 && responseIDs[0] == "" {
			expectedResponse = true
		}
		if !expectedResponse && len(responseIDs) > 0 {
			err := errors.Errorf("Expecting a proper InResponseTo value, got %#v", responseIDs)
			clientErr(w, r, err)
			return
		}

		// Try getting the IdP's cert file before using it.
		_, err = sp.GetIdPCertFile()
		if err != nil {
			internalErr(w, errors.Errorf("Failed to get private key: %v", err))
			return
		}

		// Validating message.
		signatureOK := false

		if res.Signature != nil {
			err := sp.verifySignature(samlResponseXML)
			if err != nil {
				clientErr(w, r, errors.Wrapf(err, "Unable to verify message signature"))
				return
			} else {
				signatureOK = true
			}
		}

		// Retrieve assertion
		var assertion *Assertion

		if res.EncryptedAssertion != nil {
			keyFile, err := sp.PrivkeyFile()
			if err != nil {
				internalErr(w, errors.Errorf("Failed to get private key: %v", err))
				return
			}

			plainTextAssertion, err := xmlsec.Decrypt(res.EncryptedAssertion.EncryptedData, keyFile)
			if err != nil {
				if IsSecurityException(err, &sp.SecurityOpts) {
					clientErr(w, r, errors.Wrap(err, "Unable to decrypt message"))
					return
				}
			}

			assertion = &Assertion{}
			if err := xml.Unmarshal(plainTextAssertion, assertion); err != nil {
				clientErr(w, r, errors.Wrap(err, "Unable to parse assertion"))
				return
			}

			if assertion.Signature != nil {
				err := sp.verifySignature(plainTextAssertion)
				if err != nil {
					clientErr(w, r, errors.Wrapf(err, "Unable to verify assertion signature"))
					return
				} else {
					signatureOK = true
				}
			}
		} else {
			assertion = res.Assertion
		}
		if assertion == nil {
			clientErr(w, r, errors.New("Missing assertion"))
			return
		}

		// Did we receive a signature?
		if !signatureOK {
			clientErr(w, r, errors.New("Unable to validate signature: node not found"))
			return
		}

		// Validate assertion.
		if sp.IdPMetadata.EntityID != "" {
			if assertion.Issuer.Value != sp.IdPMetadata.EntityID {
				err := errors.Errorf("Assertion issuer %q does not match expected entity ID %q", assertion.Issuer.Value, sp.IdPMetadata.EntityID)
				clientErr(w, r, errors.Wrap(err, "Assertion issuer does not match expected entity ID"))
				return
			}
		}

		if assertion.Subject.SubjectConfirmation.SubjectConfirmationData.Recipient != sp.AcsURL {
			err := errors.Errorf("Unexpected assertion recipient, expecting %q, got %q", sp.AcsURL, assertion.Subject.SubjectConfirmation.SubjectConfirmationData.Recipient)
			clientErr(w, r, errors.Wrapf(err, "Unexpected assertion recipient"))
			return
		}

		// The NotBefore and NotOnOrAfter attributes specify time limits on the
		// validity of the assertion within the context of its profile(s) of use.
		// They do not guarantee that the statements in the assertion will be
		// correct or accurate throughout the validity period. The NotBefore
		// attribute specifies the time instant at which the validity interval
		// begins. The NotOnOrAfter attribute specifies the time instant at which
		// the validity interval has ended. If the value for either NotBefore or
		// NotOnOrAfter is omitted, then it is considered unspecified.
		{
			validFrom := assertion.Conditions.NotBefore
			if !validFrom.IsZero() && validFrom.After(now.Add(ClockDriftTolerance)) {
				err := errors.Errorf("Assertion conditions are not valid yet, got %v, current time is %v", validFrom, now)
				clientErr(w, r, errors.Wrap(err, "Assertion conditions are not valid yet"))
				return
			}
		}

		{
			validUntil := assertion.Conditions.NotOnOrAfter
			if !validUntil.IsZero() && validUntil.Before(now.Add(-ClockDriftTolerance)) {
				err := errors.Errorf("Assertion conditions already expired, got %v current time is %v, extra time is %v", validUntil, now, now.Add(-ClockDriftTolerance))
				clientErr(w, r, errors.Wrap(err, "Assertion conditions already expired"))
				return
			}
		}

		// A time instant at which the subject can no longer be confirmed. The time
		// value is encoded in UTC, as described in Section 1.3.3.
		//
		// Note that the time period specified by the optional NotBefore and
		// NotOnOrAfter attributes, if present, SHOULD fall within the overall
		// assertion validity period as specified by the element's NotBefore and
		// NotOnOrAfter attributes. If both attributes are present, the value for
		// NotBefore MUST be less than (earlier than) the value for NotOnOrAfter.

		if validUntil := assertion.Subject.SubjectConfirmation.SubjectConfirmationData.NotOnOrAfter; validUntil.Before(now.Add(-ClockDriftTolerance)) {
			err := errors.Errorf("Assertion conditions already expired, got %v current time is %v", validUntil, now)
			clientErr(w, r, errors.Wrap(err, "Assertion conditions already expired"))
			return
		}

		if assertion.Conditions.AudienceRestriction != nil {
			if assertion.Conditions.AudienceRestriction.Audience.Value != sp.MetadataURL {
				// clientErr(w, fmt.Errorf("Audience restriction mismatch, got %q, expecting %q", assertion.Conditions.AudienceRestriction.Audience.Value, sp.MetadataURL), errors.New("Audience restriction mismatch"))
				// return
			}
		}

		expectedResponse = false
		for i := range responseIDs {
			if responseIDs[i] == assertion.Subject.SubjectConfirmation.SubjectConfirmationData.InResponseTo {
				expectedResponse = true
			}
		}
		if len(responseIDs) == 1 && responseIDs[0] == "" {
			expectedResponse = true
		}

		if !expectedResponse && len(responseIDs) > 0 {
			clientErr(w, r, errors.New("Unexpected assertion InResponseTo value"))
			return
		}

		ctx := context.WithValue(r.Context(), "saml.assertion", assertion)
		next.ServeHTTP(w, r.WithContext(ctx))
	})
}

func publicErrorMessage(err error) string {
	type hasCause interface {
		Cause(err error) error
	}
	// is there any better way to retrieve the error _message_ without the cause?
	if _, ok := err.(hasCause); ok {
		msg := err.Error()
		parts := strings.SplitN(msg, ":", 2)
		if len(parts) > 0 {
			return parts[0]
		}
		return msg
	}
	return err.Error()
}

func clientErr(w http.ResponseWriter, r *http.Request, err error) {
	publicError := publicErrorMessage(err)

	report := InspectRequest(r)

	Fatal(errors.Wrapf(err, "failed request: %s", report.String()))

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.WriteHeader(http.StatusBadRequest)
	w.Write([]byte(publicError))
}

func internalErr(w http.ResponseWriter, err error) {
	serverErr(w, errors.Wrap(err, "an internal error ocurred, please try again"))
}

func serverErr(w http.ResponseWriter, err error) {
	Fatal(err)

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.WriteHeader(http.StatusInternalServerError)
	w.Write([]byte(publicErrorMessage(err)))
}
<commit_msg>rename RequestAuthHandler to AuthnRequestHandler<commit_after>// Package sp provides tools for buildin an SP such as serving metadata,
// authenticating an assertion and building assertions for IdPs.
package saml

import (
	"bytes"
	"compress/flate"
	"context"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/goware/saml/xmlsec"
	"github.com/pkg/errors"
)

var idAttrs = []string{
	"urn:oasis:names:tc:SAML:2.0:protocol:Response",
	"urn:oasis:names:tc:SAML:2.0:assertion:Assertion",
}

func parseFormAndKeepBody(r *http.Request) error {
	var buf bytes.Buffer

	// Fill buf while reading r.Body
	r.Body = ioutil.NopCloser(io.TeeReader(r.Body, &buf))

	// ParseForm reads all data from r.Body and empties it because it's a buffer.
	if err := r.ParseForm(); err != nil {
		return err
	}

	// Restore body so it can be read again.
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// AuthnRequestHandler creates an authentication assert and makes the user send it
// to the IdP (via redirection).
func (sp *ServiceProvider) AuthnRequestHandler(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	destination, err := sp.GetIdPAuthResource()
	if err != nil {
		internalErr(w, errors.Errorf("GetIdPAuthResource: %v", err))
		return
	}

	authnRequest, err := sp.MakeAuthenticationRequest(destination)
	if err != nil {
		internalErr(w, errors.Errorf("Failed to make auth request to %v: %v", destination, err))
		return
	}

	buf, err := xml.MarshalIndent(authnRequest, "", "\t")
	if err != nil {
		internalErr(w, errors.Errorf("Failed to marshal auth request %v", err))
		return
	}

	// RelayState is an opaque string that can be used to keep track of this
	// session on our side.
	var relayState string
	token := ctx.Value("saml.RelayState")
	if token != nil {
		relayState, _ = token.(string)
	}

	fbuf := bytes.NewBuffer(nil)
	fwri, err := flate.NewWriter(fbuf, flate.DefaultCompression)
	if err != nil {
		internalErr(w, errors.Errorf("Failed to build buffer %v", err))
		return
	}

	_, err = fwri.Write(buf)
	if err != nil {
		internalErr(w, errors.Errorf("Failed to write to buffer %v", err))
		return
	}
	fwri.Close()
	message := base64.StdEncoding.EncodeToString(fbuf.Bytes())

	redirectURL := destination + fmt.Sprintf(`?RelayState=%s&SAMLRequest=%s`, url.QueryEscape(relayState), url.QueryEscape(message))

	w.Header().Add("Location", redirectURL)
	w.WriteHeader(http.StatusFound)
	return
}

// MetadataHandler creates and serves a metadata XML file.
func (sp *ServiceProvider) MetadataHandler(w http.ResponseWriter, r *http.Request) {
	metadata, err := sp.Metadata()
	if err != nil {
		internalErr(w, errors.Errorf("Failed to build metadata %v", err))
		return
	}
	out, err := xml.MarshalIndent(metadata, "", "\t")
	if err != nil {
		internalErr(w, errors.Errorf("Failed to format metadata %v", err))
		return
	}
	w.Header().Set("Content-Type", "application/xml; charset=utf8")
	w.Write([]byte("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"))
	w.Write(out)
}

func (sp *ServiceProvider) possibleResponseIDs() []string {
	responseIDs := []string{}
	if sp.AllowIdpInitiated {
		responseIDs = append(responseIDs, "")
	}
	return responseIDs
}

func (sp *ServiceProvider) verifySignature(plaintextMessage []byte) error {
	idpCertFile, err := sp.GetIdPCertFile()
	if err != nil {
		return err
	}

	var validationErr error
	for _, idAttr := range idAttrs {
		err := xmlsec.Verify(plaintextMessage, idpCertFile, idAttr)
		if err == nil {
			// No error, this message is OK
			return nil
		}

		// We got an error...
		if !IsSecurityException(err, &sp.SecurityOpts) {
			// ...but it was not a security exception, so we ignore it and accept
			// the verification.
			return nil
		}

		// We had an error, let's try with the next ID.
		validationErr = err
	}
	if validationErr != nil {
		return validationErr
	}

	return errors.New("could not find validation node ID")
}

// AssertionConsumer creates an HTTP handler that can be used to authenticate and
// validate an assertion. If the assertion is valid the flow it passed to the
// given grantFn function.
func (sp *ServiceProvider) AssertionConsumer(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		now := Now()

		if err := parseFormAndKeepBody(r); err != nil {
			clientErr(w, r, errors.Wrap(err, "Unable to read POST data"))
		}

		samlResponse := r.Form.Get("SAMLResponse")

		// This RelayState (if any) needs to be been validated by the invoker.
		relayState := r.Form.Get("RelayState")

		// TODO: Remove this when we're stable enough.
		Logf("SAMLResponse -> %v", samlResponse)
		Logf("relayState -> %v", relayState)

		_ = relayState // Don't know what to do with this yet.

		samlResponseXML, err := base64.StdEncoding.DecodeString(samlResponse)
		if err != nil {
			err = errors.Wrapf(err, "could not decode base64 payload: %s", samlResponse)
			clientErr(w, r, errors.Wrap(err, "Malformed payload"))
			return
		}

		Logf("SAMLResponse (XML) -> %v", string(samlResponseXML))

		var res Response
		err = xml.Unmarshal(samlResponseXML, &res)
		if err != nil {
			err = errors.Wrapf(err, "could not unmarshal XML document: %s", string(samlResponseXML))
			clientErr(w, r, errors.Wrap(err, "Malformed XML"))
			return
		}

		_, err = sp.GetIdPMetadata()
		if err != nil {
			clientErr(w, r, errors.Wrap(err, "unable to retrieve IdP metadata"))
			return
		}

		// Validate message.

		if res.Destination != sp.AcsURL {
			// Note: OneLogin triggers this error when the Recipient field
			// is left blank (or when not set to the correct ACS endpoint)
			// in the OneLogin SAML configuration page. OneLogin returns
			// Destination="{recipient}" in the SAML reponse in this case.
			err := errors.Errorf("Wrong ACS destination, expecting %q, got %q", sp.AcsURL, res.Destination)
			clientErr(w, r, errors.Wrap(err, "Wrong ACS destination"))
			return
		}

		if sp.IdPMetadata.EntityID != "" {
			if res.Issuer.Value != sp.IdPMetadata.EntityID {
				err := errors.Errorf("Issuer %q does not match expected entity ID %q", res.Issuer.Value, sp.IdPMetadata.EntityID)
				clientErr(w, r, errors.Wrap(err, "Issuer does not match expected entity ID"))
				return
			}
		}

		if res.Status.StatusCode.Value != "urn:oasis:names:tc:SAML:2.0:status:Success" {
			err := errors.Errorf("Unexpected status code: %v", res.Status.StatusCode.Value)
			clientErr(w, r, errors.Wrap(err, "Unexpected status code"))
			return
		}

		expectedResponse := false
		responseIDs := sp.possibleResponseIDs()
		for i := range responseIDs {
			if responseIDs[i] == res.InResponseTo {
				expectedResponse = true
			}
		}
		if len(responseIDs) == 1 && responseIDs[0] == "" {
			expectedResponse = true
		}
		if !expectedResponse && len(responseIDs) > 0 {
			err := errors.Errorf("Expecting a proper InResponseTo value, got %#v", responseIDs)
			clientErr(w, r, err)
			return
		}

		// Try getting the IdP's cert file before using it.
		_, err = sp.GetIdPCertFile()
		if err != nil {
			internalErr(w, errors.Errorf("Failed to get private key: %v", err))
			return
		}

		// Validating message.
		signatureOK := false

		if res.Signature != nil {
			err := sp.verifySignature(samlResponseXML)
			if err != nil {
				clientErr(w, r, errors.Wrapf(err, "Unable to verify message signature"))
				return
			} else {
				signatureOK = true
			}
		}

		// Retrieve assertion
		var assertion *Assertion

		if res.EncryptedAssertion != nil {
			keyFile, err := sp.PrivkeyFile()
			if err != nil {
				internalErr(w, errors.Errorf("Failed to get private key: %v", err))
				return
			}

			plainTextAssertion, err := xmlsec.Decrypt(res.EncryptedAssertion.EncryptedData, keyFile)
			if err != nil {
				if IsSecurityException(err, &sp.SecurityOpts) {
					clientErr(w, r, errors.Wrap(err, "Unable to decrypt message"))
					return
				}
			}

			assertion = &Assertion{}
			if err := xml.Unmarshal(plainTextAssertion, assertion); err != nil {
				clientErr(w, r, errors.Wrap(err, "Unable to parse assertion"))
				return
			}

			if assertion.Signature != nil {
				err := sp.verifySignature(plainTextAssertion)
				if err != nil {
					clientErr(w, r, errors.Wrapf(err, "Unable to verify assertion signature"))
					return
				} else {
					signatureOK = true
				}
			}
		} else {
			assertion = res.Assertion
		}
		if assertion == nil {
			clientErr(w, r, errors.New("Missing assertion"))
			return
		}

		// Did we receive a signature?
		if !signatureOK {
			clientErr(w, r, errors.New("Unable to validate signature: node not found"))
			return
		}

		// Validate assertion.
		if sp.IdPMetadata.EntityID != "" {
			if assertion.Issuer.Value != sp.IdPMetadata.EntityID {
				err := errors.Errorf("Assertion issuer %q does not match expected entity ID %q", assertion.Issuer.Value, sp.IdPMetadata.EntityID)
				clientErr(w, r, errors.Wrap(err, "Assertion issuer does not match expected entity ID"))
				return
			}
		}

		if assertion.Subject.SubjectConfirmation.SubjectConfirmationData.Recipient != sp.AcsURL {
			err := errors.Errorf("Unexpected assertion recipient, expecting %q, got %q", sp.AcsURL, assertion.Subject.SubjectConfirmation.SubjectConfirmationData.Recipient)
			clientErr(w, r, errors.Wrapf(err, "Unexpected assertion recipient"))
			return
		}

		// The NotBefore and NotOnOrAfter attributes specify time limits on the
		// validity of the assertion within the context of its profile(s) of use.
		// They do not guarantee that the statements in the assertion will be
		// correct or accurate throughout the validity period. The NotBefore
		// attribute specifies the time instant at which the validity interval
		// begins. The NotOnOrAfter attribute specifies the time instant at which
		// the validity interval has ended. If the value for either NotBefore or
		// NotOnOrAfter is omitted, then it is considered unspecified.
		{
			validFrom := assertion.Conditions.NotBefore
			if !validFrom.IsZero() && validFrom.After(now.Add(ClockDriftTolerance)) {
				err := errors.Errorf("Assertion conditions are not valid yet, got %v, current time is %v", validFrom, now)
				clientErr(w, r, errors.Wrap(err, "Assertion conditions are not valid yet"))
				return
			}
		}

		{
			validUntil := assertion.Conditions.NotOnOrAfter
			if !validUntil.IsZero() && validUntil.Before(now.Add(-ClockDriftTolerance)) {
				err := errors.Errorf("Assertion conditions already expired, got %v current time is %v, extra time is %v", validUntil, now, now.Add(-ClockDriftTolerance))
				clientErr(w, r, errors.Wrap(err, "Assertion conditions already expired"))
				return
			}
		}

		// A time instant at which the subject can no longer be confirmed. The time
		// value is encoded in UTC, as described in Section 1.3.3.
		//
		// Note that the time period specified by the optional NotBefore and
		// NotOnOrAfter attributes, if present, SHOULD fall within the overall
		// assertion validity period as specified by the element's NotBefore and
		// NotOnOrAfter attributes. If both attributes are present, the value for
		// NotBefore MUST be less than (earlier than) the value for NotOnOrAfter.

		if validUntil := assertion.Subject.SubjectConfirmation.SubjectConfirmationData.NotOnOrAfter; validUntil.Before(now.Add(-ClockDriftTolerance)) {
			err := errors.Errorf("Assertion conditions already expired, got %v current time is %v", validUntil, now)
			clientErr(w, r, errors.Wrap(err, "Assertion conditions already expired"))
			return
		}

		if assertion.Conditions.AudienceRestriction != nil {
			if assertion.Conditions.AudienceRestriction.Audience.Value != sp.MetadataURL {
				// clientErr(w, fmt.Errorf("Audience restriction mismatch, got %q, expecting %q", assertion.Conditions.AudienceRestriction.Audience.Value, sp.MetadataURL), errors.New("Audience restriction mismatch"))
				// return
			}
		}

		expectedResponse = false
		for i := range responseIDs {
			if responseIDs[i] == assertion.Subject.SubjectConfirmation.SubjectConfirmationData.InResponseTo {
				expectedResponse = true
			}
		}
		if len(responseIDs) == 1 && responseIDs[0] == "" {
			expectedResponse = true
		}

		if !expectedResponse && len(responseIDs) > 0 {
			clientErr(w, r, errors.New("Unexpected assertion InResponseTo value"))
			return
		}

		ctx := context.WithValue(r.Context(), "saml.assertion", assertion)
		next.ServeHTTP(w, r.WithContext(ctx))
	})
}

func publicErrorMessage(err error) string {
	type hasCause interface {
		Cause(err error) error
	}
	// is there any better way to retrieve the error _message_ without the cause?
	if _, ok := err.(hasCause); ok {
		msg := err.Error()
		parts := strings.SplitN(msg, ":", 2)
		if len(parts) > 0 {
			return parts[0]
		}
		return msg
	}
	return err.Error()
}

func clientErr(w http.ResponseWriter, r *http.Request, err error) {
	publicError := publicErrorMessage(err)

	report := InspectRequest(r)

	Fatal(errors.Wrapf(err, "failed request: %s", report.String()))

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.WriteHeader(http.StatusBadRequest)
	w.Write([]byte(publicError))
}

func internalErr(w http.ResponseWriter, err error) {
	serverErr(w, errors.Wrap(err, "an internal error ocurred, please try again"))
}

func serverErr(w http.ResponseWriter, err error) {
	Fatal(err)

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.WriteHeader(http.StatusInternalServerError)
	w.Write([]byte(publicErrorMessage(err)))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"sync"
)

var META_VERSION = 1
var BASE_PORT = 10000
var globalWorldBaseDir string
var globalServerMap = NewServerMap()
var globalIdSequence = NewIdSequence(0)
var globalPortMapper = NewPortMapper()

type PortMapper struct {
	portSequence int
	freePorts    []int
	mutex        sync.Mutex
}

func (pm *PortMapper) getPort() int {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	if len(pm.freePorts) == 0 {
		result := pm.portSequence
		pm.portSequence++
		return result
	}
	result := pm.freePorts[len(pm.freePorts)-1]
	pm.freePorts = pm.freePorts[:len(pm.freePorts)-1]
	return result
}

func (pm *PortMapper) freePort(port int) {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	pm.freePorts = append(pm.freePorts, port)
}

func NewPortMapper() *PortMapper {
	return &PortMapper{
		freePorts: make([]int, 0, 10),
	}
}

type IdSequence struct {
	nextValue int
	mutex     sync.Mutex
}

func (is *IdSequence) getId() int {
	is.mutex.Lock()
	defer is.mutex.Unlock()
	result := is.nextValue
	is.nextValue++
	return result
}

func (is *IdSequence) updateSequence(usedValue int) {
	if usedValue >= is.nextValue {
		is.nextValue = usedValue + 1
	}
}

func NewIdSequence(start int) *IdSequence {
	return &IdSequence{
		nextValue: start,
	}
}

type Server struct {
	Id         int
	CreatorId  int
	Name       string
	PortOffset int
	Handle     *exec.Cmd `json:"-"`
}

func NewServer(id int, creatorId int, name string) *Server {
	if id < 0 {
		id = globalIdSequence.getId()
	}
	return &Server{
		Id:         id,
		CreatorId:  creatorId,
		Name:       strings.TrimSpace(name),
		PortOffset: globalPortMapper.getPort(),
	}
}

type ServerMap struct {
	servers map[string]*Server
	mutex   sync.Mutex
}

func (sm *ServerMap) Put(server *Server) {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	sm.servers[str(server.Id)] = server
}

func (sm *ServerMap) Get(id int) *Server {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	return sm.servers[str(id)]
}

func (sm *ServerMap) Remove(id int) *Server {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	server := sm.servers[str(id)]
	delete(sm.servers, str(id))
	return server
}

func (sm *ServerMap) Encode(w io.Writer) error {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	err := json.NewEncoder(w).Encode(sm.servers)
	return err
}

func NewServerMap() *ServerMap {
	return &ServerMap{
		servers: make(map[string]*Server),
	}
}

func runServer(server *Server) {
	app := "./server"
	args := []string{
		"-client", "client",
		"-world", worldDir(server.Id),
		"-host", ":" + str(BASE_PORT+server.PortOffset),
	}

	cmd := exec.Command(app, args...)
	server.Handle = cmd
	globalServerMap.Put(server)
	err := cmd.Run()

	if err != nil {
		log.Println("Error running the server:", err)
	}
}

type ApiGeneric struct {
	ServerId int
}

type ApiCreate struct {
	CreatorId  int
	ServerName string
}

type ServerListResponse struct {
	List map[string]Server
}

func parseGenericRequest(r *http.Request) (ApiGeneric, error) {
	var result ApiGeneric
	err := json.NewDecoder(r.Body).Decode(&result)
	if err != nil {
		log.Println("Error while parsing generic request", err)
		return result, err
	}
	return result, nil
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	server := globalServerMap.Get(request.ServerId)
	if server == nil {
		http.NotFound(w, r)
		return
	}

	err = json.NewEncoder(w).Encode(server)
	if err != nil {
		log.Println("Error marshalling server")
		http.Error(w, err.Error(), 500)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	err := globalServerMap.Encode(w)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func createHandler(w http.ResponseWriter, r *http.Request) {
	var request ApiCreate
	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		log.Println("Error parsing create request:", err, "Body:", r.Body)
		http.Error(w, err.Error(), 500)
		return
	}

	server := NewServer(-1, request.CreatorId, request.ServerName)
	saveServer(server)
	go runServer(server)
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	server := globalServerMap.Remove(request.ServerId)
	err = server.Handle.Process.Kill()
	if err != nil {
		log.Println("Error while sending SIG_DEATH to running server", err)
	}
	_, err = server.Handle.Process.Wait()
	if err != nil {
		log.Println("Error while waiting on process.", err)
	}
	globalPortMapper.freePort(server.PortOffset)
}

func saveServer(server *Server) {
	err := os.MkdirAll(worldDir(server.Id), 0755)
	if err != nil {
		log.Println("Error creating required directories:", err)
		return
	}

	file, err := os.Create(path.Join(worldDir(server.Id), "meta.server"))
	if err != nil {
		log.Println("Error creating file to save server:", err)
		return
	}
	defer file.Close()

	err = json.NewEncoder(file).Encode(server)
	if err != nil {
		log.Println("Error encoding meta data.", err)
	}
}

func loadServers() {
	files, err := ioutil.ReadDir(globalWorldBaseDir)
	if err != nil {
		log.Println("Error reading base world directory")
		return
	}

	for _, fileInfo := range files {
		loadServer(fileInfo)
	}
}

func loadServer(fileInfo os.FileInfo) {
	if !fileInfo.IsDir() {
		return
	}

	file, err := os.Open(path.Join(globalWorldBaseDir, fileInfo.Name(), "meta.server"))
	if err != nil {
		log.Println("Error opening server meta data:", err)
		return
	}
	defer file.Close()

	var server Server
	err = json.NewDecoder(file).Decode(&server)
	if err != nil {
		log.Println("Error while parsing meta json for", fileInfo.Name(), err)
		return
	}

	server.PortOffset = globalPortMapper.getPort()
	globalIdSequence.updateSequence(server.Id)
	go runServer(&server)
}

func main() {
	worldFolder := flag.String("worlds", "worlds/", "Sets the base folder used to store the worlds.")
	flag.Parse()
	globalWorldBaseDir = *worldFolder

	loadServers()

	runtime.GOMAXPROCS(runtime.NumCPU())

	http.HandleFunc("/get", getHandler)
	http.HandleFunc("/list", listHandler)
	http.HandleFunc("/create", createHandler)
	http.HandleFunc("/delete", deleteHandler)
	err := http.ListenAndServe(":3001", nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

// Utility function
func str(i int) string {
	return fmt.Sprintf("%d", i)
}

func worldDir(serverId int) string {
	return path.Join(globalWorldBaseDir, "world"+str(serverId))
}
<commit_msg>Use a default name if only whitespace is provided<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path"
	"runtime"
	"strings"
	"sync"
)

var META_VERSION = 1
var BASE_PORT = 10000
var globalWorldBaseDir string
var globalServerMap = NewServerMap()
var globalIdSequence = NewIdSequence(0)
var globalPortMapper = NewPortMapper()

type PortMapper struct {
	portSequence int
	freePorts    []int
	mutex        sync.Mutex
}

func (pm *PortMapper) getPort() int {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	if len(pm.freePorts) == 0 {
		result := pm.portSequence
		pm.portSequence++
		return result
	}
	result := pm.freePorts[len(pm.freePorts)-1]
	pm.freePorts = pm.freePorts[:len(pm.freePorts)-1]
	return result
}

func (pm *PortMapper) freePort(port int) {
	pm.mutex.Lock()
	defer pm.mutex.Unlock()
	pm.freePorts = append(pm.freePorts, port)
}

func NewPortMapper() *PortMapper {
	return &PortMapper{
		freePorts: make([]int, 0, 10),
	}
}

type IdSequence struct {
	nextValue int
	mutex     sync.Mutex
}

func (is *IdSequence) getId() int {
	is.mutex.Lock()
	defer is.mutex.Unlock()
	result := is.nextValue
	is.nextValue++
	return result
}

func (is *IdSequence) updateSequence(usedValue int) {
	if usedValue >= is.nextValue {
		is.nextValue = usedValue + 1
	}
}

func NewIdSequence(start int) *IdSequence {
	return &IdSequence{
		nextValue: start,
	}
}

type Server struct {
	Id         int
	CreatorId  int
	Name       string
	PortOffset int
	Handle     *exec.Cmd `json:"-"`
}

func NewServer(id int, creatorId int, name string) *Server {
	if id < 0 {
		id = globalIdSequence.getId()
	}

	serverName := strings.TrimSpace(name)
	if len(serverName) == 0 {
		serverName = "NoName-" + str(id)
	}

	return &Server{
		Id:         id,
		CreatorId:  creatorId,
		Name:       serverName,
		PortOffset: globalPortMapper.getPort(),
	}
}

type ServerMap struct {
	servers map[string]*Server
	mutex   sync.Mutex
}

func (sm *ServerMap) Put(server *Server) {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	sm.servers[str(server.Id)] = server
}

func (sm *ServerMap) Get(id int) *Server {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	return sm.servers[str(id)]
}

func (sm *ServerMap) Remove(id int) *Server {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	server := sm.servers[str(id)]
	delete(sm.servers, str(id))
	return server
}

func (sm *ServerMap) Encode(w io.Writer) error {
	sm.mutex.Lock()
	defer sm.mutex.Unlock()
	err := json.NewEncoder(w).Encode(sm.servers)
	return err
}

func NewServerMap() *ServerMap {
	return &ServerMap{
		servers: make(map[string]*Server),
	}
}

func runServer(server *Server) {
	app := "./server"
	args := []string{
		"-client", "client",
		"-world", worldDir(server.Id),
		"-host", ":" + str(BASE_PORT+server.PortOffset),
	}

	cmd := exec.Command(app, args...)
	server.Handle = cmd
	globalServerMap.Put(server)
	err := cmd.Run()

	if err != nil {
		log.Println("Error running the server:", err)
	}
}

type ApiGeneric struct {
	ServerId int
}

type ApiCreate struct {
	CreatorId  int
	ServerName string
}

type ServerListResponse struct {
	List map[string]Server
}

func parseGenericRequest(r *http.Request) (ApiGeneric, error) {
	var result ApiGeneric
	err := json.NewDecoder(r.Body).Decode(&result)
	if err != nil {
		log.Println("Error while parsing generic request", err)
		return result, err
	}
	return result, nil
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	server := globalServerMap.Get(request.ServerId)
	if server == nil {
		http.NotFound(w, r)
		return
	}

	err = json.NewEncoder(w).Encode(server)
	if err != nil {
		log.Println("Error marshalling server")
		http.Error(w, err.Error(), 500)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	err := globalServerMap.Encode(w)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func createHandler(w http.ResponseWriter, r *http.Request) {
	var request ApiCreate
	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		log.Println("Error parsing create request:", err, "Body:", r.Body)
		http.Error(w, err.Error(), 500)
		return
	}

	server := NewServer(-1, request.CreatorId, request.ServerName)
	saveServer(server)
	go runServer(server)
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	request, err := parseGenericRequest(r)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	server := globalServerMap.Remove(request.ServerId)
	err = server.Handle.Process.Kill()
	if err != nil {
		log.Println("Error while sending SIG_DEATH to running server", err)
	}
	_, err = server.Handle.Process.Wait()
	if err != nil {
		log.Println("Error while waiting on process.", err)
	}
	globalPortMapper.freePort(server.PortOffset)
	os.RemoveAll(worldDir(server.Id))
}

func saveServer(server *Server) {
	err := os.MkdirAll(worldDir(server.Id), 0755)
	if err != nil {
		log.Println("Error creating required directories:", err)
		return
	}

	file, err := os.Create(path.Join(worldDir(server.Id), "meta.server"))
	if err != nil {
		log.Println("Error creating file to save server:", err)
		return
	}
	defer file.Close()

	err = json.NewEncoder(file).Encode(server)
	if err != nil {
		log.Println("Error encoding meta data.", err)
	}
}

func loadServers() {
	files, err := ioutil.ReadDir(globalWorldBaseDir)
	if err != nil {
		log.Println("Error reading base world directory")
		return
	}

	for _, fileInfo := range files {
		loadServer(fileInfo)
	}
}

func loadServer(fileInfo os.FileInfo) {
	if !fileInfo.IsDir() {
		return
	}

	file, err := os.Open(path.Join(globalWorldBaseDir, fileInfo.Name(), "meta.server"))
	if err != nil {
		log.Println("Error opening server meta data:", err)
		return
	}
	defer file.Close()

	var server Server
	err = json.NewDecoder(file).Decode(&server)
	if err != nil {
		log.Println("Error while parsing meta json for", fileInfo.Name(), err)
		return
	}

	server.PortOffset = globalPortMapper.getPort()
	globalIdSequence.updateSequence(server.Id)
	go runServer(&server)
}

func main() {
	worldFolder := flag.String("worlds", "worlds/", "Sets the base folder used to store the worlds.")
	flag.Parse()
	globalWorldBaseDir = *worldFolder

	loadServers()

	runtime.GOMAXPROCS(runtime.NumCPU())

	http.HandleFunc("/get", getHandler)
	http.HandleFunc("/list", listHandler)
	http.HandleFunc("/create", createHandler)
	http.HandleFunc("/delete", deleteHandler)
	err := http.ListenAndServe(":3001", nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

// Utility function
func str(i int) string {
	return fmt.Sprintf("%d", i)
}

func worldDir(serverId int) string {
	return path.Join(globalWorldBaseDir, "world"+str(serverId))
}
<|endoftext|>


<commit_before>package tests

import (
	"testing"

	"github.com/polydawn/go-errcat"

	"go.polydawn.net/rio/fs"
)

func CheckMkdirLstatRoundtrip(t *testing.T, afs fs.FS) {
	d1 := fs.MustRelPath("d1")
	if err := afs.Mkdir(d1, 0755); err != nil {
		t.Fatalf("mkdir failed: %s", err)
	}
	stat, err := afs.LStat(d1)
	if err != nil {
		t.Fatalf("lstat on just-created dir failed: %s", err)
	}
	if stat.Type != fs.Type_Dir {
		t.Fatalf("lstat on just-created dir returned bogus type: %s", stat.Type)
	}
}

func CheckDeepMkdirError(t *testing.T, afs fs.FS) {
	d1d2 := fs.MustRelPath("d1/d2")
	if err := afs.Mkdir(d1d2, 0755); err == nil {
		t.Fatalf("deep mkdir without parents should have failed: %s", err)
	}
	_, err := afs.LStat(d1d2)
	if errcat.Category(err) != fs.ErrNotExists {
		t.Fatalf("deep mkdir without parents error with category %q: got %q", fs.ErrNotExists, errcat.Category(err))
	}
}

func CheckMklinkLstatRoundtrip(t *testing.T, afs fs.FS) {
	l1 := fs.MustRelPath("l1")
	if err := afs.Mklink(l1, "./target"); err != nil {
		t.Fatalf("mklink failed: %s", err)
	}
	stat, err := afs.LStat(l1)
	if err != nil {
		t.Fatalf("lstat on just-created symlink failed: %s", err)
	}
	if stat.Type != fs.Type_Symlink {
		t.Fatalf("lstat on just-created symlink returned bogus type: %s", stat.Type)
	}
}
<commit_msg>fs/tests: correct some over-fatality.<commit_after>package tests

import (
	"testing"

	"github.com/polydawn/go-errcat"

	"go.polydawn.net/rio/fs"
)

func CheckMkdirLstatRoundtrip(t *testing.T, afs fs.FS) {
	d1 := fs.MustRelPath("d1")
	if err := afs.Mkdir(d1, 0755); err != nil {
		t.Fatalf("mkdir failed: %s", err)
	}
	stat, err := afs.LStat(d1)
	if err != nil {
		t.Fatalf("lstat on just-created dir failed: %s", err)
	}
	if stat.Type != fs.Type_Dir {
		t.Errorf("lstat on just-created dir returned bogus type: %s", stat.Type)
	}
}

func CheckDeepMkdirError(t *testing.T, afs fs.FS) {
	d1d2 := fs.MustRelPath("d1/d2")
	if err := afs.Mkdir(d1d2, 0755); err == nil {
		t.Fatalf("deep mkdir without parents should have failed: %s", err)
	}
	_, err := afs.LStat(d1d2)
	if errcat.Category(err) != fs.ErrNotExists {
		t.Errorf("deep mkdir without parents error with category %q: got %q", fs.ErrNotExists, errcat.Category(err))
	}
}

func CheckMklinkLstatRoundtrip(t *testing.T, afs fs.FS) {
	l1 := fs.MustRelPath("l1")
	if err := afs.Mklink(l1, "./target"); err != nil {
		t.Fatalf("mklink failed: %s", err)
	}
	stat, err := afs.LStat(l1)
	if err != nil {
		t.Fatalf("lstat on just-created symlink failed: %s", err)
	}
	if stat.Type != fs.Type_Symlink {
		t.Errorf("lstat on just-created symlink returned bogus type: %s", stat.Type)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 The GoSNMP Authors. All rights reserved.  Use of this
// source code is governed by a BSD-style license that can be found in the
// LICENSE file.

// This set of end-to-end integration tests execute gosnmp against a real
// SNMP MIB-2 host. Potential test systems could include a router, NAS box, printer,
// or a linux box running snmpd, snmpsimd.py, etc.
//
// Ensure "gosnmp-test-host" is defined in your hosts file, and points to your
// generic test system.

package gosnmp

import (
	"fmt"
	//"log"
	//"os"
	"strings"
	"testing"
	"time"
)

const (
	testTarget = "gosnmp-test-host" // Don't modify here - set in your hosts file.
	testPort   = 161
)

func setupConnection(t *testing.T) {
	if len(testTarget) < 1 {
		t.Skip("Skipping Generic tests! Is %s a valid SNMP host?")
	}
	Default.Target = testTarget
	Default.Port = testPort
	err := Default.Connect()
	if err != nil {
		t.Fatalf("Connection failed. Is %s defined in your hosts file? \n(err: %v)",
			testTarget, err)
	}
}

func TestGenericBasicGet(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Type != OctetString {
		t.Fatalf("Expected sysDescr to be OctetString")
	}
	sysDescr := result.Variables[0].Value.([]byte)
	if len(sysDescr) == 0 {
		t.Fatalf("Got a zero length sysDescr")
	}
}

func TestGenericMultiGet(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	oids := []string{
		".1.3.6.1.2.1.1.1.0", // SNMP MIB-2 sysDescr
		".1.3.6.1.2.1.1.5.0", // SNMP MIB-2 sysName
	}
	result, err := Default.Get(oids)
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 2 {
		t.Fatalf("Expected result of size 2")
	}
	for _, v := range result.Variables {
		if v.Type != OctetString {
			t.Fatalf("Expected OctetString")
		}
	}
}

func TestGenericGetNext(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	sysDescrOid := ".1.3.6.1.2.1.1.1.0" // SNMP MIB-2 sysDescr
	result, err := Default.GetNext([]string{sysDescrOid})
	if err != nil {
		t.Fatalf("GetNext() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Name == sysDescrOid {
		t.Fatalf("Expected next OID")
	}
}

func TestGenericWalk(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.WalkAll("")
	if err != nil {
		t.Fatalf("WalkAll() Failed with error => %v", err)
	}
	if len(result) <= 1 {
		t.Fatalf("Expected multiple values, got %d", len(result))
	}
}

func TestGenericBulkWalk(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.BulkWalkAll("")
	if err != nil {
		t.Fatalf("BulkWalkAll() Failed with error => %v", err)
	}
	if len(result) <= 1 {
		t.Fatalf("Expected multiple values, got %d", len(result))
	}
}

// Standard exception/error tests

func TestGenericFailureUnknownHost(t *testing.T) {
	unknownHost := fmt.Sprintf("gosnmp-test-unknown-host-%d", time.Now().UTC().UnixNano())
	Default.Target = unknownHost
	err := Default.Connect()
	if err == nil {
		t.Fatalf("Expected connection failure due to unknown host")
	}
	if !strings.Contains(strings.ToLower(err.Error()), "no such host") {
		t.Fatalf("Expected connection error of type 'no such host'! Got => %v", err)
	}
	_, err = Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err == nil {
		t.Fatalf("Expected get to fail due to missing connection")
	}
}

func TestGenericFailureConnectionTimeout(t *testing.T) {
	Default.Target = "198.51.100.1" // Black hole
	err := Default.Connect()
	if err != nil {
		t.Fatalf("Did not expect connection error with IP address")
	}
	_, err = Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err == nil {
		t.Fatalf("Expected Get() to fail due to invalid IP")
	}
	if !strings.Contains(err.Error(), "timeout") {
		t.Fatalf("Expected timeout error. Got => %v", err)
	}
}

func TestGenericFailureConnectionRefused(t *testing.T) {
	Default.Target = "127.0.0.1"
	Default.Port = 1 // Don't expect SNMP to be running here!
	err := Default.Connect()
	if err != nil {
		t.Fatalf("Did not expect connection error with IP address")
	}
	_, err = Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err == nil {
		t.Fatalf("Expected Get() to fail due to invalid port")
	}
	if !(strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "forcibly closed")) {
		t.Fatalf("Expected connection refused error. Got => %v", err)
	}
}

func TestSnmpV3NoAuthNoPrivBasicGet(t *testing.T) {
	Default.Version = Version3
	Default.MsgFlags = NoAuthNoPriv
	Default.SecurityModel = UserSecurityModel
	Default.SecurityParameters = &UsmSecurityParameters{UserName: "test"}
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Type != OctetString {
		t.Fatalf("Expected sysDescr to be OctetString")
	}
	sysDescr := result.Variables[0].Value.([]byte)
	if len(sysDescr) == 0 {
		t.Fatalf("Got a zero length sysDescr")
	}
}

func TestSnmpV3AuthNoPrivBasicGet(t *testing.T) {
	Default.Version = Version3
	Default.MsgFlags = AuthNoPriv
	Default.SecurityModel = UserSecurityModel
	Default.SecurityParameters = &UsmSecurityParameters{UserName: "testAuth", AuthenticationProtocol: MD5, AuthenticationPassphrase: "testingpass0123456789"}
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Type != OctetString {
		t.Fatalf("Expected sysDescr to be OctetString")
	}
	sysDescr := result.Variables[0].Value.([]byte)
	if len(sysDescr) == 0 {
		t.Fatalf("Got a zero length sysDescr")
	}
}
<commit_msg>Fix username<commit_after>// Copyright 2012-2014 The GoSNMP Authors. All rights reserved.  Use of this
// source code is governed by a BSD-style license that can be found in the
// LICENSE file.

// This set of end-to-end integration tests execute gosnmp against a real
// SNMP MIB-2 host. Potential test systems could include a router, NAS box, printer,
// or a linux box running snmpd, snmpsimd.py, etc.
//
// Ensure "gosnmp-test-host" is defined in your hosts file, and points to your
// generic test system.

package gosnmp

import (
	"fmt"
	//"log"
	//"os"
	"strings"
	"testing"
	"time"
)

const (
	testTarget = "gosnmp-test-host" // Don't modify here - set in your hosts file.
	testPort   = 161
)

func setupConnection(t *testing.T) {
	if len(testTarget) < 1 {
		t.Skip("Skipping Generic tests! Is %s a valid SNMP host?")
	}
	Default.Target = testTarget
	Default.Port = testPort
	err := Default.Connect()
	if err != nil {
		t.Fatalf("Connection failed. Is %s defined in your hosts file? \n(err: %v)",
			testTarget, err)
	}
}

func TestGenericBasicGet(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Type != OctetString {
		t.Fatalf("Expected sysDescr to be OctetString")
	}
	sysDescr := result.Variables[0].Value.([]byte)
	if len(sysDescr) == 0 {
		t.Fatalf("Got a zero length sysDescr")
	}
}

func TestGenericMultiGet(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	oids := []string{
		".1.3.6.1.2.1.1.1.0", // SNMP MIB-2 sysDescr
		".1.3.6.1.2.1.1.5.0", // SNMP MIB-2 sysName
	}
	result, err := Default.Get(oids)
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 2 {
		t.Fatalf("Expected result of size 2")
	}
	for _, v := range result.Variables {
		if v.Type != OctetString {
			t.Fatalf("Expected OctetString")
		}
	}
}

func TestGenericGetNext(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	sysDescrOid := ".1.3.6.1.2.1.1.1.0" // SNMP MIB-2 sysDescr
	result, err := Default.GetNext([]string{sysDescrOid})
	if err != nil {
		t.Fatalf("GetNext() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Name == sysDescrOid {
		t.Fatalf("Expected next OID")
	}
}

func TestGenericWalk(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.WalkAll("")
	if err != nil {
		t.Fatalf("WalkAll() Failed with error => %v", err)
	}
	if len(result) <= 1 {
		t.Fatalf("Expected multiple values, got %d", len(result))
	}
}

func TestGenericBulkWalk(t *testing.T) {
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.BulkWalkAll("")
	if err != nil {
		t.Fatalf("BulkWalkAll() Failed with error => %v", err)
	}
	if len(result) <= 1 {
		t.Fatalf("Expected multiple values, got %d", len(result))
	}
}

// Standard exception/error tests

func TestGenericFailureUnknownHost(t *testing.T) {
	unknownHost := fmt.Sprintf("gosnmp-test-unknown-host-%d", time.Now().UTC().UnixNano())
	Default.Target = unknownHost
	err := Default.Connect()
	if err == nil {
		t.Fatalf("Expected connection failure due to unknown host")
	}
	if !strings.Contains(strings.ToLower(err.Error()), "no such host") {
		t.Fatalf("Expected connection error of type 'no such host'! Got => %v", err)
	}
	_, err = Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err == nil {
		t.Fatalf("Expected get to fail due to missing connection")
	}
}

func TestGenericFailureConnectionTimeout(t *testing.T) {
	Default.Target = "198.51.100.1" // Black hole
	err := Default.Connect()
	if err != nil {
		t.Fatalf("Did not expect connection error with IP address")
	}
	_, err = Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err == nil {
		t.Fatalf("Expected Get() to fail due to invalid IP")
	}
	if !strings.Contains(err.Error(), "timeout") {
		t.Fatalf("Expected timeout error. Got => %v", err)
	}
}

func TestGenericFailureConnectionRefused(t *testing.T) {
	Default.Target = "127.0.0.1"
	Default.Port = 1 // Don't expect SNMP to be running here!
	err := Default.Connect()
	if err != nil {
		t.Fatalf("Did not expect connection error with IP address")
	}
	_, err = Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err == nil {
		t.Fatalf("Expected Get() to fail due to invalid port")
	}
	if !(strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "forcibly closed")) {
		t.Fatalf("Expected connection refused error. Got => %v", err)
	}
}

func TestSnmpV3NoAuthNoPrivBasicGet(t *testing.T) {
	Default.Version = Version3
	Default.MsgFlags = NoAuthNoPriv
	Default.SecurityModel = UserSecurityModel
	Default.SecurityParameters = &UsmSecurityParameters{UserName: "test"}
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Type != OctetString {
		t.Fatalf("Expected sysDescr to be OctetString")
	}
	sysDescr := result.Variables[0].Value.([]byte)
	if len(sysDescr) == 0 {
		t.Fatalf("Got a zero length sysDescr")
	}
}

func TestSnmpV3AuthNoPrivBasicGet(t *testing.T) {
	Default.Version = Version3
	Default.MsgFlags = AuthNoPriv
	Default.SecurityModel = UserSecurityModel
	Default.SecurityParameters = &UsmSecurityParameters{UserName: "authTest", AuthenticationProtocol: MD5, AuthenticationPassphrase: "testingpass0123456789"}
	setupConnection(t)
	defer Default.Conn.Close()

	result, err := Default.Get([]string{".1.3.6.1.2.1.1.1.0"}) // SNMP MIB-2 sysDescr
	if err != nil {
		t.Fatalf("Get() failed with error => %v", err)
	}
	if len(result.Variables) != 1 {
		t.Fatalf("Expected result of size 1")
	}
	if result.Variables[0].Type != OctetString {
		t.Fatalf("Expected sysDescr to be OctetString")
	}
	sysDescr := result.Variables[0].Value.([]byte)
	if len(sysDescr) == 0 {
		t.Fatalf("Got a zero length sysDescr")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/docopt/docopt-go"
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/calicoctl/v3/calicoctl/commands"
	"github.com/projectcalico/calicoctl/v3/calicoctl/util"
)

func main() {
	name, desc := util.NameAndDescription()
	doc := fmt.Sprintf(`Usage:
  <BINARY_NAME> [options] <command> [<args>...]

    create       Create a resource by file, directory or stdin.
    replace      Replace a resource by file, directory or stdin.
    apply        Apply a resource by file, directory or stdin.  This creates a resource
                 if it does not exist, and replaces a resource if it does exists.
    patch        Patch a pre-exisiting resource in place.
    delete       Delete a resource identified by file, directory, stdin or resource type and
                 name.
    get          Get a resource identified by file, directory, stdin or resource type and
                 name.
    label        Add or update labels of resources.
    convert      Convert config files between different API versions.
    ipam         IP address management.
    node         Calico node management.
    version      Display the version of this binary.
    export       Export the Calico datastore objects for migration
    import       Import the Calico datastore objects for migration
    datastore    Calico datastore management.

Options:
  -h --help                    Show this screen.
  -l --log-level=<level>       Set the log level (one of panic, fatal, error,
                               warn, info, debug) [default: panic]
     --context=<context>       The name of the kubeconfig context to use.
     --allow-version-mismatch  Allow client and cluster versions mismatch.

Description:
  The %s is used to manage Calico network and security
  policy, to view and manage endpoint configuration, and to manage a Calico
  node instance.

  See '<BINARY_NAME> <command> --help' to read about a specific subcommand.
`, desc)

	// Replace all instances of BINARY_NAME with the name of the binary.
	doc = strings.ReplaceAll(doc, "<BINARY_NAME>", name)

	var parser = &docopt.Parser{
		HelpHandler:   docopt.PrintHelpOnly,
		OptionsFirst:  true,
		SkipHelpFlags: false,
	}
	arguments, err := parser.ParseArgs(doc, nil, commands.VERSION_SUMMARY)
	if err != nil {
		if _, ok := err.(*docopt.UserError); ok {
			// the user gave us bad input
			fmt.Printf("Invalid option: '%s'. Use flag '--help' to read about a specific subcommand.\n", strings.Join(os.Args[1:], " "))
		}
		os.Exit(1)
	}

	if logLevel := arguments["--log-level"]; logLevel != nil {
		parsedLogLevel, err := log.ParseLevel(logLevel.(string))
		if err != nil {
			fmt.Printf("Unknown log level: %s, expected one of: \n"+
				"panic, fatal, error, warn, info, debug.\n", logLevel)
			os.Exit(1)
		} else {
			log.SetLevel(parsedLogLevel)
			log.Infof("Log level set to %v", parsedLogLevel)
		}
	}

	if context := arguments["--context"]; context != nil {
		os.Setenv("K8S_CURRENT_CONTEXT", context.(string))
	}

	if arguments["<command>"] != nil {
		command := arguments["<command>"].(string)
		args := append([]string{command}, arguments["<args>"].([]string)...)

		// Propagate the '--allow-version-mismatch' arg to override version mismatch checking.
		if allowMismatch, _ := arguments["--allow-version-mismatch"].(bool); allowMismatch {
			args = append(args, "--allow-version-mismatch")
		}

		var err error

		switch command {
		case "create":
			err = commands.Create(args)
		case "replace":
			err = commands.Replace(args)
		case "apply":
			err = commands.Apply(args)
		case "patch":
			err = commands.Patch(args)
		case "delete":
			err = commands.Delete(args)
		case "get":
			err = commands.Get(args)
		case "label":
			err = commands.Label(args)
		case "convert":
			err = commands.Convert(args)
		case "version":
			err = commands.Version(args)
		case "node":
			err = commands.Node(args)
		case "ipam":
			err = commands.IPAM(args)
		case "datastore":
			err = commands.Datastore(args)
		default:
			err = fmt.Errorf("Unknown command: %q\n%s", command, doc)
		}

		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}
	}
}
<commit_msg>Remove export and import commands from help text<commit_after>// Copyright (c) 2019 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/docopt/docopt-go"
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/calicoctl/v3/calicoctl/commands"
	"github.com/projectcalico/calicoctl/v3/calicoctl/util"
)

func main() {
	name, desc := util.NameAndDescription()
	doc := fmt.Sprintf(`Usage:
  <BINARY_NAME> [options] <command> [<args>...]

    create       Create a resource by file, directory or stdin.
    replace      Replace a resource by file, directory or stdin.
    apply        Apply a resource by file, directory or stdin.  This creates a resource
                 if it does not exist, and replaces a resource if it does exists.
    patch        Patch a pre-exisiting resource in place.
    delete       Delete a resource identified by file, directory, stdin or resource type and
                 name.
    get          Get a resource identified by file, directory, stdin or resource type and
                 name.
    label        Add or update labels of resources.
    convert      Convert config files between different API versions.
    ipam         IP address management.
    node         Calico node management.
    version      Display the version of this binary.
    datastore    Calico datastore management.

Options:
  -h --help                    Show this screen.
  -l --log-level=<level>       Set the log level (one of panic, fatal, error,
                               warn, info, debug) [default: panic]
     --context=<context>       The name of the kubeconfig context to use.
     --allow-version-mismatch  Allow client and cluster versions mismatch.

Description:
  The %s is used to manage Calico network and security
  policy, to view and manage endpoint configuration, and to manage a Calico
  node instance.

  See '<BINARY_NAME> <command> --help' to read about a specific subcommand.
`, desc)

	// Replace all instances of BINARY_NAME with the name of the binary.
	doc = strings.ReplaceAll(doc, "<BINARY_NAME>", name)

	var parser = &docopt.Parser{
		HelpHandler:   docopt.PrintHelpOnly,
		OptionsFirst:  true,
		SkipHelpFlags: false,
	}
	arguments, err := parser.ParseArgs(doc, nil, commands.VERSION_SUMMARY)
	if err != nil {
		if _, ok := err.(*docopt.UserError); ok {
			// the user gave us bad input
			fmt.Printf("Invalid option: '%s'. Use flag '--help' to read about a specific subcommand.\n", strings.Join(os.Args[1:], " "))
		}
		os.Exit(1)
	}

	if logLevel := arguments["--log-level"]; logLevel != nil {
		parsedLogLevel, err := log.ParseLevel(logLevel.(string))
		if err != nil {
			fmt.Printf("Unknown log level: %s, expected one of: \n"+
				"panic, fatal, error, warn, info, debug.\n", logLevel)
			os.Exit(1)
		} else {
			log.SetLevel(parsedLogLevel)
			log.Infof("Log level set to %v", parsedLogLevel)
		}
	}

	if context := arguments["--context"]; context != nil {
		os.Setenv("K8S_CURRENT_CONTEXT", context.(string))
	}

	if arguments["<command>"] != nil {
		command := arguments["<command>"].(string)
		args := append([]string{command}, arguments["<args>"].([]string)...)

		// Propagate the '--allow-version-mismatch' arg to override version mismatch checking.
		if allowMismatch, _ := arguments["--allow-version-mismatch"].(bool); allowMismatch {
			args = append(args, "--allow-version-mismatch")
		}

		var err error

		switch command {
		case "create":
			err = commands.Create(args)
		case "replace":
			err = commands.Replace(args)
		case "apply":
			err = commands.Apply(args)
		case "patch":
			err = commands.Patch(args)
		case "delete":
			err = commands.Delete(args)
		case "get":
			err = commands.Get(args)
		case "label":
			err = commands.Label(args)
		case "convert":
			err = commands.Convert(args)
		case "version":
			err = commands.Version(args)
		case "node":
			err = commands.Node(args)
		case "ipam":
			err = commands.IPAM(args)
		case "datastore":
			err = commands.Datastore(args)
		default:
			err = fmt.Errorf("Unknown command: %q\n%s", command, doc)
		}

		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/barnslig/torture/lib/elastic"
	"github.com/dustin/go-humanize"
	"regexp"
	"strings"
)

type ElasticSearch struct {
	url string
}

type hash map[string]interface{}

func CreateElasticSearch(host string) (es *ElasticSearch, err error) {
	es = &ElasticSearch{url: host}
	return
}

func (es *ElasticSearch) Search(stmt Statement, perPage int, page int) (result elastic.Result, err error) {
	query := strings.Join(stmt.Phrases, " ")

	matchQ := hash{
		"function_score": hash{
			"query": hash{
				"match": hash{
					"Servers.Path": hash{
						"query":     query,
						"fuzziness": 0.4,
					},
				},
			},
			"field_value_factor": hash{
				"field":    "Size",
				"missing":  1,
				"modifier": "log1p",
				"factor":   0.000000001, // Increase Bytes to Gigabytes
			},
		},
	}

	filterQ := []hash{}
	for _, treat := range stmt.Treats {

		// Filter for file extensions, e.g. extension:pdf or extension!mkv
		if treat.Key == keyExtension {
			regex := ""
			switch treat.Operator {
			case EQUALS:
				regex = fmt.Sprintf(`.+.%s`, ExtractRegexSave(treat.Value))
				break
			case NOT:
				regex = fmt.Sprintf(`@&~(.+.%s)`, ExtractRegexSave(treat.Value))
				break
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"regexp": hash{
					"Filename": regex,
				},
			})
		}

		// Filter for size, e.g. size>1gb or size<20mb
		if treat.Key == keySize {
			// Try to parse the given size
			size, err := humanize.ParseBytes(treat.Value)
			if err != nil {
				continue
			}

			rangeQ := hash{}
			switch treat.Operator {
			case LTE:
				rangeQ = hash{
					"lte": size,
				}
				break
			case GTE:
				rangeQ = hash{
					"gte": size,
				}
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"range": hash{
					"Size": rangeQ,
				},
			})
		}

	}

	data, err := elastic.Request("GET", elastic.URL(es.url, "/torture/file/_search"), hash{
		"size": perPage,
		"from": perPage * page,
		"query": hash{
			"filtered": hash{
				"query": matchQ,
				"filter": hash{
					"and": hash{
						"filters": filterQ,
					},
				},
			},
		},
	})
	if err != nil {
		return
	}

	result = elastic.Result{}
	err = json.Unmarshal(data, &result)

	return
}

// Extract only regex-save characters from a string so we can Sprintf
func ExtractRegexSave(src string) string {
	r := regexp.MustCompile(`\w+`)
	return strings.Join(r.FindAllString(src, -1), "")
}
<commit_msg>Frontend: Query using query_string, default operator is AND<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/barnslig/torture/lib/elastic"
	"github.com/dustin/go-humanize"
	"regexp"
	"strings"
)

type ElasticSearch struct {
	url string
}

type hash map[string]interface{}

func CreateElasticSearch(host string) (es *ElasticSearch, err error) {
	es = &ElasticSearch{url: host}
	return
}

func (es *ElasticSearch) Search(stmt Statement, perPage int, page int) (result elastic.Result, err error) {
	query := strings.Join(stmt.Phrases, " ")

	matchQ := hash{
		"function_score": hash{
			"query": hash{
				"query_string": hash{
					"default_field": "Servers.Path",
					"default_operator": "AND",
					"query": query,
				},
			},
			"field_value_factor": hash{
				"field":    "Size",
				"missing":  1,
				"modifier": "log1p",
				"factor":   0.000000001, // Increase Bytes to Gigabytes
			},
		},
	}

	filterQ := []hash{}
	for _, treat := range stmt.Treats {

		// Filter for file extensions, e.g. extension:pdf or extension!mkv
		if treat.Key == keyExtension {
			regex := ""
			switch treat.Operator {
			case EQUALS:
				regex = fmt.Sprintf(`.+.%s`, ExtractRegexSave(treat.Value))
				break
			case NOT:
				regex = fmt.Sprintf(`@&~(.+.%s)`, ExtractRegexSave(treat.Value))
				break
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"regexp": hash{
					"Filename": regex,
				},
			})
		}

		// Filter for size, e.g. size>1gb or size<20mb
		if treat.Key == keySize {
			// Try to parse the given size
			size, err := humanize.ParseBytes(treat.Value)
			if err != nil {
				continue
			}

			rangeQ := hash{}
			switch treat.Operator {
			case LTE:
				rangeQ = hash{
					"lte": size,
				}
				break
			case GTE:
				rangeQ = hash{
					"gte": size,
				}
			default:
				continue
			}

			filterQ = append(filterQ, hash{
				"range": hash{
					"Size": rangeQ,
				},
			})
		}

	}

	data, err := elastic.Request("GET", elastic.URL(es.url, "/torture/file/_search"), hash{
		"size": perPage,
		"from": perPage * page,
		"query": hash{
			"filtered": hash{
				"query": matchQ,
				"filter": hash{
					"and": hash{
						"filters": filterQ,
					},
				},
			},
		},
	})
	if err != nil {
		return
	}

	result = elastic.Result{}
	err = json.Unmarshal(data, &result)

	return
}

// Extract only regex-save characters from a string so we can Sprintf
func ExtractRegexSave(src string) string {
	r := regexp.MustCompile(`\w+`)
	return strings.Join(r.FindAllString(src, -1), "")
}
<|endoftext|>


<commit_before>package pkg2

import "github.com/uber-go/zap"

func Print() {
	zap.New(zap.NewJSONEncoder()).Error("test print")
}
<commit_msg>test: use new zap address<commit_after>package pkg2

import "go.uber.org/zap"

func Print() {
	l, _ := zap.NewProduction()
	l.Error("test print")
}
<|endoftext|>


<commit_before>package accounting

import (
	"encoding/json"
	"io"
	"sync"
	"time"

	"github.com/rclone/rclone/fs"
)

// TransferSnapshot represents state of an account at point in time.
type TransferSnapshot struct {
	Name        string    `json:"name"`
	Size        int64     `json:"size"`
	Bytes       int64     `json:"bytes"`
	Checked     bool      `json:"checked"`
	StartedAt   time.Time `json:"started_at"`
	CompletedAt time.Time `json:"completed_at,omitempty"`
	Error       error     `json:"-"`
}

// MarshalJSON implements json.Marshaler interface.
func (as TransferSnapshot) MarshalJSON() ([]byte, error) {
	err := ""
	if as.Error != nil {
		err = as.Error.Error()
	}
	type Alias TransferSnapshot
	return json.Marshal(&struct {
		Error string `json:"error"`
		Alias
	}{
		Error: err,
		Alias: (Alias)(as),
	})
}

// Transfer keeps track of initiated transfers and provides access to
// accounting functions.
// Transfer needs to be closed on completion.
type Transfer struct {
	stats    *StatsInfo
	remote   string
	size     int64
	checking bool

	// Protects all bellow
	mu          sync.Mutex
	acc         *Account
	err         error
	startedAt   time.Time
	completedAt time.Time
}

// newCheckingTransfer instantiates new checking of the object.
func newCheckingTransfer(stats *StatsInfo, obj fs.Object) *Transfer {
	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true)
}

// newTransfer instantiates new transfer.
func newTransfer(stats *StatsInfo, obj fs.Object) *Transfer {
	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false)
}

func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool) *Transfer {
	tr := &Transfer{
		stats:     stats,
		remote:    remote,
		size:      size,
		startedAt: time.Now(),
		checking:  checking,
	}
	stats.AddTransfer(tr)
	return tr
}

// Done ends the transfer.
// Must be called after transfer is finished to run proper cleanups.
func (tr *Transfer) Done(err error) {
	tr.mu.Lock()
	defer tr.mu.Unlock()

	if err != nil {
		tr.stats.Error(err)
		tr.err = err
	}
	if tr.acc != nil {
		if err := tr.acc.Close(); err != nil {
			fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "can't close account: %+v\n", err)
		}
	}
	if tr.checking {
		tr.stats.DoneChecking(tr.remote)
	} else {
		tr.stats.DoneTransferring(tr.remote, err == nil)
	}

	tr.completedAt = time.Now()
}

// Account returns reader that knows how to keep track of transfer progress.
func (tr *Transfer) Account(in io.ReadCloser) *Account {
	tr.mu.Lock()
	defer tr.mu.Unlock()

	if tr.acc == nil {
		tr.acc = newAccountSizeName(tr.stats, in, tr.size, tr.remote)

	}
	return tr.acc
}

// TimeRange returns the time transfer started and ended at. If not completed
// it will return zero time for end time.
func (tr *Transfer) TimeRange() (time.Time, time.Time) {
	tr.mu.Lock()
	defer tr.mu.Unlock()
	return tr.startedAt, tr.completedAt
}

// IsDone returns true if transfer is completed.
func (tr *Transfer) IsDone() bool {
	tr.mu.Lock()
	defer tr.mu.Unlock()
	return !tr.completedAt.IsZero()
}

// Snapshot produces stats for this account at point in time.
func (tr *Transfer) Snapshot() TransferSnapshot {
	tr.mu.Lock()
	defer tr.mu.Unlock()

	var s, b int64 = tr.size, 0
	if tr.acc != nil {
		b, s = tr.acc.progress()
	}
	return TransferSnapshot{
		Name:        tr.remote,
		Checked:     tr.checking,
		Size:        s,
		Bytes:       b,
		StartedAt:   tr.startedAt,
		CompletedAt: tr.completedAt,
		Error:       tr.err,
	}
}
<commit_msg>accounting: fix locking in Transfer to avoid deadlock with --progress<commit_after>package accounting

import (
	"encoding/json"
	"io"
	"sync"
	"time"

	"github.com/rclone/rclone/fs"
)

// TransferSnapshot represents state of an account at point in time.
type TransferSnapshot struct {
	Name        string    `json:"name"`
	Size        int64     `json:"size"`
	Bytes       int64     `json:"bytes"`
	Checked     bool      `json:"checked"`
	StartedAt   time.Time `json:"started_at"`
	CompletedAt time.Time `json:"completed_at,omitempty"`
	Error       error     `json:"-"`
}

// MarshalJSON implements json.Marshaler interface.
func (as TransferSnapshot) MarshalJSON() ([]byte, error) {
	err := ""
	if as.Error != nil {
		err = as.Error.Error()
	}
	type Alias TransferSnapshot
	return json.Marshal(&struct {
		Error string `json:"error"`
		Alias
	}{
		Error: err,
		Alias: (Alias)(as),
	})
}

// Transfer keeps track of initiated transfers and provides access to
// accounting functions.
// Transfer needs to be closed on completion.
type Transfer struct {
	// these are initialised at creation and may be accessed without locking
	stats     *StatsInfo
	remote    string
	size      int64
	startedAt time.Time
	checking  bool

	// Protects all below
	mu          sync.RWMutex
	acc         *Account
	err         error
	completedAt time.Time
}

// newCheckingTransfer instantiates new checking of the object.
func newCheckingTransfer(stats *StatsInfo, obj fs.Object) *Transfer {
	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true)
}

// newTransfer instantiates new transfer.
func newTransfer(stats *StatsInfo, obj fs.Object) *Transfer {
	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false)
}

func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool) *Transfer {
	tr := &Transfer{
		stats:     stats,
		remote:    remote,
		size:      size,
		startedAt: time.Now(),
		checking:  checking,
	}
	stats.AddTransfer(tr)
	return tr
}

// Done ends the transfer.
// Must be called after transfer is finished to run proper cleanups.
func (tr *Transfer) Done(err error) {
	tr.mu.Lock()

	if err != nil {
		tr.stats.Error(err)
		tr.err = err
	}
	if tr.acc != nil {
		if err := tr.acc.Close(); err != nil {
			fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "can't close account: %+v\n", err)
		}
	}

	tr.completedAt = time.Now()

	tr.mu.Unlock()

	if tr.checking {
		tr.stats.DoneChecking(tr.remote)
	} else {
		tr.stats.DoneTransferring(tr.remote, err == nil)
	}
}

// Account returns reader that knows how to keep track of transfer progress.
func (tr *Transfer) Account(in io.ReadCloser) *Account {
	tr.mu.Lock()
	if tr.acc == nil {
		tr.acc = newAccountSizeName(tr.stats, in, tr.size, tr.remote)
	}
	tr.mu.Unlock()
	return tr.acc
}

// TimeRange returns the time transfer started and ended at. If not completed
// it will return zero time for end time.
func (tr *Transfer) TimeRange() (time.Time, time.Time) {
	tr.mu.RLock()
	defer tr.mu.RUnlock()
	return tr.startedAt, tr.completedAt
}

// IsDone returns true if transfer is completed.
func (tr *Transfer) IsDone() bool {
	tr.mu.RLock()
	defer tr.mu.RUnlock()
	return !tr.completedAt.IsZero()
}

// Snapshot produces stats for this account at point in time.
func (tr *Transfer) Snapshot() TransferSnapshot {
	tr.mu.RLock()
	defer tr.mu.RUnlock()

	var s, b int64 = tr.size, 0
	if tr.acc != nil {
		b, s = tr.acc.progress()
	}
	return TransferSnapshot{
		Name:        tr.remote,
		Checked:     tr.checking,
		Size:        s,
		Bytes:       b,
		StartedAt:   tr.startedAt,
		CompletedAt: tr.completedAt,
		Error:       tr.err,
	}
}
<|endoftext|>


<commit_before>package gateway

import (
	"bytes"
	"crypto/sha1" //nolint:gosec
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway/bridgemap"
)

// handleEventFailure handles failures and reconnects bridges.
func (r *Router) handleEventFailure(msg *config.Message) {
	if msg.Event != config.EventFailure {
		return
	}
	for _, gw := range r.Gateways {
		for _, br := range gw.Bridges {
			if msg.Account == br.Account {
				go gw.reconnectBridge(br)
				return
			}
		}
	}
}

// handleEventGetChannelMembers handles channel members
func (r *Router) handleEventGetChannelMembers(msg *config.Message) {
	if msg.Event != config.EventGetChannelMembers {
		return
	}
	for _, gw := range r.Gateways {
		for _, br := range gw.Bridges {
			if msg.Account == br.Account {
				cMembers := msg.Extra[config.EventGetChannelMembers][0].(config.ChannelMembers)
				r.logger.Debugf("Syncing channelmembers from %s", msg.Account)
				br.SetChannelMembers(&cMembers)
				return
			}
		}
	}
}

// handleEventRejoinChannels handles rejoining of channels.
func (r *Router) handleEventRejoinChannels(msg *config.Message) {
	if msg.Event != config.EventRejoinChannels {
		return
	}
	for _, gw := range r.Gateways {
		for _, br := range gw.Bridges {
			if msg.Account == br.Account {
				br.Joined = make(map[string]bool)
				if err := br.JoinChannels(); err != nil {
					r.logger.Errorf("channel join failed for %s: %s", msg.Account, err)
				}
			}
		}
	}
}

// handleFiles uploads or places all files on the given msg to the MediaServer and
// adds the new URL of the file on the MediaServer onto the given msg.
func (gw *Gateway) handleFiles(msg *config.Message) {
	reg := regexp.MustCompile("[^a-zA-Z0-9]+")

	// If we don't have a attachfield or we don't have a mediaserver configured return
	if msg.Extra == nil ||
		(gw.BridgeValues().General.MediaServerUpload == "" &&
			gw.BridgeValues().General.MediaDownloadPath == "") {
		return
	}

	// If we don't have files, nothing to upload.
	if len(msg.Extra["file"]) == 0 {
		return
	}

	for i, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		ext := filepath.Ext(fi.Name)
		fi.Name = fi.Name[0 : len(fi.Name)-len(ext)]
		fi.Name = reg.ReplaceAllString(fi.Name, "_")
		fi.Name += ext

		sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec

		if gw.BridgeValues().General.MediaServerUpload != "" {
			// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
			if err := gw.handleFilesUpload(&fi); err != nil {
				gw.logger.Error(err)
				continue
			}
		} else {
			// Use MediaServerPath. Place the file on the current filesystem.
			if err := gw.handleFilesLocal(&fi); err != nil {
				gw.logger.Error(err)
				continue
			}
		}

		// Download URL.
		durl := gw.BridgeValues().General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name

		gw.logger.Debugf("mediaserver download URL = %s", durl)

		// We uploaded/placed the file successfully. Add the SHA and URL.
		extra := msg.Extra["file"][i].(config.FileInfo)
		extra.URL = durl
		extra.SHA = sha1sum
		msg.Extra["file"][i] = extra
	}
}

// handleFilesUpload uses MediaServerUpload configuration to upload the file.
// Returns error on failure.
func (gw *Gateway) handleFilesUpload(fi *config.FileInfo) error {
	client := &http.Client{
		Timeout: time.Second * 5,
	}
	// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
	url := gw.BridgeValues().General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name

	req, err := http.NewRequest("PUT", url, bytes.NewReader(*fi.Data))
	if err != nil {
		return fmt.Errorf("mediaserver upload failed, could not create request: %#v", err)
	}

	gw.logger.Debugf("mediaserver upload url: %s", url)

	req.Header.Set("Content-Type", "binary/octet-stream")
	_, err = client.Do(req)
	if err != nil {
		return fmt.Errorf("mediaserver upload failed, could not Do request: %#v", err)
	}
	return nil
}

// handleFilesLocal use MediaServerPath configuration, places the file on the current filesystem.
// Returns error on failure.
func (gw *Gateway) handleFilesLocal(fi *config.FileInfo) error {
	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
	dir := gw.BridgeValues().General.MediaDownloadPath + "/" + sha1sum
	err := os.Mkdir(dir, os.ModePerm)
	if err != nil && !os.IsExist(err) {
		return fmt.Errorf("mediaserver path failed, could not mkdir: %s %#v", err, err)
	}

	path := dir + "/" + fi.Name
	gw.logger.Debugf("mediaserver path placing file: %s", path)

	err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
	if err != nil {
		return fmt.Errorf("mediaserver path failed, could not writefile: %s %#v", err, err)
	}
	return nil
}

// ignoreEvent returns true if we need to ignore this event for the specified destination bridge.
func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
	switch event {
	case config.EventAvatarDownload:
		// Avatar downloads are only relevant for telegram and mattermost for now
		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
			return true
		}
	case config.EventJoinLeave:
		// only relay join/part when configured
		if !dest.GetBool("ShowJoinPart") {
			return true
		}
	case config.EventTopicChange:
		// only relay topic change when used in some way on other side
		if dest.GetBool("ShowTopicChange") && dest.GetBool("SyncTopic") {
			return true
		}
	}
	return false
}

// handleMessage makes sure the message get sent to the correct bridge/channels.
// Returns an array of msg ID's
func (gw *Gateway) handleMessage(rmsg *config.Message, dest *bridge.Bridge) []*BrMsgID {
	var brMsgIDs []*BrMsgID

	// Not all bridges support "user is typing" indications so skip the message
	// if the targeted bridge does not support it.
	if rmsg.Event == config.EventUserTyping {
		if _, ok := bridgemap.UserTypingSupport[dest.Protocol]; !ok {
			return nil
		}
	}

	// if we have an attached file, or other info
	if rmsg.Extra != nil && len(rmsg.Extra[config.EventFileFailureSize]) != 0 && rmsg.Text == "" {
		return brMsgIDs
	}

	if gw.ignoreEvent(rmsg.Event, dest) {
		return brMsgIDs
	}

	// broadcast to every out channel (irc QUIT)
	if rmsg.Channel == "" && rmsg.Event != config.EventJoinLeave {
		gw.logger.Debug("empty channel")
		return brMsgIDs
	}

	// Get the ID of the parent message in thread
	var canonicalParentMsgID string
	if rmsg.ParentID != "" && dest.GetBool("PreserveThreading") {
		canonicalParentMsgID = gw.FindCanonicalMsgID(rmsg.Protocol, rmsg.ParentID)
	}

	channels := gw.getDestChannel(rmsg, *dest)
	for idx := range channels {
		channel := &channels[idx]
		msgID, err := gw.SendMessage(rmsg, dest, channel, canonicalParentMsgID)
		if err != nil {
			gw.logger.Errorf("SendMessage failed: %s", err)
			continue
		}
		if msgID == "" {
			continue
		}
		brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + msgID, channel.ID})
	}
	return brMsgIDs
}

func (gw *Gateway) handleExtractNicks(msg *config.Message) {
	var err error
	br := gw.Bridges[msg.Account]
	for _, outer := range br.GetStringSlice2D("ExtractNicks") {
		search := outer[0]
		replace := outer[1]
		msg.Username, msg.Text, err = extractNick(search, replace, msg.Username, msg.Text)
		if err != nil {
			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
			break
		}
	}
}

// extractNick searches for a username (based on "search" a regular expression).
// if this matches it extracts a nick (based on "extract" another regular expression) from text
// and replaces username with this result.
// returns error if the regexp doesn't compile.
func extractNick(search, extract, username, text string) (string, string, error) {
	re, err := regexp.Compile(search)
	if err != nil {
		return username, text, err
	}
	if re.MatchString(username) {
		re, err = regexp.Compile(extract)
		if err != nil {
			return username, text, err
		}
		res := re.FindAllStringSubmatch(text, 1)
		// only replace if we have exactly 1 match
		if len(res) > 0 && len(res[0]) == 2 {
			username = res[0][1]
			text = strings.Replace(text, res[0][0], "", 1)
		}
	}
	return username, text, nil
}
<commit_msg>Fix the behavior of ShowTopicChange and SyncTopic (#1086)<commit_after>package gateway

import (
	"bytes"
	"crypto/sha1" //nolint:gosec
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway/bridgemap"
)

// handleEventFailure handles failures and reconnects bridges.
func (r *Router) handleEventFailure(msg *config.Message) {
	if msg.Event != config.EventFailure {
		return
	}
	for _, gw := range r.Gateways {
		for _, br := range gw.Bridges {
			if msg.Account == br.Account {
				go gw.reconnectBridge(br)
				return
			}
		}
	}
}

// handleEventGetChannelMembers handles channel members
func (r *Router) handleEventGetChannelMembers(msg *config.Message) {
	if msg.Event != config.EventGetChannelMembers {
		return
	}
	for _, gw := range r.Gateways {
		for _, br := range gw.Bridges {
			if msg.Account == br.Account {
				cMembers := msg.Extra[config.EventGetChannelMembers][0].(config.ChannelMembers)
				r.logger.Debugf("Syncing channelmembers from %s", msg.Account)
				br.SetChannelMembers(&cMembers)
				return
			}
		}
	}
}

// handleEventRejoinChannels handles rejoining of channels.
func (r *Router) handleEventRejoinChannels(msg *config.Message) {
	if msg.Event != config.EventRejoinChannels {
		return
	}
	for _, gw := range r.Gateways {
		for _, br := range gw.Bridges {
			if msg.Account == br.Account {
				br.Joined = make(map[string]bool)
				if err := br.JoinChannels(); err != nil {
					r.logger.Errorf("channel join failed for %s: %s", msg.Account, err)
				}
			}
		}
	}
}

// handleFiles uploads or places all files on the given msg to the MediaServer and
// adds the new URL of the file on the MediaServer onto the given msg.
func (gw *Gateway) handleFiles(msg *config.Message) {
	reg := regexp.MustCompile("[^a-zA-Z0-9]+")

	// If we don't have a attachfield or we don't have a mediaserver configured return
	if msg.Extra == nil ||
		(gw.BridgeValues().General.MediaServerUpload == "" &&
			gw.BridgeValues().General.MediaDownloadPath == "") {
		return
	}

	// If we don't have files, nothing to upload.
	if len(msg.Extra["file"]) == 0 {
		return
	}

	for i, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		ext := filepath.Ext(fi.Name)
		fi.Name = fi.Name[0 : len(fi.Name)-len(ext)]
		fi.Name = reg.ReplaceAllString(fi.Name, "_")
		fi.Name += ext

		sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec

		if gw.BridgeValues().General.MediaServerUpload != "" {
			// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
			if err := gw.handleFilesUpload(&fi); err != nil {
				gw.logger.Error(err)
				continue
			}
		} else {
			// Use MediaServerPath. Place the file on the current filesystem.
			if err := gw.handleFilesLocal(&fi); err != nil {
				gw.logger.Error(err)
				continue
			}
		}

		// Download URL.
		durl := gw.BridgeValues().General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name

		gw.logger.Debugf("mediaserver download URL = %s", durl)

		// We uploaded/placed the file successfully. Add the SHA and URL.
		extra := msg.Extra["file"][i].(config.FileInfo)
		extra.URL = durl
		extra.SHA = sha1sum
		msg.Extra["file"][i] = extra
	}
}

// handleFilesUpload uses MediaServerUpload configuration to upload the file.
// Returns error on failure.
func (gw *Gateway) handleFilesUpload(fi *config.FileInfo) error {
	client := &http.Client{
		Timeout: time.Second * 5,
	}
	// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
	url := gw.BridgeValues().General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name

	req, err := http.NewRequest("PUT", url, bytes.NewReader(*fi.Data))
	if err != nil {
		return fmt.Errorf("mediaserver upload failed, could not create request: %#v", err)
	}

	gw.logger.Debugf("mediaserver upload url: %s", url)

	req.Header.Set("Content-Type", "binary/octet-stream")
	_, err = client.Do(req)
	if err != nil {
		return fmt.Errorf("mediaserver upload failed, could not Do request: %#v", err)
	}
	return nil
}

// handleFilesLocal use MediaServerPath configuration, places the file on the current filesystem.
// Returns error on failure.
func (gw *Gateway) handleFilesLocal(fi *config.FileInfo) error {
	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
	dir := gw.BridgeValues().General.MediaDownloadPath + "/" + sha1sum
	err := os.Mkdir(dir, os.ModePerm)
	if err != nil && !os.IsExist(err) {
		return fmt.Errorf("mediaserver path failed, could not mkdir: %s %#v", err, err)
	}

	path := dir + "/" + fi.Name
	gw.logger.Debugf("mediaserver path placing file: %s", path)

	err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
	if err != nil {
		return fmt.Errorf("mediaserver path failed, could not writefile: %s %#v", err, err)
	}
	return nil
}

// ignoreEvent returns true if we need to ignore this event for the specified destination bridge.
func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
	switch event {
	case config.EventAvatarDownload:
		// Avatar downloads are only relevant for telegram and mattermost for now
		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
			return true
		}
	case config.EventJoinLeave:
		// only relay join/part when configured
		if !dest.GetBool("ShowJoinPart") {
			return true
		}
	case config.EventTopicChange:
		// only relay topic change when used in some way on other side
		if !dest.GetBool("ShowTopicChange") && !dest.GetBool("SyncTopic") {
			return true
		}
	}
	return false
}

// handleMessage makes sure the message get sent to the correct bridge/channels.
// Returns an array of msg ID's
func (gw *Gateway) handleMessage(rmsg *config.Message, dest *bridge.Bridge) []*BrMsgID {
	var brMsgIDs []*BrMsgID

	// Not all bridges support "user is typing" indications so skip the message
	// if the targeted bridge does not support it.
	if rmsg.Event == config.EventUserTyping {
		if _, ok := bridgemap.UserTypingSupport[dest.Protocol]; !ok {
			return nil
		}
	}

	// if we have an attached file, or other info
	if rmsg.Extra != nil && len(rmsg.Extra[config.EventFileFailureSize]) != 0 && rmsg.Text == "" {
		return brMsgIDs
	}

	if gw.ignoreEvent(rmsg.Event, dest) {
		return brMsgIDs
	}

	// broadcast to every out channel (irc QUIT)
	if rmsg.Channel == "" && rmsg.Event != config.EventJoinLeave {
		gw.logger.Debug("empty channel")
		return brMsgIDs
	}

	// Get the ID of the parent message in thread
	var canonicalParentMsgID string
	if rmsg.ParentID != "" && dest.GetBool("PreserveThreading") {
		canonicalParentMsgID = gw.FindCanonicalMsgID(rmsg.Protocol, rmsg.ParentID)
	}

	channels := gw.getDestChannel(rmsg, *dest)
	for idx := range channels {
		channel := &channels[idx]
		msgID, err := gw.SendMessage(rmsg, dest, channel, canonicalParentMsgID)
		if err != nil {
			gw.logger.Errorf("SendMessage failed: %s", err)
			continue
		}
		if msgID == "" {
			continue
		}
		brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + msgID, channel.ID})
	}
	return brMsgIDs
}

func (gw *Gateway) handleExtractNicks(msg *config.Message) {
	var err error
	br := gw.Bridges[msg.Account]
	for _, outer := range br.GetStringSlice2D("ExtractNicks") {
		search := outer[0]
		replace := outer[1]
		msg.Username, msg.Text, err = extractNick(search, replace, msg.Username, msg.Text)
		if err != nil {
			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
			break
		}
	}
}

// extractNick searches for a username (based on "search" a regular expression).
// if this matches it extracts a nick (based on "extract" another regular expression) from text
// and replaces username with this result.
// returns error if the regexp doesn't compile.
func extractNick(search, extract, username, text string) (string, string, error) {
	re, err := regexp.Compile(search)
	if err != nil {
		return username, text, err
	}
	if re.MatchString(username) {
		re, err = regexp.Compile(extract)
		if err != nil {
			return username, text, err
		}
		res := re.FindAllStringSubmatch(text, 1)
		// only replace if we have exactly 1 match
		if len(res) > 0 && len(res[0]) == 2 {
			username = res[0][1]
			text = strings.Replace(text, res[0][0], "", 1)
		}
	}
	return username, text, nil
}
<|endoftext|>


<commit_before>package mangadownloader

import (
	"net/url"
	"testing"
)

func testCommonServiceManga(t *testing.T, service Service, mangaUrl *url.URL, expectedMangaName string) {
	t.Parallel()

	object, err := service.Identify(mangaUrl)
	if err != nil {
		t.Fatal(err)
	}

	manga, ok := object.(*Manga)
	if !ok {
		t.Fatal("Not a manga")
	}

	name, err := manga.Name()
	if err != nil {
		t.Fatal(err)
	}
	if name != expectedMangaName {
		t.Fatal("Unexpected name")
	}

	chapters, err := manga.Chapters()
	if err != nil {
		t.Fatal(err)
	}
	if len(chapters) == 0 {
		t.Fatal("No chapter")
	}
}

func testCommonServiceChapter(t *testing.T, service Service, chapterUrl *url.URL, expectedChapterName string) {
	t.Parallel()

	object, err := service.Identify(chapterUrl)
	if err != nil {
		t.Fatal(err)
	}

	chapter, ok := object.(*Chapter)
	if !ok {
		t.Fatal("Not a chapter")
	}

	name, err := chapter.Name()
	if err != nil {
		t.Fatal(err)
	}
	if name != expectedChapterName {
		t.Fatal("Unexpected name")
	}

	pages, err := chapter.Pages()
	if err != nil {
		t.Fatal(err)
	}
	if len(pages) == 0 {
		t.Fatal("No pages")
	}

	page := pages[0]
	_, err = page.ImageUrl()
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>disable parallel tests (doesn't work well with convey)<commit_after>package mangadownloader

import (
	"net/url"
	"testing"
)

func testCommonServiceManga(t *testing.T, service Service, mangaUrl *url.URL, expectedMangaName string) {
	//t.Parallel()

	object, err := service.Identify(mangaUrl)
	if err != nil {
		t.Fatal(err)
	}

	manga, ok := object.(*Manga)
	if !ok {
		t.Fatal("Not a manga")
	}

	name, err := manga.Name()
	if err != nil {
		t.Fatal(err)
	}
	if name != expectedMangaName {
		t.Fatal("Unexpected name")
	}

	chapters, err := manga.Chapters()
	if err != nil {
		t.Fatal(err)
	}
	if len(chapters) == 0 {
		t.Fatal("No chapter")
	}
}

func testCommonServiceChapter(t *testing.T, service Service, chapterUrl *url.URL, expectedChapterName string) {
	//t.Parallel()

	object, err := service.Identify(chapterUrl)
	if err != nil {
		t.Fatal(err)
	}

	chapter, ok := object.(*Chapter)
	if !ok {
		t.Fatal("Not a chapter")
	}

	name, err := chapter.Name()
	if err != nil {
		t.Fatal(err)
	}
	if name != expectedChapterName {
		t.Fatal("Unexpected name")
	}

	pages, err := chapter.Pages()
	if err != nil {
		t.Fatal(err)
	}
	if len(pages) == 0 {
		t.Fatal("No pages")
	}

	page := pages[0]
	_, err = page.ImageUrl()
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>