<commit_before>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dns

import (
	"errors"
	"testing"

	"github.com/jetstack/cert-manager/test/util/generate"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack/cert-manager/pkg/controller/test"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/acmedns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/azuredns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/clouddns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/cloudflare"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/route53"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/util"
)

const (
	defaultTestIssuerName      = "test-issuer"
	defaultTestIssuerKind      = v1alpha1.IssuerKind
	defaultTestNamespace       = "default"
	defaultTestCertificateName = "test-cert"
)

type solverFixture struct {
	// The Solver under test
	Solver *Solver
	*test.Builder

	// Issuer to be passed to functions on the Solver (a default will be used if nil)
	Issuer v1alpha1.GenericIssuer
	// Certificate resource to use during tests
	Certificate *v1alpha1.Certificate
	// Challenge resource to use during tests
	Challenge v1alpha1.ACMEOrderChallenge

	dnsProviders *fakeDNSProviders

	// PreFn will run before the test is run, but after the fixture has been initialised.
	// This is useful if you want to load the clientset with some resources *after* the
	// fixture has been created.
	PreFn func(*testing.T, *solverFixture)
	// CheckFn should performs checks to ensure the output of the test is as expected.
	// Optional additional values may be provided, which represent the output of the
	// function under test.
	CheckFn func(*testing.T, *solverFixture, ...interface{})
	// Err should be true if an error is expected from the function under test
	Err bool

	// testResources is used to store references to resources used or created during
	// the test.
	testResources map[string]interface{}
}

func (s *solverFixture) Setup(t *testing.T) {
	if s.Issuer == nil {
		s.Issuer = generate.Issuer(generate.IssuerConfig{
			Name:      defaultTestIssuerName,
			Namespace: defaultTestNamespace,
		})
	}
	if s.testResources == nil {
		s.testResources = map[string]interface{}{}
	}
	if s.Builder == nil {
		s.Builder = &test.Builder{}
	}
	if s.dnsProviders == nil {
		s.dnsProviders = newFakeDNSProviders()
	}
	s.Solver = buildFakeSolver(s.Builder, s.dnsProviders.constructors)
	if s.PreFn != nil {
		s.PreFn(t, s)
		s.Builder.Sync()
	}
}

func (s *solverFixture) Finish(t *testing.T, args ...interface{}) {
	defer s.Builder.Stop()
	// resync listers before running checks
	s.Builder.Sync()
	// run custom checks
	if s.CheckFn != nil {
		s.CheckFn(t, s, args...)
	}
}

func buildFakeSolver(b *test.Builder, dnsProviders dnsProviderConstructors) *Solver {
	b.Start()
	s := &Solver{
		Context:                 b.Context,
		secretLister:            b.Context.KubeSharedInformerFactory.Core().V1().Secrets().Lister(),
		dnsProviderConstructors: dnsProviders,
	}
	b.Sync()
	return s
}

func strPtr(s string) *string {
	return &s
}

type fakeDNSProviderCall struct {
	name string
	args []interface{}
}

type fakeDNSProviders struct {
	constructors dnsProviderConstructors
	calls        []fakeDNSProviderCall
}

func (f *fakeDNSProviders) call(name string, args ...interface{}) {
	f.calls = append(f.calls, fakeDNSProviderCall{name: name, args: args})
}

func newFakeDNSProviders() *fakeDNSProviders {
	f := &fakeDNSProviders{
		calls: []fakeDNSProviderCall{},
	}
	f.constructors = dnsProviderConstructors{
		cloudDNS: func(project string, serviceAccountFile string, serviceAccount []byte, dns01Nameservers []string, ambient bool) (*clouddns.DNSProvider, error) {
			f.call("clouddns", project, serviceAccountFile, serviceAccount, util.RecursiveNameservers, ambient)
			return nil, nil
		},
		cloudFlare: func(email, apikey string, dns01Nameservers []string) (*cloudflare.DNSProvider, error) {
			f.call("cloudflare", email, apikey, util.RecursiveNameservers)
			if email == "" || apikey == "" {
				return nil, errors.New("invalid email or apikey")
			}
			return nil, nil
		},
		route53: func(accessKey, secretKey, hostedZoneID, region string, ambient bool, dns01Nameservers []string) (*route53.DNSProvider, error) {
			f.call("route53", accessKey, secretKey, hostedZoneID, region, ambient, util.RecursiveNameservers)
			return nil, nil
		},
		azureDNS: func(clientID, clientSecret, subscriptionID, tenentID, resourceGroupName, hostedZoneName string, dns01Nameservers []string) (*azuredns.DNSProvider, error) {
			f.call("azuredns", clientID, clientSecret, subscriptionID, tenentID, resourceGroupName, hostedZoneName, util.RecursiveNameservers)
			return nil, nil
		},
		acmeDNS: func(host string, accountJson []byte, dns01Nameservers []string) (*acmedns.DNSProvider, error) {
			f.call("acmedns", host, accountJson, dns01Nameservers)
			return nil, nil
		},
	}
	return f
}
<commit_msg>Fix up test failure<commit_after>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dns

import (
	"errors"
	"testing"

	"github.com/jetstack/cert-manager/test/util/generate"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack/cert-manager/pkg/controller/test"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/acmedns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/azuredns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/clouddns"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/cloudflare"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/route53"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/dns/util"
)

const (
	defaultTestIssuerName      = "test-issuer"
	defaultTestIssuerKind      = v1alpha1.IssuerKind
	defaultTestNamespace       = "default"
	defaultTestCertificateName = "test-cert"
)

type solverFixture struct {
	// The Solver under test
	Solver *Solver
	*test.Builder

	// Issuer to be passed to functions on the Solver (a default will be used if nil)
	Issuer v1alpha1.GenericIssuer
	// Certificate resource to use during tests
	Certificate *v1alpha1.Certificate
	// Challenge resource to use during tests
	Challenge v1alpha1.ACMEOrderChallenge

	dnsProviders *fakeDNSProviders

	// PreFn will run before the test is run, but after the fixture has been initialised.
	// This is useful if you want to load the clientset with some resources *after* the
	// fixture has been created.
	PreFn func(*testing.T, *solverFixture)
	// CheckFn should performs checks to ensure the output of the test is as expected.
	// Optional additional values may be provided, which represent the output of the
	// function under test.
	CheckFn func(*testing.T, *solverFixture, ...interface{})
	// Err should be true if an error is expected from the function under test
	Err bool

	// testResources is used to store references to resources used or created during
	// the test.
	testResources map[string]interface{}
}

func (s *solverFixture) Setup(t *testing.T) {
	if s.Issuer == nil {
		s.Issuer = generate.Issuer(generate.IssuerConfig{
			Name:      defaultTestIssuerName,
			Namespace: defaultTestNamespace,
		})
	}
	if s.testResources == nil {
		s.testResources = map[string]interface{}{}
	}
	if s.Builder == nil {
		s.Builder = &test.Builder{}
	}
	if s.dnsProviders == nil {
		s.dnsProviders = newFakeDNSProviders()
	}
	s.Solver = buildFakeSolver(s.Builder, s.dnsProviders.constructors)
	if s.PreFn != nil {
		s.PreFn(t, s)
		s.Builder.Sync()
	}
}

func (s *solverFixture) Finish(t *testing.T, args ...interface{}) {
	defer s.Builder.Stop()
	// resync listers before running checks
	s.Builder.Sync()
	// run custom checks
	if s.CheckFn != nil {
		s.CheckFn(t, s, args...)
	}
}

func buildFakeSolver(b *test.Builder, dnsProviders dnsProviderConstructors) *Solver {
	b.Start()
	s := &Solver{
		Context:                 b.Context,
		secretLister:            b.Context.KubeSharedInformerFactory.Core().V1().Secrets().Lister(),
		dnsProviderConstructors: dnsProviders,
	}
	b.Sync()
	return s
}

func strPtr(s string) *string {
	return &s
}

type fakeDNSProviderCall struct {
	name string
	args []interface{}
}

type fakeDNSProviders struct {
	constructors dnsProviderConstructors
	calls        []fakeDNSProviderCall
}

func (f *fakeDNSProviders) call(name string, args ...interface{}) {
	f.calls = append(f.calls, fakeDNSProviderCall{name: name, args: args})
}

func newFakeDNSProviders() *fakeDNSProviders {
	f := &fakeDNSProviders{
		calls: []fakeDNSProviderCall{},
	}
	f.constructors = dnsProviderConstructors{
		cloudDNS: func(project string, serviceAccount []byte, dns01Nameservers []string, ambient bool) (*clouddns.DNSProvider, error) {
			f.call("clouddns", project, serviceAccount, util.RecursiveNameservers, ambient)
			return nil, nil
		},
		cloudFlare: func(email, apikey string, dns01Nameservers []string) (*cloudflare.DNSProvider, error) {
			f.call("cloudflare", email, apikey, util.RecursiveNameservers)
			if email == "" || apikey == "" {
				return nil, errors.New("invalid email or apikey")
			}
			return nil, nil
		},
		route53: func(accessKey, secretKey, hostedZoneID, region string, ambient bool, dns01Nameservers []string) (*route53.DNSProvider, error) {
			f.call("route53", accessKey, secretKey, hostedZoneID, region, ambient, util.RecursiveNameservers)
			return nil, nil
		},
		azureDNS: func(clientID, clientSecret, subscriptionID, tenentID, resourceGroupName, hostedZoneName string, dns01Nameservers []string) (*azuredns.DNSProvider, error) {
			f.call("azuredns", clientID, clientSecret, subscriptionID, tenentID, resourceGroupName, hostedZoneName, util.RecursiveNameservers)
			return nil, nil
		},
		acmeDNS: func(host string, accountJson []byte, dns01Nameservers []string) (*acmedns.DNSProvider, error) {
			f.call("acmedns", host, accountJson, dns01Nameservers)
			return nil, nil
		},
	}
	return f
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"time"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func dbHandler(w http.ResponseWriter, r *http.Request) {
	ssl := r.FormValue("ssl") != "false"
	service := r.FormValue("service")

	err := testDBConnection(ssl, service)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	writeJson(w, map[string]interface{}{
		"success": true,
	})
}

func testDBConnection(ssl bool, service string) error {
	var err error
	var db *sql.DB

	if service == "" {
		service = "postgres"
	}

	switch service {
	case "mysql":
		dbu := os.Getenv("DATABASE_URL")
		db, err = mysqlOpen(dbu, ssl)
		if err != nil {
			return err
		}
		return testSQLConnection(db)
	case "postgres":
		dbu := os.Getenv("DATABASE_URL")
		db, err = postgresOpen(dbu, ssl)
		if err != nil {
			return err
		}
		return testSQLConnection(db)
	case "mongodb":
		vcap_services := os.Getenv("VCAP_SERVICES")
		session, err := mongoDBOpen(vcap_services, ssl)
		if err != nil {
			return err
		}
		return testMongoDBConnection(session)
	default:
		return fmt.Errorf("unknown service: " + service)
	}
}

func testSQLConnection(db *sql.DB) error {
	defer db.Close()

	_, err := db.Exec("CREATE TABLE foo(id integer)")
	if err != nil {
		return err
	}
	defer func() {
		db.Exec("DROP TABLE foo")
	}()

	_, err = db.Exec("INSERT INTO foo VALUES(42)")
	if err != nil {
		return err
	}

	var id int
	err = db.QueryRow("SELECT * FROM foo LIMIT 1").Scan(&id)
	if err != nil {
		return err
	}
	if id != 42 {
		return fmt.Errorf("Expected 42, got %d", id)
	}

	return nil
}

func postgresOpen(dbu string, ssl bool) (*sql.DB, error) {
	u, err := url.Parse(dbu)
	if err != nil {
		return nil, err
	}

	if ssl {
		u.RawQuery = "sslmode=verify-full"
	} else {
		u.RawQuery = "sslmode=disable"
	}

	return sql.Open("postgres", u.String())
}

func mysqlOpen(dbu string, ssl bool) (*sql.DB, error) {
	u, err := url.Parse(dbu)
	if err != nil {
		return nil, err
	}

	if ssl {
		u.RawQuery = "tls=true"
	} else {
		u.RawQuery = "tls=false"
	}

	connString := fmt.Sprintf("%s@tcp(%s:%s)%s?%s", u.User.String(), u.Hostname(), u.Port(), u.EscapedPath(), u.RawQuery)

	return sql.Open("mysql", connString)
}

func testMongoDBConnection(session *mgo.Session) error {
	defer session.Close()

	type Person struct {
		Name  string
		Phone string
	}

	input := &Person{Name: "John Jones", Phone: "+447777777777"}
	db := session.DB("test").C("people")
	err := db.Insert(input)
	if err != nil {
		return err
	}

	var result Person
	err = db.Find(bson.M{"name": "John Jones"}).One(&result)
	if err != nil {
		return err
	}
	if result.Name != input.Name {
		return fmt.Errorf("Name unexpectedly changed to %s in MongoDB test.", result.Name)
	}
	if result.Phone != input.Phone {
		return fmt.Errorf("Phone unexpectedly changed to %s in MongoDB test.", result.Phone)
	}
	return nil
}

func mongoDBOpen(vcap_services_str string, ssl bool) (*mgo.Session, error) {
	type VCAPService struct {
		Credentials struct {
			URI                 string `json:"uri"`
			CACertificateBase64 string `json:"ca_certificate_base64"`
		} `json:"credentials"`
	}

	type VCAPServices struct {
		MongoDB []VCAPService `json:"mongodb"`
	}

	// Retrieve the MongoDB connection URI and the CA Certificate from the `VCAP_SERVICES`
	// environment variable.
	var vcap_services VCAPServices
	if err := json.Unmarshal([]byte(vcap_services_str), &vcap_services); err != nil {
		return nil, err
	}
	mongodb_credentials := vcap_services.MongoDB[0].Credentials
	dbu := mongodb_credentials.URI
	ca_certificate_base64 := mongodb_credentials.CACertificateBase64

	// This is work around for https://github.com/go-mgo/mgo/issues/84
	u, _ := url.Parse(dbu)
	values := u.Query()
	if _, ok := values["ssl"]; ok {
		delete(values, "ssl")
	}
	u.RawQuery = values.Encode()
	uri := u.String()

	mongourl, err := mgo.ParseURL(uri)
	if err != nil {
		return nil, err
	}

	if !ssl {
		session, err := mgo.DialWithTimeout(uri, 5 * time.Second)
		if err != nil {
			return nil, err
		}
		return session, nil
	}

	// Compose has self-signed certs for mongo. Make sure we verify it against CA certificate provided in binding.
	ca, err := base64.StdEncoding.DecodeString(ca_certificate_base64)
	if err != nil {
		return nil, err
	}
	roots := x509.NewCertPool()
	roots.AppendCertsFromPEM(ca)

	tlsConfig := &tls.Config{RootCAs: roots}
	if tlsConfig.InsecureSkipVerify {
		return nil, fmt.Errorf("Verification was skipped.")
	}

	mongourl.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
		return tls.Dial("tcp", addr.String(), tlsConfig)
	}
	mongourl.Timeout = 10 * time.Second
	session, err := mgo.DialWithInfo(mongourl)
	if err != nil {
		return nil, err
	}

	return session, nil
}
<commit_msg>Use MongoDB broker-provided DB name in Healthcheck<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"time"

	_ "github.com/go-sql-driver/mysql"
	_ "github.com/lib/pq"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func dbHandler(w http.ResponseWriter, r *http.Request) {
	ssl := r.FormValue("ssl") != "false"
	service := r.FormValue("service")

	err := testDBConnection(ssl, service)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	writeJson(w, map[string]interface{}{
		"success": true,
	})
}

func testDBConnection(ssl bool, service string) error {
	var err error
	var db *sql.DB

	if service == "" {
		service = "postgres"
	}

	switch service {
	case "mysql":
		dbu := os.Getenv("DATABASE_URL")
		db, err = mysqlOpen(dbu, ssl)
		if err != nil {
			return err
		}
		return testSQLConnection(db)
	case "postgres":
		dbu := os.Getenv("DATABASE_URL")
		db, err = postgresOpen(dbu, ssl)
		if err != nil {
			return err
		}
		return testSQLConnection(db)
	case "mongodb":
		vcap_services := os.Getenv("VCAP_SERVICES")
		credentials, err := mongoDBCredentials(vcap_services)
		if err != nil {
			return err
		}
		session, err := mongoDBOpen(credentials.URI, credentials.CACertificateBase64, ssl)
		if err != nil {
			return err
		}
		return testMongoDBConnection(session, credentials.Name)
	default:
		return fmt.Errorf("unknown service: " + service)
	}
}

func testSQLConnection(db *sql.DB) error {
	defer db.Close()

	_, err := db.Exec("CREATE TABLE foo(id integer)")
	if err != nil {
		return err
	}
	defer func() {
		db.Exec("DROP TABLE foo")
	}()

	_, err = db.Exec("INSERT INTO foo VALUES(42)")
	if err != nil {
		return err
	}

	var id int
	err = db.QueryRow("SELECT * FROM foo LIMIT 1").Scan(&id)
	if err != nil {
		return err
	}
	if id != 42 {
		return fmt.Errorf("Expected 42, got %d", id)
	}

	return nil
}

func postgresOpen(dbu string, ssl bool) (*sql.DB, error) {
	u, err := url.Parse(dbu)
	if err != nil {
		return nil, err
	}

	if ssl {
		u.RawQuery = "sslmode=verify-full"
	} else {
		u.RawQuery = "sslmode=disable"
	}

	return sql.Open("postgres", u.String())
}

func mysqlOpen(dbu string, ssl bool) (*sql.DB, error) {
	u, err := url.Parse(dbu)
	if err != nil {
		return nil, err
	}

	if ssl {
		u.RawQuery = "tls=true"
	} else {
		u.RawQuery = "tls=false"
	}

	connString := fmt.Sprintf("%s@tcp(%s:%s)%s?%s", u.User.String(), u.Hostname(), u.Port(), u.EscapedPath(), u.RawQuery)

	return sql.Open("mysql", connString)
}

func testMongoDBConnection(session *mgo.Session, db_name string) error {
	defer session.Close()

	type Person struct {
		Name  string
		Phone string
	}

	input := &Person{Name: "John Jones", Phone: "+447777777777"}
	db := session.DB(db_name).C("people")
	err := db.Insert(input)
	if err != nil {
		return err
	}

	var result Person
	err = db.Find(bson.M{"name": "John Jones"}).One(&result)
	if err != nil {
		return err
	}
	if result.Name != input.Name {
		return fmt.Errorf("Name unexpectedly changed to %s in MongoDB test.", result.Name)
	}
	if result.Phone != input.Phone {
		return fmt.Errorf("Phone unexpectedly changed to %s in MongoDB test.", result.Phone)
	}
	return nil
}

func mongoDBCredentials(vcap_services_str string) (*Credentials, error) {
	type VCAPService struct {
		Credentials Credentials `json:"credentials"`
	}

	type VCAPServices struct {
		MongoDB []VCAPService `json:"mongodb"`
	}

	// Retrieve the MongoDB connection URI and the CA Certificate from the `VCAP_SERVICES`
	// environment variable.
	var vcap_services VCAPServices
	if err := json.Unmarshal([]byte(vcap_services_str), &vcap_services); err != nil {
		return nil, err
	}
	mongodb_credentials := vcap_services.MongoDB[0].Credentials
	return &mongodb_credentials, nil
}

func mongoDBOpen(db_url string, ca_certificate_base64 string, ssl bool) (*mgo.Session, error) {
	// This is work around for https://github.com/go-mgo/mgo/issues/84
	u, _ := url.Parse(db_url)
	values := u.Query()
	if _, ok := values["ssl"]; ok {
		delete(values, "ssl")
	}
	u.RawQuery = values.Encode()
	uri := u.String()

	mongourl, err := mgo.ParseURL(uri)
	if err != nil {
		return nil, err
	}

	if !ssl {
		session, err := mgo.DialWithTimeout(uri, 5*time.Second)
		if err != nil {
			return nil, err
		}
		return session, nil
	}

	// Compose has self-signed certs for mongo. Make sure we verify it against CA certificate provided in binding.
	ca, err := base64.StdEncoding.DecodeString(ca_certificate_base64)
	if err != nil {
		return nil, err
	}
	roots := x509.NewCertPool()
	roots.AppendCertsFromPEM(ca)

	tlsConfig := &tls.Config{RootCAs: roots}
	if tlsConfig.InsecureSkipVerify {
		return nil, fmt.Errorf("Verification was skipped.")
	}

	mongourl.DialServer = func(addr *mgo.ServerAddr) (net.Conn, error) {
		return tls.Dial("tcp", addr.String(), tlsConfig)
	}
	mongourl.Timeout = 10 * time.Second
	session, err := mgo.DialWithInfo(mongourl)
	if err != nil {
		return nil, err
	}

	return session, nil
}

type Credentials struct {
	Name                string `json:"name"`
	URI                 string `json:"uri"`
	CACertificateBase64 string `json:"ca_certificate_base64"`
}
<|endoftext|>


<commit_before>package synthetictests

import (
	"context"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/openshift/origin/pkg/monitor/monitorapi"
	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// staticPodFailureRegex trying to pull out information from messages like
// `static pod lifecycle failure - static pod: "etcd" in namespace: "openshift-etcd" for revision: 6 on node: "ovirt10-gh8t5-master-2" didn't show up, waited: 2m30s`
var staticPodFailureRegex = regexp.MustCompile(
	`static pod lifecycle failure - static pod: ".*" in namespace: "(.*)" for revision: (\d) on node: "(.*)" didn't show up, waited: .*`)

type staticPodFailure struct {
	namespace      string
	node           string
	revision       int64
	failureMessage string
}

func staticPodFailureFromMessage(message string) (*staticPodFailure, error) {
	matches := staticPodFailureRegex.FindStringSubmatch(message)
	if len(matches) != 4 {
		return nil, fmt.Errorf("wrong number of matches: %v", matches)
	}
	revision, err := strconv.ParseInt(matches[2], 0, 64)
	if err != nil {
		return nil, err
	}

	return &staticPodFailure{
		namespace:      matches[1],
		node:           matches[3],
		revision:       revision,
		failureMessage: message,
	}, nil
}

func testStaticPodLifecycleFailure(events monitorapi.Intervals, kubeClientConfig *rest.Config, testSuite string) []*junitapi.JUnitTestCase {
	ctx := context.TODO()
	const testName = `[sig-node] static pods should start after being created`
	failures := []string{}

	kubeClient, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return []*junitapi.JUnitTestCase{
			{
				Name: testName,
				FailureOutput: &junitapi.FailureOutput{
					Output: err.Error(),
				},
				SystemOut: err.Error(),
			},
		}
	}

	staticPodNamespaces := []string{
		"openshift-etcd-operator",
		"openshift-kube-apiserver-operator",
		"openshift-kube-controller-manager-operator",
		"openshift-kube-scheduler-operator",
	}
	staticPodFailures := []staticPodFailure{}
	for _, ns := range staticPodNamespaces {
		// we need to get all the events from the cluster, so we cannot use the monitor events.
		events, err := kubeClient.EventsV1().Events(ns).List(ctx, metav1.ListOptions{})
		if err != nil {
			failures = append(failures, err.Error())
			continue
		}

		for _, event := range events.Items {
			if event.Reason == "OperatorStatusChanged" { // skip the clusteroperator status change event.
				continue
			}
			if !strings.Contains(event.Note, "static pod lifecycle failure") {
				continue
			}

			staticPodFailure, err := staticPodFailureFromMessage(event.Note)
			if err != nil {
				failures = append(failures, fmt.Sprintf("%v", err))
				continue
			}
			staticPodFailures = append(staticPodFailures, *staticPodFailure)
		}
	}

	// now check each failure to see if we eventually reached the level.  If we eventually reached the level
	// then don't flag it
	for _, staticPodFailure := range staticPodFailures {
		events, err := kubeClient.EventsV1().Events(staticPodFailure.namespace).List(ctx, metav1.ListOptions{})
		if err != nil {
			failures = append(failures, err.Error())
			continue
		}
		foundEventForProperRevision := false
		for _, event := range events.Items {
			isRevisionUpdate := event.Reason == "NodeCurrentRevisionChanged"
			isForNode := strings.Contains(event.Note, staticPodFailure.node)
			matches := regexp.MustCompile("to ([0-9]+) because static pod is ready").FindStringSubmatch(event.Note)
			if len(matches) == 2 {
				reachedRevision, _ := strconv.ParseInt(matches[1], 0, 64)
				if isRevisionUpdate && isForNode && reachedRevision == staticPodFailure.revision {
					// If we reach the level eventually, don't fail the test. We might choose to add an "it's slow" test, but
					// it hasn't failed. It might be possible to go directly to a later revision, and if we want to account for
					// that, the above could be changed to >= instead of equality.
					foundEventForProperRevision = true
				}
			}
		}

		if !foundEventForProperRevision {
			failures = append(failures, staticPodFailure.failureMessage)
		}
	}

	if len(failures) > 0 {
		return []*junitapi.JUnitTestCase{
			{
				Name: testName,
				FailureOutput: &junitapi.FailureOutput{
					Output: strings.Join(failures, "\n"),
				},
				SystemOut: strings.Join(failures, "\n"),
			},
		}
	}

	return []*junitapi.JUnitTestCase{
		{
			Name: testName,
		},
	}
}
<commit_msg>Try to recover from test failure add debugging message for static pod test<commit_after>package synthetictests

import (
	"context"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/openshift/origin/pkg/monitor/monitorapi"
	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// staticPodFailureRegex trying to pull out information from messages like
// `static pod lifecycle failure - static pod: "etcd" in namespace: "openshift-etcd" for revision: 6 on node: "ovirt10-gh8t5-master-2" didn't show up, waited: 2m30s`
var staticPodFailureRegex = regexp.MustCompile(
	`static pod lifecycle failure - static pod: ".*" in namespace: "(.*)" for revision: (\d) on node: "(.*)" didn't show up, waited: .*`)

type staticPodFailure struct {
	namespace      string
	node           string
	revision       int64
	failureMessage string
}

func staticPodFailureFromMessage(message string) (*staticPodFailure, error) {
	matches := staticPodFailureRegex.FindStringSubmatch(message)
	if len(matches) != 4 {
		return nil, fmt.Errorf("wrong number of matches: %v", matches)
	}
	revision, err := strconv.ParseInt(matches[2], 0, 64)
	if err != nil {
		return nil, err
	}

	return &staticPodFailure{
		namespace:      matches[1],
		node:           matches[3],
		revision:       revision,
		failureMessage: message,
	}, nil
}

func testStaticPodLifecycleFailure(events monitorapi.Intervals, kubeClientConfig *rest.Config, testSuite string) []*junitapi.JUnitTestCase {
	ctx := context.TODO()
	const testName = `[sig-node] static pods should start after being created`
	failures := []string{}

	kubeClient, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return []*junitapi.JUnitTestCase{
			{
				Name: testName,
				FailureOutput: &junitapi.FailureOutput{
					Output: err.Error(),
				},
				SystemOut: err.Error(),
			},
		}
	}

	staticPodNamespaces := []string{
		"openshift-etcd-operator",
		"openshift-kube-apiserver-operator",
		"openshift-kube-controller-manager-operator",
		"openshift-kube-scheduler-operator",
	}
	staticPodFailures := []staticPodFailure{}
	for _, ns := range staticPodNamespaces {
		// we need to get all the events from the cluster, so we cannot use the monitor events.
		events, err := kubeClient.EventsV1().Events(ns).List(ctx, metav1.ListOptions{})
		if err != nil {
			failures = append(failures, err.Error())
			continue
		}

		for _, event := range events.Items {
			if event.Reason == "OperatorStatusChanged" { // skip the clusteroperator status change event.
				continue
			}
			if !strings.Contains(event.Note, "static pod lifecycle failure") {
				continue
			}

			staticPodFailure, err := staticPodFailureFromMessage(event.Note)
			if err != nil {
				failures = append(failures, fmt.Sprintf("%v", err))
				continue
			}
			staticPodFailures = append(staticPodFailures, *staticPodFailure)
		}
	}

	// now check each failure to see if we eventually reached the level.  If we eventually reached the level
	// then don't flag it
	for _, staticPodFailure := range staticPodFailures {
		events, err := kubeClient.EventsV1().Events(staticPodFailure.namespace).List(ctx, metav1.ListOptions{})
		if err != nil {
			failures = append(failures, err.Error())
			continue
		}
		foundEventForProperRevision := false
		for _, event := range events.Items {
			isRevisionUpdate := event.Reason == "NodeCurrentRevisionChanged"
			isForNode := strings.Contains(event.Note, staticPodFailure.node)
			matches := regexp.MustCompile("to ([0-9]+) because static pod is ready").FindStringSubmatch(event.Note)
			if len(matches) == 2 {
				reachedRevision, _ := strconv.ParseInt(matches[1], 0, 64)
				if isRevisionUpdate && isForNode && reachedRevision == staticPodFailure.revision {
					// If we reach the level eventually, don't fail the test. We might choose to add an "it's slow" test, but
					// it hasn't failed. It might be possible to go directly to a later revision, and if we want to account for
					// that, the above could be changed to >= instead of equality.
					foundEventForProperRevision = true
				}
			}
		}

		// We are suspecting events API and core API are not returning the same events. Double check here.
		// For debugging purpose
		if !foundEventForProperRevision {
			coreEvents, err := kubeClient.CoreV1().Events(staticPodFailure.namespace).List(ctx, metav1.ListOptions{})
			if err == nil {
				for _, event := range coreEvents.Items {
					isRevisionUpdate := event.Reason == "NodeCurrentRevisionChanged"
					isForNode := strings.Contains(event.Message, staticPodFailure.node)
					matches := regexp.MustCompile("to ([0-9]+) because static pod is ready").FindStringSubmatch(event.Message)
					if len(matches) == 2 {
						reachedRevision, _ := strconv.ParseInt(matches[1], 0, 64)
						if isRevisionUpdate && isForNode && reachedRevision == staticPodFailure.revision {
							// Found the event in events returned from core API
							foundEventForProperRevision = true
							fmt.Printf("test %s with failure message '%s' recovered\n", testName, staticPodFailure.failureMessage)
							// Log the events API events
							eventString, err := json.Marshal(events)
							if err == nil {
								fmt.Printf("test %s recovered: corresponding events from events API %s\n", testName, eventString)
							}

							// Log the core API events
							eventString, err = json.Marshal(coreEvents)
							if err == nil {
								fmt.Printf("test %s recovered: corresponding events from core API %s\n", testName, eventString)
							}
						}
					}
				}
			}
		}

		if !foundEventForProperRevision {
			failures = append(failures, staticPodFailure.failureMessage)
		}
	}

	if len(failures) > 0 {
		return []*junitapi.JUnitTestCase{
			{
				Name: testName,
				FailureOutput: &junitapi.FailureOutput{
					Output: strings.Join(failures, "\n"),
				},
				SystemOut: strings.Join(failures, "\n"),
			},
		}
	}

	return []*junitapi.JUnitTestCase{
		{
			Name: testName,
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tamir Duberstein (tamird@gmail.com)

package protoutil_test

import (
	"fmt"
	"reflect"
	"strings"
	"testing"

	"github.com/gogo/protobuf/proto"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
)

func TestCloneProto(t *testing.T) {
	testCases := []struct {
		pb          proto.Message
		shouldPanic bool
	}{
		// Uncloneable types (all contain UUID fields).
		{&roachpb.StoreIdent{}, true},
		{&enginepb.TxnMeta{}, true},
		{&roachpb.Transaction{}, true},
		{&roachpb.Error{}, true},

		// Cloneable types.
		{&config.ZoneConfig{}, false},
	}
	for _, tc := range testCases {
		var clone proto.Message
		var panicObj interface{}
		func() {
			defer func() {
				panicObj = recover()
			}()
			clone = protoutil.Clone(tc.pb)
		}()

		if tc.shouldPanic {
			if panicObj == nil {
				t.Errorf("%T: expected panic but didn't get one", tc.pb)
			} else {
				if panicStr := fmt.Sprint(panicObj); !strings.Contains(panicStr, "attempt to clone") {
					t.Errorf("%T: got unexpected panic %s", tc.pb, panicStr)
				}
			}
		} else {
			if panicObj != nil {
				t.Errorf("%T: got unexpected panic %v", tc.pb, panicObj)
			}
		}

		if panicObj == nil {
			realClone := proto.Clone(tc.pb)
			if !reflect.DeepEqual(clone, realClone) {
				t.Errorf("%T: clone did not equal original. expected:\n%+v\ngot:\n%+v", tc.pb, realClone, clone)
			}
		}
	}
}
<commit_msg>protoutil: Expand clone_test to cover all cloned types<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tamir Duberstein (tamird@gmail.com)

package protoutil_test

import (
	"fmt"
	"reflect"
	"strings"
	"testing"

	"github.com/gogo/protobuf/proto"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
	"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
	"github.com/cockroachdb/cockroach/pkg/storage/storagebase"
	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
	"github.com/cockroachdb/cockroach/pkg/util/tracing"
)

func TestCloneProto(t *testing.T) {
	testCases := []struct {
		pb          proto.Message
		shouldPanic bool
	}{
		// Uncloneable types (all contain UUID fields).
		{&roachpb.StoreIdent{}, true},
		{&enginepb.TxnMeta{}, true},
		{&roachpb.Transaction{}, true},
		{&roachpb.Error{}, true},

		// Cloneable types. This includes all types for which a
		// protoutil.Clone call exists in the codebase as of 2016-11-21.
		{&config.ZoneConfig{}, false},
		{&gossip.Info{}, false},
		{&gossip.BootstrapInfo{}, false},
		{&tracing.SpanContextCarrier{}, false},
		{&sqlbase.IndexDescriptor{}, false},
		{&roachpb.SplitTrigger{}, false},
		{&roachpb.Value{}, false},
		{&storagebase.ReplicaState{}, false},
		{&roachpb.RangeDescriptor{}, false},
	}
	for _, tc := range testCases {
		var clone proto.Message
		var panicObj interface{}
		func() {
			defer func() {
				panicObj = recover()
			}()
			clone = protoutil.Clone(tc.pb)
		}()

		if tc.shouldPanic {
			if panicObj == nil {
				t.Errorf("%T: expected panic but didn't get one", tc.pb)
			} else {
				if panicStr := fmt.Sprint(panicObj); !strings.Contains(panicStr, "attempt to clone") {
					t.Errorf("%T: got unexpected panic %s", tc.pb, panicStr)
				}
			}
		} else {
			if panicObj != nil {
				t.Errorf("%T: got unexpected panic %v", tc.pb, panicObj)
			}
		}

		if panicObj == nil {
			realClone := proto.Clone(tc.pb)
			if !reflect.DeepEqual(clone, realClone) {
				t.Errorf("%T: clone did not equal original. expected:\n%+v\ngot:\n%+v", tc.pb, realClone, clone)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
   - support for RelayState
   - does not seem to work: incl. Capitalization content-security-policy: referrer no-referrer;
   - redo no-referer - current version does not work !!!
   - MDQ lookup by location also for hub md
   - Trusted proxy
   - wayf:wayf i hub_ops metadata
        - AttributeNameFormat for Krib -> WAYF SPS - ie if none -> repeat wayf error both formats but error
        - schacHomeOrganization
        - schacHomeOrganizationType
   - collect schema errors
   - illegal attributes from IdP - ignore or provoke error
*/

package gohybrid

import (
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"github.com/gorilla/securecookie"
	"github.com/wayf-dk/go-libxml2/types"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"sync"
)

type (
	formdata struct {
		Acs          string
		Samlresponse string
		RelayState   string
	}

	idpsppair struct {
		idp string
		sp  string
	}

	Conf struct {
		DiscoveryService        string
		Domain                  string
		HubEntityID             string
		EptidSalt               string
		HubRequestedAttributes  *goxml.Xp
		Internal, External, Hub gosaml.Md
		SecureCookieHashKey     string
		PostFormTemplate        string
		Basic2uri               map[string]string
		StdTiming               gosaml.IdAndTiming
		ElementsToSign          []string
		AttributeHandler        func(*goxml.Xp, *goxml.Xp, *goxml.Xp, *goxml.Xp) error
	}
)

const (
	idpCertQuery = `./md:IDPSSODescriptor/md:KeyDescriptor[@use="signing" or not(@use)]/ds:KeyInfo/ds:X509Data/ds:X509Certificate`
)

var (
	_     = log.Printf // For debugging; delete when done.
	_     = fmt.Printf
	remap = map[string]idpsppair{
		"https://nemlogin.wayf.dk": idpsppair{"https://saml.nemlog-in.dk", "https://nemlogin.wayf.dk"},
	}

	contextmutex sync.RWMutex
	context      = make(map[*http.Request]map[string]string)
	bify         = regexp.MustCompile("^(https?://)(.*)$")
	debify       = regexp.MustCompile("^(https?://)(?:(?:birk|krib)\\.wayf.dk/(?:birk|krib)\\.php/)(.+)$")

	postform  *template.Template
	hashKey   []byte
	seccookie *securecookie.SecureCookie
	config    = Conf{}
)

func Config(configuration Conf) {
	config = configuration
	hashKey, _ := hex.DecodeString(config.SecureCookieHashKey)
	seccookie = securecookie.New(hashKey, nil)
	postform = template.Must(template.New("post").Parse(config.PostFormTemplate))
}

func SsoService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	// handle non ok urls gracefully
	// var err error
	// check issuer and acs in md
	// receiveRequest -> request, issuer md, receiver md
	//     check for IDPList 1st in md, then in request then in query
	//     sanitize idp from query or request
	request, spmd, _, relayState, err := gosaml.ReceiveSAMLRequest(r, config.Internal, config.Hub)
	if err != nil {
		return
	}
	entityID := spmd.Query1(nil, "@entityID")
	idp := spmd.Query1(nil, "//IDPList/ProviderID") // Need to find a place for IDPList
	if idp == "" {
		idp = request.Query1(nil, "IDPList/ProviderID")
	}
	if idp == "" {
		idp = r.URL.Query().Get("idpentityid")
	}
	if idp == "" {
		data := url.Values{}
		data.Set("return", "https://"+r.Host+r.RequestURI)
		data.Set("returnIDParam", "idpentityid")
		data.Set("entityID", entityID)
		http.Redirect(w, r, config.DiscoveryService+data.Encode(), http.StatusFound)
	} else {
		var idpmd *goxml.Xp
		/**/
		// check overlap btw ad-hoc feds for the idp and the sp
		kribID := bify.ReplaceAllString(entityID, "${1}krib.wayf.dk/krib.php/$2")
		if kribID == entityID {
			kribID = "urn:oid:1.3.6.1.4.1.39153:42:" + entityID
		}

		request.QueryDashP(nil, "/saml:Issuer", kribID, nil)
		acs := request.Query1(nil, "@AssertionConsumerServiceURL")
		acsurl := bify.ReplaceAllString(acs, "${1}krib.wayf.dk/krib.php/$2")
		request.QueryDashP(nil, "@AssertionConsumerServiceURL", acsurl, nil)
		/**/
		idpmd, err = config.External.MDQ(idp)
		if err != nil {
			return
		}
		const ssoquery = "./md:IDPSSODescriptor/md:SingleSignOnService[@Binding='urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect']/@Location"
		ssoservice := idpmd.Query1(nil, ssoquery)
		if ssoservice == "" {

		}
		request.QueryDashP(nil, "@Destination", ssoservice, nil)
		u, _ := gosaml.SAMLRequest2Url(request, relayState, "", "", "")
		log.Println(request.Doc.Dump(true))
		http.Redirect(w, r, u.String(), http.StatusFound)
	}
	return
}

func BirkService(w http.ResponseWriter, r *http.Request) (err error) {
	// use incoming request for crafting the new one
	// remember to add the Scoping element to inform the IdP of requesterID - if stated in metadata for the IdP
	// check ad-hoc feds overlab
	defer r.Body.Close()
	// get the sp as well to check for allowed acs
	request, _, mdbirkidp, relayState, err := gosaml.ReceiveSAMLRequest(r, config.External, config.External)
	if err != nil {
		return
	}
	// Save the issuer and destination in a cookie for when the response comes back

	cookievalue, err := seccookie.Encode("BIRK", gosaml.Deflate(request.Doc.Dump(true)))
	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: cookievalue, Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true})

	idp := debify.ReplaceAllString(mdbirkidp.Query1(nil, "@entityID"), "$1$2")

	var mdhub, mdidp *goxml.Xp
	// are we remapping - for now only use case is https://nemlogin.wayf.dk -> https://saml.nemlog-in.dk
	if rm, ok := remap[idp]; ok {
		mdidp, err = config.Internal.MDQ(rm.idp)
		mdhub, err = config.Internal.MDQ(rm.sp)
	} else {
		mdidp, err = config.Internal.MDQ(idp)
		mdhub, err = config.Hub.MDQ(config.HubEntityID)
	}
	// use a std request - we take care of NameID etc in acsService below
	newrequest := gosaml.NewAuthnRequest(config.StdTiming.Refresh(), mdhub, mdidp)
	u, _ := gosaml.SAMLRequest2Url(newrequest, relayState, "", "", "") // not signed so blank key, pw and algo
	http.Redirect(w, r, u.String(), http.StatusFound)
	return
}

func AcsService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	birk, err := r.Cookie("BIRK")
	if err != nil {
		return err
	}

	value := []byte{}
	if err = seccookie.Decode("BIRK", birk.Value, &value); err != nil {
		return
	}

	// we checked the request when we received in birkService - we can use it without fear ie. we just parse it
	log.Println("cookie", string(gosaml.Inflate(value)))
	request := goxml.NewXp(string(gosaml.Inflate(value)))

	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: "", Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true, MaxAge: -1})
	sp_md, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/saml:Issuer"))
	if err != nil {
		return
	}

	response, idp_md, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.Internal, config.Hub)
	if err != nil {
		return
	}

	err = config.AttributeHandler(idp_md, config.HubRequestedAttributes, sp_md, response)
	if err != nil {
		return
	}

	birkmd, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/@Destination"))
	if err != nil {
		return
	}
	nameid := response.Query(nil, "./saml:Assertion/saml:Subject/saml:NameID")[0]
	// respect nameID in req, give persistent id + all computed attributes + nameformat conversion
	nameidformat := sp_md.Query1(nil, "./md:SPSSODescriptor/md:NameIDFormat")
	if nameidformat == gosaml.Persistent {
		response.QueryDashP(nameid, "@Format", gosaml.Persistent, nil)
		eptid := response.Query1(nil, `./saml:Assertion/saml:AttributeStatement/saml:Attribute[@FriendlyName="eduPersonTargetedID"]/saml:AttributeValue`)
		response.QueryDashP(nameid, ".", eptid, nil)
	} else if nameidformat == gosaml.Transient {
		response.QueryDashP(nameid, ".", gosaml.Id(), nil)
	}

	newresponse := gosaml.NewResponse(config.StdTiming.Refresh(), birkmd, sp_md, request, response)

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(newresponse, q, birkmd)
		if err != nil {
			return
		}
	}

	// when consent as a service is ready - we will post to that
	acs := newresponse.Query1(nil, "@Destination")

	data := formdata{Acs: acs, Samlresponse: base64.StdEncoding.EncodeToString([]byte(newresponse.Doc.Dump(false))), RelayState: relayState}
	postform.Execute(w, data)
	return
}

func KribService(w http.ResponseWriter, r *http.Request) (err error) {
	// check ad-hoc feds overlap
	defer r.Body.Close()

	response, _, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.External, config.External)
	if err != nil {
		return
	}
	destination := debify.ReplaceAllString(response.Query1(nil, "@Destination"), "$1$2")
	response.QueryDashP(nil, "@Destination", destination, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Subject/saml:SubjectConfirmation/saml:SubjectConfirmationData/@Recipient", destination, nil)
	issuer := config.HubEntityID
	response.QueryDashP(nil, "./saml:Issuer", issuer, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Issuer", issuer, nil)
	// Krib always receives attributes with nameformat=urn. Before sending to the real SP we need to look into
	// the metadata for SP to determine the actual nameformat - as WAYF supports both for internal SPs.
	mdsp, err := config.Internal.MDQ(destination)
	if err != nil {
		return
	}
	requestedattributes := mdsp.Query(nil, "./md:SPSSODescriptor/md:AttributeConsumingService/md:RequestedAttribute")
	attributestatement := response.Query(nil, "./saml:Assertion/saml:AttributeStatement")[0]
	for _, attr := range requestedattributes {
		nameFormat, _ := attr.(types.Element).GetAttribute("NameFormat")
		if nameFormat.NodeValue() == gosaml.Basic {
			basicname, _ := attr.(types.Element).GetAttribute("Name")
			uriname := config.Basic2uri[basicname.NodeValue()]
			responseattribute := response.Query(attributestatement, "saml:Attribute[@Name='"+uriname+"']")
			if len(responseattribute) > 0 {
				responseattribute[0].(types.Element).SetAttribute("Name", basicname.NodeValue())
				responseattribute[0].(types.Element).SetAttribute("NameFormat", gosaml.Basic)
			}
		}
	}

	mdhub, err := config.Hub.MDQ(config.HubEntityID)
	if err != nil {
		return
	}

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(response, q, mdhub)
		if err != nil {
			return
		}
	}

	data := formdata{Acs: destination, Samlresponse: base64.StdEncoding.EncodeToString([]byte(response.Doc.Dump(false))), RelayState: relayState}
	postform.Execute(w, data)
	return
}
<commit_msg>prepare for using attributerelase info template<commit_after>/*
   - support for RelayState
   - does not seem to work: incl. Capitalization content-security-policy: referrer no-referrer;
   - redo no-referer - current version does not work !!!
   - MDQ lookup by location also for hub md
   - Trusted proxy
   - wayf:wayf i hub_ops metadata
        - AttributeNameFormat for Krib -> WAYF SPS - ie if none -> repeat wayf error both formats but error
        - schacHomeOrganization
        - schacHomeOrganizationType
   - collect schema errors
   - illegal attributes from IdP - ignore or provoke error
*/

package gohybrid

import (
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/gorilla/securecookie"
	"github.com/wayf-dk/go-libxml2/types"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"sync"
)

type (
	formdata struct {
		Acs          string
		Samlresponse string
		RelayState   string
		Ard template.JS
	}

	idpsppair struct {
		idp string
		sp  string
	}

	Conf struct {
		DiscoveryService        string
		Domain                  string
		HubEntityID             string
		EptidSalt               string
		HubRequestedAttributes  *goxml.Xp
		Internal, External, Hub gosaml.Md
		SecureCookieHashKey     string
		PostFormTemplate        string
		AttributeReleaseTemplate        string
		Basic2uri               map[string]string
		StdTiming               gosaml.IdAndTiming
		ElementsToSign          []string
		AttributeHandler        func(*goxml.Xp, *goxml.Xp, *goxml.Xp, *goxml.Xp) (error, map[string][]string)
	}
)

const (
	idpCertQuery = `./md:IDPSSODescriptor/md:KeyDescriptor[@use="signing" or not(@use)]/ds:KeyInfo/ds:X509Data/ds:X509Certificate`
)

var (
	_     = log.Printf // For debugging; delete when done.
	_     = fmt.Printf
	remap = map[string]idpsppair{
		"https://nemlogin.wayf.dk": idpsppair{"https://saml.nemlog-in.dk", "https://nemlogin.wayf.dk"},
	}

	contextmutex sync.RWMutex
	context      = make(map[*http.Request]map[string]string)
	bify         = regexp.MustCompile("^(https?://)(.*)$")
	debify       = regexp.MustCompile("^(https?://)(?:(?:birk|krib)\\.wayf.dk/(?:birk|krib)\\.php/)(.+)$")

	postForm,attributeReleaseForm  *template.Template
	hashKey   []byte
	seccookie *securecookie.SecureCookie
	config    = Conf{}
)

func Config(configuration Conf) {
	config = configuration
	hashKey, _ := hex.DecodeString(config.SecureCookieHashKey)
	seccookie = securecookie.New(hashKey, nil)
	postForm = template.Must(template.New("post").Parse(config.PostFormTemplate))
	attributeReleaseForm = template.Must(template.New("post").Parse(config.AttributeReleaseTemplate))
}

func SsoService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	// handle non ok urls gracefully
	// var err error
	// check issuer and acs in md
	// receiveRequest -> request, issuer md, receiver md
	//     check for IDPList 1st in md, then in request then in query
	//     sanitize idp from query or request
	request, spmd, _, relayState, err := gosaml.ReceiveSAMLRequest(r, config.Internal, config.Hub)
	if err != nil {
		return
	}
	entityID := spmd.Query1(nil, "@entityID")
	idp := spmd.Query1(nil, "//IDPList/ProviderID") // Need to find a place for IDPList
	if idp == "" {
		idp = request.Query1(nil, "IDPList/ProviderID")
	}
	if idp == "" {
		idp = r.URL.Query().Get("idpentityid")
	}
	if idp == "" {
		data := url.Values{}
		data.Set("return", "https://"+r.Host+r.RequestURI)
		data.Set("returnIDParam", "idpentityid")
		data.Set("entityID", entityID)
		http.Redirect(w, r, config.DiscoveryService+data.Encode(), http.StatusFound)
	} else {
		var idpmd *goxml.Xp
		/**/
		// check overlap btw ad-hoc feds for the idp and the sp
		kribID := bify.ReplaceAllString(entityID, "${1}krib.wayf.dk/krib.php/$2")
		if kribID == entityID {
			kribID = "urn:oid:1.3.6.1.4.1.39153:42:" + entityID
		}

		request.QueryDashP(nil, "/saml:Issuer", kribID, nil)
		acs := request.Query1(nil, "@AssertionConsumerServiceURL")
		acsurl := bify.ReplaceAllString(acs, "${1}krib.wayf.dk/krib.php/$2")
		request.QueryDashP(nil, "@AssertionConsumerServiceURL", acsurl, nil)
		/**/
		idpmd, err = config.External.MDQ(idp)
		if err != nil {
			return
		}
		const ssoquery = "./md:IDPSSODescriptor/md:SingleSignOnService[@Binding='urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect']/@Location"
		ssoservice := idpmd.Query1(nil, ssoquery)
		if ssoservice == "" {

		}
		request.QueryDashP(nil, "@Destination", ssoservice, nil)
		u, _ := gosaml.SAMLRequest2Url(request, relayState, "", "", "")
		log.Println(request.Doc.Dump(true))
		http.Redirect(w, r, u.String(), http.StatusFound)
	}
	return
}

func BirkService(w http.ResponseWriter, r *http.Request) (err error) {
	// use incoming request for crafting the new one
	// remember to add the Scoping element to inform the IdP of requesterID - if stated in metadata for the IdP
	// check ad-hoc feds overlab
	defer r.Body.Close()
	// get the sp as well to check for allowed acs
	request, _, mdbirkidp, relayState, err := gosaml.ReceiveSAMLRequest(r, config.External, config.External)
	if err != nil {
		return
	}
	// Save the issuer and destination in a cookie for when the response comes back

	cookievalue, err := seccookie.Encode("BIRK", gosaml.Deflate(request.Doc.Dump(true)))
	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: cookievalue, Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true})

	idp := debify.ReplaceAllString(mdbirkidp.Query1(nil, "@entityID"), "$1$2")

	var mdhub, mdidp *goxml.Xp
	// are we remapping - for now only use case is https://nemlogin.wayf.dk -> https://saml.nemlog-in.dk
	if rm, ok := remap[idp]; ok {
		mdidp, err = config.Internal.MDQ(rm.idp)
		mdhub, err = config.Internal.MDQ(rm.sp)
	} else {
		mdidp, err = config.Internal.MDQ(idp)
		mdhub, err = config.Hub.MDQ(config.HubEntityID)
	}
	// use a std request - we take care of NameID etc in acsService below
	newrequest := gosaml.NewAuthnRequest(config.StdTiming.Refresh(), mdhub, mdidp)
	u, _ := gosaml.SAMLRequest2Url(newrequest, relayState, "", "", "") // not signed so blank key, pw and algo
	http.Redirect(w, r, u.String(), http.StatusFound)
	return
}

func AcsService(w http.ResponseWriter, r *http.Request) (err error) {
	defer r.Body.Close()
	birk, err := r.Cookie("BIRK")
	if err != nil {
		return err
	}

	value := []byte{}
	if err = seccookie.Decode("BIRK", birk.Value, &value); err != nil {
		return
	}

	// we checked the request when we received in birkService - we can use it without fear ie. we just parse it
	log.Println("cookie", string(gosaml.Inflate(value)))
	request := goxml.NewXp(string(gosaml.Inflate(value)))

	http.SetCookie(w, &http.Cookie{Name: "BIRK", Value: "", Domain: config.Domain, Path: "/", Secure: true, HttpOnly: true, MaxAge: -1})
	sp_md, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/saml:Issuer"))
	if err != nil {
		return
	}

	response, idp_md, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.Internal, config.Hub)
	if err != nil {
		return
	}

	err, ard := config.AttributeHandler(idp_md, config.HubRequestedAttributes, sp_md, response)
	if err != nil {
		return
	}

	birkmd, err := config.External.MDQ(request.Query1(nil, "/samlp:AuthnRequest/@Destination"))
	if err != nil {
		return
	}
	nameid := response.Query(nil, "./saml:Assertion/saml:Subject/saml:NameID")[0]
	// respect nameID in req, give persistent id + all computed attributes + nameformat conversion
	nameidformat := sp_md.Query1(nil, "./md:SPSSODescriptor/md:NameIDFormat")
	if nameidformat == gosaml.Persistent {
		response.QueryDashP(nameid, "@Format", gosaml.Persistent, nil)
		eptid := response.Query1(nil, `./saml:Assertion/saml:AttributeStatement/saml:Attribute[@FriendlyName="eduPersonTargetedID"]/saml:AttributeValue`)
		response.QueryDashP(nameid, ".", eptid, nil)
	} else if nameidformat == gosaml.Transient {
		response.QueryDashP(nameid, ".", gosaml.Id(), nil)
	}

	newresponse := gosaml.NewResponse(config.StdTiming.Refresh(), birkmd, sp_md, request, response)

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(newresponse, q, birkmd)
		if err != nil {
			return
		}
	}

	// Prepare data for the attributerelease form

	// when consent as a service is ready - we will post to that
	acs := newresponse.Query1(nil, "@Destination")

    ardjson, err := json.Marshal(ard)
	data := formdata{Acs: acs, Samlresponse: base64.StdEncoding.EncodeToString([]byte(newresponse.Doc.Dump(false))), RelayState: relayState, Ard: template.JS(ardjson)}
	attributeReleaseForm.Execute(w, data)
	return
}

func KribService(w http.ResponseWriter, r *http.Request) (err error) {
	// check ad-hoc feds overlap
	defer r.Body.Close()

	response, _, _, relayState, err := gosaml.ReceiveSAMLResponse(r, config.External, config.External)
	if err != nil {
		return
	}
	destination := debify.ReplaceAllString(response.Query1(nil, "@Destination"), "$1$2")
	response.QueryDashP(nil, "@Destination", destination, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Subject/saml:SubjectConfirmation/saml:SubjectConfirmationData/@Recipient", destination, nil)
	issuer := config.HubEntityID
	response.QueryDashP(nil, "./saml:Issuer", issuer, nil)
	response.QueryDashP(nil, "./saml:Assertion/saml:Issuer", issuer, nil)
	// Krib always receives attributes with nameformat=urn. Before sending to the real SP we need to look into
	// the metadata for SP to determine the actual nameformat - as WAYF supports both for internal SPs.
	mdsp, err := config.Internal.MDQ(destination)
	if err != nil {
		return
	}
	requestedattributes := mdsp.Query(nil, "./md:SPSSODescriptor/md:AttributeConsumingService/md:RequestedAttribute")
	attributestatement := response.Query(nil, "./saml:Assertion/saml:AttributeStatement")[0]
	for _, attr := range requestedattributes {
		nameFormat, _ := attr.(types.Element).GetAttribute("NameFormat")
		if nameFormat.NodeValue() == gosaml.Basic {
			basicname, _ := attr.(types.Element).GetAttribute("Name")
			uriname := config.Basic2uri[basicname.NodeValue()]
			responseattribute := response.Query(attributestatement, "saml:Attribute[@Name='"+uriname+"']")
			if len(responseattribute) > 0 {
				responseattribute[0].(types.Element).SetAttribute("Name", basicname.NodeValue())
				responseattribute[0].(types.Element).SetAttribute("NameFormat", gosaml.Basic)
			}
		}
	}

	mdhub, err := config.Hub.MDQ(config.HubEntityID)
	if err != nil {
		return
	}

	for _, q := range config.ElementsToSign {
		err = gosaml.SignResponse(response, q, mdhub)
		if err != nil {
			return
		}
	}

	data := formdata{Acs: destination, Samlresponse: base64.StdEncoding.EncodeToString([]byte(response.Doc.Dump(false))), RelayState: relayState}
	postForm.Execute(w, data)
	return
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"
	"sync"

	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("RuntimeKafka", func() {

	var once sync.Once
	var logger *logrus.Entry
	var vm *helpers.SSHMeta

	var allowedTopic string = "allowedTopic"
	var disallowTopic string = "disallowTopic"
	var MaxMessages int = 5

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "RuntimeKafka"})
		logger.Info("Starting")
		vm = helpers.CreateNewRuntimeHelper(helpers.Runtime, logger)
	}

	containers := func(mode string) {

		images := map[string]string{
			"zook":   "digitalwonderland/zookeeper",
			"client": "cilium/kafkaclient2",
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", "wurstmeister/kafka", helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 ", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopic := func(name string) {
		logger.Infof("Creating new kafka topic %s", name)
		res := vm.ContainerExec("client", fmt.Sprintf(
			"/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
				"--replication-factor 1 --partitions 1 --topic %s", name))
		res.ExpectSuccess()
	}

	consumer := func(topic string, maxMsg int) string {
		return fmt.Sprintf(
			"docker exec client /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
				"kafka:9092 --topic %s --max-messages %d --timeout-ms 30000", topic, maxMsg)
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i client /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, topic)
		vm.Exec(cmd)
	}

	BeforeEach(func() {
		once.Do(initialize)
		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue())
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			vm.ReportFailed()
		}
		vm.Exec("policy delete --all")
		containers("delete")
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImport(vm.GetFullPath("Policies-kafka.json"), 300)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil())
		Expect(endPoints[helpers.Enabled]).To(Equal(1))
		Expect(endPoints[helpers.Disabled]).To(Equal(2))

		createTopic(allowedTopic)
		createTopic(disallowTopic)

		res := vm.ContainerExec("client",
			"/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181")
		res.ExpectSuccess("Cannot get kafka topics")

		By("Allowed topic")
		ctx, cancel := context.WithCancel(context.Background())
		data := vm.ExecContext(ctx, consumer(allowedTopic, MaxMessages))

		//TODO: wait until ready
		helpers.Sleep(5)
		for i := 1; i <= MaxMessages; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}
		cancel()

		Expect(data.Output().String()).Should(ContainSubstring(
			"Processed a total of %d messages", MaxMessages))

		By("Disable topic")
		res = vm.Exec(consumer(disallowTopic, MaxMessages))
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})
})
<commit_msg>test/runtime: mark Kafka test as validated<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"
	"sync"

	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sirupsen/logrus"
)

var _ = Describe("RuntimeValidatedKafka", func() {

	var once sync.Once
	var logger *logrus.Entry
	var vm *helpers.SSHMeta

	var allowedTopic string = "allowedTopic"
	var disallowTopic string = "disallowTopic"
	var MaxMessages int = 5

	initialize := func() {
		logger = log.WithFields(logrus.Fields{"testName": "RuntimeKafka"})
		logger.Info("Starting")
		vm = helpers.CreateNewRuntimeHelper(helpers.Runtime, logger)
	}

	containers := func(mode string) {

		images := map[string]string{
			"zook":   "digitalwonderland/zookeeper",
			"client": "cilium/kafkaclient2",
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", "wurstmeister/kafka", helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 ", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopic := func(name string) {
		logger.Infof("Creating new kafka topic %s", name)
		res := vm.ContainerExec("client", fmt.Sprintf(
			"/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
				"--replication-factor 1 --partitions 1 --topic %s", name))
		res.ExpectSuccess()
	}

	consumer := func(topic string, maxMsg int) string {
		return fmt.Sprintf(
			"docker exec client /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
				"kafka:9092 --topic %s --max-messages %d --timeout-ms 30000", topic, maxMsg)
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i client /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, topic)
		vm.Exec(cmd)
	}

	BeforeEach(func() {
		once.Do(initialize)
		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue())
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			vm.ReportFailed()
		}
		vm.Exec("policy delete --all")
		containers("delete")
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImport(vm.GetFullPath("Policies-kafka.json"), 300)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil())
		Expect(endPoints[helpers.Enabled]).To(Equal(1))
		Expect(endPoints[helpers.Disabled]).To(Equal(2))

		createTopic(allowedTopic)
		createTopic(disallowTopic)

		res := vm.ContainerExec("client",
			"/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181")
		res.ExpectSuccess("Cannot get kafka topics")

		By("Allowed topic")
		ctx, cancel := context.WithCancel(context.Background())
		data := vm.ExecContext(ctx, consumer(allowedTopic, MaxMessages))

		//TODO: wait until ready
		helpers.Sleep(5)
		for i := 1; i <= MaxMessages; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}
		cancel()

		Expect(data.Output().String()).Should(ContainSubstring(
			"Processed a total of %d messages", MaxMessages))

		By("Disable topic")
		res = vm.Exec(consumer(disallowTopic, MaxMessages))
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})
})
<|endoftext|>


<commit_before>package gdblib

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
)

type cmdDescr struct {
	cmd      string
	response chan cmdResultRecord
}

type cmdResultRecord struct {
	id         int64
	indication string
	result     string
}

type AsyncResultRecord struct {
	Indication string
	Result     map[string]interface{}
}

type GDB struct {
	// Channel of gdb console lines
	Console chan string
	// Channel of target process lines
	Target chan string
	// Channel of internal GDB log lines
	InternalLog chan string
	// Channel of async result records
	AsyncResults chan AsyncResultRecord

	gdbCmd *exec.Cmd

	// Internal channel to send a command to the gdb interpreter
	input chan cmdDescr
	// Internal channel to send result records to callers waiting for a response
	result chan cmdResultRecord

	// Registry of command descriptors for synchronous commands
	cmdRegistry map[int64]cmdDescr
	nextId      int64
}

func convertCString(cstr string) string {
	str := cstr

	if str[0] == '"' && str[len(str)-1] == '"' {
		str = str[1 : len(str)-1]
	}
	str = strings.Replace(str, `\"`, `"`, -1)
	str = strings.Replace(str, `\n`, "\n", -1)

	return str
}

func NewGDB(program string, workingDir string) (*GDB, error) {
	gdb := &GDB{}

	gdb.gdbCmd = exec.Command("gdb", program, "-i=mi")
	gdb.gdbCmd.Dir = workingDir

	errPipe, err := gdb.gdbCmd.StderrPipe()
	outPipe, err := gdb.gdbCmd.StdoutPipe()
	inPipe, err := gdb.gdbCmd.StdinPipe()

	if err != nil {
		return nil, err
	}

	gdb.Console = make(chan string)
	gdb.Target = make(chan string)
	gdb.InternalLog = make(chan string)
	gdb.AsyncResults = make(chan AsyncResultRecord)

	gdb.input = make(chan cmdDescr)
	gdb.result = make(chan cmdResultRecord)
	gdb.cmdRegistry = make(map[int64]cmdDescr)
	gdb.nextId = 0

	writer := func() {
		for {
			select {
			case newInput := <-gdb.input:
				if newInput.response != nil {
					gdb.nextId++
					id := gdb.nextId
					gdb.cmdRegistry[id] = newInput

					inPipe.Write([]byte(strconv.FormatInt(id, 10) + newInput.cmd + "\n"))
				} else {
					inPipe.Write([]byte(newInput.cmd + "\n"))
				}
			case resultRecord := <-gdb.result:
				descriptor := gdb.cmdRegistry[resultRecord.id]

				if descriptor.cmd != "" {
					descriptor.response <- resultRecord
				}
			}
		}
	}

	reader := func(r io.ReadCloser) {
		reader := bufio.NewReader(r)
		resultRecordRegex := regexp.MustCompile(`^(\d*)\^(\S+?)(,(.*))?$`)
		asyncRecordRegex := regexp.MustCompile(`^([*=])(\S+?),(.*)$`)

		for {
			// TODO what about truncated lines, we should check isPrefix and manage the line better
			lineBytes, _, err := reader.ReadLine()
			if err != nil {
				r.Close()
				break
			}

			line := string(lineBytes)

			// TODO unescape the quotes, newlines, etc.
			// stream outputs
			if line[0] == '~' {
				line = convertCString(line[1:])
				gdb.Console <- line
			} else if line[0] == '@' {
				line = convertCString(line[1:])
				gdb.Target <- line
			} else if line[0] == '&' {
				line = convertCString(line[1:])
				gdb.InternalLog <- line + "\n"
				// result record
			} else if matches := resultRecordRegex.FindStringSubmatch(line); matches != nil {
				commandId := matches[1]
				resultIndication := matches[2]
				result := ""
				if len(matches) > 4 {
					result = matches[4]
				}

				if commandId != "" {
					id, err := strconv.ParseInt(commandId, 10, 64)

					if err == nil {
						resultRecord := cmdResultRecord{id: id, indication: resultIndication, result: result}
						gdb.result <- resultRecord
					}

					// TODO handle the parse error case
				}
				//				else {
				//					fmt.Printf("[RESULT RECORD] ID:%v %v %v\n", commandId, resultIndication, result)
				//				}
				// async record
			} else if matches := asyncRecordRegex.FindStringSubmatch(line); matches != nil {
				// recordType := matches[1]
				resultIndication := matches[2]
				result := matches[3]

				resultNode, _ := createObjectNode("{" + result + "}")
				resultObj := make(map[string]interface{})
				err := json.Unmarshal([]byte(resultNode.toJSON()), &resultObj)

				if err == nil {
					resultRecord := AsyncResultRecord{Indication: resultIndication, Result: resultObj}
					gdb.AsyncResults <- resultRecord
				} else {
					fmt.Printf("Error unmarshalling JSON for async result record: %v %v\n", err.Error(), resultNode.toJSON())
				}
				// TODO handle the parse error case
				//				fmt.Printf("[ASYNC RESULT RECORD] %v %v\n", resultIndication, result)
			} else if line == "(gdb) " {
				// This is the gdb prompt. We can just throw it out
			} else {
				//fmt.Printf("%v\n", line)
				gdb.Target <- line + "\n"
			}
		}
	}

	// TODO We probably should have a separate handler for the error pipe
	go reader(errPipe)
	go reader(outPipe)
	go writer()

	err = gdb.gdbCmd.Start()
	if err != nil {
		return nil, err
	}

	// Force GDB into asynchronous non-stop mode so that it can accept
	//  commands in the middle of execution.
	inPipe.Write([]byte("-gdb-set target-async 1\n"))
	inPipe.Write([]byte("-gdb-set non-stop on\n"))

	return gdb, nil
}

func (gdb *GDB) Wait() error {
	return gdb.gdbCmd.Wait()
}

func parseResult(result cmdResultRecord, resultObj interface{}) error {
	if result.indication == "error" {
		msg := strings.Replace(result.result, `msg="`, "", 1)
		msg = msg[:len(msg)-1]

		return errors.New(msg)
	}

	if resultObj != nil {
		//		fmt.Printf("[ORIGINAL] %v\n", result.result)

		gdbNode, _ := createObjectNode("{" + result.result + "}")
		jsonStr := gdbNode.toJSON()

		//		fmt.Printf("[JSON DUMP] %v\n", jsonStr)

		err := json.Unmarshal([]byte(jsonStr), &resultObj)
		if err != nil {
			return err
		}
	}

	return nil
}

func (gdb *GDB) GdbExit() {
	descriptor := cmdDescr{}
	descriptor.cmd = "-gdb-exit"
	descriptor.response = make(chan cmdResultRecord)
	gdb.input <- descriptor
	<-descriptor.response
}
<commit_msg>Fix data race in the process management.<commit_after>package gdblib

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

type cmdDescr struct {
	cmd      string
	response chan cmdResultRecord
}

type cmdResultRecord struct {
	id         int64
	indication string
	result     string
}

type AsyncResultRecord struct {
	Indication string
	Result     map[string]interface{}
}

type GDB struct {
	// Channel of gdb console lines
	Console chan string
	// Channel of target process lines
	Target chan string
	// Channel of internal GDB log lines
	InternalLog chan string
	// Channel of async result records
	AsyncResults chan AsyncResultRecord

	gdbCmd *exec.Cmd

	// Internal channel to send a command to the gdb interpreter
	input chan cmdDescr
	// Internal channel to send result records to callers waiting for a response
	result chan cmdResultRecord

	// Registry of command descriptors for synchronous commands
	cmdRegistry map[int64]cmdDescr
	nextId      int64
}

func convertCString(cstr string) string {
	str := cstr

	if str[0] == '"' && str[len(str)-1] == '"' {
		str = str[1 : len(str)-1]
	}
	str = strings.Replace(str, `\"`, `"`, -1)
	str = strings.Replace(str, `\n`, "\n", -1)

	return str
}

func NewGDB(program string, workingDir string) (*GDB, error) {
	gdb := &GDB{}

	gdb.gdbCmd = exec.Command("gdb", program, "-i=mi")
	gdb.gdbCmd.Dir = workingDir

	gdb.Console = make(chan string)
	gdb.Target = make(chan string)
	gdb.InternalLog = make(chan string)
	gdb.AsyncResults = make(chan AsyncResultRecord)

	gdb.input = make(chan cmdDescr)
	gdb.result = make(chan cmdResultRecord)
	gdb.cmdRegistry = make(map[int64]cmdDescr)
	gdb.nextId = 0

	wg := sync.WaitGroup{}
	wg.Add(2)

	wg2 := sync.WaitGroup{}
	wg2.Add(1)

	writer := func() {
		inPipe, err := gdb.gdbCmd.StdinPipe()

		wg2.Done()

		if err != nil {
			return
		}

		wg.Done()

		// Force GDB into asynchronous non-stop mode so that it can accept
		//  commands in the middle of execution.
		inPipe.Write([]byte("-gdb-set target-async 1\n"))
		inPipe.Write([]byte("-gdb-set non-stop on\n"))

		for {
			select {
			case newInput := <-gdb.input:
				if newInput.response != nil {
					gdb.nextId++
					id := gdb.nextId
					gdb.cmdRegistry[id] = newInput

					inPipe.Write([]byte(strconv.FormatInt(id, 10) + newInput.cmd + "\n"))
				} else {
					inPipe.Write([]byte(newInput.cmd + "\n"))
				}
			case resultRecord := <-gdb.result:
				descriptor := gdb.cmdRegistry[resultRecord.id]

				if descriptor.cmd != "" {
					descriptor.response <- resultRecord
				}
			}
		}
	}

	reader := func() {
		wg2.Wait()
		outPipe, err := gdb.gdbCmd.StdoutPipe()
		gdb.gdbCmd.StderrPipe()

		if err != nil {
			return
		}

		wg.Done()

		reader := bufio.NewReader(outPipe)
		resultRecordRegex := regexp.MustCompile(`^(\d*)\^(\S+?)(,(.*))?$`)
		asyncRecordRegex := regexp.MustCompile(`^([*=])(\S+?),(.*)$`)

		for {
			// TODO what about truncated lines, we should check isPrefix and manage the line better
			lineBytes, _, err := reader.ReadLine()
			if err != nil {
				break
			}

			line := string(lineBytes)

			// TODO unescape the quotes, newlines, etc.
			// stream outputs
			if line[0] == '~' {
				line = convertCString(line[1:])
				gdb.Console <- line
			} else if line[0] == '@' {
				line = convertCString(line[1:])
				gdb.Target <- line
			} else if line[0] == '&' {
				line = convertCString(line[1:])
				gdb.InternalLog <- line + "\n"
				// result record
			} else if matches := resultRecordRegex.FindStringSubmatch(line); matches != nil {
				commandId := matches[1]
				resultIndication := matches[2]
				result := ""
				if len(matches) > 4 {
					result = matches[4]
				}

				if commandId != "" {
					id, err := strconv.ParseInt(commandId, 10, 64)

					if err == nil {
						resultRecord := cmdResultRecord{id: id, indication: resultIndication, result: result}
						gdb.result <- resultRecord
					}

					// TODO handle the parse error case
				}
				//				else {
				//					fmt.Printf("[RESULT RECORD] ID:%v %v %v\n", commandId, resultIndication, result)
				//				}
				// async record
			} else if matches := asyncRecordRegex.FindStringSubmatch(line); matches != nil {
				// recordType := matches[1]
				resultIndication := matches[2]
				result := matches[3]

				resultNode, _ := createObjectNode("{" + result + "}")
				resultObj := make(map[string]interface{})
				err := json.Unmarshal([]byte(resultNode.toJSON()), &resultObj)

				if err == nil {
					resultRecord := AsyncResultRecord{Indication: resultIndication, Result: resultObj}
					gdb.AsyncResults <- resultRecord
				} else {
					fmt.Printf("Error unmarshalling JSON for async result record: %v %v\n", err.Error(), resultNode.toJSON())
				}
				// TODO handle the parse error case
				//				fmt.Printf("[ASYNC RESULT RECORD] %v %v\n", resultIndication, result)
			} else if line == "(gdb) " {
				// This is the gdb prompt. We can just throw it out
			} else {
				//fmt.Printf("%v\n", line)
				gdb.Target <- line + "\n"
			}
		}
	}

	go reader()
	go writer()

	wg.Wait()

	err := gdb.gdbCmd.Start()
	if err != nil {
		return nil, err
	}

	return gdb, nil
}

func (gdb *GDB) Wait() error {
	return gdb.gdbCmd.Wait()
}

func parseResult(result cmdResultRecord, resultObj interface{}) error {
	if result.indication == "error" {
		msg := strings.Replace(result.result, `msg="`, "", 1)
		msg = msg[:len(msg)-1]

		return errors.New(msg)
	}

	if resultObj != nil {
		//		fmt.Printf("[ORIGINAL] %v\n", result.result)

		gdbNode, _ := createObjectNode("{" + result.result + "}")
		jsonStr := gdbNode.toJSON()

		//		fmt.Printf("[JSON DUMP] %v\n", jsonStr)

		err := json.Unmarshal([]byte(jsonStr), &resultObj)
		if err != nil {
			return err
		}
	}

	return nil
}

func (gdb *GDB) GdbExit() {
	descriptor := cmdDescr{}
	descriptor.cmd = "-gdb-exit"
	descriptor.response = make(chan cmdResultRecord)
	gdb.input <- descriptor
	<-descriptor.response
}
<|endoftext|>


<commit_before>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main is a client to a Swarming server.
//
// The reference server python implementation documentation can be found at
// https://github.com/luci/luci-py/tree/master/appengine/swarming/doc
package main

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/maruel/subcommands"
	. "github.com/smartystreets/goconvey/convey"

	"go.chromium.org/luci/client/cmd/swarming/lib"
	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// IntegrationTestEnvVar is the name of the environment variable which controls
// whether integaration tests are executed.
// The value must be "1" for integration tests to run.
const IntegrationTestEnvVar = "INTEGRATION_TESTS"

// runIntegrationTests true if integration tests should run.
func runIntegrationTests() bool {
	return os.Getenv(IntegrationTestEnvVar) == "1"
}

// runCmd runs swarming commands appending common flags.
// It skips if integration should not run.
func runCmd(t *testing.T, cmd string, args ...string) int {
	if !runIntegrationTests() {
		t.Skipf("Skip integration tests")
	}
	args = append([]string{cmd, "-server", "chromium-swarm-dev.appspot.com"}, args...)
	return subcommands.Run(getApplication(chromeinfra.DefaultAuthOptions()), args)
}

func TestBotsCommand(t *testing.T) {
	t.Parallel()
	Convey(`ok`, t, func() {
		dir := t.TempDir()
		jsonPath := filepath.Join(dir, "out.json")

		So(runCmd(t, "bots", "-quiet", "-json", jsonPath), ShouldEqual, 0)
	})
}

func TestTasksCommand(t *testing.T) {
	t.Parallel()
	Convey(`ok`, t, func() {
		dir := t.TempDir()
		jsonPath := filepath.Join(dir, "out.json")

		So(runCmd(t, "tasks", "-limit", "1", "-quiet", "-json", jsonPath), ShouldEqual, 0)
	})
}

func TestTriggerCommand(t *testing.T) {
	// Unset SWARMING_TASK_ID otherwise the trigger may fail for parent task association.
	parentTaskID := os.Getenv(lib.TaskIDEnvVar)
	os.Unsetenv(lib.TaskIDEnvVar)
	t.Cleanup(func() {
		os.Setenv(lib.TaskIDEnvVar, parentTaskID)
	})
	Convey(`ok`, t, func() {
		args := []string{
			"-d", "pool=chromium.tests",
			"-d", "os=Linux",
			"-idempotent",
			"--", "/bin/echo", "hi",
		}
		So(runCmd(t, "trigger", args...), ShouldEqual, 0)
	})
}
<commit_msg>Revert "[swarming] Add simple integration tests"<commit_after><|endoftext|>