<commit_before>package main

import (
	"bytes"
	constraintsx509  "github.com/jcjones/constraintcrypto/x509"
	"encoding/asn1"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"github.com/mozilla/OneCRL-Tools/config"
	"github.com/mozilla/OneCRL-Tools/oneCRL"
	"os"
	"strings"
	"github.com/mozilla/OneCRL-Tools/salesforce"
	"time"
)

func getDataFromURL(url string) ([]byte, error) {
	r, _ := http.Get(url)
	defer r.Body.Close()

	return ioutil.ReadAll(r.Body)
}

func exists(item string, slice []string) bool {
	for idx := range slice {
		if slice[idx] == item {
			return true
		}
	}
	return false
}

func main() {
	filePtr := flag.String("file", "", "The file to read data from")
	exceptionsPtr := flag.String("exceptions", "exceptions.json", "A JSON document containing exceptional additions")
	urlPtr := flag.String("url", "https://ccadb-public.secure.force.com/mozilla/PublicIntermediateCertsRevokedWithPEMCSV", "the URL of the salesforce data")
	bugPtr := flag.String("bug", "", "the URL of the bug relating to this change")
	whoPtr := flag.String("who", "", "who made this change")
	whyPtr := flag.String("why", "", "why is this change happening")

	config.DefineFlags()

	flag.Parse()

	conf := config.GetConfig()

	var stream io.ReadCloser

	// make a slice of additions
	additions := new(oneCRL.Records)

	if "" != *filePtr {
		fmt.Printf("loading salesforce data from %s\n", *filePtr)
		// get the stream from a file
		csvfile, err := os.Open(*filePtr)
		if err != nil {
			fmt.Printf("problem loading salesforce data from file %s\n", err)
			return
		}

		stream = io.ReadCloser(csvfile)
	} else {
		fmt.Printf("loading salesforce data from %s\n", *urlPtr)

		// get the stream from URL
		r, err := http.Get(*urlPtr)
		if err != nil {
			fmt.Printf("problem fetching salesforce data from URL %s\n", err)
			return
		}
		defer r.Body.Close()

		stream = r.Body
	}

	existing, err:= oneCRL.FetchExistingRevocations(conf.KintoCollectionURL + "/records")
	if nil != err{
		fmt.Printf("%s\n", err)
	}

	if len(*exceptionsPtr) != 0 {
		res := new(oneCRL.Records)
		data, err := ioutil.ReadFile(*exceptionsPtr)
		if nil != err {
			fmt.Printf("problem loading oneCRL exceptions from file %s\n", err)
		}
		json.Unmarshal(data, res)

		for idx := range res.Data {
			record := res.Data[idx]
			if !exists(oneCRL.StringFromRecord(record), existing) {
				additions.Data = append(additions.Data, record)
			}
		}
	}

	row := 1
	revoked := salesforce.FetchRevokedCertInfo(stream)

	for _, each := range revoked {
		row++

		if each.Status == "Ready to Add" {
			certData, err:= salesforce.CertDataFromSalesforcePEM(each.PEM)
			if err != nil {
				fmt.Printf("(%d, %s, %s) can't decode PEM %s\n", row, each.CSN, each.CertName, each.PEM)
			}

			cert, err := constraintsx509.ParseCertificate(certData)
			if err != nil {
				fmt.Printf("(%d, %s, %s) could not parse cert\n", row, each.CSN, each.CertName)
				continue
			}

			issuerString := base64.StdEncoding.EncodeToString(cert.RawIssuer)
			serialBytes, err := asn1.Marshal(cert.SerialNumber)

			if err != nil {
				fmt.Printf("(%d, %s, %s) could not marshal serial number\n", row, each.CSN, each.CertName)
				continue
			}

			serialString := base64.StdEncoding.EncodeToString(serialBytes[2:])

			stringRep := oneCRL.StringFromIssuerSerial(issuerString, serialString)
			if exists(stringRep, existing) {
				fmt.Printf("(%d, %s, %s) revocation already in OneCRL\n", row, each.CSN, each.CertName)
				continue
			}

			matchFound := false
		    lineErrors := ""
			lineWarnings := ""
			for _, CRLLocation := range each.CRLs {
				if 0 != strings.Index(strings.Trim(CRLLocation, " "), "http") {
					if (len(strings.Trim(CRLLocation, " ")) > 0) {
						lineErrors += fmt.Sprintf("Ignoring CRL at %s because it doesn't look like an HTTP url\n", CRLLocation)
					}
					continue
				}

				// fetch and parse the CRL
				res, err := http.Get(CRLLocation)
				if err != nil {
					lineErrors += fmt.Sprintf("There was a problem fetching the CRL from %s\n", CRLLocation)
					continue
				}

				buf := new(bytes.Buffer)
				_, err = buf.ReadFrom(res.Body)
				if err != nil {
					fmt.Println("Problem reading the CRL data at %s\n", CRLLocation)
					continue
				}
				crlData := buf.Bytes()

				// Maybe the CRL is PEM - try to parse as PEM and just use the raw
				// data if that fails
				pemBlock, _ := pem.Decode(crlData)
				if pemBlock != nil {
					crlData = pemBlock.Bytes
				}

				crl, err := constraintsx509.ParseCRL(crlData)
				if err != nil {
					lineErrors += fmt.Sprintf("Could not parse the CRL at \"%s\" %v\n",
											  CRLLocation, err)
					continue
				}

				// check the CRL is still current
				if crl.HasExpired(time.Now()) {
					lineErrors += fmt.Sprintf("crl %s has expired\n", CRLLocation)
					continue
				}

				// Check the cert issuer and the CRL issuer match
				crlIssuerBytes, err := asn1.Marshal(crl.TBSCertList.Issuer)
				if nil != err {
					lineErrors += fmt.Sprintf("could not marshal CRL issuer %s\n", CRLLocation)
				}

				readibleCertIssuer, err := oneCRL.DNToRFC4514(issuerString)
				if nil != err {
					lineErrors += fmt.Sprintf("could not make readible issuer %s\n", CRLLocation)
					continue
				}

				if ! (oneCRL.ByteArrayEquals(cert.RawIssuer, crlIssuerBytes)) {
					if ! oneCRL.NamesDataMatches(cert.RawIssuer, crlIssuerBytes) {
						lineErrors += fmt.Sprintf("CRL issuer from CRL at %s does not match issuer\n%s !=\n%s\nCRL issuer:  %s\nCert issuer: %s\n", CRLLocation,
						hex.EncodeToString(crlIssuerBytes),
						hex.EncodeToString(cert.RawIssuer),
						oneCRL.RFC4514ish(crl.TBSCertList.Issuer),
						readibleCertIssuer)
						continue
					} else {
						lineWarnings += fmt.Sprintf("Warning: CRL issuer from CRL at %s does not match issuer\n%s !=\n%s\nCRL issuer:  %s\nCert issuer: %s\n", CRLLocation,
						hex.EncodeToString(crlIssuerBytes),
						hex.EncodeToString(cert.RawIssuer),
						oneCRL.RFC4514ish(crl.TBSCertList.Issuer),
						readibleCertIssuer)
					}
				}

				for revoked := range crl.TBSCertList.RevokedCertificates {
					certEntry := crl.TBSCertList.RevokedCertificates[revoked]
					serialBytesFromCRL, _ := asn1.Marshal(certEntry.SerialNumber)
					if oneCRL.ByteArrayEquals(serialBytes, serialBytesFromCRL) {
						matchFound = true
					}
				}
			}
			if !matchFound {
				if lineErrors == "" {
					lineErrors = "\n"
			    }
				fmt.Printf("(%d, %s, %s) no match found in CRL: %s", row, each.CSN, each.CertName, lineErrors)
				continue
			}

			if len(lineWarnings) != 0 {
				fmt.Printf(lineWarnings)
			}

			// record the entry for output later
			rec := oneCRL.Record{}
			rec.IssuerName = issuerString
			rec.SerialNumber = serialString
			rec.Details.Bug = *bugPtr
			rec.Details.Who = *whoPtr
			rec.Details.Why = *whyPtr
			rec.Enabled = true
			additions.Data = append(additions.Data, rec)
		}
	}

	err = oneCRL.AddEntries(additions, true)
	if nil != err {
		panic(err)
	}
}
<commit_msg>20170920 exceptions via https (#67)<commit_after>package main

import (
	"bytes"
	constraintsx509  "github.com/jcjones/constraintcrypto/x509"
	"encoding/asn1"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"github.com/mozilla/OneCRL-Tools/config"
	"github.com/mozilla/OneCRL-Tools/oneCRL"
	"os"
	"strings"
	"github.com/mozilla/OneCRL-Tools/salesforce"
	"time"
)

func getDataFromURL(url string) ([]byte, error) {
	r, _ := http.Get(url)
	defer r.Body.Close()

	return ioutil.ReadAll(r.Body)
}

func exists(item string, slice []string) bool {
	for idx := range slice {
		if slice[idx] == item {
			return true
		}
	}
	return false
}

func LoadExceptions(location string, existing []string, records *oneCRL.Records) error {
	res := new(oneCRL.Records)
	var data []byte

	if 0 != strings.Index(strings.ToUpper(location), "HTTP") {
		// if it's not an HTTP URL, attempt to load from a file
		if fileData, err := ioutil.ReadFile(location); nil != err {
			fmt.Printf("problem loading oneCRL exceptions from file %s\n", err)
		} else {
			data = fileData
		}
	} else {
		// ensure it's not an HTTP location
		if 0 != strings.Index(strings.ToUpper(location), "HTTPS") {
			return errors.New("Cowardly refusing to load exceptions from a non HTTPS location")
		}
		if resp, err := http.Get(location); nil != err {
			return err;
		} else {
			defer resp.Body.Close()
			if urlData, err := ioutil.ReadAll(resp.Body); nil != err {
				return err;
			} else {
				data = urlData
			}
		}
	}

	if err := json.Unmarshal(data, res); nil != err {
		return err
	}

	for idx := range res.Data {
		record := res.Data[idx]
		if !exists(oneCRL.StringFromRecord(record), existing) {
			records.Data = append(records.Data, record)
		}
	}
	return nil
}

func main() {
	filePtr := flag.String("file", "", "The file to read data from")
	exceptionsPtr := flag.String("exceptions", "exceptions.json", "A JSON document containing exceptional additions")
	urlPtr := flag.String("url", "https://ccadb-public.secure.force.com/mozilla/PublicIntermediateCertsRevokedWithPEMCSV", "the URL of the salesforce data")
	bugPtr := flag.String("bug", "", "the URL of the bug relating to this change")
	whoPtr := flag.String("who", "", "who made this change")
	whyPtr := flag.String("why", "", "why is this change happening")

	config.DefineFlags()

	flag.Parse()

	conf := config.GetConfig()

	var stream io.ReadCloser

	// make a slice of additions
	additions := new(oneCRL.Records)

	if "" != *filePtr {
		fmt.Printf("loading salesforce data from %s\n", *filePtr)
		// get the stream from a file
		csvfile, err := os.Open(*filePtr)
		if err != nil {
			fmt.Printf("problem loading salesforce data from file %s\n", err)
			return
		}

		stream = io.ReadCloser(csvfile)
	} else {
		fmt.Printf("loading salesforce data from %s\n", *urlPtr)

		// get the stream from URL
		r, err := http.Get(*urlPtr)
		if err != nil {
			fmt.Printf("problem fetching salesforce data from URL %s\n", err)
			return
		}
		defer r.Body.Close()

		stream = r.Body
	}

	existing, err:= oneCRL.FetchExistingRevocations(conf.KintoCollectionURL + "/records")
	if nil != err{
		fmt.Printf("%s\n", err)
	}

	if len(*exceptionsPtr) != 0 {
		if err := LoadExceptions(*exceptionsPtr, existing, additions); nil != err {
			panic(err)
		}
	}

	row := 1
	revoked := salesforce.FetchRevokedCertInfo(stream)

	for _, each := range revoked {
		row++

		if each.Status == "Ready to Add" {
			certData, err:= salesforce.CertDataFromSalesforcePEM(each.PEM)
			if err != nil {
				fmt.Printf("(%d, %s, %s) can't decode PEM %s\n", row, each.CSN, each.CertName, each.PEM)
			}

			cert, err := constraintsx509.ParseCertificate(certData)
			if err != nil {
				fmt.Printf("(%d, %s, %s) could not parse cert\n", row, each.CSN, each.CertName)
				continue
			}

			issuerString := base64.StdEncoding.EncodeToString(cert.RawIssuer)
			serialBytes, err := asn1.Marshal(cert.SerialNumber)

			if err != nil {
				fmt.Printf("(%d, %s, %s) could not marshal serial number\n", row, each.CSN, each.CertName)
				continue
			}

			serialString := base64.StdEncoding.EncodeToString(serialBytes[2:])

			stringRep := oneCRL.StringFromIssuerSerial(issuerString, serialString)
			if exists(stringRep, existing) {
				fmt.Printf("(%d, %s, %s) revocation already in OneCRL\n", row, each.CSN, each.CertName)
				continue
			}

			matchFound := false
		    lineErrors := ""
			lineWarnings := ""
			for _, CRLLocation := range each.CRLs {
				if 0 != strings.Index(strings.Trim(CRLLocation, " "), "http") {
					if (len(strings.Trim(CRLLocation, " ")) > 0) {
						lineErrors += fmt.Sprintf("Ignoring CRL at %s because it doesn't look like an HTTP url\n", CRLLocation)
					}
					continue
				}

				// fetch and parse the CRL
				res, err := http.Get(CRLLocation)
				if err != nil {
					lineErrors += fmt.Sprintf("There was a problem fetching the CRL from %s\n", CRLLocation)
					continue
				}

				buf := new(bytes.Buffer)
				_, err = buf.ReadFrom(res.Body)
				if err != nil {
					fmt.Println("Problem reading the CRL data at %s\n", CRLLocation)
					continue
				}
				crlData := buf.Bytes()

				// Maybe the CRL is PEM - try to parse as PEM and just use the raw
				// data if that fails
				pemBlock, _ := pem.Decode(crlData)
				if pemBlock != nil {
					crlData = pemBlock.Bytes
				}

				crl, err := constraintsx509.ParseCRL(crlData)
				if err != nil {
					lineErrors += fmt.Sprintf("Could not parse the CRL at \"%s\" %v\n",
											  CRLLocation, err)
					continue
				}

				// check the CRL is still current
				if crl.HasExpired(time.Now()) {
					lineErrors += fmt.Sprintf("crl %s has expired\n", CRLLocation)
					continue
				}

				// Check the cert issuer and the CRL issuer match
				crlIssuerBytes, err := asn1.Marshal(crl.TBSCertList.Issuer)
				if nil != err {
					lineErrors += fmt.Sprintf("could not marshal CRL issuer %s\n", CRLLocation)
				}

				readibleCertIssuer, err := oneCRL.DNToRFC4514(issuerString)
				if nil != err {
					lineErrors += fmt.Sprintf("could not make readible issuer %s\n", CRLLocation)
					continue
				}

				if ! (oneCRL.ByteArrayEquals(cert.RawIssuer, crlIssuerBytes)) {
					if ! oneCRL.NamesDataMatches(cert.RawIssuer, crlIssuerBytes) {
						lineErrors += fmt.Sprintf("CRL issuer from CRL at %s does not match issuer\n%s !=\n%s\nCRL issuer:  %s\nCert issuer: %s\n", CRLLocation,
						hex.EncodeToString(crlIssuerBytes),
						hex.EncodeToString(cert.RawIssuer),
						oneCRL.RFC4514ish(crl.TBSCertList.Issuer),
						readibleCertIssuer)
						continue
					} else {
						lineWarnings += fmt.Sprintf("Warning: CRL issuer from CRL at %s does not match issuer\n%s !=\n%s\nCRL issuer:  %s\nCert issuer: %s\n", CRLLocation,
						hex.EncodeToString(crlIssuerBytes),
						hex.EncodeToString(cert.RawIssuer),
						oneCRL.RFC4514ish(crl.TBSCertList.Issuer),
						readibleCertIssuer)
					}
				}

				for revoked := range crl.TBSCertList.RevokedCertificates {
					certEntry := crl.TBSCertList.RevokedCertificates[revoked]
					serialBytesFromCRL, _ := asn1.Marshal(certEntry.SerialNumber)
					if oneCRL.ByteArrayEquals(serialBytes, serialBytesFromCRL) {
						matchFound = true
					}
				}
			}
			if !matchFound {
				if lineErrors == "" {
					lineErrors = "\n"
			    }
				fmt.Printf("(%d, %s, %s) no match found in CRL: %s", row, each.CSN, each.CertName, lineErrors)
				continue
			}

			if len(lineWarnings) != 0 {
				fmt.Printf(lineWarnings)
			}

			// record the entry for output later
			rec := oneCRL.Record{}
			rec.IssuerName = issuerString
			rec.SerialNumber = serialString
			rec.Details.Bug = *bugPtr
			rec.Details.Who = *whoPtr
			rec.Details.Why = *whyPtr
			rec.Enabled = true
			additions.Data = append(additions.Data, rec)
		}
	}

	err = oneCRL.AddEntries(additions, true)
	if nil != err {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package multiverse

import (
	"os"
	//"runtime"
	//"sync"
	"testing"
)

func openTestingMultiverse() Multiverse {
	file, _ := os.Open("/home/rwertman/.diorite/multiverse.mtg")
	m, _ := Read(file)

	m.initialize()
	return m
}

/*
func TestNameIsResult(t *testing.T) {
	m := openTestingMultiverse()
	var wait sync.WaitGroup
	groups := runtime.GOMAXPROCS(-1)
	wait.Add(groups)

	totalCards := m.Cards.List.Len()
	groupInterval := totalCards / groups

	for i := 0; i < groups; i++ {
		start := i * groupInterval
		end := start + groupInterval
		if i == groups-1 {
			end = totalCards
		}
		go func(list []scrubbedCard) {
			defer wait.Done()
			for _, card := range list {
				if results := m.FuzzyNameSearch(card.Card.Name, 1); results[0].Name != card.Card.Name {
					t.Errorf("Searching for '%s' should return that card. Got: %s\n", card.Card.Name, results[0].Name)
				}
			}
		}(m.Cards.List[start:end])
	}
	wait.Wait()
}
*/
func TestPhoneticCorrection(t *testing.T) {
	m := openTestingMultiverse()

	if results := m.FuzzyNameSearch("asheok", 1); results[0].Name != "Ashiok, Nightmare Weaver" {
		t.Errorf("Searching for 'asheok' should return Ashiok, Nightmare Weaver. Got: %s\n", results[0].Name)
	}
}

func BenchmarkSingleWordFuzzyNameSearch(b *testing.B) {
	m := openTestingMultiverse()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("asheok", 15)
	}
}

func BenchmarkMultiplePartialWordFuzzyNameSearch(b *testing.B) {
	m := openTestingMultiverse()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("ava me", 15)
	}
}

func BenchmarkSingleLetterFuzzyNameSearch(b *testing.B) {
	m := openTestingMultiverse()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("d", 15)
	}
}

func BenchmarkFullNameFuzzyNameSearch(b *testing.B) {
	m := openTestingMultiverse()
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("Might of Old Krosa", 15)
	}
}
<commit_msg>Adding built in test data.<commit_after>package multiverse

import (
	"testing"
)

/*
func TestNameIsResult(t *testing.T) {
	m := openTestingMultiverse()
	var wait sync.WaitGroup
	groups := runtime.GOMAXPROCS(-1)
	wait.Add(groups)

	totalCards := m.Cards.List.Len()
	groupInterval := totalCards / groups

	for i := 0; i < groups; i++ {
		start := i * groupInterval
		end := start + groupInterval
		if i == groups-1 {
			end = totalCards
		}
		go func(list []scrubbedCard) {
			defer wait.Done()
			for _, card := range list {
				if results := m.FuzzyNameSearch(card.Card.Name, 1); results[0].Name != card.Card.Name {
					t.Errorf("Searching for '%s' should return that card. Got: %s\n", card.Card.Name, results[0].Name)
				}
			}
		}(m.Cards.List[start:end])
	}
	wait.Wait()
}
*/

func TestPhoneticCorrection(t *testing.T) {
	m := getTestingMultiverse(t)

	if results := m.FuzzyNameSearch("asheok", 1); results[0].Name != "Ashiok, Nightmare Weaver" {
		t.Errorf("Searching for 'asheok' should return Ashiok, Nightmare Weaver. Got: %s\n", results[0].Name)
	}
}

func BenchmarkSingleWordFuzzyNameSearch(b *testing.B) {
	m := getTestingMultiverse(nil)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("asheok", 15)
	}
}

func BenchmarkMultiplePartialWordFuzzyNameSearch(b *testing.B) {
	m := getTestingMultiverse(nil)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("ava me", 15)
	}
}

func BenchmarkSingleLetterFuzzyNameSearch(b *testing.B) {
	m := getTestingMultiverse(nil)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("d", 15)
	}
}

func BenchmarkFullNameFuzzyNameSearch(b *testing.B) {
	m := getTestingMultiverse(nil)
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m.FuzzyNameSearch("Might of Old Krosa", 15)
	}
}
<|endoftext|>


<commit_before>package torrent

import (
	"bytes"
	"crypto/sha1" // nolint: gosec
	"errors"
	"fmt"

	"github.com/cenkalti/rain/torrent/bitfield"
	"github.com/cenkalti/rain/torrent/internal/peer"
	"github.com/cenkalti/rain/torrent/internal/peerconn/peerreader"
	"github.com/cenkalti/rain/torrent/internal/peerconn/peerwriter"
	"github.com/cenkalti/rain/torrent/internal/peerprotocol"
	"github.com/cenkalti/rain/torrent/internal/piecewriter"
	"github.com/cenkalti/rain/torrent/internal/tracker"
	"github.com/cenkalti/rain/torrent/metainfo"
)

func (t *Torrent) handlePeerMessage(pm peer.Message) {
	pe := pm.Peer
	switch msg := pm.Message.(type) {
	case peerprotocol.HaveMessage:
		// Save have messages for processesing later received while we don't have info yet.
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("unexpected piece index:", msg.Index)
			t.closePeer(pe)
			break
		}
		pi := &t.pieces[msg.Index]
		// pe.Logger().Debug("Peer ", pe.String(), " has piece #", pi.Index)
		t.piecePicker.HandleHave(pe, pi.Index)
		t.updateInterestedState(pe)
		t.startPieceDownloaders()
	case peerprotocol.BitfieldMessage:
		// Save bitfield messages while we don't have info yet.
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		bf, err := bitfield.NewBytes(msg.Data, t.info.NumPieces)
		if err != nil {
			pe.Logger().Errorln(err)
			t.closePeer(pe)
			break
		}
		pe.Logger().Debugln("Received bitfield:", bf.Hex())
		for i := uint32(0); i < bf.Len(); i++ {
			if bf.Test(i) {
				t.piecePicker.HandleHave(pe, i)
			}
		}
		t.updateInterestedState(pe)
		t.startPieceDownloaders()
	case peerprotocol.HaveAllMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		for _, pi := range t.pieces {
			t.piecePicker.HandleHave(pe, pi.Index)
		}
		t.updateInterestedState(pe)
		t.startPieceDownloaders()
	case peerprotocol.HaveNoneMessage:
	case peerprotocol.AllowedFastMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid allowed fast piece index:", msg.Index)
			t.closePeer(pe)
			break
		}
		pi := &t.pieces[msg.Index]
		pe.Logger().Debug("Peer ", pe.String(), " has allowed fast for piece #", pi.Index)
		t.piecePicker.HandleAllowedFast(pe, msg.Index)
	case peerprotocol.UnchokeMessage:
		pe.PeerChoking = false
		if pd, ok := t.pieceDownloaders[pe]; ok {
			pd.RequestBlocks(t.config.RequestQueueLength)
		}
		t.startPieceDownloaders()
	case peerprotocol.ChokeMessage:
		pe.PeerChoking = true
		if pd, ok := t.pieceDownloaders[pe]; ok {
			pd.Choked()
			t.pieceDownloadersChoked[pe] = pd
			t.startPieceDownloaders()
		}
	case peerprotocol.InterestedMessage:
		pe.PeerInterested = true
	case peerprotocol.NotInterestedMessage:
		pe.PeerInterested = false
	case peerreader.Piece:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("piece received but we don't have info")
			t.closePeer(pe)
			break
		}
		if msg.Index >= uint32(len(t.pieces)) {
			pe.Logger().Errorln("invalid piece index:", msg.Index)
			t.closePeer(pe)
			break
		}
		piece := &t.pieces[msg.Index]
		block := piece.Blocks.Find(msg.Begin, uint32(len(msg.Data)))
		if block == nil {
			pe.Logger().Errorln("invalid piece begin:", msg.Begin, "length:", len(msg.Data))
			t.closePeer(pe)
			break
		}
		pe.BytesDownlaodedInChokePeriod += int64(len(msg.Data))
		t.bytesDownloaded += int64(len(msg.Data))
		pd, ok := t.pieceDownloaders[pe]
		if !ok {
			t.bytesWasted += int64(len(msg.Data))
			break
		}
		if pd.Piece.Index != msg.Index {
			t.bytesWasted += int64(len(msg.Data))
			break
		}
		pd.GotBlock(block, msg.Data)
		peerreader.PiecePool.Put(msg.Data)
		if !pd.Done() {
			pd.RequestBlocks(t.config.RequestQueueLength)
			pe.ResetSnubTimer()
			break
		}
		// t.log.Debugln("piece download completed. index:", pd.Piece.Index)
		t.closePieceDownloader(pd)
		pe.StopSnubTimer()

		ok = piece.VerifyHash(pd.Buffer[:pd.Piece.Length], sha1.New()) // nolint: gosec
		if !ok {
			// TODO ban peers that sent corrupt piece
			t.log.Error("received corrupt piece")
			t.closePeer(pd.Peer)
			t.startPieceDownloaders()
			break
		}

		for pe := range t.piecePicker.RequestedPeers(piece.Index) {
			pd2 := t.pieceDownloaders[pe]
			t.closePieceDownloader(pd2)
			pd2.CancelPending()
		}

		if piece.Writing {
			panic("piece already writing")
		}
		piece.Writing = true

		pw := piecewriter.New(piece, pd.Buffer, pd.Piece.Length)
		t.pieceWriters[pw] = struct{}{}
		go pw.Run(t.pieceWriterResultC)

		t.startPieceDownloaders()
	case peerprotocol.RequestMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("request received but we don't have info")
			t.closePeer(pe)
			break
		}
		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid request index:", msg.Index)
			t.closePeer(pe)
			break
		}
		if msg.Begin+msg.Length > t.pieces[msg.Index].Length {
			pe.Logger().Errorln("invalid request length:", msg.Length)
			t.closePeer(pe)
			break
		}
		pi := &t.pieces[msg.Index]
		if pe.AmChoking {
			if pe.FastExtension {
				m := peerprotocol.RejectMessage{RequestMessage: msg}
				pe.SendMessage(m)
			}
		} else {
			pe.SendPiece(msg, pi.Data)
		}
	case peerprotocol.RejectMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("reject received but we don't have info")
			t.closePeer(pe)
			break
		}

		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid reject index:", msg.Index)
			t.closePeer(pe)
			break
		}
		piece := &t.pieces[msg.Index]
		block := piece.Blocks.Find(msg.Begin, msg.Length)
		if block == nil {
			pe.Logger().Errorln("invalid reject begin:", msg.Begin, "length:", msg.Length)
			t.closePeer(pe)
			break
		}
		pd, ok := t.pieceDownloaders[pe]
		if ok {
			pd.Rejected(block)
		}
	case peerprotocol.CancelMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("cancel received but we don't have info")
			t.closePeer(pe)
			break
		}

		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid cancel index:", msg.Index)
			t.closePeer(pe)
			break
		}
		pe.CancelRequest(msg)
	case peerwriter.BlockUploaded:
		t.bytesUploaded += int64(msg.Length)
		pe.BytesUploadedInChokePeriod += int64(msg.Length)
	case peerprotocol.ExtensionHandshakeMessage:
		pe.Logger().Debugln("extension handshake received:", msg)
		if pe.ExtensionHandshake != nil {
			pe.Logger().Debugln("peer changed extensions")
			break
		}
		pe.ExtensionHandshake = &msg

		if _, ok := msg.M[peerprotocol.ExtensionKeyMetadata]; ok {
			t.startInfoDownloaders()
		}
		if _, ok := msg.M[peerprotocol.ExtensionKeyPEX]; ok {
			if t.info != nil && t.info.Private != 1 {
				pe.StartPEX(t.peers)
			}
		}
	case peerprotocol.ExtensionMetadataMessage:
		switch msg.Type {
		case peerprotocol.ExtensionMetadataMessageTypeRequest:
			extMsgID, ok := pe.ExtensionHandshake.M[peerprotocol.ExtensionKeyMetadata]
			if !ok {
				break
			}
			if t.info == nil {
				t.sendMetadataReject(pe, msg.Piece, extMsgID)
				break
			}
			// TODO Clients MAY implement flood protection by rejecting request messages after a certain number of them have been served. Typically the number of pieces of metadata times a factor.
			start := 16 * 1024 * msg.Piece
			end := start + 16*1024
			totalSize := uint32(len(t.info.Bytes))
			if end > totalSize {
				end = totalSize
			}
			if start >= totalSize {
				t.sendMetadataReject(pe, msg.Piece, extMsgID)
				break
			}
			if end > totalSize {
				t.sendMetadataReject(pe, msg.Piece, extMsgID)
				break
			}
			data := t.info.Bytes[start:end]
			dataMsg := peerprotocol.ExtensionMetadataMessage{
				Type:      peerprotocol.ExtensionMetadataMessageTypeData,
				Piece:     msg.Piece,
				TotalSize: totalSize,
				Data:      data,
			}
			extDataMsg := peerprotocol.ExtensionMessage{
				ExtendedMessageID: extMsgID,
				Payload:           dataMsg,
			}
			pe.SendMessage(extDataMsg)
		case peerprotocol.ExtensionMetadataMessageTypeData:
			id, ok := t.infoDownloaders[pe]
			if !ok {
				break
			}
			err := id.GotBlock(msg.Piece, msg.Data)
			if err != nil {
				pe.Logger().Error(err)
				t.closePeer(pe)
				t.startInfoDownloaders()
				break
			}
			if !id.Done() {
				id.RequestBlocks(t.config.RequestQueueLength)
				pe.ResetSnubTimer()
				break
			}
			pe.StopSnubTimer()

			hash := sha1.New()                              // nolint: gosec
			hash.Write(id.Bytes)                            // nolint: gosec
			if !bytes.Equal(hash.Sum(nil), t.infoHash[:]) { // nolint: gosec
				pe.Logger().Errorln("received info does not match with hash")
				t.closePeer(id.Peer)
				t.startInfoDownloaders()
				break
			}
			t.stopInfoDownloaders()

			info, err := metainfo.NewInfo(id.Bytes)
			if err != nil {
				err = fmt.Errorf("cannot parse info bytes: %s", err)
				t.log.Error(err)
				t.stop(err)
				break
			}
			if info.Private == 1 {
				err = errors.New("private torrent from magnet")
				t.log.Error(err)
				t.stop(err)
				break
			}
			t.info = info
			if t.resume != nil {
				err = t.resume.WriteInfo(t.info.Bytes)
				if err != nil {
					err = fmt.Errorf("cannot write resume info: %s", err)
					t.log.Error(err)
					t.stop(err)
					break
				}
			}
			t.startAllocator()
		case peerprotocol.ExtensionMetadataMessageTypeReject:
			id, ok := t.infoDownloaders[pe]
			if ok {
				t.closePeer(id.Peer)
				t.startInfoDownloaders()
			}
		}
	case peerprotocol.ExtensionPEXMessage:
		addrs, err := tracker.DecodePeersCompact([]byte(msg.Added))
		if err != nil {
			t.log.Error(err)
			break
		}
		t.handleNewPeers(addrs, "pex")
	default:
		panic(fmt.Sprintf("unhandled peer message type: %T", msg))
	}
}

func (t *Torrent) updateInterestedState(pe *peer.Peer) {
	if t.pieces == nil || t.bitfield == nil {
		return
	}
	interested := false
	for i := uint32(0); i < t.bitfield.Len(); i++ {
		weHave := t.bitfield.Test(i)
		peerHave := t.piecePicker.DoesHave(pe, i)
		if !weHave && peerHave {
			interested = true
			break
		}
	}
	if !pe.AmInterested && interested {
		pe.AmInterested = true
		msg := peerprotocol.InterestedMessage{}
		pe.SendMessage(msg)
		return
	}
	if pe.AmInterested && !interested {
		pe.AmInterested = false
		msg := peerprotocol.NotInterestedMessage{}
		pe.SendMessage(msg)
		return
	}
}

func (t *Torrent) sendMetadataReject(pe *peer.Peer, i uint32, msgID uint8) {
	dataMsg := peerprotocol.ExtensionMetadataMessage{
		Type:  peerprotocol.ExtensionMetadataMessageTypeReject,
		Piece: i,
	}
	extDataMsg := peerprotocol.ExtensionMessage{
		ExtendedMessageID: msgID,
		Payload:           &dataMsg,
	}
	pe.SendMessage(extDataMsg)
}
<commit_msg>put buffer to pool on wrong piece<commit_after>package torrent

import (
	"bytes"
	"crypto/sha1" // nolint: gosec
	"errors"
	"fmt"

	"github.com/cenkalti/rain/torrent/bitfield"
	"github.com/cenkalti/rain/torrent/internal/peer"
	"github.com/cenkalti/rain/torrent/internal/peerconn/peerreader"
	"github.com/cenkalti/rain/torrent/internal/peerconn/peerwriter"
	"github.com/cenkalti/rain/torrent/internal/peerprotocol"
	"github.com/cenkalti/rain/torrent/internal/piecewriter"
	"github.com/cenkalti/rain/torrent/internal/tracker"
	"github.com/cenkalti/rain/torrent/metainfo"
)

func (t *Torrent) handlePeerMessage(pm peer.Message) {
	pe := pm.Peer
	switch msg := pm.Message.(type) {
	case peerprotocol.HaveMessage:
		// Save have messages for processesing later received while we don't have info yet.
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("unexpected piece index:", msg.Index)
			t.closePeer(pe)
			break
		}
		pi := &t.pieces[msg.Index]
		// pe.Logger().Debug("Peer ", pe.String(), " has piece #", pi.Index)
		t.piecePicker.HandleHave(pe, pi.Index)
		t.updateInterestedState(pe)
		t.startPieceDownloaders()
	case peerprotocol.BitfieldMessage:
		// Save bitfield messages while we don't have info yet.
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		bf, err := bitfield.NewBytes(msg.Data, t.info.NumPieces)
		if err != nil {
			pe.Logger().Errorln(err)
			t.closePeer(pe)
			break
		}
		pe.Logger().Debugln("Received bitfield:", bf.Hex())
		for i := uint32(0); i < bf.Len(); i++ {
			if bf.Test(i) {
				t.piecePicker.HandleHave(pe, i)
			}
		}
		t.updateInterestedState(pe)
		t.startPieceDownloaders()
	case peerprotocol.HaveAllMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		for _, pi := range t.pieces {
			t.piecePicker.HandleHave(pe, pi.Index)
		}
		t.updateInterestedState(pe)
		t.startPieceDownloaders()
	case peerprotocol.HaveNoneMessage:
	case peerprotocol.AllowedFastMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Messages = append(pe.Messages, msg)
			break
		}
		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid allowed fast piece index:", msg.Index)
			t.closePeer(pe)
			break
		}
		pi := &t.pieces[msg.Index]
		pe.Logger().Debug("Peer ", pe.String(), " has allowed fast for piece #", pi.Index)
		t.piecePicker.HandleAllowedFast(pe, msg.Index)
	case peerprotocol.UnchokeMessage:
		pe.PeerChoking = false
		if pd, ok := t.pieceDownloaders[pe]; ok {
			pd.RequestBlocks(t.config.RequestQueueLength)
		}
		t.startPieceDownloaders()
	case peerprotocol.ChokeMessage:
		pe.PeerChoking = true
		if pd, ok := t.pieceDownloaders[pe]; ok {
			pd.Choked()
			t.pieceDownloadersChoked[pe] = pd
			t.startPieceDownloaders()
		}
	case peerprotocol.InterestedMessage:
		pe.PeerInterested = true
	case peerprotocol.NotInterestedMessage:
		pe.PeerInterested = false
	case peerreader.Piece:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("piece received but we don't have info")
			t.closePeer(pe)
			break
		}
		if msg.Index >= uint32(len(t.pieces)) {
			pe.Logger().Errorln("invalid piece index:", msg.Index)
			t.closePeer(pe)
			break
		}
		piece := &t.pieces[msg.Index]
		block := piece.Blocks.Find(msg.Begin, uint32(len(msg.Data)))
		if block == nil {
			pe.Logger().Errorln("invalid piece begin:", msg.Begin, "length:", len(msg.Data))
			t.closePeer(pe)
			break
		}
		pe.BytesDownlaodedInChokePeriod += int64(len(msg.Data))
		t.bytesDownloaded += int64(len(msg.Data))
		pd, ok := t.pieceDownloaders[pe]
		if !ok {
			t.bytesWasted += int64(len(msg.Data))
			peerreader.PiecePool.Put(msg.Data)
			break
		}
		if pd.Piece.Index != msg.Index {
			t.bytesWasted += int64(len(msg.Data))
			peerreader.PiecePool.Put(msg.Data)
			break
		}
		pd.GotBlock(block, msg.Data)
		peerreader.PiecePool.Put(msg.Data)
		if !pd.Done() {
			pd.RequestBlocks(t.config.RequestQueueLength)
			pe.ResetSnubTimer()
			break
		}
		// t.log.Debugln("piece download completed. index:", pd.Piece.Index)
		t.closePieceDownloader(pd)
		pe.StopSnubTimer()

		ok = piece.VerifyHash(pd.Buffer[:pd.Piece.Length], sha1.New()) // nolint: gosec
		if !ok {
			// TODO ban peers that sent corrupt piece
			t.log.Error("received corrupt piece")
			t.closePeer(pd.Peer)
			t.startPieceDownloaders()
			break
		}

		for pe := range t.piecePicker.RequestedPeers(piece.Index) {
			pd2 := t.pieceDownloaders[pe]
			t.closePieceDownloader(pd2)
			pd2.CancelPending()
		}

		if piece.Writing {
			panic("piece already writing")
		}
		piece.Writing = true

		pw := piecewriter.New(piece, pd.Buffer, pd.Piece.Length)
		t.pieceWriters[pw] = struct{}{}
		go pw.Run(t.pieceWriterResultC)

		t.startPieceDownloaders()
	case peerprotocol.RequestMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("request received but we don't have info")
			t.closePeer(pe)
			break
		}
		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid request index:", msg.Index)
			t.closePeer(pe)
			break
		}
		if msg.Begin+msg.Length > t.pieces[msg.Index].Length {
			pe.Logger().Errorln("invalid request length:", msg.Length)
			t.closePeer(pe)
			break
		}
		pi := &t.pieces[msg.Index]
		if pe.AmChoking {
			if pe.FastExtension {
				m := peerprotocol.RejectMessage{RequestMessage: msg}
				pe.SendMessage(m)
			}
		} else {
			pe.SendPiece(msg, pi.Data)
		}
	case peerprotocol.RejectMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("reject received but we don't have info")
			t.closePeer(pe)
			break
		}

		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid reject index:", msg.Index)
			t.closePeer(pe)
			break
		}
		piece := &t.pieces[msg.Index]
		block := piece.Blocks.Find(msg.Begin, msg.Length)
		if block == nil {
			pe.Logger().Errorln("invalid reject begin:", msg.Begin, "length:", msg.Length)
			t.closePeer(pe)
			break
		}
		pd, ok := t.pieceDownloaders[pe]
		if ok {
			pd.Rejected(block)
		}
	case peerprotocol.CancelMessage:
		if t.pieces == nil || t.bitfield == nil {
			pe.Logger().Error("cancel received but we don't have info")
			t.closePeer(pe)
			break
		}

		if msg.Index >= t.info.NumPieces {
			pe.Logger().Errorln("invalid cancel index:", msg.Index)
			t.closePeer(pe)
			break
		}
		pe.CancelRequest(msg)
	case peerwriter.BlockUploaded:
		t.bytesUploaded += int64(msg.Length)
		pe.BytesUploadedInChokePeriod += int64(msg.Length)
	case peerprotocol.ExtensionHandshakeMessage:
		pe.Logger().Debugln("extension handshake received:", msg)
		if pe.ExtensionHandshake != nil {
			pe.Logger().Debugln("peer changed extensions")
			break
		}
		pe.ExtensionHandshake = &msg

		if _, ok := msg.M[peerprotocol.ExtensionKeyMetadata]; ok {
			t.startInfoDownloaders()
		}
		if _, ok := msg.M[peerprotocol.ExtensionKeyPEX]; ok {
			if t.info != nil && t.info.Private != 1 {
				pe.StartPEX(t.peers)
			}
		}
	case peerprotocol.ExtensionMetadataMessage:
		switch msg.Type {
		case peerprotocol.ExtensionMetadataMessageTypeRequest:
			extMsgID, ok := pe.ExtensionHandshake.M[peerprotocol.ExtensionKeyMetadata]
			if !ok {
				break
			}
			if t.info == nil {
				t.sendMetadataReject(pe, msg.Piece, extMsgID)
				break
			}
			// TODO Clients MAY implement flood protection by rejecting request messages after a certain number of them have been served. Typically the number of pieces of metadata times a factor.
			start := 16 * 1024 * msg.Piece
			end := start + 16*1024
			totalSize := uint32(len(t.info.Bytes))
			if end > totalSize {
				end = totalSize
			}
			if start >= totalSize {
				t.sendMetadataReject(pe, msg.Piece, extMsgID)
				break
			}
			if end > totalSize {
				t.sendMetadataReject(pe, msg.Piece, extMsgID)
				break
			}
			data := t.info.Bytes[start:end]
			dataMsg := peerprotocol.ExtensionMetadataMessage{
				Type:      peerprotocol.ExtensionMetadataMessageTypeData,
				Piece:     msg.Piece,
				TotalSize: totalSize,
				Data:      data,
			}
			extDataMsg := peerprotocol.ExtensionMessage{
				ExtendedMessageID: extMsgID,
				Payload:           dataMsg,
			}
			pe.SendMessage(extDataMsg)
		case peerprotocol.ExtensionMetadataMessageTypeData:
			id, ok := t.infoDownloaders[pe]
			if !ok {
				break
			}
			err := id.GotBlock(msg.Piece, msg.Data)
			if err != nil {
				pe.Logger().Error(err)
				t.closePeer(pe)
				t.startInfoDownloaders()
				break
			}
			if !id.Done() {
				id.RequestBlocks(t.config.RequestQueueLength)
				pe.ResetSnubTimer()
				break
			}
			pe.StopSnubTimer()

			hash := sha1.New()                              // nolint: gosec
			hash.Write(id.Bytes)                            // nolint: gosec
			if !bytes.Equal(hash.Sum(nil), t.infoHash[:]) { // nolint: gosec
				pe.Logger().Errorln("received info does not match with hash")
				t.closePeer(id.Peer)
				t.startInfoDownloaders()
				break
			}
			t.stopInfoDownloaders()

			info, err := metainfo.NewInfo(id.Bytes)
			if err != nil {
				err = fmt.Errorf("cannot parse info bytes: %s", err)
				t.log.Error(err)
				t.stop(err)
				break
			}
			if info.Private == 1 {
				err = errors.New("private torrent from magnet")
				t.log.Error(err)
				t.stop(err)
				break
			}
			t.info = info
			if t.resume != nil {
				err = t.resume.WriteInfo(t.info.Bytes)
				if err != nil {
					err = fmt.Errorf("cannot write resume info: %s", err)
					t.log.Error(err)
					t.stop(err)
					break
				}
			}
			t.startAllocator()
		case peerprotocol.ExtensionMetadataMessageTypeReject:
			id, ok := t.infoDownloaders[pe]
			if ok {
				t.closePeer(id.Peer)
				t.startInfoDownloaders()
			}
		}
	case peerprotocol.ExtensionPEXMessage:
		addrs, err := tracker.DecodePeersCompact([]byte(msg.Added))
		if err != nil {
			t.log.Error(err)
			break
		}
		t.handleNewPeers(addrs, "pex")
	default:
		panic(fmt.Sprintf("unhandled peer message type: %T", msg))
	}
}

func (t *Torrent) updateInterestedState(pe *peer.Peer) {
	if t.pieces == nil || t.bitfield == nil {
		return
	}
	interested := false
	for i := uint32(0); i < t.bitfield.Len(); i++ {
		weHave := t.bitfield.Test(i)
		peerHave := t.piecePicker.DoesHave(pe, i)
		if !weHave && peerHave {
			interested = true
			break
		}
	}
	if !pe.AmInterested && interested {
		pe.AmInterested = true
		msg := peerprotocol.InterestedMessage{}
		pe.SendMessage(msg)
		return
	}
	if pe.AmInterested && !interested {
		pe.AmInterested = false
		msg := peerprotocol.NotInterestedMessage{}
		pe.SendMessage(msg)
		return
	}
}

func (t *Torrent) sendMetadataReject(pe *peer.Peer, i uint32, msgID uint8) {
	dataMsg := peerprotocol.ExtensionMetadataMessage{
		Type:  peerprotocol.ExtensionMetadataMessageTypeReject,
		Piece: i,
	}
	extDataMsg := peerprotocol.ExtensionMessage{
		ExtendedMessageID: msgID,
		Payload:           &dataMsg,
	}
	pe.SendMessage(extDataMsg)
}
<|endoftext|>


<commit_before>package gateway

import (
	"errors"
	"math/rand"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

const (
	maxSharedNodes = 10
	maxAddrLength  = 100
	minPeers       = 3
)

// addNode adds an address to the set of nodes on the network.
func (g *Gateway) addNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; exists {
		return errors.New("node already added")
	}
	g.nodes[addr] = struct{}{}
	g.log.Println("INFO: added node", addr)
	return nil
}

func (g *Gateway) removeNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; !exists {
		return errors.New("no record of that node")
	}
	delete(g.nodes, addr)
	g.log.Println("INFO: removed node", addr)
	return nil
}

func (g *Gateway) randomNode() (modules.NetAddress, error) {
	if len(g.nodes) > 0 {
		r := rand.Intn(len(g.nodes))
		for node := range g.nodes {
			if r == 0 {
				return node, nil
			}
			r--
		}
	}

	return "", errNoPeers
}

// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
	id := g.mu.RLock()
	var nodes []modules.NetAddress
	for node := range g.nodes {
		if len(nodes) == maxSharedNodes {
			break
		}
		nodes = append(nodes, node)
	}
	g.mu.RUnlock(id)
	return encoding.WriteObject(conn, nodes)
}

// requestNodes is the calling end of the ShareNodes RPC.
func (g *Gateway) requestNodes(conn modules.PeerConn) error {
	var nodes []modules.NetAddress
	if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*maxAddrLength); err != nil {
		return err
	}
	g.log.Printf("INFO: %v sent us %v peers", conn.RemoteAddr(), len(nodes))
	id := g.mu.Lock()
	for _, node := range nodes {
		g.addNode(node)
	}
	g.save()
	g.mu.Unlock(id)
	return nil
}

// relayNode is the recipient end of the RelayNode RPC. It reads a node, adds
// it to the Gateway's node list, and relays it to each of the Gateway's
// peers. If the node is already in the node list, it is not relayed.
func (g *Gateway) relayNode(conn modules.PeerConn) error {
	// read address
	var addr modules.NetAddress
	if err := encoding.ReadObject(conn, &addr, maxAddrLength); err != nil {
		return err
	}
	// add node
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if err := g.addNode(addr); err != nil {
		return err
	}
	// relay
	go g.Broadcast("RelayNode", addr)
	return nil
}

// sendAddress is the calling end of the RelayNode RPC.
func (g *Gateway) sendAddress(conn modules.PeerConn) error {
	return encoding.WriteObject(conn, g.Address())
}
<commit_msg>better node filtering<commit_after>package gateway

import (
	"errors"
	"math/rand"
	"net"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

const (
	maxSharedNodes = 10
	maxAddrLength  = 100
	minPeers       = 3
)

// addNode adds an address to the set of nodes on the network.
func (g *Gateway) addNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; exists {
		return errors.New("node already added")
	} else if net.ParseIP(addr.Host()) == nil {
		return errors.New("address is not routable: " + string(addr))
	} else if net.ParseIP(addr.Host()).IsLoopback() {
		return errors.New("cannot add loopback address")
	}
	g.nodes[addr] = struct{}{}
	g.log.Println("INFO: added node", addr)
	return nil
}

func (g *Gateway) removeNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; !exists {
		return errors.New("no record of that node")
	}
	delete(g.nodes, addr)
	g.log.Println("INFO: removed node", addr)
	return nil
}

func (g *Gateway) randomNode() (modules.NetAddress, error) {
	if len(g.nodes) > 0 {
		r := rand.Intn(len(g.nodes))
		for node := range g.nodes {
			if r == 0 {
				return node, nil
			}
			r--
		}
	}

	return "", errNoPeers
}

// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
	id := g.mu.RLock()
	var nodes []modules.NetAddress
	for node := range g.nodes {
		if len(nodes) == maxSharedNodes {
			break
		}
		nodes = append(nodes, node)
	}
	g.mu.RUnlock(id)
	return encoding.WriteObject(conn, nodes)
}

// requestNodes is the calling end of the ShareNodes RPC.
func (g *Gateway) requestNodes(conn modules.PeerConn) error {
	var nodes []modules.NetAddress
	if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*maxAddrLength); err != nil {
		return err
	}
	g.log.Printf("INFO: %v sent us %v nodes", conn.RemoteAddr(), len(nodes))
	id := g.mu.Lock()
	for _, node := range nodes {
		g.addNode(node)
	}
	g.save()
	g.mu.Unlock(id)
	return nil
}

// relayNode is the recipient end of the RelayNode RPC. It reads a node, adds
// it to the Gateway's node list, and relays it to each of the Gateway's
// peers. If the node is already in the node list, it is not relayed.
func (g *Gateway) relayNode(conn modules.PeerConn) error {
	// read address
	var addr modules.NetAddress
	if err := encoding.ReadObject(conn, &addr, maxAddrLength); err != nil {
		return err
	}
	// add node
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if err := g.addNode(addr); err != nil {
		return err
	}
	// relay
	go g.Broadcast("RelayNode", addr)
	return nil
}

// sendAddress is the calling end of the RelayNode RPC.
func (g *Gateway) sendAddress(conn modules.PeerConn) error {
	return encoding.WriteObject(conn, g.Address())
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Miles Crabill <mcrabill@mozilla.com>

package github

import (
	"fmt"
	"log"
	"strings"

	"github.com/google/go-github/github"
	"go.mozilla.org/userplex/modules"
	"golang.org/x/oauth2"
)

func init() {
	modules.Register("github", new(module))
}

type module struct{}

func (m *module) NewRun(c modules.Configuration) modules.Runner {
	r := new(run)
	r.Conf = c
	return r
}

type run struct {
	Conf         modules.Configuration
	p            parameters
	c            credentials
	ghclient     *github.Client
	githubToLdap map[string]string
	ldapToGithub map[string]string
}

type organization struct {
	Name  string
	Teams []string
}

type parameters struct {
	Organization     organization
	UserplexTeamName string
	Enforce2FA       bool
}

type credentials struct {
	OAuthToken string `yaml:"oauthtoken"`
}

func (r *run) Run() (err error) {
	var resp *github.Response

	err = r.Conf.GetParameters(&r.p)
	if err != nil {
		return
	}
	err = r.Conf.GetCredentials(&r.c)
	if err != nil {
		return
	}

	if r.p.UserplexTeamName == "" {
		return fmt.Errorf("[error] github: UserplexTeamName is not set")
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: r.c.OAuthToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	r.ghclient = github.NewClient(tc)

	r.buildLdapMapping()
	ldapers := r.getLdapers()

	// get all members for the organization
	// name -> bool
	membersMap := r.getOrgMembersMap(r.p.Organization, "all")
	if r.Conf.Debug {
		log.Printf("[debug] github: found %d users for organization %s", len(membersMap), r.p.Organization.Name)
	}

	// get all teams for the organization
	// name -> team
	teamsMap := r.getOrgTeamsMap(r.p.Organization)
	if r.Conf.Debug {
		log.Printf("[debug] github: found %d teams for organization %s", len(teamsMap), r.p.Organization.Name)
	}

	teamMembersMap := make(map[string]map[string]bool)
	for _, team := range teamsMap {
		teamMembersMap[*team.Name] = make(map[string]bool)
		teamMembersMap[*team.Name] = r.getTeamMembersMap(team)
	}

	if _, ok := teamsMap[r.p.UserplexTeamName]; !ok {
		return fmt.Errorf("[error] github: could not find UserplexTeam \"%s\" for organization %s", r.p.UserplexTeamName, r.p.Organization.Name)
	}
	userplexedUsers := teamMembersMap[r.p.UserplexTeamName]

	var no2fa map[string]bool
	if r.p.Enforce2FA {
		no2fa = r.getOrgMembersMap(r.p.Organization, "2fa_disabled")
		log.Printf("[info] github: organization %s has %d total members and %d with 2fa disabled. %.2f%% have 2fa enabled.",
			r.p.Organization.Name, len(membersMap), len(no2fa), 100-100*float64(len(no2fa))/float64(len(membersMap)))
	}

	// member or admin
	membershipType := "member"

	countAdded := 0
	for user := range ldapers {
		// set to true to indicate that user in github has ldap match
		membersMap[user] = true

		userWasAddedToTeam := false
		// teams in config
		for _, teamName := range r.p.Organization.Teams {
			// if the team in config doesn't exist on github
			if team, ok := teamsMap[teamName]; !ok {
				return fmt.Errorf("[error] github: could not find team %s for organization %s", team, r.p.Organization.Name)
			} else {
				// if the user is already in the team, skip adding them
				if _, ok := teamMembersMap[teamName][user]; ok {
					continue
				}
				// user not in team, add them
				if r.Conf.ApplyChanges && r.Conf.Create {
					// add user to team
					_, resp, err = r.ghclient.Organizations.AddTeamMembership(*team.ID, user, &github.OrganizationAddTeamMembershipOptions{
						Role: membershipType,
					})
					if err != nil || resp.StatusCode != 200 {
						return fmt.Errorf("[error] github: could not add user %s to %s: %s, error: %v with status %s", user, r.p.Organization.Name, *team.Name, err, resp.Status)
					}
				}
				if r.Conf.Create {
					userWasAddedToTeam = true
				}
			}
		}
		if userWasAddedToTeam {
			if !r.Conf.ApplyChanges {
				log.Printf("[dryrun] github: would have added %s to GitHub organization %s and teams %v", user, r.p.Organization.Name, r.p.Organization.Teams)
			} else {
				countAdded++
			}
			r.notify(user, fmt.Sprintf("Userplex added %s to GitHub organization %s and teams %v", user, r.p.Organization.Name, r.p.Organization.Teams))
		}
	}

	countRemoved := 0
	for member := range membersMap {
		var ldapUsername, ldapUsernameString string
		member = strings.ToLower(member)
		if _, ok := r.githubToLdap[member]; ok {
			ldapUsername = r.githubToLdap[member]
			ldapUsernameString = ldapUsername + " / "
		}

		var userTeams []string
		// icky iterating over all these teams
		for _, team := range teamsMap {
			if _, ok := teamMembersMap[*team.Name][member]; ok {
				userTeams = append(userTeams, *team.Name)
			}
		}

		// if the member does not have 2fa
		_, no2fa := no2fa[member]

		// if the user is in ldap
		_, inLdap := membersMap[member]

		// if the member is not in the userplex team
		_, isUserplexed := userplexedUsers[member]

		shouldDelete := false
		if !inLdap {
			if r.Conf.Debug {
				log.Printf("[debug] github: user %s%s is not in ldap groups %s but is a member of github organization %s and teams %v", ldapUsernameString, member, r.Conf.LdapGroups, r.p.Organization.Name, userTeams)
			}
			shouldDelete = true
		}

		if r.p.Enforce2FA && no2fa {
			log.Printf("[info] github: user %s%s does not have 2FA enabled and is a member of github organization %s and teams %v", ldapUsernameString, member, r.p.Organization.Name, userTeams)
			shouldDelete = true
		}

		if shouldDelete && r.Conf.Delete {
			// not in UserplexTeam -> skip
			if !isUserplexed {
				if r.Conf.Debug {
					log.Printf("[debug] github: would have removed member %s in organization %s, but skipped because they are not in UserplexTeam %s", member, r.p.Organization.Name, r.p.UserplexTeamName)
				}
				continue
			}
			if !r.Conf.ApplyChanges {
				log.Printf("[dryrun] github: Userplex would have removed %s from GitHub organization %s", member, r.p.Organization.Name)
				r.notify(member, fmt.Sprintf("Userplex removed %s to GitHub organization %s", member, r.p.Organization.Name))
				continue
			}
			// applying changes, user is userplexed -> remove them
			resp, err = r.ghclient.Organizations.RemoveOrgMembership(r.p.Organization.Name, member)
			if err != nil || resp.StatusCode != 200 {
				log.Printf("[error] github: could not remove user %s from %s, error: %v with status %s", member, r.p.Organization.Name, err, resp.Status)
				continue
			}
			countRemoved++
			r.notify(member, fmt.Sprintf("Userplex removed %s to GitHub organization %s", member, r.p.Organization.Name))
		}
	}

	log.Printf("[info] github %q: summary added=%d, removed=%d",
		r.p.Organization.Name, countAdded, countRemoved)

	return nil
}

func (r *run) buildLdapMapping() {
	r.githubToLdap = make(map[string]string)
	r.ldapToGithub = make(map[string]string)
	for _, mapping := range r.Conf.UidMap {
		r.githubToLdap[mapping.LocalUID] = mapping.LdapUid
		r.ldapToGithub[mapping.LdapUid] = mapping.LocalUID
	}
}

func (r *run) getOrgMembersMap(org organization, filter string) (membersMap map[string]bool) {
	membersMap = make(map[string]bool)
	opt := &github.ListMembersOptions{
		Filter:      filter,
		ListOptions: github.ListOptions{PerPage: 100},
	}
	for {
		members, resp, err := r.ghclient.Organizations.ListMembers(org.Name, opt)
		if err != nil || resp.StatusCode != 200 {
			log.Printf("[error] github: could not list members for organization %s, error: %v with status %s", org, err, resp.Status)
			return
		}
		for _, member := range members {
			membersMap[*member.Login] = false
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	return membersMap
}

func (r *run) getTeamMembersMap(team *github.Team) (membersMap map[string]bool) {
	membersMap = make(map[string]bool)
	opt := &github.OrganizationListTeamMembersOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}
	for {
		members, resp, err := r.ghclient.Organizations.ListTeamMembers(*team.ID, opt)
		if err != nil || resp.StatusCode != 200 {
			log.Printf("[error] github: could not list members for organization %s, error: %v with status %s", team, err, resp.Status)
			return
		}
		for _, member := range members {
			membersMap[*member.Login] = false
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	return membersMap
}

func (r *run) getOrgTeamsMap(org organization) (teamsMap map[string]*github.Team) {
	teamsMap = make(map[string]*github.Team)
	opt := &github.ListOptions{
		PerPage: 100,
	}
	for {
		teams, resp, err := r.ghclient.Organizations.ListTeams(org.Name, opt)
		if err != nil || resp.StatusCode != 200 {
			log.Printf("[error] github: could not list teams for organization %s, error: %v", org.Name, err)
			return
		}
		for _, team := range teams {
			teamsMap[*team.Name] = team
		}
		if resp.NextPage == 0 {
			break
		}
		opt.Page = resp.NextPage
	}
	return teamsMap
}

func (r *run) notify(user string, body string) (err error) {
	rcpt := r.Conf.Notify.Recipient
	if rcpt == "{ldap:mail}" {
		if _, ok := r.githubToLdap[user]; ok {
			user = r.githubToLdap[user]
		}
		rcpt, err = r.Conf.LdapCli.GetUserEmailByUid(user)
		if err != nil {
			return fmt.Errorf("[error] github: couldn't find email of user %q in ldap, notification not sent: %v", user, err)
		}
	}
	r.Conf.Notify.Channel <- modules.Notification{
		Module:      "github",
		Recipient:   rcpt,
		Mode:        r.Conf.Notify.Mode,
		MustEncrypt: false,
		Body:        []byte(body),
	}
	return
}

func (r *run) getLdapers() (lgm map[string]bool) {
	lgm = make(map[string]bool)
	users, err := r.Conf.LdapCli.GetEnabledUsersInGroups(r.Conf.LdapGroups)
	if err != nil {
		return
	}
	for _, user := range users {
		shortdn := strings.Split(user, ",")[0]
		uid, err := r.Conf.LdapCli.GetUserId(shortdn)
		if err != nil {
			log.Printf("[error] github: ldap query failed with error %v", err)
			continue
		}

		if _, ok := r.ldapToGithub[uid]; ok {
			uid = r.ldapToGithub[uid]
		}

		lgm[uid] = false
	}
	return
}
<commit_msg>github module: print info about skipped users who are not in UserplexTeam<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Miles Crabill <mcrabill@mozilla.com>

package github

import (
	"fmt"
	"log"
	"strings"

	"github.com/google/go-github/github"
	"go.mozilla.org/userplex/modules"
	"golang.org/x/oauth2"
)

func init() {
	modules.Register("github", new(module))
}

type module struct{}

func (m *module) NewRun(c modules.Configuration) modules.Runner {
	r := new(run)
	r.Conf = c
	return r
}

type run struct {
	Conf         modules.Configuration
	p            parameters
	c            credentials
	ghclient     *github.Client
	githubToLdap map[string]string
	ldapToGithub map[string]string
}

type organization struct {
	Name  string
	Teams []string
}

type parameters struct {
	Organization     organization
	UserplexTeamName string
	Enforce2FA       bool
}

type credentials struct {
	OAuthToken string `yaml:"oauthtoken"`
}

func (r *run) Run() (err error) {
	var resp *github.Response

	err = r.Conf.GetParameters(&r.p)
	if err != nil {
		return
	}
	err = r.Conf.GetCredentials(&r.c)
	if err != nil {
		return
	}

	if r.p.UserplexTeamName == "" {
		return fmt.Errorf("[error] github: UserplexTeamName is not set")
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: r.c.OAuthToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	r.ghclient = github.NewClient(tc)

	r.buildLdapMapping()
	ldapers := r.getLdapers()

	// get all members for the organization
	// name -> bool
	membersMap := r.getOrgMembersMap(r.p.Organization, "all")
	if r.Conf.Debug {
		log.Printf("[debug] github: found %d users for organization %s", len(membersMap), r.p.Organization.Name)
	}

	// get all teams for the organization
	// name -> team
	teamsMap := r.getOrgTeamsMap(r.p.Organization)
	if r.Conf.Debug {
		log.Printf("[debug] github: found %d teams for organization %s", len(teamsMap), r.p.Organization.Name)
	}

	teamMembersMap := make(map[string]map[string]bool)
	for _, team := range teamsMap {
		teamMembersMap[*team.Name] = make(map[string]bool)
		teamMembersMap[*team.Name] = r.getTeamMembersMap(team)
	}

	if _, ok := teamsMap[r.p.UserplexTeamName]; !ok {
		return fmt.Errorf("[error] github: could not find UserplexTeam \"%s\" for organization %s", r.p.UserplexTeamName, r.p.Organization.Name)
	}
	userplexedUsers := teamMembersMap[r.p.UserplexTeamName]

	var no2fa map[string]bool
	if r.p.Enforce2FA {
		no2fa = r.getOrgMembersMap(r.p.Organization, "2fa_disabled")
		log.Printf("[info] github: organization %s has %d total members and %d with 2fa disabled. %.2f%% have 2fa enabled.",
			r.p.Organization.Name, len(membersMap), len(no2fa), 100-100*float64(len(no2fa))/float64(len(membersMap)))
	}

	// member or admin
	membershipType := "member"

	countAdded := 0
	for user := range ldapers {
		// set to true to indicate that user in github has ldap match
		membersMap[user] = true

		userWasAddedToTeam := false
		// teams in config
		for _, teamName := range r.p.Organization.Teams {
			// if the team in config doesn't exist on github
			if team, ok := teamsMap[teamName]; !ok {
				return fmt.Errorf("[error] github: could not find team %s for organization %s", team, r.p.Organization.Name)
			} else {
				// if the user is already in the team, skip adding them
				if _, ok := teamMembersMap[teamName][user]; ok {
					continue
				}
				// user not in team, add them
				if r.Conf.ApplyChanges && r.Conf.Create {
					// add user to team
					_, resp, err = r.ghclient.Organizations.AddTeamMembership(*team.ID, user, &github.OrganizationAddTeamMembershipOptions{
						Role: membershipType,
					})
					if err != nil || resp.StatusCode != 200 {
						return fmt.Errorf("[error] github: could not add user %s to %s: %s, error: %v with status %s", user, r.p.Organization.Name, *team.Name, err, resp.Status)
					}
				}
				if r.Conf.Create {
					userWasAddedToTeam = true
				}
			}
		}
		if userWasAddedToTeam {
			if !r.Conf.ApplyChanges {
				log.Printf("[dryrun] github: would have added %s to GitHub organization %s and teams %v", user, r.p.Organization.Name, r.p.Organization.Teams)
			} else {
				countAdded++
			}
			r.notify(user, fmt.Sprintf("Userplex added %s to GitHub organization %s and teams %v", user, r.p.Organization.Name, r.p.Organization.Teams))
		}
	}

	countRemoved := 0
	countSkipped := 0
	for member := range membersMap {
		var ldapUsername, ldapUsernameString string
		member = strings.ToLower(member)
		if _, ok := r.githubToLdap[member]; ok {
			ldapUsername = r.githubToLdap[member]
			ldapUsernameString = ldapUsername + " / "
		}

		var userTeams []string
		// icky iterating over all these teams
		for _, team := range teamsMap {
			if _, ok := teamMembersMap[*team.Name][member]; ok {
				userTeams = append(userTeams, *team.Name)
			}
		}

		// if the member does not have 2fa
		_, no2fa := no2fa[member]

		// if the user is in ldap
		_, inLdap := membersMap[member]

		// if the member is not in the userplex team
		_, isUserplexed := userplexedUsers[member]

		shouldDelete := false
		if !inLdap {
			if r.Conf.Debug {
				log.Printf("[debug] github: user %s%s is not in ldap groups %s but is a member of github organization %s and teams %v", ldapUsernameString, member, r.Conf.LdapGroups, r.p.Organization.Name, userTeams)
			}
			shouldDelete = true
		}

		if r.p.Enforce2FA && no2fa {
			log.Printf("[info] github: user %s%s does not have 2FA enabled and is a member of github organization %s and teams %v", ldapUsernameString, member, r.p.Organization.Name, userTeams)
			shouldDelete = true
		}

		if shouldDelete && r.Conf.Delete {
			// not in UserplexTeam -> skip
			if !isUserplexed {
				log.Printf("[info] github: would have removed member %s in organization %s, but skipped because they are not in UserplexTeam %q", member, r.p.Organization.Name, r.p.UserplexTeamName)
				countSkipped++
				continue
			}
			if !r.Conf.ApplyChanges {
				log.Printf("[dryrun] github: Userplex would have removed %s%s from GitHub organization %s", ldapUsernameString, member, r.p.Organization.Name)
				r.notify(member, fmt.Sprintf("Userplex removed %s to GitHub organization %s", member, r.p.Organization.Name))
			} else {
				// applying changes, user is userplexed -> remove them
				resp, err = r.ghclient.Organizations.RemoveOrgMembership(r.p.Organization.Name, member)
				if err != nil || resp.StatusCode != 200 {
					log.Printf("[error] github: could not remove user %s from %s, error: %v with status %s", member, r.p.Organization.Name, err, resp.Status)
				}
			}

			// update count and send notification here regardless of ApplyChanges
			countRemoved++
			r.notify(member, fmt.Sprintf("Userplex removed %s to GitHub organization %s", member, r.p.Organization.Name))
		}
	}

	log.Printf("[info] github %q: summary added=%d, removed=%d, skipped=%d",
		r.p.Organization.Name, countAdded, countRemoved, countSkipped)

	return nil
}

func (r *run) buildLdapMapping() {
	r.githubToLdap = make(map[string]string)
	r.ldapToGithub = make(map[string]string)
	for _, mapping := range r.Conf.UidMap {
		r.githubToLdap[mapping.LocalUID] = mapping.LdapUid
		r.ldapToGithub[mapping.LdapUid] = mapping.LocalUID
	}
}

func (r *run) getOrgMembersMap(org organization, filter string) (membersMap map[string]bool) {
	membersMap = make(map[string]bool)
	opt := &github.ListMembersOptions{
		Filter:      filter,
		ListOptions: github.ListOptions{PerPage: 100},
	}
	for {
		members, resp, err := r.ghclient.Organizations.ListMembers(org.Name, opt)
		if err != nil || resp.StatusCode != 200 {
			log.Printf("[error] github: could not list members for organization %s, error: %v with status %s", org, err, resp.Status)
			return
		}
		for _, member := range members {
			membersMap[*member.Login] = false
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	return membersMap
}

func (r *run) getTeamMembersMap(team *github.Team) (membersMap map[string]bool) {
	membersMap = make(map[string]bool)
	opt := &github.OrganizationListTeamMembersOptions{
		ListOptions: github.ListOptions{
			PerPage: 100,
		},
	}
	for {
		members, resp, err := r.ghclient.Organizations.ListTeamMembers(*team.ID, opt)
		if err != nil || resp.StatusCode != 200 {
			log.Printf("[error] github: could not list members for organization %s, error: %v with status %s", team, err, resp.Status)
			return
		}
		for _, member := range members {
			membersMap[*member.Login] = false
		}
		if resp.NextPage == 0 {
			break
		}
		opt.ListOptions.Page = resp.NextPage
	}
	return membersMap
}

func (r *run) getOrgTeamsMap(org organization) (teamsMap map[string]*github.Team) {
	teamsMap = make(map[string]*github.Team)
	opt := &github.ListOptions{
		PerPage: 100,
	}
	for {
		teams, resp, err := r.ghclient.Organizations.ListTeams(org.Name, opt)
		if err != nil || resp.StatusCode != 200 {
			log.Printf("[error] github: could not list teams for organization %s, error: %v", org.Name, err)
			return
		}
		for _, team := range teams {
			teamsMap[*team.Name] = team
		}
		if resp.NextPage == 0 {
			break
		}
		opt.Page = resp.NextPage
	}
	return teamsMap
}

func (r *run) notify(user string, body string) (err error) {
	rcpt := r.Conf.Notify.Recipient
	if rcpt == "{ldap:mail}" {
		if _, ok := r.githubToLdap[user]; ok {
			user = r.githubToLdap[user]
		}
		rcpt, err = r.Conf.LdapCli.GetUserEmailByUid(user)
		if err != nil {
			return fmt.Errorf("[error] github: couldn't find email of user %q in ldap, notification not sent: %v", user, err)
		}
	}
	r.Conf.Notify.Channel <- modules.Notification{
		Module:      "github",
		Recipient:   rcpt,
		Mode:        r.Conf.Notify.Mode,
		MustEncrypt: false,
		Body:        []byte(body),
	}
	return
}

func (r *run) getLdapers() (lgm map[string]bool) {
	lgm = make(map[string]bool)
	users, err := r.Conf.LdapCli.GetEnabledUsersInGroups(r.Conf.LdapGroups)
	if err != nil {
		return
	}
	for _, user := range users {
		shortdn := strings.Split(user, ",")[0]
		uid, err := r.Conf.LdapCli.GetUserId(shortdn)
		if err != nil {
			log.Printf("[error] github: ldap query failed with error %v", err)
			continue
		}

		if _, ok := r.ldapToGithub[uid]; ok {
			uid = r.ldapToGithub[uid]
		}

		lgm[uid] = false
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"strconv"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	clientv1 "k8s.io/client-go/pkg/api/v1"
	"k8s.io/kubernetes/pkg/api/v1"
	batchv1 "k8s.io/kubernetes/pkg/apis/batch/v1"
	batchv2alpha1 "k8s.io/kubernetes/pkg/apis/batch/v2alpha1"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func stagingClientPod(name, value string) clientv1.Pod {
	return clientv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"name": "foo",
				"time": value,
			},
		},
		Spec: clientv1.PodSpec{
			Containers: []clientv1.Container{
				{
					Name:  "nginx",
					Image: "gcr.io/google_containers/nginx-slim:0.7",
					Ports: []clientv1.ContainerPort{{ContainerPort: 80}},
				},
			},
		},
	}
}

func testingPod(name, value string) v1.Pod {
	return v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"name": "foo",
				"time": value,
			},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "nginx",
					Image: "gcr.io/google_containers/nginx-slim:0.7",
					Ports: []v1.ContainerPort{{ContainerPort: 80}},
					LivenessProbe: &v1.Probe{
						Handler: v1.Handler{
							HTTPGet: &v1.HTTPGetAction{
								Path: "/index.html",
								Port: intstr.FromInt(8080),
							},
						},
						InitialDelaySeconds: 30,
					},
				},
			},
		},
	}
}

func observeCreation(w watch.Interface) {
	select {
	case event, _ := <-w.ResultChan():
		if event.Type != watch.Added {
			framework.Failf("Failed to observe the creation: %v", event)
		}
	case <-time.After(30 * time.Second):
		framework.Failf("Timeout while waiting for observing the creation")
	}
}

func observeObjectDeletion(w watch.Interface) (obj runtime.Object) {
	deleted := false
	timeout := false
	timer := time.After(60 * time.Second)
	for !deleted && !timeout {
		select {
		case event, _ := <-w.ResultChan():
			if event.Type == watch.Deleted {
				obj = event.Object
				deleted = true
			}
		case <-timer:
			timeout = true
		}
	}
	if !deleted {
		framework.Failf("Failed to observe pod deletion")
	}
	return
}

var _ = framework.KubeDescribe("Generated release_1_5 clientset", func() {
	f := framework.NewDefaultFramework("clientset")
	It("should create pods, delete pods, watch pods", func() {
		podClient := f.ClientSet.Core().Pods(f.Namespace.Name)
		By("constructing the pod")
		name := "pod" + string(uuid.NewUUID())
		value := strconv.Itoa(time.Now().Nanosecond())
		podCopy := testingPod(name, value)
		pod := &podCopy
		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})).String()
		options := metav1.ListOptions{LabelSelector: selector}
		pods, err := podClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for pods: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: pods.ListMeta.ResourceVersion,
		}
		w, err := podClient.Watch(options)
		if err != nil {
			framework.Failf("Failed to set up watch: %v", err)
		}

		By("creating the pod")
		pod, err = podClient.Create(pod)
		if err != nil {
			framework.Failf("Failed to create pod: %v", err)
		}

		By("verifying the pod is in kubernetes")
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: pod.ResourceVersion,
		}
		pods, err = podClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for pods: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(1))

		By("verifying pod creation was observed")
		observeCreation(w)

		// We need to wait for the pod to be scheduled, otherwise the deletion
		// will be carried out immediately rather than gracefully.
		framework.ExpectNoError(f.WaitForPodRunning(pod.Name))

		By("deleting the pod gracefully")
		if err := podClient.Delete(pod.Name, metav1.NewDeleteOptions(30)); err != nil {
			framework.Failf("Failed to delete pod: %v", err)
		}

		By("verifying pod deletion was observed")
		obj := observeObjectDeletion(w)
		lastPod := obj.(*v1.Pod)
		Expect(lastPod.DeletionTimestamp).ToNot(BeNil())
		Expect(lastPod.Spec.TerminationGracePeriodSeconds).ToNot(BeZero())

		options = metav1.ListOptions{LabelSelector: selector}
		pods, err = podClient.List(options)
		if err != nil {
			framework.Failf("Failed to list pods to verify deletion: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(0))
	})
})

func newTestingCronJob(name string, value string) *batchv2alpha1.CronJob {
	parallelism := int32(1)
	completions := int32(1)
	return &batchv2alpha1.CronJob{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"time": value,
			},
		},
		Spec: batchv2alpha1.CronJobSpec{
			Schedule:          "*/1 * * * ?",
			ConcurrencyPolicy: batchv2alpha1.AllowConcurrent,
			JobTemplate: batchv2alpha1.JobTemplateSpec{
				Spec: batchv1.JobSpec{
					Parallelism: &parallelism,
					Completions: &completions,
					Template: v1.PodTemplateSpec{
						Spec: v1.PodSpec{
							RestartPolicy: v1.RestartPolicyOnFailure,
							Volumes: []v1.Volume{
								{
									Name: "data",
									VolumeSource: v1.VolumeSource{
										EmptyDir: &v1.EmptyDirVolumeSource{},
									},
								},
							},
							Containers: []v1.Container{
								{
									Name:  "c",
									Image: "gcr.io/google_containers/busybox:1.24",
									VolumeMounts: []v1.VolumeMount{
										{
											MountPath: "/data",
											Name:      "data",
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}
}

var _ = framework.KubeDescribe("Generated release_1_5 clientset", func() {
	f := framework.NewDefaultFramework("clientset")
	It("should create v2alpha1 cronJobs, delete cronJobs, watch cronJobs", func() {
		var enabled bool
		groupList, err := f.ClientSet.Discovery().ServerGroups()
		framework.ExpectNoError(err)
		for _, group := range groupList.Groups {
			if group.Name == batchv2alpha1.GroupName {
				for _, version := range group.Versions {
					if version.Version == batchv2alpha1.SchemeGroupVersion.Version {
						enabled = true
						break
					}
				}
			}
		}
		if !enabled {
			framework.Logf("%s is not enabled, test skipped", batchv2alpha1.SchemeGroupVersion)
			return
		}
		cronJobClient := f.ClientSet.BatchV2alpha1().CronJobs(f.Namespace.Name)
		By("constructing the cronJob")
		name := "cronjob" + string(uuid.NewUUID())
		value := strconv.Itoa(time.Now().Nanosecond())
		cronJob := newTestingCronJob(name, value)
		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})).String()
		options := metav1.ListOptions{LabelSelector: selector}
		cronJobs, err := cronJobClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for cronJobs: %v", err)
		}
		Expect(len(cronJobs.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: cronJobs.ListMeta.ResourceVersion,
		}
		w, err := cronJobClient.Watch(options)
		if err != nil {
			framework.Failf("Failed to set up watch: %v", err)
		}

		By("creating the cronJob")
		cronJob, err = cronJobClient.Create(cronJob)
		if err != nil {
			framework.Failf("Failed to create cronJob: %v", err)
		}

		By("verifying the cronJob is in kubernetes")
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: cronJob.ResourceVersion,
		}
		cronJobs, err = cronJobClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for cronJobs: %v", err)
		}
		Expect(len(cronJobs.Items)).To(Equal(1))

		By("verifying cronJob creation was observed")
		observeCreation(w)

		By("deleting the cronJob")
		if err := cronJobClient.Delete(cronJob.Name, nil); err != nil {
			framework.Failf("Failed to delete cronJob: %v", err)
		}

		options = metav1.ListOptions{LabelSelector: selector}
		cronJobs, err = cronJobClient.List(options)
		if err != nil {
			framework.Failf("Failed to list cronJobs to verify deletion: %v", err)
		}
		Expect(len(cronJobs.Items)).To(Equal(0))
	})
})

var _ = framework.KubeDescribe("Staging client repo client", func() {
	f := framework.NewDefaultFramework("clientset")
	It("should create pods, delete pods, watch pods", func() {
		podClient := f.StagingClient.Core().Pods(f.Namespace.Name)
		By("constructing the pod")
		name := "pod" + string(uuid.NewUUID())
		value := strconv.Itoa(time.Now().Nanosecond())
		podCopy := stagingClientPod(name, value)
		pod := &podCopy
		By("verifying no pod exists before the test")
		pods, err := podClient.List(metav1.ListOptions{})
		if err != nil {
			framework.Failf("Failed to query for pods: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(0))
		By("creating the pod")
		pod, err = podClient.Create(pod)
		if err != nil {
			framework.Failf("Failed to create pod: %v", err)
		}

		By("verifying the pod is in kubernetes")
		timeout := 1 * time.Minute
		if err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
			pods, err = podClient.List(metav1.ListOptions{})
			if err != nil {
				return false, err
			}
			if len(pods.Items) == 1 {
				return true, nil
			}
			return false, nil
		}); err != nil {
			framework.Failf("Err : %s\n. Failed to wait for 1 pod to be created", err)
		}
	})
})
<commit_msg>wait for the deletionTimestamp set instead of waiting for the final deletion<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"strconv"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	clientv1 "k8s.io/client-go/pkg/api/v1"
	"k8s.io/kubernetes/pkg/api/v1"
	batchv1 "k8s.io/kubernetes/pkg/apis/batch/v1"
	batchv2alpha1 "k8s.io/kubernetes/pkg/apis/batch/v2alpha1"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func stagingClientPod(name, value string) clientv1.Pod {
	return clientv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"name": "foo",
				"time": value,
			},
		},
		Spec: clientv1.PodSpec{
			Containers: []clientv1.Container{
				{
					Name:  "nginx",
					Image: "gcr.io/google_containers/nginx-slim:0.7",
					Ports: []clientv1.ContainerPort{{ContainerPort: 80}},
				},
			},
		},
	}
}

func testingPod(name, value string) v1.Pod {
	return v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"name": "foo",
				"time": value,
			},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:  "nginx",
					Image: "gcr.io/google_containers/nginx-slim:0.7",
					Ports: []v1.ContainerPort{{ContainerPort: 80}},
					LivenessProbe: &v1.Probe{
						Handler: v1.Handler{
							HTTPGet: &v1.HTTPGetAction{
								Path: "/index.html",
								Port: intstr.FromInt(8080),
							},
						},
						InitialDelaySeconds: 30,
					},
				},
			},
		},
	}
}

func observeCreation(w watch.Interface) {
	select {
	case event, _ := <-w.ResultChan():
		if event.Type != watch.Added {
			framework.Failf("Failed to observe the creation: %v", event)
		}
	case <-time.After(30 * time.Second):
		framework.Failf("Timeout while waiting for observing the creation")
	}
}

func observeObjectDeletion(w watch.Interface) (obj runtime.Object) {
	deleted := false
	timeout := false
	timer := time.After(60 * time.Second)
	for !deleted && !timeout {
		select {
		case event, _ := <-w.ResultChan():
			if event.Type == watch.Deleted {
				obj = event.Object
				deleted = true
			}
		case <-timer:
			timeout = true
		}
	}
	if !deleted {
		framework.Failf("Failed to observe pod deletion")
	}
	return
}

func observerUpdate(w watch.Interface, expectedUpdate func(runtime.Object) bool) {
	timer := time.After(30 * time.Second)
	updated := false
	timeout := false
	for !updated && !timeout {
		select {
		case event, _ := <-w.ResultChan():
			if event.Type == watch.Modified {
				if expectedUpdate(event.Object) {
					updated = true
				}
			}
		case <-timer:
			timeout = true
		}
	}
	if !updated {
		framework.Failf("Failed to observe pod update")
	}
	return
}

var _ = framework.KubeDescribe("Generated release_1_5 clientset", func() {
	f := framework.NewDefaultFramework("clientset")
	It("should create pods, set the deletionTimestamp and deletionGracePeriodSeconds of the pod", func() {
		podClient := f.ClientSet.Core().Pods(f.Namespace.Name)
		By("constructing the pod")
		name := "pod" + string(uuid.NewUUID())
		value := strconv.Itoa(time.Now().Nanosecond())
		podCopy := testingPod(name, value)
		pod := &podCopy
		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})).String()
		options := metav1.ListOptions{LabelSelector: selector}
		pods, err := podClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for pods: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: pods.ListMeta.ResourceVersion,
		}
		w, err := podClient.Watch(options)
		if err != nil {
			framework.Failf("Failed to set up watch: %v", err)
		}

		By("creating the pod")
		pod, err = podClient.Create(pod)
		if err != nil {
			framework.Failf("Failed to create pod: %v", err)
		}

		By("verifying the pod is in kubernetes")
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: pod.ResourceVersion,
		}
		pods, err = podClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for pods: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(1))

		By("verifying pod creation was observed")
		observeCreation(w)

		// We need to wait for the pod to be scheduled, otherwise the deletion
		// will be carried out immediately rather than gracefully.
		framework.ExpectNoError(f.WaitForPodRunning(pod.Name))

		By("deleting the pod gracefully")
		gracePeriod := int64(31)
		if err := podClient.Delete(pod.Name, metav1.NewDeleteOptions(gracePeriod)); err != nil {
			framework.Failf("Failed to delete pod: %v", err)
		}

		By("verifying the deletionTimestamp and deletionGracePeriodSeconds of the pod is set")
		observerUpdate(w, func(obj runtime.Object) bool {
			pod := obj.(*v1.Pod)
			return pod.ObjectMeta.DeletionTimestamp != nil && *pod.ObjectMeta.DeletionGracePeriodSeconds == gracePeriod
		})
	})
})

func newTestingCronJob(name string, value string) *batchv2alpha1.CronJob {
	parallelism := int32(1)
	completions := int32(1)
	return &batchv2alpha1.CronJob{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
			Labels: map[string]string{
				"time": value,
			},
		},
		Spec: batchv2alpha1.CronJobSpec{
			Schedule:          "*/1 * * * ?",
			ConcurrencyPolicy: batchv2alpha1.AllowConcurrent,
			JobTemplate: batchv2alpha1.JobTemplateSpec{
				Spec: batchv1.JobSpec{
					Parallelism: &parallelism,
					Completions: &completions,
					Template: v1.PodTemplateSpec{
						Spec: v1.PodSpec{
							RestartPolicy: v1.RestartPolicyOnFailure,
							Volumes: []v1.Volume{
								{
									Name: "data",
									VolumeSource: v1.VolumeSource{
										EmptyDir: &v1.EmptyDirVolumeSource{},
									},
								},
							},
							Containers: []v1.Container{
								{
									Name:  "c",
									Image: "gcr.io/google_containers/busybox:1.24",
									VolumeMounts: []v1.VolumeMount{
										{
											MountPath: "/data",
											Name:      "data",
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}
}

var _ = framework.KubeDescribe("Generated release_1_5 clientset", func() {
	f := framework.NewDefaultFramework("clientset")
	It("should create v2alpha1 cronJobs, delete cronJobs, watch cronJobs", func() {
		var enabled bool
		groupList, err := f.ClientSet.Discovery().ServerGroups()
		framework.ExpectNoError(err)
		for _, group := range groupList.Groups {
			if group.Name == batchv2alpha1.GroupName {
				for _, version := range group.Versions {
					if version.Version == batchv2alpha1.SchemeGroupVersion.Version {
						enabled = true
						break
					}
				}
			}
		}
		if !enabled {
			framework.Logf("%s is not enabled, test skipped", batchv2alpha1.SchemeGroupVersion)
			return
		}
		cronJobClient := f.ClientSet.BatchV2alpha1().CronJobs(f.Namespace.Name)
		By("constructing the cronJob")
		name := "cronjob" + string(uuid.NewUUID())
		value := strconv.Itoa(time.Now().Nanosecond())
		cronJob := newTestingCronJob(name, value)
		By("setting up watch")
		selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})).String()
		options := metav1.ListOptions{LabelSelector: selector}
		cronJobs, err := cronJobClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for cronJobs: %v", err)
		}
		Expect(len(cronJobs.Items)).To(Equal(0))
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: cronJobs.ListMeta.ResourceVersion,
		}
		w, err := cronJobClient.Watch(options)
		if err != nil {
			framework.Failf("Failed to set up watch: %v", err)
		}

		By("creating the cronJob")
		cronJob, err = cronJobClient.Create(cronJob)
		if err != nil {
			framework.Failf("Failed to create cronJob: %v", err)
		}

		By("verifying the cronJob is in kubernetes")
		options = metav1.ListOptions{
			LabelSelector:   selector,
			ResourceVersion: cronJob.ResourceVersion,
		}
		cronJobs, err = cronJobClient.List(options)
		if err != nil {
			framework.Failf("Failed to query for cronJobs: %v", err)
		}
		Expect(len(cronJobs.Items)).To(Equal(1))

		By("verifying cronJob creation was observed")
		observeCreation(w)

		By("deleting the cronJob")
		if err := cronJobClient.Delete(cronJob.Name, nil); err != nil {
			framework.Failf("Failed to delete cronJob: %v", err)
		}

		options = metav1.ListOptions{LabelSelector: selector}
		cronJobs, err = cronJobClient.List(options)
		if err != nil {
			framework.Failf("Failed to list cronJobs to verify deletion: %v", err)
		}
		Expect(len(cronJobs.Items)).To(Equal(0))
	})
})

var _ = framework.KubeDescribe("Staging client repo client", func() {
	f := framework.NewDefaultFramework("clientset")
	It("should create pods, delete pods, watch pods", func() {
		podClient := f.StagingClient.Core().Pods(f.Namespace.Name)
		By("constructing the pod")
		name := "pod" + string(uuid.NewUUID())
		value := strconv.Itoa(time.Now().Nanosecond())
		podCopy := stagingClientPod(name, value)
		pod := &podCopy
		By("verifying no pod exists before the test")
		pods, err := podClient.List(metav1.ListOptions{})
		if err != nil {
			framework.Failf("Failed to query for pods: %v", err)
		}
		Expect(len(pods.Items)).To(Equal(0))
		By("creating the pod")
		pod, err = podClient.Create(pod)
		if err != nil {
			framework.Failf("Failed to create pod: %v", err)
		}

		By("verifying the pod is in kubernetes")
		timeout := 1 * time.Minute
		if err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
			pods, err = podClient.List(metav1.ListOptions{})
			if err != nil {
				return false, err
			}
			if len(pods.Items) == 1 {
				return true, nil
			}
			return false, nil
		}); err != nil {
			framework.Failf("Err : %s\n. Failed to wait for 1 pod to be created", err)
		}
	})
})
<|endoftext|>


<commit_before>package boleto

const templateBoletoDefault = `

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">

    <link rel="stylesheet" href="https://code.ionicframework.com/ionicons/2.0.1/css/ionicons.min.css">

    {{if eq .Format "html"}}

            <button class="no-print btnDefault print" onclick="window.print()">

                &nbspImprimir
            </button>
            <button class="no-print btnDefault print" onclick="window.location='./boleto?fmt=pdf&id={{.View.ID}}'">

                &nbspGerar PDF
            </button>

    {{end}}
    {{template "boletoForm" .}}

    {{template "boletoForm" .}}

`

const boletoFormDefault = `
{{define "boletoForm"}}

                    {{.ConfigBank.Logo}}

                {{.View.BankNumber}}

                    Local de Pagamento

                    ATÉ O VENCIMENTO EM QUALQUER BANCO OU CORRESPONDENTE NÃO BANCÁRIO

                    Data de Vencimento

{{.View.Boleto.Title.ExpireDateTime | brdate}}

                    Nome do Beneficiário / CNPJ / CPF / Endereço:

                            {{.View.Boleto.Recipient.Name}}
                            {{.View.Boleto.Recipient.Document.Type}} {{fmtDoc .View.Boleto.Recipient.Document}}

                    {{.View.Boleto.Recipient.Address.Street}},
                    {{.View.Boleto.Recipient.Address.Number}} -
                    {{.View.Boleto.Recipient.Address.District}},
                    {{.View.Boleto.Recipient.Address.StateCode}} -
                    {{.View.Boleto.Recipient.Address.ZipCode}}

                    Agência/Código Beneficiário

                        {{.View.Boleto.Agreement.Agency}} / {{if eq .View.BankNumber "033-7"}}
                            {{.View.Boleto.Agreement.AgreementNumber}}
                        {{else if eq .View.BankNumber "341-7" }}
                            {{.View.Boleto.Agreement.Account}}-{{.View.Boleto.Agreement.AccountDigit}}
                        {{else if eq .View.BankNumber "104-0" }}
                            {{.View.Boleto.Agreement.AgreementNumber}}
                        {{else}}
                            {{.View.Boleto.Agreement.Account}}
                        {{end}}

                    Data do Documento

{{.View.Boleto.Title.CreateDate | brdate}}

                    Num. do Documento

{{.View.Boleto.Title.DocumentNumber}}

                    Espécie doc

{{.ConfigBank.EspecieDoc}}

                    Aceite

{{.ConfigBank.Aceite}}

                    Data Processamento

{{.View.Boleto.Title.CreateDate | brdate}}

                    {{if eq .View.BankNumber "104-0"}}
                        Nosso Número

{{.View.Boleto.Title.OurNumber}}-{{mod11dv .View.Boleto.Title.OurNumber true}}

                    {{else}}
                        Carteira/Nosso Número

{{.View.Boleto.Agreement.Wallet}}/{{.View.Boleto.Title.OurNumber}}

                    {{end}}

                {{if eq .View.BankNumber "033-7"}}

                                Carteira

COBRANCA SIMPLES RCR

                {{else}}

                    Uso do Banco

                            {{if eq .View.BankNumber "237-2"}}

                                    Cip

865

                            {{end}}

                                Carteira

                                {{if eq .View.BankNumber "104-0"}}
                                    RG
                                {{else}}
                                    {{.View.Boleto.Agreement.Wallet}}
                                {{end}}

                {{end}}

                    Espécie

{{.ConfigBank.Moeda}}

                    Quantidade

{{.ConfigBank.Quantidade}}

                    Valor

{{.ConfigBank.ValorCotacao}}

                    (=) Valor do Documento

{{fmtNumber .View.Boleto.Title.AmountInCents}}

                    Instruções de responsabilidade do BENEFICIÁRIO. Qualquer dúvida sobre este boleto contate o beneficiário.

{{.View.Boleto.Title.Instructions }}

                    (-) Descontos/Abatimento

                    (+) Juros/Multa

                    (=) Valor Pago

                            Nome do Pagador:  {{.View.Boleto.Buyer.Name}}
                            CNPJ/CPF:  {{fmtDoc .View.Boleto.Buyer.Document}}

                            Endereço:  {{.View.Boleto.Buyer.Address.Street}} {{.View.Boleto.Buyer.Address.Number}}, {{.View.Boleto.Buyer.Address.District}} - {{.View.Boleto.Buyer.Address.City}}, {{.View.Boleto.Buyer.Address.StateCode}} - {{.View.Boleto.Buyer.Address.ZipCode}}

                            Sacador/Avalista:
                            CNPJ/CPF:

            {{if eq .View.BankNumber "104-0" }}

                        Contato do Banco:

                        Telefone:

                        {{.ConfigBank.Telefone}}

                        Site:

                        {{.ConfigBank.Site}}

            {{end}}

	{{end}}
`

func getTemplateDefault() (string, string) {
	return templateBoletoDefault, boletoFormDefault
}
<commit_msg>clean template default<commit_after>package boleto

const templateBoletoDefault = `

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">

    <link rel="stylesheet" href="https://code.ionicframework.com/ionicons/2.0.1/css/ionicons.min.css">

    {{if eq .Format "html"}}

            <button class="no-print btnDefault print" onclick="window.print()">

                &nbspImprimir
            </button>
            <button class="no-print btnDefault print" onclick="window.location='./boleto?fmt=pdf&id={{.View.ID}}'">

                &nbspGerar PDF
            </button>

    {{end}}
    {{template "boletoForm" .}}

    {{template "boletoForm" .}}

`

const boletoFormDefault = `
{{define "boletoForm"}}

                    {{.ConfigBank.Logo}}

                {{.View.BankNumber}}

                    Local de Pagamento

                    ATÉ O VENCIMENTO EM QUALQUER BANCO OU CORRESPONDENTE NÃO BANCÁRIO

                    Data de Vencimento

{{.View.Boleto.Title.ExpireDateTime | brdate}}

                    Nome do Beneficiário / CNPJ / CPF / Endereço:

                            {{.View.Boleto.Recipient.Name}}
                            {{.View.Boleto.Recipient.Document.Type}} {{fmtDoc .View.Boleto.Recipient.Document}}

                    {{.View.Boleto.Recipient.Address.Street}},
                    {{.View.Boleto.Recipient.Address.Number}} -
                    {{.View.Boleto.Recipient.Address.District}},
                    {{.View.Boleto.Recipient.Address.StateCode}} -
                    {{.View.Boleto.Recipient.Address.ZipCode}}

                    Agência/Código Beneficiário

                        {{.View.Boleto.Agreement.Agency}} / {{if eq .View.BankNumber "033-7"}}
                            {{.View.Boleto.Agreement.AgreementNumber}}
                        {{else if eq .View.BankNumber "341-7" }}
                            {{.View.Boleto.Agreement.Account}}-{{.View.Boleto.Agreement.AccountDigit}}
                        {{else if eq .View.BankNumber "104-0" }}
                            {{.View.Boleto.Agreement.AgreementNumber}}
                        {{else}}
                            {{.View.Boleto.Agreement.Account}}
                        {{end}}

                    Data do Documento

{{.View.Boleto.Title.CreateDate | brdate}}

                    Num. do Documento

{{.View.Boleto.Title.DocumentNumber}}

                    Espécie doc

{{.ConfigBank.EspecieDoc}}

                    Aceite

{{.ConfigBank.Aceite}}

                    Data Processamento

{{.View.Boleto.Title.CreateDate | brdate}}

                    {{if eq .View.BankNumber "104-0"}}
                        Nosso Número

{{.View.Boleto.Title.OurNumber}}-{{mod11dv .View.Boleto.Title.OurNumber true}}

                    {{else}}
                        Carteira/Nosso Número

{{.View.Boleto.Agreement.Wallet}}/{{.View.Boleto.Title.OurNumber}}

                    {{end}}

                {{if eq .View.BankNumber "033-7"}}

                                Carteira

COBRANCA SIMPLES RCR

                {{else}}

                    Uso do Banco

                            {{if eq .View.BankNumber "237-2"}}

                                    Cip

865

                            {{end}}

                                Carteira

                                {{if eq .View.BankNumber "104-0"}}
                                    RG
                                {{else}}
                                    {{.View.Boleto.Agreement.Wallet}}
                                {{end}}

                {{end}}

                    Espécie

{{.ConfigBank.Moeda}}

                    Quantidade

{{.ConfigBank.Quantidade}}

                    Valor

{{.ConfigBank.ValorCotacao}}

                    (=) Valor do Documento

{{fmtNumber .View.Boleto.Title.AmountInCents}}

                    Instruções de responsabilidade do BENEFICIÁRIO. Qualquer dúvida sobre este boleto contate o beneficiário.

{{.View.Boleto.Title.Instructions }}

                    (-) Descontos/Abatimento

                    (+) Juros/Multa

                    (=) Valor Pago

                            Nome do Pagador:  {{.View.Boleto.Buyer.Name}}
                            CNPJ/CPF:  {{fmtDoc .View.Boleto.Buyer.Document}}

                            Endereço:  {{.View.Boleto.Buyer.Address.Street}} {{.View.Boleto.Buyer.Address.Number}}, {{.View.Boleto.Buyer.Address.District}} - {{.View.Boleto.Buyer.Address.City}}, {{.View.Boleto.Buyer.Address.StateCode}} - {{.View.Boleto.Buyer.Address.ZipCode}}

                            Sacador/Avalista:
                            CNPJ/CPF:

	{{end}}
`

func getTemplateDefault() (string, string) {
	return templateBoletoDefault, boletoFormDefault
}
<|endoftext|>