<commit_before>package java

import (
	"fmt"
	"io"
	"strings"

	"github.com/anchore/syft/internal/log"

	"github.com/anchore/syft/internal"
	"github.com/anchore/syft/internal/file"
	"github.com/anchore/syft/syft/cataloger/common"
	"github.com/anchore/syft/syft/pkg"
)

// integrity check
var _ common.ParserFn = parseJavaArchive

var archiveFormatGlobs = []string{
	"**/*.jar",
	"**/*.war",
	"**/*.ear",
	"**/*.jpi",
	"**/*.hpi",
}

type archiveParser struct {
	discoveredPkgs internal.StringSet
	fileManifest   file.ZipFileManifest
	virtualPath    string
	archivePath    string
	contentPath    string
	fileInfo       archiveFilename
	detectNested   bool
}

// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
	parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
	// note: even on error, we should always run cleanup functions
	defer cleanupFn()
	if err != nil {
		return nil, err
	}
	return parser.parse()
}

// uniquePkgKey creates a unique string to identify the given package.
func uniquePkgKey(p *pkg.Package) string {
	if p == nil {
		return ""
	}
	return fmt.Sprintf("%s|%s", p.Name, p.Version)
}

// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
// and parse nested archives or ignore them.
func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) {
	contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
	}

	fileManifest, err := file.NewZipFileManifest(archivePath)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
	}

	// fetch the last element of the virtual path
	virtualElements := strings.Split(virtualPath, ":")
	currentFilepath := virtualElements[len(virtualElements)-1]

	return &archiveParser{
		discoveredPkgs: internal.NewStringSet(),
		fileManifest:   fileManifest,
		virtualPath:    virtualPath,
		archivePath:    archivePath,
		contentPath:    contentPath,
		fileInfo:       newJavaArchiveFilename(currentFilepath),
		detectNested:   detectNested,
	}, cleanupFn, nil
}

// parse the loaded archive and return all packages found.
func (j *archiveParser) parse() ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	// find the parent package from the java manifest
	parentPkg, err := j.discoverMainPackage()
	if err != nil {
		return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
	}

	// don't add the parent package yet, we still may discover aux info to add to the metadata (but still track it as added to prevent duplicates)
	parentKey := uniquePkgKey(parentPkg)
	if parentKey != "" {
		j.discoveredPkgs.Add(parentKey)
	}

	// find aux packages from pom.properties
	auxPkgs, err := j.discoverPkgsFromPomProperties(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, auxPkgs...)

	// find nested java archive packages
	nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, nestedPkgs...)

	// lastly, add the parent package to the list (assuming the parent exists)
	if parentPkg != nil {
		// only the parent package gets the type, nested packages may be of a different package type (or not of a package type at all, since they may not be bundled)
		parentPkg.Type = j.fileInfo.pkgType()
		pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
	}

	return pkgs, nil
}

// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
	// search and parse java manifest files
	manifestMatches := j.fileManifest.GlobMatch(manifestGlob)
	if len(manifestMatches) > 1 {
		return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
	} else if len(manifestMatches) == 0 {
		// we did not find any manifests, but that may not be a problem (there may be other information to generate packages for)
		return nil, nil
	}

	// fetch the manifest file
	contents, err := file.ContentsFromZip(j.archivePath, manifestMatches...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.virtualPath, err)
	}

	// parse the manifest file into a rich object
	manifestContents := contents[manifestMatches[0]]
	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
	if err != nil {
		return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
	}

	return &pkg.Package{
		Name:         selectName(manifest, j.fileInfo),
		Version:      selectVersion(manifest, j.fileInfo),
		Language:     pkg.Java,
		Type:         pkg.JavaPkg,
		MetadataType: pkg.JavaMetadataType,
		Metadata: pkg.JavaMetadata{
			VirtualPath: j.virtualPath,
			Manifest:    manifest,
		},
	}, nil
}

// discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and
// associating each discovered package to the given parent package.
// nolint:funlen,gocognit
func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)
	parentKey := uniquePkgKey(parentPkg)

	// search and parse pom.properties files & fetch the contents
	contents, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract pom.properties: %w", err)
	}

	// parse the manifest file into a rich object
	for propsPath, propsContents := range contents {
		propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents))
		if err != nil {
			return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", j.virtualPath, err)
		}

		if propsObj != nil {
			if propsObj.Version != "" && propsObj.ArtifactID != "" {
				// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package

				// keep the artifact name within the virtual path if this package does not match the parent package
				vPathSuffix := ""
				if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
					vPathSuffix += ":" + propsObj.ArtifactID
				}
				virtualPath := j.virtualPath + vPathSuffix

				// discovered props = new package
				p := pkg.Package{
					Name:         propsObj.ArtifactID,
					Version:      propsObj.Version,
					Language:     pkg.Java,
					Type:         pkg.JavaPkg,
					MetadataType: pkg.JavaMetadataType,
					Metadata: pkg.JavaMetadata{
						VirtualPath:   virtualPath,
						PomProperties: propsObj,
						Parent:        parentPkg,
					},
				}

				pkgKey := uniquePkgKey(&p)

				// the name/version pair matches...
				matchesParentPkg := pkgKey == parentKey

				// the virtual path matches...
				matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath

				// the pom artifactId has the parent name or vice versa
				if propsObj.ArtifactID != "" {
					matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name)
				}

				if matchesParentPkg {
					// we've run across more information about our parent package, add this info to the parent package metadata
					// the pom properties is typically a better source of information for name and version than the manifest
					if p.Name != parentPkg.Name {
						parentPkg.Name = p.Name
					}
					if p.Version != parentPkg.Version {
						parentPkg.Version = p.Version
					}

					parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
					if ok {
						parentMetadata.PomProperties = propsObj
						parentPkg.Metadata = parentMetadata
					}
				} else if !j.discoveredPkgs.Contains(pkgKey) {
					// only keep packages we haven't seen yet (and are not related to the parent package)
					pkgs = append(pkgs, p)
				}
			}
		}
	}
	return pkgs, nil
}

// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	if !j.detectNested {
		return pkgs, nil
	}

	// search and parse pom.properties files & fetch the contents
	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
	}

	// discover nested artifacts
	for archivePath, archiveOpener := range openers {
		archiveReadCloser, err := archiveOpener.Open()
		if err != nil {
			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
		}
		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
		if err != nil {
			if closeErr := archiveReadCloser.Close(); closeErr != nil {
				log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
			}
			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
		}
		if err = archiveReadCloser.Close(); err != nil {
			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
		}

		// attach the parent package to all discovered packages that are not already associated with a java archive
		for _, p := range nestedPkgs {
			if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
				if metadata.Parent == nil {
					metadata.Parent = parentPkg
				}
				p.Metadata = metadata
			}
			pkgs = append(pkgs, p)
		}
	}

	return pkgs, nil
}
<commit_msg>ensure java parent pkg ref isnt nil when looking for parent matches<commit_after>package java

import (
	"fmt"
	"io"
	"strings"

	"github.com/anchore/syft/internal/log"

	"github.com/anchore/syft/internal"
	"github.com/anchore/syft/internal/file"
	"github.com/anchore/syft/syft/cataloger/common"
	"github.com/anchore/syft/syft/pkg"
)

// integrity check
var _ common.ParserFn = parseJavaArchive

var archiveFormatGlobs = []string{
	"**/*.jar",
	"**/*.war",
	"**/*.ear",
	"**/*.jpi",
	"**/*.hpi",
}

type archiveParser struct {
	discoveredPkgs internal.StringSet
	fileManifest   file.ZipFileManifest
	virtualPath    string
	archivePath    string
	contentPath    string
	fileInfo       archiveFilename
	detectNested   bool
}

// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
	parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
	// note: even on error, we should always run cleanup functions
	defer cleanupFn()
	if err != nil {
		return nil, err
	}
	return parser.parse()
}

// uniquePkgKey creates a unique string to identify the given package.
func uniquePkgKey(p *pkg.Package) string {
	if p == nil {
		return ""
	}
	return fmt.Sprintf("%s|%s", p.Name, p.Version)
}

// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
// and parse nested archives or ignore them.
func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) {
	contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
	}

	fileManifest, err := file.NewZipFileManifest(archivePath)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
	}

	// fetch the last element of the virtual path
	virtualElements := strings.Split(virtualPath, ":")
	currentFilepath := virtualElements[len(virtualElements)-1]

	return &archiveParser{
		discoveredPkgs: internal.NewStringSet(),
		fileManifest:   fileManifest,
		virtualPath:    virtualPath,
		archivePath:    archivePath,
		contentPath:    contentPath,
		fileInfo:       newJavaArchiveFilename(currentFilepath),
		detectNested:   detectNested,
	}, cleanupFn, nil
}

// parse the loaded archive and return all packages found.
func (j *archiveParser) parse() ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	// find the parent package from the java manifest
	parentPkg, err := j.discoverMainPackage()
	if err != nil {
		return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
	}

	// don't add the parent package yet, we still may discover aux info to add to the metadata (but still track it as added to prevent duplicates)
	parentKey := uniquePkgKey(parentPkg)
	if parentKey != "" {
		j.discoveredPkgs.Add(parentKey)
	}

	// find aux packages from pom.properties
	auxPkgs, err := j.discoverPkgsFromPomProperties(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, auxPkgs...)

	// find nested java archive packages
	nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, nestedPkgs...)

	// lastly, add the parent package to the list (assuming the parent exists)
	if parentPkg != nil {
		// only the parent package gets the type, nested packages may be of a different package type (or not of a package type at all, since they may not be bundled)
		parentPkg.Type = j.fileInfo.pkgType()
		pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
	}

	return pkgs, nil
}

// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
	// search and parse java manifest files
	manifestMatches := j.fileManifest.GlobMatch(manifestGlob)
	if len(manifestMatches) > 1 {
		return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
	} else if len(manifestMatches) == 0 {
		// we did not find any manifests, but that may not be a problem (there may be other information to generate packages for)
		return nil, nil
	}

	// fetch the manifest file
	contents, err := file.ContentsFromZip(j.archivePath, manifestMatches...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.virtualPath, err)
	}

	// parse the manifest file into a rich object
	manifestContents := contents[manifestMatches[0]]
	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
	if err != nil {
		return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
	}

	return &pkg.Package{
		Name:         selectName(manifest, j.fileInfo),
		Version:      selectVersion(manifest, j.fileInfo),
		Language:     pkg.Java,
		Type:         pkg.JavaPkg,
		MetadataType: pkg.JavaMetadataType,
		Metadata: pkg.JavaMetadata{
			VirtualPath: j.virtualPath,
			Manifest:    manifest,
		},
	}, nil
}

// discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and
// associating each discovered package to the given parent package.
// nolint:funlen,gocognit
func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)
	parentKey := uniquePkgKey(parentPkg)

	// search and parse pom.properties files & fetch the contents
	contents, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract pom.properties: %w", err)
	}

	// parse the manifest file into a rich object
	for propsPath, propsContents := range contents {
		propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents))
		if err != nil {
			return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", j.virtualPath, err)
		}

		if propsObj != nil {
			if propsObj.Version != "" && propsObj.ArtifactID != "" {
				// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package

				// keep the artifact name within the virtual path if this package does not match the parent package
				vPathSuffix := ""
				if parentPkg != nil && !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
					vPathSuffix += ":" + propsObj.ArtifactID
				}
				virtualPath := j.virtualPath + vPathSuffix

				// discovered props = new package
				p := pkg.Package{
					Name:         propsObj.ArtifactID,
					Version:      propsObj.Version,
					Language:     pkg.Java,
					Type:         pkg.JavaPkg,
					MetadataType: pkg.JavaMetadataType,
					Metadata: pkg.JavaMetadata{
						VirtualPath:   virtualPath,
						PomProperties: propsObj,
						Parent:        parentPkg,
					},
				}

				pkgKey := uniquePkgKey(&p)

				// the name/version pair matches...
				matchesParentPkg := pkgKey == parentKey

				if parentPkg != nil {
					// the virtual path matches...
					matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath

					// the pom artifactId has the parent name or vice versa
					if propsObj.ArtifactID != "" {
						matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name)
					}

					if matchesParentPkg {
						// we've run across more information about our parent package, add this info to the parent package metadata
						// the pom properties is typically a better source of information for name and version than the manifest
						if p.Name != parentPkg.Name {
							parentPkg.Name = p.Name
						}
						if p.Version != parentPkg.Version {
							parentPkg.Version = p.Version
						}

						parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
						if ok {
							parentMetadata.PomProperties = propsObj
							parentPkg.Metadata = parentMetadata
						}
					}
				}

				if !matchesParentPkg && !j.discoveredPkgs.Contains(pkgKey) {
					// only keep packages we haven't seen yet (and are not related to the parent package)
					pkgs = append(pkgs, p)
				}
			}
		}
	}
	return pkgs, nil
}

// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	if !j.detectNested {
		return pkgs, nil
	}

	// search and parse pom.properties files & fetch the contents
	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
	}

	// discover nested artifacts
	for archivePath, archiveOpener := range openers {
		archiveReadCloser, err := archiveOpener.Open()
		if err != nil {
			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
		}
		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
		if err != nil {
			if closeErr := archiveReadCloser.Close(); closeErr != nil {
				log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
			}
			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
		}
		if err = archiveReadCloser.Close(); err != nil {
			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
		}

		// attach the parent package to all discovered packages that are not already associated with a java archive
		for _, p := range nestedPkgs {
			if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
				if metadata.Parent == nil {
					metadata.Parent = parentPkg
				}
				p.Metadata = metadata
			}
			pkgs = append(pkgs, p)
		}
	}

	return pkgs, nil
}
<|endoftext|>


<commit_before>package cred_test

import (
	"testing"
	"github.com/viant/toolbox/cred"
	"github.com/stretchr/testify/assert"
)

func TestNewBlowfishCipher(t *testing.T) {
	cipher, err := cred.NewBlowfishCipher(cred.DefaultKey)
	if assert.Nil(t, err) {

		{
			var secret = "This is secret pass12312312321"
			encrypted := cipher.Encrypt([]byte(secret))
			decrypted := cipher.Decrypt(encrypted)
			assert.Equal(t, secret, string(decrypted))
		}

		{
			var secret = "abc"
			encrypted := cipher.Encrypt([]byte(secret))
			decrypted := cipher.Decrypt(encrypted)
			assert.Equal(t, secret, string(decrypted))
		}
	}
}

<commit_msg>added more test<commit_after>package cred_test

import (
	"testing"
	"github.com/viant/toolbox/cred"
	"github.com/stretchr/testify/assert"
)

func TestNewBlowfishCipher(t *testing.T) {
	cipher, err := cred.NewBlowfishCipher(cred.DefaultKey)
	if assert.Nil(t, err) {

		{
			var secret = "This is secret pass12312312321"
			encrypted := cipher.Encrypt([]byte(secret))
			decrypted := cipher.Decrypt(encrypted)
			assert.Equal(t, secret, string(decrypted))
		}

		{
			var secret = "abc"
			encrypted := cipher.Encrypt([]byte(secret))
			decrypted := cipher.Decrypt(encrypted)
			assert.Equal(t, secret, string(decrypted))
		}

		{
			var secret = "123!abc"
			encrypted := cipher.Encrypt([]byte(secret))
			decrypted := cipher.Decrypt(encrypted)
			assert.Equal(t, secret, string(decrypted))
		}

		{
			var secret = "test123@423 #!424"
			encrypted := cipher.Encrypt([]byte(secret))
			decrypted := cipher.Decrypt(encrypted)
			assert.Equal(t, secret, string(decrypted))
		}
	}
}

<|endoftext|>


<commit_before>package sockd

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/md5"
	"crypto/rand"
	"errors"
	"github.com/HouzuoGuo/laitos/daemon/dnsd"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/testingstub"
	"io"
	pseudoRand "math/rand"
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

const (
	MD5SumLength         = 16
	IOTimeoutSec         = time.Duration(300 * time.Second)
	RateLimitIntervalSec = 1
	MaxPacketSize        = 9038
)

var BlockedReservedCIDR = []net.IPNet{
	{IP: net.IPv4(10, 0, 0, 0), Mask: net.CIDRMask(8, 32)},
	{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)},
	{IP: net.IPv4(127, 0, 0, 0), Mask: net.CIDRMask(8, 32)},
	{IP: net.IPv4(169, 254, 0, 0), Mask: net.CIDRMask(16, 32)},
	{IP: net.IPv4(172, 16, 0, 0), Mask: net.CIDRMask(12, 32)},
	{IP: net.IPv4(192, 0, 0, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(192, 0, 2, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(192, 168, 0, 0), Mask: net.CIDRMask(16, 32)},
	{IP: net.IPv4(198, 18, 0, 0), Mask: net.CIDRMask(15, 32)},
	{IP: net.IPv4(198, 51, 100, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(203, 0, 113, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(240, 0, 0, 0), Mask: net.CIDRMask(4, 32)},
}

func IsReservedAddr(addr net.IP) bool {
	if addr == nil {
		return false
	}
	for _, reservedCIDR := range BlockedReservedCIDR {
		if reservedCIDR.Contains(addr) {
			return true
		}
	}
	return false
}

// Daemon is intentionally undocumented magic ^____^
type Daemon struct {
	Address    string `json:"Address"`
	Password   string `json:"Password"`
	PerIPLimit int    `json:"PerIPLimit"`
	TCPPort    int    `json:"TCPPort"`
	UDPPort    int    `json:"UDPPort"`

	DNSDaemon *dnsd.Daemon `json:"-"` // it is assumed to be already initialised

	tcpListener  net.Listener
	rateLimitTCP *misc.RateLimit

	udpBackLog       *UDPBackLog
	udpListener      *net.UDPConn
	udpTable         *UDPTable
	rateLimitUDP     *misc.RateLimit
	udpLoopIsRunning int32
	stopUDP          chan bool

	cipher *Cipher
	logger misc.Logger
}

func (daemon *Daemon) Initialise() error {
	if daemon.Address == "" {
		daemon.Address = "0.0.0.0"
	}
	if daemon.PerIPLimit < 1 {
		daemon.PerIPLimit = 96
	}
	daemon.logger = misc.Logger{
		ComponentName: "sockd",
		ComponentID:   []misc.LoggerIDField{{"Addr", daemon.Address}, {"TCP", daemon.TCPPort}, {"UDP", daemon.UDPPort}},
	}
	if daemon.DNSDaemon == nil {
		return errors.New("sockd.Initialise: dns daemon must be assigned")
	}
	if daemon.TCPPort < 1 {
		return errors.New("sockd.Initialise: TCP listen port must be greater than 0")
	}
	if len(daemon.Password) < 7 {
		return errors.New("sockd.Initialise: password must be at least 7 characters long")
	}
	daemon.rateLimitTCP = &misc.RateLimit{
		Logger:   daemon.logger,
		MaxCount: daemon.PerIPLimit,
		UnitSecs: RateLimitIntervalSec,
	}
	daemon.rateLimitTCP.Initialise()
	daemon.rateLimitUDP = &misc.RateLimit{
		Logger:   daemon.logger,
		MaxCount: daemon.PerIPLimit * 100,
		UnitSecs: RateLimitIntervalSec,
	}
	daemon.rateLimitUDP.Initialise()

	daemon.cipher = &Cipher{}
	daemon.cipher.Initialise(daemon.Password)

	daemon.udpBackLog = &UDPBackLog{backlog: make(map[string][]byte), mutex: new(sync.Mutex)}

	daemon.stopUDP = make(chan bool)
	return nil
}

func (daemon *Daemon) StartAndBlock() error {
	numListeners := 0
	errChan := make(chan error, 2)
	if daemon.TCPPort != 0 {
		numListeners++
		go func() {
			err := daemon.StartAndBlockTCP()
			errChan <- err
		}()
	}
	if daemon.UDPPort != 0 {
		numListeners++
		go func() {
			err := daemon.StartAndBlockUDP()
			errChan <- err
		}()
	}
	for i := 0; i < numListeners; i++ {
		if err := <-errChan; err != nil {
			daemon.Stop()
			return err
		}
	}
	return nil
}

func (daemon *Daemon) Stop() {
	if listener := daemon.tcpListener; listener != nil {
		if err := listener.Close(); err != nil {
			daemon.logger.Warning("Stop", "", err, "failed to close TCP listener")
		}
	}
	if listener := daemon.udpListener; listener != nil {
		if atomic.CompareAndSwapInt32(&daemon.udpLoopIsRunning, 1, 0) {
			daemon.stopUDP <- true
		}
		if err := listener.Close(); err != nil {
			daemon.logger.Warning("Stop", "", err, "failed to close UDP listener")
		}
	}
}

type Cipher struct {
	EncryptionStream cipher.Stream
	DecryptionStream cipher.Stream
	Key              []byte
	IV               []byte
	KeyLength        int
	IVLength         int
}

func md5Sum(d []byte) []byte {
	md5Digest := md5.New()
	md5Digest.Write(d)
	return md5Digest.Sum(nil)
}

func (cip *Cipher) Initialise(password string) {
	cip.KeyLength = 32
	cip.IVLength = 16

	segmentLength := (cip.KeyLength-1)/MD5SumLength + 1
	buf := make([]byte, segmentLength*MD5SumLength)
	copy(buf, md5Sum([]byte(password)))
	destinationBuf := make([]byte, MD5SumLength+len(password))
	start := 0
	for i := 1; i < segmentLength; i++ {
		start += MD5SumLength
		copy(destinationBuf, buf[start-MD5SumLength:start])
		copy(destinationBuf[MD5SumLength:], password)
		copy(buf[start:], md5Sum(destinationBuf))
	}
	cip.Key = buf[:cip.KeyLength]
}

func (cip *Cipher) GetCipherStream(key, iv []byte) (cipher.Stream, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	return cipher.NewCTR(block, iv), nil
}

func (cip *Cipher) InitEncryptionStream() (iv []byte) {
	var err error
	if cip.IV == nil {
		iv = make([]byte, cip.IVLength)
		if _, err = io.ReadFull(rand.Reader, iv); err != nil {
			panic(err)
		}
		cip.IV = iv
	} else {
		iv = cip.IV
	}
	cip.EncryptionStream, err = cip.GetCipherStream(cip.Key, iv)
	if err != nil {
		panic(err)
	}
	return
}

func (cip *Cipher) Encrypt(dest, src []byte) {
	cip.EncryptionStream.XORKeyStream(dest, src)
}

func (cip *Cipher) InitDecryptionStream(iv []byte) {
	var err error
	cip.DecryptionStream, err = cip.GetCipherStream(cip.Key, iv)
	if err != nil {
		panic(err)
	}
}

func (cip *Cipher) Decrypt(dest, src []byte) {
	cip.DecryptionStream.XORKeyStream(dest, src)
}

func (cip *Cipher) Copy() *Cipher {
	newCipher := *cip
	newCipher.EncryptionStream = nil
	newCipher.DecryptionStream = nil
	return &newCipher
}

var randSeed = int(time.Now().UnixNano())

func RandNum(absMin, variableLower, randMore int) int {
	return absMin + randSeed%variableLower + pseudoRand.Intn(randMore)
}

func TestSockd(sockd *Daemon, t testingstub.T) {
	var stopped bool
	go func() {
		if err := sockd.StartAndBlock(); err != nil {
			t.Fatal(err)
		}
		stopped = true
	}()
	time.Sleep(2 * time.Second)
	if conn, err := net.Dial("tcp", sockd.Address+":"+strconv.Itoa(sockd.TCPPort)); err != nil {
		t.Fatal(err)
	} else if n, err := conn.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); err != nil && n != 10 {
		t.Fatal(err, n)
	}
	// Daemon should stop within a second
	sockd.Stop()
	time.Sleep(1 * time.Second)
	if !stopped {
		t.Fatal("did not stop")
	}
	// Repeatedly stopping the daemon should have no negative consequence
	sockd.Stop()
	sockd.Stop()
}
<commit_msg>set iotimeout to 900 seconds in sockd<commit_after>package sockd

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/md5"
	"crypto/rand"
	"errors"
	"github.com/HouzuoGuo/laitos/daemon/dnsd"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/testingstub"
	"io"
	pseudoRand "math/rand"
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

const (
	MD5SumLength         = 16
	IOTimeoutSec         = time.Duration(900 * time.Second)
	RateLimitIntervalSec = 1
	MaxPacketSize        = 9038
)

var BlockedReservedCIDR = []net.IPNet{
	{IP: net.IPv4(10, 0, 0, 0), Mask: net.CIDRMask(8, 32)},
	{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)},
	{IP: net.IPv4(127, 0, 0, 0), Mask: net.CIDRMask(8, 32)},
	{IP: net.IPv4(169, 254, 0, 0), Mask: net.CIDRMask(16, 32)},
	{IP: net.IPv4(172, 16, 0, 0), Mask: net.CIDRMask(12, 32)},
	{IP: net.IPv4(192, 0, 0, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(192, 0, 2, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(192, 168, 0, 0), Mask: net.CIDRMask(16, 32)},
	{IP: net.IPv4(198, 18, 0, 0), Mask: net.CIDRMask(15, 32)},
	{IP: net.IPv4(198, 51, 100, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(203, 0, 113, 0), Mask: net.CIDRMask(24, 32)},
	{IP: net.IPv4(240, 0, 0, 0), Mask: net.CIDRMask(4, 32)},
}

func IsReservedAddr(addr net.IP) bool {
	if addr == nil {
		return false
	}
	for _, reservedCIDR := range BlockedReservedCIDR {
		if reservedCIDR.Contains(addr) {
			return true
		}
	}
	return false
}

// Daemon is intentionally undocumented magic ^____^
type Daemon struct {
	Address    string `json:"Address"`
	Password   string `json:"Password"`
	PerIPLimit int    `json:"PerIPLimit"`
	TCPPort    int    `json:"TCPPort"`
	UDPPort    int    `json:"UDPPort"`

	DNSDaemon *dnsd.Daemon `json:"-"` // it is assumed to be already initialised

	tcpListener  net.Listener
	rateLimitTCP *misc.RateLimit

	udpBackLog       *UDPBackLog
	udpListener      *net.UDPConn
	udpTable         *UDPTable
	rateLimitUDP     *misc.RateLimit
	udpLoopIsRunning int32
	stopUDP          chan bool

	cipher *Cipher
	logger misc.Logger
}

func (daemon *Daemon) Initialise() error {
	if daemon.Address == "" {
		daemon.Address = "0.0.0.0"
	}
	if daemon.PerIPLimit < 1 {
		daemon.PerIPLimit = 96
	}
	daemon.logger = misc.Logger{
		ComponentName: "sockd",
		ComponentID:   []misc.LoggerIDField{{"Addr", daemon.Address}, {"TCP", daemon.TCPPort}, {"UDP", daemon.UDPPort}},
	}
	if daemon.DNSDaemon == nil {
		return errors.New("sockd.Initialise: dns daemon must be assigned")
	}
	if daemon.TCPPort < 1 {
		return errors.New("sockd.Initialise: TCP listen port must be greater than 0")
	}
	if len(daemon.Password) < 7 {
		return errors.New("sockd.Initialise: password must be at least 7 characters long")
	}
	daemon.rateLimitTCP = &misc.RateLimit{
		Logger:   daemon.logger,
		MaxCount: daemon.PerIPLimit,
		UnitSecs: RateLimitIntervalSec,
	}
	daemon.rateLimitTCP.Initialise()
	daemon.rateLimitUDP = &misc.RateLimit{
		Logger:   daemon.logger,
		MaxCount: daemon.PerIPLimit * 100,
		UnitSecs: RateLimitIntervalSec,
	}
	daemon.rateLimitUDP.Initialise()

	daemon.cipher = &Cipher{}
	daemon.cipher.Initialise(daemon.Password)

	daemon.udpBackLog = &UDPBackLog{backlog: make(map[string][]byte), mutex: new(sync.Mutex)}

	daemon.stopUDP = make(chan bool)
	return nil
}

func (daemon *Daemon) StartAndBlock() error {
	numListeners := 0
	errChan := make(chan error, 2)
	if daemon.TCPPort != 0 {
		numListeners++
		go func() {
			err := daemon.StartAndBlockTCP()
			errChan <- err
		}()
	}
	if daemon.UDPPort != 0 {
		numListeners++
		go func() {
			err := daemon.StartAndBlockUDP()
			errChan <- err
		}()
	}
	for i := 0; i < numListeners; i++ {
		if err := <-errChan; err != nil {
			daemon.Stop()
			return err
		}
	}
	return nil
}

func (daemon *Daemon) Stop() {
	if listener := daemon.tcpListener; listener != nil {
		if err := listener.Close(); err != nil {
			daemon.logger.Warning("Stop", "", err, "failed to close TCP listener")
		}
	}
	if listener := daemon.udpListener; listener != nil {
		if atomic.CompareAndSwapInt32(&daemon.udpLoopIsRunning, 1, 0) {
			daemon.stopUDP <- true
		}
		if err := listener.Close(); err != nil {
			daemon.logger.Warning("Stop", "", err, "failed to close UDP listener")
		}
	}
}

type Cipher struct {
	EncryptionStream cipher.Stream
	DecryptionStream cipher.Stream
	Key              []byte
	IV               []byte
	KeyLength        int
	IVLength         int
}

func md5Sum(d []byte) []byte {
	md5Digest := md5.New()
	md5Digest.Write(d)
	return md5Digest.Sum(nil)
}

func (cip *Cipher) Initialise(password string) {
	cip.KeyLength = 32
	cip.IVLength = 16

	segmentLength := (cip.KeyLength-1)/MD5SumLength + 1
	buf := make([]byte, segmentLength*MD5SumLength)
	copy(buf, md5Sum([]byte(password)))
	destinationBuf := make([]byte, MD5SumLength+len(password))
	start := 0
	for i := 1; i < segmentLength; i++ {
		start += MD5SumLength
		copy(destinationBuf, buf[start-MD5SumLength:start])
		copy(destinationBuf[MD5SumLength:], password)
		copy(buf[start:], md5Sum(destinationBuf))
	}
	cip.Key = buf[:cip.KeyLength]
}

func (cip *Cipher) GetCipherStream(key, iv []byte) (cipher.Stream, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	return cipher.NewCTR(block, iv), nil
}

func (cip *Cipher) InitEncryptionStream() (iv []byte) {
	var err error
	if cip.IV == nil {
		iv = make([]byte, cip.IVLength)
		if _, err = io.ReadFull(rand.Reader, iv); err != nil {
			panic(err)
		}
		cip.IV = iv
	} else {
		iv = cip.IV
	}
	cip.EncryptionStream, err = cip.GetCipherStream(cip.Key, iv)
	if err != nil {
		panic(err)
	}
	return
}

func (cip *Cipher) Encrypt(dest, src []byte) {
	cip.EncryptionStream.XORKeyStream(dest, src)
}

func (cip *Cipher) InitDecryptionStream(iv []byte) {
	var err error
	cip.DecryptionStream, err = cip.GetCipherStream(cip.Key, iv)
	if err != nil {
		panic(err)
	}
}

func (cip *Cipher) Decrypt(dest, src []byte) {
	cip.DecryptionStream.XORKeyStream(dest, src)
}

func (cip *Cipher) Copy() *Cipher {
	newCipher := *cip
	newCipher.EncryptionStream = nil
	newCipher.DecryptionStream = nil
	return &newCipher
}

var randSeed = int(time.Now().UnixNano())

func RandNum(absMin, variableLower, randMore int) int {
	return absMin + randSeed%variableLower + pseudoRand.Intn(randMore)
}

func TestSockd(sockd *Daemon, t testingstub.T) {
	var stopped bool
	go func() {
		if err := sockd.StartAndBlock(); err != nil {
			t.Fatal(err)
		}
		stopped = true
	}()
	time.Sleep(2 * time.Second)
	if conn, err := net.Dial("tcp", sockd.Address+":"+strconv.Itoa(sockd.TCPPort)); err != nil {
		t.Fatal(err)
	} else if n, err := conn.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); err != nil && n != 10 {
		t.Fatal(err, n)
	}
	// Daemon should stop within a second
	sockd.Stop()
	time.Sleep(1 * time.Second)
	if !stopped {
		t.Fatal("did not stop")
	}
	// Repeatedly stopping the daemon should have no negative consequence
	sockd.Stop()
	sockd.Stop()
}
<|endoftext|>


<commit_before>package report

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"sync"
	"time"
)

type ErrorStat struct {
	Count     int            `json:"count"`
	Locations map[string]int `json:"locations"`
}

type ReportMgr struct {
	lock        sync.Mutex
	JobName     string    `json:"job"`
	BuildNum    int       `json:"build"`
	Deployments int       `json:"deployments"`
	Errors      int       `json:"errors"`
	StartTime   time.Time `json:"startTime"`
	Duration    string    `json:"duration"`
	// Failure map: key=error, value=locations
	Failures map[string]*ErrorStat `json:"failures"`
}

var errorRegexpMap map[string]string

func init() {
	errorRegexpMap = map[string]string{
		"azcli run":  "_init__.py",
		"azcli load": "Error loading command module",

		"VMStartTimedOut":                                "VMStartTimedOut",
		"OSProvisioningTimedOut":                         "OSProvisioningTimedOut",
		"VMExtensionProvisioningError":                   "VMExtensionProvisioningError",
		"VMExtensionProvisioningTimeout":                 "VMExtensionProvisioningTimeout",
		"InternalExecutionError":                         "InternalExecutionError",
		"SkuNotAvailable":                                "SkuNotAvailable",
		"MaxStorageAccountsCountPerSubscriptionExceeded": "MaxStorageAccountsCountPerSubscriptionExceeded",
		"ImageManagementOperationError":                  "ImageManagementOperationError",
		"DiskProcessingError":                            "DiskProcessingError",
		"DiskServiceInternalError":                       "DiskServiceInternalError",
		"AllocationFailed":                               "AllocationFailed",
		"NetworkingInternalOperationError":               "NetworkingInternalOperationError",

		"K8S curl error":                  "curl_error",
		"K8S no external IP":              "gave up waiting for loadbalancer to get an ingress ip",
		"K8S nodes not ready":             "gave up waiting for apiserver",
		"K8S service unreachable":         "gave up waiting for service to be externally reachable",
		"K8S nginx unreachable":           "failed to get expected response from nginx through the loadbalancer",
		"DCOS nodes not ready":            "gave up waiting for DCOS nodes",
		"DCOS marathon validation failed": "dcos/test.sh] marathon validation failed",
		"DCOS marathon not added":         "dcos/test.sh] gave up waiting for marathon to be added",
		"DCOS marathon-lb not installed":  "Failed to install marathon-lb",
	}
}

func New(jobName string, buildNum int, nDeploys int) *ReportMgr {
	h := &ReportMgr{}
	h.JobName = jobName
	h.BuildNum = buildNum
	h.Deployments = nDeploys
	h.Errors = 0
	h.StartTime = time.Now().UTC()
	h.Failures = make(map[string]*ErrorStat)
	return h
}

func (h *ReportMgr) Copy() *ReportMgr {
	n := New(h.JobName, h.BuildNum, h.Deployments)
	n.Errors = h.Errors
	n.StartTime = h.StartTime
	for e, f := range h.Failures {
		locs := make(map[string]int)
		for l, c := range f.Locations {
			locs[l] = c
		}
		n.Failures[e] = &ErrorStat{Count: f.Count, Locations: locs}
	}
	return n
}

func (h *ReportMgr) Process(txt, location string) {
	for key, regex := range errorRegexpMap {
		if match, _ := regexp.MatchString(regex, txt); match {
			h.addFailure(key, map[string]int{location: 1})
			return
		}
	}
	h.addFailure("Unspecified error", map[string]int{location: 1})
}

func (h *ReportMgr) addFailure(key string, locations map[string]int) {
	h.lock.Lock()
	defer h.lock.Unlock()

	cnt := 0

	if failure, ok := h.Failures[key]; !ok {
		locs := make(map[string]int)
		for l, c := range locations {
			locs[l] = c
			cnt += c
		}
		h.Failures[key] = &ErrorStat{Count: cnt, Locations: locs}
	} else {
		for l, c := range locations {
			cnt += c
			if _, ok := failure.Locations[l]; !ok {
				failure.Locations[l] = c
			} else {
				failure.Locations[l] += c
			}
		}
		failure.Count += cnt
	}
	h.Errors += cnt
}

func (h *ReportMgr) CreateTestReport(filepath string) error {
	h.Duration = time.Now().UTC().Sub(h.StartTime).String()
	data, err := json.MarshalIndent(h, "", "  ")
	if err != nil {
		return err
	}
	file, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = file.Write(data)
	if err != nil {
		return err
	}
	return nil
}

func (h *ReportMgr) CreateCombinedReport(filepath, testReportFname string) error {
	now := time.Now().UTC()
	combinedReport := h.Copy()
	for n := h.BuildNum - 1; n > 0; n-- {
		data, err := ioutil.ReadFile(fmt.Sprintf("%s/%d/%s/%s",
			os.Getenv("JOB_BUILD_ROOTDIR"), n, os.Getenv("JOB_BUILD_SUBDIR"), testReportFname))
		if err != nil {
			break
		}
		testReport := &ReportMgr{}
		if err := json.Unmarshal(data, &testReport); err != nil {
			break
		}
		// get combined report for past 24 hours
		if now.Sub(testReport.StartTime) > time.Duration(time.Hour*24) {
			break
		}
		combinedReport.StartTime = testReport.StartTime
		combinedReport.Deployments += testReport.Deployments

		for e, f := range testReport.Failures {
			combinedReport.addFailure(e, f.Locations)
		}
	}
	return combinedReport.CreateTestReport(filepath)
}
<commit_msg>configure number of recent test reports in the combined report (#908)<commit_after>package report

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"sync"
	"time"
)

type ErrorStat struct {
	Count     int            `json:"count"`
	Locations map[string]int `json:"locations"`
}

type ReportMgr struct {
	lock        sync.Mutex
	JobName     string    `json:"job"`
	BuildNum    int       `json:"build"`
	Deployments int       `json:"deployments"`
	Errors      int       `json:"errors"`
	StartTime   time.Time `json:"startTime"`
	Duration    string    `json:"duration"`
	// Failure map: key=error, value=locations
	Failures map[string]*ErrorStat `json:"failures"`
}

var errorRegexpMap map[string]string

func init() {
	errorRegexpMap = map[string]string{
		"azcli run":  "_init__.py",
		"azcli load": "Error loading command module",

		"VMStartTimedOut":                                "VMStartTimedOut",
		"OSProvisioningTimedOut":                         "OSProvisioningTimedOut",
		"VMExtensionProvisioningError":                   "VMExtensionProvisioningError",
		"VMExtensionProvisioningTimeout":                 "VMExtensionProvisioningTimeout",
		"InternalExecutionError":                         "InternalExecutionError",
		"SkuNotAvailable":                                "SkuNotAvailable",
		"MaxStorageAccountsCountPerSubscriptionExceeded": "MaxStorageAccountsCountPerSubscriptionExceeded",
		"ImageManagementOperationError":                  "ImageManagementOperationError",
		"DiskProcessingError":                            "DiskProcessingError",
		"DiskServiceInternalError":                       "DiskServiceInternalError",
		"AllocationFailed":                               "AllocationFailed",
		"NetworkingInternalOperationError":               "NetworkingInternalOperationError",

		"K8S curl error":                  "curl_error",
		"K8S no external IP":              "gave up waiting for loadbalancer to get an ingress ip",
		"K8S nodes not ready":             "gave up waiting for apiserver",
		"K8S service unreachable":         "gave up waiting for service to be externally reachable",
		"K8S nginx unreachable":           "failed to get expected response from nginx through the loadbalancer",
		"DCOS nodes not ready":            "gave up waiting for DCOS nodes",
		"DCOS marathon validation failed": "dcos/test.sh] marathon validation failed",
		"DCOS marathon not added":         "dcos/test.sh] gave up waiting for marathon to be added",
		"DCOS marathon-lb not installed":  "Failed to install marathon-lb",
	}
}

func New(jobName string, buildNum int, nDeploys int) *ReportMgr {
	h := &ReportMgr{}
	h.JobName = jobName
	h.BuildNum = buildNum
	h.Deployments = nDeploys
	h.Errors = 0
	h.StartTime = time.Now().UTC()
	h.Failures = make(map[string]*ErrorStat)
	return h
}

func (h *ReportMgr) Copy() *ReportMgr {
	n := New(h.JobName, h.BuildNum, h.Deployments)
	n.Errors = h.Errors
	n.StartTime = h.StartTime
	for e, f := range h.Failures {
		locs := make(map[string]int)
		for l, c := range f.Locations {
			locs[l] = c
		}
		n.Failures[e] = &ErrorStat{Count: f.Count, Locations: locs}
	}
	return n
}

func (h *ReportMgr) Process(txt, location string) {
	for key, regex := range errorRegexpMap {
		if match, _ := regexp.MatchString(regex, txt); match {
			h.addFailure(key, map[string]int{location: 1})
			return
		}
	}
	h.addFailure("Unspecified error", map[string]int{location: 1})
}

func (h *ReportMgr) addFailure(key string, locations map[string]int) {
	h.lock.Lock()
	defer h.lock.Unlock()

	cnt := 0

	if failure, ok := h.Failures[key]; !ok {
		locs := make(map[string]int)
		for l, c := range locations {
			locs[l] = c
			cnt += c
		}
		h.Failures[key] = &ErrorStat{Count: cnt, Locations: locs}
	} else {
		for l, c := range locations {
			cnt += c
			if _, ok := failure.Locations[l]; !ok {
				failure.Locations[l] = c
			} else {
				failure.Locations[l] += c
			}
		}
		failure.Count += cnt
	}
	h.Errors += cnt
}

func (h *ReportMgr) CreateTestReport(filepath string) error {
	h.Duration = time.Now().UTC().Sub(h.StartTime).String()
	data, err := json.MarshalIndent(h, "", "  ")
	if err != nil {
		return err
	}
	file, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = file.Write(data)
	if err != nil {
		return err
	}
	return nil
}

func (h *ReportMgr) CreateCombinedReport(filepath, testReportFname string) error {
	// "COMBINED_PAST_REPORTS" is the number of recent reports in the combined report
	reports, err := strconv.Atoi(os.Getenv("COMBINED_PAST_REPORTS"))
	if err != nil || reports <= 0 {
		fmt.Println("Warning: COMBINED_PAST_REPORTS is not set or invalid. Ignoring")
		return nil
	}
	combinedReport := h.Copy()
	for i := 1; i <= reports; i++ {
		data, err := ioutil.ReadFile(fmt.Sprintf("%s/%d/%s/%s",
			os.Getenv("JOB_BUILD_ROOTDIR"), h.BuildNum-i, os.Getenv("JOB_BUILD_SUBDIR"), testReportFname))
		if err != nil {
			break
		}
		testReport := &ReportMgr{}
		if err := json.Unmarshal(data, &testReport); err != nil {
			break
		}
		combinedReport.StartTime = testReport.StartTime
		combinedReport.Deployments += testReport.Deployments

		for e, f := range testReport.Failures {
			combinedReport.addFailure(e, f.Locations)
		}
	}
	return combinedReport.CreateTestReport(filepath)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package podlogs enables live capturing of all events and log
// messages for some or all pods in a namespace as they get generated.
// This helps debugging both a running test (what is currently going
// on?) and the output of a CI run (events appear in chronological
// order and output that normally isn't available like the command
// stdout messages are available).
package podlogs

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"github.com/pkg/errors"
	"io"
	"os"
	"path"
	"regexp"
	"strings"
	"sync"

	"k8s.io/api/core/v1"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
)

// LogsForPod starts reading the logs for a certain pod. If the pod has more than one
// container, opts.Container must be set. Reading stops when the context is done.
// The stream includes formatted error messages and ends with
//    rpc error: code = Unknown desc = Error: No such container: 41a...
// when the pod gets deleted while streaming.
func LogsForPod(ctx context.Context, cs clientset.Interface, ns, pod string, opts *v1.PodLogOptions) (io.ReadCloser, error) {
	req := cs.Core().Pods(ns).GetLogs(pod, opts)
	return req.Context(ctx).Stream()
}

// LogOutput determines where output from CopyAllLogs goes.
type LogOutput struct {
	// If not nil, errors will be logged here.
	StatusWriter io.Writer

	// If not nil, all output goes to this writer with "<pod>/<container>:" as prefix.
	LogWriter io.Writer

	// Base directory for one log file per container.
	// The full path of each log file will be <log path prefix><pod>-<container>.log.
	LogPathPrefix string
}

// Matches harmless errors from pkg/kubelet/kubelet_pods.go.
var expectedErrors = regexp.MustCompile(`container .* in pod .* is (terminated|waiting to start|not available)|the server could not find the requested resource`)

// CopyAllLogs follows the logs of all containers in all pods,
// including those that get created in the future, and writes each log
// line as configured in the output options. It does that until the
// context is done or until an error occurs.
//
// Beware that there is currently no way to force log collection
// before removing pods, which means that there is a known race
// between "stop pod" and "collecting log entries". The alternative
// would be a blocking function with collects logs from all currently
// running pods, but that then would have the disadvantage that
// already deleted pods aren't covered.
func CopyAllLogs(ctx context.Context, cs clientset.Interface, ns string, to LogOutput) error {
	watcher, err := cs.Core().Pods(ns).Watch(meta.ListOptions{})
	if err != nil {
		return errors.Wrap(err, "cannot create Pod event watcher")
	}

	go func() {
		var m sync.Mutex
		logging := map[string]bool{}
		check := func() {
			m.Lock()
			defer m.Unlock()

			pods, err := cs.Core().Pods(ns).List(meta.ListOptions{})
			if err != nil {
				if to.StatusWriter != nil {
					fmt.Fprintf(to.StatusWriter, "ERROR: get pod list in %s: %s\n", ns, err)
				}
				return
			}

			for _, pod := range pods.Items {
				for _, c := range pod.Spec.Containers {
					name := pod.ObjectMeta.Name + "/" + c.Name
					if logging[name] {
						continue
					}
					readCloser, err := LogsForPod(ctx, cs, ns, pod.ObjectMeta.Name,
						&v1.PodLogOptions{
							Container: c.Name,
							Follow:    true,
						})
					if err != nil {
						// We do get "normal" errors here, like trying to read too early.
						// We can ignore those.
						if to.StatusWriter != nil &&
							expectedErrors.FindStringIndex(err.Error()) == nil {
							fmt.Fprintf(to.StatusWriter, "WARNING: pod log: %s: %s\n", name, err)
						}
						continue
					}

					// Determine where we write. If this fails, we intentionally return without clearing
					// the logging[name] flag, which prevents trying over and over again to
					// create the output file.
					var out io.Writer
					var closer io.Closer
					var prefix string
					if to.LogWriter != nil {
						out = to.LogWriter
						prefix = name + ": "
					} else {
						var err error
						filename := to.LogPathPrefix + pod.ObjectMeta.Name + "-" + c.Name + ".log"
						err = os.MkdirAll(path.Dir(filename), 0755)
						if err != nil {
							if to.StatusWriter != nil {
								fmt.Fprintf(to.StatusWriter, "ERROR: pod log: create directory for %s: %s\n", filename, err)
							}
							return
						}
						// The test suite might run the same test multiple times,
						// so we have to append here.
						file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
						if err != nil {
							if to.StatusWriter != nil {
								fmt.Fprintf(to.StatusWriter, "ERROR: pod log: create file %s: %s\n", filename, err)
							}
							return
						}
						closer = file
						out = file
					}
					go func() {
						if closer != nil {
							defer closer.Close()
						}
						defer func() {
							m.Lock()
							logging[name] = false
							m.Unlock()
							readCloser.Close()
						}()
						scanner := bufio.NewScanner(readCloser)
						first := true
						for scanner.Scan() {
							line := scanner.Text()
							// Filter out the expected "end of stream" error message,
							// it would just confuse developers who don't know about it.
							// Same for attempts to read logs from a container that
							// isn't ready (yet?!).
							if !strings.HasPrefix(line, "rpc error: code = Unknown desc = Error: No such container:") &&
								!strings.HasPrefix(line, "Unable to retrieve container logs for ") {
								if first {
									if to.LogWriter == nil {
										// Because the same log might be written to multiple times
										// in different test instances, log an extra line to separate them.
										// Also provides some useful extra information.
										fmt.Fprintf(out, "==== start of log for container %s ====\n", name)
									}
									first = false
								}
								fmt.Fprintf(out, "%s%s\n", prefix, scanner.Text())
							}
						}
					}()
					logging[name] = true
				}
			}
		}

		// Watch events to see whether we can start logging
		// and log interesting ones.
		check()
		for {
			select {
			case <-watcher.ResultChan():
				check()
			case <-ctx.Done():
				return
			}
		}
	}()

	return nil
}

// WatchPods prints pod status events for a certain namespace or all namespaces
// when namespace name is empty.
func WatchPods(ctx context.Context, cs clientset.Interface, ns string, to io.Writer) error {
	watcher, err := cs.Core().Pods(ns).Watch(meta.ListOptions{})
	if err != nil {
		return errors.Wrap(err, "cannot create Pod event watcher")
	}

	go func() {
		defer watcher.Stop()
		for {
			select {
			case e := <-watcher.ResultChan():
				if e.Object == nil {
					continue
				}

				pod, ok := e.Object.(*v1.Pod)
				if !ok {
					continue
				}
				buffer := new(bytes.Buffer)
				fmt.Fprintf(buffer,
					"pod event: %s: %s/%s %s: %s %s\n",
					e.Type,
					pod.Namespace,
					pod.Name,
					pod.Status.Phase,
					pod.Status.Reason,
					pod.Status.Conditions,
				)
				for _, cst := range pod.Status.ContainerStatuses {
					fmt.Fprintf(buffer, "   %s: ", cst.Name)
					if cst.State.Waiting != nil {
						fmt.Fprintf(buffer, "WAITING: %s - %s",
							cst.State.Waiting.Reason,
							cst.State.Waiting.Message,
						)
					} else if cst.State.Running != nil {
						fmt.Fprintf(buffer, "RUNNING")
					} else if cst.State.Waiting != nil {
						fmt.Fprintf(buffer, "TERMINATED: %s - %s",
							cst.State.Waiting.Reason,
							cst.State.Waiting.Message,
						)
					}
					fmt.Fprintf(buffer, "\n")
				}
				to.Write(buffer.Bytes())
			case <-ctx.Done():
				return
			}
		}
	}()

	return nil
}
<commit_msg>e2e/framework: capture logs only from pods that actually started<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package podlogs enables live capturing of all events and log
// messages for some or all pods in a namespace as they get generated.
// This helps debugging both a running test (what is currently going
// on?) and the output of a CI run (events appear in chronological
// order and output that normally isn't available like the command
// stdout messages are available).
package podlogs

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"github.com/pkg/errors"
	"io"
	"os"
	"path"
	"regexp"
	"strings"
	"sync"

	"k8s.io/api/core/v1"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	clientset "k8s.io/client-go/kubernetes"
)

// LogsForPod starts reading the logs for a certain pod. If the pod has more than one
// container, opts.Container must be set. Reading stops when the context is done.
// The stream includes formatted error messages and ends with
//    rpc error: code = Unknown desc = Error: No such container: 41a...
// when the pod gets deleted while streaming.
func LogsForPod(ctx context.Context, cs clientset.Interface, ns, pod string, opts *v1.PodLogOptions) (io.ReadCloser, error) {
	req := cs.Core().Pods(ns).GetLogs(pod, opts)
	return req.Context(ctx).Stream()
}

// LogOutput determines where output from CopyAllLogs goes.
type LogOutput struct {
	// If not nil, errors will be logged here.
	StatusWriter io.Writer

	// If not nil, all output goes to this writer with "<pod>/<container>:" as prefix.
	LogWriter io.Writer

	// Base directory for one log file per container.
	// The full path of each log file will be <log path prefix><pod>-<container>.log.
	LogPathPrefix string
}

// Matches harmless errors from pkg/kubelet/kubelet_pods.go.
var expectedErrors = regexp.MustCompile(`container .* in pod .* is (terminated|waiting to start|not available)|the server could not find the requested resource`)

// CopyAllLogs follows the logs of all containers in all pods,
// including those that get created in the future, and writes each log
// line as configured in the output options. It does that until the
// context is done or until an error occurs.
//
// Beware that there is currently no way to force log collection
// before removing pods, which means that there is a known race
// between "stop pod" and "collecting log entries". The alternative
// would be a blocking function with collects logs from all currently
// running pods, but that then would have the disadvantage that
// already deleted pods aren't covered.
func CopyAllLogs(ctx context.Context, cs clientset.Interface, ns string, to LogOutput) error {
	watcher, err := cs.Core().Pods(ns).Watch(meta.ListOptions{})
	if err != nil {
		return errors.Wrap(err, "cannot create Pod event watcher")
	}

	go func() {
		var m sync.Mutex
		logging := map[string]bool{}
		check := func() {
			m.Lock()
			defer m.Unlock()

			pods, err := cs.Core().Pods(ns).List(meta.ListOptions{})
			if err != nil {
				if to.StatusWriter != nil {
					fmt.Fprintf(to.StatusWriter, "ERROR: get pod list in %s: %s\n", ns, err)
				}
				return
			}

			for _, pod := range pods.Items {
				for i, c := range pod.Spec.Containers {
					name := pod.ObjectMeta.Name + "/" + c.Name
					if logging[name] ||
						// sanity check, array should have entry for each container
						len(pod.Status.ContainerStatuses) <= i ||
						// Don't attempt to get logs for a container unless it is running or has terminated.
						// Trying to get a log would just end up with an error that we would have to suppress.
						(pod.Status.ContainerStatuses[i].State.Running == nil &&
							pod.Status.ContainerStatuses[i].State.Terminated == nil) {
						continue
					}
					readCloser, err := LogsForPod(ctx, cs, ns, pod.ObjectMeta.Name,
						&v1.PodLogOptions{
							Container: c.Name,
							Follow:    true,
						})
					if err != nil {
						// We do get "normal" errors here, like trying to read too early.
						// We can ignore those.
						if to.StatusWriter != nil &&
							expectedErrors.FindStringIndex(err.Error()) == nil {
							fmt.Fprintf(to.StatusWriter, "WARNING: pod log: %s: %s\n", name, err)
						}
						continue
					}

					// Determine where we write. If this fails, we intentionally return without clearing
					// the logging[name] flag, which prevents trying over and over again to
					// create the output file.
					var out io.Writer
					var closer io.Closer
					var prefix string
					if to.LogWriter != nil {
						out = to.LogWriter
						prefix = name + ": "
					} else {
						var err error
						filename := to.LogPathPrefix + pod.ObjectMeta.Name + "-" + c.Name + ".log"
						err = os.MkdirAll(path.Dir(filename), 0755)
						if err != nil {
							if to.StatusWriter != nil {
								fmt.Fprintf(to.StatusWriter, "ERROR: pod log: create directory for %s: %s\n", filename, err)
							}
							return
						}
						// The test suite might run the same test multiple times,
						// so we have to append here.
						file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
						if err != nil {
							if to.StatusWriter != nil {
								fmt.Fprintf(to.StatusWriter, "ERROR: pod log: create file %s: %s\n", filename, err)
							}
							return
						}
						closer = file
						out = file
					}
					go func() {
						if closer != nil {
							defer closer.Close()
						}
						defer func() {
							m.Lock()
							logging[name] = false
							m.Unlock()
							readCloser.Close()
						}()
						scanner := bufio.NewScanner(readCloser)
						first := true
						for scanner.Scan() {
							line := scanner.Text()
							// Filter out the expected "end of stream" error message,
							// it would just confuse developers who don't know about it.
							// Same for attempts to read logs from a container that
							// isn't ready (yet?!).
							if !strings.HasPrefix(line, "rpc error: code = Unknown desc = Error: No such container:") &&
								!strings.HasPrefix(line, "Unable to retrieve container logs for ") {
								if first {
									if to.LogWriter == nil {
										// Because the same log might be written to multiple times
										// in different test instances, log an extra line to separate them.
										// Also provides some useful extra information.
										fmt.Fprintf(out, "==== start of log for container %s ====\n", name)
									}
									first = false
								}
								fmt.Fprintf(out, "%s%s\n", prefix, scanner.Text())
							}
						}
					}()
					logging[name] = true
				}
			}
		}

		// Watch events to see whether we can start logging
		// and log interesting ones.
		check()
		for {
			select {
			case <-watcher.ResultChan():
				check()
			case <-ctx.Done():
				return
			}
		}
	}()

	return nil
}

// WatchPods prints pod status events for a certain namespace or all namespaces
// when namespace name is empty.
func WatchPods(ctx context.Context, cs clientset.Interface, ns string, to io.Writer) error {
	watcher, err := cs.Core().Pods(ns).Watch(meta.ListOptions{})
	if err != nil {
		return errors.Wrap(err, "cannot create Pod event watcher")
	}

	go func() {
		defer watcher.Stop()
		for {
			select {
			case e := <-watcher.ResultChan():
				if e.Object == nil {
					continue
				}

				pod, ok := e.Object.(*v1.Pod)
				if !ok {
					continue
				}
				buffer := new(bytes.Buffer)
				fmt.Fprintf(buffer,
					"pod event: %s: %s/%s %s: %s %s\n",
					e.Type,
					pod.Namespace,
					pod.Name,
					pod.Status.Phase,
					pod.Status.Reason,
					pod.Status.Conditions,
				)
				for _, cst := range pod.Status.ContainerStatuses {
					fmt.Fprintf(buffer, "   %s: ", cst.Name)
					if cst.State.Waiting != nil {
						fmt.Fprintf(buffer, "WAITING: %s - %s",
							cst.State.Waiting.Reason,
							cst.State.Waiting.Message,
						)
					} else if cst.State.Running != nil {
						fmt.Fprintf(buffer, "RUNNING")
					} else if cst.State.Waiting != nil {
						fmt.Fprintf(buffer, "TERMINATED: %s - %s",
							cst.State.Waiting.Reason,
							cst.State.Waiting.Message,
						)
					}
					fmt.Fprintf(buffer, "\n")
				}
				to.Write(buffer.Bytes())
			case <-ctx.Done():
				return
			}
		}
	}()

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
	"k8s.io/kubernetes/test/e2e/upgrades"

	"github.com/onsi/ginkgo"
)

// ServiceUpgradeTest tests that a service is available before and
// after a cluster upgrade. During a master-only upgrade, it will test
// that a service remains available during the upgrade.
type ServiceUpgradeTest struct {
	jig          *e2eservice.TestJig
	tcpService   *v1.Service
	tcpIngressIP string
	svcPort      int
}

// Name returns the tracking name of the test.
func (ServiceUpgradeTest) Name() string { return "service-upgrade" }

func shouldTestPDBs() bool { return framework.ProviderIs("gce", "gke") }

// Setup creates a service with a load balancer and makes sure it's reachable.
func (t *ServiceUpgradeTest) Setup(f *framework.Framework) {
	serviceName := "service-test"
	jig := e2eservice.NewTestJig(f.ClientSet, f.Namespace.Name, serviceName)

	ns := f.Namespace
	cs := f.ClientSet

	ginkgo.By("creating a TCP service " + serviceName + " with type=LoadBalancer in namespace " + ns.Name)
	_, err := jig.CreateTCPService(func(s *v1.Service) {
		s.Spec.Type = v1.ServiceTypeLoadBalancer
	})
	framework.ExpectNoError(err)
	tcpService, err := jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
	framework.ExpectNoError(err)

	// Get info to hit it with
	tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
	svcPort := int(tcpService.Spec.Ports[0].Port)

	ginkgo.By("creating pod to be part of service " + serviceName)
	rc, err := jig.Run(jig.AddRCAntiAffinity)
	framework.ExpectNoError(err)

	if shouldTestPDBs() {
		ginkgo.By("creating a PodDisruptionBudget to cover the ReplicationController")
		_, err = jig.CreatePDB(rc)
		framework.ExpectNoError(err)
	}

	// Hit it once before considering ourselves ready
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	timeout := e2eservice.LoadBalancerLagTimeoutDefault
	if framework.ProviderIs("aws") {
		timeout = e2eservice.LoadBalancerLagTimeoutAWS
	}
	e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, timeout)

	t.jig = jig
	t.tcpService = tcpService
	t.tcpIngressIP = tcpIngressIP
	t.svcPort = svcPort
}

// Test runs a connectivity check to the service.
func (t *ServiceUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	switch upgrade {
	case upgrades.MasterUpgrade, upgrades.ClusterUpgrade:
		t.test(f, done, true, true)
	case upgrades.NodeUpgrade:
		// Node upgrades should test during disruption only on GCE/GKE for now.
		t.test(f, done, shouldTestPDBs(), false)
	default:
		t.test(f, done, false, false)
	}
}

// Teardown cleans up any remaining resources.
func (t *ServiceUpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
}

func (t *ServiceUpgradeTest) test(f *framework.Framework, done <-chan struct{}, testDuringDisruption, testFinalizer bool) {
	if testDuringDisruption {
		// Continuous validation
		ginkgo.By("continuously hitting the pod through the service's LoadBalancer")
		wait.Until(func() {
			e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
		}, framework.Poll, done)
	} else {
		// Block until upgrade is done
		ginkgo.By("waiting for upgrade to finish without checking if service remains up")
		<-done
	}

	// Hit it once more
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
	if testFinalizer {
		defer func() {
			ginkgo.By("Check that service can be deleted with finalizer")
			e2eservice.WaitForServiceDeletedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name)
		}()
		ginkgo.By("Check that finalizer is present on loadBalancer type service")
		e2eservice.WaitForServiceUpdatedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name, true)
	}
}
<commit_msg>UPSTREAM: <carry>: Always test PDB's during service upgrade test<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
	"k8s.io/kubernetes/test/e2e/upgrades"

	"github.com/onsi/ginkgo"
)

// ServiceUpgradeTest tests that a service is available before and
// after a cluster upgrade. During a master-only upgrade, it will test
// that a service remains available during the upgrade.
type ServiceUpgradeTest struct {
	jig          *e2eservice.TestJig
	tcpService   *v1.Service
	tcpIngressIP string
	svcPort      int
}

// Name returns the tracking name of the test.
func (ServiceUpgradeTest) Name() string { return "service-upgrade" }

func shouldTestPDBs() bool { return true }

// Setup creates a service with a load balancer and makes sure it's reachable.
func (t *ServiceUpgradeTest) Setup(f *framework.Framework) {
	serviceName := "service-test"
	jig := e2eservice.NewTestJig(f.ClientSet, f.Namespace.Name, serviceName)

	ns := f.Namespace
	cs := f.ClientSet

	ginkgo.By("creating a TCP service " + serviceName + " with type=LoadBalancer in namespace " + ns.Name)
	_, err := jig.CreateTCPService(func(s *v1.Service) {
		s.Spec.Type = v1.ServiceTypeLoadBalancer
	})
	framework.ExpectNoError(err)
	tcpService, err := jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
	framework.ExpectNoError(err)

	// Get info to hit it with
	tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
	svcPort := int(tcpService.Spec.Ports[0].Port)

	ginkgo.By("creating pod to be part of service " + serviceName)
	rc, err := jig.Run(jig.AddRCAntiAffinity)
	framework.ExpectNoError(err)

	if shouldTestPDBs() {
		ginkgo.By("creating a PodDisruptionBudget to cover the ReplicationController")
		_, err = jig.CreatePDB(rc)
		framework.ExpectNoError(err)
	}

	// Hit it once before considering ourselves ready
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	timeout := e2eservice.LoadBalancerLagTimeoutDefault
	if framework.ProviderIs("aws") {
		timeout = e2eservice.LoadBalancerLagTimeoutAWS
	}
	e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, timeout)

	t.jig = jig
	t.tcpService = tcpService
	t.tcpIngressIP = tcpIngressIP
	t.svcPort = svcPort
}

// Test runs a connectivity check to the service.
func (t *ServiceUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	switch upgrade {
	case upgrades.MasterUpgrade, upgrades.ClusterUpgrade:
		t.test(f, done, true, true)
	case upgrades.NodeUpgrade:
		// Node upgrades should test during disruption only on GCE/GKE for now.
		t.test(f, done, shouldTestPDBs(), false)
	default:
		t.test(f, done, false, false)
	}
}

// Teardown cleans up any remaining resources.
func (t *ServiceUpgradeTest) Teardown(f *framework.Framework) {
	// rely on the namespace deletion to clean up everything
}

func (t *ServiceUpgradeTest) test(f *framework.Framework, done <-chan struct{}, testDuringDisruption, testFinalizer bool) {
	if testDuringDisruption {
		// Continuous validation
		ginkgo.By("continuously hitting the pod through the service's LoadBalancer")
		wait.Until(func() {
			e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
		}, framework.Poll, done)
	} else {
		// Block until upgrade is done
		ginkgo.By("waiting for upgrade to finish without checking if service remains up")
		<-done
	}

	// Hit it once more
	ginkgo.By("hitting the pod through the service's LoadBalancer")
	e2eservice.TestReachableHTTP(t.tcpIngressIP, t.svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
	if testFinalizer {
		defer func() {
			ginkgo.By("Check that service can be deleted with finalizer")
			e2eservice.WaitForServiceDeletedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name)
		}()
		ginkgo.By("Check that finalizer is present on loadBalancer type service")
		e2eservice.WaitForServiceUpdatedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name, true)
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"github.com/jpillora/go-ogle-analytics"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/controller/flash_message"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/google_analytics/measurement"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/util"
	"github.com/stvp/rollbar"
	"github.com/uber-go/zap"
)

const (
	APITokenCookieName   = "apiToken"
	TrackingIDCookieName = "trackingId"
)

func TemplateDir() string {
	if util.IsProductionEnv() {
		return "static"
	} else {
		return "src/html"
	}
}

func TemplatePath(file string) string {
	return path.Join(TemplateDir(), file)
}

func ParseHTMLTemplates(files ...string) *template.Template {
	f := []string{
		TemplatePath("_base.html"),
		TemplatePath("_flashMessage.html"),
	}
	f = append(f, files...)
	return template.Must(template.ParseFiles(f...))
}

func InternalServerError(w http.ResponseWriter, err error) {
	//switch _ := errors.Cause(err).(type) { // TODO:
	//default:
	// unknown error
	if rollbar.Token != "" {
		rollbar.Error(rollbar.ERR, err)
	}
	fields := []zap.Field{
		zap.Error(err),
	}
	if e, ok := err.(errors.StackTracer); ok {
		b := &bytes.Buffer{}
		for _, f := range e.StackTrace() {
			fmt.Fprintf(b, "%+v\n", f)
		}
		fields = append(fields, zap.String("stacktrace", b.String()))
	}
	logger.App.Error("InternalServerError", fields...)

	http.Error(w, fmt.Sprintf("Internal Server Error\n\n%v", err), http.StatusInternalServerError)
	if !config.IsProductionEnv() {
		fmt.Fprintf(w, "----- stacktrace -----\n")
		if e, ok := err.(errors.StackTracer); ok {
			for _, f := range e.StackTrace() {
				fmt.Fprintf(w, "%+v\n", f)
			}
		}
	}
}

func JSON(w http.ResponseWriter, code int, body interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	//w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(code)
	if err := json.NewEncoder(w).Encode(body); err != nil {
		http.Error(w, `{ "status": "Failed to Encode as JSON" }`, http.StatusInternalServerError)
		return
	}
}

type commonTemplateData struct {
	StaticURL         string
	GoogleAnalyticsID string
	CurrentURL        string
	CanonicalURL      string
	TrackingID        string
	UserID            string
	NavigationItems   []navigationItem
	FlashMessage      *flash_message.FlashMessage
}

type navigationItem struct {
	Text string
	URL  string
}

var loggedInNavigationItems = []navigationItem{
	{"ホーム", "/me"},
	{"設定", "/me/setting"},
	{"ログアウト", "/me/logout"},
}

var loggedOutNavigationItems = []navigationItem{
	{"ホーム", "/"},
}

func getCommonTemplateData(req *http.Request, loggedIn bool, userID uint32) commonTemplateData {
	canonicalURL := fmt.Sprintf("%s://%s%s", config.WebURLScheme(req), req.Host, req.RequestURI)
	canonicalURL = (strings.SplitN(canonicalURL, "?", 2))[0] // TODO: use url.Parse
	data := commonTemplateData{
		StaticURL:         config.StaticURL(),
		GoogleAnalyticsID: config.GoogleAnalyticsID(),
		CurrentURL:        req.RequestURI,
		CanonicalURL:      canonicalURL,
	}
	if loggedIn {
		data.NavigationItems = loggedInNavigationItems
	} else {
		data.NavigationItems = loggedOutNavigationItems
	}
	if flashMessageKey := req.FormValue("flashMessageKey"); flashMessageKey != "" {
		flashMessage, _ := flash_message.MustStore(req.Context()).Load(flashMessageKey)
		data.FlashMessage = flashMessage
	}
	data.TrackingID = model.MustTrackingID(req.Context())
	if userID != 0 {
		data.UserID = fmt.Sprint(userID)
	}

	return data
}

var measurementClient = measurement.NewClient(&http.Client{
	//Transport: &logger.LoggingHTTPTransport{DumpHeaderBody: true},
	Timeout: time.Second * 7,
})

var gaHTTPClient *http.Client = &http.Client{
	Transport: &logger.LoggingHTTPTransport{DumpHeaderBody: true},
	Timeout:   time.Second * 7,
}

const (
	eventCategoryUser = "user"
)

func sendMeasurementEvent2(req *http.Request, category, action, label string, value int64, userID uint32) {
	gaClient, err := ga.NewClient(os.Getenv("GOOGLE_ANALYTICS_ID"))
	if err != nil {
		logger.App.Warn("ga.NewClient() failed", zap.Error(err))
	}
	gaClient.HttpClient = gaHTTPClient
	gaClient.UserAgentOverride(req.UserAgent())

	var clientID string
	if cookie, err := req.Cookie("_ga"); err == nil {
		clientID, err = measurement.GetClientID(cookie)
		if err != nil {
			logger.App.Warn("measurement.GetClientID() failed", zap.Error(err))
		}
	} else {
		clientID = GetRemoteAddress(req)
	}
	gaClient.ClientID(clientID)
	gaClient.DocumentHostName(req.Host)
	gaClient.DocumentPath(req.URL.Path)
	gaClient.DocumentTitle(req.URL.Path)
	gaClient.DocumentReferrer(req.Referer())
	gaClient.IPOverride(GetRemoteAddress(req))

	logFields := []zap.Field{
		zap.String("category", category),
		zap.String("action", action),
	}
	event := ga.NewEvent(category, action)
	if label != "" {
		event.Label(label)
		logFields = append(logFields, zap.String("label", label))
	}
	if value != 0 {
		event.Value(value)
		logFields = append(logFields, zap.Int64("value", value))
	}
	if userID != 0 {
		gaClient.UserID(fmt.Sprint(userID))
		logFields = append(logFields, zap.Uint("userID", uint(userID)))
	}
	if err := gaClient.Send(event); err == nil {
		// TODO: stats log
		logger.App.Debug("sendMeasurementEvent() success", logFields...)
	} else {
		logger.App.Warn("gaClient.Send() failed", zap.Error(err))
	}
}

func sendMeasurementEvent(req *http.Request, category, action, label string, value int64, userID uint32) {
	trackingID := os.Getenv("GOOGLE_ANALYTICS_ID")
	var clientID string
	if cookie, err := req.Cookie("_ga"); err == nil {
		clientID, err = measurement.GetClientID(cookie)
		if err != nil {
			logger.App.Warn("measurement.GetClientID() failed", zap.Error(err))
		}
	} else {
		clientID = GetRemoteAddress(req)
	}

	params := measurement.NewEventParams(req.UserAgent(), trackingID, clientID, category, action)
	params.DataSource = "server"
	if label != "" {
		params.EventLabel = label
	}
	if value != 0 {
		params.EventValue = value
	}
	if userID != 0 {
		params.UserID = fmt.Sprint(userID)
	}

	if err := measurementClient.Do(params); err == nil {
		logger.App.Debug(
			"sendMeasurementEvent() success",
			zap.String("category", category),
			zap.String("action", action),
			zap.String("label", label),
			zap.Int64("value", value),
			zap.Uint("userID", uint(userID)),
		)
	} else {
		logger.App.Warn("measurementClient.Do() failed", zap.Error(err))
	}
}

func GetRemoteAddress(req *http.Request) string {
	xForwardedFor := req.Header.Get("X-Forwarded-For")
	if xForwardedFor == "" {
		return (strings.Split(req.RemoteAddr, ":"))[0]
	}
	return strings.TrimSpace((strings.Split(xForwardedFor, ","))[0])
}
<commit_msg>Use trackingID for GA measurement protocol<commit_after>package controller

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"github.com/jpillora/go-ogle-analytics"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/controller/flash_message"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/google_analytics/measurement"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/util"
	"github.com/stvp/rollbar"
	"github.com/uber-go/zap"
)

const (
	APITokenCookieName   = "apiToken"
	TrackingIDCookieName = "trackingId"
)

func TemplateDir() string {
	if util.IsProductionEnv() {
		return "static"
	} else {
		return "src/html"
	}
}

func TemplatePath(file string) string {
	return path.Join(TemplateDir(), file)
}

func ParseHTMLTemplates(files ...string) *template.Template {
	f := []string{
		TemplatePath("_base.html"),
		TemplatePath("_flashMessage.html"),
	}
	f = append(f, files...)
	return template.Must(template.ParseFiles(f...))
}

func InternalServerError(w http.ResponseWriter, err error) {
	//switch _ := errors.Cause(err).(type) { // TODO:
	//default:
	// unknown error
	if rollbar.Token != "" {
		rollbar.Error(rollbar.ERR, err)
	}
	fields := []zap.Field{
		zap.Error(err),
	}
	if e, ok := err.(errors.StackTracer); ok {
		b := &bytes.Buffer{}
		for _, f := range e.StackTrace() {
			fmt.Fprintf(b, "%+v\n", f)
		}
		fields = append(fields, zap.String("stacktrace", b.String()))
	}
	logger.App.Error("InternalServerError", fields...)

	http.Error(w, fmt.Sprintf("Internal Server Error\n\n%v", err), http.StatusInternalServerError)
	if !config.IsProductionEnv() {
		fmt.Fprintf(w, "----- stacktrace -----\n")
		if e, ok := err.(errors.StackTracer); ok {
			for _, f := range e.StackTrace() {
				fmt.Fprintf(w, "%+v\n", f)
			}
		}
	}
}

func JSON(w http.ResponseWriter, code int, body interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	//w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(code)
	if err := json.NewEncoder(w).Encode(body); err != nil {
		http.Error(w, `{ "status": "Failed to Encode as JSON" }`, http.StatusInternalServerError)
		return
	}
}

type commonTemplateData struct {
	StaticURL         string
	GoogleAnalyticsID string
	CurrentURL        string
	CanonicalURL      string
	TrackingID        string
	UserID            string
	NavigationItems   []navigationItem
	FlashMessage      *flash_message.FlashMessage
}

type navigationItem struct {
	Text string
	URL  string
}

var loggedInNavigationItems = []navigationItem{
	{"ホーム", "/me"},
	{"設定", "/me/setting"},
	{"ログアウト", "/me/logout"},
}

var loggedOutNavigationItems = []navigationItem{
	{"ホーム", "/"},
}

func getCommonTemplateData(req *http.Request, loggedIn bool, userID uint32) commonTemplateData {
	canonicalURL := fmt.Sprintf("%s://%s%s", config.WebURLScheme(req), req.Host, req.RequestURI)
	canonicalURL = (strings.SplitN(canonicalURL, "?", 2))[0] // TODO: use url.Parse
	data := commonTemplateData{
		StaticURL:         config.StaticURL(),
		GoogleAnalyticsID: config.GoogleAnalyticsID(),
		CurrentURL:        req.RequestURI,
		CanonicalURL:      canonicalURL,
	}
	if loggedIn {
		data.NavigationItems = loggedInNavigationItems
	} else {
		data.NavigationItems = loggedOutNavigationItems
	}
	if flashMessageKey := req.FormValue("flashMessageKey"); flashMessageKey != "" {
		flashMessage, _ := flash_message.MustStore(req.Context()).Load(flashMessageKey)
		data.FlashMessage = flashMessage
	}
	data.TrackingID = model.MustTrackingID(req.Context())
	if userID != 0 {
		data.UserID = fmt.Sprint(userID)
	}

	return data
}

var measurementClient = measurement.NewClient(&http.Client{
	//Transport: &logger.LoggingHTTPTransport{DumpHeaderBody: true},
	Timeout: time.Second * 7,
})

var gaHTTPClient *http.Client = &http.Client{
	Transport: &logger.LoggingHTTPTransport{DumpHeaderBody: true},
	Timeout:   time.Second * 7,
}

const (
	eventCategoryUser = "user"
)

func sendMeasurementEvent2(req *http.Request, category, action, label string, value int64, userID uint32) {
	gaClient, err := ga.NewClient(os.Getenv("GOOGLE_ANALYTICS_ID"))
	if err != nil {
		logger.App.Warn("ga.NewClient() failed", zap.Error(err))
	}
	gaClient.HttpClient = gaHTTPClient
	gaClient.UserAgentOverride(req.UserAgent())

	gaClient.ClientID(model.MustTrackingID(req.Context()))
	gaClient.DocumentHostName(req.Host)
	gaClient.DocumentPath(req.URL.Path)
	gaClient.DocumentTitle(req.URL.Path)
	gaClient.DocumentReferrer(req.Referer())
	gaClient.IPOverride(GetRemoteAddress(req))

	logFields := []zap.Field{
		zap.String("category", category),
		zap.String("action", action),
	}
	event := ga.NewEvent(category, action)
	if label != "" {
		event.Label(label)
		logFields = append(logFields, zap.String("label", label))
	}
	if value != 0 {
		event.Value(value)
		logFields = append(logFields, zap.Int64("value", value))
	}
	if userID != 0 {
		gaClient.UserID(fmt.Sprint(userID))
		logFields = append(logFields, zap.Uint("userID", uint(userID)))
	}
	if err := gaClient.Send(event); err == nil {
		// TODO: stats log
		logger.App.Debug("sendMeasurementEvent() success", logFields...)
	} else {
		logger.App.Warn("gaClient.Send() failed", zap.Error(err))
	}
}

func sendMeasurementEvent(req *http.Request, category, action, label string, value int64, userID uint32) {
	trackingID := os.Getenv("GOOGLE_ANALYTICS_ID")
	var clientID string
	if cookie, err := req.Cookie("_ga"); err == nil {
		clientID, err = measurement.GetClientID(cookie)
		if err != nil {
			logger.App.Warn("measurement.GetClientID() failed", zap.Error(err))
		}
	} else {
		clientID = GetRemoteAddress(req)
	}

	params := measurement.NewEventParams(req.UserAgent(), trackingID, clientID, category, action)
	params.DataSource = "server"
	if label != "" {
		params.EventLabel = label
	}
	if value != 0 {
		params.EventValue = value
	}
	if userID != 0 {
		params.UserID = fmt.Sprint(userID)
	}

	if err := measurementClient.Do(params); err == nil {
		logger.App.Debug(
			"sendMeasurementEvent() success",
			zap.String("category", category),
			zap.String("action", action),
			zap.String("label", label),
			zap.Int64("value", value),
			zap.Uint("userID", uint(userID)),
		)
	} else {
		logger.App.Warn("measurementClient.Do() failed", zap.Error(err))
	}
}

func GetRemoteAddress(req *http.Request) string {
	xForwardedFor := req.Header.Get("X-Forwarded-For")
	if xForwardedFor == "" {
		return (strings.Split(req.RemoteAddr, ":"))[0]
	}
	return strings.TrimSpace((strings.Split(xForwardedFor, ","))[0])
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/logger"
	"github.com/stretchr/testify/assert"
)

func findArtifact(artifacts []*api.Artifact, search string) *api.Artifact {
	for _, a := range artifacts {
		if filepath.Base(a.Path) == search {
			return a
		}
	}

	return nil
}

func TestCollect(t *testing.T) {
	t.Parallel()

	wd, _ := os.Getwd()
	root := filepath.Join(wd, "..")
	os.Chdir(root)
	defer os.Chdir(wd)

	volumeName := filepath.VolumeName(root)
	rootWithoutVolume := strings.TrimPrefix(root, volumeName)

	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
		Paths: fmt.Sprintf("%s;%s",
			filepath.Join("test", "fixtures", "artifacts", "**/*.jpg"),
			filepath.Join(root, "test", "fixtures", "artifacts", "**/*.gif"),
		),
	})

	artifacts, err := uploader.Collect()
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, len(artifacts), 4)

	var testCases = []struct {
		Name         string
		Path         string
		AbsolutePath string
		GlobPath     string
		FileSize     int
		Sha1Sum      string
	}{
		{
			"Mr Freeze.jpg",
			strings.Join([]string{"test", "fixtures", "artifacts", "Mr Freeze.jpg"}, "/"),
			filepath.Join(root, "test", "fixtures", "artifacts", "Mr Freeze.jpg"),
			filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
			362371,
			"f5bc7bc9f5f9c3e543dde0eb44876c6f9acbfb6b",
		},
		{
			"Commando.jpg",
			strings.Join([]string{"test", "fixtures", "artifacts", "folder", "Commando.jpg"}, "/"),
			filepath.Join(root, "test", "fixtures", "artifacts", "folder", "Commando.jpg"),
			filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
			113000,
			"811d7cb0317582e22ebfeb929d601cdabea4b3c0",
		},
		{
			"The Terminator.jpg",
			strings.Join([]string{"test", "fixtures", "artifacts", "this is a folder with a space", "The Terminator.jpg"}, "/"),
			filepath.Join(root, "test", "fixtures", "artifacts", "this is a folder with a space", "The Terminator.jpg"),
			filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
			47301,
			"ed76566ede9cb6edc975fcadca429665aad8785a",
		},
		{
			"Smile.gif",
			strings.Join([]string{rootWithoutVolume[1:], "test", "fixtures", "artifacts", "gifs", "Smile.gif"}, "/"),
			filepath.Join(root, "test", "fixtures", "artifacts", "gifs", "Smile.gif"),
			filepath.Join(root, "test", "fixtures", "artifacts", "**", "*.gif"),
			2038453,
			"bd4caf2e01e59777744ac1d52deafa01c2cb9bfd",
		},
	}

	for _, tc := range testCases {
		t.Run(tc.Name, func(t *testing.T) {
			a := findArtifact(artifacts, tc.Name)
			if a == nil {
				t.Fatalf("Failed to find artifact %q", tc.Name)
			}

			assert.Equal(t, tc.Path, a.Path)
			assert.Equal(t, tc.AbsolutePath, a.AbsolutePath)
			assert.Equal(t, tc.GlobPath, a.GlobPath)
			assert.Equal(t, tc.FileSize, int(a.FileSize))
			assert.Equal(t, tc.Sha1Sum, a.Sha1Sum)
		})
	}
}

func TestCollectThatDoesntMatchAnyFiles(t *testing.T) {
	wd, _ := os.Getwd()
	root := filepath.Join(wd, "..")
	os.Chdir(root)
	defer os.Chdir(wd)

	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
		Paths: strings.Join([]string{
			filepath.Join("log", "*"),
			filepath.Join("tmp", "capybara", "**", "*"),
			filepath.Join("mkmf.log"),
			filepath.Join("log", "mkmf.log"),
		}, ";"),
	})

	artifacts, err := uploader.Collect()
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, len(artifacts), 0)
}

func TestCollectWithSomeGlobsThatDontMatchAnything(t *testing.T) {
	wd, _ := os.Getwd()
	root := filepath.Join(wd, "..")
	os.Chdir(root)
	defer os.Chdir(wd)

	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
		Paths: strings.Join([]string{
			filepath.Join("dontmatchanything", "*"),
			filepath.Join("dontmatchanything.zip"),
			filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
		}, ";"),
	})

	artifacts, err := uploader.Collect()
	if err != nil {
		t.Fatal(err)
	}

	if len(artifacts) != 3 {
		t.Fatalf("Expected to match 3 artifacts, found %d", len(artifacts))
	}
}
<commit_msg>Pinch the non-positional struct form from #638<commit_after>package agent

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/logger"
	"github.com/stretchr/testify/assert"
)

func findArtifact(artifacts []*api.Artifact, search string) *api.Artifact {
	for _, a := range artifacts {
		if filepath.Base(a.Path) == search {
			return a
		}
	}

	return nil
}

func TestCollect(t *testing.T) {
	t.Parallel()

	wd, _ := os.Getwd()
	root := filepath.Join(wd, "..")
	os.Chdir(root)
	defer os.Chdir(wd)

	volumeName := filepath.VolumeName(root)
	rootWithoutVolume := strings.TrimPrefix(root, volumeName)

	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
		Paths: fmt.Sprintf("%s;%s",
			filepath.Join("test", "fixtures", "artifacts", "**/*.jpg"),
			filepath.Join(root, "test", "fixtures", "artifacts", "**/*.gif"),
		),
	})

	artifacts, err := uploader.Collect()
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, len(artifacts), 4)

	var testCases = []struct {
		Name         string
		Path         string
		AbsolutePath string
		GlobPath     string
		FileSize     int
		Sha1Sum      string
	}{
		{
			Name:         "Mr Freeze.jpg",
			Path:         strings.Join([]string{"test", "fixtures", "artifacts", "Mr Freeze.jpg"}, "/"),
			AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "Mr Freeze.jpg"),
			GlobPath:     filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
			FileSize:     362371,
			Sha1Sum:      "f5bc7bc9f5f9c3e543dde0eb44876c6f9acbfb6b",
		},
		{
			Name:         "Commando.jpg",
			Path:         strings.Join([]string{"test", "fixtures", "artifacts", "folder", "Commando.jpg"}, "/"),
			AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "folder", "Commando.jpg"),
			GlobPath:     filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
			FileSize:     113000,
			Sha1Sum:      "811d7cb0317582e22ebfeb929d601cdabea4b3c0",
		},
		{
			Name:         "The Terminator.jpg",
			Path:         strings.Join([]string{"test", "fixtures", "artifacts", "this is a folder with a space", "The Terminator.jpg"}, "/"),
			AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "this is a folder with a space", "The Terminator.jpg"),
			GlobPath:     filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
			FileSize:     47301,
			Sha1Sum:      "ed76566ede9cb6edc975fcadca429665aad8785a",
		},
		{
			Name:         "Smile.gif",
			Path:         strings.Join([]string{rootWithoutVolume[1:], "test", "fixtures", "artifacts", "gifs", "Smile.gif"}, "/"),
			AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "gifs", "Smile.gif"),
			GlobPath:     filepath.Join(root, "test", "fixtures", "artifacts", "**", "*.gif"),
			FileSize:     2038453,
			Sha1Sum:      "bd4caf2e01e59777744ac1d52deafa01c2cb9bfd",
		},
	}

	for _, tc := range testCases {
		t.Run(tc.Name, func(t *testing.T) {
			a := findArtifact(artifacts, tc.Name)
			if a == nil {
				t.Fatalf("Failed to find artifact %q", tc.Name)
			}

			assert.Equal(t, tc.Path, a.Path)
			assert.Equal(t, tc.AbsolutePath, a.AbsolutePath)
			assert.Equal(t, tc.GlobPath, a.GlobPath)
			assert.Equal(t, tc.FileSize, int(a.FileSize))
			assert.Equal(t, tc.Sha1Sum, a.Sha1Sum)
		})
	}
}

func TestCollectThatDoesntMatchAnyFiles(t *testing.T) {
	wd, _ := os.Getwd()
	root := filepath.Join(wd, "..")
	os.Chdir(root)
	defer os.Chdir(wd)

	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
		Paths: strings.Join([]string{
			filepath.Join("log", "*"),
			filepath.Join("tmp", "capybara", "**", "*"),
			filepath.Join("mkmf.log"),
			filepath.Join("log", "mkmf.log"),
		}, ";"),
	})

	artifacts, err := uploader.Collect()
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, len(artifacts), 0)
}

func TestCollectWithSomeGlobsThatDontMatchAnything(t *testing.T) {
	wd, _ := os.Getwd()
	root := filepath.Join(wd, "..")
	os.Chdir(root)
	defer os.Chdir(wd)

	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
		Paths: strings.Join([]string{
			filepath.Join("dontmatchanything", "*"),
			filepath.Join("dontmatchanything.zip"),
			filepath.Join("test", "fixtures", "artifacts", "**", "*.jpg"),
		}, ";"),
	})

	artifacts, err := uploader.Collect()
	if err != nil {
		t.Fatal(err)
	}

	if len(artifacts) != 3 {
		t.Fatalf("Expected to match 3 artifacts, found %d", len(artifacts))
	}
}
<|endoftext|>