<commit_before>package bagman

import (
	"archive/tar"
	"crypto/md5"
	"crypto/sha256"
	"fmt"
	"github.com/APTrust/bagins"
	"github.com/satori/go.uuid"
	"io"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"time"
)

// Untars the file at the specified tarFilePath and returns a list
// of files that were untarred from the archive. Check
// result.Error to ensure there were no errors.
// tarFilePath is the tarFilePath to the tar file that you want to unpack.
// instDomain is the domain name of the institution that owns the bag.
// bagName is the name of the tar file, minus the ".tar" extension.
//
// If param buildIngestData is true, the untar operation will run
// md5 and sha256 checksums on all of the data files in the archive,
// and it will use MimeMagic to figure out each file's mime type.
// We must do this during ingest, so buildIngestData must be true
// when we're running this on our servers.
//
// If we're running in a client environment--that is, an APTrust partner
// is running the bag validation application on their laptop or desktop--
// buildIngestData should be false. There's no need to calculate sha256
// checksums in that case, and we cannot assume that the client would have
// the external MimeMagic C libraries installed. The lack of those libraries
// would cause the application to crash. So buildIngestData == false on
// the client!!
func Untar(tarFilePath, instDomain, bagName string, buildIngestData bool) (result *TarResult) {

	// Set up our result
	tarResult := new(TarResult)
	absInputFile, err := filepath.Abs(tarFilePath)
	if err != nil {
		tarResult.ErrorMessage = fmt.Sprintf("Before untarring, could not determine "+
			"absolute path to downloaded file: %v", err)
		return tarResult
	}
	tarResult.InputFile = absInputFile

	_, err = GetInstitutionFromBagName(filepath.Base(absInputFile))
	if err != nil {
		tarResult.ErrorMessage = err.Error()
		return tarResult
	}

	// Open the tar file for reading.
	file, err := os.Open(tarFilePath)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		tarResult.ErrorMessage = fmt.Sprintf("Could not open file %s for untarring: %v",
			tarFilePath, err)
		return tarResult
	}

	// Record the name of the top-level directory in the tar
	// file. Our spec says that the name of the directory into
	// which the file untars should be the same as the tar file
	// name, minus the .tar extension. So uva-123.tar should
	// untar into a directory called uva-123. This is required
	// so that IntellectualObject and GenericFile identifiers
	// in Fedora can be traced back to the named bag from which
	// they came. Other parts of bagman, such as the file cleanup
	// routines, assume that the untarred directory name will
	// match the tar file name, as the spec demands. When the names
	// do not match, the cleanup routines will not clean up the
	// untarred files, and we'll end up losing a lot of disk space.
	topLevelDir := ""

	// Untar the file and record the results.
	tarReader := tar.NewReader(file)

	for {
		header, err := tarReader.Next()
		if err != nil && err.Error() == "EOF" {
			break // end of archive
		}
		if err != nil {
			tarResult.ErrorMessage = fmt.Sprintf(
				"Error reading tar file header: %v. " +
					"Either this is not a tar file, or the file is corrupt.", err)
			return tarResult
		}

		// Top-level dir will be the first header entry.
		if header.Typeflag == tar.TypeDir && topLevelDir == "" {
			topLevelDir = strings.Replace(header.Name, "/", "", 1)
			// Fix for Windows
			systemNormalizedPath := tarFilePath
			if runtime.GOOS == "windows" && strings.Contains(tarFilePath, "\\") {
				systemNormalizedPath = strings.Replace(tarFilePath, "\\", "/", -1)
			}
			expectedDir := path.Base(systemNormalizedPath)
			if strings.HasSuffix(expectedDir, ".tar") {
				expectedDir = expectedDir[0 : len(expectedDir)-4]
			}
			if topLevelDir != expectedDir {
				tarResult.ErrorMessage = fmt.Sprintf(
					"Bag '%s' should untar to a folder named '%s', but "+
						"it untars to '%s'. Please repackage this bag and try again.",
					path.Base(tarFilePath), expectedDir, topLevelDir)
				return tarResult
			}
		}

		outputPath := filepath.Join(filepath.Dir(absInputFile), header.Name)
		// PT #114804083 - Make sure there's a slash in the path before setting
		// tarfile.OutputDir, or we'll set it incorrectly and when we call
		// bagman.ReadBag(tarResult.OutputDir), it will fail with "file not found".
		pathParts := strings.Split(header.Name, "/")
		tarDirectory := pathParts[0]
		if tarResult.OutputDir == "" && len(pathParts) > 1 {
			tarResult.OutputDir = filepath.Join(filepath.Dir(absInputFile), tarDirectory)
		}

		// Make sure the directory that we're about to write into exists.
		err = os.MkdirAll(filepath.Dir(outputPath), 0755)
		if err != nil {
			tarResult.ErrorMessage = fmt.Sprintf("Could not create destination file '%s' "+
				"while unpacking tar archive: %v", outputPath, err)
			return tarResult
		}

		// Copy the file, if it's an actual file. Otherwise, ignore it and record
		// a warning. The bag library does not deal with items like symlinks.
		if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA {
			if strings.Contains(header.Name, "/data/") {
				var dataFile *File = nil
				dataFile = buildFile(tarReader, filepath.Dir(absInputFile), header.Name,
					header.Size, header.ModTime, buildIngestData)
				cleanBagName, _ := CleanBagName(bagName)
				dataFile.Identifier = fmt.Sprintf("%s/%s", cleanBagName, dataFile.Path)
				dataFile.IdentifierAssigned = time.Now()
				tarResult.Files = append(tarResult.Files, dataFile)
			} else {
				err = saveFile(outputPath, tarReader)
				if err != nil {
					tarResult.ErrorMessage = fmt.Sprintf("Error copying file from tar archive "+
						"to '%s': %v", outputPath, err)
					return tarResult
				}
			}

			outputRelativePath := strings.Replace(outputPath, tarResult.OutputDir+"/", "", 1)
			tarResult.FilesUnpacked = append(tarResult.FilesUnpacked, outputRelativePath)

		} else if header.Typeflag != tar.TypeDir {
			tarResult.Warnings = append(tarResult.Warnings,
				fmt.Sprintf("Ignoring item %s of type %c because it's neither a file nor a directory",
					header.Name, header.Typeflag))
		}
	}
	sort.Strings(tarResult.FilesUnpacked)
	return tarResult
}

// Reads an untarred bag. The tarFilePath parameter should be a path to
// a directory that contains the bag, info and manifest files.
// The bag content should be in the data directory under tarFilePath.
// Check result.Error to ensure there were no errors.
func ReadBag(tarFilePath string) (result *BagReadResult) {
	bagReadResult := new(BagReadResult)
	bagReadResult.Path = tarFilePath

	// Final param to bagins.ReadBag is the name of the checksum file.
	// That param defaults to manifest-md5.txt, which is what it
	// should be for bags we're fetching from the S3 receiving buckets.
	bag, err := bagins.ReadBag(tarFilePath, []string{"bagit.txt", "bag-info.txt", "aptrust-info.txt"}, "")
	if err != nil {
		bagReadResult.ErrorMessage = fmt.Sprintf("Error unpacking bag: %v", err)
		return bagReadResult
	}

	fileNames, err := bag.ListFiles()
	if err != nil {
		bagReadResult.ErrorMessage = fmt.Sprintf("Could not list bag files: %v ", err)
		return bagReadResult
	}

	dataDirPrefix := "data/"
	if runtime.GOOS == "windows" {
		dataDirPrefix = "data\\"
	}

	errMsg := ""
	bagReadResult.Files = make([]string, len(fileNames))
	hasBagit := false
	hasAPTrustInfo := false
	hasMd5Manifest := false
	hasDataFiles := false
	for index, fileName := range fileNames {
		bagReadResult.Files[index] = fileName
		if fileName == "bagit.txt" {
			hasBagit = true
		} else if fileName == "aptrust-info.txt" {
			hasAPTrustInfo = true
		} else if fileName == "manifest-md5.txt" {
			hasMd5Manifest = true
		} else if strings.HasPrefix(fileName, dataDirPrefix) {
			hasDataFiles = true
		}
	}
	if !hasBagit {
		errMsg += " Bag is missing bagit.txt file.\n"
	}
	if !hasAPTrustInfo {
		errMsg += " Bag is missing aptrust-info.txt file.\n"
	}
	if !hasMd5Manifest {
		errMsg += " Bag is missing manifest-md5.txt file.\n"
	}
	if !hasDataFiles {
		errMsg += " Bag's data directory is missing or empty.\n"
	}

	extractTags(bag, bagReadResult)

	checksumErrors := bag.Manifest.RunChecksums()
	if len(checksumErrors) > 0 {
		errMsg += "The following checksums could not be verified:\n"
		bagReadResult.ChecksumErrors = make([]error, len(checksumErrors))
		for i, err := range checksumErrors {
			bagReadResult.ChecksumErrors[i] = err
			errMsg += "  " + err.Error() + ".\n"
		}
	}

	if errMsg != "" {
		bagReadResult.ErrorMessage += fmt.Sprintf(errMsg)
	}

	return bagReadResult
}

// Extract all of the tags from tag files "bagit.txt", "bag-info.txt",
// and "aptrust-info.txt", and put those tags into the Tags member
// of the BagReadResult structure.
func extractTags(bag *bagins.Bag, bagReadResult *BagReadResult) {
	tagFiles := []string{"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	accessRights := ""
	bagTitle := ""
	for _, file := range tagFiles {
		tagFile, err := bag.TagFile(file)
		if err != nil {
			bagReadResult.ErrorMessage = fmt.Sprintf("Error reading tags from bag: %v", err)
			return
		}
		tagFields := tagFile.Data.Fields()

		for _, tagField := range tagFields {
			tag := Tag{tagField.Label(), strings.TrimSpace(tagField.Value())}
			bagReadResult.Tags = append(bagReadResult.Tags, tag)

			lcLabel := strings.ToLower(tag.Label)
			if lcLabel == "access" {
				accessRights = strings.TrimSpace(strings.ToLower(tag.Value))
			} else if accessRights == "" && lcLabel == "rights" {
				accessRights = strings.TrimSpace(strings.ToLower(tag.Value))
			} else if lcLabel == "title" {
				bagTitle = strings.TrimSpace(tag.Value)
			}
		}
	}

	// Make sure access rights are valid, or Fluctus will reject
	// this data when we try to register it.
	accessValid := false
	for _, value := range AccessRights {
		if accessRights == value {
			accessValid = true
		}
	}
	if false == accessValid {
		bagReadResult.ErrorMessage += fmt.Sprintf(
			"In tag file, access (rights) value '%s' is not valid.\n", accessRights)
	}

	// Fluctus will reject IntellectualObjects that don't have a title.
	if bagTitle == "" {
		bagReadResult.ErrorMessage +=
			"Required field Title is missing from tag file.\n"
	}
}

// Saves a file from the tar archive to local disk. This function
// used to save non-data files (manifests, tag files, etc.)
func saveFile(destination string, tarReader *tar.Reader) error {
	outputWriter, err := os.OpenFile(destination, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		return err
	}
	_, err = io.Copy(outputWriter, tarReader)
	if err != nil {
		return err
	}
	return nil
}

// buildFile saves a data file from the tar archive to disk,
// then returns a struct with data we'll need to construct the
// GenericFile object in Fedora later.
func buildFile(tarReader *tar.Reader, tarDirectory string, fileName string, size int64, modTime time.Time, buildIngestData bool) (file *File) {
	file = NewFile()
	file.Path = fileName[strings.Index(fileName, "/data/")+1 : len(fileName)]
	absPath, err := filepath.Abs(filepath.Join(tarDirectory, fileName))
	if err != nil {
		file.ErrorMessage = fmt.Sprintf("Path error: %v", err)
		return file
	}
	uuid := uuid.NewV4()
	file.Uuid = uuid.String()
	file.UuidGenerated = time.Now().UTC()
	file.Size = size
	file.Modified = modTime

	// Set up a MultiWriter to stream data ONCE to file,
	// md5 and sha256. We don't want to process the stream
	// three separate times.
	outputWriter, err := os.OpenFile(absPath, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		file.ErrorMessage = fmt.Sprintf("Error opening writing to %s: %v", absPath, err)
		return file
	}

	// If we're just validating the bag format, don't bother
	// doing the checksums or figuring out mime-type.
	// Partners run our basic validators on their machines,
	// which probably don't even have the required MagicMime C
	// libraries. If we're doing full ingest, we'll calculate
	// the additional sums & take a guess at mime type.
	if buildIngestData == false {
		io.Copy(outputWriter, tarReader)
	} else {
		md5Hash := md5.New()
		shaHash := sha256.New()
		multiWriter := io.MultiWriter(md5Hash, shaHash, outputWriter)
		io.Copy(multiWriter, tarReader)

		file.Md5 = fmt.Sprintf("%x", md5Hash.Sum(nil))
		file.Sha256 = fmt.Sprintf("%x", shaHash.Sum(nil))
		file.Sha256Generated = time.Now().UTC()

		file.MimeType, err = GuessMimeType(absPath)
	}

	return file
}
<commit_msg>Finished fix to PT #114804083<commit_after>package bagman

import (
	"archive/tar"
	"crypto/md5"
	"crypto/sha256"
	"fmt"
	"github.com/APTrust/bagins"
	"github.com/satori/go.uuid"
	"io"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"time"
)

// Untars the file at the specified tarFilePath and returns a list
// of files that were untarred from the archive. Check
// result.Error to ensure there were no errors.
// tarFilePath is the tarFilePath to the tar file that you want to unpack.
// instDomain is the domain name of the institution that owns the bag.
// bagName is the name of the tar file, minus the ".tar" extension.
//
// If param buildIngestData is true, the untar operation will run
// md5 and sha256 checksums on all of the data files in the archive,
// and it will use MimeMagic to figure out each file's mime type.
// We must do this during ingest, so buildIngestData must be true
// when we're running this on our servers.
//
// If we're running in a client environment--that is, an APTrust partner
// is running the bag validation application on their laptop or desktop--
// buildIngestData should be false. There's no need to calculate sha256
// checksums in that case, and we cannot assume that the client would have
// the external MimeMagic C libraries installed. The lack of those libraries
// would cause the application to crash. So buildIngestData == false on
// the client!!
func Untar(tarFilePath, instDomain, bagName string, buildIngestData bool) (result *TarResult) {

	// Set up our result
	tarResult := new(TarResult)
	absInputFile, err := filepath.Abs(tarFilePath)
	if err != nil {
		tarResult.ErrorMessage = fmt.Sprintf("Before untarring, could not determine "+
			"absolute path to downloaded file: %v", err)
		return tarResult
	}
	tarResult.InputFile = absInputFile

	_, err = GetInstitutionFromBagName(filepath.Base(absInputFile))
	if err != nil {
		tarResult.ErrorMessage = err.Error()
		return tarResult
	}

	// Open the tar file for reading.
	file, err := os.Open(tarFilePath)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		tarResult.ErrorMessage = fmt.Sprintf("Could not open file %s for untarring: %v",
			tarFilePath, err)
		return tarResult
	}

	// Record the name of the top-level directory in the tar
	// file. Our spec says that the name of the directory into
	// which the file untars should be the same as the tar file
	// name, minus the .tar extension. So uva-123.tar should
	// untar into a directory called uva-123. This is required
	// so that IntellectualObject and GenericFile identifiers
	// in Fedora can be traced back to the named bag from which
	// they came. Other parts of bagman, such as the file cleanup
	// routines, assume that the untarred directory name will
	// match the tar file name, as the spec demands. When the names
	// do not match, the cleanup routines will not clean up the
	// untarred files, and we'll end up losing a lot of disk space.
	topLevelDir := ""

	// Untar the file and record the results.
	tarReader := tar.NewReader(file)

	for {
		header, err := tarReader.Next()
		if err != nil && err.Error() == "EOF" {
			break // end of archive
		}
		if err != nil {
			tarResult.ErrorMessage = fmt.Sprintf(
				"Error reading tar file header: %v. " +
					"Either this is not a tar file, or the file is corrupt.", err)
			return tarResult
		}

		// Top-level dir will be the first header entry.
		if header.Typeflag == tar.TypeDir && topLevelDir == "" {
			topLevelDir = strings.Replace(header.Name, "/", "", 1)
			// Fix for Windows
			systemNormalizedPath := tarFilePath
			if runtime.GOOS == "windows" && strings.Contains(tarFilePath, "\\") {
				systemNormalizedPath = strings.Replace(tarFilePath, "\\", "/", -1)
			}
			expectedDir := path.Base(systemNormalizedPath)
			if strings.HasSuffix(expectedDir, ".tar") {
				expectedDir = expectedDir[0 : len(expectedDir)-4]
			}
			if topLevelDir != expectedDir {
				tarResult.ErrorMessage = fmt.Sprintf(
					"Bag '%s' should untar to a folder named '%s', but "+
						"it untars to '%s'. Please repackage this bag and try again.",
					path.Base(tarFilePath), expectedDir, topLevelDir)
				return tarResult
			}
		}

		outputPath := filepath.Join(filepath.Dir(absInputFile), header.Name)
		// PT #114804083 - Make sure there's a slash in the path before setting
		// tarfile.OutputDir, or we'll set it incorrectly and when we call
		// bagman.ReadBag(tarResult.OutputDir), it will fail with "file not found".
		pathParts := strings.Split(header.Name, "/")
		tarDirectory := pathParts[0]
		if (tarResult.OutputDir == "" || tarResult.OutputDir == filepath.Dir(absInputFile)) && len(pathParts) > 1 {
			tarResult.OutputDir = filepath.Join(filepath.Dir(absInputFile), tarDirectory)
		}

		// Make sure the directory that we're about to write into exists.
		err = os.MkdirAll(filepath.Dir(outputPath), 0755)
		if err != nil {
			tarResult.ErrorMessage = fmt.Sprintf("Could not create destination file '%s' "+
				"while unpacking tar archive: %v", outputPath, err)
			return tarResult
		}

		// Copy the file, if it's an actual file. Otherwise, ignore it and record
		// a warning. The bag library does not deal with items like symlinks.
		if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA {
			if strings.Contains(header.Name, "/data/") {
				var dataFile *File = nil
				dataFile = buildFile(tarReader, filepath.Dir(absInputFile), header.Name,
					header.Size, header.ModTime, buildIngestData)
				cleanBagName, _ := CleanBagName(bagName)
				dataFile.Identifier = fmt.Sprintf("%s/%s", cleanBagName, dataFile.Path)
				dataFile.IdentifierAssigned = time.Now()
				tarResult.Files = append(tarResult.Files, dataFile)
			} else {
				err = saveFile(outputPath, tarReader)
				if err != nil {
					tarResult.ErrorMessage = fmt.Sprintf("Error copying file from tar archive "+
						"to '%s': %v", outputPath, err)
					return tarResult
				}
			}

			outputRelativePath := strings.Replace(outputPath, tarResult.OutputDir+"/", "", 1)
			tarResult.FilesUnpacked = append(tarResult.FilesUnpacked, outputRelativePath)

		} else if header.Typeflag != tar.TypeDir {
			tarResult.Warnings = append(tarResult.Warnings,
				fmt.Sprintf("Ignoring item %s of type %c because it's neither a file nor a directory",
					header.Name, header.Typeflag))
		}
	}
	sort.Strings(tarResult.FilesUnpacked)
	return tarResult
}

// Reads an untarred bag. The tarFilePath parameter should be a path to
// a directory that contains the bag, info and manifest files.
// The bag content should be in the data directory under tarFilePath.
// Check result.Error to ensure there were no errors.
func ReadBag(tarFilePath string) (result *BagReadResult) {
	bagReadResult := new(BagReadResult)
	bagReadResult.Path = tarFilePath

	// Final param to bagins.ReadBag is the name of the checksum file.
	// That param defaults to manifest-md5.txt, which is what it
	// should be for bags we're fetching from the S3 receiving buckets.
	bag, err := bagins.ReadBag(tarFilePath, []string{"bagit.txt", "bag-info.txt", "aptrust-info.txt"}, "")
	if err != nil {
		bagReadResult.ErrorMessage = fmt.Sprintf("Error unpacking bag: %v", err)
		return bagReadResult
	}

	fileNames, err := bag.ListFiles()
	if err != nil {
		bagReadResult.ErrorMessage = fmt.Sprintf("Could not list bag files: %v ", err)
		return bagReadResult
	}

	dataDirPrefix := "data/"
	if runtime.GOOS == "windows" {
		dataDirPrefix = "data\\"
	}

	errMsg := ""
	bagReadResult.Files = make([]string, len(fileNames))
	hasBagit := false
	hasAPTrustInfo := false
	hasMd5Manifest := false
	hasDataFiles := false
	for index, fileName := range fileNames {
		bagReadResult.Files[index] = fileName
		if fileName == "bagit.txt" {
			hasBagit = true
		} else if fileName == "aptrust-info.txt" {
			hasAPTrustInfo = true
		} else if fileName == "manifest-md5.txt" {
			hasMd5Manifest = true
		} else if strings.HasPrefix(fileName, dataDirPrefix) {
			hasDataFiles = true
		}
	}
	if !hasBagit {
		errMsg += " Bag is missing bagit.txt file.\n"
	}
	if !hasAPTrustInfo {
		errMsg += " Bag is missing aptrust-info.txt file.\n"
	}
	if !hasMd5Manifest {
		errMsg += " Bag is missing manifest-md5.txt file.\n"
	}
	if !hasDataFiles {
		errMsg += " Bag's data directory is missing or empty.\n"
	}

	extractTags(bag, bagReadResult)

	checksumErrors := bag.Manifest.RunChecksums()
	if len(checksumErrors) > 0 {
		errMsg += "The following checksums could not be verified:\n"
		bagReadResult.ChecksumErrors = make([]error, len(checksumErrors))
		for i, err := range checksumErrors {
			bagReadResult.ChecksumErrors[i] = err
			errMsg += "  " + err.Error() + ".\n"
		}
	}

	if errMsg != "" {
		bagReadResult.ErrorMessage += fmt.Sprintf(errMsg)
	}

	return bagReadResult
}

// Extract all of the tags from tag files "bagit.txt", "bag-info.txt",
// and "aptrust-info.txt", and put those tags into the Tags member
// of the BagReadResult structure.
func extractTags(bag *bagins.Bag, bagReadResult *BagReadResult) {
	tagFiles := []string{"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	accessRights := ""
	bagTitle := ""
	for _, file := range tagFiles {
		tagFile, err := bag.TagFile(file)
		if err != nil {
			bagReadResult.ErrorMessage = fmt.Sprintf("Error reading tags from bag: %v", err)
			return
		}
		tagFields := tagFile.Data.Fields()

		for _, tagField := range tagFields {
			tag := Tag{tagField.Label(), strings.TrimSpace(tagField.Value())}
			bagReadResult.Tags = append(bagReadResult.Tags, tag)

			lcLabel := strings.ToLower(tag.Label)
			if lcLabel == "access" {
				accessRights = strings.TrimSpace(strings.ToLower(tag.Value))
			} else if accessRights == "" && lcLabel == "rights" {
				accessRights = strings.TrimSpace(strings.ToLower(tag.Value))
			} else if lcLabel == "title" {
				bagTitle = strings.TrimSpace(tag.Value)
			}
		}
	}

	// Make sure access rights are valid, or Fluctus will reject
	// this data when we try to register it.
	accessValid := false
	for _, value := range AccessRights {
		if accessRights == value {
			accessValid = true
		}
	}
	if false == accessValid {
		bagReadResult.ErrorMessage += fmt.Sprintf(
			"In tag file, access (rights) value '%s' is not valid.\n", accessRights)
	}

	// Fluctus will reject IntellectualObjects that don't have a title.
	if bagTitle == "" {
		bagReadResult.ErrorMessage +=
			"Required field Title is missing from tag file.\n"
	}
}

// Saves a file from the tar archive to local disk. This function
// used to save non-data files (manifests, tag files, etc.)
func saveFile(destination string, tarReader *tar.Reader) error {
	outputWriter, err := os.OpenFile(destination, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		return err
	}
	_, err = io.Copy(outputWriter, tarReader)
	if err != nil {
		return err
	}
	return nil
}

// buildFile saves a data file from the tar archive to disk,
// then returns a struct with data we'll need to construct the
// GenericFile object in Fedora later.
func buildFile(tarReader *tar.Reader, tarDirectory string, fileName string, size int64, modTime time.Time, buildIngestData bool) (file *File) {
	file = NewFile()
	file.Path = fileName[strings.Index(fileName, "/data/")+1 : len(fileName)]
	absPath, err := filepath.Abs(filepath.Join(tarDirectory, fileName))
	if err != nil {
		file.ErrorMessage = fmt.Sprintf("Path error: %v", err)
		return file
	}
	uuid := uuid.NewV4()
	file.Uuid = uuid.String()
	file.UuidGenerated = time.Now().UTC()
	file.Size = size
	file.Modified = modTime

	// Set up a MultiWriter to stream data ONCE to file,
	// md5 and sha256. We don't want to process the stream
	// three separate times.
	outputWriter, err := os.OpenFile(absPath, os.O_CREATE|os.O_WRONLY, 0644)
	if outputWriter != nil {
		defer outputWriter.Close()
	}
	if err != nil {
		file.ErrorMessage = fmt.Sprintf("Error opening writing to %s: %v", absPath, err)
		return file
	}

	// If we're just validating the bag format, don't bother
	// doing the checksums or figuring out mime-type.
	// Partners run our basic validators on their machines,
	// which probably don't even have the required MagicMime C
	// libraries. If we're doing full ingest, we'll calculate
	// the additional sums & take a guess at mime type.
	if buildIngestData == false {
		io.Copy(outputWriter, tarReader)
	} else {
		md5Hash := md5.New()
		shaHash := sha256.New()
		multiWriter := io.MultiWriter(md5Hash, shaHash, outputWriter)
		io.Copy(multiWriter, tarReader)

		file.Md5 = fmt.Sprintf("%x", md5Hash.Sum(nil))
		file.Sha256 = fmt.Sprintf("%x", shaHash.Sum(nil))
		file.Sha256Generated = time.Now().UTC()

		file.MimeType, err = GuessMimeType(absPath)
	}

	return file
}
<|endoftext|>


<commit_before>package csvutil

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/grokify/gocharts/data/table"
	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/type/stringsutil"
)

var DebugReadCSV = false // should not need to edit

/*

For UTF-8 BOM, csv.Reader.Read() will return error = "line 1, column 1: bare \" in non-quoted-field"

If you encounter this close the file and call again with stripBom = true

*/

// NewReader will create a csv.Reader and optionally strip off the
// byte order mark (BOM) if requested. Close file reader with
// `defer f.Close()`.
func NewReader(path string, comma rune, stripBom bool) (*csv.Reader, *os.File, error) {
	var csvReader *csv.Reader
	var file *os.File
	file, err := os.Open(path)
	if err != nil {
		return csvReader, file, err
	}
	if stripBom {
		b3 := make([]byte, 3)
		_, err := file.Read(b3)
		if err != nil {
			return csvReader, file, err
		}
	}
	csvReader = csv.NewReader(file)
	csvReader.Comma = comma
	return csvReader, file, nil
}

// Writer is a struct for a CSV/TSV writer.
type Writer struct {
	Separator        string
	StripRepeatedSep bool
	ReplaceSeparator bool
	SeparatorAlt     string
	File             *os.File
}

// NewWriter returns a Writer with the separator params set.
func NewWriter(filepath, sep string, replaceSeparator bool, alt string) (Writer, error) {
	w := Writer{
		Separator:        sep,
		StripRepeatedSep: false,
		ReplaceSeparator: replaceSeparator,
		SeparatorAlt:     alt}
	return w, w.open(filepath)
}

// Open opens a filepath.
func (w *Writer) open(filepath string) error {
	f, err := os.Create(filepath)
	if err != nil {
		return err
	}
	w.File = f
	return nil
}

// AddLine adds an []interface{} to the file.
func (w *Writer) AddLine(cells []interface{}) error {
	_, err := w.File.WriteString(
		stringsutil.JoinInterface(
			cells, w.Separator, false, w.ReplaceSeparator, w.SeparatorAlt) + "\n")
	if err != nil {
		return err
	}
	return w.File.Sync()
}

// Close closes the file.
func (w *Writer) Close() error {
	return w.File.Close()
}

func NewWriterFile(filename string) (*csv.Writer, *os.File, error) {
	file, err := os.Create(filename)
	if err != nil {
		return nil, file, err
	}
	//defer file.Close()

	writer := csv.NewWriter(file)
	//defer writer.Flush()
	return writer, file, nil
}

type CSVHeader struct {
	Columns []string
}

func (ch *CSVHeader) Index(want string) int {
	for i, try := range ch.Columns {
		if want == try {
			return i
		}
	}
	return -1
}

func (ch *CSVHeader) RecordMatch(row []string, andFilter map[string]stringsutil.MatchInfo) (bool, error) {
	for colName, matchInfo := range andFilter {
		idx := ch.Index(colName)
		if idx >= len(row) {
			return false, nil
		}
		match, err := stringsutil.Match(row[idx], matchInfo)
		if err != nil {
			return false, err
		}
		if !match {
			return false, nil
		}
	}
	return true, nil
}

func (ch *CSVHeader) RecordToMSS(row []string) map[string]string {
	mss := map[string]string{}
	l := len(row)
	for i, key := range ch.Columns {
		if i < l {
			mss[key] = row[i]
		}
	}
	return mss
}

func FilterCSVFile(inPath, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error {
	reader, inFile, err := NewReader(inPath, inComma, inStripBom)
	if err != nil {
		return err
	}
	defer inFile.Close()
	writer, outFile, err := NewWriterFile(outPath)
	if err != nil {
		return err
	}
	defer writer.Flush()
	defer outFile.Close()
	return WriteCSVFiltered(reader, writer, andFilter, true)
}

// MergeFilterCSVFiles can merge and filter multiple CSV files. It expects row definitions to be the same
// across all input files.
func MergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error {
	writer, outFile, err := NewWriterFile(outPath)
	if err != nil {
		return err
	}
	defer writer.Flush()
	defer outFile.Close()

	for i, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return err
		}
		defer inFile.Close()

		writeHeader := false
		if i == 0 {
			writeHeader = true
		}
		err = WriteCSVFiltered(reader, writer, andFilter, writeHeader)
		if err != nil {
			return err
		}
	}
	return nil
}

/*
type JsonRecordsInfo struct {
	Meta    JsonRecordsInfoMeta `json:"meta"`
	Records []map[string]string `json:"records"`
}

type JsonRecordsInfoMeta struct {
	Count int `json:"count"`
}*/

func ReadMergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) (table.DocumentsSet, error) {
	//data := JsonRecordsInfo{Records: []map[string]string{}}
	data := table.NewDocumentsSet()

	for _, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return data, err
		}

		csvHeader := CSVHeader{}
		j := -1

		for {
			line, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return data, err
			}
			j += 1

			if j == 0 {
				csvHeader.Columns = line
				continue
			}
			match, err := csvHeader.RecordMatch(line, andFilter)
			if err != nil {
				return data, err
			}
			if !match {
				continue
			}

			mss := csvHeader.RecordToMSS(line)
			data.Documents = append(data.Documents, mss)
		}
		err = inFile.Close()
		if err != nil {
			return data, err
		}
	}
	data.Inflate()
	return data, nil
}

func MergeFilterCSVFilesToJSON(inPaths []string, outPath string, inComma rune, inStripBom bool, perm os.FileMode, andFilter map[string]stringsutil.MatchInfo) error {
	data, err := ReadMergeFilterCSVFiles(inPaths, outPath, inComma, inStripBom, andFilter)
	if err != nil {
		return err
	}
	bytes, err := jsonutil.MarshalSimple(data, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(outPath, bytes, perm)
}

func MergeFilterCSVFilesToJSONL(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error {
	outFh, err := os.Create(outPath)
	if err != nil {
		return err
	}

	for _, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return err
		}

		csvHeader := CSVHeader{}
		j := -1

		for {
			line, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}
			j += 1

			if j == 0 {
				csvHeader.Columns = line
				continue
			}
			match, err := csvHeader.RecordMatch(line, andFilter)
			if err != nil {
				return err
			}
			if !match {
				continue
			}

			mss := csvHeader.RecordToMSS(line)

			bytes, err := json.Marshal(mss)
			if err != nil {
				return err
			}
			_, err = outFh.Write(bytes)
			if err != nil {
				return err
			}
			_, err = outFh.Write([]byte("\n"))
			if err != nil {
				return err
			}
			outFh.Sync()
		}
		err = inFile.Close()
		if err != nil {
			return err
		}
	}
	return outFh.Close()
}

// WriteCSVFiltered filters an existing CSV and writes the matching lines
// to a *csv.Writer.
func WriteCSVFiltered(reader *csv.Reader, writer *csv.Writer, andFilter map[string]stringsutil.MatchInfo, writeHeader bool) error {
	csvHeader := CSVHeader{}
	i := -1
	for {
		line, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		i += 1
		if i == 0 {
			csvHeader.Columns = line
			if writeHeader {
				err := writer.Write(line)
				if err != nil {
					return err
				}
			}
			continue
		}
		match, err := csvHeader.RecordMatch(line, andFilter)
		if err != nil {
			return err
		}
		if match {
			err := writer.Write(line)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// NewTableDataFileCSV reads in a CSV file and returns a TableData struct.
func NewTableDataFileCSV(path string, comma rune, stripBom bool) (table.TableData, error) {
	tbl := table.NewTableData()
	csvReader, f, err := NewReader(path, comma, stripBom)
	if err != nil {
		return tbl, err
	}
	defer f.Close()
	if DebugReadCSV {
		i := -1
		for {
			line, err := csvReader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return tbl, err
			}
			i += 1
			if i == 0 {
				tbl.Columns = line
				continue
			}
			tbl.Records = append(tbl.Records, line)
			if i > 2500 {
				fmt.Printf("[%v] %v\n", i, strings.Join(line, ","))
			}
		}

	} else {
		lines, err := csvReader.ReadAll()
		if err != nil {
			return tbl, err
		}
		tbl.LoadMergedRows(lines)
	}
	return tbl, nil
}
<commit_msg>add csvutil.NewTableDataFileSimple()<commit_after>package csvutil

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/grokify/gocharts/data/table"
	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/type/stringsutil"
)

var DebugReadCSV = false // should not need to edit

/*

For UTF-8 BOM, csv.Reader.Read() will return error = "line 1, column 1: bare \" in non-quoted-field"

If you encounter this close the file and call again with stripBom = true

*/

// NewReader will create a csv.Reader and optionally strip off the
// byte order mark (BOM) if requested. Close file reader with
// `defer f.Close()`.
func NewReader(path string, comma rune, stripBom bool) (*csv.Reader, *os.File, error) {
	var csvReader *csv.Reader
	var file *os.File
	file, err := os.Open(path)
	if err != nil {
		return csvReader, file, err
	}
	if stripBom {
		b3 := make([]byte, 3)
		_, err := file.Read(b3)
		if err != nil {
			return csvReader, file, err
		}
	}
	csvReader = csv.NewReader(file)
	csvReader.Comma = comma
	return csvReader, file, nil
}

// Writer is a struct for a CSV/TSV writer.
type Writer struct {
	Separator        string
	StripRepeatedSep bool
	ReplaceSeparator bool
	SeparatorAlt     string
	File             *os.File
}

// NewWriter returns a Writer with the separator params set.
func NewWriter(filepath, sep string, replaceSeparator bool, alt string) (Writer, error) {
	w := Writer{
		Separator:        sep,
		StripRepeatedSep: false,
		ReplaceSeparator: replaceSeparator,
		SeparatorAlt:     alt}
	return w, w.open(filepath)
}

// Open opens a filepath.
func (w *Writer) open(filepath string) error {
	f, err := os.Create(filepath)
	if err != nil {
		return err
	}
	w.File = f
	return nil
}

// AddLine adds an []interface{} to the file.
func (w *Writer) AddLine(cells []interface{}) error {
	_, err := w.File.WriteString(
		stringsutil.JoinInterface(
			cells, w.Separator, false, w.ReplaceSeparator, w.SeparatorAlt) + "\n")
	if err != nil {
		return err
	}
	return w.File.Sync()
}

// Close closes the file.
func (w *Writer) Close() error {
	return w.File.Close()
}

func NewWriterFile(filename string) (*csv.Writer, *os.File, error) {
	file, err := os.Create(filename)
	if err != nil {
		return nil, file, err
	}
	//defer file.Close()

	writer := csv.NewWriter(file)
	//defer writer.Flush()
	return writer, file, nil
}

type CSVHeader struct {
	Columns []string
}

func (ch *CSVHeader) Index(want string) int {
	for i, try := range ch.Columns {
		if want == try {
			return i
		}
	}
	return -1
}

func (ch *CSVHeader) RecordMatch(row []string, andFilter map[string]stringsutil.MatchInfo) (bool, error) {
	for colName, matchInfo := range andFilter {
		idx := ch.Index(colName)
		if idx >= len(row) {
			return false, nil
		}
		match, err := stringsutil.Match(row[idx], matchInfo)
		if err != nil {
			return false, err
		}
		if !match {
			return false, nil
		}
	}
	return true, nil
}

func (ch *CSVHeader) RecordToMSS(row []string) map[string]string {
	mss := map[string]string{}
	l := len(row)
	for i, key := range ch.Columns {
		if i < l {
			mss[key] = row[i]
		}
	}
	return mss
}

func FilterCSVFile(inPath, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error {
	reader, inFile, err := NewReader(inPath, inComma, inStripBom)
	if err != nil {
		return err
	}
	defer inFile.Close()
	writer, outFile, err := NewWriterFile(outPath)
	if err != nil {
		return err
	}
	defer writer.Flush()
	defer outFile.Close()
	return WriteCSVFiltered(reader, writer, andFilter, true)
}

// MergeFilterCSVFiles can merge and filter multiple CSV files. It expects row definitions to be the same
// across all input files.
func MergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error {
	writer, outFile, err := NewWriterFile(outPath)
	if err != nil {
		return err
	}
	defer writer.Flush()
	defer outFile.Close()

	for i, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return err
		}
		defer inFile.Close()

		writeHeader := false
		if i == 0 {
			writeHeader = true
		}
		err = WriteCSVFiltered(reader, writer, andFilter, writeHeader)
		if err != nil {
			return err
		}
	}
	return nil
}

/*
type JsonRecordsInfo struct {
	Meta    JsonRecordsInfoMeta `json:"meta"`
	Records []map[string]string `json:"records"`
}

type JsonRecordsInfoMeta struct {
	Count int `json:"count"`
}*/

func ReadMergeFilterCSVFiles(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) (table.DocumentsSet, error) {
	//data := JsonRecordsInfo{Records: []map[string]string{}}
	data := table.NewDocumentsSet()

	for _, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return data, err
		}

		csvHeader := CSVHeader{}
		j := -1

		for {
			line, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return data, err
			}
			j += 1

			if j == 0 {
				csvHeader.Columns = line
				continue
			}
			match, err := csvHeader.RecordMatch(line, andFilter)
			if err != nil {
				return data, err
			}
			if !match {
				continue
			}

			mss := csvHeader.RecordToMSS(line)
			data.Documents = append(data.Documents, mss)
		}
		err = inFile.Close()
		if err != nil {
			return data, err
		}
	}
	data.Inflate()
	return data, nil
}

func MergeFilterCSVFilesToJSON(inPaths []string, outPath string, inComma rune, inStripBom bool, perm os.FileMode, andFilter map[string]stringsutil.MatchInfo) error {
	data, err := ReadMergeFilterCSVFiles(inPaths, outPath, inComma, inStripBom, andFilter)
	if err != nil {
		return err
	}
	bytes, err := jsonutil.MarshalSimple(data, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(outPath, bytes, perm)
}

func MergeFilterCSVFilesToJSONL(inPaths []string, outPath string, inComma rune, inStripBom bool, andFilter map[string]stringsutil.MatchInfo) error {
	outFh, err := os.Create(outPath)
	if err != nil {
		return err
	}

	for _, inPath := range inPaths {
		reader, inFile, err := NewReader(inPath, inComma, inStripBom)
		if err != nil {
			return err
		}

		csvHeader := CSVHeader{}
		j := -1

		for {
			line, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return err
			}
			j += 1

			if j == 0 {
				csvHeader.Columns = line
				continue
			}
			match, err := csvHeader.RecordMatch(line, andFilter)
			if err != nil {
				return err
			}
			if !match {
				continue
			}

			mss := csvHeader.RecordToMSS(line)

			bytes, err := json.Marshal(mss)
			if err != nil {
				return err
			}
			_, err = outFh.Write(bytes)
			if err != nil {
				return err
			}
			_, err = outFh.Write([]byte("\n"))
			if err != nil {
				return err
			}
			outFh.Sync()
		}
		err = inFile.Close()
		if err != nil {
			return err
		}
	}
	return outFh.Close()
}

// WriteCSVFiltered filters an existing CSV and writes the matching lines
// to a *csv.Writer.
func WriteCSVFiltered(reader *csv.Reader, writer *csv.Writer, andFilter map[string]stringsutil.MatchInfo, writeHeader bool) error {
	csvHeader := CSVHeader{}
	i := -1
	for {
		line, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}
		i += 1
		if i == 0 {
			csvHeader.Columns = line
			if writeHeader {
				err := writer.Write(line)
				if err != nil {
					return err
				}
			}
			continue
		}
		match, err := csvHeader.RecordMatch(line, andFilter)
		if err != nil {
			return err
		}
		if match {
			err := writer.Write(line)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func NewTableDataFileSimple(path string, sep string, hasHeader, trimSpace bool) (table.TableData, error) {
	tbl := table.NewTableData()
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return tbl, err
	}
	lines := strings.Split(string(data), "\n")
	for i, line := range lines {
		if trimSpace {
			line = strings.TrimSpace(line)
		}
		parts := strings.Split(line, sep)
		parts = stringsutil.SliceLinesTrimSpace(parts, false)
		if hasHeader && i == 0 {
			tbl.Columns = parts
		} else {
			tbl.Records = append(tbl.Records, parts)
		}
	}
	return tbl, nil
}

// NewTableDataFileCSV reads in a CSV file and returns a TableData struct.
func NewTableDataFileCSV(path string, comma rune, stripBom bool) (table.TableData, error) {
	tbl := table.NewTableData()
	csvReader, f, err := NewReader(path, comma, stripBom)
	if err != nil {
		return tbl, err
	}
	defer f.Close()
	if DebugReadCSV {
		i := -1
		for {
			line, err := csvReader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				return tbl, err
			}
			i += 1
			if i == 0 {
				tbl.Columns = line
				continue
			}
			tbl.Records = append(tbl.Records, line)
			if i > 2500 {
				fmt.Printf("[%v] %v\n", i, strings.Join(line, ","))
			}
		}

	} else {
		lines, err := csvReader.ReadAll()
		if err != nil {
			return tbl, err
		}
		tbl.LoadMergedRows(lines)
	}
	return tbl, nil
}
<|endoftext|>


<commit_before>package main // import "kkn.fi/cmd/vanity"

import (
	"fmt"
	"net/http"
	"strings"
)

type (
	// packageConfig defines Go package that has vanity import defined by Path,
	// VCS system type and VCS URL.
	packageConfig struct {
		// Name is the name of the Go package.
		Name string
		// VCS is version control system used by the project.
		VCS string
		// URL of the git repository
		URL string
	}
	// vanityServer is the actual HTTP server for Go vanity domains.
	vanityServer struct {
		// Domain is the vanity domain.
		Domain string
		// Packages contains settings for vanity Packages.
		Packages map[string]*packageConfig
	}
)

// newPackage returns a new Package given a path and VCS.
func newPackage(name, vcs, url string) *packageConfig {
	return &packageConfig{
		Name: name,
		VCS:  vcs,
		URL:  url,
	}
}

// newServer returns a new Vanity Server given domain name and
// vanity package configuration.
func newServer(domain string, config map[string]*packageConfig) *vanityServer {
	return &vanityServer{
		Domain:   domain,
		Packages: config,
	}
}

// path returns the path portition of the package name
func (p packageConfig) path() string {
	i := strings.Index(p.Name, "/")
	return p.Name[i+1:]
}

// goDocURL returns the HTTP URL to godoc.org.
func (p packageConfig) goDocURL(domain string) string {
	return fmt.Sprintf("https://godoc.org/%v%v", domain, p.Name)
}

// goImportMeta creates the <meta/> HTML tag containing name and content attributes.
func (p packageConfig) goImportMeta(domain string) string {
	s := `<meta name="go-import" content="%v/%v %v %v">`
	return fmt.Sprintf(s, domain, p.path(), p.VCS, p.URL)
}

func (s vanityServer) find(path string) *packageConfig {
	p, ok := s.Packages[path]
	if !ok {
		return nil
	}
	return p
}

// ServeHTTP is an HTTP Handler for Go vanity domain.
func (s vanityServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	if r.Method != http.MethodGet {
		status := http.StatusMethodNotAllowed
		http.Error(w, http.StatusText(status), status)
		return
	}

	pack := s.find(r.URL.Path)
	if pack == nil {
		http.NotFound(w, r)
		return
	}
	if r.FormValue("go-get") != "1" {
		url := pack.goDocURL(s.Domain)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
		return
	}
	fmt.Fprint(w, pack.goImportMeta(s.Domain))
}
<commit_msg>Remove find()<commit_after>package main // import "kkn.fi/cmd/vanity"

import (
	"fmt"
	"net/http"
	"strings"
)

type (
	// packageConfig defines Go package that has vanity import defined by Path,
	// VCS system type and VCS URL.
	packageConfig struct {
		// Name is the name of the Go package.
		Name string
		// VCS is version control system used by the project.
		VCS string
		// URL of the git repository
		URL string
	}
	// vanityServer is the actual HTTP server for Go vanity domains.
	vanityServer struct {
		// Domain is the vanity domain.
		Domain string
		// Packages contains settings for vanity Packages.
		Packages map[string]*packageConfig
	}
)

// newPackage returns a new Package given a path and VCS.
func newPackage(name, vcs, url string) *packageConfig {
	return &packageConfig{
		Name: name,
		VCS:  vcs,
		URL:  url,
	}
}

// newServer returns a new Vanity Server given domain name and
// vanity package configuration.
func newServer(domain string, config map[string]*packageConfig) *vanityServer {
	return &vanityServer{
		Domain:   domain,
		Packages: config,
	}
}

// path returns the path portition of the package name
func (p packageConfig) path() string {
	i := strings.Index(p.Name, "/")
	return p.Name[i+1:]
}

// goDocURL returns the HTTP URL to godoc.org.
func (p packageConfig) goDocURL(domain string) string {
	return fmt.Sprintf("https://godoc.org/%v%v", domain, p.Name)
}

// goImportMeta creates the <meta/> HTML tag containing name and content attributes.
func (p packageConfig) goImportMeta(domain string) string {
	s := `<meta name="go-import" content="%v/%v %v %v">`
	return fmt.Sprintf(s, domain, p.path(), p.VCS, p.URL)
}

// ServeHTTP is an HTTP Handler for Go vanity domain.
func (s vanityServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	if r.Method != http.MethodGet {
		status := http.StatusMethodNotAllowed
		http.Error(w, http.StatusText(status), status)
		return
	}

	pack, ok := s.Packages[r.URL.Path]
	if !ok {
		http.NotFound(w, r)
		return
	}
	if r.FormValue("go-get") != "1" {
		url := pack.goDocURL(s.Domain)
		http.Redirect(w, r, url, http.StatusTemporaryRedirect)
		return
	}
	fmt.Fprint(w, pack.goImportMeta(s.Domain))
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Package trace provides methods to submit Zipkin style Span to tcollector Server.
package trace

import (
	"encoding/base64"
	"encoding/binary"
	"net"
	"strconv"
	"strings"
	"time"

	tc "github.com/uber/tchannel/golang"
	"github.com/uber/tchannel/golang/thrift"
	"github.com/uber/tchannel/golang/trace/thrift/gen-go/tcollector"
)

const (
	tcollectorServiceName = "tcollector"
	chanBufferSize        = 100
)

type zipkinData struct {
	Span              tc.Span
	Annotations       []tc.Annotation
	BinaryAnnotations []tc.BinaryAnnotation
	TargetEndpoint    tc.TargetEndpoint
}

// ZipkinTraceReporter is a trace reporter that submits trace spans in to zipkin trace server.
type ZipkinTraceReporter struct {
	tchannel *tc.Channel
	client   tcollector.TChanTCollector
	c        chan zipkinData
}

// NewZipkinTraceReporter returns a zipkin trace reporter that submits span to tcollector service.
func NewZipkinTraceReporter(ch *tc.Channel) *ZipkinTraceReporter {
	thriftClient := thrift.NewClient(ch, tcollectorServiceName, nil)
	client := tcollector.NewTChanTCollectorClient(thriftClient)
	// create the goroutine method to actually to the submit Span.
	reporter := &ZipkinTraceReporter{tchannel: ch, client: client, c: make(chan zipkinData, chanBufferSize)}
	go reporter.zipkinSpanWorker()
	return reporter
}

// Report method will submit trace span to tcollector server.
func (r *ZipkinTraceReporter) Report(
	span tc.Span, annotations []tc.Annotation, binaryAnnotations []tc.BinaryAnnotation, targetEndpoint tc.TargetEndpoint) {
	data := zipkinData{
		Span:              span,
		Annotations:       annotations,
		BinaryAnnotations: binaryAnnotations,
		TargetEndpoint:    targetEndpoint,
	}

	select {
	case r.c <- data:
	default:
	}
}

func (r *ZipkinTraceReporter) zipkinReport(data *zipkinData) error {
	ctx, cancel := tc.NewContextBuilder(time.Second).
		SetShardKey(base64Encode(data.Span.TraceID())).Build()
	defer cancel()

	thriftSpan := buildZipkinSpan(data.Span, data.Annotations, data.BinaryAnnotations, data.TargetEndpoint)
	// client submit
	// ignore the response result because TChannel shouldn't care about it.
	_, err := r.client.Submit(ctx, thriftSpan)
	return err
}

func (r *ZipkinTraceReporter) zipkinSpanWorker() {
	for {
		data := <-r.c
		if err := r.zipkinReport(&data); err != nil {
			r.tchannel.Logger().Infof("Zipkin Span submit failed. Get error: %v", err)
		}
	}
}

// buildZipkinSpan builds zipkin span based on tchannel span.
func buildZipkinSpan(span tc.Span, annotations []tc.Annotation, binaryAnnotations []tc.BinaryAnnotation, targetEndpoint tc.TargetEndpoint) *tcollector.Span {
	hostport := strings.Split(targetEndpoint.HostPort, ":")
	port, _ := strconv.ParseInt(hostport[1], 10, 32)
	host := tcollector.Endpoint{
		Ipv4:        int32(inetAton(hostport[0])),
		Port:        int32(port),
		ServiceName: targetEndpoint.ServiceName,
	}

	// TODO Add BinaryAnnotations
	thriftSpan := tcollector.Span{
		TraceId:     uint64ToBytes(span.TraceID()),
		Host:        &host,
		Name:        targetEndpoint.Operation,
		Id:          uint64ToBytes(span.SpanID()),
		ParentId:    uint64ToBytes(span.ParentID()),
		Annotations: buildZipkinAnnotations(annotations),
		Debug:       false,
	}

	return &thriftSpan
}

// buildZipkinAnnotations builds zipkin Annotations based on tchannel annotations.
func buildZipkinAnnotations(anns []tc.Annotation) []*tcollector.Annotation {
	zipkinAnns := make([]*tcollector.Annotation, len(anns))
	for i, ann := range anns {
		zipkinAnns[i] = &tcollector.Annotation{
			Timestamp: (float64)(ann.Timestamp.UnixNano() / 1e6),
			Value:     (string)(ann.Key),
		}
	}
	return zipkinAnns
}

// inetAton converts string Ipv4 to uint32
func inetAton(ip string) uint32 {
	ipBytes := net.ParseIP(ip).To4()
	return binary.BigEndian.Uint32(ipBytes)
}

// base64Encode encodes uint64 with base64 StdEncoding.
func base64Encode(data uint64) string {
	return base64.StdEncoding.EncodeToString(uint64ToBytes(data))
}

// uint64ToBytes converts uint64 to bytes.
func uint64ToBytes(i uint64) []byte {
	buf := make([]byte, 8)
	binary.BigEndian.PutUint64(buf, uint64(i))
	return buf
}

// ZipkinTraceReporterFactory builds ZipkinTraceReporter by given TChannel instance.
func ZipkinTraceReporterFactory(tchannel *tc.Channel) tc.TraceReporter {
	return NewZipkinTraceReporter(tchannel)
}
<commit_msg>use for loop around the channel add log when channel is full<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Package trace provides methods to submit Zipkin style Span to tcollector Server.
package trace

import (
	"encoding/base64"
	"encoding/binary"
	"net"
	"strconv"
	"strings"
	"time"

	tc "github.com/uber/tchannel/golang"
	"github.com/uber/tchannel/golang/thrift"
	"github.com/uber/tchannel/golang/trace/thrift/gen-go/tcollector"
)

const (
	tcollectorServiceName = "tcollector"
	chanBufferSize        = 100
)

type zipkinData struct {
	Span              tc.Span
	Annotations       []tc.Annotation
	BinaryAnnotations []tc.BinaryAnnotation
	TargetEndpoint    tc.TargetEndpoint
}

// ZipkinTraceReporter is a trace reporter that submits trace spans in to zipkin trace server.
type ZipkinTraceReporter struct {
	tchannel *tc.Channel
	client   tcollector.TChanTCollector
	c        chan zipkinData
	logger   tc.Logger
}

// NewZipkinTraceReporter returns a zipkin trace reporter that submits span to tcollector service.
func NewZipkinTraceReporter(ch *tc.Channel) *ZipkinTraceReporter {
	thriftClient := thrift.NewClient(ch, tcollectorServiceName, nil)
	client := tcollector.NewTChanTCollectorClient(thriftClient)
	// create the goroutine method to actually to the submit Span.
	reporter := &ZipkinTraceReporter{
		tchannel: ch,
		client:   client,
		c:        make(chan zipkinData, chanBufferSize),
		logger:   ch.Logger(),
	}
	go reporter.zipkinSpanWorker()
	return reporter
}

// Report method will submit trace span to tcollector server.
func (r *ZipkinTraceReporter) Report(
	span tc.Span, annotations []tc.Annotation, binaryAnnotations []tc.BinaryAnnotation, targetEndpoint tc.TargetEndpoint) {
	data := zipkinData{
		Span:              span,
		Annotations:       annotations,
		BinaryAnnotations: binaryAnnotations,
		TargetEndpoint:    targetEndpoint,
	}

	select {
	case r.c <- data:
	default:
		r.logger.Infof("Buffer channel for zipkin trace report is full.")
	}
}

func (r *ZipkinTraceReporter) zipkinReport(data *zipkinData) error {
	ctx, cancel := tc.NewContextBuilder(time.Second).
		SetShardKey(base64Encode(data.Span.TraceID())).Build()
	defer cancel()

	thriftSpan := buildZipkinSpan(data.Span, data.Annotations, data.BinaryAnnotations, data.TargetEndpoint)
	// client submit
	// ignore the response result because TChannel shouldn't care about it.
	_, err := r.client.Submit(ctx, thriftSpan)
	return err
}

func (r *ZipkinTraceReporter) zipkinSpanWorker() {
	for data := range r.c {
		if err := r.zipkinReport(&data); err != nil {
			r.tchannel.Logger().Infof("Zipkin Span submit failed. Get error: %v", err)
		}
	}
}

// buildZipkinSpan builds zipkin span based on tchannel span.
func buildZipkinSpan(span tc.Span, annotations []tc.Annotation, binaryAnnotations []tc.BinaryAnnotation, targetEndpoint tc.TargetEndpoint) *tcollector.Span {
	hostport := strings.Split(targetEndpoint.HostPort, ":")
	port, _ := strconv.ParseInt(hostport[1], 10, 32)
	host := tcollector.Endpoint{
		Ipv4:        int32(inetAton(hostport[0])),
		Port:        int32(port),
		ServiceName: targetEndpoint.ServiceName,
	}

	// TODO Add BinaryAnnotations
	thriftSpan := tcollector.Span{
		TraceId:     uint64ToBytes(span.TraceID()),
		Host:        &host,
		Name:        targetEndpoint.Operation,
		Id:          uint64ToBytes(span.SpanID()),
		ParentId:    uint64ToBytes(span.ParentID()),
		Annotations: buildZipkinAnnotations(annotations),
		Debug:       false,
	}

	return &thriftSpan
}

// buildZipkinAnnotations builds zipkin Annotations based on tchannel annotations.
func buildZipkinAnnotations(anns []tc.Annotation) []*tcollector.Annotation {
	zipkinAnns := make([]*tcollector.Annotation, len(anns))
	for i, ann := range anns {
		zipkinAnns[i] = &tcollector.Annotation{
			Timestamp: (float64)(ann.Timestamp.UnixNano() / 1e6),
			Value:     (string)(ann.Key),
		}
	}
	return zipkinAnns
}

// inetAton converts string Ipv4 to uint32
func inetAton(ip string) uint32 {
	ipBytes := net.ParseIP(ip).To4()
	return binary.BigEndian.Uint32(ipBytes)
}

// base64Encode encodes uint64 with base64 StdEncoding.
func base64Encode(data uint64) string {
	return base64.StdEncoding.EncodeToString(uint64ToBytes(data))
}

// uint64ToBytes converts uint64 to bytes.
func uint64ToBytes(i uint64) []byte {
	buf := make([]byte, 8)
	binary.BigEndian.PutUint64(buf, uint64(i))
	return buf
}

// ZipkinTraceReporterFactory builds ZipkinTraceReporter by given TChannel instance.
func ZipkinTraceReporterFactory(tchannel *tc.Channel) tc.TraceReporter {
	return NewZipkinTraceReporter(tchannel)
}
<|endoftext|>


<commit_before>package gophermail

import (
	"bytes"
	"encoding/base64"
	"io"
)

var delimiter = []byte("\r\n") // CRLF

// Lines should be no longer than 78 characters excluding the CRLF.
// See RFC 2822.
const maxLength = 78

// splittingWriter is an io.WriteCloser that delimits
// the written data into fixed length chunks using
// a separator character sequence.
type splittingWriter struct {
	b       *bytes.Buffer
	w       io.Writer
	flushed bool
}

func (t *splittingWriter) Write(p []byte) (n int, err error) {

	// Write everything to our buffer.
	n, err = t.b.Write(p)

	// Check if our buffer can be flushed.
	for t.b.Len() >= maxLength {
		// If this isn't the first time flushing
		// the buffer then we need to write a delimiter.
		if t.flushed {
			_, err = t.w.Write(delimiter)
			if err != nil {
				return
			}
		}

		// Copy the next chunk out of the buffer.
		var n2 int64
		n2, err = io.CopyN(t.w, t.b, maxLength)
		if n2 > 0 {
			t.flushed = true
		}
	}

	return
}

func (t *splittingWriter) Close() (err error) {
	// Flush everything in the buffer.
	if t.b.Len() > 0 {
		// If this isn't the first time flushing
		// the buffer then we need to write a delimiter.
		if t.flushed {
			_, err = t.w.Write(delimiter)
			if err != nil {
				return
			}
		}

		var n int64
		n, err = io.Copy(t.w, t.b)
		if n > 0 {
			t.flushed = true
		}
	}

	return
}

type base64MimeEncoder struct {
	enc io.WriteCloser
	w   io.WriteCloser
}

func (t *base64MimeEncoder) Write(p []byte) (n int, err error) {
	n, err = t.enc.Write(p)
	return
}

func (t *base64MimeEncoder) Close() (err error) {
	err = t.enc.Close()
	if err != nil {
		return err
	}

	err = t.w.Close()
	return
}

func NewBase64MimeEncoder(w io.Writer) io.WriteCloser {
	splitter := &splittingWriter{
		w: w,
		b: &bytes.Buffer{},
	}
	t := &base64MimeEncoder{w: splitter}
	t.enc = base64.NewEncoder(base64.StdEncoding, splitter)
	return t
}
<commit_msg>Change max line length to 76<commit_after>package gophermail

import (
	"bytes"
	"encoding/base64"
	"io"
)

var delimiter = []byte("\r\n") // CRLF

// Lines should be no longer than 76 characters excluding the CRLF.
// See RFC 2045 and GitHub Issue #13.
const maxLength = 76

// splittingWriter is an io.WriteCloser that delimits
// the written data into fixed length chunks using
// a separator character sequence.
type splittingWriter struct {
	b       *bytes.Buffer
	w       io.Writer
	flushed bool
}

func (t *splittingWriter) Write(p []byte) (n int, err error) {

	// Write everything to our buffer.
	n, err = t.b.Write(p)

	// Check if our buffer can be flushed.
	for t.b.Len() >= maxLength {
		// If this isn't the first time flushing
		// the buffer then we need to write a delimiter.
		if t.flushed {
			_, err = t.w.Write(delimiter)
			if err != nil {
				return
			}
		}

		// Copy the next chunk out of the buffer.
		var n2 int64
		n2, err = io.CopyN(t.w, t.b, maxLength)
		if n2 > 0 {
			t.flushed = true
		}
	}

	return
}

func (t *splittingWriter) Close() (err error) {
	// Flush everything in the buffer.
	if t.b.Len() > 0 {
		// If this isn't the first time flushing
		// the buffer then we need to write a delimiter.
		if t.flushed {
			_, err = t.w.Write(delimiter)
			if err != nil {
				return
			}
		}

		var n int64
		n, err = io.Copy(t.w, t.b)
		if n > 0 {
			t.flushed = true
		}
	}

	return
}

type base64MimeEncoder struct {
	enc io.WriteCloser
	w   io.WriteCloser
}

func (t *base64MimeEncoder) Write(p []byte) (n int, err error) {
	n, err = t.enc.Write(p)
	return
}

func (t *base64MimeEncoder) Close() (err error) {
	err = t.enc.Close()
	if err != nil {
		return err
	}

	err = t.w.Close()
	return
}

func NewBase64MimeEncoder(w io.Writer) io.WriteCloser {
	splitter := &splittingWriter{
		w: w,
		b: &bytes.Buffer{},
	}
	t := &base64MimeEncoder{w: splitter}
	t.enc = base64.NewEncoder(base64.StdEncoding, splitter)
	return t
}
<|endoftext|>


<commit_before>package g

import (
	"encoding/json"
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/toolkits/file"
)

type HttpConfig struct {
	Enabled    bool   `json:"enabled"`
	Listen     string `json:"listen"`
	Cookie     string `json:"cookie"`
	ViewPath   string `json:"view_path"`
	StaticPath string `json:"static_path"`
}

type TimeoutConfig struct {
	Conn  int64 `json:"conn"`
	Read  int64 `json:"read"`
	Write int64 `json:"write"`
}

type CacheConfig struct {
	Enabled bool           `json:"enabled"`
	Redis   string         `json:"redis"`
	Idle    int            `json:"idle"`
	Max     int            `json:"max"`
	Timeout *TimeoutConfig `json:"timeout"`
}

type UicConfig struct {
	Addr string `json:"addr"`
	Idle int    `json:"idle"`
	Max  int    `json:"max"`
}

type GraphDBConfig struct {
	Addr           string `json:"addr"`
	Idle           int    `json:"idle"`
	Max            int    `json:"max"`
	Limit          int    `json:"limit"`
	LimitHostGroup int    `json:"limitHostGroup"`
}

type FalconPortalConfig struct {
	Addr  string `json:"addr"`
	Idle  int    `json:"idle"`
	Max   int    `json:"max"`
	Limit int    `json:"limit"`
}

type BossConfig struct {
	Addr string `json:"addr"`
	Idle int    `json:"idle"`
	Max  int    `json:"max"`
}

type ShortcutConfig struct {
	FalconPortal     string `json:"falconPortal"`
	FalconDashboard  string `json:"falconDashboard"`
	GrafanaDashboard string `json:"grafanaDashboard"`
	FalconAlarm      string `json:"falconAlarm"`
}

type LdapConfig struct {
	Enabled    bool     `json:"enabled"`
	Addr       string   `json:"addr"`
	BindDN     string   `json:"bindDN"`
	BaseDN     string   `json:"baseDN"`
	BindPasswd string   `json:"bindPasswd"`
	UserField  string   `json:"userField"`
	Attributes []string `json:"attributes"`
}

type ApiConfig struct {
	Key      string `json:"key"`
	Redirect string `json:"redirect"`
	Login    string `json:"login"`
	Access   string `json:"access"`
	Role     string `json:"role"`
	Logout   string `json:"logout"`
	Name     string `json:"name"`
	Token    string `json:"token"`
	Map      string `json:"map"`
	Contact  string `json:"contact"`
}

type GraphConfig struct {
	ConnTimeout int32             `json:"connTimeout"`
	CallTimeout int32             `json:"callTimeout"`
	MaxConns    int32             `json:"maxConns"`
	MaxIdle     int32             `json:"maxIdle"`
	Replicas    int32             `json:"replicas"`
	Cluster     map[string]string `json:"cluster"`
}

type GrpcConfig struct {
	Enabled bool `json:"enabled"`
	Port    int  `json:"port"`
}

type MqConfig struct {
	Enabled  bool   `json:"enabled"`
	Queue    string `json:"queue"`
	Consumer string `json:"consumer"`
}

type GlobalConfig struct {
	Log          string              `json:"log"`
	Company      string              `json:"company"`
	Cache        *CacheConfig        `json:"cache"`
	Http         *HttpConfig         `json:"http"`
	Salt         string              `json:"salt"`
	CanRegister  bool                `json:"canRegister"`
	Ldap         *LdapConfig         `json:"ldap"`
	Uic          *UicConfig          `json:"uic"`
	GraphDB      *GraphDBConfig      `json:"graphdb"`
	BossDB       *BossConfig         `json:"boss"`
	FalconPortal *FalconPortalConfig `json:"falcon_portal"`
	Shortcut     *ShortcutConfig     `json:"shortcut"`
	Api          *ApiConfig          `json:"api"`
	Graph        *GraphConfig        `json:"graph"`
	Grpc         *GrpcConfig         `json:"grpc"`
	Mq           *MqConfig           `json:"mq"`
}

var (
	ConfigFile string
	config     *GlobalConfig
	configLock = new(sync.RWMutex)
)

func Config() *GlobalConfig {
	configLock.RLock()
	defer configLock.RUnlock()
	return config
}

func ParseConfig(cfg string) error {
	if cfg == "" {
		return fmt.Errorf("use -c to specify configuration file")
	}

	if !file.IsExist(cfg) {
		return fmt.Errorf("config file %s is nonexistent", cfg)
	}

	ConfigFile = cfg

	configContent, err := file.ToTrimString(cfg)
	if err != nil {
		return fmt.Errorf("read config file %s fail %s", cfg, err)
	}

	var c GlobalConfig
	err = json.Unmarshal([]byte(configContent), &c)
	if err != nil {
		return fmt.Errorf("parse config file %s fail %s", cfg, err)
	}

	configLock.Lock()
	defer configLock.Unlock()

	config = &c

	log.Println("read config file:", cfg, "successfully")
	return nil
}
<commit_msg>[OWL-1164][fe] add Cookie in ApiConfig<commit_after>package g

import (
	"encoding/json"
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/toolkits/file"
)

type HttpConfig struct {
	Enabled    bool   `json:"enabled"`
	Listen     string `json:"listen"`
	Cookie     string `json:"cookie"`
	ViewPath   string `json:"view_path"`
	StaticPath string `json:"static_path"`
}

type TimeoutConfig struct {
	Conn  int64 `json:"conn"`
	Read  int64 `json:"read"`
	Write int64 `json:"write"`
}

type CacheConfig struct {
	Enabled bool           `json:"enabled"`
	Redis   string         `json:"redis"`
	Idle    int            `json:"idle"`
	Max     int            `json:"max"`
	Timeout *TimeoutConfig `json:"timeout"`
}

type UicConfig struct {
	Addr string `json:"addr"`
	Idle int    `json:"idle"`
	Max  int    `json:"max"`
}

type GraphDBConfig struct {
	Addr           string `json:"addr"`
	Idle           int    `json:"idle"`
	Max            int    `json:"max"`
	Limit          int    `json:"limit"`
	LimitHostGroup int    `json:"limitHostGroup"`
}

type FalconPortalConfig struct {
	Addr  string `json:"addr"`
	Idle  int    `json:"idle"`
	Max   int    `json:"max"`
	Limit int    `json:"limit"`
}

type BossConfig struct {
	Addr string `json:"addr"`
	Idle int    `json:"idle"`
	Max  int    `json:"max"`
}

type ShortcutConfig struct {
	FalconPortal     string `json:"falconPortal"`
	FalconDashboard  string `json:"falconDashboard"`
	GrafanaDashboard string `json:"grafanaDashboard"`
	FalconAlarm      string `json:"falconAlarm"`
}

type LdapConfig struct {
	Enabled    bool     `json:"enabled"`
	Addr       string   `json:"addr"`
	BindDN     string   `json:"bindDN"`
	BaseDN     string   `json:"baseDN"`
	BindPasswd string   `json:"bindPasswd"`
	UserField  string   `json:"userField"`
	Attributes []string `json:"attributes"`
}

type ApiConfig struct {
	Cookie   string `json:"cookie"`
	Key      string `json:"key"`
	Redirect string `json:"redirect"`
	Login    string `json:"login"`
	Access   string `json:"access"`
	Role     string `json:"role"`
	Logout   string `json:"logout"`
	Name     string `json:"name"`
	Token    string `json:"token"`
	Map      string `json:"map"`
	Contact  string `json:"contact"`
}

type GraphConfig struct {
	ConnTimeout int32             `json:"connTimeout"`
	CallTimeout int32             `json:"callTimeout"`
	MaxConns    int32             `json:"maxConns"`
	MaxIdle     int32             `json:"maxIdle"`
	Replicas    int32             `json:"replicas"`
	Cluster     map[string]string `json:"cluster"`
}

type GrpcConfig struct {
	Enabled bool `json:"enabled"`
	Port    int  `json:"port"`
}

type MqConfig struct {
	Enabled  bool   `json:"enabled"`
	Queue    string `json:"queue"`
	Consumer string `json:"consumer"`
}

type GlobalConfig struct {
	Log          string              `json:"log"`
	Company      string              `json:"company"`
	Cache        *CacheConfig        `json:"cache"`
	Http         *HttpConfig         `json:"http"`
	Salt         string              `json:"salt"`
	CanRegister  bool                `json:"canRegister"`
	Ldap         *LdapConfig         `json:"ldap"`
	Uic          *UicConfig          `json:"uic"`
	GraphDB      *GraphDBConfig      `json:"graphdb"`
	BossDB       *BossConfig         `json:"boss"`
	FalconPortal *FalconPortalConfig `json:"falcon_portal"`
	Shortcut     *ShortcutConfig     `json:"shortcut"`
	Api          *ApiConfig          `json:"api"`
	Graph        *GraphConfig        `json:"graph"`
	Grpc         *GrpcConfig         `json:"grpc"`
	Mq           *MqConfig           `json:"mq"`
}

var (
	ConfigFile string
	config     *GlobalConfig
	configLock = new(sync.RWMutex)
)

func Config() *GlobalConfig {
	configLock.RLock()
	defer configLock.RUnlock()
	return config
}

func ParseConfig(cfg string) error {
	if cfg == "" {
		return fmt.Errorf("use -c to specify configuration file")
	}

	if !file.IsExist(cfg) {
		return fmt.Errorf("config file %s is nonexistent", cfg)
	}

	ConfigFile = cfg

	configContent, err := file.ToTrimString(cfg)
	if err != nil {
		return fmt.Errorf("read config file %s fail %s", cfg, err)
	}

	var c GlobalConfig
	err = json.Unmarshal([]byte(configContent), &c)
	if err != nil {
		return fmt.Errorf("parse config file %s fail %s", cfg, err)
	}

	configLock.Lock()
	defer configLock.Unlock()

	config = &c

	log.Println("read config file:", cfg, "successfully")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vulns

import (
	"fmt"
	"io"
	"net/url"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/google/osv/vulnfeeds/cves"
)

type Event struct {
	Introduced string `json:"introduced,omitempty" yaml:"introduced,omitempty"`
	Fixed      string `json:"fixed,omitempty" yaml:"fixed,omitempty"`
	Limit      string `json:"limit,omitempty" yaml:"limit,omitempty"`
}

type Affected struct {
	Package struct {
		Name      string `json:"name" yaml:"name"`
		Ecosystem string `json:"ecosystem" yaml:"ecosystem"`
		Purl      string `json:"purl,omitempty" yaml:"purl,omitempty"`
	} `json:"package"`
	Ranges   []AffectedRange `json:"ranges" yaml:"ranges"`
	Versions []string        `json:"versions" yaml:"versions,omitempty"`
}

type AffectedRange struct {
	Type   string  `json:"type" yaml:"type"`
	Repo   string  `json:"repo,omitempty" yaml:"repo,omitempty"`
	Events []Event `json:"events", yaml:"events"`
}

type Reference struct {
	Type string `json:"type" yaml:"type"`
	URL  string `json:"url" yaml:"url"`
}

type Vulnerability struct {
	ID         string      `json:"id" yaml:"id"`
	Summary    string      `json:"summary,omitempty" yaml:"summary,omitempty"`
	Details    string      `json:"details" yaml:"details"`
	Affected   []Affected  `json:"affected" yaml:"affected"`
	References []Reference `json:"references" yaml:"references"`
	Aliases    []string    `json:"aliases,omitempty" yaml:"aliases,omitempty"`
	Modified   string      `json:"modified" yaml:"modified"`
	Published  string      `json:"published" yaml:"published"`
}

func timestampToRFC3339(timestamp string) (string, error) {
	t, err := cves.ParseTimestamp(timestamp)
	if err != nil {
		return "", err
	}

	return t.Format(time.RFC3339), nil
}

func classifyReferenceLink(link string) string {
	u, err := url.Parse(link)
	if err != nil {
		return "WEB"
	}

	pathParts := strings.Split(u.Path, "/")

	// Index 0 will always be "", so the length must be at least 2 to be relevant
	if len(pathParts) >= 2 {
		if u.Host == "github.com" {
			// Example: https://github.com/google/osv/commit/cd4e934d0527e5010e373e7fed54ef5daefba2f5
			if pathParts[len(pathParts)-2] == "commit" {
				return "FIX"
			}

			// Example: https://github.com/advisories/GHSA-fr26-qjc8-mvjx
			// Example: https://github.com/dpgaspar/Flask-AppBuilder/security/advisories/GHSA-624f-cqvr-3qw4
			if pathParts[1] == "advisories" || pathParts[len(pathParts)-2] == "advisories" {
				return "ADVISORY"
			}
		}

		if u.Host == "snyk.io" {
			//Example: https://snyk.io/vuln/SNYK-PYTHON-TRYTOND-1730329
			if pathParts[1] == "vuln" {
				return "ADVISORY"
			}
		}

		if u.Host == "nvd.nist.gov" {
			//Example: https://nvd.nist.gov/vuln/detail/CVE-2021-23336
			if len(pathParts) == 4 && pathParts[1] == "vuln" && pathParts[2] == "detail" {
				return "ADVISORY"
			}
		}

		if u.Host == "pypi.org" {
			//Example: "https://pypi.org/project/flask"
			if pathParts[1] == "project" {
				return "PACKAGE"
			}
		}
	}

	if strings.Contains(link, "advisory") || strings.Contains(link, "advisories") {
		return "ADVISORY"
	}

	if strings.Contains(link, "blog") {
		return "ARTICLE"
	}

	return "WEB"
}

func extractAliases(id string, cve cves.CVE) []string {
	var aliases []string
	if id != cve.CVEDataMeta.ID {
		aliases = append(aliases, cve.CVEDataMeta.ID)
	}

	for _, reference := range cve.References.ReferenceData {
		u, err := url.Parse(reference.URL)
		if err == nil {
			pathParts := strings.Split(u.Path, "/")

			// Index 0 will always be "", so the length must be at least 3 here to be relevant
			if len(pathParts) >= 3 {
				if u.Host == "github.com" {
					// Example: https://github.com/advisories/GHSA-fr26-qjc8-mvjx
					// Example: https://github.com/dpgaspar/Flask-AppBuilder/security/advisories/GHSA-624f-cqvr-3qw4
					if pathParts[len(pathParts)-2] == "advisories" {
						a := pathParts[len(pathParts)-1]

						if id != a && strings.HasPrefix(a, "GHSA-") {
							aliases = append(aliases, a)
						}
					}
				}

				if u.Host == "snyk.io" {
					//Example: https://snyk.io/vuln/SNYK-PYTHON-TRYTOND-1730329
					if pathParts[1] == "vuln" {
						a := pathParts[len(pathParts)-1]

						if id != a && strings.HasPrefix(a, "SNYK-") {
							aliases = append(aliases, a)
						}
					}
				}
			}
		}
	}

	return aliases
}

func FromCVE(id string, cve cves.CVEItem, pkg, ecosystem, purl, versionType string, validVersions []string) (*Vulnerability, []string) {
	v := Vulnerability{
		ID:      id,
		Details: cves.EnglishDescription(cve.CVE),
		Aliases: extractAliases(id, cve.CVE),
	}

	affected := Affected{}
	affected.Package.Name = pkg
	affected.Package.Ecosystem = ecosystem
	affected.Package.Purl = purl

	var err error
	var notes []string
	v.Published, err = timestampToRFC3339(cve.PublishedDate)
	if err != nil {
		notes = append(notes, fmt.Sprintf("Failed to parse published date: %v\n", err))
	}

	v.Modified, err = timestampToRFC3339(cve.LastModifiedDate)
	if err != nil {
		notes = append(notes, fmt.Sprintf("Failed to parse modified date: %v\n", err))
	}

	for _, reference := range cve.CVE.References.ReferenceData {
		v.References = append(v.References, Reference{
			Type: classifyReferenceLink(reference.URL),
			URL:  reference.URL,
		})
	}

	// Extract version information where we can.
	version, versionNotes := cves.ExtractVersionInfo(cve, validVersions)
	notes = append(notes, versionNotes...)

	repoToCommits := map[string][]string{}
	for _, fixCommit := range version.FixCommits {
		repoToCommits[fixCommit.Repo] = append(repoToCommits[fixCommit.Repo], fixCommit.Commit)
	}

	for repo, commits := range repoToCommits {
		gitRange := AffectedRange{
			Type: "GIT",
			Repo: repo,
		}
		gitRange.Events = append(gitRange.Events, Event{Introduced: "0"})
		for _, commit := range commits {
			gitRange.Events = append(gitRange.Events, Event{Fixed: commit})
		}
		affected.Ranges = append(affected.Ranges, gitRange)
	}

	versionRange := AffectedRange{
		Type: "ECOSYSTEM",
	}
	seenIntroduced := map[string]bool{}
	seenFixed := map[string]bool{}

	for _, v := range version.AffectedVersions {
		var introduced string
		if v.Introduced == "" {
			introduced = "0"
		} else {
			introduced = v.Introduced
		}

		if _, seen := seenIntroduced[introduced]; !seen {
			versionRange.Events = append(versionRange.Events, Event{
				Introduced: introduced,
			})
			seenIntroduced[introduced] = true
		}

		if _, seen := seenFixed[v.Fixed]; v.Fixed != "" && !seen {
			versionRange.Events = append(versionRange.Events, Event{
				Fixed: v.Fixed,
			})
			seenFixed[v.Fixed] = true
		}
	}
	affected.Ranges = append(affected.Ranges, versionRange)
	v.Affected = append(v.Affected, affected)
	return &v, notes
}

func FromYAML(r io.Reader) (*Vulnerability, error) {
	decoder := yaml.NewDecoder(r)
	var vuln Vulnerability
	err := decoder.Decode(&vuln)
	if err != nil {
		return nil, err
	}

	return &vuln, nil
}

func (v *Vulnerability) ToYAML(w io.Writer) error {
	encoder := yaml.NewEncoder(w)
	return encoder.Encode(v)
}
<commit_msg>vulnfeeds: classify GitHub issues ref links as type REPORT (#250)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vulns

import (
	"fmt"
	"io"
	"net/url"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/google/osv/vulnfeeds/cves"
)

type Event struct {
	Introduced string `json:"introduced,omitempty" yaml:"introduced,omitempty"`
	Fixed      string `json:"fixed,omitempty" yaml:"fixed,omitempty"`
	Limit      string `json:"limit,omitempty" yaml:"limit,omitempty"`
}

type Affected struct {
	Package struct {
		Name      string `json:"name" yaml:"name"`
		Ecosystem string `json:"ecosystem" yaml:"ecosystem"`
		Purl      string `json:"purl,omitempty" yaml:"purl,omitempty"`
	} `json:"package"`
	Ranges   []AffectedRange `json:"ranges" yaml:"ranges"`
	Versions []string        `json:"versions" yaml:"versions,omitempty"`
}

type AffectedRange struct {
	Type   string  `json:"type" yaml:"type"`
	Repo   string  `json:"repo,omitempty" yaml:"repo,omitempty"`
	Events []Event `json:"events", yaml:"events"`
}

type Reference struct {
	Type string `json:"type" yaml:"type"`
	URL  string `json:"url" yaml:"url"`
}

type Vulnerability struct {
	ID         string      `json:"id" yaml:"id"`
	Summary    string      `json:"summary,omitempty" yaml:"summary,omitempty"`
	Details    string      `json:"details" yaml:"details"`
	Affected   []Affected  `json:"affected" yaml:"affected"`
	References []Reference `json:"references" yaml:"references"`
	Aliases    []string    `json:"aliases,omitempty" yaml:"aliases,omitempty"`
	Modified   string      `json:"modified" yaml:"modified"`
	Published  string      `json:"published" yaml:"published"`
}

func timestampToRFC3339(timestamp string) (string, error) {
	t, err := cves.ParseTimestamp(timestamp)
	if err != nil {
		return "", err
	}

	return t.Format(time.RFC3339), nil
}

func classifyReferenceLink(link string) string {
	u, err := url.Parse(link)
	if err != nil {
		return "WEB"
	}

	pathParts := strings.Split(u.Path, "/")

	// Index 0 will always be "", so the length must be at least 2 to be relevant
	if len(pathParts) >= 2 {
		if u.Host == "github.com" {
			// Example: https://github.com/google/osv/commit/cd4e934d0527e5010e373e7fed54ef5daefba2f5
			if pathParts[len(pathParts)-2] == "commit" {
				return "FIX"
			}

			// Example: https://github.com/advisories/GHSA-fr26-qjc8-mvjx
			// Example: https://github.com/dpgaspar/Flask-AppBuilder/security/advisories/GHSA-624f-cqvr-3qw4
			if pathParts[len(pathParts)-2] == "advisories" {
				return "ADVISORY"
			}

			// Example: https://github.com/Netflix/lemur/issues/117
			if pathParts[len(pathParts)-2] == "issues" {
				return "REPORT"
			}
		}

		if u.Host == "snyk.io" {
			//Example: https://snyk.io/vuln/SNYK-PYTHON-TRYTOND-1730329
			if pathParts[1] == "vuln" {
				return "ADVISORY"
			}
		}

		if u.Host == "nvd.nist.gov" {
			//Example: https://nvd.nist.gov/vuln/detail/CVE-2021-23336
			if len(pathParts) == 4 && pathParts[1] == "vuln" && pathParts[2] == "detail" {
				return "ADVISORY"
			}
		}

		if u.Host == "pypi.org" {
			//Example: "https://pypi.org/project/flask"
			if pathParts[1] == "project" {
				return "PACKAGE"
			}
		}
	}

	if strings.Contains(link, "advisory") || strings.Contains(link, "advisories") {
		return "ADVISORY"
	}

	if strings.Contains(link, "blog") {
		return "ARTICLE"
	}

	return "WEB"
}

func extractAliases(id string, cve cves.CVE) []string {
	var aliases []string
	if id != cve.CVEDataMeta.ID {
		aliases = append(aliases, cve.CVEDataMeta.ID)
	}

	for _, reference := range cve.References.ReferenceData {
		u, err := url.Parse(reference.URL)
		if err == nil {
			pathParts := strings.Split(u.Path, "/")

			// Index 0 will always be "", so the length must be at least 3 here to be relevant
			if len(pathParts) >= 3 {
				if u.Host == "github.com" {
					// Example: https://github.com/advisories/GHSA-fr26-qjc8-mvjx
					// Example: https://github.com/dpgaspar/Flask-AppBuilder/security/advisories/GHSA-624f-cqvr-3qw4
					if pathParts[len(pathParts)-2] == "advisories" {
						a := pathParts[len(pathParts)-1]

						if id != a && strings.HasPrefix(a, "GHSA-") {
							aliases = append(aliases, a)
						}
					}
				}

				if u.Host == "snyk.io" {
					//Example: https://snyk.io/vuln/SNYK-PYTHON-TRYTOND-1730329
					if pathParts[1] == "vuln" {
						a := pathParts[len(pathParts)-1]

						if id != a && strings.HasPrefix(a, "SNYK-") {
							aliases = append(aliases, a)
						}
					}
				}
			}
		}
	}

	return aliases
}

func FromCVE(id string, cve cves.CVEItem, pkg, ecosystem, purl, versionType string, validVersions []string) (*Vulnerability, []string) {
	v := Vulnerability{
		ID:      id,
		Details: cves.EnglishDescription(cve.CVE),
		Aliases: extractAliases(id, cve.CVE),
	}

	affected := Affected{}
	affected.Package.Name = pkg
	affected.Package.Ecosystem = ecosystem
	affected.Package.Purl = purl

	var err error
	var notes []string
	v.Published, err = timestampToRFC3339(cve.PublishedDate)
	if err != nil {
		notes = append(notes, fmt.Sprintf("Failed to parse published date: %v\n", err))
	}

	v.Modified, err = timestampToRFC3339(cve.LastModifiedDate)
	if err != nil {
		notes = append(notes, fmt.Sprintf("Failed to parse modified date: %v\n", err))
	}

	for _, reference := range cve.CVE.References.ReferenceData {
		v.References = append(v.References, Reference{
			Type: classifyReferenceLink(reference.URL),
			URL:  reference.URL,
		})
	}

	// Extract version information where we can.
	version, versionNotes := cves.ExtractVersionInfo(cve, validVersions)
	notes = append(notes, versionNotes...)

	repoToCommits := map[string][]string{}
	for _, fixCommit := range version.FixCommits {
		repoToCommits[fixCommit.Repo] = append(repoToCommits[fixCommit.Repo], fixCommit.Commit)
	}

	for repo, commits := range repoToCommits {
		gitRange := AffectedRange{
			Type: "GIT",
			Repo: repo,
		}
		gitRange.Events = append(gitRange.Events, Event{Introduced: "0"})
		for _, commit := range commits {
			gitRange.Events = append(gitRange.Events, Event{Fixed: commit})
		}
		affected.Ranges = append(affected.Ranges, gitRange)
	}

	versionRange := AffectedRange{
		Type: "ECOSYSTEM",
	}
	seenIntroduced := map[string]bool{}
	seenFixed := map[string]bool{}

	for _, v := range version.AffectedVersions {
		var introduced string
		if v.Introduced == "" {
			introduced = "0"
		} else {
			introduced = v.Introduced
		}

		if _, seen := seenIntroduced[introduced]; !seen {
			versionRange.Events = append(versionRange.Events, Event{
				Introduced: introduced,
			})
			seenIntroduced[introduced] = true
		}

		if _, seen := seenFixed[v.Fixed]; v.Fixed != "" && !seen {
			versionRange.Events = append(versionRange.Events, Event{
				Fixed: v.Fixed,
			})
			seenFixed[v.Fixed] = true
		}
	}
	affected.Ranges = append(affected.Ranges, versionRange)
	v.Affected = append(v.Affected, affected)
	return &v, notes
}

func FromYAML(r io.Reader) (*Vulnerability, error) {
	decoder := yaml.NewDecoder(r)
	var vuln Vulnerability
	err := decoder.Decode(&vuln)
	if err != nil {
		return nil, err
	}

	return &vuln, nil
}

func (v *Vulnerability) ToYAML(w io.Writer) error {
	encoder := yaml.NewEncoder(w)
	return encoder.Encode(v)
}
<|endoftext|>


<commit_before>// Copyright 2013 Artur Grabowski. All rights reserved.
// Use of this source code is governed by a ISC-style
// license that can be found in the LICENSE file.
package bconf

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
)

// Split function for the scanner. This is what removes comments, whitespace and
// empty lines and returns just the valid bconf lines.
func scanLines(data []byte, atEOF bool) (int, []byte, error) {
	totadvance := 0
	for true {
		advance, token, err := bufio.ScanLines(data, atEOF)
		if advance == 0 || err != nil {
			return advance, token, err
		}
		totadvance += advance
		// Eat leading whitespace
		i := 0
		for ; i < len(token); i++ {
			if token[i] != ' ' && token[i] != '\t' {
				break
			}
		}
		token = token[i:]

		// Empty lines and comments.
		if len(token) == 0 || token[0] == '#' {
			data = data[advance:]
			continue
		}
		return totadvance, token, nil
	}
	return -1, nil, nil
}

// Populate a Bconf with data from the bconf-formatted file provided here as a name.
func (bc *Bconf) LoadConfFile(fname string) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	return bc.LoadConfData(f)
}

// Populate a Bconf with data from the bconf-formatted io.Reader.
func (bc *Bconf) LoadConfData(f io.Reader) error {
	scanner := bufio.NewScanner(f)
	scanner.Split(scanLines)
	for scanner.Scan() {
		t := scanner.Text()
		if strings.HasPrefix(t, "include") {
			incpath := strings.Split(t, " \t")
			if len(incpath) < 2 {
				return errors.New(fmt.Sprintf("include without path: %v", t))
			}
			bc.LoadConfFile(incpath[len(incpath)-1])
		}
		skv := strings.SplitN(t, "=", 2)
		if len(skv) != 2 {
			return errors.New(fmt.Sprintf("malformed bconf line: %v", t))
		}
		k, v := skv[0], skv[1]
		bc.AddValue(strings.Split(k, "."), v)

	}
	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix parsing of the include path.<commit_after>// Copyright 2013 Artur Grabowski. All rights reserved.
// Use of this source code is governed by a ISC-style
// license that can be found in the LICENSE file.
package bconf

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"unicode"
)

// Split function for the scanner. This is what removes comments, whitespace and
// empty lines and returns just the valid bconf lines.
func scanLines(data []byte, atEOF bool) (int, []byte, error) {
	totadvance := 0
	for true {
		advance, token, err := bufio.ScanLines(data, atEOF)
		if advance == 0 || err != nil {
			return advance, token, err
		}
		totadvance += advance
		// Eat leading whitespace
		i := 0
		for ; i < len(token); i++ {
			if token[i] != ' ' && token[i] != '\t' {
				break
			}
		}
		token = token[i:]

		// Empty lines and comments.
		if len(token) == 0 || token[0] == '#' {
			data = data[advance:]
			continue
		}
		return totadvance, token, nil
	}
	return -1, nil, nil
}

// Populate a Bconf with data from the bconf-formatted file provided here as a name.
func (bc *Bconf) LoadConfFile(fname string) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	return bc.LoadConfData(f)
}

// Populate a Bconf with data from the bconf-formatted io.Reader.
func (bc *Bconf) LoadConfData(f io.Reader) error {
	scanner := bufio.NewScanner(f)
	scanner.Split(scanLines)
	for scanner.Scan() {
		t := scanner.Text()
		if strings.HasPrefix(t, "include") {
			incpath := strings.TrimLeftFunc(t[7:], unicode.IsSpace)
			if incpath == "" {
				return errors.New(fmt.Sprintf("include without path: '%v'", t))
			}
			if err := bc.LoadConfFile(incpath); err != nil {
				return err
			}
		} else {
			skv := strings.SplitN(t, "=", 2)
			if len(skv) != 2 {
				return errors.New(fmt.Sprintf("malformed bconf line: %v", t))
			}
			k, v := skv[0], skv[1]
			bc.AddValue(strings.Split(k, "."), v)
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package etcd

import (
	"context"
	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"

	exutil "github.com/openshift/origin/test/extended/util"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
)

var _ = g.Describe("[sig-etcd] etcd", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLIWithoutNamespace("etcd-invariants").AsAdmin()

	g.It("cluster has the same number of master nodes and voting members from the endpoints configmap [Early]", func() {
		masterNodeLabelSelectorString := "node-role.kubernetes.io/master"
		masterNodeList, err := oc.KubeClient().CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: masterNodeLabelSelectorString})
		o.Expect(err).ToNot(o.HaveOccurred())
		currentMasterNodesIPListSet := sets.NewString()
		for _, masterNode := range masterNodeList.Items {
			for _, masterNodeAddress := range masterNode.Status.Addresses {
				if masterNodeAddress.Type == corev1.NodeInternalIP {
					currentMasterNodesIPListSet.Insert(masterNodeAddress.Address)
				}
			}
		}

		etcdEndpointsConfigMap, err := oc.KubeClient().CoreV1().ConfigMaps("openshift-etcd").Get(context.TODO(), "etcd-endpoints", metav1.GetOptions{})
		o.Expect(err).ToNot(o.HaveOccurred())
		currentVotingMemberIPListSet := sets.NewString()
		for _, votingMemberIP := range etcdEndpointsConfigMap.Data {
			currentVotingMemberIPListSet.Insert(votingMemberIP)
		}

		if currentVotingMemberIPListSet.Len() != currentMasterNodesIPListSet.Len() {
			g.GinkgoT().Fatalf(
				"incorrect number of voting members found in openshift-etcd/etcd-endpoints, expected it to match the number of master nodes = %d, members from the cm = %v, master nodes = %v",
				currentMasterNodesIPListSet.Len(),
				currentVotingMemberIPListSet.List(),
				currentMasterNodesIPListSet.List())
		}

		if !currentVotingMemberIPListSet.Equal(currentMasterNodesIPListSet) {
			g.GinkgoT().Fatalf("IPs of voting members from openshift-etcd/etcd-endpoints =%v don't match master nodes IPs = %v ", currentVotingMemberIPListSet.List(), currentMasterNodesIPListSet.List())
		}
	})
})
<commit_msg>ExternalControlPlaneTopology: Skip etcd master endpoint verification<commit_after>package etcd

import (
	"context"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"

	exutil "github.com/openshift/origin/test/extended/util"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
)

var _ = g.Describe("[sig-etcd] etcd", func() {
	defer g.GinkgoRecover()
	oc := exutil.NewCLIWithoutNamespace("etcd-invariants").AsAdmin()

	g.It("cluster has the same number of master nodes and voting members from the endpoints configmap [Early]", func() {
		exutil.SkipIfExternalControlplaneTopology(oc, "clusters with external controlplane topology don't have master nodes")
		masterNodeLabelSelectorString := "node-role.kubernetes.io/master"
		masterNodeList, err := oc.KubeClient().CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: masterNodeLabelSelectorString})
		o.Expect(err).ToNot(o.HaveOccurred())
		currentMasterNodesIPListSet := sets.NewString()
		for _, masterNode := range masterNodeList.Items {
			for _, masterNodeAddress := range masterNode.Status.Addresses {
				if masterNodeAddress.Type == corev1.NodeInternalIP {
					currentMasterNodesIPListSet.Insert(masterNodeAddress.Address)
				}
			}
		}

		etcdEndpointsConfigMap, err := oc.KubeClient().CoreV1().ConfigMaps("openshift-etcd").Get(context.TODO(), "etcd-endpoints", metav1.GetOptions{})
		o.Expect(err).ToNot(o.HaveOccurred())
		currentVotingMemberIPListSet := sets.NewString()
		for _, votingMemberIP := range etcdEndpointsConfigMap.Data {
			currentVotingMemberIPListSet.Insert(votingMemberIP)
		}

		if currentVotingMemberIPListSet.Len() != currentMasterNodesIPListSet.Len() {
			g.GinkgoT().Fatalf(
				"incorrect number of voting members found in openshift-etcd/etcd-endpoints, expected it to match the number of master nodes = %d, members from the cm = %v, master nodes = %v",
				currentMasterNodesIPListSet.Len(),
				currentVotingMemberIPListSet.List(),
				currentMasterNodesIPListSet.List())
		}

		if !currentVotingMemberIPListSet.Equal(currentMasterNodesIPListSet) {
			g.GinkgoT().Fatalf("IPs of voting members from openshift-etcd/etcd-endpoints =%v don't match master nodes IPs = %v ", currentVotingMemberIPListSet.List(), currentMasterNodesIPListSet.List())
		}
	})
})
<|endoftext|>


<commit_before>package main

import (
		"os"
		"fmt"
		"io/ioutil"
		"encoding/xml"
		"encoding/json"
		"github.com/aws/aws-sdk-go/aws"
		"github.com/aws/aws-sdk-go/aws/session"
		"github.com/aws/aws-sdk-go/service/dynamodb"
)

func main() {
// new comment
	type Dependency struct {
		GroupId string `xml:"groupId"`
		ArtifactId string `xml:"artifactId"`
		Version string `xml:"version"`
	}
	type ProjectXml struct {
		Dependencies []Dependency `xml:"dependencies>dependency"`
	}

	type ProjectJson struct {
		Dependencies interface{} `json:"dependencies"`
	}

	// collect all dependencies
	var allDepend []Dependency

	xmlFile, err := os.Open("pom.xml")
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}

	defer xmlFile.Close()

	b, _ := ioutil.ReadAll(xmlFile)

	var pom ProjectXml
	xml.Unmarshal(b, &pom)

	for _, depend := range pom.Dependencies {
		allDepend = append(allDepend, depend)
		// fmt.Println(depend)
	}

	jsonFile, err := os.Open("package.json")
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}

	defer jsonFile.Close()

	c, _ := ioutil.ReadAll(jsonFile)

	var pack ProjectJson

	json.Unmarshal(c, &pack)

	m := pack.Dependencies.(map[string]interface{})

	for k,v := range m {
		switch vv := v.(type) {
		case string:
			allDepend = append(allDepend, Dependency{"no group",k, vv})
			// fmt.Println(k, vv)
    	default:
    	}
	}

	// Load everything into dynamodb
	svc := dynamodb.New(session.New(&aws.Config{Region: aws.String("us-east-1")}))

	for _,value := range allDepend {
		fmt.Println(value.GroupId, value.ArtifactId, value.Version)

		params := &dynamodb.PutItemInput {
			Item: map[string]*dynamodb.AttributeValue {
				"packageID": {S: aws.String(value.ArtifactId)},
				"version": {S: aws.String(value.Version) },
			},
			TableName: aws.String("owjjh-test"),
		}

		_, err := svc.PutItem(params)

		if err != nil {
			fmt.Println(err.Error())
			return
		}

		// fmt.Println(resp)
	}

}
<commit_msg>Revert "Test update"<commit_after>package main

import (
		"os"
		"fmt"
		"io/ioutil"
		"encoding/xml"
		"encoding/json"
		"github.com/aws/aws-sdk-go/aws"
		"github.com/aws/aws-sdk-go/aws/session"
		"github.com/aws/aws-sdk-go/service/dynamodb"
)

func main() {
	type Dependency struct {
		GroupId string `xml:"groupId"`
		ArtifactId string `xml:"artifactId"`
		Version string `xml:"version"`
	}
	type ProjectXml struct {
		Dependencies []Dependency `xml:"dependencies>dependency"`
	}

	type ProjectJson struct {
		Dependencies interface{} `json:"dependencies"`
	}

	// collect all dependencies
	var allDepend []Dependency

	xmlFile, err := os.Open("pom.xml")
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}

	defer xmlFile.Close()

	b, _ := ioutil.ReadAll(xmlFile)

	var pom ProjectXml
	xml.Unmarshal(b, &pom)

	for _, depend := range pom.Dependencies {
		allDepend = append(allDepend, depend)
		// fmt.Println(depend)
	}

	jsonFile, err := os.Open("package.json")
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}

	defer jsonFile.Close()

	c, _ := ioutil.ReadAll(jsonFile)

	var pack ProjectJson

	json.Unmarshal(c, &pack)

	m := pack.Dependencies.(map[string]interface{})

	for k,v := range m {
		switch vv := v.(type) {
		case string:
			allDepend = append(allDepend, Dependency{"no group",k, vv})
			// fmt.Println(k, vv)
    	default:
    	}
	}

	// Load everything into dynamodb
	svc := dynamodb.New(session.New(&aws.Config{Region: aws.String("us-east-1")}))

	for _,value := range allDepend {
		fmt.Println(value.GroupId, value.ArtifactId, value.Version)

		params := &dynamodb.PutItemInput {
			Item: map[string]*dynamodb.AttributeValue {
				"packageID": {S: aws.String(value.ArtifactId)},
				"version": {S: aws.String(value.Version) },
			},
			TableName: aws.String("owjjh-test"),
		}

		_, err := svc.PutItem(params)

		if err != nil {
			fmt.Println(err.Error())
			return
		}

		// fmt.Println(resp)
	}

}
<|endoftext|>