<commit_before>package main

import (
	"fmt"
	"io"
	"strings"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/uploader"
)

func push(buildPath string, spec string) {
	must(doPush(buildPath, spec))
}

func doPush(buildPath string, spec string) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan *tlc.Container)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs)

	target, channel, err := parseSpec(spec)
	if err != nil {
		return err
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return err
	}

	newBuildRes, err := client.CreateBuild(target, channel)
	if err != nil {
		return err
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", channel)
		targetSignature = make([]sync.BlockHash, 0)
		targetContainer = &tlc.Container{}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return err
		}

		var signatureFileID int64
		for _, f := range buildFiles.Files {
			if f.Type == itchio.BuildFileType_SIGNATURE {
				signatureFileID = f.ID
				break
			}
		}

		if signatureFileID == 0 {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFileID)
		if err != nil {
			return err
		}

		targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return err
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewMultipartUpload(newPatchRes.File.UploadURL,
		newPatchRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	signatureWriter, err := uploader.NewMultipartUpload(newSignatureRes.File.UploadURL,
		newSignatureRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container

	comm.Debugf("Waiting for source container")
	select {
	case err := <-walkErrs:
		return err
	case sourceContainer = <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		break
	}

	comm.Logf("")
	comm.Opf("Pushing %s (%s)", humanize.Bytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes) * 0.97)

		// input bytes that aren't in output, for esxample:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > 10*1024 {
			comm.ProgressLabel(fmt.Sprintf("%s left", humanize.Bytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)
	}
	patchWriter.OnProgress = updateProgress

	stateConsumer := &pwr.StateConsumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		SourcePath:      buildPath,

		TargetContainer: targetContainer,
		TargetSignature: targetSignature,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return err
	}

	// close in a goroutine to avoid deadlocking
	doClose := func(c io.Closer, done chan bool, errs chan error) {
		err := c.Close()
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}

	go doClose(patchWriter, uploadDone, uploadErrs)
	go doClose(signatureWriter, uploadDone, uploadErrs)

	for c := 0; c < 4; c++ {
		select {
		case err := <-uploadErrs:
			return err
		case <-uploadDone:
		}
	}
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return err
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.Bytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))

		comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
	}
	return nil
}

type fileSlot struct {
	Type     itchio.BuildFileType
	Response itchio.NewBuildFileResponse
}

func createBothFiles(client *itchio.Client, buildID int64) (patch itchio.NewBuildFileResponse, signature itchio.NewBuildFileResponse, err error) {
	createFile := func(buildType itchio.BuildFileType, done chan fileSlot, errs chan error) {
		var res itchio.NewBuildFileResponse
		res, err = client.CreateBuildFile(buildID, buildType)
		if err != nil {
			errs <- err
		}
		comm.Debugf("Created %s build file: %+v", buildType, res.File)
		done <- fileSlot{buildType, res}
	}

	done := make(chan fileSlot)
	errs := make(chan error)

	go createFile(itchio.BuildFileType_PATCH, done, errs)
	go createFile(itchio.BuildFileType_SIGNATURE, done, errs)

	for i := 0; i < 2; i++ {
		select {
		case err = <-errs:
			return
		case slot := <-done:
			switch slot.Type {
			case itchio.BuildFileType_PATCH:
				patch = slot.Response
			case itchio.BuildFileType_SIGNATURE:
				signature = slot.Response
			}
		}
	}

	return
}

func doWalk(path string, result chan *tlc.Container, errs chan error) {
	container, err := tlc.Walk(path, filterPaths)
	if err != nil {
		errs <- err
	}

	result <- container
}

func parseSpec(spec string) (string, string, error) {
	tokens := strings.Split(spec, ":")

	if len(tokens) == 1 {
		return "", "", fmt.Errorf("invalid spec: %s, missing channel (examples: %s:windows-32-beta, %s:linux-64)", spec, spec, spec)
	} else if len(tokens) != 2 {
		return "", "", fmt.Errorf("invalid spec: %s, expected something of the form user/page:channel", spec)
	}

	return tokens[0], tokens[1], nil
}

func min(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Don't print negative savings, or 0%-reused<commit_after>package main

import (
	"fmt"
	"io"
	"math"
	"strings"

	"github.com/dustin/go-humanize"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/sync"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/uploader"
)

func push(buildPath string, spec string) {
	must(doPush(buildPath, spec))
}

func doPush(buildPath string, spec string) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan *tlc.Container)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs)

	target, channel, err := parseSpec(spec)
	if err != nil {
		return err
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return err
	}

	newBuildRes, err := client.CreateBuild(target, channel)
	if err != nil {
		return err
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature []sync.BlockHash
	var targetContainer *tlc.Container

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", channel)
		targetSignature = make([]sync.BlockHash, 0)
		targetContainer = &tlc.Container{}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return err
		}

		var signatureFileID int64
		for _, f := range buildFiles.Files {
			if f.Type == itchio.BuildFileType_SIGNATURE {
				signatureFileID = f.ID
				break
			}
		}

		if signatureFileID == 0 {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFileID)
		if err != nil {
			return err
		}

		targetContainer, targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return err
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewMultipartUpload(newPatchRes.File.UploadURL,
		newPatchRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	signatureWriter, err := uploader.NewMultipartUpload(newSignatureRes.File.UploadURL,
		newSignatureRes.File.UploadParams, fmt.Sprintf("%d-%d.pwr", parentID, buildID),
		uploadDone, uploadErrs, comm.NewStateConsumer())
	if err != nil {
		return err
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container

	comm.Debugf("Waiting for source container")
	select {
	case err := <-walkErrs:
		return err
	case sourceContainer = <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		break
	}

	comm.Logf("")
	comm.Opf("Pushing %s (%s)", humanize.Bytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes) * 0.97)

		// input bytes that aren't in output, for esxample:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > 10*1024 {
			comm.ProgressLabel(fmt.Sprintf("%s left", humanize.Bytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)
	}
	patchWriter.OnProgress = updateProgress

	stateConsumer := &pwr.StateConsumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		SourcePath:      buildPath,

		TargetContainer: targetContainer,
		TargetSignature: targetSignature,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return err
	}

	// close in a goroutine to avoid deadlocking
	doClose := func(c io.Closer, done chan bool, errs chan error) {
		err := c.Close()
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}

	go doClose(patchWriter, uploadDone, uploadErrs)
	go doClose(signatureWriter, uploadDone, uploadErrs)

	for c := 0; c < 4; c++ {
		select {
		case err := <-uploadErrs:
			return err
		case <-uploadDone:
		}
	}
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- err
			return
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return err
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.Bytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.Bytes(uint64(dctx.FreshBytes))
		savings := 100.0 - relToNew

		if dctx.ReusedBytes > 0 {
			comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		} else {
			comm.Statf("Added %s fresh data", percReused, prettyFreshSize)
		}

		if savings > 0 && !math.IsNaN(savings) {
			comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
		} else {
			comm.Statf("%s patch (no savings)", prettyPatchSize)
		}
	}
	return nil
}

type fileSlot struct {
	Type     itchio.BuildFileType
	Response itchio.NewBuildFileResponse
}

func createBothFiles(client *itchio.Client, buildID int64) (patch itchio.NewBuildFileResponse, signature itchio.NewBuildFileResponse, err error) {
	createFile := func(buildType itchio.BuildFileType, done chan fileSlot, errs chan error) {
		var res itchio.NewBuildFileResponse
		res, err = client.CreateBuildFile(buildID, buildType)
		if err != nil {
			errs <- err
		}
		comm.Debugf("Created %s build file: %+v", buildType, res.File)
		done <- fileSlot{buildType, res}
	}

	done := make(chan fileSlot)
	errs := make(chan error)

	go createFile(itchio.BuildFileType_PATCH, done, errs)
	go createFile(itchio.BuildFileType_SIGNATURE, done, errs)

	for i := 0; i < 2; i++ {
		select {
		case err = <-errs:
			return
		case slot := <-done:
			switch slot.Type {
			case itchio.BuildFileType_PATCH:
				patch = slot.Response
			case itchio.BuildFileType_SIGNATURE:
				signature = slot.Response
			}
		}
	}

	return
}

func doWalk(path string, result chan *tlc.Container, errs chan error) {
	container, err := tlc.Walk(path, filterPaths)
	if err != nil {
		errs <- err
	}

	result <- container
}

func parseSpec(spec string) (string, string, error) {
	tokens := strings.Split(spec, ":")

	if len(tokens) == 1 {
		return "", "", fmt.Errorf("invalid spec: %s, missing channel (examples: %s:windows-32-beta, %s:linux-64)", spec, spec, spec)
	} else if len(tokens) != 2 {
		return "", "", fmt.Errorf("invalid spec: %s, expected something of the form user/page:channel", spec)
	}

	return tokens[0], tokens[1], nil
}

func min(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>// ratt operates on a Debian .changes file of a just-built package, identifies
// all reverse-build-dependencies and rebuilds them with the .debs from the
// .changes file.
//
// The intended use-case is, for example, to package a new snapshot of a Go
// library and verify that the new version does not break any other Go
// libraries/binaries.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/version"
)

var (
	logDir = flag.String("log_dir",
		"buildlogs",
		"Directory in which sbuild(1) logs for all reverse-build-dependencies are stored")

	dryRun = flag.Bool("dry_run",
		false,
		"Print sbuild command lines, but do not actually build the reverse-build-dependencies")

	sbuildDist = flag.String("sbuild_dist",
		"",
		"sbuild --dist= value (e.g. \"sid\"). Defaults to the Distribution: entry from the specified .changes file")

	listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`)
)

func dependsOn(src control.SourceIndex, binaries map[string]bool) bool {
	buildDepends := src.GetBuildDepends()
	for _, possibility := range buildDepends.GetAllPossibilities() {
		if binaries[possibility.Name] {
			return true
		}
	}
	return false
}

func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error {
	log.Printf("Loading sources index %q\n", sourcesPath)
	s, err := os.Open(sourcesPath)
	if err != nil {
		return err
	}
	defer s.Close()
	idx, err := control.ParseSourceIndex(bufio.NewReader(s))
	if err != nil {
		return err
	}

	for _, src := range idx {
		if dependsOn(src, binaries) {
			rebuild[src.Package] = append(rebuild[src.Package], src.Version)
		}
	}

	return nil
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0])
	}

	changesPath := flag.Arg(0)
	log.Printf("Loading changes file %q\n", changesPath)

	c, err := os.Open(changesPath)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	changes, err := control.ParseChanges(bufio.NewReader(c), changesPath)
	if err != nil {
		log.Fatal(err)
	}

	binaries := make(map[string]bool)
	var debs []string
	for _, file := range changes.Files {
		if filepath.Ext(file.Filename) == ".deb" {
			debs = append(debs, file.Filename)
		}
	}
	for _, binary := range changes.Binaries {
		binaries[binary] = true
	}

	log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " "))
	log.Printf(" - corresponding .debs (will be injected when building):\n")
	for _, deb := range debs {
		log.Printf("    %s\n", deb)
	}

	var sourcesPaths []string
	indexTargets := exec.Command("apt-get",
		"indextargets",
		"--format",
		"$(FILENAME)",
		"Codename: sid",
		"ShortDesc: Sources")
	if lines, err := indexTargets.Output(); err == nil {
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				sourcesPaths = append(sourcesPaths, line)
			}
		}
	} else {
		// Fallback for older versions of apt-get. See
		// https://bugs.debian.org/801594 for context.
		releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease")
		if err != nil {
			log.Fatal(err)
		}
		for _, releasepath := range releaseMatches {
			r, err := os.Open(releasepath)
			if err != nil {
				log.Fatal(err)
			}
			defer r.Close()
			release, err := control.ParseParagraph(bufio.NewReader(r))
			if err != nil {
				log.Fatal(err)
			}
			if release.Values["Suite"] != "unstable" {
				continue
			}

			listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath)
			if len(listsPrefix) != 2 {
				log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe)
			}
			sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			sourcesPaths = append(sourcesPaths, sourceMatches...)
		}
	}

	if len(sourcesPaths) == 0 {
		log.Fatal("Could not find InRelease file for unstable. Are you missing unstable in your /etc/apt/sources.list?")
	}

	rebuild := make(map[string][]version.Version)

	for _, sourcesPath := range sourcesPaths {
		if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil {
			log.Fatal(err)
		}
	}

	// TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time).

	// TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable.

	if strings.TrimSpace(*sbuildDist) == "" {
		*sbuildDist = changes.Distribution
		log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist)
	}

	buildresults := make(map[string]bool)
	for src, versions := range rebuild {
		sort.Sort(sort.Reverse(version.Slice(versions)))
		newest := versions[0]
		target := fmt.Sprintf("%s_%s", src, newest)
		// TODO: discard resulting package immediately?
		args := []string{
			"--arch-all",
			"--dist=" + *sbuildDist,
			"--nolog",
			target,
		}
		for _, filename := range debs {
			args = append(args, fmt.Sprintf("--extra-package=%s", filename))
		}
		cmd := exec.Command("sbuild", args...)
		if err := os.MkdirAll(*logDir, 0755); err != nil {
			log.Fatal(err)
		}
		log.Printf("Building %s (commandline: %v)\n", target, cmd.Args)
		if *dryRun {
			continue
		}
		buildlog, err := os.Create(filepath.Join(*logDir, target))
		if err != nil {
			log.Fatal(err)
		}
		defer buildlog.Close()
		cmd.Stdout = buildlog
		cmd.Stderr = buildlog
		if err := cmd.Run(); err != nil {
			log.Printf("building %s failed: %v\n", target, err)
			buildresults[target] = false
		} else {
			buildresults[target] = true
		}
	}

	log.Printf("Build results:\n")
	// Print all successful builds first (not as interesting), then failed ones.
	for target, result := range buildresults {
		if !result {
			continue
		}
		log.Printf("PASSED: %s\n", target)
	}

	for target, result := range buildresults {
		if result {
			continue
		}
		log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target))
	}
}
<commit_msg>Use dose-ceve(1) if available for determining revers-build-deps.<commit_after>// ratt operates on a Debian .changes file of a just-built package, identifies
// all reverse-build-dependencies and rebuilds them with the .debs from the
// .changes file.
//
// The intended use-case is, for example, to package a new snapshot of a Go
// library and verify that the new version does not break any other Go
// libraries/binaries.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"pault.ag/go/debian/control"
	"pault.ag/go/debian/version"
)

var (
	logDir = flag.String("log_dir",
		"buildlogs",
		"Directory in which sbuild(1) logs for all reverse-build-dependencies are stored")

	dryRun = flag.Bool("dry_run",
		false,
		"Print sbuild command lines, but do not actually build the reverse-build-dependencies")

	sbuildDist = flag.String("sbuild_dist",
		"",
		"sbuild --dist= value (e.g. \"sid\"). Defaults to the Distribution: entry from the specified .changes file")

	listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`)
)

func dependsOn(src control.SourceIndex, binaries map[string]bool) bool {
	buildDepends := src.GetBuildDepends()
	for _, possibility := range buildDepends.GetAllPossibilities() {
		if binaries[possibility.Name] {
			return true
		}
	}
	return false
}

func addReverseBuildDeps(sourcesPath string, binaries map[string]bool, rebuild map[string][]version.Version) error {
	log.Printf("Loading sources index %q\n", sourcesPath)
	s, err := os.Open(sourcesPath)
	if err != nil {
		return err
	}
	defer s.Close()
	idx, err := control.ParseSourceIndex(bufio.NewReader(s))
	if err != nil {
		return err
	}

	for _, src := range idx {
		if dependsOn(src, binaries) {
			rebuild[src.Package] = append(rebuild[src.Package], src.Version)
		}
	}

	return nil
}

func main() {
	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Usage: %s [options] <path-to-changes-file>\n", os.Args[0])
	}

	changesPath := flag.Arg(0)
	log.Printf("Loading changes file %q\n", changesPath)

	c, err := os.Open(changesPath)
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()
	changes, err := control.ParseChanges(bufio.NewReader(c), changesPath)
	if err != nil {
		log.Fatal(err)
	}

	binaries := make(map[string]bool)
	var debs []string
	for _, file := range changes.Files {
		if filepath.Ext(file.Filename) == ".deb" {
			debs = append(debs, file.Filename)
		}
	}
	for _, binary := range changes.Binaries {
		binaries[binary] = true
	}

	log.Printf(" - %d binary packages: %s\n", len(changes.Binaries), strings.Join(changes.Binaries, " "))
	log.Printf(" - corresponding .debs (will be injected when building):\n")
	for _, deb := range debs {
		log.Printf("    %s\n", deb)
	}

	var sourcesPaths []string
	var packagesPaths []string
	indexTargets := exec.Command("apt-get",
		"indextargets",
		"--format",
		"$(FILENAME)",
		"Codename: sid",
		"ShortDesc: Sources")
	if lines, err := indexTargets.Output(); err == nil {
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				sourcesPaths = append(sourcesPaths, line)
			}
		}
		binaryIndexTargets := exec.Command(
			"apt-get",
			"indextargets",
			"--format",
			"$(FILENAME)",
			"Codename: sid",
			"ShortDesc: Packages")
		lines, err = binaryIndexTargets.Output()
		if err != nil {
			log.Fatal("Could not get packages files using %+v: %v", binaryIndexTargets.Args, err)
		}
		for _, line := range strings.Split(string(lines), "\n") {
			trimmed := strings.TrimSpace(line)
			if trimmed != "" {
				packagesPaths = append(packagesPaths, line)
			}
		}
	} else {
		// Fallback for older versions of apt-get. See
		// https://bugs.debian.org/801594 for context.
		releaseMatches, err := filepath.Glob("/var/lib/apt/lists/*_InRelease")
		if err != nil {
			log.Fatal(err)
		}
		for _, releasepath := range releaseMatches {
			r, err := os.Open(releasepath)
			if err != nil {
				log.Fatal(err)
			}
			defer r.Close()
			release, err := control.ParseParagraph(bufio.NewReader(r))
			if err != nil {
				log.Fatal(err)
			}
			if release.Values["Suite"] != "unstable" {
				continue
			}

			listsPrefix := listsPrefixRe.FindStringSubmatch(releasepath)
			if len(listsPrefix) != 2 {
				log.Fatalf("release file path %q does not match regexp %q\n", releasepath, listsPrefixRe)
			}
			sourceMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Sources", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			sourcesPaths = append(sourcesPaths, sourceMatches...)
			packagesMatches, err := filepath.Glob(fmt.Sprintf("/var/lib/apt/lists/%s_*_Packages", listsPrefix[1]))
			if err != nil {
				log.Fatal(err)
			}
			packagesPaths = append(packagesPaths, packagesMatches...)
		}
	}

	if len(sourcesPaths) == 0 {
		log.Fatal("Could not find InRelease file for unstable. Are you missing unstable in your /etc/apt/sources.list?")
	}

	rebuild := make(map[string][]version.Version)

	archCmd := exec.Command(
		"dpkg-architecture",
		"--query=DEB_BUILD_ARCH")
	archOut, err := archCmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	arch := strings.TrimSpace(string(archOut))

	// TODO: Cache this output based on the .changes file. dose-ceve takes quite a while.
	ceve := exec.Command(
		"dose-ceve",
		"--deb-native-arch="+arch,
		"-T", "debsrc",
		"-r", strings.Join(changes.Binaries, ","),
		"-G", "pkg")
	for _, packagesPath := range packagesPaths {
		ceve.Args = append(ceve.Args, "deb://"+packagesPath)
	}
	for _, sourcesPath := range sourcesPaths {
		ceve.Args = append(ceve.Args, "debsrc://"+sourcesPath)
	}

	log.Printf("Figuring out reverse build dependencies using dose-ceve(1). This might take a while\n")
	if out, err := ceve.Output(); err == nil {
		r := bufio.NewReader(strings.NewReader(string(out)))
		for {
			paragraph, err := control.ParseParagraph(r)
			if paragraph == nil || err == io.EOF {
				break
			}
			pkg := paragraph.Values["Package"]
			ver, err := version.Parse(paragraph.Values["Version"])
			if err != nil {
				log.Fatalf("Cannot parse version number %q in dose-ceve(1) output: %v", paragraph.Values["Version"], err)
			}
			rebuild[pkg] = append(rebuild[pkg], ver)
		}
	} else {
		log.Printf("dose-ceve(1) failed (%v), falling back to interpreting Sources directly\n", err)
		for _, sourcesPath := range sourcesPaths {
			if err := addReverseBuildDeps(sourcesPath, binaries, rebuild); err != nil {
				log.Fatal(err)
			}
		}
	}

	// TODO: add -recursive flag to also cover dependencies which are not DIRECT dependencies. use http://godoc.org/pault.ag/go/debian/control#OrderDSCForBuild (topsort) to build dependencies in the right order (saving CPU time).

	// TODO: what’s a good integration method for doing this in more setups, e.g. on a cloud provider or something? mapreri from #debian-qa says jenkins.debian.net is suitable.

	if strings.TrimSpace(*sbuildDist) == "" {
		*sbuildDist = changes.Distribution
		log.Printf("Setting -sbuild_dist=%s (from .changes file)\n", *sbuildDist)
	}

	buildresults := make(map[string]bool)
	for src, versions := range rebuild {
		sort.Sort(sort.Reverse(version.Slice(versions)))
		newest := versions[0]
		target := fmt.Sprintf("%s_%s", src, newest)
		// TODO: discard resulting package immediately?
		args := []string{
			"--arch-all",
			"--dist=" + *sbuildDist,
			"--nolog",
			target,
		}
		for _, filename := range debs {
			args = append(args, fmt.Sprintf("--extra-package=%s", filename))
		}
		cmd := exec.Command("sbuild", args...)
		if err := os.MkdirAll(*logDir, 0755); err != nil {
			log.Fatal(err)
		}
		log.Printf("Building %s (commandline: %v)\n", target, cmd.Args)
		if *dryRun {
			continue
		}
		buildlog, err := os.Create(filepath.Join(*logDir, target))
		if err != nil {
			log.Fatal(err)
		}
		defer buildlog.Close()
		cmd.Stdout = buildlog
		cmd.Stderr = buildlog
		if err := cmd.Run(); err != nil {
			log.Printf("building %s failed: %v\n", target, err)
			buildresults[target] = false
		} else {
			buildresults[target] = true
		}
	}

	log.Printf("Build results:\n")
	// Print all successful builds first (not as interesting), then failed ones.
	for target, result := range buildresults {
		if !result {
			continue
		}
		log.Printf("PASSED: %s\n", target)
	}

	for target, result := range buildresults {
		if result {
			continue
		}
		log.Printf("FAILED: %s (see %s)\n", target, filepath.Join(*logDir, target))
	}
}
<|endoftext|>


<commit_before>// A quick and easy way to setup a RESTful JSON API
//
// Go-JSON-REST is a thin layer on top of net/http that helps building RESTful JSON APIs easily.
// It provides fast URL routing using https://github.com/ant0ine/go-urlrouter, and helpers to deal
// with JSON requests and responses. It is not a high-level REST framework that transparently maps
// HTTP requests to language procedure calls, on the opposite, you constantly have access to the
// underlying net/http objects.
//
// Example:
//
//      package main
//
//      import (
//              "github.com/ant0ine/go-json-rest"
//              "net/http"
//      )
//
//      type User struct {
//              Id   string
//              Name string
//      }
//
//      func GetUser(w *rest.ResponseWriter, req *rest.Request) {
//              user := User{
//                      Id:   req.PathParam("id"),
//                      Name: "Antoine",
//              }
//              w.WriteJSON(&user)
//      }
//
//      func main() {
//              handler := ResourceHandler{}
//              handler.SetRoutes(
//                      rest.Route{"GET", "/users/:id", GetUser},
//              )
//              http.ListenAndServe(":8080", &handler)
//      }
//
package rest

import (
	"compress/gzip"
	"encoding/json"
	"github.com/ant0ine/go-urlrouter"
	"io/ioutil"
	"net/http"
	"strings"
)

// Implement the http.Handler interface and act as a router for the defined Routes.
// The defaults are intended to be developemnt friendly, for production you may want
// to turn on gzip and disable the JSON indentation.
type ResourceHandler struct {
	router urlrouter.Router
	// If true and if the client accepts the Gzip encoding, the response payloads
	// will be compressed using gzip, and the corresponding response header will set.
	EnableGzip bool
	// If true the JSON payload will be written in one line with no space.
	DisableJsonIndent bool
}

// Used with SetRoutes.
type Route struct {
	Method  string
	PathExp string
	Dest    func(*ResponseWriter, *Request)
}

// Define the Routes. The order the Routes matters,
// if a request matches multiple Routes, the first one will be used.
// Note that the underlying router is https://github.com/ant0ine/go-urlrouter.
func (self *ResourceHandler) SetRoutes(routes ...Route) error {
	self.router = urlrouter.Router{
		Routes: []urlrouter.Route{},
	}
	for _, route := range routes {
		// make sure the method is uppercase
		method := strings.ToUpper(route.Method)

		self.router.Routes = append(
			self.router.Routes,
			urlrouter.Route{
				PathExp: method + route.PathExp,
				Dest:    route.Dest,
			},
		)
	}
	return self.router.Start()
}

// This makes ResourceHandler implement the http.Handler interface
func (self *ResourceHandler) ServeHTTP(orig_writer http.ResponseWriter, orig_request *http.Request) {

	// find the route
	route, params, err := self.router.FindRoute(
		strings.ToUpper(orig_request.Method) + orig_request.URL.Path,
	)
	if err != nil {
		// should never happen has the URL has already been parsed
		panic(err)
	}
	if route == nil {
		// no route found
		http.NotFound(orig_writer, orig_request)
		return
	}

	// determine if gzip is needed
	is_gzipped := self.EnableGzip == true &&
		strings.Contains(orig_request.Header.Get("Accept-Encoding"), "gzip")

	is_indented := !self.DisableJsonIndent

	request := Request{
		orig_request,
		params,
	}

	writer := ResponseWriter{
		orig_writer,
		is_gzipped,
		is_indented,
	}

	// run the user code
	handler := route.Dest.(func(*ResponseWriter, *Request))
	handler(&writer, &request)
}

// Inherit from http.Request, and provide additional methods.
type Request struct {
	*http.Request
	// map of parameters that have been matched in the URL Path.
	PathParams map[string]string
}

// Provide a convenient access to the PathParams map
func (self *Request) PathParam(name string) string {
	return self.PathParams[name]
}

// Read the request body and decode the JSON using json.Unmarshal
func (self *Request) DecodeJSONPayload(v interface{}) error {
	content, err := ioutil.ReadAll(self.Body)
	self.Body.Close()
	if err != nil {
		return err
	}
	err = json.Unmarshal(content, v)
	if err != nil {
		return err
	}
	return nil
}

// Inherit from an object implementing the http.ResponseWriter interface, and provide additional methods.
type ResponseWriter struct {
	http.ResponseWriter
	is_gzipped  bool
	is_indented bool
}

// Overloading of the http.ResponseWriter method.
// Provide additional capabilities, like transparent gzip encoding.
func (self *ResponseWriter) Write(b []byte) (int, error) {
	if self.is_gzipped {
		self.Header().Set("Content-Encoding", "gzip")
		gzip_writer := gzip.NewWriter(self.ResponseWriter)
		defer gzip_writer.Close()
		return gzip_writer.Write(b)
	}
	return self.ResponseWriter.Write(b)
}

// Encode the object in JSON, set the content-type header,
// and call Write
func (self *ResponseWriter) WriteJSON(v interface{}) error {
	self.Header().Set("content-type", "application/json")
	var b []byte
	var err error
	if self.is_indented {
		b, err = json.MarshalIndent(v, "", "  ")
	} else {
		b, err = json.Marshal(v)
	}
	if err != nil {
		return err
	}
	self.Write(b)
	return nil
}
<commit_msg>make it more explicit that Dest is a function<commit_after>// A quick and easy way to setup a RESTful JSON API
//
// Go-JSON-REST is a thin layer on top of net/http that helps building RESTful JSON APIs easily.
// It provides fast URL routing using https://github.com/ant0ine/go-urlrouter, and helpers to deal
// with JSON requests and responses. It is not a high-level REST framework that transparently maps
// HTTP requests to language procedure calls, on the opposite, you constantly have access to the
// underlying net/http objects.
//
// Example:
//
//      package main
//
//      import (
//              "github.com/ant0ine/go-json-rest"
//              "net/http"
//      )
//
//      type User struct {
//              Id   string
//              Name string
//      }
//
//      func GetUser(w *rest.ResponseWriter, req *rest.Request) {
//              user := User{
//                      Id:   req.PathParam("id"),
//                      Name: "Antoine",
//              }
//              w.WriteJSON(&user)
//      }
//
//      func main() {
//              handler := ResourceHandler{}
//              handler.SetRoutes(
//                      rest.Route{"GET", "/users/:id", GetUser},
//              )
//              http.ListenAndServe(":8080", &handler)
//      }
//
package rest

import (
	"compress/gzip"
	"encoding/json"
	"github.com/ant0ine/go-urlrouter"
	"io/ioutil"
	"net/http"
	"strings"
)

// Implement the http.Handler interface and act as a router for the defined Routes.
// The defaults are intended to be developemnt friendly, for production you may want
// to turn on gzip and disable the JSON indentation.
type ResourceHandler struct {
	router urlrouter.Router
	// If true and if the client accepts the Gzip encoding, the response payloads
	// will be compressed using gzip, and the corresponding response header will set.
	EnableGzip bool
	// If true the JSON payload will be written in one line with no space.
	DisableJsonIndent bool
}

// Used with SetRoutes.
type Route struct {
	Method  string
	PathExp string
	Func    func(*ResponseWriter, *Request)
}

// Define the Routes. The order the Routes matters,
// if a request matches multiple Routes, the first one will be used.
// Note that the underlying router is https://github.com/ant0ine/go-urlrouter.
func (self *ResourceHandler) SetRoutes(routes ...Route) error {
	self.router = urlrouter.Router{
		Routes: []urlrouter.Route{},
	}
	for _, route := range routes {
		// make sure the method is uppercase
		method := strings.ToUpper(route.Method)

		self.router.Routes = append(
			self.router.Routes,
			urlrouter.Route{
				PathExp: method + route.PathExp,
				Dest:    route.Func,
			},
		)
	}
	return self.router.Start()
}

// This makes ResourceHandler implement the http.Handler interface
func (self *ResourceHandler) ServeHTTP(orig_writer http.ResponseWriter, orig_request *http.Request) {

	// find the route
	route, params, err := self.router.FindRoute(
		strings.ToUpper(orig_request.Method) + orig_request.URL.Path,
	)
	if err != nil {
		// should never happen has the URL has already been parsed
		panic(err)
	}
	if route == nil {
		// no route found
		http.NotFound(orig_writer, orig_request)
		return
	}

	// determine if gzip is needed
	is_gzipped := self.EnableGzip == true &&
		strings.Contains(orig_request.Header.Get("Accept-Encoding"), "gzip")

	is_indented := !self.DisableJsonIndent

	request := Request{
		orig_request,
		params,
	}

	writer := ResponseWriter{
		orig_writer,
		is_gzipped,
		is_indented,
	}

	// run the user code
	handler := route.Dest.(func(*ResponseWriter, *Request))
	handler(&writer, &request)
}

// Inherit from http.Request, and provide additional methods.
type Request struct {
	*http.Request
	// map of parameters that have been matched in the URL Path.
	PathParams map[string]string
}

// Provide a convenient access to the PathParams map
func (self *Request) PathParam(name string) string {
	return self.PathParams[name]
}

// Read the request body and decode the JSON using json.Unmarshal
func (self *Request) DecodeJSONPayload(v interface{}) error {
	content, err := ioutil.ReadAll(self.Body)
	self.Body.Close()
	if err != nil {
		return err
	}
	err = json.Unmarshal(content, v)
	if err != nil {
		return err
	}
	return nil
}

// Inherit from an object implementing the http.ResponseWriter interface, and provide additional methods.
type ResponseWriter struct {
	http.ResponseWriter
	is_gzipped  bool
	is_indented bool
}

// Overloading of the http.ResponseWriter method.
// Provide additional capabilities, like transparent gzip encoding.
func (self *ResponseWriter) Write(b []byte) (int, error) {
	if self.is_gzipped {
		self.Header().Set("Content-Encoding", "gzip")
		gzip_writer := gzip.NewWriter(self.ResponseWriter)
		defer gzip_writer.Close()
		return gzip_writer.Write(b)
	}
	return self.ResponseWriter.Write(b)
}

// Encode the object in JSON, set the content-type header,
// and call Write
func (self *ResponseWriter) WriteJSON(v interface{}) error {
	self.Header().Set("content-type", "application/json")
	var b []byte
	var err error
	if self.is_indented {
		b, err = json.MarshalIndent(v, "", "  ")
	} else {
		b, err = json.Marshal(v)
	}
	if err != nil {
		return err
	}
	self.Write(b)
	return nil
}
<|endoftext|>


<commit_before>package runc

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Format is the type of log formatting options avaliable
type Format string

const (
	none Format = ""
	JSON Format = "json"
	Text Format = "text"
	// DefaultCommand is the default command for Runc
	DefaultCommand string = "runc"
)

// Runc is the client to the runc cli
type Runc struct {
	//If command is empty, DefaultCommand is used
	Command      string
	Root         string
	Debug        bool
	Log          string
	LogFormat    Format
	PdeathSignal syscall.Signal
}

// List returns all containers created inside the provided runc root directory
func (r *Runc) List(context context.Context) ([]*Container, error) {
	data, err := r.command(context, "list", "--format=json").Output()
	if err != nil {
		return nil, err
	}
	var out []*Container
	if err := json.Unmarshal(data, &out); err != nil {
		return nil, err
	}
	return out, nil
}

// State returns the state for the container provided by id
func (r *Runc) State(context context.Context, id string) (*Container, error) {
	data, err := r.command(context, "state", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var c Container
	if err := json.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return &c, nil
}

type CreateOpts struct {
	IO
	// PidFile is a path to where a pid file should be created
	PidFile       string
	ConsoleSocket *ConsoleSocket
	Detach        bool
	NoPivot       bool
	NoNewKeyring  bool
}

func (o *CreateOpts) args() (out []string, err error) {
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.NoPivot {
		out = append(out, "--no-pivot")
	}
	if o.NoNewKeyring {
		out = append(out, "--no-new-keyring")
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	return out, nil
}

// Create creates a new container and returns its pid if it was created successfully
func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
	args := []string{"create", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	if cmd.Stdout == nil && cmd.Stderr == nil {
		data, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("%s: %s", err, data)
		}
		return nil
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	return cmd.Wait()
}

// Start will start an already created container
func (r *Runc) Start(context context.Context, id string) error {
	return runOrError(r.command(context, "start", id))
}

type ExecOpts struct {
	IO
	PidFile       string
	Uid           int
	Gid           int
	Cwd           string
	Tty           bool
	ConsoleSocket *ConsoleSocket
	Detach        bool
}

func (o *ExecOpts) args() (out []string, err error) {
	out = append(out, "--user", fmt.Sprintf("%d:%d", o.Uid, o.Gid))
	if o.Tty {
		out = append(out, "--tty")
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.Cwd != "" {
		out = append(out, "--cwd", o.Cwd)
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	return out, nil
}

// Exec executres and additional process inside the container based on a full
// OCI Process specification
func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error {
	f, err := ioutil.TempFile("", "-process")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	err = json.NewEncoder(f).Encode(spec)
	f.Close()
	if err != nil {
		return err
	}
	args := []string{"exec", "--process", f.Name()}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	return runOrError(cmd)
}

// Run runs the create, start, delete lifecycle of the container
// and returns its exit status after it has exited
func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) {
	args := []string{"run", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return -1, err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil {
		opts.Set(cmd)
	}
	if err := cmd.Start(); err != nil {
		return -1, err
	}
	status, err := cmd.Process.Wait()
	if err != nil {
		return -1, err
	}
	return status.Sys().(syscall.WaitStatus).ExitStatus(), nil
}

// Delete deletes the container
func (r *Runc) Delete(context context.Context, id string) error {
	return runOrError(r.command(context, "delete", id))
}

// KillOpts specifies options for killing a container and its processes
type KillOpts struct {
	All bool
}

func (o *KillOpts) args() (out []string) {
	if o.All {
		out = append(out, "--all")
	}
	return out
}

// Kill sends the specified signal to the container
func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) error {
	args := []string{
		"kill",
	}
	if opts != nil {
		args = append(args, opts.args()...)
	}
	return runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...))
}

// Stats return the stats for a container like cpu, memory, and io
func (r *Runc) Stats(context context.Context, id string) (*Stats, error) {
	cmd := r.command(context, "events", "--stats", id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer func() {
		rd.Close()
		cmd.Wait()
	}()
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	var e Event
	if err := json.NewDecoder(rd).Decode(&e); err != nil {
		return nil, err
	}
	return e.Stats, nil
}

// Events returns an event stream from runc for a container with stats and OOM notifications
func (r *Runc) Events(context context.Context, id string, interval time.Duration) (chan *Event, error) {
	cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		rd.Close()
		return nil, err
	}
	var (
		dec = json.NewDecoder(rd)
		c   = make(chan *Event, 128)
	)
	go func() {
		defer func() {
			close(c)
			rd.Close()
			cmd.Wait()
		}()
		for {
			var e Event
			if err := dec.Decode(&e); err != nil {
				if err == io.EOF {
					return
				}
				e = Event{
					Type: "error",
					Err:  err,
				}
			}
			c <- &e
		}
	}()
	return c, nil
}

// Pause the container with the provided id
func (r *Runc) Pause(context context.Context, id string) error {
	return runOrError(r.command(context, "pause", id))
}

// Resume the container with the provided id
func (r *Runc) Resume(context context.Context, id string) error {
	return runOrError(r.command(context, "resume", id))
}

// Ps lists all the processes inside the container returning their pids
func (r *Runc) Ps(context context.Context, id string) ([]int, error) {
	data, err := r.command(context, "ps", "--format", "json", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var pids []int
	if err := json.Unmarshal(data, &pids); err != nil {
		return nil, err
	}
	return pids, nil
}

func (r *Runc) args() (out []string) {
	if r.Root != "" {
		out = append(out, "--root", r.Root)
	}
	if r.Debug {
		out = append(out, "--debug")
	}
	if r.Log != "" {
		out = append(out, "--log", r.Log)
	}
	if r.LogFormat != none {
		out = append(out, "--log-format", string(r.LogFormat))
	}
	return out
}

func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
	command := r.Command
	if command == "" {
		command = DefaultCommand
	}
	cmd := exec.CommandContext(context, command, append(r.args(), args...)...)
	if r.PdeathSignal != 0 {
		cmd.SysProcAttr = &syscall.SysProcAttr{
			Pdeathsig: r.PdeathSignal,
		}
	}
	return cmd
}
<commit_msg>Close exec pipes after start<commit_after>package runc

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	specs "github.com/opencontainers/runtime-spec/specs-go"
)

// Format is the type of log formatting options avaliable
type Format string

const (
	none Format = ""
	JSON Format = "json"
	Text Format = "text"
	// DefaultCommand is the default command for Runc
	DefaultCommand string = "runc"
)

// Runc is the client to the runc cli
type Runc struct {
	//If command is empty, DefaultCommand is used
	Command      string
	Root         string
	Debug        bool
	Log          string
	LogFormat    Format
	PdeathSignal syscall.Signal
}

// List returns all containers created inside the provided runc root directory
func (r *Runc) List(context context.Context) ([]*Container, error) {
	data, err := r.command(context, "list", "--format=json").Output()
	if err != nil {
		return nil, err
	}
	var out []*Container
	if err := json.Unmarshal(data, &out); err != nil {
		return nil, err
	}
	return out, nil
}

// State returns the state for the container provided by id
func (r *Runc) State(context context.Context, id string) (*Container, error) {
	data, err := r.command(context, "state", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var c Container
	if err := json.Unmarshal(data, &c); err != nil {
		return nil, err
	}
	return &c, nil
}

type CreateOpts struct {
	IO
	// PidFile is a path to where a pid file should be created
	PidFile       string
	ConsoleSocket *ConsoleSocket
	Detach        bool
	NoPivot       bool
	NoNewKeyring  bool
}

func (o *CreateOpts) args() (out []string, err error) {
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.NoPivot {
		out = append(out, "--no-pivot")
	}
	if o.NoNewKeyring {
		out = append(out, "--no-new-keyring")
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	return out, nil
}

// Create creates a new container and returns its pid if it was created successfully
func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOpts) error {
	args := []string{"create", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	if cmd.Stdout == nil && cmd.Stderr == nil {
		data, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("%s: %s", err, data)
		}
		return nil
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	return cmd.Wait()
}

// Start will start an already created container
func (r *Runc) Start(context context.Context, id string) error {
	return runOrError(r.command(context, "start", id))
}

type ExecOpts struct {
	IO
	PidFile       string
	Uid           int
	Gid           int
	Cwd           string
	Tty           bool
	ConsoleSocket *ConsoleSocket
	Detach        bool
}

func (o *ExecOpts) args() (out []string, err error) {
	out = append(out, "--user", fmt.Sprintf("%d:%d", o.Uid, o.Gid))
	if o.Tty {
		out = append(out, "--tty")
	}
	if o.ConsoleSocket != nil {
		out = append(out, "--console-socket", o.ConsoleSocket.Path())
	}
	if o.Cwd != "" {
		out = append(out, "--cwd", o.Cwd)
	}
	if o.Detach {
		out = append(out, "--detach")
	}
	if o.PidFile != "" {
		abs, err := filepath.Abs(o.PidFile)
		if err != nil {
			return nil, err
		}
		out = append(out, "--pid-file", abs)
	}
	return out, nil
}

// Exec executres and additional process inside the container based on a full
// OCI Process specification
func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error {
	f, err := ioutil.TempFile("", "-process")
	if err != nil {
		return err
	}
	defer os.Remove(f.Name())
	err = json.NewEncoder(f).Encode(spec)
	f.Close()
	if err != nil {
		return err
	}
	args := []string{"exec", "--process", f.Name()}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil && opts.IO != nil {
		opts.Set(cmd)
	}
	if cmd.Stdout == nil && cmd.Stderr == nil {
		data, err := cmd.CombinedOutput()
		if err != nil {
			return fmt.Errorf("%s: %s", err, data)
		}
		return nil
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	if opts != nil && opts.IO != nil {
		if c, ok := opts.IO.(StartCloser); ok {
			if err := c.CloseAfterStart(); err != nil {
				return err
			}
		}
	}
	return cmd.Wait()
}

// Run runs the create, start, delete lifecycle of the container
// and returns its exit status after it has exited
func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) (int, error) {
	args := []string{"run", "--bundle", bundle}
	if opts != nil {
		oargs, err := opts.args()
		if err != nil {
			return -1, err
		}
		args = append(args, oargs...)
	}
	cmd := r.command(context, append(args, id)...)
	if opts != nil {
		opts.Set(cmd)
	}
	if err := cmd.Start(); err != nil {
		return -1, err
	}
	status, err := cmd.Process.Wait()
	if err != nil {
		return -1, err
	}
	return status.Sys().(syscall.WaitStatus).ExitStatus(), nil
}

// Delete deletes the container
func (r *Runc) Delete(context context.Context, id string) error {
	return runOrError(r.command(context, "delete", id))
}

// KillOpts specifies options for killing a container and its processes
type KillOpts struct {
	All bool
}

func (o *KillOpts) args() (out []string) {
	if o.All {
		out = append(out, "--all")
	}
	return out
}

// Kill sends the specified signal to the container
func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) error {
	args := []string{
		"kill",
	}
	if opts != nil {
		args = append(args, opts.args()...)
	}
	return runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...))
}

// Stats return the stats for a container like cpu, memory, and io
func (r *Runc) Stats(context context.Context, id string) (*Stats, error) {
	cmd := r.command(context, "events", "--stats", id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	defer func() {
		rd.Close()
		cmd.Wait()
	}()
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	var e Event
	if err := json.NewDecoder(rd).Decode(&e); err != nil {
		return nil, err
	}
	return e.Stats, nil
}

// Events returns an event stream from runc for a container with stats and OOM notifications
func (r *Runc) Events(context context.Context, id string, interval time.Duration) (chan *Event, error) {
	cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id)
	rd, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		rd.Close()
		return nil, err
	}
	var (
		dec = json.NewDecoder(rd)
		c   = make(chan *Event, 128)
	)
	go func() {
		defer func() {
			close(c)
			rd.Close()
			cmd.Wait()
		}()
		for {
			var e Event
			if err := dec.Decode(&e); err != nil {
				if err == io.EOF {
					return
				}
				e = Event{
					Type: "error",
					Err:  err,
				}
			}
			c <- &e
		}
	}()
	return c, nil
}

// Pause the container with the provided id
func (r *Runc) Pause(context context.Context, id string) error {
	return runOrError(r.command(context, "pause", id))
}

// Resume the container with the provided id
func (r *Runc) Resume(context context.Context, id string) error {
	return runOrError(r.command(context, "resume", id))
}

// Ps lists all the processes inside the container returning their pids
func (r *Runc) Ps(context context.Context, id string) ([]int, error) {
	data, err := r.command(context, "ps", "--format", "json", id).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s: %s", err, data)
	}
	var pids []int
	if err := json.Unmarshal(data, &pids); err != nil {
		return nil, err
	}
	return pids, nil
}

func (r *Runc) args() (out []string) {
	if r.Root != "" {
		out = append(out, "--root", r.Root)
	}
	if r.Debug {
		out = append(out, "--debug")
	}
	if r.Log != "" {
		out = append(out, "--log", r.Log)
	}
	if r.LogFormat != none {
		out = append(out, "--log-format", string(r.LogFormat))
	}
	return out
}

func (r *Runc) command(context context.Context, args ...string) *exec.Cmd {
	command := r.Command
	if command == "" {
		command = DefaultCommand
	}
	cmd := exec.CommandContext(context, command, append(r.args(), args...)...)
	if r.PdeathSignal != 0 {
		cmd.SysProcAttr = &syscall.SysProcAttr{
			Pdeathsig: r.PdeathSignal,
		}
	}
	return cmd
}
<|endoftext|>


<commit_before>package main

import (
	_ "time"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"github.com/gorilla/sessions"
	"io"
	"net/http"
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"runtime"
	"errors"
	"html/template"
	"os/exec"
	"bytes"
)

var templates = template.Must(template.ParseFiles("index.html"))

const MESSAGE_QUEUE_SIZE = 10

const STATUS_FAILURE = "{\"status\":\"failure\"}"

var authKey = []byte("somesecretauth")
var store sessions.Store

var pool *Pool
var clients map[int64]*Client

var db *sql.DB

type Pool struct {
	in  chan *Client
	out chan *Room
}

type Client struct {
	id      int64
	otherid int64
	in      chan string
	out     chan string
	retChan chan *Room
}

type Room struct {
	id      []byte
	client1 *Client
	client2 *Client
	question *Question
}

type Question struct {
	Id 	int64			`json:"id"`
	Title string		`json:"title"`
	Body string			`json:"body"`
	Difficulty int 		`json:"diff"`
}

func (p *Pool) Pair() {
	for {
		c1, c2 := <-p.in, <-p.in

		for c1.id == c2.id {
			c2 = <- p.in
		}

		fmt.Println("match found for ", c1.id, " and ", c2.id)

		b := make([]byte, 32)
		n, err := io.ReadFull(rand.Reader, b)
		if err != nil || n != 32 {
			return
		}

		row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
		q := new(Question)
		err = row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)
		if err !=  nil {
			fmt.Println(err)
		}

		room := &Room{b, c1, c2, q}

		c1.otherid, c2.otherid = c2.id, c1.id
		c1.in, c2.in = c2.out, c1.out

		c1.retChan <- room
		c2.retChan <- room
	}
}

func newPool() *Pool {
	pool := &Pool{
		in:  make(chan *Client),
		out: make(chan *Room),
	}

	go pool.Pair()

	return pool
}

func UIDFromSession(w http.ResponseWriter, r *http.Request) (int64, error) {
	session, _ := store.Get(r, "session")
	userid := session.Values["userid"]

	if userid == nil {
		return 0, errors.New("no cookie set")
	}
	return userid.(int64), nil
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, _ = sql.Open("mysql", "root:pass@/suitup")
	defer db.Close()

	store = sessions.NewCookieStore(authKey)

	pool = newPool()
	clients = make(map[int64]*Client)

	http.HandleFunc("/", mainHandle)

	http.HandleFunc("/login", login)

	http.HandleFunc("/message/check", checkMessage)
	http.HandleFunc("/message/send", sendMessage)

	http.HandleFunc("/question/new", newQuestion)
	http.HandleFunc("/question/submit", testCode)

	http.HandleFunc("/chatroom/join", joinChatRoom)
	http.HandleFunc("/chatroom/leave", leaveChatRoom)

	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("/home/suitup/hackmit/assets/"))))
	http.ListenAndServe(":8080", nil)
}

func mainHandle(w http.ResponseWriter, r *http.Request) {
	templates.ExecuteTemplate(w, "index.html", nil)
}

func joinChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("join ", uid)

	retChan := make(chan *Room)
	client := &Client{
		id:      uid,
		in:      nil,
		out:     make(chan string, MESSAGE_QUEUE_SIZE),
		retChan: retChan,
	}

	clients[uid] = client
	pool.in <- client

	chatroom := <- retChan

	fmt.Println("joinChatRoom-chatroom.id: ", chatroom.id)

	qjs, err := json.Marshal(chatroom.question)

	fmt.Fprint(w, "{\"status\":\"success\",\"crid\":\"", asciify(chatroom.id), "\",\"question\":", qjs, "}")
}

func asciify(ba []byte) string {
	ret := make([]byte, len(ba))
	for i, b := range ba {
		ret[i] = (b % 26) + 97
	}
	return string(ret)
}

func testCode(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	code := r.FormValue("submission")
	_ = r.FormValue("cvid")
	qid := string(1)
	app := "./secure.sh"
	cmd := exec.Command(app, qid, code)
    var b bytes.Buffer
    cmd.Stdout = &b
    err := cmd.Run()
    if err != nil {
    	fmt.Println(err)
    }
    fmt.Fprint(w, "{\"status\":\"success\", \"data\": ", b.String(), "}" )
    return
}

func leaveChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, _ := UIDFromSession(w, r)
	client := clients[uid]

	if client != nil {
		fmt.Println("leaving")
		delete(clients, client.otherid)
		delete(clients, uid)
	}

	fmt.Println("leave ", uid)

	fmt.Fprint(w, "{\"status\":\"success\"}")
}

func sendMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("send ", uid)

	r.ParseForm()
	message := r.FormValue("s")

	client := clients[uid]

	if client != nil {
		client.out <- message
		fmt.Fprint(w, "{\"status\":\"success\"}")
	} else {
		fmt.Fprint(w, STATUS_FAILURE)
	}
}

func checkMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("check ", uid)

	client := clients[uid]

	if client != nil {
		select {
		case message, ok := <- client.in:
			if ok {
				fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"", message, "\"}")
			} else {
				delete(clients, uid)
				fmt.Fprint(w, "{\"status\":\"failure\"}")
			}
		default:
			fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"\"}")
		}

	} else {
		fmt.Fprint(w, "{\"status\":\"failure\",\"s\":\"\"}")
	}
}

type User struct {
    Id            	int64      	`json:"id"`
    FacebookId  	string		`json:fbid"`
    Username 		string		`json:username"`
    Email 			string		`json:email"`
    Level 			int64		`json:lvl"`
    Score 			int64		`json:score"`
}

func newQuestion(w http.ResponseWriter, r *http.Request) {
	row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
	q := new(Question)
	err := row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)

	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	b, err := json.Marshal(q)
	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	fmt.Fprint(w, string(b))
}

func login(w http.ResponseWriter, r *http.Request) {
	inputToken := r.FormValue("access_token")
	if len(inputToken) != 0 {
		uid := GetMe(inputToken)

		// row := db.QueryRow("SELECT id FROM users")
		row := db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
		user := new(User)
		err := row.Scan(&user.Id)

		if err != nil {
			_, err = db.Exec("INSERT INTO users (facebook_id, username, email, level, points) VALUES (?, ?, ?, 0, 0)", uid, "", "")
			if err != nil {
				fmt.Fprint(w, STATUS_FAILURE)
				return
			} else {
				row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
				err = row.Scan(&user.Id)
				if err != nil {
					fmt.Fprint(w, STATUS_FAILURE)
					return
				}
			}

		}

		session, _ := store.Get(r, "session")
		session.Values["userid"] = user.Id
		session.Save(r, w)

		fmt.Fprint(w, "{\"status\":\"success\"}")
	}
}

func readHttpBody(response *http.Response) string {
	bodyBuffer := make([]byte, 1000)
	var str string

	count, err := response.Body.Read(bodyBuffer)

	for ; count > 0; count, err = response.Body.Read(bodyBuffer) {

		if err != nil {

		}

		str += string(bodyBuffer[:count])
	}

	return str

}

func getUncachedResponse(uri string) (*http.Response, error) {
	request, err := http.NewRequest("GET", uri, nil)

	if err == nil {
		request.Header.Add("Cache-Control", "no-cache")

		client := new(http.Client)

		return client.Do(request)
	}

	if (err != nil) {
	}
	return nil, err

}

func GetMe(token string) string {
	response, err := getUncachedResponse("https://graph.facebook.com/me?access_token="+token)

	if err == nil {

		var jsonBlob interface{}

		responseBody := readHttpBody(response)

		if responseBody != "" {
			err = json.Unmarshal([]byte(responseBody), &jsonBlob)

			if err == nil {
				jsonObj := jsonBlob.(map[string]interface{})
				return jsonObj["id"].(string)
			}
		}
		return err.Error()
	}

	return err.Error()
}

func handleError(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>slkjf<commit_after>package main

import (
	_ "time"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"github.com/gorilla/sessions"
	"io"
	"net/http"
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"runtime"
	"errors"
	"html/template"
	"os/exec"
	"bytes"
)

var templates = template.Must(template.ParseFiles("index.html"))

const MESSAGE_QUEUE_SIZE = 10

const STATUS_FAILURE = "{\"status\":\"failure\"}"

var authKey = []byte("somesecretauth")
var store sessions.Store

var pool *Pool
var clients map[int64]*Client

var db *sql.DB

type Pool struct {
	in  chan *Client
	out chan *Room
}

type Client struct {
	id      int64
	otherid int64
	in      chan string
	out     chan string
	retChan chan *Room
}

type Room struct {
	id      []byte
	client1 *Client
	client2 *Client
	Question *Question
}

type Question struct {
	Id 	int64			`json:"id"`
	Title string		`json:"title"`
	Body string			`json:"body"`
	Difficulty int 		`json:"diff"`
}

func (p *Pool) Pair() {
	for {
		c1, c2 := <-p.in, <-p.in

		for c1.id == c2.id {
			c2 = <- p.in
		}

		fmt.Println("match found for ", c1.id, " and ", c2.id)

		b := make([]byte, 32)
		n, err := io.ReadFull(rand.Reader, b)
		if err != nil || n != 32 {
			return
		}

		row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
		q := new(Question)
		err = row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)
		if err !=  nil {
			fmt.Println(err)
		}

		room := &Room{b, c1, c2, q}

		c1.otherid, c2.otherid = c2.id, c1.id
		c1.in, c2.in = c2.out, c1.out

		c1.retChan <- room
		c2.retChan <- room
	}
}

func newPool() *Pool {
	pool := &Pool{
		in:  make(chan *Client),
		out: make(chan *Room),
	}

	go pool.Pair()

	return pool
}

func UIDFromSession(w http.ResponseWriter, r *http.Request) (int64, error) {
	session, _ := store.Get(r, "session")
	userid := session.Values["userid"]

	if userid == nil {
		return 0, errors.New("no cookie set")
	}
	return userid.(int64), nil
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, _ = sql.Open("mysql", "root:pass@/suitup")
	defer db.Close()

	store = sessions.NewCookieStore(authKey)

	pool = newPool()
	clients = make(map[int64]*Client)

	http.HandleFunc("/", mainHandle)

	http.HandleFunc("/login", login)

	http.HandleFunc("/message/check", checkMessage)
	http.HandleFunc("/message/send", sendMessage)

	http.HandleFunc("/question/new", newQuestion)
	http.HandleFunc("/question/submit", testCode)

	http.HandleFunc("/chatroom/join", joinChatRoom)
	http.HandleFunc("/chatroom/leave", leaveChatRoom)

	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir("/home/suitup/hackmit/assets/"))))
	http.ListenAndServe(":8080", nil)
}

func mainHandle(w http.ResponseWriter, r *http.Request) {
	templates.ExecuteTemplate(w, "index.html", nil)
}

func joinChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("join ", uid)

	retChan := make(chan *Room)
	client := &Client{
		id:      uid,
		in:      nil,
		out:     make(chan string, MESSAGE_QUEUE_SIZE),
		retChan: retChan,
	}

	clients[uid] = client
	pool.in <- client

	chatroom := <- retChan

	fmt.Println("joinChatRoom-chatroom.id: ", chatroom.id)

	qjs, err := json.Marshal(chatroom.Question)

	fmt.Println("qjs: ", qjs)

	fmt.Fprint(w, "{\"status\":\"success\",\"crid\":\"", asciify(chatroom.id), "\",\"question\":", qjs, "}")
}

func asciify(ba []byte) string {
	ret := make([]byte, len(ba))
	for i, b := range ba {
		ret[i] = (b % 26) + 97
	}
	return string(ret)
}

func testCode(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	code := r.FormValue("submission")
	_ = r.FormValue("cvid")
	qid := string(1)
	app := "./secure.sh"
	cmd := exec.Command(app, qid, code)
    var b bytes.Buffer
    cmd.Stdout = &b
    err := cmd.Run()
    if err != nil {
    	fmt.Println(err)
    }
    fmt.Fprint(w, "{\"status\":\"success\", \"data\": ", b.String(), "}" )
    return
}

func leaveChatRoom(w http.ResponseWriter, r *http.Request) {
	uid, _ := UIDFromSession(w, r)
	client := clients[uid]

	if client != nil {
		fmt.Println("leaving")
		delete(clients, client.otherid)
		delete(clients, uid)
	}

	fmt.Println("leave ", uid)

	fmt.Fprint(w, "{\"status\":\"success\"}")
}

func sendMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("send ", uid)

	r.ParseForm()
	message := r.FormValue("s")

	client := clients[uid]

	if client != nil {
		client.out <- message
		fmt.Fprint(w, "{\"status\":\"success\"}")
	} else {
		fmt.Fprint(w, STATUS_FAILURE)
	}
}

func checkMessage(w http.ResponseWriter, r *http.Request) {
	uid, err := UIDFromSession(w, r)
	handleError(err)

	fmt.Println("check ", uid)

	client := clients[uid]

	if client != nil {
		select {
		case message, ok := <- client.in:
			if ok {
				fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"", message, "\"}")
			} else {
				delete(clients, uid)
				fmt.Fprint(w, "{\"status\":\"failure\"}")
			}
		default:
			fmt.Fprint(w, "{\"status\":\"success\",\"s\":\"\"}")
		}

	} else {
		fmt.Fprint(w, "{\"status\":\"failure\",\"s\":\"\"}")
	}
}

type User struct {
    Id            	int64      	`json:"id"`
    FacebookId  	string		`json:fbid"`
    Username 		string		`json:username"`
    Email 			string		`json:email"`
    Level 			int64		`json:lvl"`
    Score 			int64		`json:score"`
}

func newQuestion(w http.ResponseWriter, r *http.Request) {
	row := db.QueryRow("SELECT * FROM questions ORDER BY RAND()")
	q := new(Question)
	err := row.Scan(&q.Id, &q.Title, &q.Body, &q.Difficulty)

	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	b, err := json.Marshal(q)
	if err != nil {
		fmt.Println(err)
		fmt.Fprint(w, STATUS_FAILURE)
	}

	fmt.Fprint(w, string(b))
}

func login(w http.ResponseWriter, r *http.Request) {
	inputToken := r.FormValue("access_token")
	if len(inputToken) != 0 {
		uid := GetMe(inputToken)

		// row := db.QueryRow("SELECT id FROM users")
		row := db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
		user := new(User)
		err := row.Scan(&user.Id)

		if err != nil {
			_, err = db.Exec("INSERT INTO users (facebook_id, username, email, level, points) VALUES (?, ?, ?, 0, 0)", uid, "", "")
			if err != nil {
				fmt.Fprint(w, STATUS_FAILURE)
				return
			} else {
				row = db.QueryRow("SELECT id FROM users WHERE facebook_id=?", string(uid))
				err = row.Scan(&user.Id)
				if err != nil {
					fmt.Fprint(w, STATUS_FAILURE)
					return
				}
			}

		}

		session, _ := store.Get(r, "session")
		session.Values["userid"] = user.Id
		session.Save(r, w)

		fmt.Fprint(w, "{\"status\":\"success\"}")
	}
}

func readHttpBody(response *http.Response) string {
	bodyBuffer := make([]byte, 1000)
	var str string

	count, err := response.Body.Read(bodyBuffer)

	for ; count > 0; count, err = response.Body.Read(bodyBuffer) {

		if err != nil {

		}

		str += string(bodyBuffer[:count])
	}

	return str

}

func getUncachedResponse(uri string) (*http.Response, error) {
	request, err := http.NewRequest("GET", uri, nil)

	if err == nil {
		request.Header.Add("Cache-Control", "no-cache")

		client := new(http.Client)

		return client.Do(request)
	}

	if (err != nil) {
	}
	return nil, err

}

func GetMe(token string) string {
	response, err := getUncachedResponse("https://graph.facebook.com/me?access_token="+token)

	if err == nil {

		var jsonBlob interface{}

		responseBody := readHttpBody(response)

		if responseBody != "" {
			err = json.Unmarshal([]byte(responseBody), &jsonBlob)

			if err == nil {
				jsonObj := jsonBlob.(map[string]interface{})
				return jsonObj["id"].(string)
			}
		}
		return err.Error()
	}

	return err.Error()
}

func handleError(err error) {
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>// Package shor implements parsing and querying of the shor language, a simple hierarchical
// ordered representation for data.
package shor

import (
	"io"
	"strconv"
	"strings"

	"github.com/skelterjohn/gopp"
)

const grammar = `
ignore: /^#.*/
ignore: /^\s+/

Doc => {field=Key} {/} <List>
List => {field=Type} {list} {field=Kids} <<Node>>*
Node => {field=Key} <id> ':' <Value>
Node => <Value>
Value => '{' <List> '}'
Value => {field=Type} {num} {field=Val} <num>
Value => {field=Type} {bool} {field=Val} <bool>
Value => {field=Type} {str} {field=Val} <str>

num = /([-+]?\d*\.?\d+([eE][-+]?\d+)?)/
bool = /(true|false)/
str = /"((?:[^"\\]|\\.)*)"/
id = /([\pL][\pL\pN\-_]*)/
`

var (
	decFact *gopp.DecoderFactory
)

func init() {
	var err error
	decFact, err = gopp.NewDecoderFactory(grammar, "Doc")
	if err != nil {
		panic(err)
	}
}

// Node holds a shor node and its children.
type Node struct {
	Key    string // id if keyed, empty if keyless, or "/" if root
	Val    string // string representation of value, empty for list
	Type   string // str, num, bool, or list
	Kids   []*Node
	Parent *Node // nil if root node
}

// linkNodes updates parent links for entire tree.
func (n *Node) linkNodes(par *Node) {
	n.Parent = par
	for i := range n.Kids {
		n.Kids[i].linkNodes(n)
	}
}

// Append adds kid at the end of the node's list of children.
func (n *Node) Append(kid *Node) {
	n.Kids = append(n.Kids, kid)
	kid.Parent = n
}

// Insert places kid in the node's list of children at the given position.
func (n *Node) Insert(kid *Node, pos int) bool {
	if pos < 0 || pos > len(n.Kids) {
		return false
	}
	n.Kids = append(n.Kids[:pos], append([]*Node{kid}, n.Kids[pos:]...)...)
	kid.Parent = n
	return true
}

// Remove removes kide from the node's list of children.
func (n *Node) Remove(kid *Node) bool {
	for i, k := range n.Kids {
		if k == kid {
			k.Parent = nil
			n.Kids = append(n.Kids[:i], n.Kids[i:]...)
			return true
		}
	}
	return false
}

// String returns a tree in single-line format.
func (n *Node) String() string {
	return n.Format(-1, "")
}

// Format returns a formatted string representation of a tree.
// If depth is negative, single-line format is produced.
func (n *Node) Format(depth int, indent string) string {
	isRoot := (n.Key == "/")
	isMultiline := (depth >= 0)
	ind, keySep, eltSep, listSep := "", "", " ", ""
	if isMultiline {
		ind = strings.Repeat(indent, depth)
		keySep, eltSep, listSep = " ", "\n", "\n"
	}

	s := ""
	switch n.Type {
	case "list":
		kidDepth := depth
		if !isRoot && isMultiline {
			kidDepth++
		}
		for i := range n.Kids {
			s += n.Kids[i].Format(kidDepth, indent)
			if isMultiline || i < len(n.Kids)-1 {
				s += eltSep
			}
		}
		if !isRoot {
			s = "{" + listSep + s + ind + "}"
		}
	case "str":
		s = strconv.Quote(n.Val)
	default:
		s = n.Val
	}
	if n.Key != "" && !isRoot {
		s = n.Key + ":" + keySep + s
	}
	if !isRoot {
		s = ind + s
	}
	return s
}

// Query returns a Query object populated with n.
func (n *Node) Query() Query {
	return Query{n}
}

// Parse parses input and returns a tree of parsed shor nodes.
func Parse(in io.Reader) (*Node, error) {
	dec := decFact.NewDecoder(in)
	tree := &Node{}
	err := dec.Decode(tree)
	if err != nil {
		return nil, err
	}
	tree.linkNodes(nil)
	return tree, nil
}
<commit_msg>Use int for node type instead of string.<commit_after>// Package shor implements parsing and querying of the shor language, a simple hierarchical
// ordered representation for data.
package shor

import (
	"fmt"
	"io"
	"strconv"
	"strings"

	"github.com/skelterjohn/gopp"
)

const (
	Num = iota
	Bool
	Str
	List
)

var decFact *gopp.DecoderFactory

var grammar = fmt.Sprintf(`
ignore: /^#.*/
ignore: /^\s+/

Doc => {field=Key} {/} <List>
List => {field=Type} {%d} {field=Kids} <<Node>>*
Node => {field=Key} <id> ':' <Value>
Node => <Value>
Value => '{' <List> '}'
Value => {field=Type} {%d} {field=Val} <num>
Value => {field=Type} {%d} {field=Val} <bool>
Value => {field=Type} {%d} {field=Val} <str>

num = /([-+]?\d*\.?\d+([eE][-+]?\d+)?)/
bool = /(true|false)/
str = /"((?:[^"\\]|\\.)*)"/
id = /([\pL][\pL\pN\-_]*)/`, List, Num, Bool, Str)

func init() {
	var err error
	decFact, err = gopp.NewDecoderFactory(grammar, "Doc")
	if err != nil {
		panic(err)
	}
}

// Node holds a shor node and its children.
type Node struct {
	Key    string // id if keyed, empty if keyless, or "/" if root
	Val    string // string representation of value, empty for list
	Type   int    // str, num, bool, or list
	Parent *Node  // nil if root node
	Kids   []*Node
}

// linkNodes updates parent links for entire tree.
func (n *Node) linkNodes(par *Node) {
	n.Parent = par
	for i := range n.Kids {
		n.Kids[i].linkNodes(n)
	}
}

// Append adds kid at the end of the node's list of children.
func (n *Node) Append(kid *Node) {
	n.Kids = append(n.Kids, kid)
	kid.Parent = n
}

// Insert places kid in the node's list of children at the given position.
func (n *Node) Insert(kid *Node, pos int) bool {
	if pos < 0 || pos > len(n.Kids) {
		return false
	}
	n.Kids = append(n.Kids[:pos], append([]*Node{kid}, n.Kids[pos:]...)...)
	kid.Parent = n
	return true
}

// Remove removes kide from the node's list of children.
func (n *Node) Remove(kid *Node) bool {
	for i, k := range n.Kids {
		if k == kid {
			k.Parent = nil
			n.Kids = append(n.Kids[:i], n.Kids[i:]...)
			return true
		}
	}
	return false
}

// String returns a tree in single-line format.
func (n *Node) String() string {
	return n.Format(-1, "")
}

// Format returns a formatted string representation of a tree.
// If depth is negative, single-line format is produced.
func (n *Node) Format(depth int, indent string) string {
	isRoot := (n.Key == "/")
	isMultiline := (depth >= 0)
	ind, keySep, eltSep, listSep := "", "", " ", ""
	if isMultiline {
		ind = strings.Repeat(indent, depth)
		keySep, eltSep, listSep = " ", "\n", "\n"
	}

	s := ""
	switch n.Type {
	case List:
		kidDepth := depth
		if !isRoot && isMultiline {
			kidDepth++
		}
		for i := range n.Kids {
			s += n.Kids[i].Format(kidDepth, indent)
			if isMultiline || i < len(n.Kids)-1 {
				s += eltSep
			}
		}
		if !isRoot {
			s = "{" + listSep + s + ind + "}"
		}
	case Str:
		s = strconv.Quote(n.Val)
	default:
		s = n.Val
	}
	if n.Key != "" && !isRoot {
		s = n.Key + ":" + keySep + s
	}
	if !isRoot {
		s = ind + s
	}
	return s
}

// Query returns a Query object populated with n.
func (n *Node) Query() Query {
	return Query{n}
}

// Parse parses input and returns a tree of parsed shor nodes.
func Parse(in io.Reader) (*Node, error) {
	dec := decFact.NewDecoder(in)
	tree := &Node{}
	err := dec.Decode(tree)
	if err != nil {
		return nil, err
	}
	tree.linkNodes(nil)
	return tree, nil
}
<|endoftext|>


<commit_before>package routers

import (
	"github.com/astaxie/beego"
	"github.com/shwinpiocess/cc/controllers"
)

func init() {
	beego.Router("/", &controllers.MainController{}, "*:Index")
	beego.Router("/login", &controllers.MainController{}, "*:Login")
	beego.Router("/logout", &controllers.MainController{}, "*:Logout")

	// 业务管理
	beego.Router("/app/index", &controllers.AppController{}, "*:Index")
	beego.Router("/app/newapp", &controllers.AppController{}, "*:NewApp")
	beego.Router("/app/add", &controllers.AppController{}, "*:AddApp")
	beego.Router("/app/delete", &controllers.AppController{}, "*:DeleteApp")
	beego.Router("/welcome/setDefaultApp", &controllers.AppController{}, "*:SetDefaultApp")

	beego.Router("/topology/index", &controllers.AppController{}, "*:TopologyIndex")

	// 快速分配
	beego.Router("/host/quickImport", &controllers.AppController{}, "*:QuickImport")
}
<commit_msg>新增集群控制路由<commit_after>package routers

import (
	"github.com/astaxie/beego"
	"github.com/shwinpiocess/cc/controllers"
)

func init() {
	beego.Router("/", &controllers.MainController{}, "*:Index")
	beego.Router("/login", &controllers.MainController{}, "*:Login")
	beego.Router("/logout", &controllers.MainController{}, "*:Logout")

	// 业务管理
	beego.Router("/app/index", &controllers.AppController{}, "*:Index")
	beego.Router("/app/newapp", &controllers.AppController{}, "*:NewApp")
	beego.Router("/app/add", &controllers.AppController{}, "*:AddApp")
	beego.Router("/app/delete", &controllers.AppController{}, "*:DeleteApp")
	beego.Router("/welcome/setDefaultApp", &controllers.AppController{}, "*:SetDefaultApp")

	beego.Router("/topology/index", &controllers.AppController{}, "*:TopologyIndex")

	beego.Router("/Set/getAllSetInfo", &controllers.SetController{}, "*:GetAllSetInfo")
	beego.Router("/Set/newSet", &controllers.SetController{}, "*:NewSet")

	// 快速分配
	beego.Router("/host/quickImport", &controllers.AppController{}, "*:QuickImport")
}
<|endoftext|>


<commit_before>// Copyright 2017 Corey Scott http://www.sage42.org/
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generator

import (
	"bufio"
	"fmt"
	"go/parser"
	"go/token"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/corsc/go-commons/iocloser"
	"github.com/corsc/go-tools/package-coverage/utils"
)

const coverageFilename = "profile.cov"

var fakeTestFilename = "fake_test.go"

func processAllDirs(basePath string, exclusionsMatcher *regexp.Regexp, logTag string, actionFunc func(string)) {
	paths, err := utils.FindAllGoDirs(basePath)
	if err != nil {
		return
	}

	for _, path := range paths {
		if exclusionsMatcher.FindString(path) != "" {
			utils.LogWhenVerbose("[%s] path '%s' skipped due to skipDir regex '%s'",
				logTag, path, exclusionsMatcher.String())
			continue
		}

		utils.LogWhenVerbose("[%s] processing path '%s'", logTag, path)
		actionFunc(path)
	}
}

// this function will generate the test coverage for the supplied directory
func generateCoverage(path string, exclusions *regexp.Regexp, quietMode bool, tags string) {
	packageName := findPackageName(path)

	fakeTestFile := addFakeTest(path, packageName)
	defer removeFakeTest(fakeTestFile)

	err := execCoverage(path, quietMode, tags)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error generating coverage %s", err)
	}

	if exclusions == nil {
		return
	}

	err = filterCoverage(filepath.Join(path, coverageFilename), exclusions)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error filtering files: %s", err)
	}
}

// add a fake test to ensure that there is at least 1 test in this directory
func addFakeTest(path string, packageName string) string {
	testFilename := createTestFilename(path)

	createTestFile(packageName, testFilename)

	return testFilename
}

func createTestFilename(path string) string {
	return path + fakeTestFilename
}

// find the package name by using the go AST
func findPackageName(path string) string {
	fileSet := token.NewFileSet()
	pkgs, err := parser.ParseDir(fileSet, path, nil, 0)
	if err != nil {
		utils.LogWhenVerbose("[coverage] err while parsing the '%s' into Go AST Err: '%s", path, err)
		return UnknownPackage
	}

	for pkgName := range pkgs {
		return pkgName
	}
	return UnknownPackage
}

// create a fake test so that all directories are guaranteed to contain tests (and therefore coverage will be generated)
func createTestFile(packageName string, testFilename string) {
	utils.LogWhenVerbose("[coverage] created test for package %s file @ %s", packageName, testFilename)

	if _, err := os.Stat(testFilename); err == nil {
		utils.LogWhenVerbose("[coverage] file already exists @ %s cowardly refusing to overwrite", testFilename)
		return
	}

	file, err := os.OpenFile(testFilename, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while creating test file %s", err)
		return
	}

	_, err = file.WriteString(`package ` + packageName + `

import "testing"

func TestThisTestDoesntReallyTestAnything(t *testing.T) {}
`)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while writing test file %s", err)
		return
	}

	err = file.Close()
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while closing file '%s", err)
	}
}

// remove the previously added fake test (i.e. clean up)
func removeFakeTest(filename string) {
	utils.LogWhenVerbose("[coverage] remove test file @ %s", filename)

	err := os.Remove(filename)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while removing test file @ %s, err: %s", filename, err)
	}
}

// essentially call `go test` to generate the coverage
func execCoverage(dir string, quiet bool, tags string) error {
	arguments := []string{
		"test",
		"-coverprofile=" + coverageFilename,
	}

	if len(tags) > 0 {
		arguments = append(arguments, `-tags="`+tags+`"`)
	}

	cmd := exec.Command("go", arguments...)
	cmd.Dir = dir

	if !quiet {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	err := cmd.Run()
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while running go test. err: %s", err)
		utils.LogWhenVerbose("[coverage] error while running go test. err: %s", cmd.Stderr)
		return err
	}

	utils.LogWhenVerbose("[coverage] created coverage file @ %s%s", dir, coverageFilename)
	utils.LogWhenVerbose("[coverage] test output %s", cmd.Stdout)
	return nil
}

func filterCoverage(coverageFilename string, exclusionsMatcher *regexp.Regexp) error {
	coverageTempFilename := coverageFilename + "~"

	coverageTempFile, err := os.OpenFile(coverageTempFilename, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while opening file %s, err: %s", coverageTempFilename, err)
		return err
	}

	defer iocloser.Close(coverageTempFile)

	coverageFile, err := os.OpenFile(coverageFilename, os.O_RDWR, 0)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while opening file %s, err: %s", coverageFilename, err)
		return err
	}

	defer func() {
		if err := coverageFile.Close(); err != nil {
			utils.LogWhenVerbose("[coverage] Cannot close coverage file %s: %s", coverageFilename, err)
		}
	}()

	if err := filterCoverageContents(exclusionsMatcher, coverageFile, coverageTempFile); err != nil {
		utils.LogWhenVerbose("[coverage] error while filtering coverage file %s: %s", coverageFilename, err)
		return err
	}

	if err := os.Remove(coverageFilename); err != nil {
		utils.LogWhenVerbose("[coverage] cannot remove old coverage file %s: %s", coverageFilename, err)
	}

	if err := os.Rename(coverageTempFilename, coverageFilename); err != nil {
		utils.LogWhenVerbose("[coverage] cannot rename filtered coverage file %s: %s", coverageTempFilename, err)
	}

	return nil
}

func filterCoverageContents(exclusionsMatcher *regexp.Regexp, in io.Reader, out io.Writer) error {
	coverageFileScanner := bufio.NewScanner(in)
	for coverageFileScanner.Scan() {
		line := coverageFileScanner.Text()

		fileNameEndIndex := strings.LastIndex(line, ":")
		if fileNameEndIndex == -1 {
			utils.LogWhenVerbose("[coverage] error in line '%s'", line)
			continue
		}

		fileName := line[:fileNameEndIndex]

		if exclusionsMatcher.MatchString(fileName) {
			utils.LogWhenVerbose("[coverage] skipped file %s", fileName)
			continue
		}

		if _, err := fmt.Fprintln(out, line); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Turn on Go modules and add more debugging to test failures<commit_after>// Copyright 2017 Corey Scott http://www.sage42.org/
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generator

import (
	"bufio"
	"fmt"
	"go/parser"
	"go/token"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/corsc/go-commons/iocloser"
	"github.com/corsc/go-tools/package-coverage/utils"
)

const coverageFilename = "profile.cov"

var fakeTestFilename = "fake_test.go"

func processAllDirs(basePath string, exclusionsMatcher *regexp.Regexp, logTag string, actionFunc func(string)) {
	paths, err := utils.FindAllGoDirs(basePath)
	if err != nil {
		return
	}

	for _, path := range paths {
		if exclusionsMatcher.FindString(path) != "" {
			utils.LogWhenVerbose("[%s] path '%s' skipped due to skipDir regex '%s'",
				logTag, path, exclusionsMatcher.String())
			continue
		}

		utils.LogWhenVerbose("[%s] processing path '%s'", logTag, path)
		actionFunc(path)
	}
}

// this function will generate the test coverage for the supplied directory
func generateCoverage(path string, exclusions *regexp.Regexp, quietMode bool, tags string) {
	packageName := findPackageName(path)

	fakeTestFile := addFakeTest(path, packageName)
	defer removeFakeTest(fakeTestFile)

	err := execCoverage(path, quietMode, tags)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error generating coverage %s", err)
	}

	if exclusions == nil {
		return
	}

	err = filterCoverage(filepath.Join(path, coverageFilename), exclusions)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error filtering files: %s", err)
	}
}

// add a fake test to ensure that there is at least 1 test in this directory
func addFakeTest(path string, packageName string) string {
	testFilename := createTestFilename(path)

	createTestFile(packageName, testFilename)

	return testFilename
}

func createTestFilename(path string) string {
	return path + fakeTestFilename
}

// find the package name by using the go AST
func findPackageName(path string) string {
	fileSet := token.NewFileSet()
	pkgs, err := parser.ParseDir(fileSet, path, nil, 0)
	if err != nil {
		utils.LogWhenVerbose("[coverage] err while parsing the '%s' into Go AST Err: '%s", path, err)
		return UnknownPackage
	}

	for pkgName := range pkgs {
		return pkgName
	}
	return UnknownPackage
}

// create a fake test so that all directories are guaranteed to contain tests (and therefore coverage will be generated)
func createTestFile(packageName string, testFilename string) {
	utils.LogWhenVerbose("[coverage] created test for package %s file @ %s", packageName, testFilename)

	if _, err := os.Stat(testFilename); err == nil {
		utils.LogWhenVerbose("[coverage] file already exists @ %s cowardly refusing to overwrite", testFilename)
		return
	}

	file, err := os.OpenFile(testFilename, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while creating test file %s", err)
		return
	}

	_, err = file.WriteString(`package ` + packageName + `

import "testing"

func TestThisTestDoesntReallyTestAnything(t *testing.T) {}
`)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while writing test file %s", err)
		return
	}

	err = file.Close()
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while closing file '%s", err)
	}
}

// remove the previously added fake test (i.e. clean up)
func removeFakeTest(filename string) {
	utils.LogWhenVerbose("[coverage] remove test file @ %s", filename)

	err := os.Remove(filename)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while removing test file @ %s, err: %s", filename, err)
	}
}

// essentially call `go test` to generate the coverage
func execCoverage(dir string, quiet bool, tags string) error {
	arguments := []string{
		"test",
		"-coverprofile=" + coverageFilename,
	}

	if len(tags) > 0 {
		arguments = append(arguments, `-tags="`+tags+`"`)
	}

	cmd := exec.Command("go", arguments...)
	cmd.Dir = dir

	if !quiet {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	err := cmd.Run()
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while running go test. err: %s", err)
		utils.LogWhenVerbose("[coverage] test output.\n%s", cmd.Stdout)
		utils.LogWhenVerbose("[coverage] test output error\n%s", cmd.Stderr)
		return err
	}

	utils.LogWhenVerbose("[coverage] created coverage file @ %s%s", dir, coverageFilename)
	utils.LogWhenVerbose("[coverage] test output.\n%s", cmd.Stdout)
	return nil
}

func filterCoverage(coverageFilename string, exclusionsMatcher *regexp.Regexp) error {
	coverageTempFilename := coverageFilename + "~"

	coverageTempFile, err := os.OpenFile(coverageTempFilename, os.O_RDWR|os.O_CREATE, 0600)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while opening file %s, err: %s", coverageTempFilename, err)
		return err
	}

	defer iocloser.Close(coverageTempFile)

	coverageFile, err := os.OpenFile(coverageFilename, os.O_RDWR, 0)
	if err != nil {
		utils.LogWhenVerbose("[coverage] error while opening file %s, err: %s", coverageFilename, err)
		return err
	}

	defer func() {
		if err := coverageFile.Close(); err != nil {
			utils.LogWhenVerbose("[coverage] Cannot close coverage file %s: %s", coverageFilename, err)
		}
	}()

	if err := filterCoverageContents(exclusionsMatcher, coverageFile, coverageTempFile); err != nil {
		utils.LogWhenVerbose("[coverage] error while filtering coverage file %s: %s", coverageFilename, err)
		return err
	}

	if err := os.Remove(coverageFilename); err != nil {
		utils.LogWhenVerbose("[coverage] cannot remove old coverage file %s: %s", coverageFilename, err)
	}

	if err := os.Rename(coverageTempFilename, coverageFilename); err != nil {
		utils.LogWhenVerbose("[coverage] cannot rename filtered coverage file %s: %s", coverageTempFilename, err)
	}

	return nil
}

func filterCoverageContents(exclusionsMatcher *regexp.Regexp, in io.Reader, out io.Writer) error {
	coverageFileScanner := bufio.NewScanner(in)
	for coverageFileScanner.Scan() {
		line := coverageFileScanner.Text()

		fileNameEndIndex := strings.LastIndex(line, ":")
		if fileNameEndIndex == -1 {
			utils.LogWhenVerbose("[coverage] error in line '%s'", line)
			continue
		}

		fileName := line[:fileNameEndIndex]

		if exclusionsMatcher.MatchString(fileName) {
			utils.LogWhenVerbose("[coverage] skipped file %s", fileName)
			continue
		}

		if _, err := fmt.Fprintln(out, line); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>