<commit_before>package vsolver

import "sort"

// smAdapter is an adapter and around a proper SourceManager.
//
// It provides localized caching that's tailored to the requirements of a
// particular solve run.
//
// It also performs transformations between ProjectIdentifiers, which is what
// the solver primarily deals in, and ProjectName, which is what the
// SourceManager primarily deals in. This separation is helpful because it keeps
// the complexities of deciding what a particular name "means" entirely within
// the solver, while the SourceManager can traffic exclusively in
// globally-unique network names.
type smAdapter struct {
	// The underlying, adapted-to SourceManager
	sm SourceManager
	// Direction to sort the version list. False indicates sorting for upgrades;
	// true for downgrades.
	sortdown bool
	// Map of project root name to their available version list. This cache is
	// layered on top of the proper SourceManager's cache; the only difference
	// is that this keeps the versions sorted in the direction required by the
	// current solve run
	vlists map[ProjectName][]Version
}

func (c *smAdapter) getProjectInfo(pa ProjectAtom) (ProjectInfo, error) {
	return c.sm.GetProjectInfo(ProjectName(pa.Ident.netName()), pa.Version)
}

func (c *smAdapter) key(id ProjectIdentifier) ProjectName {
	k := ProjectName(id.NetworkName)
	if k == "" {
		k = id.LocalName
	}

	return k
}

func (c *smAdapter) listVersions(id ProjectIdentifier) ([]Version, error) {
	k := c.key(id)

	if vl, exists := c.vlists[k]; exists {
		return vl, nil
	}

	vl, err := c.sm.ListVersions(k)
	// TODO cache errors, too?
	if err != nil {
		return nil, err
	}

	if c.sortdown {
		sort.Sort(downgradeVersionSorter(vl))
	} else {
		sort.Sort(upgradeVersionSorter(vl))
	}

	c.vlists[k] = vl
	return vl, nil
}

func (c *smAdapter) repoExists(id ProjectIdentifier) (bool, error) {
	k := c.key(id)
	return c.sm.RepoExists(k)
}

func (c *smAdapter) vendorCodeExists(id ProjectIdentifier) (bool, error) {
	k := c.key(id)
	return c.sm.VendorCodeExists(k)
}

func (c *smAdapter) pairVersion(id ProjectIdentifier, v UnpairedVersion) PairedVersion {
	vl, err := c.listVersions(id)
	if err != nil {
		return nil
	}

	// doing it like this is a bit sloppy
	for _, v2 := range vl {
		if p, ok := v2.(PairedVersion); ok {
			if p.Matches(v) {
				return p
			}
		}
	}

	return nil
}

func (c *smAdapter) pairRevision(id ProjectIdentifier, r Revision) []Version {
	vl, err := c.listVersions(id)
	if err != nil {
		return nil
	}

	p := []Version{r}
	// doing it like this is a bit sloppy
	for _, v2 := range vl {
		if pv, ok := v2.(PairedVersion); ok {
			if pv.Matches(r) {
				p = append(p, pv)
			}
		}
	}

	return p
}

type upgradeVersionSorter []Version
type downgradeVersionSorter []Version

func (vs upgradeVersionSorter) Len() int {
	return len(vs)
}

func (vs upgradeVersionSorter) Swap(i, j int) {
	vs[i], vs[j] = vs[j], vs[i]
}

func (vs downgradeVersionSorter) Len() int {
	return len(vs)
}

func (vs downgradeVersionSorter) Swap(i, j int) {
	vs[i], vs[j] = vs[j], vs[i]
}

func (vs upgradeVersionSorter) Less(i, j int) bool {
	l, r := vs[i], vs[j]

	if tl, ispair := l.(versionPair); ispair {
		l = tl.v
	}
	if tr, ispair := r.(versionPair); ispair {
		r = tr.v
	}

	switch compareVersionType(l, r) {
	case -1:
		return true
	case 1:
		return false
	case 0:
		break
	default:
		panic("unreachable")
	}

	switch l.(type) {
	// For these, now nothing to do but alpha sort
	case Revision, branchVersion, plainVersion:
		return l.String() < r.String()
	}

	// This ensures that pre-release versions are always sorted after ALL
	// full-release versions
	lsv, rsv := l.(semVersion).sv, r.(semVersion).sv
	lpre, rpre := lsv.Prerelease() == "", rsv.Prerelease() == ""
	if (lpre && !rpre) || (!lpre && rpre) {
		return lpre
	}
	return lsv.GreaterThan(rsv)
}

func (vs downgradeVersionSorter) Less(i, j int) bool {
	l, r := vs[i], vs[j]

	if tl, ispair := l.(versionPair); ispair {
		l = tl.v
	}
	if tr, ispair := r.(versionPair); ispair {
		r = tr.v
	}

	switch compareVersionType(l, r) {
	case -1:
		return true
	case 1:
		return false
	case 0:
		break
	default:
		panic("unreachable")
	}

	switch l.(type) {
	// For these, now nothing to do but alpha
	case Revision, branchVersion, plainVersion:
		return l.String() < r.String()
	}

	// This ensures that pre-release versions are always sorted after ALL
	// full-release versions
	lsv, rsv := l.(semVersion).sv, r.(semVersion).sv
	lpre, rpre := lsv.Prerelease() == "", rsv.Prerelease() == ""
	if (lpre && !rpre) || (!lpre && rpre) {
		return lpre
	}
	return lsv.LessThan(rsv)
}
<commit_msg>Impl authoritative matches() method<commit_after>package vsolver

import "sort"

// smAdapter is an adapter and around a proper SourceManager.
//
// It provides localized caching that's tailored to the requirements of a
// particular solve run.
//
// It also performs transformations between ProjectIdentifiers, which is what
// the solver primarily deals in, and ProjectName, which is what the
// SourceManager primarily deals in. This separation is helpful because it keeps
// the complexities of deciding what a particular name "means" entirely within
// the solver, while the SourceManager can traffic exclusively in
// globally-unique network names.
//
// Finally, it provides authoritative version/constraint operations, ensuring
// that any possible approach to a match - even those not literally encoded in
// the inputs - is achieved.
type smAdapter struct {
	// The underlying, adapted-to SourceManager
	sm SourceManager
	// Direction to sort the version list. False indicates sorting for upgrades;
	// true for downgrades.
	sortdown bool
	// Map of project root name to their available version list. This cache is
	// layered on top of the proper SourceManager's cache; the only difference
	// is that this keeps the versions sorted in the direction required by the
	// current solve run
	vlists map[ProjectName][]Version
}

func (c *smAdapter) getProjectInfo(pa ProjectAtom) (ProjectInfo, error) {
	return c.sm.GetProjectInfo(ProjectName(pa.Ident.netName()), pa.Version)
}

func (c *smAdapter) key(id ProjectIdentifier) ProjectName {
	k := ProjectName(id.NetworkName)
	if k == "" {
		k = id.LocalName
	}

	return k
}

func (c *smAdapter) listVersions(id ProjectIdentifier) ([]Version, error) {
	k := c.key(id)

	if vl, exists := c.vlists[k]; exists {
		return vl, nil
	}

	vl, err := c.sm.ListVersions(k)
	// TODO cache errors, too?
	if err != nil {
		return nil, err
	}

	if c.sortdown {
		sort.Sort(downgradeVersionSorter(vl))
	} else {
		sort.Sort(upgradeVersionSorter(vl))
	}

	c.vlists[k] = vl
	return vl, nil
}

func (c *smAdapter) repoExists(id ProjectIdentifier) (bool, error) {
	k := c.key(id)
	return c.sm.RepoExists(k)
}

func (c *smAdapter) vendorCodeExists(id ProjectIdentifier) (bool, error) {
	k := c.key(id)
	return c.sm.VendorCodeExists(k)
}

func (c *smAdapter) pairVersion(id ProjectIdentifier, v UnpairedVersion) PairedVersion {
	vl, err := c.listVersions(id)
	if err != nil {
		return nil
	}

	// doing it like this is a bit sloppy
	for _, v2 := range vl {
		if p, ok := v2.(PairedVersion); ok {
			if p.Matches(v) {
				return p
			}
		}
	}

	return nil
}

func (c *smAdapter) pairRevision(id ProjectIdentifier, r Revision) []Version {
	vl, err := c.listVersions(id)
	if err != nil {
		return nil
	}

	p := []Version{r}
	// doing it like this is a bit sloppy
	for _, v2 := range vl {
		if pv, ok := v2.(PairedVersion); ok {
			if pv.Matches(r) {
				p = append(p, pv)
			}
		}
	}

	return p
}

// matches performs a typical match check between the provided version and
// constraint. If that basic check fails and the provided version is incomplete
// (e.g. an unpaired version or bare revision), it will attempt to gather more
// information on one or the other and re-perform the comparison.
func (c *smAdapter) matches(id ProjectIdentifier, c2 Constraint, v Version) bool {
	if c2.Matches(v) {
		return true
	}

	// There's a wide field of possible ways that pairing might result in a
	// match. For each possible type of version, start by carving out all the
	// cases where the constraint would have provided an authoritative match
	// result.
	switch tv := v.(type) {
	case PairedVersion:
		switch tc := c2.(type) {
		case PairedVersion, Revision, noneConstraint:
			// These three would all have been authoritative matches
			return false
		case UnpairedVersion:
			// Only way paired and unpaired could match is if they share an
			// underlying rev
			pv := c.pairVersion(id, tc)
			if pv == nil {
				return false
			}
			return pv.Matches(v)
		case semverConstraint:
			// Have to check all the possible versions for that rev to see if
			// any match the semver constraint
			for _, pv := range c.pairRevision(id, tv.Underlying()) {
				if tc.Matches(pv) {
					return true
				}
			}
			return false
		}

	case Revision:
		switch tc := c2.(type) {
		case PairedVersion, Revision, noneConstraint:
			// These three would all have been authoritative matches
			return false
		case UnpairedVersion:
			// Only way paired and unpaired could match is if they share an
			// underlying rev
			pv := c.pairVersion(id, tc)
			if pv == nil {
				return false
			}
			return pv.Matches(v)
		case semverConstraint:
			// Have to check all the possible versions for the rev to see if
			// any match the semver constraint
			for _, pv := range c.pairRevision(id, tv) {
				if tc.Matches(pv) {
					return true
				}
			}
			return false
		}

	// UnpairedVersion as input has the most weird cases. It's also the one
	// we'll probably see the least
	case UnpairedVersion:
		switch tc := c2.(type) {
		case noneConstraint:
			// obviously
			return false
		case Revision, PairedVersion:
			// Easy case for both - just pair the uv and see if it matches the revision
			// constraint
			pv := c.pairVersion(id, tv)
			if pv == nil {
				return false
			}
			return tc.Matches(pv)
		case UnpairedVersion:
			// Both are unpaired versions. See if they share an underlying rev.
			pv := c.pairVersion(id, tv)
			if pv == nil {
				return false
			}

			pc := c.pairVersion(id, tc)
			if pc == nil {
				return false
			}
			return pc.Matches(pv)

		case semverConstraint:
			// semverConstraint can't ever match a rev, but we do need to check
			// if any other versions corresponding to this rev work.
			pv := c.pairVersion(id, tv)
			if pv == nil {
				return false
			}

			for _, ttv := range c.pairRevision(id, pv.Underlying()) {
				if c2.Matches(ttv) {
					return true
				}
			}
			return false
		}
	default:
		panic("unreachable")
	}
}

type upgradeVersionSorter []Version
type downgradeVersionSorter []Version

func (vs upgradeVersionSorter) Len() int {
	return len(vs)
}

func (vs upgradeVersionSorter) Swap(i, j int) {
	vs[i], vs[j] = vs[j], vs[i]
}

func (vs downgradeVersionSorter) Len() int {
	return len(vs)
}

func (vs downgradeVersionSorter) Swap(i, j int) {
	vs[i], vs[j] = vs[j], vs[i]
}

func (vs upgradeVersionSorter) Less(i, j int) bool {
	l, r := vs[i], vs[j]

	if tl, ispair := l.(versionPair); ispair {
		l = tl.v
	}
	if tr, ispair := r.(versionPair); ispair {
		r = tr.v
	}

	switch compareVersionType(l, r) {
	case -1:
		return true
	case 1:
		return false
	case 0:
		break
	default:
		panic("unreachable")
	}

	switch l.(type) {
	// For these, now nothing to do but alpha sort
	case Revision, branchVersion, plainVersion:
		return l.String() < r.String()
	}

	// This ensures that pre-release versions are always sorted after ALL
	// full-release versions
	lsv, rsv := l.(semVersion).sv, r.(semVersion).sv
	lpre, rpre := lsv.Prerelease() == "", rsv.Prerelease() == ""
	if (lpre && !rpre) || (!lpre && rpre) {
		return lpre
	}
	return lsv.GreaterThan(rsv)
}

func (vs downgradeVersionSorter) Less(i, j int) bool {
	l, r := vs[i], vs[j]

	if tl, ispair := l.(versionPair); ispair {
		l = tl.v
	}
	if tr, ispair := r.(versionPair); ispair {
		r = tr.v
	}

	switch compareVersionType(l, r) {
	case -1:
		return true
	case 1:
		return false
	case 0:
		break
	default:
		panic("unreachable")
	}

	switch l.(type) {
	// For these, now nothing to do but alpha
	case Revision, branchVersion, plainVersion:
		return l.String() < r.String()
	}

	// This ensures that pre-release versions are always sorted after ALL
	// full-release versions
	lsv, rsv := l.(semVersion).sv, r.(semVersion).sv
	lpre, rpre := lsv.Prerelease() == "", rsv.Prerelease() == ""
	if (lpre && !rpre) || (!lpre && rpre) {
		return lpre
	}
	return lsv.LessThan(rsv)
}
<|endoftext|>


<commit_before>// +build linux

package devmapper

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"

	"github.com/Sirupsen/logrus"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/devicemapper"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/go-units"
)

func init() {
	graphdriver.Register("devicemapper", Init)
}

// Driver contains the device set mounted and the home directory
type Driver struct {
	*DeviceSet
	home    string
	uidMaps []idtools.IDMap
	gidMaps []idtools.IDMap
	ctr     *graphdriver.RefCounter
}

// Init creates a driver with the given home and the set of options.
func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
	deviceSet, err := NewDeviceSet(home, true, options, uidMaps, gidMaps)
	if err != nil {
		return nil, err
	}

	if err := mount.MakePrivate(home); err != nil {
		return nil, err
	}

	d := &Driver{
		DeviceSet: deviceSet,
		home:      home,
		uidMaps:   uidMaps,
		gidMaps:   gidMaps,
		ctr:       graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
	}

	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
}

func (d *Driver) String() string {
	return "devicemapper"
}

// Status returns the status about the driver in a printable format.
// Information returned contains Pool Name, Data File, Metadata file, disk usage by
// the data and metadata, etc.
func (d *Driver) Status() [][2]string {
	s := d.DeviceSet.Status()

	status := [][2]string{
		{"Pool Name", s.PoolName},
		{"Pool Blocksize", fmt.Sprintf("%s", units.HumanSize(float64(s.SectorSize)))},
		{"Base Device Size", fmt.Sprintf("%s", units.HumanSize(float64(s.BaseDeviceSize)))},
		{"Backing Filesystem", s.BaseDeviceFS},
		{"Data file", s.DataFile},
		{"Metadata file", s.MetadataFile},
		{"Data Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Used)))},
		{"Data Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Total)))},
		{"Data Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Available)))},
		{"Metadata Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Used)))},
		{"Metadata Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Total)))},
		{"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))},
		{"Thin Pool Minimum Free Space", fmt.Sprintf("%s", units.HumanSize(float64(s.MinFreeSpace)))},
		{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
		{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
		{"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)},
		{"Deferred Deleted Device Count", fmt.Sprintf("%v", s.DeferredDeletedDeviceCount)},
	}
	if len(s.DataLoopback) > 0 {
		status = append(status, [2]string{"Data loop file", s.DataLoopback})
	}
	if len(s.MetadataLoopback) > 0 {
		status = append(status, [2]string{"Metadata loop file", s.MetadataLoopback})
	}
	if vStr, err := devicemapper.GetLibraryVersion(); err == nil {
		status = append(status, [2]string{"Library Version", vStr})
	}
	return status
}

// GetMetadata returns a map of information about the device.
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	m, err := d.DeviceSet.exportDeviceMetadata(id)

	if err != nil {
		return nil, err
	}

	metadata := make(map[string]string)
	metadata["DeviceId"] = strconv.Itoa(m.deviceID)
	metadata["DeviceSize"] = strconv.FormatUint(m.deviceSize, 10)
	metadata["DeviceName"] = m.deviceName
	return metadata, nil
}

// Cleanup unmounts a device.
func (d *Driver) Cleanup() error {
	err := d.DeviceSet.Shutdown(d.home)

	if err2 := mount.Unmount(d.home); err == nil {
		err = err2
	}

	return err
}

// CreateReadWrite creates a layer that is writable for use as a container
// file system.
func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
	return d.Create(id, parent, mountLabel, storageOpt)
}

// Create adds a device with a given id and the parent.
func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error {
	if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil {
		return err
	}

	return nil
}

// Remove removes a device with a given id, unmounts the filesystem.
func (d *Driver) Remove(id string) error {
	if !d.DeviceSet.HasDevice(id) {
		// Consider removing a non-existing device a no-op
		// This is useful to be able to progress on container removal
		// if the underlying device has gone away due to earlier errors
		return nil
	}

	// This assumes the device has been properly Get/Put:ed and thus is unmounted
	if err := d.DeviceSet.DeleteDevice(id, false); err != nil {
		return err
	}

	mp := path.Join(d.home, "mnt", id)
	if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Get mounts a device with given id into the root filesystem
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mp := path.Join(d.home, "mnt", id)
	if count := d.ctr.Increment(mp); count > 1 {
		return mp, nil
	}

	uid, gid, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		d.ctr.Decrement(mp)
		return "", err
	}

	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(path.Join(d.home, "mnt"), 0755, uid, gid); err != nil && !os.IsExist(err) {
		d.ctr.Decrement(mp)
		return "", err
	}
	if err := idtools.MkdirAs(mp, 0755, uid, gid); err != nil && !os.IsExist(err) {
		d.ctr.Decrement(mp)
		return "", err
	}

	// Mount the device
	if err := d.DeviceSet.MountDevice(id, mp, mountLabel); err != nil {
		d.ctr.Decrement(mp)
		return "", err
	}

	rootFs := path.Join(mp, "rootfs")
	if err := idtools.MkdirAllAs(rootFs, 0755, uid, gid); err != nil && !os.IsExist(err) {
		d.ctr.Decrement(mp)
		d.DeviceSet.UnmountDevice(id, mp)
		return "", err
	}

	idFile := path.Join(mp, "id")
	if _, err := os.Stat(idFile); err != nil && os.IsNotExist(err) {
		// Create an "id" file with the container/image id in it to help reconstruct this in case
		// of later problems
		if err := ioutil.WriteFile(idFile, []byte(id), 0600); err != nil {
			d.ctr.Decrement(mp)
			d.DeviceSet.UnmountDevice(id, mp)
			return "", err
		}
	}

	return rootFs, nil
}

// Put unmounts a device and removes it.
func (d *Driver) Put(id string) error {
	mp := path.Join(d.home, "mnt", id)
	if count := d.ctr.Decrement(mp); count > 0 {
		return nil
	}
	err := d.DeviceSet.UnmountDevice(id, mp)
	if err != nil {
		logrus.Errorf("devmapper: Error unmounting device %s: %s", id, err)
	}
	return err
}

// Exists checks to see if the device exists.
func (d *Driver) Exists(id string) bool {
	return d.DeviceSet.HasDevice(id)
}
<commit_msg>Fixes Issue # 22992: docker commit failing.<commit_after>// +build linux

package devmapper

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"

	"github.com/Sirupsen/logrus"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/docker/docker/pkg/devicemapper"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/go-units"
)

func init() {
	graphdriver.Register("devicemapper", Init)
}

// Driver contains the device set mounted and the home directory
type Driver struct {
	*DeviceSet
	home    string
	uidMaps []idtools.IDMap
	gidMaps []idtools.IDMap
	ctr     *graphdriver.RefCounter
}

// Init creates a driver with the given home and the set of options.
func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
	deviceSet, err := NewDeviceSet(home, true, options, uidMaps, gidMaps)
	if err != nil {
		return nil, err
	}

	if err := mount.MakePrivate(home); err != nil {
		return nil, err
	}

	d := &Driver{
		DeviceSet: deviceSet,
		home:      home,
		uidMaps:   uidMaps,
		gidMaps:   gidMaps,
		ctr:       graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
	}

	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
}

func (d *Driver) String() string {
	return "devicemapper"
}

// Status returns the status about the driver in a printable format.
// Information returned contains Pool Name, Data File, Metadata file, disk usage by
// the data and metadata, etc.
func (d *Driver) Status() [][2]string {
	s := d.DeviceSet.Status()

	status := [][2]string{
		{"Pool Name", s.PoolName},
		{"Pool Blocksize", fmt.Sprintf("%s", units.HumanSize(float64(s.SectorSize)))},
		{"Base Device Size", fmt.Sprintf("%s", units.HumanSize(float64(s.BaseDeviceSize)))},
		{"Backing Filesystem", s.BaseDeviceFS},
		{"Data file", s.DataFile},
		{"Metadata file", s.MetadataFile},
		{"Data Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Used)))},
		{"Data Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Total)))},
		{"Data Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Available)))},
		{"Metadata Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Used)))},
		{"Metadata Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Total)))},
		{"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))},
		{"Thin Pool Minimum Free Space", fmt.Sprintf("%s", units.HumanSize(float64(s.MinFreeSpace)))},
		{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
		{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
		{"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)},
		{"Deferred Deleted Device Count", fmt.Sprintf("%v", s.DeferredDeletedDeviceCount)},
	}
	if len(s.DataLoopback) > 0 {
		status = append(status, [2]string{"Data loop file", s.DataLoopback})
	}
	if len(s.MetadataLoopback) > 0 {
		status = append(status, [2]string{"Metadata loop file", s.MetadataLoopback})
	}
	if vStr, err := devicemapper.GetLibraryVersion(); err == nil {
		status = append(status, [2]string{"Library Version", vStr})
	}
	return status
}

// GetMetadata returns a map of information about the device.
func (d *Driver) GetMetadata(id string) (map[string]string, error) {
	m, err := d.DeviceSet.exportDeviceMetadata(id)

	if err != nil {
		return nil, err
	}

	metadata := make(map[string]string)
	metadata["DeviceId"] = strconv.Itoa(m.deviceID)
	metadata["DeviceSize"] = strconv.FormatUint(m.deviceSize, 10)
	metadata["DeviceName"] = m.deviceName
	return metadata, nil
}

// Cleanup unmounts a device.
func (d *Driver) Cleanup() error {
	err := d.DeviceSet.Shutdown(d.home)

	if err2 := mount.Unmount(d.home); err == nil {
		err = err2
	}

	return err
}

// CreateReadWrite creates a layer that is writable for use as a container
// file system.
func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
	return d.Create(id, parent, mountLabel, storageOpt)
}

// Create adds a device with a given id and the parent.
func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error {
	if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil {
		return err
	}

	return nil
}

// Remove removes a device with a given id, unmounts the filesystem.
func (d *Driver) Remove(id string) error {
	if !d.DeviceSet.HasDevice(id) {
		// Consider removing a non-existing device a no-op
		// This is useful to be able to progress on container removal
		// if the underlying device has gone away due to earlier errors
		return nil
	}

	// This assumes the device has been properly Get/Put:ed and thus is unmounted
	if err := d.DeviceSet.DeleteDevice(id, false); err != nil {
		return err
	}

	mp := path.Join(d.home, "mnt", id)
	if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Get mounts a device with given id into the root filesystem
func (d *Driver) Get(id, mountLabel string) (string, error) {
	mp := path.Join(d.home, "mnt", id)
	rootFs := path.Join(mp, "rootfs")
	if count := d.ctr.Increment(mp); count > 1 {
		return rootFs, nil
	}

	uid, gid, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
	if err != nil {
		d.ctr.Decrement(mp)
		return "", err
	}

	// Create the target directories if they don't exist
	if err := idtools.MkdirAllAs(path.Join(d.home, "mnt"), 0755, uid, gid); err != nil && !os.IsExist(err) {
		d.ctr.Decrement(mp)
		return "", err
	}
	if err := idtools.MkdirAs(mp, 0755, uid, gid); err != nil && !os.IsExist(err) {
		d.ctr.Decrement(mp)
		return "", err
	}

	// Mount the device
	if err := d.DeviceSet.MountDevice(id, mp, mountLabel); err != nil {
		d.ctr.Decrement(mp)
		return "", err
	}

	if err := idtools.MkdirAllAs(rootFs, 0755, uid, gid); err != nil && !os.IsExist(err) {
		d.ctr.Decrement(mp)
		d.DeviceSet.UnmountDevice(id, mp)
		return "", err
	}

	idFile := path.Join(mp, "id")
	if _, err := os.Stat(idFile); err != nil && os.IsNotExist(err) {
		// Create an "id" file with the container/image id in it to help reconstruct this in case
		// of later problems
		if err := ioutil.WriteFile(idFile, []byte(id), 0600); err != nil {
			d.ctr.Decrement(mp)
			d.DeviceSet.UnmountDevice(id, mp)
			return "", err
		}
	}

	return rootFs, nil
}

// Put unmounts a device and removes it.
func (d *Driver) Put(id string) error {
	mp := path.Join(d.home, "mnt", id)
	if count := d.ctr.Decrement(mp); count > 0 {
		return nil
	}
	err := d.DeviceSet.UnmountDevice(id, mp)
	if err != nil {
		logrus.Errorf("devmapper: Error unmounting device %s: %s", id, err)
	}
	return err
}

// Exists checks to see if the device exists.
func (d *Driver) Exists(id string) bool {
	return d.DeviceSet.HasDevice(id)
}
<|endoftext|>


<commit_before>package fs

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"

	"gotest.tools/assert"
)

// resourcePath is an adaptor for resources so they can be used as a Path
// with PathOps.
type resourcePath struct{}

func (p *resourcePath) Path() string {
	return "manifest: not a filesystem path"
}

func (p *resourcePath) Remove() {}

type filePath struct {
	resourcePath
	file *file
}

func (p *filePath) SetContent(content io.ReadCloser) {
	p.file.content = content
}

func (p *filePath) SetUID(uid uint32) {
	p.file.uid = uid
}

func (p *filePath) SetGID(gid uint32) {
	p.file.gid = gid
}

type directoryPath struct {
	resourcePath
	directory *directory
}

func (p *directoryPath) SetUID(uid uint32) {
	p.directory.uid = uid
}

func (p *directoryPath) SetGID(gid uint32) {
	p.directory.gid = gid
}

func (p *directoryPath) AddSymlink(path, target string) error {
	p.directory.items[path] = &symlink{
		resource: newResource(defaultSymlinkMode),
		target:   target,
	}
	return nil
}

func (p *directoryPath) AddFile(path string, ops ...PathOp) error {
	newFile := &file{resource: newResource(0)}
	p.directory.items[path] = newFile
	exp := &filePath{file: newFile}
	return applyPathOps(exp, ops)
}

func (p *directoryPath) AddGlobFiles(glob string, ops ...PathOp) error {
	newFile := &file{resource: newResource(0)}
	newFilePath := &filePath{file: newFile}
	p.directory.filepathGlobs[glob] = newFilePath
	return applyPathOps(newFilePath, ops)
}

func (p *directoryPath) AddDirectory(path string, ops ...PathOp) error {
	newDir := newDirectoryWithDefaults()
	p.directory.items[path] = newDir
	exp := &directoryPath{directory: newDir}
	return applyPathOps(exp, ops)
}

// Expected returns a Manifest with a directory structured created by ops. The
// PathOp operations are applied to the manifest as expectations of the
// filesystem structure and properties.
func Expected(t assert.TestingT, ops ...PathOp) Manifest {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}

	newDir := newDirectoryWithDefaults()
	e := &directoryPath{directory: newDir}
	assert.NilError(t, applyPathOps(e, ops))
	return Manifest{root: newDir}
}

func newDirectoryWithDefaults() *directory {
	return &directory{
		resource:      newResource(defaultRootDirMode),
		items:         make(map[string]dirEntry),
		filepathGlobs: make(map[string]*filePath),
	}
}

func newResource(mode os.FileMode) resource {
	return resource{
		mode: mode,
		uid:  currentUID(),
		gid:  currentGID(),
	}
}

func currentUID() uint32 {
	return normalizeID(os.Getuid())
}

func currentGID() uint32 {
	return normalizeID(os.Getgid())
}

func normalizeID(id int) uint32 {
	// ids will be -1 on windows
	if id < 0 {
		return 0
	}
	return uint32(id)
}

var anyFileContent = ioutil.NopCloser(bytes.NewReader(nil))

// MatchAnyFileContent is a PathOp that updates a Manifest so that the file
// at path may contain any content.
func MatchAnyFileContent(path Path) error {
	if m, ok := path.(*filePath); ok {
		m.SetContent(anyFileContent)
	}
	return nil
}

// MatchContentIgnoreCarriageReturn is a PathOp that ignores cariage return
// discrepancies.
func MatchContentIgnoreCarriageReturn(path Path) error {
	if m, ok := path.(*filePath); ok {
		m.file.ignoreCariageReturn = true
	}
	return nil
}

const anyFile = "*"

// MatchExtraFiles is a PathOp that updates a Manifest to allow a directory
// to contain unspecified files.
func MatchExtraFiles(path Path) error {
	if m, ok := path.(*directoryPath); ok {
		m.AddFile(anyFile)
	}
	return nil
}

// CompareResult is the result of comparison.
//
// See gotest.tools/assert/cmp.StringResult for a convenient implementation of
// this interface.
type CompareResult interface {
	Success() bool
	FailureMessage() string
}

// MatchFileContent is a PathOp that updates a Manifest to use the provided
// function to determine if a file's content matches the expectation.
func MatchFileContent(f func([]byte) CompareResult) PathOp {
	return func(path Path) error {
		if m, ok := path.(*filePath); ok {
			m.file.compareContentFunc = f
		}
		return nil
	}
}

// MatchFilesWithGlob is a PathOp that updates a Manifest to mathc files using
// glob pattern, and check them using the ops.
func MatchFilesWithGlob(glob string, ops ...PathOp) PathOp {
	return func(path Path) error {
		if m, ok := path.(*directoryPath); ok {
			m.AddGlobFiles(glob, ops...)
		}
		return nil
	}
}

// anyFileMode is represented by uint32_max
const anyFileMode os.FileMode = 4294967295

// MatchAnyFileMode is a PathOp that updates a Manifest so that the resource at path
// will match any file mode.
func MatchAnyFileMode(path Path) error {
	if m, ok := path.(manifestResource); ok {
		m.SetMode(anyFileMode)
	}
	return nil
}
<commit_msg>fix typo<commit_after>package fs

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"

	"gotest.tools/assert"
)

// resourcePath is an adaptor for resources so they can be used as a Path
// with PathOps.
type resourcePath struct{}

func (p *resourcePath) Path() string {
	return "manifest: not a filesystem path"
}

func (p *resourcePath) Remove() {}

type filePath struct {
	resourcePath
	file *file
}

func (p *filePath) SetContent(content io.ReadCloser) {
	p.file.content = content
}

func (p *filePath) SetUID(uid uint32) {
	p.file.uid = uid
}

func (p *filePath) SetGID(gid uint32) {
	p.file.gid = gid
}

type directoryPath struct {
	resourcePath
	directory *directory
}

func (p *directoryPath) SetUID(uid uint32) {
	p.directory.uid = uid
}

func (p *directoryPath) SetGID(gid uint32) {
	p.directory.gid = gid
}

func (p *directoryPath) AddSymlink(path, target string) error {
	p.directory.items[path] = &symlink{
		resource: newResource(defaultSymlinkMode),
		target:   target,
	}
	return nil
}

func (p *directoryPath) AddFile(path string, ops ...PathOp) error {
	newFile := &file{resource: newResource(0)}
	p.directory.items[path] = newFile
	exp := &filePath{file: newFile}
	return applyPathOps(exp, ops)
}

func (p *directoryPath) AddGlobFiles(glob string, ops ...PathOp) error {
	newFile := &file{resource: newResource(0)}
	newFilePath := &filePath{file: newFile}
	p.directory.filepathGlobs[glob] = newFilePath
	return applyPathOps(newFilePath, ops)
}

func (p *directoryPath) AddDirectory(path string, ops ...PathOp) error {
	newDir := newDirectoryWithDefaults()
	p.directory.items[path] = newDir
	exp := &directoryPath{directory: newDir}
	return applyPathOps(exp, ops)
}

// Expected returns a Manifest with a directory structured created by ops. The
// PathOp operations are applied to the manifest as expectations of the
// filesystem structure and properties.
func Expected(t assert.TestingT, ops ...PathOp) Manifest {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}

	newDir := newDirectoryWithDefaults()
	e := &directoryPath{directory: newDir}
	assert.NilError(t, applyPathOps(e, ops))
	return Manifest{root: newDir}
}

func newDirectoryWithDefaults() *directory {
	return &directory{
		resource:      newResource(defaultRootDirMode),
		items:         make(map[string]dirEntry),
		filepathGlobs: make(map[string]*filePath),
	}
}

func newResource(mode os.FileMode) resource {
	return resource{
		mode: mode,
		uid:  currentUID(),
		gid:  currentGID(),
	}
}

func currentUID() uint32 {
	return normalizeID(os.Getuid())
}

func currentGID() uint32 {
	return normalizeID(os.Getgid())
}

func normalizeID(id int) uint32 {
	// ids will be -1 on windows
	if id < 0 {
		return 0
	}
	return uint32(id)
}

var anyFileContent = ioutil.NopCloser(bytes.NewReader(nil))

// MatchAnyFileContent is a PathOp that updates a Manifest so that the file
// at path may contain any content.
func MatchAnyFileContent(path Path) error {
	if m, ok := path.(*filePath); ok {
		m.SetContent(anyFileContent)
	}
	return nil
}

// MatchContentIgnoreCarriageReturn is a PathOp that ignores cariage return
// discrepancies.
func MatchContentIgnoreCarriageReturn(path Path) error {
	if m, ok := path.(*filePath); ok {
		m.file.ignoreCariageReturn = true
	}
	return nil
}

const anyFile = "*"

// MatchExtraFiles is a PathOp that updates a Manifest to allow a directory
// to contain unspecified files.
func MatchExtraFiles(path Path) error {
	if m, ok := path.(*directoryPath); ok {
		m.AddFile(anyFile)
	}
	return nil
}

// CompareResult is the result of comparison.
//
// See gotest.tools/assert/cmp.StringResult for a convenient implementation of
// this interface.
type CompareResult interface {
	Success() bool
	FailureMessage() string
}

// MatchFileContent is a PathOp that updates a Manifest to use the provided
// function to determine if a file's content matches the expectation.
func MatchFileContent(f func([]byte) CompareResult) PathOp {
	return func(path Path) error {
		if m, ok := path.(*filePath); ok {
			m.file.compareContentFunc = f
		}
		return nil
	}
}

// MatchFilesWithGlob is a PathOp that updates a Manifest to match files using
// glob pattern, and check them using the ops.
func MatchFilesWithGlob(glob string, ops ...PathOp) PathOp {
	return func(path Path) error {
		if m, ok := path.(*directoryPath); ok {
			m.AddGlobFiles(glob, ops...)
		}
		return nil
	}
}

// anyFileMode is represented by uint32_max
const anyFileMode os.FileMode = 4294967295

// MatchAnyFileMode is a PathOp that updates a Manifest so that the resource at path
// will match any file mode.
func MatchAnyFileMode(path Path) error {
	if m, ok := path.(manifestResource); ok {
		m.SetMode(anyFileMode)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

const (
	hash1 = "1111111111111111111111111111111111111111"
	hash2 = "2222222222222222222222222222222222222222"
	hash3 = "3333333333333333333333333333333333333333"
)

func TestWebhookData(t *testing.T) {
	t.Run("webhhook consumes first hash value", func(t *testing.T) {
		whd := NewWebhookData()

		whd.UpdateAndTrigger(hash1)
		whd.WaitForChange()

		hash := whd.Hash()
		if hash1 != hash {
			t.Fatalf("expected hash %s but got %s", hash1, hash)
		}
	})

	t.Run("second update wins", func(t *testing.T) {
		whd := NewWebhookData()

		whd.UpdateAndTrigger(hash1)
		whd.UpdateAndTrigger(hash2)
		whd.WaitForChange()

		hash := whd.Hash()
		if hash2 != hash {
			t.Fatalf("expected hash %s but got %s", hash2, hash)
		}
	})

	t.Run("same hash value does not lead to an update", func(t *testing.T) {
		whd := NewWebhookData()

		whd.UpdateAndTrigger(hash1)
		whd.WaitForChange()
		hash := whd.Hash()
		if hash1 != hash {
			t.Fatalf("expected hash %s but got %s", hash1, hash)
		}

		whd.UpdateAndTrigger(hash1)
		changed := whd.Wait()

		if changed {
			t.Fatalf("no change expected")
		}

		hash = whd.Hash()
		if hash1 != hash {
			t.Fatalf("expected hash %s but got %s", hash1, hash)
		}
	})
}
<commit_msg>Cleanup test.<commit_after>package main

import (
	"testing"
)

const (
	hash1 = "1111111111111111111111111111111111111111"
	hash2 = "2222222222222222222222222222222222222222"
)

func TestWebhookData(t *testing.T) {
	t.Run("webhhook consumes first hash value", func(t *testing.T) {
		whd := NewWebhookData()

		whd.UpdateAndTrigger(hash1)
		whd.WaitForChange()

		hash := whd.Hash()
		if hash1 != hash {
			t.Fatalf("expected hash %s but got %s", hash1, hash)
		}
	})

	t.Run("second update wins", func(t *testing.T) {
		whd := NewWebhookData()

		whd.UpdateAndTrigger(hash1)
		whd.UpdateAndTrigger(hash2)
		whd.WaitForChange()

		hash := whd.Hash()
		if hash2 != hash {
			t.Fatalf("expected hash %s but got %s", hash2, hash)
		}
	})

	t.Run("same hash value does not lead to an update", func(t *testing.T) {
		whd := NewWebhookData()

		whd.UpdateAndTrigger(hash1)
		whd.WaitForChange()
		hash := whd.Hash()
		if hash1 != hash {
			t.Fatalf("expected hash %s but got %s", hash1, hash)
		}

		whd.UpdateAndTrigger(hash1)
		changed := whd.Wait()

		if changed {
			t.Fatalf("no change expected")
		}

		hash = whd.Hash()
		if hash1 != hash {
			t.Fatalf("expected hash %s but got %s", hash1, hash)
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"regexp"
)

func acquireFile(specFilename string, xmlSpecURL string) error {
	if f, err := os.Open(specFilename); err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		f.Close()
		return nil
	}

	resp, err := http.Get(xmlSpecURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("could not download spec %q from %q: ",
			specFilename, xmlSpecURL, resp.Status)
	}

	tmpFilename := specFilename + ".download"
	w, err := os.Create(tmpFilename)
	if err != nil {
		return err
	}
	defer w.Close()

	_, err = io.Copy(w, resp.Body)
	if err != nil {
		return err
	}

	return os.Rename(tmpFilename, specFilename)
}

func globFiles(pattern string, archive *zip.ReadCloser) []*zip.File {
	var files []*zip.File
	for _, f := range archive.File {
		if matched, err := path.Match(pattern, f.Name); err != nil {
			// This shouldn't happen - all patterns are hard-coded, errors in them
			// are a programming error.
			panic(err)
		} else if matched {
			files = append(files, f)
		}
	}
	return files
}

func unmarshalXmlFile(file *zip.File, data interface{}) error {
	r, err := file.Open()
	if err != nil {
		return err
	}
	decoder := xml.NewDecoder(r)
	defer r.Close()
	return decoder.Decode(data)
}

var scpdFilenameRe = regexp.MustCompile(
	`.*/([a-zA-Z0-9]+)([0-9]+)\.xml`)

func urnPartsFromSCPDFilename(filename string) (*URNParts, error) {
	parts := scpdFilenameRe.FindStringSubmatch(filename)
	if len(parts) != 3 {
		return nil, fmt.Errorf("SCPD filename %q does not have expected number of parts", filename)
	}
	name, version := parts[1], parts[2]
	return &URNParts{
		URN:     serviceURNPrefix + name + ":" + version,
		Name:    name,
		Version: version,
	}, nil
}
<commit_msg>Fix minor bug in how errors are displayed in codegen.<commit_after>package main

import (
	"archive/zip"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"regexp"
)

func acquireFile(specFilename string, xmlSpecURL string) error {
	if f, err := os.Open(specFilename); err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		f.Close()
		return nil
	}

	resp, err := http.Get(xmlSpecURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("could not download spec %q from %q: %s",
			specFilename, xmlSpecURL, resp.Status)
	}

	tmpFilename := specFilename + ".download"
	w, err := os.Create(tmpFilename)
	if err != nil {
		return err
	}
	defer w.Close()

	_, err = io.Copy(w, resp.Body)
	if err != nil {
		return err
	}

	return os.Rename(tmpFilename, specFilename)
}

func globFiles(pattern string, archive *zip.ReadCloser) []*zip.File {
	var files []*zip.File
	for _, f := range archive.File {
		if matched, err := path.Match(pattern, f.Name); err != nil {
			// This shouldn't happen - all patterns are hard-coded, errors in them
			// are a programming error.
			panic(err)
		} else if matched {
			files = append(files, f)
		}
	}
	return files
}

func unmarshalXmlFile(file *zip.File, data interface{}) error {
	r, err := file.Open()
	if err != nil {
		return err
	}
	decoder := xml.NewDecoder(r)
	defer r.Close()
	return decoder.Decode(data)
}

var scpdFilenameRe = regexp.MustCompile(
	`.*/([a-zA-Z0-9]+)([0-9]+)\.xml`)

func urnPartsFromSCPDFilename(filename string) (*URNParts, error) {
	parts := scpdFilenameRe.FindStringSubmatch(filename)
	if len(parts) != 3 {
		return nil, fmt.Errorf("SCPD filename %q does not have expected number of parts", filename)
	}
	name, version := parts[1], parts[2]
	return &URNParts{
		URN:     serviceURNPrefix + name + ":" + version,
		Name:    name,
		Version: version,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	flag "github.com/spf13/pflag"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"

	habclientset "github.com/habitat-sh/habitat-operator/pkg/client/clientset/versioned"
	habinformers "github.com/habitat-sh/habitat-operator/pkg/client/informers/externalversions"
	habv1beta1controller "github.com/habitat-sh/habitat-operator/pkg/controller/v1beta1"
	habv1beta2controller "github.com/habitat-sh/habitat-operator/pkg/controller/v1beta2"
)

const resyncPeriod = 30 * time.Second

type Clientsets struct {
	KubeClientset          *kubernetes.Clientset
	HabClientset           *habclientset.Clientset
	ApiextensionsClientset *apiextensionsclient.Clientset
}

func run() int {
	// Parse config flags.
	kubeconfig := flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
	verbose := flag.BoolP("verbose", "v", false, "Enable verbose logging.")
	flag.Parse()

	// Set up logging.
	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
	logger = log.With(logger, "ts", log.DefaultTimestamp)

	if *verbose {
		logger = level.NewFilter(logger, level.AllowDebug())
	} else {
		logger = level.NewFilter(logger, level.AllowInfo())
	}

	// Build operator config.
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	// This is the clientset for interacting with the apiextensions group.
	apiextensionsClientset, err := apiextensionsclient.NewForConfig(config)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	// This is the clientset for interacting with the Habitat API.
	habClientset, err := habclientset.NewForConfig(config)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	// This is the clientset for interacting with the stable API group.
	kubeClientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	ctx, cancelFunc := context.WithCancel(context.Background())
	defer cancelFunc()

	cSets := Clientsets{
		KubeClientset:          kubeClientset,
		HabClientset:           habClientset,
		ApiextensionsClientset: apiextensionsClientset,
	}

	if err := v1beta1(ctx, cSets, logger); err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	if err := v1beta2(ctx, cSets, logger); err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	term := make(chan os.Signal)
	// Relay these signals to the `term` channel.
	signal.Notify(term, syscall.SIGINT, syscall.SIGTERM)

	select {
	case <-term:
		level.Info(logger).Log("msg", "received SIGTERM, exiting gracefully...")
	case <-ctx.Done():
		level.Info(logger).Log("msg", "context channel closed, exiting")
	}

	return 0
}

func v1beta1(ctx context.Context, cSets Clientsets, logger log.Logger) error {
	// Create Habitat CRD.
	_, err := habv1beta1controller.CreateCRD(cSets.ApiextensionsClientset)
	if err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return err
		}

		level.Info(logger).Log("msg", "Habitat CRD already exists, continuing")
	} else {
		level.Info(logger).Log("msg", "created Habitat CRD")
	}

	config := habv1beta1controller.Config{
		HabitatClient:       cSets.HabClientset.HabitatV1beta1().RESTClient(),
		KubernetesClientset: cSets.KubeClientset,
	}
	controller, err := habv1beta1controller.New(config, log.With(logger, "component", "controller/v1beta1"))
	if err != nil {
		return err
	}

	go controller.Run(runtime.NumCPU(), ctx)

	return nil
}

func v1beta2(ctx context.Context, cSets Clientsets, logger log.Logger) error {
	// Create Habitat CRD.
	_, err := habv1beta2controller.CreateCRD(cSets.ApiextensionsClientset)
	if err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return err
		}

		level.Info(logger).Log("msg", "Habitat CRD already exists, continuing")
	} else {
		level.Info(logger).Log("msg", "created Habitat CRD")
	}

	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(cSets.KubeClientset, resyncPeriod)
	habInformerFactory := habinformers.NewSharedInformerFactory(cSets.HabClientset, resyncPeriod)

	config := habv1beta2controller.Config{
		// NOTE: The v1beta2 controller still needs to use a v1beta1 client,
		// because we _have_ only one client.  This is due to the fact that it's
		// not currently possible to have multiple versions of a CRD (and
		// therefore, of a client), running at the same time
		HabitatClient:          cSets.HabClientset.HabitatV1beta1().RESTClient(),
		KubernetesClientset:    cSets.KubeClientset,
		KubeInformerFactory:    kubeInformerFactory,
		HabitatInformerFactory: habInformerFactory,
	}
	controller, err := habv1beta2controller.New(config, log.With(logger, "component", "controller/v1beta2"))
	if err != nil {
		return err
	}

	go kubeInformerFactory.Start(ctx.Done())
	go habInformerFactory.Start(ctx.Done())

	go controller.Run(runtime.NumCPU(), ctx)

	return nil
}

func main() {
	os.Exit(run())
}
<commit_msg>main: Remove dependency on pflag library<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"os"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"

	habclientset "github.com/habitat-sh/habitat-operator/pkg/client/clientset/versioned"
	habinformers "github.com/habitat-sh/habitat-operator/pkg/client/informers/externalversions"
	habv1beta1controller "github.com/habitat-sh/habitat-operator/pkg/controller/v1beta1"
	habv1beta2controller "github.com/habitat-sh/habitat-operator/pkg/controller/v1beta2"
)

const resyncPeriod = 30 * time.Second

type Clientsets struct {
	KubeClientset          *kubernetes.Clientset
	HabClientset           *habclientset.Clientset
	ApiextensionsClientset *apiextensionsclient.Clientset
}

func run() int {
	// Parse config flags.
	kubeconfig := flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
	verbose := flag.Bool("verbose", false, "Enable verbose logging.")
	flag.Parse()

	// Set up logging.
	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
	logger = log.With(logger, "ts", log.DefaultTimestamp)

	if *verbose {
		logger = level.NewFilter(logger, level.AllowDebug())
	} else {
		logger = level.NewFilter(logger, level.AllowInfo())
	}

	// Build operator config.
	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	// This is the clientset for interacting with the apiextensions group.
	apiextensionsClientset, err := apiextensionsclient.NewForConfig(config)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	// This is the clientset for interacting with the Habitat API.
	habClientset, err := habclientset.NewForConfig(config)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	// This is the clientset for interacting with the stable API group.
	kubeClientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	ctx, cancelFunc := context.WithCancel(context.Background())
	defer cancelFunc()

	cSets := Clientsets{
		KubeClientset:          kubeClientset,
		HabClientset:           habClientset,
		ApiextensionsClientset: apiextensionsClientset,
	}

	if err := v1beta1(ctx, cSets, logger); err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	if err := v1beta2(ctx, cSets, logger); err != nil {
		level.Error(logger).Log("msg", err)
		return 1
	}

	term := make(chan os.Signal)
	// Relay these signals to the `term` channel.
	signal.Notify(term, syscall.SIGINT, syscall.SIGTERM)

	select {
	case <-term:
		level.Info(logger).Log("msg", "received SIGTERM, exiting gracefully...")
	case <-ctx.Done():
		level.Info(logger).Log("msg", "context channel closed, exiting")
	}

	return 0
}

func v1beta1(ctx context.Context, cSets Clientsets, logger log.Logger) error {
	// Create Habitat CRD.
	_, err := habv1beta1controller.CreateCRD(cSets.ApiextensionsClientset)
	if err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return err
		}

		level.Info(logger).Log("msg", "Habitat CRD already exists, continuing")
	} else {
		level.Info(logger).Log("msg", "created Habitat CRD")
	}

	config := habv1beta1controller.Config{
		HabitatClient:       cSets.HabClientset.HabitatV1beta1().RESTClient(),
		KubernetesClientset: cSets.KubeClientset,
	}
	controller, err := habv1beta1controller.New(config, log.With(logger, "component", "controller/v1beta1"))
	if err != nil {
		return err
	}

	go controller.Run(runtime.NumCPU(), ctx)

	return nil
}

func v1beta2(ctx context.Context, cSets Clientsets, logger log.Logger) error {
	// Create Habitat CRD.
	_, err := habv1beta2controller.CreateCRD(cSets.ApiextensionsClientset)
	if err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return err
		}

		level.Info(logger).Log("msg", "Habitat CRD already exists, continuing")
	} else {
		level.Info(logger).Log("msg", "created Habitat CRD")
	}

	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(cSets.KubeClientset, resyncPeriod)
	habInformerFactory := habinformers.NewSharedInformerFactory(cSets.HabClientset, resyncPeriod)

	config := habv1beta2controller.Config{
		// NOTE: The v1beta2 controller still needs to use a v1beta1 client,
		// because we _have_ only one client.  This is due to the fact that it's
		// not currently possible to have multiple versions of a CRD (and
		// therefore, of a client), running at the same time
		HabitatClient:          cSets.HabClientset.HabitatV1beta1().RESTClient(),
		KubernetesClientset:    cSets.KubeClientset,
		KubeInformerFactory:    kubeInformerFactory,
		HabitatInformerFactory: habInformerFactory,
	}
	controller, err := habv1beta2controller.New(config, log.With(logger, "component", "controller/v1beta2"))
	if err != nil {
		return err
	}

	go kubeInformerFactory.Start(ctx.Done())
	go habInformerFactory.Start(ctx.Done())

	go controller.Run(runtime.NumCPU(), ctx)

	return nil
}

func main() {
	os.Exit(run())
}
<|endoftext|>


<commit_before>package gateway

import (
	"encoding/binary"
	"io"
)

func writeI16(writer io.Writer, buf []byte, v int16) error {
	b := buf[:2]
	binary.BigEndian.PutUint16(b, uint16(v))
	_, e := writer.Write(b)
	return e
}

func writeI32(writer io.Writer, buf []byte, v int32) error {
	b := buf[:4]
	binary.BigEndian.PutUint32(b, uint32(v))
	_, e := writer.Write(b)
	return e
}

func writeI64(writer io.Writer, buf []byte, v int64) error {
	b := buf[:8]
	binary.BigEndian.PutUint64(b, uint64(v))
	_, e := writer.Write(b)
	return e
}

type Message struct {
	Partition int32
	Offset    int64
	Value     []byte
}

func DecodeMessageSet(messageSet []byte) []Message {
	r := make([]Message, 0)

	idx := 0
	for {
		m := Message{}
		m.Partition = int32(binary.BigEndian.Uint32(messageSet[idx : idx+4]))

		idx += 4
		m.Offset = int64(binary.BigEndian.Uint64(messageSet[idx : idx+8]))

		idx += 8
		messageSetLen := int(binary.BigEndian.Uint32(messageSet[idx : idx+4]))

		idx += 4
		m.Value = messageSet[idx : idx+messageSetLen]

		r = append(r, m)

		idx += messageSetLen
		if idx == len(messageSet) {
			return r
		}
	}

}
<commit_msg>fix the wrong var naming<commit_after>package gateway

import (
	"encoding/binary"
	"io"
)

func writeI16(writer io.Writer, buf []byte, v int16) error {
	b := buf[:2]
	binary.BigEndian.PutUint16(b, uint16(v))
	_, e := writer.Write(b)
	return e
}

func writeI32(writer io.Writer, buf []byte, v int32) error {
	b := buf[:4]
	binary.BigEndian.PutUint32(b, uint32(v))
	_, e := writer.Write(b)
	return e
}

func writeI64(writer io.Writer, buf []byte, v int64) error {
	b := buf[:8]
	binary.BigEndian.PutUint64(b, uint64(v))
	_, e := writer.Write(b)
	return e
}

type Message struct {
	Partition int32
	Offset    int64
	Value     []byte
}

func DecodeMessageSet(messageSet []byte) []Message {
	r := make([]Message, 0)

	idx := 0
	for {
		m := Message{}
		m.Partition = int32(binary.BigEndian.Uint32(messageSet[idx : idx+4]))

		idx += 4
		m.Offset = int64(binary.BigEndian.Uint64(messageSet[idx : idx+8]))

		idx += 8
		msgLen := int(binary.BigEndian.Uint32(messageSet[idx : idx+4]))

		idx += 4
		m.Value = messageSet[idx : idx+msgLen]

		r = append(r, m)

		idx += msgLen
		if idx == len(messageSet) {
			return r
		}
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog"
	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
	utilstaticpod "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
	"k8s.io/kubernetes/pkg/util/initsystem"
	utilsexec "k8s.io/utils/exec"
)

// NewCmdReset returns the "kubeadm reset" command
func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command {
	var certsDir string
	var criSocketPath string
	var ignorePreflightErrors []string
	var forceReset bool
	kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath()

	cmd := &cobra.Command{
		Use:   "reset",
		Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.",
		Run: func(cmd *cobra.Command, args []string) {
			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors)
			kubeadmutil.CheckErr(err)

			kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile)
			client, err := getClientset(kubeConfigFile, false)
			kubeadmutil.CheckErr(err)

			r, err := NewReset(in, ignorePreflightErrorsSet, forceReset, certsDir, criSocketPath)
			kubeadmutil.CheckErr(err)
			kubeadmutil.CheckErr(r.Run(out, client))
		},
	}

	options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors)
	options.AddKubeConfigFlag(cmd.PersistentFlags(), &kubeConfigFile)

	cmd.PersistentFlags().StringVar(
		&certsDir, "cert-dir", kubeadmapiv1beta1.DefaultCertificatesDir,
		"The path to the directory where the certificates are stored. If specified, clean this directory.",
	)

	cmd.PersistentFlags().StringVar(
		&criSocketPath, "cri-socket", kubeadmapiv1beta1.DefaultCRISocket,
		"The path to the CRI socket to use with crictl when cleaning up containers.",
	)

	cmd.PersistentFlags().BoolVarP(
		&forceReset, "force", "f", false,
		"Reset the node without prompting for confirmation.",
	)

	return cmd
}

// Reset defines struct used for kubeadm reset command
type Reset struct {
	certsDir      string
	criSocketPath string
}

// NewReset instantiate Reset struct
func NewReset(in io.Reader, ignorePreflightErrors sets.String, forceReset bool, certsDir, criSocketPath string) (*Reset, error) {
	if !forceReset {
		fmt.Println("[reset] WARNING: changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.")
		fmt.Print("[reset] are you sure you want to proceed? [y/N]: ")
		s := bufio.NewScanner(in)
		s.Scan()
		if err := s.Err(); err != nil {
			return nil, err
		}
		if strings.ToLower(s.Text()) != "y" {
			return nil, errors.New("Aborted reset operation")
		}
	}

	fmt.Println("[preflight] running pre-flight checks")
	if err := preflight.RunRootCheckOnly(ignorePreflightErrors); err != nil {
		return nil, err
	}

	return &Reset{
		certsDir:      certsDir,
		criSocketPath: criSocketPath,
	}, nil
}

// Run reverts any changes made to this host by "kubeadm init" or "kubeadm join".
func (r *Reset) Run(out io.Writer, client clientset.Interface) error {
	var dirsToClean []string
	// Only clear etcd data when using local etcd.
	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")

	klog.V(1).Infof("[reset] checking for etcd config")
	etcdDataDir, err := getEtcdDataDir(etcdManifestPath, client)
	if err == nil {
		dirsToClean = append(dirsToClean, etcdDataDir)
	} else {
		fmt.Println("[reset] no etcd config found. Assuming external etcd")
		fmt.Println("[reset] please manually reset etcd to prevent further issues")
	}

	// Try to stop the kubelet service
	klog.V(1).Infof("[reset] getting init system")
	initSystem, err := initsystem.GetInitSystem()
	if err != nil {
		klog.Warningln("[reset] the kubelet service could not be stopped by kubeadm. Unable to detect a supported init system!")
		klog.Warningln("[reset] please ensure kubelet is stopped manually")
	} else {
		fmt.Println("[reset] stopping the kubelet service")
		if err := initSystem.ServiceStop("kubelet"); err != nil {
			klog.Warningf("[reset] the kubelet service could not be stopped by kubeadm: [%v]\n", err)
			klog.Warningln("[reset] please ensure kubelet is stopped manually")
		}
	}

	// Try to unmount mounted directories under kubeadmconstants.KubeletRunDirectory in order to be able to remove the kubeadmconstants.KubeletRunDirectory directory later
	fmt.Printf("[reset] unmounting mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
	umountDirsCmd := fmt.Sprintf("awk '$2 ~ path {print $2}' path=%s /proc/mounts | xargs -r umount", kubeadmconstants.KubeletRunDirectory)

	klog.V(1).Infof("[reset] executing command %q", umountDirsCmd)
	umountOutputBytes, err := exec.Command("sh", "-c", umountDirsCmd).Output()
	if err != nil {
		klog.Errorf("[reset] failed to unmount mounted directories in %s: %s\n", kubeadmconstants.KubeletRunDirectory, string(umountOutputBytes))
	}

	klog.V(1).Info("[reset] removing Kubernetes-managed containers")
	if err := removeContainers(utilsexec.New(), r.criSocketPath); err != nil {
		klog.Errorf("[reset] failed to remove containers: %+v", err)
	}

	dirsToClean = append(dirsToClean, []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}...)

	// Then clean contents from the stateful kubelet, etcd and cni directories
	fmt.Printf("[reset] deleting contents of stateful directories: %v\n", dirsToClean)
	for _, dir := range dirsToClean {
		klog.V(1).Infof("[reset] deleting content of %s", dir)
		cleanDir(dir)
	}

	// Remove contents from the config and pki directories
	klog.V(1).Infoln("[reset] removing contents from the config and pki directories")
	if r.certsDir != kubeadmapiv1beta1.DefaultCertificatesDir {
		klog.Warningf("[reset] WARNING: cleaning a non-default certificates directory: %q\n", r.certsDir)
	}
	resetConfigDir(kubeadmconstants.KubernetesDir, r.certsDir)

	// Output help text instructing user how to remove iptables rules
	msg := `
	The reset process does not reset or clean up iptables rules or IPVS tables.
	If you wish to reset iptables, you must do so manually.
	For example:
	iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X

	If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
	to reset your system's IPVS tables.
	`
	fmt.Print(msg)

	return nil
}

func getEtcdDataDir(manifestPath string, client clientset.Interface) (string, error) {
	const etcdVolumeName = "etcd-data"
	var dataDir string

	cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "reset", "", false)
	if err == nil {
		return cfg.Etcd.Local.DataDir, nil
	}
	klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap, using etcd pod spec as fallback: %v", err)

	etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath)
	if err != nil {
		return "", err
	}

	for _, volumeMount := range etcdPod.Spec.Volumes {
		if volumeMount.Name == etcdVolumeName {
			dataDir = volumeMount.HostPath.Path
			break
		}
	}
	if dataDir == "" {
		return dataDir, fmt.Errorf("invalid etcd pod manifest")
	}
	return dataDir, nil
}

func removeContainers(execer utilsexec.Interface, criSocketPath string) error {
	containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath)
	if err != nil {
		return err
	}
	containers, err := containerRuntime.ListKubeContainers()
	if err != nil {
		return err
	}
	if err := containerRuntime.RemoveContainers(containers); err != nil {
		return err
	}
	return nil
}

// cleanDir removes everything in a directory, but not the directory itself
func cleanDir(filePath string) error {
	// If the directory doesn't even exist there's nothing to do, and we do
	// not consider this an error
	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		return nil
	}

	d, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, name := range names {
		if err = os.RemoveAll(filepath.Join(filePath, name)); err != nil {
			return err
		}
	}
	return nil
}

// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/.
func resetConfigDir(configPathDir, pkiPathDir string) {
	dirsToClean := []string{
		filepath.Join(configPathDir, kubeadmconstants.ManifestsSubDirName),
		pkiPathDir,
	}
	fmt.Printf("[reset] deleting contents of config directories: %v\n", dirsToClean)
	for _, dir := range dirsToClean {
		if err := cleanDir(dir); err != nil {
			klog.Errorf("[reset] failed to remove directory: %q [%v]\n", dir, err)
		}
	}

	filesToClean := []string{
		filepath.Join(configPathDir, kubeadmconstants.AdminKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.KubeletKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.SchedulerKubeConfigFileName),
	}
	fmt.Printf("[reset] deleting files: %v\n", filesToClean)
	for _, path := range filesToClean {
		if err := os.RemoveAll(path); err != nil {
			klog.Errorf("[reset] failed to remove file: %q [%v]\n", path, err)
		}
	}
}
<commit_msg>reset helptext alignment<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	"k8s.io/apimachinery/pkg/util/sets"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog"
	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
	utilstaticpod "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
	"k8s.io/kubernetes/pkg/util/initsystem"
	utilsexec "k8s.io/utils/exec"
)

// NewCmdReset returns the "kubeadm reset" command
func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command {
	var certsDir string
	var criSocketPath string
	var ignorePreflightErrors []string
	var forceReset bool
	kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath()

	cmd := &cobra.Command{
		Use:   "reset",
		Short: "Run this to revert any changes made to this host by 'kubeadm init' or 'kubeadm join'.",
		Run: func(cmd *cobra.Command, args []string) {
			ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors)
			kubeadmutil.CheckErr(err)

			kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile)
			client, err := getClientset(kubeConfigFile, false)
			kubeadmutil.CheckErr(err)

			r, err := NewReset(in, ignorePreflightErrorsSet, forceReset, certsDir, criSocketPath)
			kubeadmutil.CheckErr(err)
			kubeadmutil.CheckErr(r.Run(out, client))
		},
	}

	options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors)
	options.AddKubeConfigFlag(cmd.PersistentFlags(), &kubeConfigFile)

	cmd.PersistentFlags().StringVar(
		&certsDir, "cert-dir", kubeadmapiv1beta1.DefaultCertificatesDir,
		"The path to the directory where the certificates are stored. If specified, clean this directory.",
	)

	cmd.PersistentFlags().StringVar(
		&criSocketPath, "cri-socket", kubeadmapiv1beta1.DefaultCRISocket,
		"The path to the CRI socket to use with crictl when cleaning up containers.",
	)

	cmd.PersistentFlags().BoolVarP(
		&forceReset, "force", "f", false,
		"Reset the node without prompting for confirmation.",
	)

	return cmd
}

// Reset defines struct used for kubeadm reset command
type Reset struct {
	certsDir      string
	criSocketPath string
}

// NewReset instantiate Reset struct
func NewReset(in io.Reader, ignorePreflightErrors sets.String, forceReset bool, certsDir, criSocketPath string) (*Reset, error) {
	if !forceReset {
		fmt.Println("[reset] WARNING: changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.")
		fmt.Print("[reset] are you sure you want to proceed? [y/N]: ")
		s := bufio.NewScanner(in)
		s.Scan()
		if err := s.Err(); err != nil {
			return nil, err
		}
		if strings.ToLower(s.Text()) != "y" {
			return nil, errors.New("Aborted reset operation")
		}
	}

	fmt.Println("[preflight] running pre-flight checks")
	if err := preflight.RunRootCheckOnly(ignorePreflightErrors); err != nil {
		return nil, err
	}

	return &Reset{
		certsDir:      certsDir,
		criSocketPath: criSocketPath,
	}, nil
}

// Run reverts any changes made to this host by "kubeadm init" or "kubeadm join".
func (r *Reset) Run(out io.Writer, client clientset.Interface) error {
	var dirsToClean []string
	// Only clear etcd data when using local etcd.
	etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml")

	klog.V(1).Infof("[reset] checking for etcd config")
	etcdDataDir, err := getEtcdDataDir(etcdManifestPath, client)
	if err == nil {
		dirsToClean = append(dirsToClean, etcdDataDir)
	} else {
		fmt.Println("[reset] no etcd config found. Assuming external etcd")
		fmt.Println("[reset] please manually reset etcd to prevent further issues")
	}

	// Try to stop the kubelet service
	klog.V(1).Infof("[reset] getting init system")
	initSystem, err := initsystem.GetInitSystem()
	if err != nil {
		klog.Warningln("[reset] the kubelet service could not be stopped by kubeadm. Unable to detect a supported init system!")
		klog.Warningln("[reset] please ensure kubelet is stopped manually")
	} else {
		fmt.Println("[reset] stopping the kubelet service")
		if err := initSystem.ServiceStop("kubelet"); err != nil {
			klog.Warningf("[reset] the kubelet service could not be stopped by kubeadm: [%v]\n", err)
			klog.Warningln("[reset] please ensure kubelet is stopped manually")
		}
	}

	// Try to unmount mounted directories under kubeadmconstants.KubeletRunDirectory in order to be able to remove the kubeadmconstants.KubeletRunDirectory directory later
	fmt.Printf("[reset] unmounting mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory)
	umountDirsCmd := fmt.Sprintf("awk '$2 ~ path {print $2}' path=%s /proc/mounts | xargs -r umount", kubeadmconstants.KubeletRunDirectory)

	klog.V(1).Infof("[reset] executing command %q", umountDirsCmd)
	umountOutputBytes, err := exec.Command("sh", "-c", umountDirsCmd).Output()
	if err != nil {
		klog.Errorf("[reset] failed to unmount mounted directories in %s: %s\n", kubeadmconstants.KubeletRunDirectory, string(umountOutputBytes))
	}

	klog.V(1).Info("[reset] removing Kubernetes-managed containers")
	if err := removeContainers(utilsexec.New(), r.criSocketPath); err != nil {
		klog.Errorf("[reset] failed to remove containers: %+v", err)
	}

	dirsToClean = append(dirsToClean, []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}...)

	// Then clean contents from the stateful kubelet, etcd and cni directories
	fmt.Printf("[reset] deleting contents of stateful directories: %v\n", dirsToClean)
	for _, dir := range dirsToClean {
		klog.V(1).Infof("[reset] deleting content of %s", dir)
		cleanDir(dir)
	}

	// Remove contents from the config and pki directories
	klog.V(1).Infoln("[reset] removing contents from the config and pki directories")
	if r.certsDir != kubeadmapiv1beta1.DefaultCertificatesDir {
		klog.Warningf("[reset] WARNING: cleaning a non-default certificates directory: %q\n", r.certsDir)
	}
	resetConfigDir(kubeadmconstants.KubernetesDir, r.certsDir)

	// Output help text instructing user how to remove iptables rules
	msg := `
The reset process does not reset or clean up iptables rules or IPVS tables.
If you wish to reset iptables, you must do so manually.
For example:
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X

If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
to reset your system's IPVS tables.

`
	fmt.Print(msg)

	return nil
}

func getEtcdDataDir(manifestPath string, client clientset.Interface) (string, error) {
	const etcdVolumeName = "etcd-data"
	var dataDir string

	cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "reset", "", false)
	if err == nil {
		return cfg.Etcd.Local.DataDir, nil
	}
	klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap, using etcd pod spec as fallback: %v", err)

	etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath)
	if err != nil {
		return "", err
	}

	for _, volumeMount := range etcdPod.Spec.Volumes {
		if volumeMount.Name == etcdVolumeName {
			dataDir = volumeMount.HostPath.Path
			break
		}
	}
	if dataDir == "" {
		return dataDir, fmt.Errorf("invalid etcd pod manifest")
	}
	return dataDir, nil
}

func removeContainers(execer utilsexec.Interface, criSocketPath string) error {
	containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath)
	if err != nil {
		return err
	}
	containers, err := containerRuntime.ListKubeContainers()
	if err != nil {
		return err
	}
	if err := containerRuntime.RemoveContainers(containers); err != nil {
		return err
	}
	return nil
}

// cleanDir removes everything in a directory, but not the directory itself
func cleanDir(filePath string) error {
	// If the directory doesn't even exist there's nothing to do, and we do
	// not consider this an error
	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		return nil
	}

	d, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer d.Close()
	names, err := d.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, name := range names {
		if err = os.RemoveAll(filepath.Join(filePath, name)); err != nil {
			return err
		}
	}
	return nil
}

// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/.
func resetConfigDir(configPathDir, pkiPathDir string) {
	dirsToClean := []string{
		filepath.Join(configPathDir, kubeadmconstants.ManifestsSubDirName),
		pkiPathDir,
	}
	fmt.Printf("[reset] deleting contents of config directories: %v\n", dirsToClean)
	for _, dir := range dirsToClean {
		if err := cleanDir(dir); err != nil {
			klog.Errorf("[reset] failed to remove directory: %q [%v]\n", dir, err)
		}
	}

	filesToClean := []string{
		filepath.Join(configPathDir, kubeadmconstants.AdminKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.KubeletKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
		filepath.Join(configPathDir, kubeadmconstants.SchedulerKubeConfigFileName),
	}
	fmt.Printf("[reset] deleting files: %v\n", filesToClean)
	for _, path := range filesToClean {
		if err := os.RemoveAll(path); err != nil {
			klog.Errorf("[reset] failed to remove file: %q [%v]\n", path, err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/scipipe/scipipe"
)

// AuditReport is a container for data to be parsed into an audit report, in
// HTML, TeX or other format
type auditReport struct {
	FileName    string
	ScipipeVer  string
	RunTime     time.Duration
	AuditInfos  []*scipipe.AuditInfo
	ColorDef    string
	ChartHeight string
}

func auditInfoToHTML(inFilePath string, outFilePath string, flatten bool) error {
	ip := scipipe.NewFileIP(strings.Replace(inFilePath, ".audit.json", "", 1))
	auditInfo := ip.AuditInfo()

	outHTML := fmt.Sprintf(headHTMLPattern, ip.Path())
	if flatten {
		auditInfosByID := extractAuditInfosByID(auditInfo)
		auditInfosByStartTime := sortAuditInfosByStartTime(auditInfosByID)
		for _, ai := range auditInfosByStartTime {
			ai.Upstream = nil
			outHTML += formatTaskHTML(ai.ProcessName, ai)
		}
	} else {
		outHTML += formatTaskHTML(ip.Path(), auditInfo)
	}
	outHTML += bottomHTML

	if _, err := os.Stat(outFilePath); os.IsExist(err) {
		return errWrap(err, "File already exists:"+outFilePath)
	}
	outFile, err := os.Create(outFilePath)
	if err != nil {
		return errWrap(err, "Could not create file:"+outFilePath)
	}
	outFile.WriteString(outHTML)
	outFile.Close()
	fmt.Println("Wrote audit HTML file to: " + outFilePath)
	return nil
}

func formatTaskHTML(fileName string, auditInfo *scipipe.AuditInfo) (outHTML string) {
	outHTML = "
\n"
	outHTML += fmt.Sprintf("
%s / %s
\n", auditInfo.ProcessName, auditInfo.ID, auditInfo.ID, auditInfo.ID)
	outHTML += fmt.Sprintf("

%s

\n", auditInfo.Command)

	params := []string{}
	for pname, p := range auditInfo.Params {
		params = append(params, fmt.Sprintf("%s: %s", pname, p))
	}
	outHTML += fmt.Sprintf("
Parameters:	%s
\n", strings.Join(params, ", "))
	tags := []string{}
	for pname, p := range auditInfo.Tags {
		tags = append(tags, fmt.Sprintf("%s: %s", pname, p))
	}
	outHTML += fmt.Sprintf("
Tags:
%v

\n", strings.Join(tags, ", "))

	outHTML += fmt.Sprintf("
Start time:	%s
\n", auditInfo.StartTime.Format(`2006-01-02 15:04:05<span class="greyout">.000 -0700 MST</span>`))
	outHTML += fmt.Sprintf("
Finish time:	%s
\n", auditInfo.FinishTime.Format(`2006-01-02 15:04:05<span class="greyout">.000 -0700 MST</span>`))
	et := auditInfo.ExecTimeNS
	outHTML += fmt.Sprintf("
Execution time:	%s
\n", et.Truncate(time.Millisecond).String())
	//upStreamHTML := ""
	//for filePath, uai := range auditInfo.Upstream {
	//	upStreamHTML += formatTaskHTML(filePath, uai)
	//}
	//if outHTML != "" {
	//	outHTML += "
Upstreams:	" + upStreamHTML + "
\n"
	//}
	outHTML += "
\n"
	return
}

func auditInfoToTeX(inFilePath string, outFilePath string, flatten bool) error {
	outFile, err := os.Create(outFilePath)
	scipipe.CheckWithMsg(err, "Could not create TeX file")

	auditInfo := scipipe.UnmarshalAuditInfoJSONFile(inFilePath)
	auditInfosByID := extractAuditInfosByID(auditInfo)
	auditInfosByStartTime := sortAuditInfosByStartTime(auditInfosByID)

	texTpl := template.New("TeX").Funcs(
		template.FuncMap{
			"strrepl":        func(subj string, find string, repl string) string { return strings.Replace(subj, find, repl, -1) },
			"sub":            func(val1 int, val2 int) int { return val1 - val2 },
			"timesub":        func(t1 time.Time, t2 time.Time) time.Duration { return t1.Sub(t2) },
			"durtomillis":    func(exact time.Duration) (rounded time.Duration) { return exact.Truncate(1e6 * time.Nanosecond) },
			"timetomillis":   func(exact time.Time) (rounded time.Time) { return exact.Truncate(1e6 * time.Nanosecond) },
			"durtomillisint": func(exact time.Duration) (millis int) { return int(exact.Nanoseconds() / 1000000) },
		})
	texTpl, err = texTpl.Parse(texTemplate)
	scipipe.CheckWithMsg(err, "Could not parse TeX template")

	report := auditReport{
		FileName:    inFilePath,
		ScipipeVer:  scipipe.Version,
		RunTime:     auditInfosByStartTime[len(auditInfosByStartTime)-1].FinishTime.Sub(auditInfosByStartTime[0].StartTime),
		AuditInfos:  auditInfosByStartTime,
		ChartHeight: fmt.Sprintf("%.03f", 1.0+float64(len(auditInfosByStartTime))*0.6),
	}

	palette := palettes[1]
	if len(report.AuditInfos) <= 50 {
		palette = palettes[len(report.AuditInfos)]
	} else {
		palette = palettes[len(report.AuditInfos)%50]
	}
	for i, p := range palette {
		report.ColorDef += fmt.Sprintf("\\definecolor{color%d}{RGB}{%d,%d,%d}\n", i, p.r, p.g, p.b)
	}

	texTpl.Execute(outFile, report)
	fmt.Println("Wrote audit TeX file to: " + outFilePath)
	return nil
}

func extractAuditInfosByID(auditInfo *scipipe.AuditInfo) (auditInfosByID map[string]*scipipe.AuditInfo) {
	auditInfosByID = make(map[string]*scipipe.AuditInfo)
	auditInfosByID[auditInfo.ID] = auditInfo
	for _, ai := range auditInfo.Upstream {
		auditInfosByID = mergeStringAuditInfoMaps(auditInfosByID, extractAuditInfosByID(ai))
	}
	return auditInfosByID
}

func mergeStringAuditInfoMaps(ms ...map[string]*scipipe.AuditInfo) (merged map[string]*scipipe.AuditInfo) {
	merged = make(map[string]*scipipe.AuditInfo)
	for _, m := range ms {
		for k, v := range m {
			merged[k] = v
		}
	}
	return merged
}

func sortAuditInfosByStartTime(auditInfosByID map[string]*scipipe.AuditInfo) []*scipipe.AuditInfo {
	sorted := []*scipipe.AuditInfo{}

	auditInfosByStartTime := map[time.Time]*scipipe.AuditInfo{}
	startTimes := []time.Time{}
	for _, ai := range auditInfosByID {
		auditInfosByStartTime[ai.StartTime] = ai
		startTimes = append(startTimes, ai.StartTime)
	}
	sort.Slice(startTimes, func(i, j int) bool { return startTimes[i].Before(startTimes[j]) })
	for _, t := range startTimes {
		sorted = append(sorted, auditInfosByStartTime[t])
	}
	return sorted
}

const headHTMLPattern = `

<title>Audit info for: %s</title>

`
const bottomHTML = `
`

// LaTeX code from vision.tex:
const texTemplate = `\documentclass[11pt,oneside,openright]{memoir}

\usepackage{tcolorbox}
\usepackage[scaled]{beramono}
\renewcommand*\familydefault{\ttdefault}
\usepackage[T1]{fontenc}
\usepackage{tabularx}
\usepackage{listings}
\usepackage{graphicx}
\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{pgfplotstable}
\usepackage{xcolor}

{{ .ColorDef }}

% from https://tex.stackexchange.com/a/128040/110842
% filter to only get the current row in \pgfplotsinvokeforeach
\pgfplotsset{
    select row/.style={
        x filter/.code={\ifnum\coordindex=#1\else\def\pgfmathresult{}\fi}
    }
}

\pgfplotstableread[col sep=comma]{
start,end,Name,color
{{ $startTime := (index .AuditInfos 0).StartTime }}
{{ range $i, $v := .AuditInfos }}{{ durtomillisint (timesub $v.StartTime $startTime) }},{{ durtomillisint (timesub $v.FinishTime $startTime) }},{{ strrepl .ProcessName "_" "\\_" }},color{{ $i }}
{{ end }}
}\loadedtable
\pgfplotstablegetrowsof{\loadedtable}
\pgfplotsset{compat=1.13}
\pgfmathsetmacro{\tablerows}{int(\pgfplotsretval-1)}

\begin{document}
\pagestyle{plain}
\noindent
\begin{minipage}{\textwidth}
    \vspace{-8em}\hspace{-8em}
    %\includegraphics[width=9em]{images/scipipe_logo_bluegrey.png}
\end{minipage}

\noindent
{\huge\textbf{SciPipe Audit Report}} \\
\vspace{10pt}

    \begin{tcolorbox}[ title=Workflow for file: {{ (strrepl (strrepl .FileName ".audit.json" "") "_" "\\_") }} ]
    \small
\begin{tabular}{rp{0.72\linewidth}}
SciPipe version: & {{ .ScipipeVer }} \\
Start time:  & {{ timetomillis (index .AuditInfos 0).StartTime }} \\
Finish time: & {{ timetomillis (index .AuditInfos (sub (len .AuditInfos) 1)).FinishTime }} \\
Run time: & {{ durtomillis .RunTime }}  \\
\end{tabular}
    \end{tcolorbox}

\setlength{\fboxsep}{0pt}
\noindent

%\hspace{-0.1725\textwidth}\fbox{\includegraphics[width=1.35\textwidth]{images/cawpre.pdf}}

\section*{Execution timeline}

\begin{tikzpicture}
\begin{axis}[
    xbar, xmin=0,
    y axis line style = { opacity = 0 },
    tickwidth         = 0pt,
	width=10cm,
	height={{ .ChartHeight }}cm,
    % next two lines also from https://tex.stackexchange.com/a/128040/110842,
    ytick={0,...,\tablerows},
    yticklabels from table={\loadedtable}{Name},
    xbar stacked,
    bar shift=0pt,
    y dir=reverse,
    xtick={1, 60000, 120000, 180000, 240000, 300000, 600000, 900000, 1200000},
    xticklabels={0, 1 min, 2 min, 3 min, 4 min, 5 min, 10 min, 15 min, 20 min},
    scaled x ticks=false,
]

\pgfplotsinvokeforeach{0,...,\tablerows}{
    % get color from table, commands defined must be individual for each plot
    % because the color is used in \end{axis} and therefore would otherwise
    % use the last definition
    \pgfplotstablegetelem{#1}{color}\of{\loadedtable}
    \expandafter\edef\csname barcolor.#1\endcsname{\pgfplotsretval}
    \addplot+[color=\csname barcolor.#1\endcsname] table [select row=#1, x expr=\thisrow{end}-\thisrow{start}, y expr=#1]{\loadedtable};
}
\end{axis}
\end{tikzpicture}

\newpage

\section*{Tasks}
    \lstset{ breaklines=true,
            postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space},
            aboveskip=8pt,belowskip=8pt}

{{ range $i, $v := .AuditInfos }}
   \begin{tcolorbox}[ title={{ (strrepl $v.ProcessName "_" "\\_") }},
                      colbacktitle=color{{ $i }}!63!white,
                      colback=color{{ $i }}!37!white,
                      coltitle=black ]
       \small
       \begin{tabular}{rp{0.72\linewidth}}
ID: & {{ $v.ID }} \\
Process: & {{ (strrepl $v.ProcessName "_" "\\_") }} \\
Command: & \begin{lstlisting}
{{ strrepl $v.Command "_" "\\_" }}
\end{lstlisting} \\
Parameters:& {{ range $k, $v := $v.Params }}{{- $k -}}={{- $v -}}{{ end }} \\
Tags: & {{ range $k, $v := $v.Tags }}{{- $k -}}={{- $v -}}{{ end }} \\
Start time:  & {{ timetomillis $v.StartTime }} \\
Finish time: & {{ timetomillis $v.FinishTime }} \\
Execution time: & {{ durtomillis $v.ExecTimeNS }} \\
        \end{tabular}
	\end{tcolorbox}
{{ end }}

\end{document}`
<commit_msg>Refactor audit2tex a bit and fix calculation of total runtime<commit_after>package main

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/scipipe/scipipe"
)

// AuditReport is a container for data to be parsed into an audit report, in
// HTML, TeX or other format
type auditReport struct {
	FileName    string
	ScipipeVer  string
	RunTime     time.Duration
	AuditInfos  []*scipipe.AuditInfo
	ColorDef    string
	ChartHeight string
}

var (
	tplFuncs = template.FuncMap{
		"strrepl":        func(subj string, find string, repl string) string { return strings.Replace(subj, find, repl, -1) },
		"sub":            func(val1 int, val2 int) int { return val1 - val2 },
		"timesub":        func(t1 time.Time, t2 time.Time) time.Duration { return t1.Sub(t2) },
		"durtomillis":    func(exact time.Duration) (rounded time.Duration) { return exact.Truncate(1e6 * time.Nanosecond) },
		"timetomillis":   func(exact time.Time) (rounded time.Time) { return exact.Truncate(1e6 * time.Nanosecond) },
		"durtomillisint": func(exact time.Duration) (millis int) { return int(exact.Nanoseconds() / 1000000) },
	}
)

func auditInfoToHTML(inFilePath string, outFilePath string, flatten bool) error {
	ip := scipipe.NewFileIP(strings.Replace(inFilePath, ".audit.json", "", 1))
	auditInfo := ip.AuditInfo()

	outHTML := fmt.Sprintf(headHTMLPattern, ip.Path())
	if flatten {
		auditInfosByID := extractAuditInfosByID(auditInfo)
		auditInfosByStartTime := sortAuditInfosByStartTime(auditInfosByID)
		for _, ai := range auditInfosByStartTime {
			ai.Upstream = nil
			outHTML += formatTaskHTML(ai.ProcessName, ai)
		}
	} else {
		outHTML += formatTaskHTML(ip.Path(), auditInfo)
	}
	outHTML += bottomHTML

	if _, err := os.Stat(outFilePath); os.IsExist(err) {
		return errWrap(err, "File already exists:"+outFilePath)
	}
	outFile, err := os.Create(outFilePath)
	if err != nil {
		return errWrap(err, "Could not create file:"+outFilePath)
	}
	outFile.WriteString(outHTML)
	outFile.Close()
	fmt.Println("Wrote audit HTML file to: " + outFilePath)
	return nil
}

func formatTaskHTML(fileName string, auditInfo *scipipe.AuditInfo) (outHTML string) {
	outHTML = "
\n"
	outHTML += fmt.Sprintf("
%s / %s
\n", auditInfo.ProcessName, auditInfo.ID, auditInfo.ID, auditInfo.ID)
	outHTML += fmt.Sprintf("

%s

\n", auditInfo.Command)

	params := []string{}
	for pname, p := range auditInfo.Params {
		params = append(params, fmt.Sprintf("%s: %s", pname, p))
	}
	outHTML += fmt.Sprintf("
Parameters:	%s
\n", strings.Join(params, ", "))
	tags := []string{}
	for pname, p := range auditInfo.Tags {
		tags = append(tags, fmt.Sprintf("%s: %s", pname, p))
	}
	outHTML += fmt.Sprintf("
Tags:
%v

\n", strings.Join(tags, ", "))

	outHTML += fmt.Sprintf("
Start time:	%s
\n", auditInfo.StartTime.Format(`2006-01-02 15:04:05<span class="greyout">.000 -0700 MST</span>`))
	outHTML += fmt.Sprintf("
Finish time:	%s
\n", auditInfo.FinishTime.Format(`2006-01-02 15:04:05<span class="greyout">.000 -0700 MST</span>`))
	et := auditInfo.ExecTimeNS
	outHTML += fmt.Sprintf("
Execution time:	%s
\n", et.Truncate(time.Millisecond).String())
	//upStreamHTML := ""
	//for filePath, uai := range auditInfo.Upstream {
	//	upStreamHTML += formatTaskHTML(filePath, uai)
	//}
	//if outHTML != "" {
	//	outHTML += "
Upstreams:	" + upStreamHTML + "
\n"
	//}
	outHTML += "
\n"
	return
}

func auditInfoToTeX(inFilePath string, outFilePath string, flatten bool) error {
	outFile, err := os.Create(outFilePath)
	scipipe.CheckWithMsg(err, "Could not create TeX file")

	auditInfo := scipipe.UnmarshalAuditInfoJSONFile(inFilePath)
	auditInfosByID := extractAuditInfosByID(auditInfo)
	auditInfosByStartTime := sortAuditInfosByStartTime(auditInfosByID)

	texTpl := template.New("TeX").Funcs(tplFuncs)
	texTpl, err = texTpl.Parse(texTemplate)
	scipipe.CheckWithMsg(err, "Could not parse TeX template")

	runTime := time.Duration(0)
	for _, auInfo := range auditInfosByStartTime {
		runTime += auInfo.ExecTimeNS
	}

	report := auditReport{
		FileName:    inFilePath,
		ScipipeVer:  scipipe.Version,
		RunTime:     runTime,
		AuditInfos:  auditInfosByStartTime,
		ChartHeight: fmt.Sprintf("%.03f", 1.0+float64(len(auditInfosByStartTime))*0.5),
	}

	palette := palettes[1]
	if len(report.AuditInfos) <= 50 {
		palette = palettes[len(report.AuditInfos)]
	} else {
		palette = palettes[len(report.AuditInfos)%50]
	}
	for i, p := range palette {
		report.ColorDef += fmt.Sprintf("\\definecolor{color%d}{RGB}{%d,%d,%d}\n", i, p.r, p.g, p.b)
	}

	texTpl.Execute(outFile, report)
	fmt.Println("Wrote audit TeX file to: " + outFilePath)
	return nil
}

func extractAuditInfosByID(auditInfo *scipipe.AuditInfo) (auditInfosByID map[string]*scipipe.AuditInfo) {
	auditInfosByID = make(map[string]*scipipe.AuditInfo)
	auditInfosByID[auditInfo.ID] = auditInfo
	for _, ai := range auditInfo.Upstream {
		auditInfosByID = mergeStringAuditInfoMaps(auditInfosByID, extractAuditInfosByID(ai))
	}
	return auditInfosByID
}

func mergeStringAuditInfoMaps(ms ...map[string]*scipipe.AuditInfo) (merged map[string]*scipipe.AuditInfo) {
	merged = make(map[string]*scipipe.AuditInfo)
	for _, m := range ms {
		for k, v := range m {
			merged[k] = v
		}
	}
	return merged
}

func sortAuditInfosByStartTime(auditInfosByID map[string]*scipipe.AuditInfo) []*scipipe.AuditInfo {
	sorted := []*scipipe.AuditInfo{}

	auditInfosByStartTime := map[time.Time]*scipipe.AuditInfo{}
	startTimes := []time.Time{}
	for _, ai := range auditInfosByID {
		auditInfosByStartTime[ai.StartTime] = ai
		startTimes = append(startTimes, ai.StartTime)
	}
	sort.Slice(startTimes, func(i, j int) bool { return startTimes[i].Before(startTimes[j]) })
	for _, t := range startTimes {
		sorted = append(sorted, auditInfosByStartTime[t])
	}
	return sorted
}

const headHTMLPattern = `

<title>Audit info for: %s</title>

`
const bottomHTML = `
`

// LaTeX code from vision.tex:
const texTemplate = `\documentclass[11pt,oneside,openright]{memoir}

\usepackage{tcolorbox}
\usepackage[scaled]{beramono}
\renewcommand*\familydefault{\ttdefault}
\usepackage[T1]{fontenc}
\usepackage{tabularx}
\usepackage{listings}
\usepackage{graphicx}
\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{pgfplotstable}
\usepackage{xcolor}

{{ .ColorDef }}

% from https://tex.stackexchange.com/a/128040/110842
% filter to only get the current row in \pgfplotsinvokeforeach
\pgfplotsset{
    select row/.style={
        x filter/.code={\ifnum\coordindex=#1\else\def\pgfmathresult{}\fi}
    }
}

\pgfplotstableread[col sep=comma]{
start,end,Name,color
{{ $startTime := (index .AuditInfos 0).StartTime }}
{{ range $i, $v := .AuditInfos }}{{ durtomillisint (timesub $v.StartTime $startTime) }},{{ durtomillisint (timesub $v.FinishTime $startTime) }},{{ strrepl .ProcessName "_" "\\_" }},color{{ $i }}
{{ end }}
}\loadedtable
\pgfplotstablegetrowsof{\loadedtable}
\pgfplotsset{compat=1.13}
\pgfmathsetmacro{\tablerows}{int(\pgfplotsretval-1)}

\begin{document}
\pagestyle{plain}
\noindent
\begin{minipage}{\textwidth}
    \vspace{-8em}\hspace{-8em}
    %\includegraphics[width=9em]{images/scipipe_logo_bluegrey.png}
\end{minipage}

\noindent
{\huge\textbf{SciPipe Audit Report}} \\
\vspace{10pt}

    \begin{tcolorbox}[ title=Workflow for file: {{ (strrepl (strrepl .FileName ".audit.json" "") "_" "\\_") }} ]
    \small
\begin{tabular}{rp{0.72\linewidth}}
SciPipe version: & {{ .ScipipeVer }} \\
Start time:  & {{ timetomillis (index .AuditInfos 0).StartTime }} \\
Finish time: & {{ timetomillis (index .AuditInfos (sub (len .AuditInfos) 1)).FinishTime }} \\
Run time: & {{ durtomillis .RunTime }}  \\
\end{tabular}
    \end{tcolorbox}

\setlength{\fboxsep}{0pt}
\noindent

%\hspace{-0.1725\textwidth}\fbox{\includegraphics[width=1.35\textwidth]{images/cawpre.pdf}}

\section*{Execution timeline}

\begin{tikzpicture}
\begin{axis}[
    xbar, xmin=0,
    y axis line style = { opacity = 0 },
    tickwidth         = 0pt,
	width=10cm,
	height={{ .ChartHeight }}cm,
    % next two lines also from https://tex.stackexchange.com/a/128040/110842,
    ytick={0,...,\tablerows},
    yticklabels from table={\loadedtable}{Name},
    xbar stacked,
    bar shift=0pt,
    y dir=reverse,
    xtick={1, 60000, 120000, 180000, 240000, 300000, 600000, 900000, 1200000},
    xticklabels={0, 1 min, 2 min, 3 min, 4 min, 5 min, 10 min, 15 min, 20 min},
    scaled x ticks=false,
]

\pgfplotsinvokeforeach{0,...,\tablerows}{
    % get color from table, commands defined must be individual for each plot
    % because the color is used in \end{axis} and therefore would otherwise
    % use the last definition
    \pgfplotstablegetelem{#1}{color}\of{\loadedtable}
    \expandafter\edef\csname barcolor.#1\endcsname{\pgfplotsretval}
    \addplot+[color=\csname barcolor.#1\endcsname] table [select row=#1, x expr=\thisrow{end}-\thisrow{start}, y expr=#1]{\loadedtable};
}
\end{axis}
\end{tikzpicture}

\newpage

\section*{Tasks}
    \lstset{ breaklines=true,
            postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space},
            aboveskip=8pt,belowskip=8pt}

{{ range $i, $v := .AuditInfos }}
   \begin{tcolorbox}[ title={{ (strrepl $v.ProcessName "_" "\\_") }},
                      colbacktitle=color{{ $i }}!63!white,
                      colback=color{{ $i }}!37!white,
                      coltitle=black ]
       \small
       \begin{tabular}{rp{0.72\linewidth}}
ID: & {{ $v.ID }} \\
Process: & {{ (strrepl $v.ProcessName "_" "\\_") }} \\
Command: & \begin{lstlisting}
{{ strrepl $v.Command "_" "\\_" }}
\end{lstlisting} \\
Parameters:& {{ range $k, $v := $v.Params }}{{- $k -}}={{- $v -}}{{ end }} \\
Tags: & {{ range $k, $v := $v.Tags }}{{- $k -}}={{- $v -}}{{ end }} \\
Start time:  & {{ timetomillis $v.StartTime }} \\
Finish time: & {{ timetomillis $v.FinishTime }} \\
Execution time: & {{ durtomillis $v.ExecTimeNS }} \\
        \end{tabular}
	\end{tcolorbox}
{{ end }}

\end{document}`
<|endoftext|>