<commit_before>package tvrage

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"
)

type Show struct {
	ID             int      `xml:"showid"`
	Name           string   `xml:"name"`
	Link           string   `xml:"link"`
	Country        string   `xml:"country"`
	Started        int      `xml:"started"`
	Ended          int      `xml:"ended"`
	Seasons        int      `xml:"seasons"`
	Status         string   `xml:"status"`
	Classification string   `xml:"classification"`
	Genres         []string `xml:"genres>genre"`
}

func (s Show) String() string {
	return fmt.Sprintf("%s [%d - %s]", s.Name, s.Started, s.Status)
}

type tvrageTime struct {
	time.Time
}

func (t *tvrageTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var v string
	d.DecodeElement(&v, &start)
	parsed, err := time.Parse(TIMEFMT, v)
	if err != nil {
		return nil
	}
	*t = tvrageTime{parsed}
	return nil
}

type Episode struct {
	Season     int
	Ordinal    int        `xml:"epnum"`
	Number     int        `xml:"seasonnum"`
	Production string     `xml:"prodnum"`
	AirDate    tvrageTime `xml:"airdate"`
	Link       string     `xml:"link"`
	Title      string     `xml:"title"`
}

func (e Episode) String() string {
	return fmt.Sprintf(`S%02dE%02d "%s"`, e.Season, e.Number, e.Title)
}

func (e *Episode) DeltaDays() string {
	d := int(e.AirDate.Sub(time.Now()).Hours() / 24.0)
	if d < 0 {
		if d == -1 {
			return "yesterday"
		} else {
			return fmt.Sprintf("%d days ago", -d)
		}
	} else if d > 0 {
		if d == 1 {
			return "tomorrow"
		} else {
			return fmt.Sprintf("in %d days", d+1)
		}
	} else {
		return "today"
	}
}

type Episodes []Episode

func (es Episodes) Last() (Episode, bool) {
	var r Episode
	t := time.Now()
	for _, e := range es {
		if e.AirDate.IsZero() {
			continue
		}
		if e.AirDate.Before(t) {
			r = e
		}
	}
	if r.AirDate.IsZero() {
		return r, false
	} else {
		return r, true
	}
}

func (es Episodes) Next() (Episode, bool) {
	var r Episode
	t := time.Now()
	for _, e := range es {
		if e.AirDate.IsZero() {
			continue
		}
		if e.AirDate.After(t) {
			return e, true
		}
	}
	return r, false
}

type resultSeason struct {
	Number   int       `xml:"no,attr"`
	Episodes []Episode `xml:"episode"`
}

type resultEpisodeList struct {
	Total   int            `xml:"totalseasons"`
	Seasons []resultSeason `xml:"Episodelist>Season"`
}

type resultSearch struct {
	Shows []Show `xml:"show"`
}

const (
	SEARCHURL = `http://services.tvrage.com/feeds/search.php?show=%s`      // URL for show searching
	EPLISTURL = `http://services.tvrage.com/feeds/episode_list.php?sid=%d` // URL for episode list
	TIMEFMT   = `2006-01-02`                                               // time.Parse format string for air date
	VERSION   = `0.0.1`                                                    // library version
)

var (
	Client = &http.Client{} // default HTTP client
)

func parseSearchResult(in io.Reader) ([]Show, error) {
	r := resultSearch{}
	x := xml.NewDecoder(in)
	if err := x.Decode(&r); err != nil {
		return nil, err
	}
	return r.Shows, nil
}

func Search(name string) ([]Show, error) {
	q := fmt.Sprintf(SEARCHURL, url.QueryEscape(name))
	r, err := Client.Get(q)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return parseSearchResult(r.Body)
}

func parseEpisodeListResult(in io.Reader) (Episodes, error) {
	var es Episodes
	r := resultEpisodeList{}
	x := xml.NewDecoder(in)
	if err := x.Decode(&r); err != nil {
		return nil, err
	}
	for _, s := range r.Seasons {
		for _, e := range s.Episodes {
			e.Season = s.Number
			es = append(es, e)
		}
	}
	return es, nil
}

func EpisodeList(id int) (Episodes, error) {
	q := fmt.Sprintf(EPLISTURL, id)
	r, err := Client.Get(q)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return parseEpisodeListResult(r.Body)
}
<commit_msg>Add basic docs<commit_after>// Package tvrage provides basic access to tvrage.com services for finding out the last
// and next episodes of a given TV show (plus a bit more), no API key required.
package tvrage

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"time"
)

// Show maps all available show data, as retrieved via Search.
type Show struct {
	ID             int      `xml:"showid"`
	Name           string   `xml:"name"`
	Link           string   `xml:"link"`
	Country        string   `xml:"country"`
	Started        int      `xml:"started"`
	Ended          int      `xml:"ended"`
	Seasons        int      `xml:"seasons"`
	Status         string   `xml:"status"`
	Classification string   `xml:"classification"`
	Genres         []string `xml:"genres>genre"`
}

// String returns a pretty string for a given Show.
func (s Show) String() string {
	return fmt.Sprintf("%s [%d - %s]", s.Name, s.Started, s.Status)
}

// tvrageTime is a thin shim over time.Time used to implement XML unmarshaling.
type tvrageTime struct {
	time.Time
}

// UnmarshalXML implements time.Time XML unmarshaling for tvrage.com air date format.
func (t *tvrageTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var v string
	d.DecodeElement(&v, &start)
	parsed, err := time.Parse(TIMEFMT, v)
	if err != nil {
		return nil
	}
	*t = tvrageTime{parsed}
	return nil
}

// Episode maps all available episode data, as retrieved via EpisodeList.
type Episode struct {
	Season     int
	Ordinal    int        `xml:"epnum"`
	Number     int        `xml:"seasonnum"`
	Production string     `xml:"prodnum"`
	AirDate    tvrageTime `xml:"airdate"`
	Link       string     `xml:"link"`
	Title      string     `xml:"title"`
}

// String returns a pretty string for a given Episode.
func (e Episode) String() string {
	return fmt.Sprintf(`S%02dE%02d "%s"`, e.Season, e.Number, e.Title)
}

// DeltaDays returns a pretty string indicating the delta in days between now
// and the episode air date.
func (e *Episode) DeltaDays() string {
	d := int(e.AirDate.Sub(time.Now()).Hours() / 24.0)
	if d < 0 {
		if d == -1 {
			return "yesterday"
		} else {
			return fmt.Sprintf("%d days ago", -d)
		}
	} else if d > 0 {
		if d == 1 {
			return "tomorrow"
		} else {
			return fmt.Sprintf("in %d days", d+1)
		}
	} else {
		return "today"
	}
}

// Episodes is a thin shim over []Episodes to enable methods on Episode slices.
type Episodes []Episode

// Last returns the last aired episode from the given slice of Episodes and true
// if it was possible to find such episode.
func (es Episodes) Last() (Episode, bool) {
	var r Episode
	t := time.Now()
	for _, e := range es {
		if e.AirDate.IsZero() {
			continue
		}
		if e.AirDate.Before(t) {
			r = e
		}
	}
	if r.AirDate.IsZero() {
		return r, false
	} else {
		return r, true
	}
}

// Next returns the next episode to air from the given slice of Episodes and true
// if it was possible to find such episode.
func (es Episodes) Next() (Episode, bool) {
	var r Episode
	t := time.Now()
	for _, e := range es {
		if e.AirDate.IsZero() {
			continue
		}
		if e.AirDate.After(t) {
			return e, true
		}
	}
	return r, false
}

// resultSeason is an internal intermediate struct used for processing EpisodeList results.
type resultSeason struct {
	Number   int       `xml:"no,attr"`
	Episodes []Episode `xml:"episode"`
}

// resultEpisodeList is an internal final struct used for processing EpisodeList results.
type resultEpisodeList struct {
	Total   int            `xml:"totalseasons"`
	Seasons []resultSeason `xml:"Episodelist>Season"`
}

// resultSearch is an internal final struct used for processing Search results.
type resultSearch struct {
	Shows []Show `xml:"show"`
}

const (
	SEARCHURL = `http://services.tvrage.com/feeds/search.php?show=%s`      // URL for show searching
	EPLISTURL = `http://services.tvrage.com/feeds/episode_list.php?sid=%d` // URL for episode list
	TIMEFMT   = `2006-01-02`                                               // time.Parse format string for air date
	VERSION   = `0.0.1`                                                    // library version
)

var (
	Client = &http.Client{} // default HTTP client
)

// parseSearchResult parses the XML as retrieved by Search.
func parseSearchResult(in io.Reader) ([]Show, error) {
	r := resultSearch{}
	x := xml.NewDecoder(in)
	if err := x.Decode(&r); err != nil {
		return nil, err
	}
	return r.Shows, nil
}

// Search retrieves matched shows for the given name.
func Search(name string) ([]Show, error) {
	q := fmt.Sprintf(SEARCHURL, url.QueryEscape(name))
	r, err := Client.Get(q)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return parseSearchResult(r.Body)
}

// parseEpisodeListResults parses the XML as retrieved by EpisodeList.
// It fills in the season number and returns a slice of Episodes.
func parseEpisodeListResult(in io.Reader) (Episodes, error) {
	var es Episodes
	r := resultEpisodeList{}
	x := xml.NewDecoder(in)
	if err := x.Decode(&r); err != nil {
		return nil, err
	}
	for _, s := range r.Seasons {
		for _, e := range s.Episodes {
			e.Season = s.Number
			es = append(es, e)
		}
	}
	return es, nil
}

// EpisodeList retrieves the list of episodes for the given show id.
func EpisodeList(id int) (Episodes, error) {
	q := fmt.Sprintf(EPLISTURL, id)
	r, err := Client.Get(q)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return parseEpisodeListResult(r.Body)
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

var btrfsVersion string
var btrfsLoaded bool

type btrfs struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *btrfs) load() error {
	if btrfsLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"btrfs"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if btrfsVersion == "" {
		out, err := shared.RunCommand("btrfs", "version")
		if err != nil {
			return err
		}

		count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
		if err != nil || count != 1 {
			return fmt.Errorf("The 'btrfs' tool isn't working properly")
		}
	}

	btrfsLoaded = true
	return nil
}

// Info returns info about the driver and its environment.
func (d *btrfs) Info() Info {
	return Info{
		Name:                  "btrfs",
		Version:               btrfsVersion,
		OptimizedImages:       true,
		PreservesInodes:       !d.state.OS.RunningInUserNS,
		Remote:                false,
		VolumeTypes:           []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:          false,
		RunningQuotaResize:    true,
		RunningSnapshotFreeze: false,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *btrfs) Create() error {
	// Store the provided source as we are likely to be mangling it.
	d.config["volatile.initial_source"] = d.config["source"]

	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	if d.config["source"] == "" || d.config["source"] == loopPath {
		// Create a loop based pool.
		d.config["source"] = loopPath

		// Create the loop file itself.
		size, err := units.ParseByteSizeString(d.config["size"])
		if err != nil {
			return err
		}

		err = createSparseFile(d.config["source"], size)
		if err != nil {
			return fmt.Errorf("Failed to create the sparse file: %v", err)
		}

		// Format the file.
		_, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return fmt.Errorf("Failed to format sparse file: %v", err)
		}
	} else if shared.IsBlockdevPath(d.config["source"]) {
		// Format the block device.
		_, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return fmt.Errorf("Failed to format block device: %v", err)
		}

		// Record the UUID as the source.
		devUUID, err := fsUUID(d.config["source"])
		if err != nil {
			return err
		}

		// Confirm that the symlink is appearing (give it 10s).
		if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
			// Override the config to use the UUID.
			d.config["source"] = devUUID
		}
	} else if d.config["source"] != "" {
		hostPath := shared.HostPath(d.config["source"])
		if d.isSubvolume(hostPath) {
			// Existing btrfs subvolume.
			subvols, err := d.getSubvolumes(hostPath)
			if err != nil {
				return fmt.Errorf("Could not determine if existing btrfs subvolume is empty: %v", err)
			}

			// Check that the provided subvolume is empty.
			if len(subvols) > 0 {
				return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
			}
		} else {
			// New btrfs subvolume on existing btrfs filesystem.
			cleanSource := filepath.Clean(hostPath)
			lxdDir := shared.VarPath()

			if shared.PathExists(hostPath) && !hasFilesystem(hostPath, util.FilesystemSuperMagicBtrfs) {
				return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
			} else if strings.HasPrefix(cleanSource, lxdDir) {
				if cleanSource != GetPoolMountPath(d.name) {
					return fmt.Errorf("Only allowed source path under %s is %s", shared.VarPath(), GetPoolMountPath(d.name))
				} else if !hasFilesystem(shared.VarPath("storage-pools"), util.FilesystemSuperMagicBtrfs) {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}

				// Delete the current directory to replace by subvolume.
				err := os.Remove(cleanSource)
				if err != nil {
					return err
				}
			}

			// Create the subvolume.
			_, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
			if err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf("Invalid \"source\" property")
	}

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *btrfs) Delete(op *operations.Operation) error {
	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// Delete potential intermediate btrfs subvolumes.
	for _, volType := range d.Info().VolumeTypes {
		for _, dir := range BaseDirectories[volType] {
			path := filepath.Join(GetPoolMountPath(d.name), dir)
			if !shared.PathExists(path) {
				continue
			}

			if !d.isSubvolume(path) {
				continue
			}

			err := d.deleteSubvolume(path, true)
			if err != nil {
				return fmt.Errorf("Could not delete btrfs subvolume: %s", path)
			}
		}
	}

	// On delete, wipe everything in the directory.
	err := wipeDirectory(GetPoolMountPath(d.name))
	if err != nil {
		return err
	}

	// Unmount the path.
	_, err = d.Unmount()
	if err != nil {
		return err
	}

	// If the pool path is a subvolume itself, delete it.
	if d.isSubvolume(GetPoolMountPath(d.name)) {
		err := d.deleteSubvolume(GetPoolMountPath(d.name), false)
		if err != nil {
			return err
		}

		// And re-create as an empty directory to make the backend happy.
		err = os.Mkdir(GetPoolMountPath(d.name), 0700)
		if err != nil {
			return err
		}
	}

	// Delete any loop file we may have used.
	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	if shared.PathExists(loopPath) {
		err = os.Remove(loopPath)
		if err != nil {
			return err
		}
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *btrfs) Validate(config map[string]string) error {
	return nil
}

// Update applies any driver changes required from a configuration change.
func (d *btrfs) Update(changedConfig map[string]string) error {
	// We only care about btrfs.mount_options.
	val, ok := changedConfig["btrfs.mount_options"]
	if !ok {
		return nil
	}

	// Trigger a re-mount.
	d.config["btrfs.mount_options"] = val
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
	mntFlags |= unix.MS_REMOUNT

	err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
	if err != nil {
		return err
	}

	return nil
}

// Mount mounts the storage pool.
func (d *btrfs) Mount() (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(GetPoolMountPath(d.name)) {
		return false, nil
	}

	// Setup mount options.
	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	mntSrc := ""
	mntDst := GetPoolMountPath(d.name)
	mntFilesystem := "btrfs"
	if d.config["source"] == loopPath {
		// Bring up the loop device.
		loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
		if err != nil {
			return false, err
		}
		defer loopF.Close()

		mntSrc = loopF.Name()
	} else if filepath.IsAbs(d.config["source"]) {
		// Bring up an existing device or path.
		mntSrc = shared.HostPath(d.config["source"])

		if !shared.IsBlockdevPath(mntSrc) {
			mntFilesystem = "none"

			if !hasFilesystem(mntSrc, util.FilesystemSuperMagicBtrfs) {
				return false, fmt.Errorf("Source path '%s' isn't btrfs", mntSrc)
			}
		}
	} else {
		// Mount using UUID.
		mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
	}

	// Get the custom mount flags/options.
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())

	// Handle bind-mounts first.
	if mntFilesystem == "none" {
		// Setup the bind-mount itself.
		err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
		if err != nil {
			return false, err
		}

		// Now apply the custom options.
		mntFlags |= unix.MS_REMOUNT
		err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
		if err != nil {
			return false, err
		}

		return true, nil
	}

	// Handle traditional mounts.
	err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Unmount unmounts the storage pool.
func (d *btrfs) Unmount() (bool, error) {
	// Unmount the pool.
	ourUnmount, err := forceUnmount(GetPoolMountPath(d.name))
	if err != nil {
		return false, err
	}

	// If loop backed, force release the loop device.
	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	if d.config["source"] == loopPath {
		releaseLoopDev(loopPath)
	}

	return ourUnmount, nil
}

// GetResources returns the pool resource usage information.
func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
	return d.vfsGetResources()
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	if contentType != ContentTypeFS {
		return nil
	}

	// When performing a refresh, always use rsync. Using btrfs send/receive
	// here doesn't make sense since it would need to send everything again
	// which defeats the purpose of a refresh.
	if refresh {
		return []migration.Type{
			{
				FSType:   migration.MigrationFSType_RSYNC,
				Features: []string{"xattrs", "delete", "compress", "bidirectional"},
			},
		}
	}

	return []migration.Type{
		{
			FSType: migration.MigrationFSType_BTRFS,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: []string{"xattrs", "delete", "compress", "bidirectional"},
		},
	}
}
<commit_msg>lxd/storage/btrfs: Disable send/receive inside containers<commit_after>package drivers

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

var btrfsVersion string
var btrfsLoaded bool

type btrfs struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *btrfs) load() error {
	if btrfsLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"btrfs"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if btrfsVersion == "" {
		out, err := shared.RunCommand("btrfs", "version")
		if err != nil {
			return err
		}

		count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
		if err != nil || count != 1 {
			return fmt.Errorf("The 'btrfs' tool isn't working properly")
		}
	}

	btrfsLoaded = true
	return nil
}

// Info returns info about the driver and its environment.
func (d *btrfs) Info() Info {
	return Info{
		Name:                  "btrfs",
		Version:               btrfsVersion,
		OptimizedImages:       true,
		PreservesInodes:       !d.state.OS.RunningInUserNS,
		Remote:                false,
		VolumeTypes:           []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:          false,
		RunningQuotaResize:    true,
		RunningSnapshotFreeze: false,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *btrfs) Create() error {
	// Store the provided source as we are likely to be mangling it.
	d.config["volatile.initial_source"] = d.config["source"]

	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	if d.config["source"] == "" || d.config["source"] == loopPath {
		// Create a loop based pool.
		d.config["source"] = loopPath

		// Create the loop file itself.
		size, err := units.ParseByteSizeString(d.config["size"])
		if err != nil {
			return err
		}

		err = createSparseFile(d.config["source"], size)
		if err != nil {
			return fmt.Errorf("Failed to create the sparse file: %v", err)
		}

		// Format the file.
		_, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return fmt.Errorf("Failed to format sparse file: %v", err)
		}
	} else if shared.IsBlockdevPath(d.config["source"]) {
		// Format the block device.
		_, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return fmt.Errorf("Failed to format block device: %v", err)
		}

		// Record the UUID as the source.
		devUUID, err := fsUUID(d.config["source"])
		if err != nil {
			return err
		}

		// Confirm that the symlink is appearing (give it 10s).
		if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
			// Override the config to use the UUID.
			d.config["source"] = devUUID
		}
	} else if d.config["source"] != "" {
		hostPath := shared.HostPath(d.config["source"])
		if d.isSubvolume(hostPath) {
			// Existing btrfs subvolume.
			subvols, err := d.getSubvolumes(hostPath)
			if err != nil {
				return fmt.Errorf("Could not determine if existing btrfs subvolume is empty: %v", err)
			}

			// Check that the provided subvolume is empty.
			if len(subvols) > 0 {
				return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
			}
		} else {
			// New btrfs subvolume on existing btrfs filesystem.
			cleanSource := filepath.Clean(hostPath)
			lxdDir := shared.VarPath()

			if shared.PathExists(hostPath) && !hasFilesystem(hostPath, util.FilesystemSuperMagicBtrfs) {
				return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
			} else if strings.HasPrefix(cleanSource, lxdDir) {
				if cleanSource != GetPoolMountPath(d.name) {
					return fmt.Errorf("Only allowed source path under %s is %s", shared.VarPath(), GetPoolMountPath(d.name))
				} else if !hasFilesystem(shared.VarPath("storage-pools"), util.FilesystemSuperMagicBtrfs) {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}

				// Delete the current directory to replace by subvolume.
				err := os.Remove(cleanSource)
				if err != nil {
					return err
				}
			}

			// Create the subvolume.
			_, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
			if err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf("Invalid \"source\" property")
	}

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *btrfs) Delete(op *operations.Operation) error {
	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// Delete potential intermediate btrfs subvolumes.
	for _, volType := range d.Info().VolumeTypes {
		for _, dir := range BaseDirectories[volType] {
			path := filepath.Join(GetPoolMountPath(d.name), dir)
			if !shared.PathExists(path) {
				continue
			}

			if !d.isSubvolume(path) {
				continue
			}

			err := d.deleteSubvolume(path, true)
			if err != nil {
				return fmt.Errorf("Could not delete btrfs subvolume: %s", path)
			}
		}
	}

	// On delete, wipe everything in the directory.
	err := wipeDirectory(GetPoolMountPath(d.name))
	if err != nil {
		return err
	}

	// Unmount the path.
	_, err = d.Unmount()
	if err != nil {
		return err
	}

	// If the pool path is a subvolume itself, delete it.
	if d.isSubvolume(GetPoolMountPath(d.name)) {
		err := d.deleteSubvolume(GetPoolMountPath(d.name), false)
		if err != nil {
			return err
		}

		// And re-create as an empty directory to make the backend happy.
		err = os.Mkdir(GetPoolMountPath(d.name), 0700)
		if err != nil {
			return err
		}
	}

	// Delete any loop file we may have used.
	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	if shared.PathExists(loopPath) {
		err = os.Remove(loopPath)
		if err != nil {
			return err
		}
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *btrfs) Validate(config map[string]string) error {
	return nil
}

// Update applies any driver changes required from a configuration change.
func (d *btrfs) Update(changedConfig map[string]string) error {
	// We only care about btrfs.mount_options.
	val, ok := changedConfig["btrfs.mount_options"]
	if !ok {
		return nil
	}

	// Trigger a re-mount.
	d.config["btrfs.mount_options"] = val
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
	mntFlags |= unix.MS_REMOUNT

	err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
	if err != nil {
		return err
	}

	return nil
}

// Mount mounts the storage pool.
func (d *btrfs) Mount() (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(GetPoolMountPath(d.name)) {
		return false, nil
	}

	// Setup mount options.
	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	mntSrc := ""
	mntDst := GetPoolMountPath(d.name)
	mntFilesystem := "btrfs"
	if d.config["source"] == loopPath {
		// Bring up the loop device.
		loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
		if err != nil {
			return false, err
		}
		defer loopF.Close()

		mntSrc = loopF.Name()
	} else if filepath.IsAbs(d.config["source"]) {
		// Bring up an existing device or path.
		mntSrc = shared.HostPath(d.config["source"])

		if !shared.IsBlockdevPath(mntSrc) {
			mntFilesystem = "none"

			if !hasFilesystem(mntSrc, util.FilesystemSuperMagicBtrfs) {
				return false, fmt.Errorf("Source path '%s' isn't btrfs", mntSrc)
			}
		}
	} else {
		// Mount using UUID.
		mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
	}

	// Get the custom mount flags/options.
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())

	// Handle bind-mounts first.
	if mntFilesystem == "none" {
		// Setup the bind-mount itself.
		err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
		if err != nil {
			return false, err
		}

		// Now apply the custom options.
		mntFlags |= unix.MS_REMOUNT
		err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
		if err != nil {
			return false, err
		}

		return true, nil
	}

	// Handle traditional mounts.
	err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Unmount unmounts the storage pool.
func (d *btrfs) Unmount() (bool, error) {
	// Unmount the pool.
	ourUnmount, err := forceUnmount(GetPoolMountPath(d.name))
	if err != nil {
		return false, err
	}

	// If loop backed, force release the loop device.
	loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
	if d.config["source"] == loopPath {
		releaseLoopDev(loopPath)
	}

	return ourUnmount, nil
}

// GetResources returns the pool resource usage information.
func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
	return d.vfsGetResources()
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	if contentType != ContentTypeFS {
		return nil
	}

	// Only use rsync for refreshes and if running in an unprivileged container.
	if refresh || d.state.OS.RunningInUserNS {
		return []migration.Type{
			{
				FSType:   migration.MigrationFSType_RSYNC,
				Features: []string{"xattrs", "delete", "compress", "bidirectional"},
			},
		}
	}

	return []migration.Type{
		{
			FSType: migration.MigrationFSType_BTRFS,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: []string{"xattrs", "delete", "compress", "bidirectional"},
		},
	}
}
<|endoftext|>


<commit_before>// What's the delivery status?
// dRbiG, 2014
// See LICENSE.txt

package main

import (
	"errors"
	"fmt"
	"github.com/kennygrant/sanitize"
	"github.com/moovweb/gokogiri"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
)

type DlFunc func(base, id string) (body []byte, err error)

type Service struct {
	Name       string
	Downloader DlFunc
	Matcher    string
	URL        string
	XPath      string
	Extractor  *regexp.Regexp
}

func (s *Service) IsMatch(id string) bool {
	matched, err := regexp.MatchString(s.Matcher, id)
	if err != nil {
		panic(err)
	}
	return matched
}

func (s *Service) Check(id string) (status string, err error) {
	body, err := s.Downloader(s.URL, id)
	if err != nil {
		return
	}

	if s.Extractor != nil {
		parts := s.Extractor.FindSubmatch(body)
		if parts == nil {
			return "", nil
		}

		status = string(parts[1])
	} else {
		doc, err := gokogiri.ParseHtml(body)
		if err != nil {
			return "", err
		}
		defer doc.Free()

		res, err := doc.Search(s.XPath)
		if err != nil {
			return "", err
		}
		if len(res) < 1 {
			return "", nil
		}

		status = sanitize.HTML(res[0].String())
		status = replacer.ReplaceAllString(status, " ")
		status = strings.TrimSpace(status)
	}

	return
}

func dlSimpleGet(base, id string) (body []byte, err error) {
	url := fmt.Sprintf(base, id)

	res, err := client.Get(url)
	if err != nil {
		return
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("DownloadPage: %d %s", res.StatusCode, url))
		return
	}

	body, err = ioutil.ReadAll(res.Body)
	return
}

func dlPocztex(base, id string) (body []byte, err error) {
	form := url.Values{}
	form.Add("n", id)
	form.Add("s", "1")

	req, err := http.NewRequest("POST", base, strings.NewReader(form.Encode()))
	if err != nil {
		return
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.AddCookie(&http.Cookie{Name: "PHPSESSID", Value: "1"})

	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("DownloadPocztex: %d %s", res.StatusCode, base))
		return
	}

	body, err = ioutil.ReadAll(res.Body)
	return
}

func dlDPD(base, id string) (body []byte, err error) {
	form := url.Values{}
	form.Add("q", id)
	form.Add("typ", "1")

	req, err := http.NewRequest("POST", base, strings.NewReader(form.Encode()))
	if err != nil {
		return
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("DownloadDPD: %d %s", res.StatusCode, base))
		return
	}

	body, err = ioutil.ReadAll(res.Body)
	return
}

func checkService(id string, s Service) {
	res, err := s.Check(id)
	if (err == nil) && (res != "") {
		fmt.Printf("%20s %-8s %s\n", id, s.Name, res)
	}
	wg.Done()
}

var (
	client   *http.Client
	replacer *regexp.Regexp
	wg       sync.WaitGroup
)

func main() {
	services := [...]Service{
		Service{
			"DHL",
			dlSimpleGet,
			"^\\d{11}$",
			"http://www.dhl.com.pl/sledzenieprzesylkikrajowej/szukaj.aspx?m=0&sn=%s",
			"//*[@id='middle']/table/tbody/tr[2]/td[4]/text()[1]",
			nil,
		},
		Service{
			"DPD",
			dlDPD,
			"^\\w{14}$",
			"https://tracktrace.dpd.com.pl/findPackage",
			"//table/tr[2]/td[3]/text()",
			nil,
		},
		Service{
			"SIÓDEMKA",
			dlSimpleGet,
			"^\\d{13}$",
			"https://siodemka.com/tracking/%s/",
			"//*[@id='page']/div[2]/table[2]/tbody/tr[4]/td[4]",
			nil,
		},
		Service{
			"UPS",
			dlSimpleGet,
			"^1Z\\w{16}$",
			"http://wwwapps.ups.com/WebTracking/track?loc=pl_PL&HTMLVersion=5.0&Requester=UPSHome&WBPM_lid=homepage/ct1.html_pnl_trk&trackNums=%s&track.x=Monitoruj",
			"//*[@id='tt_spStatus']/text()",
			nil,
		},
		Service{
			"GLS",
			dlSimpleGet,
			"^\\d{11}$",
			"https://gls-group.eu/app/service/open/rest/PL/pl/rstt001?match=%s&caller=witt002",
			"",
			regexp.MustCompile("\"statusText\":\"(.*?)\""),
		},
		Service{
			"K-EX",
			dlSimpleGet,
			"^\\d{9}$",
			"http://kurier.k-ex.pl/tnt_szczegoly.php?nr=%s",
			"//table[last()]/tr[last()]/td[4]/text()",
			nil,
		},
		Service{
			"Pocztex",
			dlPocztex,
			"^\\d{20}$",
			"http://www.pocztex.pl/sledzenie/wssClient.php",
			"//table[@id='zadarzenia_td']/tr[last()]/td[1]/text()",
			nil,
		},
	}

	client = &http.Client{}
	replacer = regexp.MustCompile("\\s{2,}|\\t+|\\\\n")

	for _, id := range os.Args[1:] {
		for _, s := range services {
			if s.IsMatch(id) {
				wg.Add(1)
				go checkService(id, s)
			}
		}
	}

	wg.Wait()
}
<commit_msg>kurier.go: add InPost service<commit_after>// What's the delivery status?
// dRbiG, 2014
// See LICENSE.txt

package main

import (
	"errors"
	"fmt"
	"github.com/kennygrant/sanitize"
	"github.com/moovweb/gokogiri"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
)

type DlFunc func(base, id string) (body []byte, err error)

type Service struct {
	Name       string
	Downloader DlFunc
	Matcher    string
	URL        string
	XPath      string
	Extractor  *regexp.Regexp
}

func (s *Service) IsMatch(id string) bool {
	matched, err := regexp.MatchString(s.Matcher, id)
	if err != nil {
		panic(err)
	}
	return matched
}

func (s *Service) Check(id string) (status string, err error) {
	body, err := s.Downloader(s.URL, id)
	if err != nil {
		return
	}

	if s.Extractor != nil {
		parts := s.Extractor.FindSubmatch(body)
		if parts == nil {
			return "", nil
		}

		status = string(parts[1])
	} else {
		doc, err := gokogiri.ParseHtml(body)
		if err != nil {
			return "", err
		}
		defer doc.Free()

		res, err := doc.Search(s.XPath)
		if err != nil {
			return "", err
		}
		if len(res) < 1 {
			return "", nil
		}

		status = sanitize.HTML(res[0].String())
		status = replacer.ReplaceAllString(status, " ")
		status = strings.TrimSpace(status)
	}

	return
}

func dlSimpleGet(base, id string) (body []byte, err error) {
	url := fmt.Sprintf(base, id)

	res, err := client.Get(url)
	if err != nil {
		return
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("DownloadPage: %d %s", res.StatusCode, url))
		return
	}

	body, err = ioutil.ReadAll(res.Body)
	return
}

func dlPocztex(base, id string) (body []byte, err error) {
	form := url.Values{}
	form.Add("n", id)
	form.Add("s", "1")

	req, err := http.NewRequest("POST", base, strings.NewReader(form.Encode()))
	if err != nil {
		return
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.AddCookie(&http.Cookie{Name: "PHPSESSID", Value: "1"})

	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("DownloadPocztex: %d %s", res.StatusCode, base))
		return
	}

	body, err = ioutil.ReadAll(res.Body)
	return
}

func dlDPD(base, id string) (body []byte, err error) {
	form := url.Values{}
	form.Add("q", id)
	form.Add("typ", "1")

	req, err := http.NewRequest("POST", base, strings.NewReader(form.Encode()))
	if err != nil {
		return
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("DownloadDPD: %d %s", res.StatusCode, base))
		return
	}

	body, err = ioutil.ReadAll(res.Body)
	return
}

func checkService(id string, s Service) {
	res, err := s.Check(id)
	if (err == nil) && (res != "") {
		fmt.Printf("%20s %-8s %s\n", id, s.Name, res)
	}
	wg.Done()
}

var (
	client   *http.Client
	replacer *regexp.Regexp
	wg       sync.WaitGroup
)

func main() {
	services := [...]Service{
		Service{
			"DHL",
			dlSimpleGet,
			"^\\d{11}$",
			"http://www.dhl.com.pl/sledzenieprzesylkikrajowej/szukaj.aspx?m=0&sn=%s",
			"//*[@id='middle']/table/tbody/tr[2]/td[4]/text()[1]",
			nil,
		},
		Service{
			"DPD",
			dlDPD,
			"^\\w{14}$",
			"https://tracktrace.dpd.com.pl/findPackage",
			"//table/tr[2]/td[3]/text()",
			nil,
		},
		Service{
			"SIÓDEMKA",
			dlSimpleGet,
			"^\\d{13}$",
			"https://siodemka.com/tracking/%s/",
			"//*[@id='page']/div[2]/table[2]/tbody/tr[4]/td[4]",
			nil,
		},
		Service{
			"UPS",
			dlSimpleGet,
			"^1Z\\w{16}$",
			"http://wwwapps.ups.com/WebTracking/track?loc=pl_PL&HTMLVersion=5.0&Requester=UPSHome&WBPM_lid=homepage/ct1.html_pnl_trk&trackNums=%s&track.x=Monitoruj",
			"//*[@id='tt_spStatus']/text()",
			nil,
		},
		Service{
			"GLS",
			dlSimpleGet,
			"^\\d{11}$",
			"https://gls-group.eu/app/service/open/rest/PL/pl/rstt001?match=%s&caller=witt002",
			"",
			regexp.MustCompile("\"statusText\":\"(.*?)\""),
		},
		Service{
			"K-EX",
			dlSimpleGet,
			"^\\d{9}$",
			"http://kurier.k-ex.pl/tnt_szczegoly.php?nr=%s",
			"//table[last()]/tr[last()]/td[4]/text()",
			nil,
		},
		Service{
			"Pocztex",
			dlPocztex,
			"^\\d{20}$",
			"http://www.pocztex.pl/sledzenie/wssClient.php",
			"//table[@id='zadarzenia_td']/tr[last()]/td[1]/text()",
			nil,
		},
		Service{
			"InPost",
			dlSimpleGet,
			"^\\d{24}$",
			"https://paczkomaty.pl/pl/znajdz-paczke?parcel=%s",
			"//*[@id='find-parcel']/div[3]/div/table/tbody/tr[3]/td",
			nil,
		},
	}

	client = &http.Client{}
	replacer = regexp.MustCompile("\\s{2,}|\\t+|\\\\n")

	for _, id := range os.Args[1:] {
		for _, s := range services {
			if s.IsMatch(id) {
				wg.Add(1)
				go checkService(id, s)
			}
		}
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package terraform

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
)

const appName string = "concourse-up"

// IClient is an interface for the terraform Client
type IClient interface {
	Output() (*Metadata, error)
	Apply() error
	Destroy() error
	Cleanup() error
}

// Client wraps common terraform commands
type Client struct {
	configDir string
	stdout    io.Writer
	stderr    io.Writer
}

// ClientFactory is a function that builds a client interface
type ClientFactory func(config []byte, stdout, stderr io.Writer) (IClient, error)

// NewClient is a concrete implementation of ClientFactory
func NewClient(config []byte, stdout, stderr io.Writer) (IClient, error) {
	if err := checkTerraformOnPath(stderr, stderr); err != nil {
		return nil, err
	}

	configDir, err := initConfig(config, stderr, stderr)
	if err != nil {
		return nil, err
	}

	return &Client{
		configDir: configDir,
		stdout:    stdout,
		stderr:    stderr,
	}, nil
}

// Cleanup cleans up the temporary directory used by terraform
func (client *Client) Cleanup() error {
	return os.RemoveAll(client.configDir)
}

// Output fetches the terraform output/metadata
func (client *Client) Output() (*Metadata, error) {
	stdoutBuffer := bytes.NewBuffer(nil)
	if err := terraform([]string{
		"output",
		"-json",
	}, client.configDir, stdoutBuffer, client.stderr); err != nil {
		return nil, err
	}

	metadata := Metadata{}
	if err := json.NewDecoder(stdoutBuffer).Decode(&metadata); err != nil {
		return nil, err
	}

	return &metadata, nil
}

// Apply takes a terraform config and applies it
func (client *Client) Apply() error {
	return terraform([]string{
		"apply",
		"-input=false",
	}, client.configDir, client.stdout, client.stderr)
}

// Destroy destroys the given terraform config
func (client *Client) Destroy() error {
	return terraform([]string{
		"destroy",
		"-force",
	}, client.configDir, client.stdout, client.stderr)
}

func checkTerraformOnPath(stdout, stderr io.Writer) error {
	if err := terraform([]string{"version"}, "", stdout, stderr); err != nil {
		return fmt.Errorf("Error running `terraform version`, is terraform in your PATH?\n%s", err.Error())
	}
	return nil
}

func terraform(args []string, dir string, stdout, stderr io.Writer) error {
	cmd := exec.Command("terraform", args...)
	cmd.Dir = dir
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	return cmd.Run()
}

func initConfig(config []byte, stdout, stderr io.Writer) (string, error) {
	// write out config
	tmpDir, err := ioutil.TempDir("", appName)
	if err != nil {
		return "", err
	}

	configPath := filepath.Join(tmpDir, "main.tf")
	if err := ioutil.WriteFile(configPath, config, 0777); err != nil {
		return "", err
	}

	if err := terraform([]string{
		"init",
	}, tmpDir, stdout, stderr); err != nil {
		return "", err
	}

	return tmpDir, nil
}
<commit_msg>improve error message when terraform missing from path<commit_after>package terraform

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
)

const appName string = "concourse-up"

// IClient is an interface for the terraform Client
type IClient interface {
	Output() (*Metadata, error)
	Apply() error
	Destroy() error
	Cleanup() error
}

// Client wraps common terraform commands
type Client struct {
	configDir string
	stdout    io.Writer
	stderr    io.Writer
}

// ClientFactory is a function that builds a client interface
type ClientFactory func(config []byte, stdout, stderr io.Writer) (IClient, error)

// NewClient is a concrete implementation of ClientFactory
func NewClient(config []byte, stdout, stderr io.Writer) (IClient, error) {
	if err := checkTerraformOnPath(stderr, stderr); err != nil {
		return nil, err
	}

	configDir, err := initConfig(config, stderr, stderr)
	if err != nil {
		return nil, err
	}

	return &Client{
		configDir: configDir,
		stdout:    stdout,
		stderr:    stderr,
	}, nil
}

// Cleanup cleans up the temporary directory used by terraform
func (client *Client) Cleanup() error {
	return os.RemoveAll(client.configDir)
}

// Output fetches the terraform output/metadata
func (client *Client) Output() (*Metadata, error) {
	stdoutBuffer := bytes.NewBuffer(nil)
	if err := terraform([]string{
		"output",
		"-json",
	}, client.configDir, stdoutBuffer, client.stderr); err != nil {
		return nil, err
	}

	metadata := Metadata{}
	if err := json.NewDecoder(stdoutBuffer).Decode(&metadata); err != nil {
		return nil, err
	}

	return &metadata, nil
}

// Apply takes a terraform config and applies it
func (client *Client) Apply() error {
	return terraform([]string{
		"apply",
		"-input=false",
	}, client.configDir, client.stdout, client.stderr)
}

// Destroy destroys the given terraform config
func (client *Client) Destroy() error {
	return terraform([]string{
		"destroy",
		"-force",
	}, client.configDir, client.stdout, client.stderr)
}

func checkTerraformOnPath(stdout, stderr io.Writer) error {
	if err := terraform([]string{"version"}, "", stdout, stderr); err != nil {
		return fmt.Errorf("Error running `terraform version`, is terraform in your PATH?\n%s\n\nDownload terraform here: https://www.terraform.io/downloads.html\n", err.Error())
	}
	return nil
}

func terraform(args []string, dir string, stdout, stderr io.Writer) error {
	cmd := exec.Command("terraform", args...)
	cmd.Dir = dir
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	return cmd.Run()
}

func initConfig(config []byte, stdout, stderr io.Writer) (string, error) {
	// write out config
	tmpDir, err := ioutil.TempDir("", appName)
	if err != nil {
		return "", err
	}

	configPath := filepath.Join(tmpDir, "main.tf")
	if err := ioutil.WriteFile(configPath, config, 0777); err != nil {
		return "", err
	}

	if err := terraform([]string{
		"init",
	}, tmpDir, stdout, stderr); err != nil {
		return "", err
	}

	return tmpDir, nil
}
<|endoftext|>


<commit_before>package s3api

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/gorilla/mux"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {

	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	_, err := validateContentMd5(r.Header)
	if err != nil {
		writeErrorResponse(w, ErrInvalidDigest, r.URL)
		return
	}

	rAuthType := getRequestAuthType(r)
	dataReader := r.Body
	if rAuthType == authTypeStreamingSigned {
		dataReader = newSignV4ChunkedReader(r)
	}

	uploadUrl := fmt.Sprintf("http://%s%s/%s%s?collection=%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object, bucket)

	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

	if errCode != ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	setEtag(w, etag)

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	if strings.HasSuffix(r.URL.Path, "/") {
		writeErrorResponse(w, ErrNotImplemented, r.URL)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroghResponse)

}

func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroghResponse)

}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, func(proxyResonse *http.Response, w http.ResponseWriter) {
		for k, v := range proxyResonse.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(http.StatusNoContent)
	})

}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
	// TODO
	writeErrorResponse(w, ErrNotImplemented, r.URL)
}

func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) {

	glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)

	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", destUrl, err)
		writeErrorResponse(w, ErrInternalError, r.URL)
		return
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)
	proxyReq.Header.Set("Etag-MD5", "True")

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		writeErrorResponse(w, ErrInternalError, r.URL)
		return
	}
	defer resp.Body.Close()

	responseFn(resp, w)
}
func passThroghResponse(proxyResonse *http.Response, w http.ResponseWriter) {
	for k, v := range proxyResonse.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(proxyResonse.StatusCode)
	io.Copy(w, proxyResonse.Body)
}

func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.ReadCloser) (etag string, code ErrorCode) {

	hash := md5.New()
	var body io.Reader = io.TeeReader(dataReader, hash)

	proxyReq, err := http.NewRequest("PUT", uploadUrl, body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", uploadUrl, err)
		return "", ErrInternalError
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	dataReader.Close()

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		return "", ErrInternalError
	}
	defer resp.Body.Close()

	etag = fmt.Sprintf("%x", hash.Sum(nil))

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.Errorf("upload to filer response read: %v", ra_err)
		return etag, ErrInternalError
	}
	var ret weed_server.FilerPostResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body))
		return "", ErrInternalError
	}
	if ret.Error != "" {
		glog.Errorf("upload to filer error: %v", ret.Error)
		return "", ErrInternalError
	}

	return etag, ErrNone
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getObject(vars map[string]string) string {
	object := vars["object"]
	if !strings.HasPrefix(object, "/") {
		object = "/" + object
	}
	return object
}
<commit_msg>fix spelling<commit_after>package s3api

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/gorilla/mux"
)

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {

	// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	_, err := validateContentMd5(r.Header)
	if err != nil {
		writeErrorResponse(w, ErrInvalidDigest, r.URL)
		return
	}

	rAuthType := getRequestAuthType(r)
	dataReader := r.Body
	if rAuthType == authTypeStreamingSigned {
		dataReader = newSignV4ChunkedReader(r)
	}

	uploadUrl := fmt.Sprintf("http://%s%s/%s%s?collection=%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object, bucket)

	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

	if errCode != ErrNone {
		writeErrorResponse(w, errCode, r.URL)
		return
	}

	setEtag(w, etag)

	writeSuccessResponseEmpty(w)
}

func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	if strings.HasSuffix(r.URL.Path, "/") {
		writeErrorResponse(w, ErrNotImplemented, r.URL)
		return
	}

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)

}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	vars := mux.Vars(r)
	bucket := vars["bucket"]
	object := getObject(vars)

	destUrl := fmt.Sprintf("http://%s%s/%s%s",
		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)

	s3a.proxyToFiler(w, r, destUrl, func(proxyResonse *http.Response, w http.ResponseWriter) {
		for k, v := range proxyResonse.Header {
			w.Header()[k] = v
		}
		w.WriteHeader(http.StatusNoContent)
	})

}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
	// TODO
	writeErrorResponse(w, ErrNotImplemented, r.URL)
}

func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResonse *http.Response, w http.ResponseWriter)) {

	glog.V(2).Infof("s3 proxying %s to %s", r.Method, destUrl)

	proxyReq, err := http.NewRequest(r.Method, destUrl, r.Body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", destUrl, err)
		writeErrorResponse(w, ErrInternalError, r.URL)
		return
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)
	proxyReq.Header.Set("Etag-MD5", "True")

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		writeErrorResponse(w, ErrInternalError, r.URL)
		return
	}
	defer resp.Body.Close()

	responseFn(resp, w)
}
func passThroughResponse(proxyResonse *http.Response, w http.ResponseWriter) {
	for k, v := range proxyResonse.Header {
		w.Header()[k] = v
	}
	w.WriteHeader(proxyResonse.StatusCode)
	io.Copy(w, proxyResonse.Body)
}

func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.ReadCloser) (etag string, code ErrorCode) {

	hash := md5.New()
	var body io.Reader = io.TeeReader(dataReader, hash)

	proxyReq, err := http.NewRequest("PUT", uploadUrl, body)

	if err != nil {
		glog.Errorf("NewRequest %s: %v", uploadUrl, err)
		return "", ErrInternalError
	}

	proxyReq.Header.Set("Host", s3a.option.Filer)
	proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr)

	for header, values := range r.Header {
		for _, value := range values {
			proxyReq.Header.Add(header, value)
		}
	}

	resp, postErr := client.Do(proxyReq)

	dataReader.Close()

	if postErr != nil {
		glog.Errorf("post to filer: %v", postErr)
		return "", ErrInternalError
	}
	defer resp.Body.Close()

	etag = fmt.Sprintf("%x", hash.Sum(nil))

	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		glog.Errorf("upload to filer response read: %v", ra_err)
		return etag, ErrInternalError
	}
	var ret weed_server.FilerPostResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.Errorf("failing to read upload to %s : %v", uploadUrl, string(resp_body))
		return "", ErrInternalError
	}
	if ret.Error != "" {
		glog.Errorf("upload to filer error: %v", ret.Error)
		return "", ErrInternalError
	}

	return etag, ErrNone
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getObject(vars map[string]string) string {
	object := vars["object"]
	if !strings.HasPrefix(object, "/") {
		object = "/" + object
	}
	return object
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"

	"code.google.com/p/gogoprotobuf/proto"

	"github.com/ninjasphere/go-zigbee/nwkmgr"

	"github.com/ninjasphere/go-zigbee/gateway"
	"github.com/ninjasphere/go-zigbee/otasrvr"
)

const (
	hostname    = "beaglebone.local"
	otasrvrPort = 2525
	gatewayPort = 2541
	nwkmgrPort  = 2540
)

// ZStackServer holds the connection to one of the Z-Stack servers (nwkmgr, gateway and otasrvr)
type ZStackServer struct {
	Incoming chan *[]byte // Incoming raw protobuf packets

	name               string
	subsystem          int8
	conn               net.Conn
	outgoing           chan *ZStackCommand // Outgoing protobuf messages
	pendingByCommandID map[int8]*ZStackPendingResponse
}

func (s *ZStackServer) sendCommand(message *ZStackCommand) {
	s.outgoing <- message
}

func (s *ZStackServer) sendRequest(request *ZStackCommand, response *ZStackCommand) (chan error, error) {
	s.outgoing <- request

	pending := &ZStackPendingResponse{
		message:  response.message,
		complete: make(chan error),
	}

	if s.pendingByCommandID[response.commandID] != nil {
		return nil, fmt.Errorf("There is already a pending command waiting for the same response (Command ID: %X)", response.commandID)
	}

	s.pendingByCommandID[response.commandID] = pending

	return pending.complete, nil
}

// ZStackCommand contains a protobuf message and a command id
type ZStackCommand struct {
	message   proto.Message
	commandID int8
}

// ZStackPendingResponse contains the protobuf command to be filled with the response, and a 'complete' channel to indicate when done
type ZStackPendingResponse struct {
	message  proto.Message
	complete chan error
}

type ZStackNwkMgrServer struct {
	*ZStackServer
}

// SendCommand sends a protobuf Message to the Z-Stack server
func (s *ZStackNwkMgrServer) SendCommand(command zStackNwkCommand) {
	s.sendCommand(&ZStackCommand{
		message:   command,
		commandID: int8(command.GetCmdId()),
	})
}

// SendRequest sends a protobuf Message to the Z-Stack server, and waits for the response
func (s *ZStackNwkMgrServer) SendRequest(request zStackNwkCommand, response zStackNwkCommand) error {

	complete, err := s.sendRequest(&ZStackCommand{
		message:   request,
		commandID: int8(request.GetCmdId()),
	}, &ZStackCommand{
		message:   response,
		commandID: int8(response.GetCmdId()),
	})

	if err != nil {
		return err
	}

	return <-complete
}

type zStackNwkCommand interface {
	proto.Message
	GetCmdId() nwkmgr.NwkMgrCmdIdT
}

type zStackGatewayMessage interface {
	GetCmdId() gateway.GwCmdIdT
}

type zStackOtaMgrMessage interface {
	GetCmdId() otasrvr.OtaMgrCmdIdT
}

func (s *ZStackServer) outgoingLoop() {
	for {
		command := <-s.outgoing

		proto.SetDefaults(command.message)

		packet, err := proto.Marshal(command.message)
		if err != nil {
			log.Fatal("marshaling error: ", err)
		}

		log.Printf("Protobuf packet %x", packet)

		buffer := new(bytes.Buffer)

		if err != nil {
			// handle error
			log.Printf("Error connecting %s", err)
		}

		// Add the Z-Stack 4-byte header
		err = binary.Write(buffer, binary.LittleEndian, uint16(len(packet))) // Packet length
		err = binary.Write(buffer, binary.LittleEndian, s.subsystem)         // Subsystem
		err = binary.Write(buffer, binary.LittleEndian, command.commandID)   // Command Id

		_, err = buffer.Write(packet)

		log.Printf("%s: Sending packet: % X", s.name, buffer.Bytes())

		// Send it to the Z-Stack server
		_, err = s.conn.Write(buffer.Bytes())
	}
}

func (s *ZStackServer) incomingLoop() {
	for {
		buf := make([]byte, 1024)
		n, err := s.conn.Read(buf)
		//log.Printf("Read %d from %s", n, s.name)
		if err != nil {
			log.Fatalf("%s: Error reading socket %s", s.name, err)
		}
		pos := 0

		for {
			var length uint16
			var incomingSubsystem uint8
			reader := bytes.NewReader(buf[pos:])
			err := binary.Read(reader, binary.LittleEndian, &length)
			if err != nil {
				log.Fatalf("%s: Failed to read packet length %s", s.name, err)
			}

			err = binary.Read(reader, binary.LittleEndian, &incomingSubsystem)
			if err != nil {
				log.Fatalf("%s: Failed to read packet subsystem %s", s.name, err)
			}

			log.Printf("%s: Incoming subsystem %d (wanted: %d)", s.name, incomingSubsystem, s.subsystem)

			log.Printf("%s: Found packet of size : %d", s.name, length)

			packet := buf[pos+4 : pos+4+int(length)]

			commandID := int8(packet[1])

			// Check if this packet has a ZCL request id... TODO

			// Check if we have any pending requests that want this command id...
			pending := s.pendingByCommandID[commandID]

			if pending != nil {
				pending.complete <- proto.Unmarshal(packet, pending.message)

			} else { // Or just send it out to be handled elsewhere
				s.Incoming <- &packet
			}

			log.Printf("%s: Command ID:0x%X Packet: % X", s.name, commandID, packet)
			pos += int(length) + 4

			if pos >= n {
				break
			}
		}
		//fmt.Printf("Received from %s (len:%d) : % X", s.name, n, buf[:n])
	}
}

func connectToServer(name string, subsystem int8, port int) (*ZStackServer, error) {

	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", hostname, port))

	if err != nil {
		return nil, err
	}

	server := &ZStackServer{
		name:               name,
		subsystem:          subsystem,
		conn:               conn,
		outgoing:           make(chan *ZStackCommand),
		Incoming:           make(chan *[]byte),
		pendingByCommandID: make(map[int8]*ZStackPendingResponse),
	}

	go server.incomingLoop()
	go server.outgoingLoop()

	return server, nil
}

func main() {
	log.Println("Starting")

	_, err := connectToServer("otasrvr", int8(otasrvr.ZStackOTASysIDs_RPC_SYS_PB_OTA_MGR), otasrvrPort)
	if err != nil {
		// handle error
		log.Printf("Error connecting otasrvr %s", err)
	}

	nwkmgrTemp, err := connectToServer("nwkmgr", int8(nwkmgr.ZStackNwkMgrSysIdT_RPC_SYS_PB_NWK_MGR), nwkmgrPort)
	if err != nil {
		// handle error
		log.Printf("Error connecting nwkmgr %s", err)
	}
	nwkmgrConn := &ZStackNwkMgrServer{nwkmgrTemp}

	gatewayConn, err := connectToServer("gateway", int8(gateway.ZStackGwSysIdT_RPC_SYS_PB_GW), gatewayPort)
	if err != nil {
		// handle error
		log.Printf("Error connecting gateway %s", err)
	}

	go func() {
		for {
			bytes := <-gatewayConn.Incoming

			log.Printf("Got gateway message % X", bytes)

			var message = &gateway.GwAttributeReportingInd{}
			err = proto.Unmarshal(*bytes, message)
			outJSON(message)

		}
	}()

	//command := &nwkmgr.NwkZigbeeNwkInfoReq{}
	response := &nwkmgr.NwkGetLocalDeviceInfoCnf{}
	err = nwkmgrConn.SendRequest(&nwkmgr.NwkGetLocalDeviceInfoReq{}, response)

	if err != nil {
		log.Fatalf("Failed to get local device info: %s", err)
	}
	log.Println("Local device info: ")
	outJSON(response)

	joinTime := uint32(30)
	permitJoinRequest := &nwkmgr.NwkSetPermitJoinReq{
		PermitJoinTime: &joinTime,
		PermitJoin:     nwkmgr.NwkPermitJoinTypeT_PERMIT_ALL.Enum(),
	}

	permitJoinResponse := &nwkmgr.NwkZigbeeGenericCnf{}

	err = nwkmgrConn.SendRequest(permitJoinRequest, permitJoinResponse)
	if err != nil {
		log.Fatalf("Failed to enable joining: %s", err)
	}
	log.Println("Permit join response: ")
	outJSON(permitJoinResponse)

	deviceListResponse := &nwkmgr.NwkGetDeviceListCnf{}

	err = nwkmgrConn.SendRequest(&nwkmgr.NwkGetDeviceListReq{}, deviceListResponse)
	if err != nil {
		log.Fatalf("Failed to get device list: %s", err)
	}
	log.Println("Device list: ")
	outJSON(deviceListResponse)

	//time.Sleep(2000 * time.Millisecond)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	fmt.Println("Got signal:", <-c)

}

func outJSON(thing interface{}) {
	jsonOut, _ := json.Marshal(thing)

	log.Printf("%s", jsonOut)
}
<commit_msg>Toggling the power on and off!<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"time"

	"code.google.com/p/gogoprotobuf/proto"

	"github.com/ninjasphere/go-zigbee/nwkmgr"

	"github.com/ninjasphere/go-zigbee/gateway"
	"github.com/ninjasphere/go-zigbee/otasrvr"
)

const (
	hostname    = "beaglebone.local"
	otasrvrPort = 2525
	gatewayPort = 2541
	nwkmgrPort  = 2540
)

// ZStackServer holds the connection to one of the Z-Stack servers (nwkmgr, gateway and otasrvr)
type ZStackServer struct {
	Incoming chan *[]byte // Incoming raw protobuf packets

	name               string
	subsystem          int8
	conn               net.Conn
	outgoing           chan *ZStackCommand // Outgoing protobuf messages
	pendingByCommandID map[int8]*ZStackPendingResponse
}

func (s *ZStackServer) sendCommand(message *ZStackCommand) {
	s.outgoing <- message
}

func (s *ZStackServer) sendRequest(request *ZStackCommand, response *ZStackCommand) (chan error, error) {
	s.outgoing <- request

	pending := &ZStackPendingResponse{
		message:  response.message,
		complete: make(chan error),
	}

	if s.pendingByCommandID[response.commandID] != nil {
		return nil, fmt.Errorf("There is already a pending command waiting for the same response (Command ID: %X)", response.commandID)
	}

	s.pendingByCommandID[response.commandID] = pending

	return pending.complete, nil
}

// ZStackCommand contains a protobuf message and a command id
type ZStackCommand struct {
	message   proto.Message
	commandID int8
}

// ZStackPendingResponse contains the protobuf command to be filled with the response, and a 'complete' channel to indicate when done
type ZStackPendingResponse struct {
	message  proto.Message
	complete chan error
}

type ZStackNwkMgrServer struct {
	*ZStackServer
}

// SendCommand sends a protobuf Message to the Z-Stack server
func (s *ZStackNwkMgrServer) SendCommand(command zStackNwkCommand) {
	s.sendCommand(&ZStackCommand{
		message:   command,
		commandID: int8(command.GetCmdId()),
	})
}

// SendRequest sends a protobuf Message to the Z-Stack server, and waits for the response
func (s *ZStackNwkMgrServer) SendRequest(request zStackNwkCommand, response zStackNwkCommand) error {

	complete, err := s.sendRequest(&ZStackCommand{
		message:   request,
		commandID: int8(request.GetCmdId()),
	}, &ZStackCommand{
		message:   response,
		commandID: int8(response.GetCmdId()),
	})

	if err != nil {
		return err
	}

	return <-complete
}

type ZStackGatewayServer struct {
	*ZStackServer
}

// SendCommand sends a protobuf Message to the Z-Stack server
func (s *ZStackGatewayServer) SendCommand(command zStackGatewayCommand) {
	s.sendCommand(&ZStackCommand{
		message:   command,
		commandID: int8(command.GetCmdId()),
	})
}

// SendRequest sends a protobuf Message to the Z-Stack server, and waits for the response
func (s *ZStackGatewayServer) SendRequest(request zStackGatewayCommand, response zStackGatewayCommand) error {

	complete, err := s.sendRequest(&ZStackCommand{
		message:   request,
		commandID: int8(request.GetCmdId()),
	}, &ZStackCommand{
		message:   response,
		commandID: int8(response.GetCmdId()),
	})

	if err != nil {
		return err
	}

	return <-complete
}

type zStackNwkCommand interface {
	proto.Message
	GetCmdId() nwkmgr.NwkMgrCmdIdT
}

type zStackGatewayCommand interface {
	proto.Message
	GetCmdId() gateway.GwCmdIdT
}

type zStackOtaMgrCommand interface {
	proto.Message
	GetCmdId() otasrvr.OtaMgrCmdIdT
}

func (s *ZStackServer) outgoingLoop() {
	for {
		command := <-s.outgoing

		proto.SetDefaults(command.message)

		packet, err := proto.Marshal(command.message)
		if err != nil {
			log.Fatal("marshaling error: ", err)
		}

		log.Printf("Protobuf packet %x", packet)

		buffer := new(bytes.Buffer)

		if err != nil {
			// handle error
			log.Printf("Error connecting %s", err)
		}

		// Add the Z-Stack 4-byte header
		err = binary.Write(buffer, binary.LittleEndian, uint16(len(packet))) // Packet length
		err = binary.Write(buffer, binary.LittleEndian, s.subsystem)         // Subsystem
		err = binary.Write(buffer, binary.LittleEndian, command.commandID)   // Command Id

		_, err = buffer.Write(packet)

		log.Printf("%s: Sending packet: % X", s.name, buffer.Bytes())

		// Send it to the Z-Stack server
		_, err = s.conn.Write(buffer.Bytes())
	}
}

func (s *ZStackServer) incomingLoop() {
	for {
		buf := make([]byte, 1024)
		n, err := s.conn.Read(buf)
		//log.Printf("Read %d from %s", n, s.name)
		if err != nil {
			log.Fatalf("%s: Error reading socket %s", s.name, err)
		}
		pos := 0

		for {
			var length uint16
			var incomingSubsystem uint8
			reader := bytes.NewReader(buf[pos:])
			err := binary.Read(reader, binary.LittleEndian, &length)
			if err != nil {
				log.Fatalf("%s: Failed to read packet length %s", s.name, err)
			}

			err = binary.Read(reader, binary.LittleEndian, &incomingSubsystem)
			if err != nil {
				log.Fatalf("%s: Failed to read packet subsystem %s", s.name, err)
			}

			log.Printf("%s: Incoming subsystem %d (wanted: %d)", s.name, incomingSubsystem, s.subsystem)

			log.Printf("%s: Found packet of size : %d", s.name, length)

			packet := buf[pos+4 : pos+4+int(length)]

			commandID := int8(packet[1])

			log.Printf("%s: Command ID:0x%X Packet: % X", s.name, commandID, packet)

			// Check if this packet has a ZCL request id... TODO

			// Check if we have any pending requests that want this command id...
			pending := s.pendingByCommandID[commandID]

			if pending != nil {
				s.pendingByCommandID[commandID] = nil
				pending.complete <- proto.Unmarshal(packet, pending.message)

			} else { // Or just send it out to be handled elsewhere
				s.Incoming <- &packet
			}

			pos += int(length) + 4

			if pos >= n {
				break
			}
		}
		//fmt.Printf("Received from %s (len:%d) : % X", s.name, n, buf[:n])
	}
}

func connectToServer(name string, subsystem int8, port int) (*ZStackServer, error) {

	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", hostname, port))

	if err != nil {
		return nil, err
	}

	server := &ZStackServer{
		name:               name,
		subsystem:          subsystem,
		conn:               conn,
		outgoing:           make(chan *ZStackCommand),
		Incoming:           make(chan *[]byte),
		pendingByCommandID: make(map[int8]*ZStackPendingResponse),
	}

	go server.incomingLoop()
	go server.outgoingLoop()

	return server, nil
}

func main() {
	log.Println("Starting")

	_, err := connectToServer("otasrvr", int8(otasrvr.ZStackOTASysIDs_RPC_SYS_PB_OTA_MGR), otasrvrPort)
	if err != nil {
		// handle error
		log.Printf("Error connecting otasrvr %s", err)
	}

	nwkmgrTemp, err := connectToServer("nwkmgr", int8(nwkmgr.ZStackNwkMgrSysIdT_RPC_SYS_PB_NWK_MGR), nwkmgrPort)
	if err != nil {
		// handle error
		log.Printf("Error connecting nwkmgr %s", err)
	}
	nwkmgrConn := &ZStackNwkMgrServer{nwkmgrTemp}

	gatewayTemp, err := connectToServer("gateway", int8(gateway.ZStackGwSysIdT_RPC_SYS_PB_GW), gatewayPort)
	if err != nil {
		// handle error
		log.Printf("Error connecting gateway %s", err)
	}

	gatewayConn := &ZStackGatewayServer{gatewayTemp}

	go func() {
		for {
			bytes := <-gatewayConn.Incoming

			log.Printf("Got gateway message % X", bytes)

			var message = &gateway.GwAttributeReportingInd{}
			err = proto.Unmarshal(*bytes, message)
			outJSON(message)

		}
	}()

	//command := &nwkmgr.NwkZigbeeNwkInfoReq{}
	response := &nwkmgr.NwkGetLocalDeviceInfoCnf{}
	err = nwkmgrConn.SendRequest(&nwkmgr.NwkGetLocalDeviceInfoReq{}, response)

	if err != nil {
		log.Fatalf("Failed to get local device info: %s", err)
	}
	log.Println("Local device info: ")
	outJSON(response)

	joinTime := uint32(30)
	permitJoinRequest := &nwkmgr.NwkSetPermitJoinReq{
		PermitJoinTime: &joinTime,
		PermitJoin:     nwkmgr.NwkPermitJoinTypeT_PERMIT_ALL.Enum(),
	}

	permitJoinResponse := &nwkmgr.NwkZigbeeGenericCnf{}

	err = nwkmgrConn.SendRequest(permitJoinRequest, permitJoinResponse)
	if err != nil {
		log.Fatalf("Failed to enable joining: %s", err)
	}
	if permitJoinResponse.Status.String() != "STATUS_SUCCESS" {
		log.Fatalf("Failed to enable joining: %s", permitJoinResponse.Status)
	}
	log.Println("Permit join response: ")
	outJSON(permitJoinResponse)

	deviceListResponse := &nwkmgr.NwkGetDeviceListCnf{}

	err = nwkmgrConn.SendRequest(&nwkmgr.NwkGetDeviceListReq{}, deviceListResponse)
	if err != nil {
		log.Fatalf("Failed to get device list: %s", err)
	}
	log.Printf("Found %d device(s): ", len(deviceListResponse.DeviceList))
	outJSON(deviceListResponse)

	for _, device := range deviceListResponse.DeviceList {
		log.Printf("Got device : %d", device.IeeeAddress)
		for _, endpoint := range device.SimpleDescList {
			log.Printf("Got endpoint : %d", endpoint.EndpointId)

			if containsUInt32(endpoint.InputClusters, 0x06) {
				log.Printf("This endpoint has on/off cluster")

				onOffReq := &gateway.DevSetOnOffStateReq{
					DstAddress: &gateway.GwAddressStructT{
						AddressType: gateway.GwAddressTypeT_UNICAST.Enum(),
						IeeeAddr:    device.IeeeAddress,
					},
					State: gateway.GwOnOffStateT_TOGGLE_STATE.Enum(),
				}

				res := &gateway.GwZigbeeGenericCnf{}

				for i := 0; i < 3; i++ {
					log.Println("Toggling on/off device")

					err = gatewayConn.SendRequest(onOffReq, res)
					if err != nil {
						log.Fatalf("Failed to toggle device: ", err)
					}
					log.Printf("Got on/off response")
					outJSON(res)
					time.Sleep(2000 * time.Millisecond)
				}

			}

		}
		//GwOnOffStateT_TOGGLE_STATE
	}

	//time.Sleep(2000 * time.Millisecond)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	fmt.Println("Got signal:", <-c)

}

func containsUInt32(hackstack []uint32, needle uint32) bool {
	for _, cluster := range hackstack {
		if cluster == needle {
			return true
		}
	}
	return false
}

func outJSON(thing interface{}) {
	jsonOut, _ := json.Marshal(thing)

	log.Printf("%s", jsonOut)
}
<|endoftext|>


<commit_before>package cache

/*
TODO:

This is extremely ugly. We should further parameterize the
SBFactories and use them directly instead of repeating code.

*/

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/dockerutil"

	docker "github.com/fsouza/go-dockerclient"
	sb "github.com/open-lambda/open-lambda/worker/sandbox"
)

var unshareFlags []string = []string{"-fimu"}

const rootCacheSandboxDir = "/tmp/olcache"

func InitCacheFactory(opts *config.Config, cluster string) (cf *BufferedCacheFactory, root sb.ContainerSandbox, rootDir string, err error) {
	cf, root, rootDir, err = NewBufferedCacheFactory(opts, cluster)
	if err != nil {
		return nil, nil, "", err
	}

	return cf, root, rootDir, nil
}

// emptySBInfo wraps sandbox information necessary for the buffer.
type emptySBInfo struct {
	sandbox    sb.ContainerSandbox
	sandboxDir string
}

// BufferedCacheFactory maintains a buffer of sandboxes created by another factory.
type BufferedCacheFactory struct {
	delegate CacheFactory
	buffer   chan *emptySBInfo
	errors   chan error
	dir      string
	idxPtr   *int64
}

type CacheFactory interface {
	Create(sandboxDir string, rootCmd []string) (sb.ContainerSandbox, error)
	Cleanup()
}

// DockerCacheFactory is a SandboxFactory that creates docker sandboxes for the cache.
type DockerCacheFactory struct {
	client  *docker.Client
	cmd     []string
	caps    []string
	labels  map[string]string
	pkgsDir string
}

// NewDockerCacheFactory creates a CacheFactory that uses Docker containers.
func NewDockerCacheFactory(cluster, pkgsDir string) (*DockerCacheFactory, error) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}

	cmd := []string{"/init"}

	caps := []string{"SYS_ADMIN"}

	labels := map[string]string{
		dockerutil.DOCKER_LABEL_CLUSTER: cluster,
		dockerutil.DOCKER_LABEL_TYPE:    dockerutil.POOL,
	}

	cf := &DockerCacheFactory{client, cmd, caps, labels, pkgsDir}
	return cf, nil
}

// Create creates a docker container from the pool directory.
func (cf *DockerCacheFactory) Create(sandboxDir string, cmd []string) (sb.ContainerSandbox, error) {
	volumes := []string{
		fmt.Sprintf("%s:%s", sandboxDir, "/host"),
		fmt.Sprintf("%s:%s:ro", cf.pkgsDir, "/packages"),
	}

	container, err := cf.client.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Image:  dockerutil.CACHE_IMAGE,
				Labels: cf.labels,
				Cmd:    cmd,
			},
			HostConfig: &docker.HostConfig{
				Binds:      volumes,
				PidMode:    "host",
				CapAdd:     cf.caps,
				AutoRemove: true,
			},
		},
	)
	if err != nil {
		return nil, err
	}

	sandbox := sb.NewDockerSandbox("", sandboxDir, "", "", container, cf.client)

	return sandbox, nil
}

func (cf *DockerCacheFactory) Cleanup() {
	return
}

// OLContainerCacheFactory is a SandboxFactory that creates olcontainers for the cache.
type OLContainerCacheFactory struct {
	opts    *config.Config
	cgf     *sb.CgroupFactory
	cmd     []string
	baseDir string
	pkgsDir string
}

// NewOLContainerCacheFactory creates a CacheFactory that uses olcontainers.
func NewOLContainerCacheFactory(opts *config.Config, cluster, baseDir, pkgsDir string) (*OLContainerCacheFactory, error) {
	for _, cgroup := range sb.CGroupList {
		cgroupPath := path.Join("/sys/fs/cgroup", cgroup, sb.OLCGroupName)
		if err := os.MkdirAll(cgroupPath, 0700); err != nil {
			return nil, err
		}
	}

	cgf, err := sb.NewCgroupFactory("cache", opts.Cg_pool_size)
	if err != nil {
		return nil, err
	}

	if err := os.MkdirAll(rootCacheSandboxDir, 0777); err != nil {
		return nil, fmt.Errorf("failed to make root sandbox dir :: %v", err.Error())
	} else if err := syscall.Mount(rootCacheSandboxDir, rootCacheSandboxDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind root sandbox dir: %v", err.Error())
	} else if err := syscall.Mount("none", rootCacheSandboxDir, "", sb.PRIVATE, ""); err != nil {
		return nil, fmt.Errorf("failed to make root cache sandbox dir private :: %v", err.Error())
	}

	sbPkgsDir := path.Join(baseDir, "packages")
	if err := syscall.Mount(pkgsDir, sbPkgsDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind packages dir: %s -> %s :: %v", opts.Pkgs_dir, sbPkgsDir, err)
	} else if err := syscall.Mount("none", sbPkgsDir, "", sb.BIND_RO, ""); err != nil {
		return nil, fmt.Errorf("failed to bind packages dir RO: %s -> %s :: %v", opts.Pkgs_dir, sbPkgsDir, err)
	}

	return &OLContainerCacheFactory{opts, cgf, []string{"/init"}, baseDir, pkgsDir}, nil
}

// Create creates a docker sandbox from the pool directory.
func (cf *OLContainerCacheFactory) Create(hostDir string, startCmd []string) (sb.ContainerSandbox, error) {
	id_bytes, err := exec.Command("uuidgen").Output()
	if err != nil {
		return nil, err
	}
	id := strings.TrimSpace(string(id_bytes[:]))

	rootDir := path.Join(rootCacheSandboxDir, fmt.Sprintf("%s", id))
	if err := os.Mkdir(rootDir, 0700); err != nil {
		return nil, err
	}

	// NOTE: mount points are expected to exist in OLContainer_handler_base directory

	if err := syscall.Mount(cf.baseDir, rootDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind root dir: %s -> %s :: %v\n", cf.baseDir, rootDir, err)
	} else if err := syscall.Mount("none", rootDir, "", sb.BIND_RO, ""); err != nil {
		return nil, fmt.Errorf("failed to bind root dir RO: %s :: %v\n", rootDir, err)
	} else if err := syscall.Mount("none", rootDir, "", sb.PRIVATE, ""); err != nil {
		return nil, fmt.Errorf("failed to make root dir private :: %v", err)
	}

	sandbox, err := sb.NewOLContainerSandbox(cf.cgf, cf.opts, rootDir, id, startCmd, unshareFlags)
	if err != nil {
		return nil, err
	}

	sbHostDir := filepath.Join(rootDir, "host")
	if err := syscall.Mount(sbHostDir, sbHostDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind sandbox host dir onto itself :: %v\n", err)
	} else if err := syscall.Mount("none", sbHostDir, "", sb.SHARED, ""); err != nil {
		return nil, fmt.Errorf("failed to make sbHostDir shared :: %v\n", err)
	}

	if err := sandbox.MountDirs(hostDir, ""); err != nil {
		return nil, err
	}

	return sandbox, nil
}

func (cf *OLContainerCacheFactory) Cleanup() {
	for _, cgroup := range sb.CGroupList {
		cgroupPath := path.Join("/sys/fs/cgroup", cgroup, sb.OLCGroupName)
		os.Remove(cgroupPath)
	}

	runCmd([]string{"/bin/umount", "/tmp/cache_*/*"})
	runCmd([]string{"/bin/umount", "/tmp/cache_*"})
	runCmd([]string{"/bin/rm", "-rf", "/tmp/cache_*"})
}

// NewBufferedCacheFactory creates a BufferedCacheFactory and starts a go routine to
// fill the sandbox buffer.
func NewBufferedCacheFactory(opts *config.Config, cluster string) (*BufferedCacheFactory, sb.ContainerSandbox, string, error) {
	cacheDir := opts.Import_cache_dir
	pkgsDir := opts.Pkgs_dir
	buffer := opts.Import_cache_buffer
	indexHost := opts.Index_host
	indexPort := opts.Index_port

	rootCmd := []string{"/usr/bin/python", "/server.py"}
	if indexHost != "" && indexPort != "" {
		rootCmd = append(rootCmd, indexHost, indexPort)
	}

	var delegate CacheFactory
	var err error
	if opts.Sandbox == "docker" {
		delegate, err = NewDockerCacheFactory(cluster, pkgsDir)
		if err != nil {
			return nil, nil, "", err
		}
	} else if opts.Sandbox == "olcontainer" {
		delegate, err = NewOLContainerCacheFactory(opts, cluster, opts.OLContainer_cache_base, pkgsDir)
		if err != nil {
			return nil, nil, "", err
		}
	}

	bf := &BufferedCacheFactory{
		delegate: delegate,
		buffer:   make(chan *emptySBInfo, buffer),
		errors:   make(chan error, buffer),
		dir:      cacheDir,
	}

	if err := os.MkdirAll(cacheDir, os.ModeDir); err != nil {
		return nil, nil, "", fmt.Errorf("failed to create pool directory at %s: %v", cacheDir, err)
	}

	// create the root container
	rootDir := filepath.Join(bf.dir, "root")
	if err := os.MkdirAll(rootDir, os.ModeDir); err != nil {
		return nil, nil, "", fmt.Errorf("failed to create cache entry directory at %s: %v", cacheDir, err)
	}

	root, err := bf.delegate.Create(rootDir, rootCmd)
	if err != nil {
		return nil, nil, "", fmt.Errorf("failed to create cache entry sandbox: %v", err)
	} else if err := root.Start(); err != nil {
		return nil, nil, "", fmt.Errorf("failed to start cache entry sandbox: %v", err)
	}

	// fill the sandbox buffer
	var sharedIdx int64 = -1
	bf.idxPtr = &sharedIdx
	for i := 0; i < 5; i++ {
		go func(idxPtr *int64) {
			for {
				newIdx := atomic.AddInt64(idxPtr, 1)
				if newIdx < 0 {
					return // kill signal
				}

				sandboxDir := filepath.Join(bf.dir, fmt.Sprintf("%d", newIdx))
				if err := os.MkdirAll(sandboxDir, os.ModeDir); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else if sandbox, err := bf.delegate.Create(sandboxDir, []string{"/init"}); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else if err := sandbox.Start(); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else if err := sandbox.Pause(); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else {
					bf.buffer <- &emptySBInfo{sandbox, sandboxDir}
					bf.errors <- nil
				}
			}
		}(bf.idxPtr)
	}

	log.Printf("filling cache buffer")
	for len(bf.buffer) < cap(bf.buffer) {
		time.Sleep(20 * time.Millisecond)
	}
	log.Printf("cache buffer full")

	return bf, root, rootDir, nil
}

// Returns a sandbox ready for a cache interpreter
func (bf *BufferedCacheFactory) Create() (sb.ContainerSandbox, string, error) {
	info, err := <-bf.buffer, <-bf.errors
	if err != nil {
		return nil, "", err
	}

	if err := info.sandbox.Unpause(); err != nil {
		return nil, "", err
	}

	return info.sandbox, info.sandboxDir, nil
}

func (bf *BufferedCacheFactory) Cleanup() {
	// kill signal must be negative for all producers
	atomic.StoreInt64(bf.idxPtr, -1000)

	// empty the buffer
	for {
		select {
		case info := <-bf.buffer:
			if info == nil {
				continue
			}
			info.sandbox.Unpause()
			info.sandbox.Stop()
			info.sandbox.Remove()
		default:
			// clean up mount points once buffer is empty
			bf.delegate.Cleanup()
			return
		}
	}
}

func runCmd(args []string) error {
	c := exec.Cmd{Path: args[0], Args: args}
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr

	return c.Run()
}
<commit_msg>Remove unnecessary mount in cacheFactory<commit_after>package cache

/*
TODO:

This is extremely ugly. We should further parameterize the
SBFactories and use them directly instead of repeating code.

*/

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/open-lambda/open-lambda/worker/config"
	"github.com/open-lambda/open-lambda/worker/dockerutil"

	docker "github.com/fsouza/go-dockerclient"
	sb "github.com/open-lambda/open-lambda/worker/sandbox"
)

var unshareFlags []string = []string{"-fimu"}

const rootCacheSandboxDir = "/tmp/olcache"

func InitCacheFactory(opts *config.Config, cluster string) (cf *BufferedCacheFactory, root sb.ContainerSandbox, rootDir string, err error) {
	cf, root, rootDir, err = NewBufferedCacheFactory(opts, cluster)
	if err != nil {
		return nil, nil, "", err
	}

	return cf, root, rootDir, nil
}

// emptySBInfo wraps sandbox information necessary for the buffer.
type emptySBInfo struct {
	sandbox    sb.ContainerSandbox
	sandboxDir string
}

// BufferedCacheFactory maintains a buffer of sandboxes created by another factory.
type BufferedCacheFactory struct {
	delegate CacheFactory
	buffer   chan *emptySBInfo
	errors   chan error
	dir      string
	idxPtr   *int64
}

type CacheFactory interface {
	Create(sandboxDir string, rootCmd []string) (sb.ContainerSandbox, error)
	Cleanup()
}

// DockerCacheFactory is a SandboxFactory that creates docker sandboxes for the cache.
type DockerCacheFactory struct {
	client  *docker.Client
	cmd     []string
	caps    []string
	labels  map[string]string
	pkgsDir string
}

// NewDockerCacheFactory creates a CacheFactory that uses Docker containers.
func NewDockerCacheFactory(cluster, pkgsDir string) (*DockerCacheFactory, error) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return nil, err
	}

	cmd := []string{"/init"}

	caps := []string{"SYS_ADMIN"}

	labels := map[string]string{
		dockerutil.DOCKER_LABEL_CLUSTER: cluster,
		dockerutil.DOCKER_LABEL_TYPE:    dockerutil.POOL,
	}

	cf := &DockerCacheFactory{client, cmd, caps, labels, pkgsDir}
	return cf, nil
}

// Create creates a docker container from the pool directory.
func (cf *DockerCacheFactory) Create(sandboxDir string, cmd []string) (sb.ContainerSandbox, error) {
	volumes := []string{
		fmt.Sprintf("%s:%s", sandboxDir, "/host"),
		fmt.Sprintf("%s:%s:ro", cf.pkgsDir, "/packages"),
	}

	container, err := cf.client.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Image:  dockerutil.CACHE_IMAGE,
				Labels: cf.labels,
				Cmd:    cmd,
			},
			HostConfig: &docker.HostConfig{
				Binds:      volumes,
				PidMode:    "host",
				CapAdd:     cf.caps,
				AutoRemove: true,
			},
		},
	)
	if err != nil {
		return nil, err
	}

	sandbox := sb.NewDockerSandbox("", sandboxDir, "", "", container, cf.client)

	return sandbox, nil
}

func (cf *DockerCacheFactory) Cleanup() {
	return
}

// OLContainerCacheFactory is a SandboxFactory that creates olcontainers for the cache.
type OLContainerCacheFactory struct {
	opts    *config.Config
	cgf     *sb.CgroupFactory
	cmd     []string
	baseDir string
	pkgsDir string
}

// NewOLContainerCacheFactory creates a CacheFactory that uses olcontainers.
func NewOLContainerCacheFactory(opts *config.Config, cluster, baseDir, pkgsDir string) (*OLContainerCacheFactory, error) {
	for _, cgroup := range sb.CGroupList {
		cgroupPath := path.Join("/sys/fs/cgroup", cgroup, sb.OLCGroupName)
		if err := os.MkdirAll(cgroupPath, 0700); err != nil {
			return nil, err
		}
	}

	cgf, err := sb.NewCgroupFactory("cache", opts.Cg_pool_size)
	if err != nil {
		return nil, err
	}

	if err := os.MkdirAll(rootCacheSandboxDir, 0777); err != nil {
		return nil, fmt.Errorf("failed to make root sandbox dir :: %v", err.Error())
	} else if err := syscall.Mount(rootCacheSandboxDir, rootCacheSandboxDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind root sandbox dir: %v", err.Error())
	} else if err := syscall.Mount("none", rootCacheSandboxDir, "", sb.PRIVATE, ""); err != nil {
		return nil, fmt.Errorf("failed to make root cache sandbox dir private :: %v", err.Error())
	}

	sbPkgsDir := path.Join(baseDir, "packages")
	if err := syscall.Mount(pkgsDir, sbPkgsDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind packages dir: %s -> %s :: %v", opts.Pkgs_dir, sbPkgsDir, err)
	} else if err := syscall.Mount("none", sbPkgsDir, "", sb.BIND_RO, ""); err != nil {
		return nil, fmt.Errorf("failed to bind packages dir RO: %s -> %s :: %v", opts.Pkgs_dir, sbPkgsDir, err)
	}

	return &OLContainerCacheFactory{opts, cgf, []string{"/init"}, baseDir, pkgsDir}, nil
}

// Create creates a docker sandbox from the pool directory.
func (cf *OLContainerCacheFactory) Create(hostDir string, startCmd []string) (sb.ContainerSandbox, error) {
	id_bytes, err := exec.Command("uuidgen").Output()
	if err != nil {
		return nil, err
	}
	id := strings.TrimSpace(string(id_bytes[:]))

	rootDir := path.Join(rootCacheSandboxDir, id)
	if err := os.Mkdir(rootDir, 0700); err != nil {
		return nil, err
	}

	// NOTE: mount points are expected to exist in OLContainer_handler_base directory

	if err := syscall.Mount(cf.baseDir, rootDir, "", sb.BIND, ""); err != nil {
		return nil, fmt.Errorf("failed to bind root dir: %s -> %s :: %v\n", cf.baseDir, rootDir, err)
	} else if err := syscall.Mount("none", rootDir, "", sb.BIND_RO, ""); err != nil {
		return nil, fmt.Errorf("failed to bind root dir RO: %s :: %v\n", rootDir, err)
	} else if err := syscall.Mount("none", rootDir, "", sb.PRIVATE, ""); err != nil {
		return nil, fmt.Errorf("failed to make root dir private :: %v", err)
	}

	sandbox, err := sb.NewOLContainerSandbox(cf.cgf, cf.opts, rootDir, id, startCmd, unshareFlags)
	if err != nil {
		return nil, err
	}

	if err := sandbox.MountDirs(hostDir, ""); err != nil {
		return nil, err
	}

	return sandbox, nil
}

func (cf *OLContainerCacheFactory) Cleanup() {
	for _, cgroup := range sb.CGroupList {
		cgroupPath := path.Join("/sys/fs/cgroup", cgroup, sb.OLCGroupName)
		os.Remove(cgroupPath)
	}

	runCmd([]string{"/bin/umount", "/tmp/cache_*/*"})
	runCmd([]string{"/bin/umount", "/tmp/cache_*"})
	runCmd([]string{"/bin/rm", "-rf", "/tmp/cache_*"})
}

// NewBufferedCacheFactory creates a BufferedCacheFactory and starts a go routine to
// fill the sandbox buffer.
func NewBufferedCacheFactory(opts *config.Config, cluster string) (*BufferedCacheFactory, sb.ContainerSandbox, string, error) {
	cacheDir := opts.Import_cache_dir
	pkgsDir := opts.Pkgs_dir
	buffer := opts.Import_cache_buffer
	indexHost := opts.Index_host
	indexPort := opts.Index_port

	rootCmd := []string{"/usr/bin/python", "/server.py"}
	if indexHost != "" && indexPort != "" {
		rootCmd = append(rootCmd, indexHost, indexPort)
	}

	var delegate CacheFactory
	var err error
	if opts.Sandbox == "docker" {
		delegate, err = NewDockerCacheFactory(cluster, pkgsDir)
		if err != nil {
			return nil, nil, "", err
		}
	} else if opts.Sandbox == "olcontainer" {
		delegate, err = NewOLContainerCacheFactory(opts, cluster, opts.OLContainer_cache_base, pkgsDir)
		if err != nil {
			return nil, nil, "", err
		}
	}

	bf := &BufferedCacheFactory{
		delegate: delegate,
		buffer:   make(chan *emptySBInfo, buffer),
		errors:   make(chan error, buffer),
		dir:      cacheDir,
	}

	if err := os.MkdirAll(cacheDir, os.ModeDir); err != nil {
		return nil, nil, "", fmt.Errorf("failed to create pool directory at %s: %v", cacheDir, err)
	}

	// create the root container
	rootDir := filepath.Join(bf.dir, "root")
	if err := os.MkdirAll(rootDir, os.ModeDir); err != nil {
		return nil, nil, "", fmt.Errorf("failed to create cache entry directory at %s: %v", cacheDir, err)
	}

	root, err := bf.delegate.Create(rootDir, rootCmd)
	if err != nil {
		return nil, nil, "", fmt.Errorf("failed to create cache entry sandbox: %v", err)
	} else if err := root.Start(); err != nil {
		return nil, nil, "", fmt.Errorf("failed to start cache entry sandbox: %v", err)
	}

	// fill the sandbox buffer
	var sharedIdx int64 = -1
	bf.idxPtr = &sharedIdx
	for i := 0; i < 5; i++ {
		go func(idxPtr *int64) {
			for {
				newIdx := atomic.AddInt64(idxPtr, 1)
				if newIdx < 0 {
					return // kill signal
				}

				sandboxDir := filepath.Join(bf.dir, fmt.Sprintf("%d", newIdx))
				if err := os.MkdirAll(sandboxDir, os.ModeDir); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else if sandbox, err := bf.delegate.Create(sandboxDir, []string{"/init"}); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else if err := sandbox.Start(); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else if err := sandbox.Pause(); err != nil {
					bf.buffer <- nil
					bf.errors <- err
				} else {
					bf.buffer <- &emptySBInfo{sandbox, sandboxDir}
					bf.errors <- nil
				}
			}
		}(bf.idxPtr)
	}

	log.Printf("filling cache buffer")
	for len(bf.buffer) < cap(bf.buffer) {
		time.Sleep(20 * time.Millisecond)
	}
	log.Printf("cache buffer full")

	return bf, root, rootDir, nil
}

// Returns a sandbox ready for a cache interpreter
func (bf *BufferedCacheFactory) Create() (sb.ContainerSandbox, string, error) {
	info, err := <-bf.buffer, <-bf.errors
	if err != nil {
		return nil, "", err
	}

	if err := info.sandbox.Unpause(); err != nil {
		return nil, "", err
	}

	return info.sandbox, info.sandboxDir, nil
}

func (bf *BufferedCacheFactory) Cleanup() {
	// kill signal must be negative for all producers
	atomic.StoreInt64(bf.idxPtr, -1000)

	// empty the buffer
	for {
		select {
		case info := <-bf.buffer:
			if info == nil {
				continue
			}
			info.sandbox.Unpause()
			info.sandbox.Stop()
			info.sandbox.Remove()
		default:
			// clean up mount points once buffer is empty
			bf.delegate.Cleanup()
			return
		}
	}
}

func runCmd(args []string) error {
	c := exec.Cmd{Path: args[0], Args: args}
	c.Stdout = os.Stdout
	c.Stderr = os.Stderr

	return c.Run()
}
<|endoftext|>