<commit_before>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/gotetra/render/halo"
	rgeom "github.com/phil-mansfield/gotetra/render/geom"
	"github.com/phil-mansfield/gotetra/los/geom"
	"github.com/phil-mansfield/gotetra/los/analyze"
)

type Params struct {
	MaxMult float64
}

func main() {
	p := parseCmd()
	ids, snaps, coeffs, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	snapBins, coeffBins, idxBins := binBySnap(snaps, ids, coeffs)

	masses := make([]float64, len(ids))

	for snap, snapIDs := range snapBins {
		idxs := idxBins[snap]
		snapCoeffs := coeffBins[snap]
		if snap == -1 { continue }

		hds, files, err := readHeaders(snap)
		if err != nil { err.Error() }
		hBounds, err := boundingSpheres(snap, &hds[0], snapIDs, p)

		intrBins := binIntersections(hds, hBounds)

		xs := []rgeom.Vec{}
		for i := range hds {
			if len(intrBins[i]) == 0 { continue }
			hd := &hds[i]

			n := hd.SegmentWidth*hd.SegmentWidth*hd.SegmentWidth
			if len(xs) == 0 { xs = make([]rgeom.Vec, n) }
			io.ReadSheetPositionsAt(files[i], xs)

			for j := range idxs {
				masses[idxs[j]] += massContained(
					&hds[i], xs, snapCoeffs[j], hBounds[j],
				)
			}
		}
	}

	printMasses(ids, snaps, masses)
}

func parseStdin() (ids, snaps []int, coeffs [][]float64, err error) {
	ids, snaps, coeffs = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

		var (
			id, snap int
			hCoeffs []float64
			err error
		)
		switch {
		case len(tokens) == 0:
			continue
		case len(tokens) <= 2:
			if tokens[0] == "" { continue }
			return nil, nil, nil, fmt.Errorf(
				"Line %d of stdin has 1 token, but >2 are required.", i + 1,
			)
		case len(tokens) > 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}

			hCoeffs = make([]float64, len(tokens) - 2)
			for i := range hCoeffs {
				hCoeffs[i], err = strconv.ParseFloat(tokens[i + 2], 64)
			}
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
		coeffs = append(coeffs, hCoeffs)
	}

	return ids, snaps, coeffs, nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseCmd() *Params {
	p := &Params{}
	flag.Float64Var(&p.MaxMult, "MaxMult", 3,
		"Ending radius of LoSs as a multiple of R_200m. " +
			"Should be the same value as used in gtet_shell.")
	flag.Parse()
	return p
}

func binBySnap(
	snaps, ids []int, coeffs [][]float64,
) (snapBins map[int][]int,coeffBins map[int][][]float64,idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	coeffBins = make(map[int][][]float64)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		snapBins[snap] = append(snapBins[snap], ids[i])
		coeffBins[snap] = append(coeffBins[snap], coeffs[i])
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, coeffBins, idxBins
}

func readHeaders(snap int) ([]io.SheetHeader, []string, error) {
	memoDir := os.Getenv("GTET_MEMO_DIR")
	if memoDir == "" {
		// You don't want to memoize? Fine. Deal with the consequences.
		return readHeadersFromSheet(snap)
	}
	if _, err := os.Stat(memoDir); err != nil {
		return nil, nil, err
	}

	memoFile := path.Join(memoDir, fmt.Sprintf("hd_snap%d.dat", snap))

	if _, err := os.Stat(memoFile); err != nil {
		// File not written yet.
		hds, files, err := readHeadersFromSheet(snap)
		if err != nil { return nil, nil, err }

        f, err := os.Create(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()
        binary.Write(f, binary.LittleEndian, hds)

		return hds, files, nil
	} else {
		// File exists: read from it instead.

		f, err := os.Open(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()

		n, err := sheetNum(snap)
		if err != nil { return nil, nil, err }
		hds := make([]io.SheetHeader, n)
        binary.Read(f, binary.LittleEndian, hds)

		gtetFmt := os.Getenv("GTET_FMT")
		dir := fmt.Sprintf(gtetFmt, snap)
		files, err := dirContents(dir)
		if err != nil { return nil, nil, err }

		return hds, files, nil
	}
}

func sheetNum(snap int) (int, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return 0, err }
	return len(files), nil
}

func readHeadersFromSheet(snap int) ([]io.SheetHeader, []string, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return nil, nil, err }

	hds := make([]io.SheetHeader, len(files))
	for i := range files {
		err = io.ReadSheetHeaderAt(files[i], &hds[i])
		if err != nil { return nil, nil, err }
	}
	return hds, files, nil
}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func wrapDist(x1, x2, width float32) float32 {
	dist := x1 - x2
	if dist > width / 2 {
		return dist - width
	} else if dist < width / -2 {
		return dist + width
	} else {
		return dist
	}
}

func inRange(x, r, low, width, tw float32) bool {
	return wrapDist(x, low, tw) > -r && wrapDist(x, low + width, tw) < r
}

// SheetIntersect returns true if the given halo and sheet intersect one another
// and false otherwise.
func sheetIntersect(s geom.Sphere, hd *io.SheetHeader) bool {
	tw := float32(hd.TotalWidth)
	return inRange(s.C[0], s.R, hd.Origin[0], hd.Width[0], tw) &&
		inRange(s.C[1], s.R, hd.Origin[1], hd.Width[1], tw) &&
		inRange(s.C[2], s.R, hd.Origin[2], hd.Width[2], tw)
}

func binIntersections(
	hds []io.SheetHeader, spheres []geom.Sphere,
) [][]geom.Sphere {
	bins := make([][]geom.Sphere, len(hds))
	for i := range hds {
		for si := range spheres {
			if sheetIntersect(spheres[si], &hds[i]) {
				bins[i] = append(bins[i], spheres[si])
			}
		}
	}
	return bins
}

func boundingSpheres(
	snap int, hd *io.SheetHeader, ids []int, p *Params,
) ([]geom.Sphere, error) {
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	hlists, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	rids, vals, err := halo.ReadRockstarVals(
		hlists[snap - 1], &hd.Cosmo, halo.X, halo.Y, halo.Z, halo.Rad200b,
	)
	xs, ys, zs, rs := vals[0], vals[1], vals[2], vals[3]

	spheres := make([]geom.Sphere, len(rids))
	for i := range spheres {
		spheres[i].C = geom.Vec{float32(xs[i]), float32(ys[i]), float32(zs[i])}
		spheres[i].R = float32(rs[i])
	}

	return spheres, nil
}

func findOrder(coeffs []float64) int {
	i := 1
	for {
		if i*i == len(coeffs) {
			return i
		} else if i*i > len(coeffs) {
			panic("Impossible")
		}
		i++
	}
}

func massContained(
	hd *io.SheetHeader, xs []rgeom.Vec, coeffs []float64, sphere geom.Sphere,
) float64 {
	sum := 0.0
	ptMass := hd.Mass

	order := findOrder(coeffs)
	shell := analyze.PennaFunc(coeffs, order, order, 2)

	for i := range xs {
		x, y, z := float64(xs[i][0]), float64(xs[i][1]), float64(xs[i][2])
		if shell.Contains(x, y, z) { sum += ptMass }
	}
	return sum
}

func printMasses(ids, snaps []int, masses []float64) {
	idWidth, snapWidth, massWidth := 0, 0, 0
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		mWidth := len(fmt.Sprintf("%.5g", masses[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		if mWidth > massWidth { massWidth = mWidth }
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd %%%d.5g\n",
		idWidth, snapWidth, massWidth)
	for i := range ids { fmt.Printf(rowFmt, ids[i], snaps[i], masses[i]) }
}
<commit_msg>Fixed bugs in gtet_mass.<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/cosmo"
	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/gotetra/render/halo"
	rgeom "github.com/phil-mansfield/gotetra/render/geom"
	"github.com/phil-mansfield/gotetra/los/geom"
	"github.com/phil-mansfield/gotetra/los/analyze"
)

type Params struct {
	MaxMult float64
}

func main() {
	p := parseCmd()
	ids, snaps, coeffs, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	snapBins, coeffBins, idxBins := binBySnap(snaps, ids, coeffs)

	masses := make([]float64, len(ids))

	for snap, snapIDs := range snapBins {
		log.Println(snap)
		idxs := idxBins[snap]
		snapCoeffs := coeffBins[snap]
		if snap == -1 { continue }

		hds, files, err := readHeaders(snap)
		if err != nil { log.Fatal(err.Error()) }
		hBounds, err := boundingSpheres(snap, &hds[0], snapIDs, p)
		if err != nil { log.Fatal(err.Error()) }
		intrBins := binIntersections(hds, hBounds)

		xs := []rgeom.Vec{}
		for i := range hds {
			if len(intrBins[i]) == 0 { continue }
			hd := &hds[i]

			n := hd.GridWidth*hd.GridWidth*hd.GridWidth
			if len(xs) == 0 { xs = make([]rgeom.Vec, n) }
			err := io.ReadSheetPositionsAt(files[i], xs)
			if err != nil { log.Fatal(err.Error()) }

			for j := range idxs {
				masses[idxs[j]] += massContained(
					&hds[i], xs, snapCoeffs[j], hBounds[j],
				)
			}
			log.Printf("%.3g\n", masses)
		}
	}

	printMasses(ids, snaps, masses)
}

func parseStdin() (ids, snaps []int, coeffs [][]float64, err error) {
	ids, snaps, coeffs = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

		var (
			id, snap int
			hCoeffs []float64
			err error
		)
		switch {
		case len(tokens) == 0:
			continue
		case len(tokens) <= 2:
			if tokens[0] == "" { continue }
			return nil, nil, nil, fmt.Errorf(
				"Line %d of stdin has 1 token, but >2 are required.", i + 1,
			)
		case len(tokens) > 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}

			hCoeffs = make([]float64, len(tokens) - 2)
			for i := range hCoeffs {
				hCoeffs[i], err = strconv.ParseFloat(tokens[i + 2], 64)
			}
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
		coeffs = append(coeffs, hCoeffs)
	}

	return ids, snaps, coeffs, nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseCmd() *Params {
	p := &Params{}
	flag.Float64Var(&p.MaxMult, "MaxMult", 3,
		"Ending radius of LoSs as a multiple of R_200m. " +
			"Should be the same value as used in gtet_shell.")
	flag.Parse()
	return p
}

func binBySnap(
	snaps, ids []int, coeffs [][]float64,
) (snapBins map[int][]int,coeffBins map[int][][]float64,idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	coeffBins = make(map[int][][]float64)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		snapBins[snap] = append(snapBins[snap], ids[i])
		coeffBins[snap] = append(coeffBins[snap], coeffs[i])
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, coeffBins, idxBins
}

func readHeaders(snap int) ([]io.SheetHeader, []string, error) {
	memoDir := os.Getenv("GTET_MEMO_DIR")
	if memoDir == "" {
		// You don't want to memoize? Fine. Deal with the consequences.
		return readHeadersFromSheet(snap)
	}
	if _, err := os.Stat(memoDir); err != nil {
		return nil, nil, err
	}

	memoFile := path.Join(memoDir, fmt.Sprintf("hd_snap%d.dat", snap))

	if _, err := os.Stat(memoFile); err != nil {
		// File not written yet.
		hds, files, err := readHeadersFromSheet(snap)
		if err != nil { return nil, nil, err }

        f, err := os.Create(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()
        binary.Write(f, binary.LittleEndian, hds)

		return hds, files, nil
	} else {
		// File exists: read from it instead.

		f, err := os.Open(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()

		n, err := sheetNum(snap)
		if err != nil { return nil, nil, err }
		hds := make([]io.SheetHeader, n)
        binary.Read(f, binary.LittleEndian, hds)

		gtetFmt := os.Getenv("GTET_FMT")
		dir := fmt.Sprintf(gtetFmt, snap)
		files, err := dirContents(dir)
		if err != nil { return nil, nil, err }

		return hds, files, nil
	}
}

func sheetNum(snap int) (int, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return 0, err }
	return len(files), nil
}

func readHeadersFromSheet(snap int) ([]io.SheetHeader, []string, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return nil, nil, err }

	hds := make([]io.SheetHeader, len(files))
	for i := range files {
		err = io.ReadSheetHeaderAt(files[i], &hds[i])
		if err != nil { return nil, nil, err }
	}
	return hds, files, nil
}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func wrapDist(x1, x2, width float32) float32 {
	dist := x1 - x2
	if dist > width / 2 {
		return dist - width
	} else if dist < width / -2 {
		return dist + width
	} else {
		return dist
	}
}

func inRange(x, r, low, width, tw float32) bool {
	return wrapDist(x, low, tw) > -r && wrapDist(x, low + width, tw) < r
}

// SheetIntersect returns true if the given halo and sheet intersect one another
// and false otherwise.
func sheetIntersect(s geom.Sphere, hd *io.SheetHeader) bool {
	tw := float32(hd.TotalWidth)
	return inRange(s.C[0], s.R, hd.Origin[0], hd.Width[0], tw) &&
		inRange(s.C[1], s.R, hd.Origin[1], hd.Width[1], tw) &&
		inRange(s.C[2], s.R, hd.Origin[2], hd.Width[2], tw)
}

func binIntersections(
	hds []io.SheetHeader, spheres []geom.Sphere,
) [][]geom.Sphere {
	bins := make([][]geom.Sphere, len(hds))
	for i := range hds {
		for si := range spheres {
			if sheetIntersect(spheres[si], &hds[i]) {
				bins[i] = append(bins[i], spheres[si])
			}
		}
	}
	return bins
}

func boundingSpheres(
	snap int, hd *io.SheetHeader, ids []int, p *Params,
) ([]geom.Sphere, error) {
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	hlists, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	rids, vals, err := halo.ReadRockstarVals(
		hlists[snap - 1], &hd.Cosmo, halo.X, halo.Y, halo.Z, halo.Rad200b,
	)
	xs, ys, zs, rs := vals[0], vals[1], vals[2], vals[3]

	spheres := make([]geom.Sphere, len(ids))
	for i := range spheres {
		j := -1
		for idx := range xs {
			if rids[idx] == ids[i] {
				j = idx
				break
			}
		}

		if j == -1 {
			return nil, fmt.Errorf("Halo %d not found in snap %d.",
				ids[i], snap)
		}
		spheres[i].C = geom.Vec{float32(xs[j]), float32(ys[j]), float32(zs[j])}
		spheres[i].R = float32(rs[j])
	}

	return spheres, nil
}

func findOrder(coeffs []float64) int {
	i := 1
	for {
		if 2*i*i == len(coeffs) {
			return i
		} else if 2*i*i > len(coeffs) {
			panic("Impossible")
		}
		i++
	}
}

func wrap(x, tw2 float32) float32 {
	if x > tw2 {
		return x - tw2
	} else if x < -tw2 {
		return x + tw2
	}
	return x
}

func coords(idx, cells int64) (x, y, z int64) {
    x = idx % cells
    y = (idx % (cells * cells)) / cells
    z = idx / (cells * cells)
    return x, y, z
}

func massContained(
	hd *io.SheetHeader, xs []rgeom.Vec, coeffs []float64, sphere geom.Sphere,
) float64 {
	c := &hd.Cosmo
	rhoM := cosmo.RhoAverage(c.H100 * 100, c.OmegaM, c.OmegaL, c.Z )
	dx := hd.TotalWidth / float64(hd.CountWidth) / (1 + c.Z)
	ptMass := rhoM * (dx*dx*dx)
	tw2 := float32(hd.TotalWidth) / 2

	order := findOrder(coeffs)
	shell := analyze.PennaFunc(coeffs, order, order, 2)

	sum := 0.0
	sw := hd.SegmentWidth
	for si := int64(0); si < sw*sw*sw; si++ {
		xi, yi, zi := coords(si, hd.SegmentWidth)
		i := xi + yi*sw + zi*sw*sw
		x, y, z := xs[i][0], xs[i][1], xs[i][2]
		x, y, z = x - sphere.C[0], y - sphere.C[1], z - sphere.C[2]
		x = wrap(x, tw2)
		y = wrap(y, tw2)
		z = wrap(z, tw2)
		if shell.Contains(float64(x), float64(y), float64(z)) { sum += ptMass }
	}
	return sum
}

func printMasses(ids, snaps []int, masses []float64) {
	idWidth, snapWidth, massWidth := 0, 0, 0
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		mWidth := len(fmt.Sprintf("%.5g", masses[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		if mWidth > massWidth { massWidth = mWidth }
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd %%%d.5g\n",
		idWidth, snapWidth, massWidth)
	for i := range ids { fmt.Printf(rowFmt, ids[i], snaps[i], masses[i]) }
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/cloud66/habitus/configuration"

	"gopkg.in/yaml.v2"
)

// Artefact holds a parsed source for a build artefact
type Artefact struct {
	Order  int
	Step   Step
	Source string
	Dest   string // this is only the folder. Filename comes from the source
}

// Cleanup holds everything that's needed for a cleanup
type Cleanup struct {
	Commands []string
}

// Step Holds a single step in the build process
// Public structs. They are used to store the build for the builders
type Step struct {
	Order      int
	Name       string
	Dockerfile string
	Artefacts  []Artefact
	Manifest   Manifest
	Cleanup    *Cleanup
}

// Manifest Holds the whole build process
type Manifest struct {
	Steps        []Step
	IsPrivileged bool
}

type cleanup struct {
	Commands []string
}

// Private structs. They are used to load from yaml
type step struct {
	Name       string
	Dockerfile string
	Artefacts  []string
	Cleanup    *cleanup
}

type build struct {
	Workdir string
	Steps   []step
	Config  *configuration.Config
}

// LoadBuildFromFile loads Build from a yaml file
func LoadBuildFromFile(config *configuration.Config) (*Manifest, error) {
	config.Logger.Notice("Using '%s' as build file", config.Buildfile)

	t := build{Config: config}

	data, err := ioutil.ReadFile(config.Buildfile)
	if err != nil {
		return nil, err
	}

	data = parseForEnvVars(config, data)

	err = yaml.Unmarshal([]byte(data), &t)
	if err != nil {
		return nil, err
	}

	return t.convertToBuild()
}

func (b *build) convertToBuild() (*Manifest, error) {
	r := Manifest{}
	r.IsPrivileged = false
	r.Steps = []Step{}

	for idx, s := range b.Steps {
		convertedStep := Step{}

		convertedStep.Manifest = r
		convertedStep.Dockerfile = s.Dockerfile
		convertedStep.Name = s.Name
		convertedStep.Order = idx
		convertedStep.Artefacts = []Artefact{}
		if s.Cleanup != nil && !b.Config.NoSquash {
			convertedStep.Cleanup = &Cleanup{Commands: s.Cleanup.Commands}
			r.IsPrivileged = true
		} else {
			convertedStep.Cleanup = &Cleanup{}
		}

		for kdx, a := range s.Artefacts {
			convertedArt := Artefact{}

			convertedArt.Order = kdx
			convertedArt.Step = convertedStep
			parts := strings.Split(a, ":")
			convertedArt.Source = parts[0]
			if len(parts) == 1 {
				// only one use the base
				convertedArt.Dest = "."
			} else {
				convertedArt.Dest = parts[1]
			}

			convertedStep.Artefacts = append(convertedStep.Artefacts, convertedArt)
		}

		// is it unique?
		for _, s := range r.Steps {
			if s.Name == convertedStep.Name {
				return nil, fmt.Errorf("Step name '%s' is not unique", convertedStep.Name)
			}
		}

		r.Steps = append(r.Steps, convertedStep)
	}

	return &r, nil
}

// FindStepByName finds a step by name. Returns nil if not found
func (m *Manifest) FindStepByName(name string) (*Step, error) {
	for _, step := range m.Steps {
		if step.Name == name {
			return &step, nil
		}
	}

	return nil, nil
}

func parseForEnvVars(config *configuration.Config, value []byte) []byte {
	r, _ := regexp.Compile("_env\\((.*)\\)")

	matched := r.ReplaceAllFunc(value, func(s []byte) []byte {
		m := string(s)
		parts := r.FindStringSubmatch(m)

		if len(config.EnvVars) == 0 {
			return []byte(os.Getenv(parts[1]))
		} else {
			return []byte(config.EnvVars.Find(parts[1]))
		}
	})

	return matched
}
<commit_msg>service dependency logic<commit_after>package build

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	"github.com/cloud66/habitus/configuration"

	"gopkg.in/yaml.v2"
)

// Artefact holds a parsed source for a build artefact
type Artefact struct {
	Step   Step
	Source string
	Dest   string // this is only the folder. Filename comes from the source
}

// Cleanup holds everything that's needed for a cleanup
type Cleanup struct {
	Commands []string
}

// Step Holds a single step in the build process
// Public structs. They are used to store the build for the builders
type Step struct {
	Name       string
	Dockerfile string
	Artefacts  []Artefact
	Manifest   Manifest
	Cleanup    *Cleanup
	DependsOn  []*Step
}

// Manifest Holds the whole build process
type Manifest struct {
	Steps        []Step
	IsPrivileged bool
}

type cleanup struct {
	Commands []string
}

// Private structs. They are used to load from yaml
type step struct {
	Name       string
	Dockerfile string
	Artefacts  []string
	Cleanup    *cleanup
	DependsOn  []string
}

// This is loaded from the build.yml file
type build struct {
	Workdir string
	Steps   []step
	Config  *configuration.Config
}

// LoadBuildFromFile loads Build from a yaml file
func LoadBuildFromFile(config *configuration.Config) (*Manifest, error) {
	config.Logger.Notice("Using '%s' as build file", config.Buildfile)

	t := build{Config: config}

	data, err := ioutil.ReadFile(config.Buildfile)
	if err != nil {
		return nil, err
	}

	data = parseForEnvVars(config, data)

	err = yaml.Unmarshal([]byte(data), &t)
	if err != nil {
		return nil, err
	}

	return t.convertToBuild()
}

// finds a step in the loaded build.yml by name
func (b *build) findStepByName(name string) (*step, error) {
	for _, step := range b.Steps {
		if step.Name == name {
			return &step, nil
		}
	}

	return nil, nil
}

func (b *build) convertToBuild() (*Manifest, error) {
	r := Manifest{}
	r.IsPrivileged = false
	r.Steps = []Step{}

	for _, s := range b.Steps {
		convertedStep := Step{}

		convertedStep.Manifest = r
		convertedStep.Dockerfile = s.Dockerfile
		convertedStep.Name = s.Name
		convertedStep.Artefacts = []Artefact{}
		if s.Cleanup != nil && !b.Config.NoSquash {
			convertedStep.Cleanup = &Cleanup{Commands: s.Cleanup.Commands}
			r.IsPrivileged = true
		} else {
			convertedStep.Cleanup = &Cleanup{}
		}

		for _, a := range s.Artefacts {
			convertedArt := Artefact{}

			convertedArt.Step = convertedStep
			parts := strings.Split(a, ":")
			convertedArt.Source = parts[0]
			if len(parts) == 1 {
				// only one use the base
				convertedArt.Dest = "."
			} else {
				convertedArt.Dest = parts[1]
			}

			convertedStep.Artefacts = append(convertedStep.Artefacts, convertedArt)
		}

		// is it unique?
		for _, s := range r.Steps {
			if s.Name == convertedStep.Name {
				return nil, fmt.Errorf("Step name '%s' is not unique", convertedStep.Name)
			}
		}

		r.Steps = append(r.Steps, convertedStep)
	}

	// now that we have the Manifest built from the file, we can resolve dependencies
	for idx, s := range r.Steps {
		bStep, err := b.findStepByName(s.Name)
		if err != nil {
			return nil, err
		}
		if bStep == nil {
			return nil, fmt.Errorf("step not found %s", s.Name)
		}

		for _, d := range bStep.DependsOn {
			convertedStep, err := r.FindStepByName(d)
			if err != nil {
				return nil, err
			}
			if convertedStep == nil {
				return nil, fmt.Errorf("can't find step %s", d)
			}

			r.Steps[idx].DependsOn = append(r.Steps[idx].DependsOn, convertedStep)
		}
	}

	// build the dependency tree
	_, err := r.serviceOrder(r.Steps)
	if err != nil {
		return nil, err
	}

	return &r, nil
}

// takes in a list of steps and returns an array of steps ordered by their dependency order
// result[0] will be an array of all steps with no dependency
// result[1] will be an array of steps depending on one or more of result[0] steps and so on
func (m *Manifest) serviceOrder(list []Step) ([][]Step, error) {
	if len(list) == 0 {
		return [][]Step{}, nil
	}

	var result [][]Step

	// find all steps with no dependencies
	for {
		var level []Step
		for _, step := range list {
			if len(step.DependsOn) == 0 {
				level = append(level, step)
			}
		}

		// if none is found while there where items in the list, then we have a circular dependency somewhere
		if len(list) != 0 && len(level) == 0 {
			return nil, errors.New("Found circular dependency in services")
		}

		result = append(result, level)

		// now take out all of those found from the list of other items (they are now 'resolved')
		for idx, step := range list { // for every step
			stepDeps := append([]*Step(nil), step.DependsOn...) // clone the dependency list so we can remove items from it
			for kdx, dep := range stepDeps {                    // iterate through its dependeneis
				for _, resolved := range level { // and find any resolved step in them and take it out
					if resolved.Name == dep.Name {
						list[idx].DependsOn = append(list[idx].DependsOn[:kdx], list[idx].DependsOn[kdx+1:]...)
					}
				}
			}
		}

		// take out everything we have in this level from the list
		for _, s := range level {
			listCopy := append([]Step(nil), list...)
			for idx, l := range listCopy {
				if s.Name == l.Name {
					list = append(list[:idx], list[idx+1:]...)
				}
			}
		}

		// we are done
		if len(list) == 0 {
			break
		}
	}

	return result, nil
}

// FindStepByName finds a step by name. Returns nil if not found
func (m *Manifest) FindStepByName(name string) (*Step, error) {
	for _, step := range m.Steps {
		if step.Name == name {
			return &step, nil
		}
	}

	return nil, nil
}

func parseForEnvVars(config *configuration.Config, value []byte) []byte {
	r, _ := regexp.Compile("_env\\((.*)\\)")

	matched := r.ReplaceAllFunc(value, func(s []byte) []byte {
		m := string(s)
		parts := r.FindStringSubmatch(m)

		if len(config.EnvVars) == 0 {
			return []byte(os.Getenv(parts[1]))
		} else {
			return []byte(config.EnvVars.Find(parts[1]))
		}
	})

	return matched
}
<|endoftext|>


<commit_before>//   Copyright 2010 Yves Junqueira
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

// The rsync package is a very simple interface to the command line rsync tool.
package rsync

import (
	"os"
	"io/ioutil"
	"exec"
	"log"
)

// Rsync sends a file via rsync to a remote host.
// Password input is not supported. You must use an SSH key in a standard location.
func Rsync(source string, user string, host string, dest string) (err os.Error) {
	r_path, err := exec.LookPath("rsync")
	if err != nil {
		log.Stderrf("rsync command not found (%s)\n", err)
		return
	}
	// MergeWithStdout makes error messages disappear.
	cmd, err := exec.Run(r_path, []string{r_path, "-az", source, user + "@" + host + ":" + dest},
		os.Environ(), exec.DevNull, exec.DevNull, exec.Pipe)
	// I love this in Go...
	defer cmd.Close()
	// .. but, man, these error checks look ugly.
	if err != nil {
		log.Stderrf("rsync run error (%s)\n", err)
		return
	}
	waitmsg, err := cmd.Wait(0)
	if err != nil {
		log.Stderrf("rsync wait error (%s)\n", err)
		return
	}
	buf, err := ioutil.ReadAll(cmd.Stderr)
	if err != nil {
		log.Stderrf("Error reading from stderr (%s)\n", err)
	}
	log.Stdout(string(buf))
	if waitmsg.ExitStatus() != 0 {
		log.Stderrf("rsync returned with an error status (%s)\n", waitmsg)
		return
	}
	return
}
<commit_msg>Fix for the new exec.Run() interface.<commit_after>//   Copyright 2010 Yves Junqueira
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

// The rsync package is a very simple interface to the command line rsync tool.
package rsync

import (
	"os"
	"io/ioutil"
	"exec"
	"log"
)

// Rsync sends a file via rsync to a remote host.
// Password input is not supported. You must use an SSH key in a standard location.
func Rsync(source string, user string, host string, dest string) (err os.Error) {
	r_path, err := exec.LookPath("rsync")
	if err != nil {
		log.Stderrf("rsync command not found (%s)\n", err)
		return
	}
	cmdArguments := []string{r_path, "-az", source, user + "@" + host + ":" + dest}
	// MergeWithStdout makes error messages disappear.
	cmd, err := exec.Run(r_path, cmdArguments, os.Environ(), "",
		exec.DevNull, exec.DevNull, exec.Pipe)
	// I love this in Go...
	defer cmd.Close()
	// .. but, man, these error checks look ugly.
	if err != nil {
		log.Stderrf("rsync run error (%s)\n", err)
		return
	}
	waitmsg, err := cmd.Wait(0)
	if err != nil {
		log.Stderrf("rsync wait error (%s)\n", err)
		return
	}
	buf, err := ioutil.ReadAll(cmd.Stderr)
	if err != nil {
		log.Stderrf("Error reading from stderr (%s)\n", err)
	}
	if len(buf) > 0 {
		log.Stdout(string(buf))
	}
	if waitmsg.ExitStatus() != 0 {
		log.Stderrf("rsync returned with an error status (%s)\n", waitmsg)
		return
	}
	log.Stderrf("All happy.\n")
	return
}
<|endoftext|>


<commit_before>package webdriver_test

import (
	"os"
	"os/exec"
	"testing"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/sclevine/agouti"
)

func TestTest(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Test Suite")
}

var (
	testServer   *gexec.Session
	chromeDriver = agouti.ChromeDriver(
		agouti.Desired(agouti.Capabilities{
			"loggingPrefs": map[string]string{
				"browser": "INFO",
			},
			"browserName": "chrome",
		}),
		// Unfortunately headless doesn't seem to work quite yet,
		// seems lock up loading the page.
		// (tried Google Chrome 59.0.3071.115)
		// https://developers.google.com/web/updates/2017/04/headless-chrome#drivers
		/*agouti.ChromeOptions(
			"args", []string{
				"--headless",
				"--disable-gpu",
			},
		),
		agouti.ChromeOptions(
			"binary", "/usr/bin/google-chrome-stable",
		),*/
	)
	seleniumDriver = agouti.Selenium(
		agouti.Browser("firefox"),
		agouti.Desired(agouti.NewCapabilities("acceptInsecureCerts")),
	)
)

var _ = BeforeSuite(func() {
	var binPath string
	By("Building the server", func() {
		var err error
		binPath, err = gexec.Build("./server/main.go")
		Expect(err).NotTo(HaveOccurred())
	})

	By("Running the server", func() {
		var err error
		testServer, err = gexec.Start(exec.Command(binPath), GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())
	})

	By("Starting the WebDrivers", func() {
		if os.Getenv("GOPHERJS_SERVER_ADDR") == "" {
			Expect(chromeDriver.Start()).NotTo(HaveOccurred())
			//Expect(seleniumDriver.Start()).NotTo(HaveOccurred())
		}
	})
})

var _ = AfterSuite(func() {
	By("Stopping the WebDrivers", func() {
		if os.Getenv("GOPHERJS_SERVER_ADDR") == "" {
			Expect(chromeDriver.Stop()).NotTo(HaveOccurred())
			//Expect(seleniumDriver.Stop()).NotTo(HaveOccurred())
		}
	})

	By("Stopping the server", func() {
		testServer.Terminate()
		testServer.Wait()
		Expect(testServer).To(gexec.Exit())
	})

	By("Cleaning up built artifacts", func() {
		gexec.CleanupBuildArtifacts()
	})
})
<commit_msg>Rename test suite<commit_after>package webdriver_test

import (
	"os"
	"os/exec"
	"testing"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/sclevine/agouti"
)

func TestWebdriver(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Webdriver Suite")
}

var (
	testServer   *gexec.Session
	chromeDriver = agouti.ChromeDriver(
		agouti.Desired(agouti.Capabilities{
			"loggingPrefs": map[string]string{
				"browser": "INFO",
			},
			"browserName": "chrome",
		}),
		// Unfortunately headless doesn't seem to work quite yet,
		// seems lock up loading the page.
		// (tried Google Chrome 59.0.3071.115)
		// https://developers.google.com/web/updates/2017/04/headless-chrome#drivers
		/*agouti.ChromeOptions(
			"args", []string{
				"--headless",
				"--disable-gpu",
			},
		),
		agouti.ChromeOptions(
			"binary", "/usr/bin/google-chrome-stable",
		),*/
	)
	seleniumDriver = agouti.Selenium(
		agouti.Browser("firefox"),
		agouti.Desired(agouti.NewCapabilities("acceptInsecureCerts")),
	)
)

var _ = BeforeSuite(func() {
	var binPath string
	By("Building the server", func() {
		var err error
		binPath, err = gexec.Build("./server/main.go")
		Expect(err).NotTo(HaveOccurred())
	})

	By("Running the server", func() {
		var err error
		testServer, err = gexec.Start(exec.Command(binPath), GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())
	})

	By("Starting the WebDrivers", func() {
		if os.Getenv("GOPHERJS_SERVER_ADDR") == "" {
			Expect(chromeDriver.Start()).NotTo(HaveOccurred())
			//Expect(seleniumDriver.Start()).NotTo(HaveOccurred())
		}
	})
})

var _ = AfterSuite(func() {
	By("Stopping the WebDrivers", func() {
		if os.Getenv("GOPHERJS_SERVER_ADDR") == "" {
			Expect(chromeDriver.Stop()).NotTo(HaveOccurred())
			//Expect(seleniumDriver.Stop()).NotTo(HaveOccurred())
		}
	})

	By("Stopping the server", func() {
		testServer.Terminate()
		testServer.Wait()
		Expect(testServer).To(gexec.Exit())
	})

	By("Cleaning up built artifacts", func() {
		gexec.CleanupBuildArtifacts()
	})
})
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fs

import (
	. "launchpad.net/gocheck"
	"os"
	"testing"
)

type S struct{}

var _ = Suite(&S{})

func Test(t *testing.T) {
	TestingT(t)
}

func (s *S) TestOsFsImplementsFS(c *C) {
	var _ Fs = OsFs{}
}

func (s *S) TestOsFsCreatesTheFileInTheDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	os.Remove(path)
	defer os.Remove(path)
	fs := OsFs{}
	f, err := fs.Create(path)
	c.Assert(err, IsNil)
	defer f.Close()
	_, err = os.Stat(path)
	c.Assert(err, IsNil)
}

func (s *S) TestOsFsMkdirWritesTheDirectoryInTheDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	os.RemoveAll(path)
	defer os.RemoveAll(path)
	fs := OsFs{}
	err := fs.Mkdir(path, 0755)
	c.Assert(err, IsNil)
	fi, err := os.Stat(path)
	c.Assert(err, IsNil)
	c.Assert(fi.IsDir(), Equals, true)
}

func (s *S) TestOsFsMkdirAllWritesAllDirectoriesInTheDisc(c *C) {
	root := "/tmp/test-fs-tsuru"
	path := root + "/path"
	paths := []string{root, path}
	for _, path := range paths {
		os.RemoveAll(path)
		defer os.RemoveAll(path)
	}
	fs := OsFs{}
	err := fs.MkdirAll(path, 0755)
	c.Assert(err, IsNil)
	for _, path := range paths {
		fi, err := os.Stat(path)
		c.Assert(err, IsNil)
		c.Assert(fi.IsDir(), Equals, true)
	}
}

func (s *S) TestOsFsOpenOpensTheFileFromDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	unknownPath := "/tmp/test-fs-tsuru-unknown"
	os.Remove(unknownPath)
	defer os.Remove(path)
	f, err := os.Create(path)
	c.Assert(err, IsNil)
	f.Close()
	fs := OsFs{}
	file, err := fs.Open(path)
	c.Assert(err, IsNil)
	file.Close()
	_, err = fs.Open(unknownPath)
	c.Assert(err, NotNil)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestOsFsRemoveDeletesTheFileFromDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	unknownPath := "/tmp/test-fs-tsuru-unknown"
	os.Remove(unknownPath)
	// Remove the file even if the test fails.
	defer os.Remove(path)
	f, err := os.Create(path)
	c.Assert(err, IsNil)
	f.Close()
	fs := OsFs{}
	err = fs.Remove(path)
	c.Assert(err, IsNil)
	_, err = os.Stat(path)
	c.Assert(os.IsNotExist(err), Equals, true)
	err = fs.Remove(unknownPath)
	c.Assert(err, NotNil)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestOsFsRemoveAllDeletesDirectoryFromDisc(c *C) {
	path := "/tmp/tsuru/test-fs-tsuru"
	err := os.MkdirAll(path, 0755)
	c.Assert(err, IsNil)
	// Remove the directory even if the test fails.
	defer os.RemoveAll(path)
	fs := OsFs{}
	err = fs.RemoveAll(path)
	c.Assert(err, IsNil)
	_, err = os.Stat(path)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestOsFsStatChecksTheFileInTheDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	unknownPath := "/tmp/test-fs-tsuru-unknown"
	os.Remove(unknownPath)
	defer os.Remove(path)
	f, err := os.Create(path)
	c.Assert(err, IsNil)
	f.Close()
	fs := OsFs{}
	_, err = fs.Stat(path)
	c.Assert(err, IsNil)
	_, err = fs.Stat(unknownPath)
	c.Assert(os.IsNotExist(err), Equals, true)
}
<commit_msg>fs: added test for OpenFile<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fs

import (
	. "launchpad.net/gocheck"
	"os"
	"testing"
)

type S struct{}

var _ = Suite(&S{})

func Test(t *testing.T) {
	TestingT(t)
}

func (s *S) TestOsFsImplementsFS(c *C) {
	var _ Fs = OsFs{}
}

func (s *S) TestOsFsCreatesTheFileInTheDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	os.Remove(path)
	defer os.Remove(path)
	fs := OsFs{}
	f, err := fs.Create(path)
	c.Assert(err, IsNil)
	defer f.Close()
	_, err = os.Stat(path)
	c.Assert(err, IsNil)
}

func (s *S) TestOsFsOpenFile(c *C) {
	path := "/tmp/test-fs-tsuru"
	os.Remove(path)
	defer os.Remove(path)
	fs := OsFs{}
	f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
	c.Assert(err, IsNil)
	defer f.Close()
	_, ok := f.(*os.File)
	c.Assert(ok, Equals, true)
}

func (s *S) TestOsFsMkdirWritesTheDirectoryInTheDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	os.RemoveAll(path)
	defer os.RemoveAll(path)
	fs := OsFs{}
	err := fs.Mkdir(path, 0755)
	c.Assert(err, IsNil)
	fi, err := os.Stat(path)
	c.Assert(err, IsNil)
	c.Assert(fi.IsDir(), Equals, true)
}

func (s *S) TestOsFsMkdirAllWritesAllDirectoriesInTheDisc(c *C) {
	root := "/tmp/test-fs-tsuru"
	path := root + "/path"
	paths := []string{root, path}
	for _, path := range paths {
		os.RemoveAll(path)
		defer os.RemoveAll(path)
	}
	fs := OsFs{}
	err := fs.MkdirAll(path, 0755)
	c.Assert(err, IsNil)
	for _, path := range paths {
		fi, err := os.Stat(path)
		c.Assert(err, IsNil)
		c.Assert(fi.IsDir(), Equals, true)
	}
}

func (s *S) TestOsFsOpenOpensTheFileFromDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	unknownPath := "/tmp/test-fs-tsuru-unknown"
	os.Remove(unknownPath)
	defer os.Remove(path)
	f, err := os.Create(path)
	c.Assert(err, IsNil)
	f.Close()
	fs := OsFs{}
	file, err := fs.Open(path)
	c.Assert(err, IsNil)
	file.Close()
	_, err = fs.Open(unknownPath)
	c.Assert(err, NotNil)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestOsFsRemoveDeletesTheFileFromDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	unknownPath := "/tmp/test-fs-tsuru-unknown"
	os.Remove(unknownPath)
	// Remove the file even if the test fails.
	defer os.Remove(path)
	f, err := os.Create(path)
	c.Assert(err, IsNil)
	f.Close()
	fs := OsFs{}
	err = fs.Remove(path)
	c.Assert(err, IsNil)
	_, err = os.Stat(path)
	c.Assert(os.IsNotExist(err), Equals, true)
	err = fs.Remove(unknownPath)
	c.Assert(err, NotNil)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestOsFsRemoveAllDeletesDirectoryFromDisc(c *C) {
	path := "/tmp/tsuru/test-fs-tsuru"
	err := os.MkdirAll(path, 0755)
	c.Assert(err, IsNil)
	// Remove the directory even if the test fails.
	defer os.RemoveAll(path)
	fs := OsFs{}
	err = fs.RemoveAll(path)
	c.Assert(err, IsNil)
	_, err = os.Stat(path)
	c.Assert(os.IsNotExist(err), Equals, true)
}

func (s *S) TestOsFsStatChecksTheFileInTheDisc(c *C) {
	path := "/tmp/test-fs-tsuru"
	unknownPath := "/tmp/test-fs-tsuru-unknown"
	os.Remove(unknownPath)
	defer os.Remove(path)
	f, err := os.Create(path)
	c.Assert(err, IsNil)
	f.Close()
	fs := OsFs{}
	_, err = fs.Stat(path)
	c.Assert(err, IsNil)
	_, err = fs.Stat(unknownPath)
	c.Assert(os.IsNotExist(err), Equals, true)
}
<|endoftext|>


<commit_before>package caddyfile

import (
	"io"
	"os"
	"path/filepath"
	"strings"
)

// Parse parses the input just enough to group tokens, in
// order, by server block. No further parsing is performed.
// Server blocks are returned in the order in which they appear.
// Directives that do not appear in validDirectives will cause
// an error. If you do not want to check for valid directives,
// pass in nil instead.
func Parse(filename string, input io.Reader, validDirectives []string) ([]ServerBlock, error) {
	p := parser{Dispenser: NewDispenser(filename, input), validDirectives: validDirectives}
	blocks, err := p.parseAll()
	return blocks, err
}

// allTokens lexes the entire input, but does not parse it.
// It returns all the tokens from the input, unstructured
// and in order.
func allTokens(input io.Reader) (tokens []Token) {
	l := new(lexer)
	l.load(input)
	for l.next() {
		tokens = append(tokens, l.token)
	}
	return
}

type parser struct {
	Dispenser
	block           ServerBlock // current server block being parsed
	validDirectives []string    // a directive must be valid or it's an error
	eof             bool        // if we encounter a valid EOF in a hard place
}

func (p *parser) parseAll() ([]ServerBlock, error) {
	var blocks []ServerBlock

	for p.Next() {
		err := p.parseOne()
		if err != nil {
			return blocks, err
		}
		if len(p.block.Keys) > 0 {
			blocks = append(blocks, p.block)
		}
	}

	return blocks, nil
}

func (p *parser) parseOne() error {
	p.block = ServerBlock{Tokens: make(map[string][]Token)}

	err := p.begin()
	if err != nil {
		return err
	}

	return nil
}

func (p *parser) begin() error {
	if len(p.tokens) == 0 {
		return nil
	}

	err := p.addresses()
	if err != nil {
		return err
	}

	if p.eof {
		// this happens if the Caddyfile consists of only
		// a line of addresses and nothing else
		return nil
	}

	err = p.blockContents()
	if err != nil {
		return err
	}

	return nil
}

func (p *parser) addresses() error {
	var expectingAnother bool

	for {
		tkn := replaceEnvVars(p.Val())

		// special case: import directive replaces tokens during parse-time
		if tkn == "import" && p.isNewLine() {
			err := p.doImport()
			if err != nil {
				return err
			}
			continue
		}

		// Open brace definitely indicates end of addresses
		if tkn == "{" {
			if expectingAnother {
				return p.Errf("Expected another address but had '%s' - check for extra comma", tkn)
			}
			break
		}

		if tkn != "" { // empty token possible if user typed ""
			// Trailing comma indicates another address will follow, which
			// may possibly be on the next line
			if tkn[len(tkn)-1] == ',' {
				tkn = tkn[:len(tkn)-1]
				expectingAnother = true
			} else {
				expectingAnother = false // but we may still see another one on this line
			}

			p.block.Keys = append(p.block.Keys, tkn)
		}

		// Advance token and possibly break out of loop or return error
		hasNext := p.Next()
		if expectingAnother && !hasNext {
			return p.EOFErr()
		}
		if !hasNext {
			p.eof = true
			break // EOF
		}
		if !expectingAnother && p.isNewLine() {
			break
		}
	}

	return nil
}

func (p *parser) blockContents() error {
	errOpenCurlyBrace := p.openCurlyBrace()
	if errOpenCurlyBrace != nil {
		// single-server configs don't need curly braces
		p.cursor--
	}

	err := p.directives()
	if err != nil {
		return err
	}

	// Only look for close curly brace if there was an opening
	if errOpenCurlyBrace == nil {
		err = p.closeCurlyBrace()
		if err != nil {
			return err
		}
	}

	return nil
}

// directives parses through all the lines for directives
// and it expects the next token to be the first
// directive. It goes until EOF or closing curly brace
// which ends the server block.
func (p *parser) directives() error {
	for p.Next() {
		// end of server block
		if p.Val() == "}" {
			break
		}

		// special case: import directive replaces tokens during parse-time
		if p.Val() == "import" {
			err := p.doImport()
			if err != nil {
				return err
			}
			p.cursor-- // cursor is advanced when we continue, so roll back one more
			continue
		}

		// normal case: parse a directive on this line
		if err := p.directive(); err != nil {
			return err
		}
	}
	return nil
}

// doImport swaps out the import directive and its argument
// (a total of 2 tokens) with the tokens in the specified file
// or globbing pattern. When the function returns, the cursor
// is on the token before where the import directive was. In
// other words, call Next() to access the first token that was
// imported.
func (p *parser) doImport() error {
	// syntax check
	if !p.NextArg() {
		return p.ArgErr()
	}
	importPattern := p.Val()
	if p.NextArg() {
		return p.Err("Import takes only one argument (glob pattern or file)")
	}

	// make path relative to Caddyfile rather than current working directory (issue #867)
	// and then use glob to get list of matching filenames
	absFile, err := filepath.Abs(p.Dispenser.filename)
	if err != nil {
		return p.Errf("Failed to get absolute path of file: %s", p.Dispenser.filename)
	}

	var matches []string
	relImportPattern := filepath.Join(filepath.Dir(absFile), importPattern)
	matches, err = filepath.Glob(relImportPattern)

	if err != nil {
		return p.Errf("Failed to use import pattern %s: %v", importPattern, err)
	}
	if len(matches) == 0 {
		return p.Errf("No files matching import pattern %s", importPattern)
	}

	// splice out the import directive and its argument (2 tokens total)
	tokensBefore := p.tokens[:p.cursor-1]
	tokensAfter := p.tokens[p.cursor+1:]

	// collect all the imported tokens
	var importedTokens []Token
	for _, importFile := range matches {
		newTokens, err := p.doSingleImport(importFile)
		if err != nil {
			return err
		}
		importedTokens = append(importedTokens, newTokens...)
	}

	// splice the imported tokens in the place of the import statement
	// and rewind cursor so Next() will land on first imported token
	p.tokens = append(tokensBefore, append(importedTokens, tokensAfter...)...)
	p.cursor--

	return nil
}

// doSingleImport lexes the individual file at importFile and returns
// its tokens or an error, if any.
func (p *parser) doSingleImport(importFile string) ([]Token, error) {
	file, err := os.Open(importFile)
	if err != nil {
		return nil, p.Errf("Could not import %s: %v", importFile, err)
	}
	defer file.Close()
	importedTokens := allTokens(file)

	// Tack the filename onto these tokens so errors show the imported file's name
	filename := filepath.Base(importFile)
	for i := 0; i < len(importedTokens); i++ {
		importedTokens[i].File = filename
	}

	return importedTokens, nil
}

// directive collects tokens until the directive's scope
// closes (either end of line or end of curly brace block).
// It expects the currently-loaded token to be a directive
// (or } that ends a server block). The collected tokens
// are loaded into the current server block for later use
// by directive setup functions.
func (p *parser) directive() error {
	dir := p.Val()
	nesting := 0

	// TODO: More helpful error message ("did you mean..." or "maybe you need to install its server type")
	if !p.validDirective(dir) {
		return p.Errf("Unknown directive '%s'", dir)
	}

	// The directive itself is appended as a relevant token
	p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])

	for p.Next() {
		if p.Val() == "{" {
			nesting++
		} else if p.isNewLine() && nesting == 0 {
			p.cursor-- // read too far
			break
		} else if p.Val() == "}" && nesting > 0 {
			nesting--
		} else if p.Val() == "}" && nesting == 0 {
			return p.Err("Unexpected '}' because no matching opening brace")
		}
		p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text)
		p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
	}

	if nesting > 0 {
		return p.EOFErr()
	}
	return nil
}

// openCurlyBrace expects the current token to be an
// opening curly brace. This acts like an assertion
// because it returns an error if the token is not
// a opening curly brace. It does NOT advance the token.
func (p *parser) openCurlyBrace() error {
	if p.Val() != "{" {
		return p.SyntaxErr("{")
	}
	return nil
}

// closeCurlyBrace expects the current token to be
// a closing curly brace. This acts like an assertion
// because it returns an error if the token is not
// a closing curly brace. It does NOT advance the token.
func (p *parser) closeCurlyBrace() error {
	if p.Val() != "}" {
		return p.SyntaxErr("}")
	}
	return nil
}

// validDirective returns true if dir is in p.validDirectives.
func (p *parser) validDirective(dir string) bool {
	if p.validDirectives == nil {
		return true
	}
	for _, d := range p.validDirectives {
		if d == dir {
			return true
		}
	}
	return false
}

// replaceEnvVars replaces environment variables that appear in the token
// and understands both the $UNIX and %WINDOWS% syntaxes.
func replaceEnvVars(s string) string {
	s = replaceEnvReferences(s, "{%", "%}")
	s = replaceEnvReferences(s, "{$", "}")
	return s
}

// replaceEnvReferences performs the actual replacement of env variables
// in s, given the placeholder start and placeholder end strings.
func replaceEnvReferences(s, refStart, refEnd string) string {
	index := strings.Index(s, refStart)
	for index != -1 {
		endIndex := strings.Index(s, refEnd)
		if endIndex != -1 {
			ref := s[index : endIndex+len(refEnd)]
			s = strings.Replace(s, ref, os.Getenv(ref[len(refStart):len(ref)-len(refEnd)]), -1)
		} else {
			return s
		}
		index = strings.Index(s, refStart)
	}
	return s
}

// ServerBlock associates any number of keys (usually addresses
// of some sort) with tokens (grouped by directive name).
type ServerBlock struct {
	Keys   []string
	Tokens map[string][]Token
}
<commit_msg>fix: import should always be relative to current file directory<commit_after>package caddyfile

import (
	"io"
	"os"
	"path/filepath"
	"strings"
)

// Parse parses the input just enough to group tokens, in
// order, by server block. No further parsing is performed.
// Server blocks are returned in the order in which they appear.
// Directives that do not appear in validDirectives will cause
// an error. If you do not want to check for valid directives,
// pass in nil instead.
func Parse(filename string, input io.Reader, validDirectives []string) ([]ServerBlock, error) {
	p := parser{Dispenser: NewDispenser(filename, input), validDirectives: validDirectives}
	blocks, err := p.parseAll()
	return blocks, err
}

// allTokens lexes the entire input, but does not parse it.
// It returns all the tokens from the input, unstructured
// and in order.
func allTokens(input io.Reader) (tokens []Token) {
	l := new(lexer)
	l.load(input)
	for l.next() {
		tokens = append(tokens, l.token)
	}
	return
}

type parser struct {
	Dispenser
	block           ServerBlock // current server block being parsed
	validDirectives []string    // a directive must be valid or it's an error
	eof             bool        // if we encounter a valid EOF in a hard place
}

func (p *parser) parseAll() ([]ServerBlock, error) {
	var blocks []ServerBlock

	for p.Next() {
		err := p.parseOne()
		if err != nil {
			return blocks, err
		}
		if len(p.block.Keys) > 0 {
			blocks = append(blocks, p.block)
		}
	}

	return blocks, nil
}

func (p *parser) parseOne() error {
	p.block = ServerBlock{Tokens: make(map[string][]Token)}

	err := p.begin()
	if err != nil {
		return err
	}

	return nil
}

func (p *parser) begin() error {
	if len(p.tokens) == 0 {
		return nil
	}

	err := p.addresses()
	if err != nil {
		return err
	}

	if p.eof {
		// this happens if the Caddyfile consists of only
		// a line of addresses and nothing else
		return nil
	}

	err = p.blockContents()
	if err != nil {
		return err
	}

	return nil
}

func (p *parser) addresses() error {
	var expectingAnother bool

	for {
		tkn := replaceEnvVars(p.Val())

		// special case: import directive replaces tokens during parse-time
		if tkn == "import" && p.isNewLine() {
			err := p.doImport()
			if err != nil {
				return err
			}
			continue
		}

		// Open brace definitely indicates end of addresses
		if tkn == "{" {
			if expectingAnother {
				return p.Errf("Expected another address but had '%s' - check for extra comma", tkn)
			}
			break
		}

		if tkn != "" { // empty token possible if user typed ""
			// Trailing comma indicates another address will follow, which
			// may possibly be on the next line
			if tkn[len(tkn)-1] == ',' {
				tkn = tkn[:len(tkn)-1]
				expectingAnother = true
			} else {
				expectingAnother = false // but we may still see another one on this line
			}

			p.block.Keys = append(p.block.Keys, tkn)
		}

		// Advance token and possibly break out of loop or return error
		hasNext := p.Next()
		if expectingAnother && !hasNext {
			return p.EOFErr()
		}
		if !hasNext {
			p.eof = true
			break // EOF
		}
		if !expectingAnother && p.isNewLine() {
			break
		}
	}

	return nil
}

func (p *parser) blockContents() error {
	errOpenCurlyBrace := p.openCurlyBrace()
	if errOpenCurlyBrace != nil {
		// single-server configs don't need curly braces
		p.cursor--
	}

	err := p.directives()
	if err != nil {
		return err
	}

	// Only look for close curly brace if there was an opening
	if errOpenCurlyBrace == nil {
		err = p.closeCurlyBrace()
		if err != nil {
			return err
		}
	}

	return nil
}

// directives parses through all the lines for directives
// and it expects the next token to be the first
// directive. It goes until EOF or closing curly brace
// which ends the server block.
func (p *parser) directives() error {
	for p.Next() {
		// end of server block
		if p.Val() == "}" {
			break
		}

		// special case: import directive replaces tokens during parse-time
		if p.Val() == "import" {
			err := p.doImport()
			if err != nil {
				return err
			}
			p.cursor-- // cursor is advanced when we continue, so roll back one more
			continue
		}

		// normal case: parse a directive on this line
		if err := p.directive(); err != nil {
			return err
		}
	}
	return nil
}

// doImport swaps out the import directive and its argument
// (a total of 2 tokens) with the tokens in the specified file
// or globbing pattern. When the function returns, the cursor
// is on the token before where the import directive was. In
// other words, call Next() to access the first token that was
// imported.
func (p *parser) doImport() error {
	// syntax check
	if !p.NextArg() {
		return p.ArgErr()
	}
	importPattern := p.Val()
	if p.NextArg() {
		return p.Err("Import takes only one argument (glob pattern or file)")
	}

	// make path relative to Caddyfile rather than current working directory (issue #867)
	// and then use glob to get list of matching filenames
	absFile, err := filepath.Abs(p.Dispenser.filename)
	if err != nil {
		return p.Errf("Failed to get absolute path of file: %s", p.Dispenser.filename)
	}

	var matches []string
	var globPattern string
	if !filepath.IsAbs(importPattern) {
		globPattern = filepath.Join(filepath.Dir(absFile), importPattern)
	} else {
		globPattern = importPattern
	}
	matches, err = filepath.Glob(globPattern)

	if err != nil {
		return p.Errf("Failed to use import pattern %s: %v", importPattern, err)
	}
	if len(matches) == 0 {
		return p.Errf("No files matching import pattern %s", importPattern)
	}

	// splice out the import directive and its argument (2 tokens total)
	tokensBefore := p.tokens[:p.cursor-1]
	tokensAfter := p.tokens[p.cursor+1:]

	// collect all the imported tokens
	var importedTokens []Token
	for _, importFile := range matches {
		newTokens, err := p.doSingleImport(importFile)
		if err != nil {
			return err
		}
		var importLine int
		importDir := filepath.Dir(importFile)
		for i, token := range newTokens {
			if token.Text == "import" {
				importLine = token.Line
				continue
			}
			if token.Line == importLine {
				var abs string
				if !filepath.IsAbs(importFile) {
					abs = filepath.Join(filepath.Dir(absFile), token.Text)
				} else {
					abs = filepath.Join(importDir, token.Text)
				}
				newTokens[i] = Token{
					Text: abs,
					Line: token.Line,
					File: token.File,
				}
			}
		}
		importedTokens = append(importedTokens, newTokens...)
	}

	// splice the imported tokens in the place of the import statement
	// and rewind cursor so Next() will land on first imported token
	p.tokens = append(tokensBefore, append(importedTokens, tokensAfter...)...)
	p.cursor--

	return nil
}

// doSingleImport lexes the individual file at importFile and returns
// its tokens or an error, if any.
func (p *parser) doSingleImport(importFile string) ([]Token, error) {
	file, err := os.Open(importFile)
	if err != nil {
		return nil, p.Errf("Could not import %s: %v", importFile, err)
	}
	defer file.Close()
	importedTokens := allTokens(file)

	// Tack the filename onto these tokens so errors show the imported file's name
	filename := filepath.Base(importFile)
	for i := 0; i < len(importedTokens); i++ {
		importedTokens[i].File = filename
	}

	return importedTokens, nil
}

// directive collects tokens until the directive's scope
// closes (either end of line or end of curly brace block).
// It expects the currently-loaded token to be a directive
// (or } that ends a server block). The collected tokens
// are loaded into the current server block for later use
// by directive setup functions.
func (p *parser) directive() error {
	dir := p.Val()
	nesting := 0

	// TODO: More helpful error message ("did you mean..." or "maybe you need to install its server type")
	if !p.validDirective(dir) {
		return p.Errf("Unknown directive '%s'", dir)
	}

	// The directive itself is appended as a relevant token
	p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])

	for p.Next() {
		if p.Val() == "{" {
			nesting++
		} else if p.isNewLine() && nesting == 0 {
			p.cursor-- // read too far
			break
		} else if p.Val() == "}" && nesting > 0 {
			nesting--
		} else if p.Val() == "}" && nesting == 0 {
			return p.Err("Unexpected '}' because no matching opening brace")
		}
		p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text)
		p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
	}

	if nesting > 0 {
		return p.EOFErr()
	}
	return nil
}

// openCurlyBrace expects the current token to be an
// opening curly brace. This acts like an assertion
// because it returns an error if the token is not
// a opening curly brace. It does NOT advance the token.
func (p *parser) openCurlyBrace() error {
	if p.Val() != "{" {
		return p.SyntaxErr("{")
	}
	return nil
}

// closeCurlyBrace expects the current token to be
// a closing curly brace. This acts like an assertion
// because it returns an error if the token is not
// a closing curly brace. It does NOT advance the token.
func (p *parser) closeCurlyBrace() error {
	if p.Val() != "}" {
		return p.SyntaxErr("}")
	}
	return nil
}

// validDirective returns true if dir is in p.validDirectives.
func (p *parser) validDirective(dir string) bool {
	if p.validDirectives == nil {
		return true
	}
	for _, d := range p.validDirectives {
		if d == dir {
			return true
		}
	}
	return false
}

// replaceEnvVars replaces environment variables that appear in the token
// and understands both the $UNIX and %WINDOWS% syntaxes.
func replaceEnvVars(s string) string {
	s = replaceEnvReferences(s, "{%", "%}")
	s = replaceEnvReferences(s, "{$", "}")
	return s
}

// replaceEnvReferences performs the actual replacement of env variables
// in s, given the placeholder start and placeholder end strings.
func replaceEnvReferences(s, refStart, refEnd string) string {
	index := strings.Index(s, refStart)
	for index != -1 {
		endIndex := strings.Index(s, refEnd)
		if endIndex != -1 {
			ref := s[index : endIndex+len(refEnd)]
			s = strings.Replace(s, ref, os.Getenv(ref[len(refStart):len(ref)-len(refEnd)]), -1)
		} else {
			return s
		}
		index = strings.Index(s, refStart)
	}
	return s
}

// ServerBlock associates any number of keys (usually addresses
// of some sort) with tokens (grouped by directive name).
type ServerBlock struct {
	Keys   []string
	Tokens map[string][]Token
}
<|endoftext|>


<commit_before>package agent

import (
	"io/ioutil"
	"os"
	"sync/atomic"
	"testing"
)

var nextPort uint32 = 17000

func getPort() int {
	return int(atomic.AddUint32(&nextPort, 1))
}

func tmpDir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "nomad")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	return dir
}

func makeAgent(t *testing.T, cb func(*Config)) (string, *Agent) {
	dir := tmpDir(t)
	conf := DevConfig()

	if cb != nil {
		cb(conf)
	}

	agent, err := NewAgent(conf, os.Stderr)
	if err != nil {
		os.RemoveAll(dir)
		t.Fatalf("err: %v", err)
	}
	return dir, agent
}
<commit_msg>agent: adding basic test<commit_after>package agent

import (
	"io/ioutil"
	"os"
	"sync/atomic"
	"testing"
)

var nextPort uint32 = 17000

func getPort() int {
	return int(atomic.AddUint32(&nextPort, 1))
}

func tmpDir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "nomad")
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	return dir
}

func makeAgent(t *testing.T, cb func(*Config)) (string, *Agent) {
	dir := tmpDir(t)
	conf := DevConfig()

	if cb != nil {
		cb(conf)
	}

	agent, err := NewAgent(conf, os.Stderr)
	if err != nil {
		os.RemoveAll(dir)
		t.Fatalf("err: %v", err)
	}
	return dir, agent
}

func TestAgent_RPCPing(t *testing.T) {
	dir, agent := makeAgent(t, nil)
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	var out struct{}
	if err := agent.RPC("Status.Ping", struct{}{}, &out); err != nil {
		t.Fatalf("err: %v", err)
	}
}
<|endoftext|>


<commit_before>package container

import (
	"fmt"
	"strings"

	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/cli"
	"github.com/docker/docker/cli/command"
	"github.com/docker/docker/opts"
	runconfigopts "github.com/docker/docker/runconfig/opts"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
)

type updateOptions struct {
	blkioWeight        uint16
	cpuPeriod          int64
	cpuQuota           int64
	cpuRealtimePeriod  int64
	cpuRealtimeRuntime int64
	cpusetCpus         string
	cpusetMems         string
	cpuShares          int64
	memory             opts.MemBytes
	memoryReservation  opts.MemBytes
	memorySwap         opts.MemSwapBytes
	kernelMemory       opts.MemBytes
	restartPolicy      string

	nFlag int

	containers []string
}

// NewUpdateCommand creates a new cobra.Command for `docker update`
func NewUpdateCommand(dockerCli *command.DockerCli) *cobra.Command {
	var opts updateOptions

	cmd := &cobra.Command{
		Use:   "update [OPTIONS] CONTAINER [CONTAINER...]",
		Short: "Update configuration of one or more containers",
		Args:  cli.RequiresMinArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.containers = args
			opts.nFlag = cmd.Flags().NFlag()
			return runUpdate(dockerCli, &opts)
		},
	}

	flags := cmd.Flags()
	flags.Uint16Var(&opts.blkioWeight, "blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)")
	flags.Int64Var(&opts.cpuPeriod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period")
	flags.Int64Var(&opts.cpuQuota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota")
	flags.Int64Var(&opts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds")
	flags.SetAnnotation("cpu-rt-period", "version", []string{"1.25"})
	flags.Int64Var(&opts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit the CPU real-time runtime in microseconds")
	flags.SetAnnotation("cpu-rt-runtime", "version", []string{"1.25"})
	flags.StringVar(&opts.cpusetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)")
	flags.StringVar(&opts.cpusetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)")
	flags.Int64VarP(&opts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
	flags.VarP(&opts.memory, "memory", "m", "Memory limit")
	flags.Var(&opts.memoryReservation, "memory-reservation", "Memory soft limit")
	flags.Var(&opts.memorySwap, "memory-swap", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap")
	flags.Var(&opts.kernelMemory, "kernel-memory", "Kernel memory limit")
	flags.StringVar(&opts.restartPolicy, "restart", "", "Restart policy to apply when a container exits")

	return cmd
}

func runUpdate(dockerCli *command.DockerCli, opts *updateOptions) error {
	var err error

	if opts.nFlag == 0 {
		return errors.New("You must provide one or more flags when using this command.")
	}

	var restartPolicy containertypes.RestartPolicy
	if opts.restartPolicy != "" {
		restartPolicy, err = runconfigopts.ParseRestartPolicy(opts.restartPolicy)
		if err != nil {
			return err
		}
	}

	resources := containertypes.Resources{
		BlkioWeight:        opts.blkioWeight,
		CpusetCpus:         opts.cpusetCpus,
		CpusetMems:         opts.cpusetMems,
		CPUShares:          opts.cpuShares,
		Memory:             opts.memory.Value(),
		MemoryReservation:  opts.memoryReservation.Value(),
		MemorySwap:         opts.memorySwap.Value(),
		KernelMemory:       opts.kernelMemory.Value(),
		CPUPeriod:          opts.cpuPeriod,
		CPUQuota:           opts.cpuQuota,
		CPURealtimePeriod:  opts.cpuRealtimePeriod,
		CPURealtimeRuntime: opts.cpuRealtimeRuntime,
	}

	updateConfig := containertypes.UpdateConfig{
		Resources:     resources,
		RestartPolicy: restartPolicy,
	}

	ctx := context.Background()

	var (
		warns []string
		errs  []string
	)
	for _, container := range opts.containers {
		r, err := dockerCli.Client().ContainerUpdate(ctx, container, updateConfig)
		if err != nil {
			errs = append(errs, err.Error())
		} else {
			fmt.Fprintln(dockerCli.Out(), container)
		}
		warns = append(warns, r.Warnings...)
	}
	if len(warns) > 0 {
		fmt.Fprintln(dockerCli.Out(), strings.Join(warns, "\n"))
	}
	if len(errs) > 0 {
		return errors.New(strings.Join(errs, "\n"))
	}
	return nil
}
<commit_msg>Add `--cpus` support for `docker update`<commit_after>package container

import (
	"fmt"
	"strings"

	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/cli"
	"github.com/docker/docker/cli/command"
	"github.com/docker/docker/opts"
	runconfigopts "github.com/docker/docker/runconfig/opts"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"golang.org/x/net/context"
)

type updateOptions struct {
	blkioWeight        uint16
	cpuPeriod          int64
	cpuQuota           int64
	cpuRealtimePeriod  int64
	cpuRealtimeRuntime int64
	cpusetCpus         string
	cpusetMems         string
	cpuShares          int64
	memory             opts.MemBytes
	memoryReservation  opts.MemBytes
	memorySwap         opts.MemSwapBytes
	kernelMemory       opts.MemBytes
	restartPolicy      string
	cpus               opts.NanoCPUs

	nFlag int

	containers []string
}

// NewUpdateCommand creates a new cobra.Command for `docker update`
func NewUpdateCommand(dockerCli *command.DockerCli) *cobra.Command {
	var opts updateOptions

	cmd := &cobra.Command{
		Use:   "update [OPTIONS] CONTAINER [CONTAINER...]",
		Short: "Update configuration of one or more containers",
		Args:  cli.RequiresMinArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.containers = args
			opts.nFlag = cmd.Flags().NFlag()
			return runUpdate(dockerCli, &opts)
		},
	}

	flags := cmd.Flags()
	flags.Uint16Var(&opts.blkioWeight, "blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)")
	flags.Int64Var(&opts.cpuPeriod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period")
	flags.Int64Var(&opts.cpuQuota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota")
	flags.Int64Var(&opts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds")
	flags.SetAnnotation("cpu-rt-period", "version", []string{"1.25"})
	flags.Int64Var(&opts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit the CPU real-time runtime in microseconds")
	flags.SetAnnotation("cpu-rt-runtime", "version", []string{"1.25"})
	flags.StringVar(&opts.cpusetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)")
	flags.StringVar(&opts.cpusetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)")
	flags.Int64VarP(&opts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
	flags.VarP(&opts.memory, "memory", "m", "Memory limit")
	flags.Var(&opts.memoryReservation, "memory-reservation", "Memory soft limit")
	flags.Var(&opts.memorySwap, "memory-swap", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap")
	flags.Var(&opts.kernelMemory, "kernel-memory", "Kernel memory limit")
	flags.StringVar(&opts.restartPolicy, "restart", "", "Restart policy to apply when a container exits")

	flags.Var(&opts.cpus, "cpus", "Number of CPUs")
	flags.SetAnnotation("cpus", "version", []string{"1.29"})

	return cmd
}

func runUpdate(dockerCli *command.DockerCli, opts *updateOptions) error {
	var err error

	if opts.nFlag == 0 {
		return errors.New("You must provide one or more flags when using this command.")
	}

	var restartPolicy containertypes.RestartPolicy
	if opts.restartPolicy != "" {
		restartPolicy, err = runconfigopts.ParseRestartPolicy(opts.restartPolicy)
		if err != nil {
			return err
		}
	}

	resources := containertypes.Resources{
		BlkioWeight:        opts.blkioWeight,
		CpusetCpus:         opts.cpusetCpus,
		CpusetMems:         opts.cpusetMems,
		CPUShares:          opts.cpuShares,
		Memory:             opts.memory.Value(),
		MemoryReservation:  opts.memoryReservation.Value(),
		MemorySwap:         opts.memorySwap.Value(),
		KernelMemory:       opts.kernelMemory.Value(),
		CPUPeriod:          opts.cpuPeriod,
		CPUQuota:           opts.cpuQuota,
		CPURealtimePeriod:  opts.cpuRealtimePeriod,
		CPURealtimeRuntime: opts.cpuRealtimeRuntime,
		NanoCPUs:           opts.cpus.Value(),
	}

	updateConfig := containertypes.UpdateConfig{
		Resources:     resources,
		RestartPolicy: restartPolicy,
	}

	ctx := context.Background()

	var (
		warns []string
		errs  []string
	)
	for _, container := range opts.containers {
		r, err := dockerCli.Client().ContainerUpdate(ctx, container, updateConfig)
		if err != nil {
			errs = append(errs, err.Error())
		} else {
			fmt.Fprintln(dockerCli.Out(), container)
		}
		warns = append(warns, r.Warnings...)
	}
	if len(warns) > 0 {
		fmt.Fprintln(dockerCli.Out(), strings.Join(warns, "\n"))
	}
	if len(errs) > 0 {
		return errors.New(strings.Join(errs, "\n"))
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

func readUint(path string) (uint64, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}

	value, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 64)
	if err != nil {
		return 0, err
	}

	return value, nil
}

func networkAutoAttach(cluster *db.Cluster, devName string) error {
	_, dbInfo, err := cluster.GetNetworkWithInterface(devName)
	if err != nil {
		// No match found, move on
		return nil
	}

	return network.AttachInterface(dbInfo.Name, devName)
}

func networkGetInterfaces(cluster *db.Cluster) ([]string, error) {
	networks, err := cluster.GetNetworks()
	if err != nil {
		return nil, err
	}

	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		// Ignore veth pairs (for performance reasons)
		if strings.HasPrefix(iface.Name, "veth") {
			continue
		}

		// Append to the list
		if !shared.StringInSlice(iface.Name, networks) {
			networks = append(networks, iface.Name)
		}
	}

	return networks, nil
}

// networkUpdateForkdnsServersTask runs every 30s and refreshes the forkdns servers list.
func networkUpdateForkdnsServersTask(s *state.State, heartbeatData *cluster.APIHeartbeat) error {
	// Get a list of managed networks
	networks, err := s.Cluster.GetNonPendingNetworks()
	if err != nil {
		return err
	}

	for _, name := range networks {
		n, err := network.LoadByName(s, name)
		if err != nil {
			logger.Errorf("Failed to load network %q for heartbeat", name)
			continue
		}

		if n.Type() == "bridge" && n.Config()["bridge.mode"] == "fan" {
			err := n.HandleHeartbeat(heartbeatData)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func networkGetState(netIf net.Interface) api.NetworkState {
	netState := "down"
	netType := "unknown"

	if netIf.Flags&net.FlagBroadcast > 0 {
		netType = "broadcast"
	}

	if netIf.Flags&net.FlagPointToPoint > 0 {
		netType = "point-to-point"
	}

	if netIf.Flags&net.FlagLoopback > 0 {
		netType = "loopback"
	}

	if netIf.Flags&net.FlagUp > 0 {
		netState = "up"
	}

	network := api.NetworkState{
		Addresses: []api.NetworkStateAddress{},
		Counters:  api.NetworkStateCounters{},
		Hwaddr:    netIf.HardwareAddr.String(),
		Mtu:       netIf.MTU,
		State:     netState,
		Type:      netType,
	}

	// Populate address information.
	addrs, err := netIf.Addrs()
	if err == nil {
		for _, addr := range addrs {
			fields := strings.SplitN(addr.String(), "/", 2)
			if len(fields) != 2 {
				continue
			}

			family := "inet"
			if strings.Contains(fields[0], ":") {
				family = "inet6"
			}

			scope := "global"
			if strings.HasPrefix(fields[0], "127") {
				scope = "local"
			}

			if fields[0] == "::1" {
				scope = "local"
			}

			if strings.HasPrefix(fields[0], "169.254") {
				scope = "link"
			}

			if strings.HasPrefix(fields[0], "fe80:") {
				scope = "link"
			}

			address := api.NetworkStateAddress{}
			address.Family = family
			address.Address = fields[0]
			address.Netmask = fields[1]
			address.Scope = scope

			network.Addresses = append(network.Addresses, address)
		}
	}

	// Populate bond details.
	bondPath := fmt.Sprintf("/sys/class/net/%s/bonding", netIf.Name)
	if shared.PathExists(bondPath) {
		bonding := api.NetworkStateBond{}

		// Bond mode.
		strValue, err := ioutil.ReadFile(filepath.Join(bondPath, "mode"))
		if err == nil {
			bonding.Mode = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Bond transmit policy.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "xmit_hash_policy"))
		if err == nil {
			bonding.TransmitPolicy = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Up delay.
		uintValue, err := readUint(filepath.Join(bondPath, "updelay"))
		if err == nil {
			bonding.UpDelay = uintValue
		}

		// Down delay.
		uintValue, err = readUint(filepath.Join(bondPath, "downdelay"))
		if err == nil {
			bonding.DownDelay = uintValue
		}

		// MII frequency.
		uintValue, err = readUint(filepath.Join(bondPath, "miimon"))
		if err == nil {
			bonding.MIIFrequency = uintValue
		}

		// MII state.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "mii_status"))
		if err == nil {
			bonding.MIIState = strings.TrimSpace(string(strValue))
		}

		// Lower devices.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "slaves"))
		if err == nil {
			bonding.LowerDevices = strings.Split(strings.TrimSpace(string(strValue)), " ")
		}

		network.Bond = &bonding
	}

	// Populate bridge details.
	bridgePath := fmt.Sprintf("/sys/class/net/%s/bridge", netIf.Name)
	if shared.PathExists(bridgePath) {
		bridge := api.NetworkStateBridge{}

		// Bridge ID.
		strValue, err := ioutil.ReadFile(filepath.Join(bridgePath, "bridge_id"))
		if err == nil {
			bridge.ID = strings.TrimSpace(string(strValue))
		}

		// Bridge STP.
		uintValue, err := readUint(filepath.Join(bridgePath, "stp_state"))
		if err == nil {
			bridge.STP = uintValue == 1
		}

		// Bridge forward delay.
		uintValue, err = readUint(filepath.Join(bridgePath, "forward_delay"))
		if err == nil {
			bridge.ForwardDelay = uintValue
		}

		// Bridge default VLAN.
		uintValue, err = readUint(filepath.Join(bridgePath, "default_pvid"))
		if err == nil {
			bridge.VLANDefault = uintValue
		}

		// Bridge VLAN filtering.
		uintValue, err = readUint(filepath.Join(bridgePath, "vlan_filtering"))
		if err == nil {
			bridge.VLANFiltering = uintValue == 1
		}

		// Upper devices.
		bridgeIfPath := fmt.Sprintf("/sys/class/net/%s/brif", netIf.Name)
		if shared.PathExists(bridgeIfPath) {
			entries, err := ioutil.ReadDir(bridgeIfPath)
			if err == nil {
				bridge.UpperDevices = []string{}
				for _, entry := range entries {
					bridge.UpperDevices = append(bridge.UpperDevices, entry.Name())
				}
			}
		}

		network.Bridge = &bridge
	}

	// Get counters.
	network.Counters = shared.NetworkGetCounters(netIf.Name)
	return network
}
<commit_msg>lxd/networks/utils: Removes networkGetInterfaces function<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/network"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

func readUint(path string) (uint64, error) {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}

	value, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 64)
	if err != nil {
		return 0, err
	}

	return value, nil
}

func networkAutoAttach(cluster *db.Cluster, devName string) error {
	_, dbInfo, err := cluster.GetNetworkWithInterface(devName)
	if err != nil {
		// No match found, move on
		return nil
	}

	return network.AttachInterface(dbInfo.Name, devName)
}

// networkUpdateForkdnsServersTask runs every 30s and refreshes the forkdns servers list.
func networkUpdateForkdnsServersTask(s *state.State, heartbeatData *cluster.APIHeartbeat) error {
	// Get a list of managed networks
	networks, err := s.Cluster.GetNonPendingNetworks()
	if err != nil {
		return err
	}

	for _, name := range networks {
		n, err := network.LoadByName(s, name)
		if err != nil {
			logger.Errorf("Failed to load network %q for heartbeat", name)
			continue
		}

		if n.Type() == "bridge" && n.Config()["bridge.mode"] == "fan" {
			err := n.HandleHeartbeat(heartbeatData)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func networkGetState(netIf net.Interface) api.NetworkState {
	netState := "down"
	netType := "unknown"

	if netIf.Flags&net.FlagBroadcast > 0 {
		netType = "broadcast"
	}

	if netIf.Flags&net.FlagPointToPoint > 0 {
		netType = "point-to-point"
	}

	if netIf.Flags&net.FlagLoopback > 0 {
		netType = "loopback"
	}

	if netIf.Flags&net.FlagUp > 0 {
		netState = "up"
	}

	network := api.NetworkState{
		Addresses: []api.NetworkStateAddress{},
		Counters:  api.NetworkStateCounters{},
		Hwaddr:    netIf.HardwareAddr.String(),
		Mtu:       netIf.MTU,
		State:     netState,
		Type:      netType,
	}

	// Populate address information.
	addrs, err := netIf.Addrs()
	if err == nil {
		for _, addr := range addrs {
			fields := strings.SplitN(addr.String(), "/", 2)
			if len(fields) != 2 {
				continue
			}

			family := "inet"
			if strings.Contains(fields[0], ":") {
				family = "inet6"
			}

			scope := "global"
			if strings.HasPrefix(fields[0], "127") {
				scope = "local"
			}

			if fields[0] == "::1" {
				scope = "local"
			}

			if strings.HasPrefix(fields[0], "169.254") {
				scope = "link"
			}

			if strings.HasPrefix(fields[0], "fe80:") {
				scope = "link"
			}

			address := api.NetworkStateAddress{}
			address.Family = family
			address.Address = fields[0]
			address.Netmask = fields[1]
			address.Scope = scope

			network.Addresses = append(network.Addresses, address)
		}
	}

	// Populate bond details.
	bondPath := fmt.Sprintf("/sys/class/net/%s/bonding", netIf.Name)
	if shared.PathExists(bondPath) {
		bonding := api.NetworkStateBond{}

		// Bond mode.
		strValue, err := ioutil.ReadFile(filepath.Join(bondPath, "mode"))
		if err == nil {
			bonding.Mode = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Bond transmit policy.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "xmit_hash_policy"))
		if err == nil {
			bonding.TransmitPolicy = strings.Split(strings.TrimSpace(string(strValue)), " ")[0]
		}

		// Up delay.
		uintValue, err := readUint(filepath.Join(bondPath, "updelay"))
		if err == nil {
			bonding.UpDelay = uintValue
		}

		// Down delay.
		uintValue, err = readUint(filepath.Join(bondPath, "downdelay"))
		if err == nil {
			bonding.DownDelay = uintValue
		}

		// MII frequency.
		uintValue, err = readUint(filepath.Join(bondPath, "miimon"))
		if err == nil {
			bonding.MIIFrequency = uintValue
		}

		// MII state.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "mii_status"))
		if err == nil {
			bonding.MIIState = strings.TrimSpace(string(strValue))
		}

		// Lower devices.
		strValue, err = ioutil.ReadFile(filepath.Join(bondPath, "slaves"))
		if err == nil {
			bonding.LowerDevices = strings.Split(strings.TrimSpace(string(strValue)), " ")
		}

		network.Bond = &bonding
	}

	// Populate bridge details.
	bridgePath := fmt.Sprintf("/sys/class/net/%s/bridge", netIf.Name)
	if shared.PathExists(bridgePath) {
		bridge := api.NetworkStateBridge{}

		// Bridge ID.
		strValue, err := ioutil.ReadFile(filepath.Join(bridgePath, "bridge_id"))
		if err == nil {
			bridge.ID = strings.TrimSpace(string(strValue))
		}

		// Bridge STP.
		uintValue, err := readUint(filepath.Join(bridgePath, "stp_state"))
		if err == nil {
			bridge.STP = uintValue == 1
		}

		// Bridge forward delay.
		uintValue, err = readUint(filepath.Join(bridgePath, "forward_delay"))
		if err == nil {
			bridge.ForwardDelay = uintValue
		}

		// Bridge default VLAN.
		uintValue, err = readUint(filepath.Join(bridgePath, "default_pvid"))
		if err == nil {
			bridge.VLANDefault = uintValue
		}

		// Bridge VLAN filtering.
		uintValue, err = readUint(filepath.Join(bridgePath, "vlan_filtering"))
		if err == nil {
			bridge.VLANFiltering = uintValue == 1
		}

		// Upper devices.
		bridgeIfPath := fmt.Sprintf("/sys/class/net/%s/brif", netIf.Name)
		if shared.PathExists(bridgeIfPath) {
			entries, err := ioutil.ReadDir(bridgeIfPath)
			if err == nil {
				bridge.UpperDevices = []string{}
				for _, entry := range entries {
					bridge.UpperDevices = append(bridge.UpperDevices, entry.Name())
				}
			}
		}

		network.Bridge = &bridge
	}

	// Get counters.
	network.Counters = shared.NetworkGetCounters(netIf.Name)
	return network
}
<|endoftext|>