<commit_before>// Copyright 2011 Kevin Bulusek. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gameboy

import (
	"⚛sdl"
	"time"
)

const (
	modeHBlank = byte(iota)
	modeVBlank
	modeOAM
	modeVRAM
)

const (
	tilesX   = 20
	tilesY   = 18
	tileW    = 8
	tileH    = 8
	mapW     = 32
	mapH     = 32
	displayW = tileW * tilesX
	displayH = tileH * tilesY

	hblankTicks = 204 / 4
	vblankTicks = 4560 / 4
	oamTicks    = 80 / 4
	vramTicks   = 172 / 4

	scanlineTicks = oamTicks + vramTicks + hblankTicks
	refreshTicks  = scanlineTicks*displayH + vblankTicks
)

type display struct {
	*memory
	*sdl.Surface
	pal       [4]uint32
	frameTime int64
	screenW   int
	screenH   int

	clock int

	// LCDC flags
	enable       bool
	windowMap    bool
	windowEnable bool
	tileData     bool
	bgMap        bool
	spriteSize   bool
	spriteEnable bool
	bgEnable     bool

	// STAT flags
	lycInterrupt    bool
	oamInterrupt    bool
	vblankInterrupt bool
	hblankInterrupt bool
	mode            byte

	// LCD registers
	ly       byte
	scy, scx byte
	wy, wx   byte

	bgp [4]byte
	obp [2][4]byte

	// Scanlines are rendered to here first, and then drawn to the
	// display - rather than each layer accessing the display
	// separately (which is slow).
	lineBuf [displayW]byte

	// When rendering a scanline this is zeroed out, then
	// logical-ORed with the pixels from the BG and window. This
	// is then used to lookup which pixels can be painted in
	// sprites that are to be obscured by the background layers.
	oamLineMask [displayW]byte
}

func newDisplay(m *memory) *display {
	sdl.WM_SetCaption(m.rom.title(), "")
	flags := uint32(sdl.DOUBLEBUF)
	if m.config.Fullscreen {
		flags |= sdl.FULLSCREEN
	}
	lcd := display{memory: m}
	lcd.screenW = displayW * m.config.Scale
	lcd.screenH = displayH * m.config.Scale
	lcd.Surface = sdl.SetVideoMode(lcd.screenW, lcd.screenH, 0, flags)
	lcd.pal[0] = sdl.MapRGBA(lcd.Format, 0x9B, 0xBC, 0x0F, 0)
	lcd.pal[1] = sdl.MapRGBA(lcd.Format, 0x8B, 0xAC, 0x0F, 0)
	lcd.pal[2] = sdl.MapRGBA(lcd.Format, 0x30, 0x62, 0x30, 0)
	lcd.pal[3] = sdl.MapRGBA(lcd.Format, 0x0F, 0x38, 0x0F, 0)
	sdl.ShowCursor(sdl.DISABLE)
	lcd.FillRect(nil, lcd.pal[0])
	lcd.Flip()
	lcd.frameTime = time.Nanoseconds()
	return &lcd
}

func (lcd *display) toggleFullScreen() {
	sdl.WM_ToggleFullScreen(lcd.Surface)
}

func (lcd *display) step(t int) {
	lcd.clock += t
	if lcd.clock >= refreshTicks {
		lcd.clock -= refreshTicks
	}

	lcd.ly = byte(lcd.clock / scanlineTicks)
	lcd.hram[portLY-0xFF00] = lcd.ly
	mode := calcMode(lcd.clock, lcd.ly)
	if mode == lcd.mode {
		return
	}
	lcd.mode = mode

	stat := lcd.readPort(portSTAT)&^3 | mode
	irq := lcd.readPort(portIF)

	switch mode {
	case modeOAM:
		if lcd.oamInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
		if lyc := lcd.readPort(portLYC); lcd.ly-1 == lyc {
			stat |= 0x04
			if lcd.lycInterrupt {
				lcd.writePort(portIF, irq|0x02)
			}
		} else {
			stat &^= 0x04
		}
	case modeVRAM:
		if lcd.enable {
			lcd.scanline()
		}
	case modeHBlank:
		if lcd.hblankInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
	case modeVBlank:
		if lcd.vblankInterrupt {
			irq |= 0x02
		}
		lcd.writePort(portIF, irq|0x01)
		lcd.Flip()

		// while audio is playing, we let it control the
		// emulation speed
		if !lcd.audio.enable {
			lcd.delay()
		}
	}

	lcd.writePort(portSTAT, stat)
}

func (lcd *display) delay() {
	now := time.Nanoseconds()
	delta := now - lcd.frameTime
	target := 16742706 - delta
	if target > 0 {
		time.Sleep(target)
	}
	lcd.frameTime = time.Nanoseconds()
}

func (lcd *display) scanline() {
	for i := 0; i < displayW; i++ {
		lcd.oamLineMask[i] = 0
		lcd.lineBuf[i] = 0
	}

	if lcd.bgEnable {
		lcd.mapline(lcd.bgMap, byte(0), lcd.scx, lcd.scy)
	}

	if lcd.windowEnable {
		if lcd.wx < 167 && lcd.wy < 144 && lcd.ly >= lcd.wy {
			x := int(lcd.wx) - 7
			xoff := -x
			if x < 0 {
				x = 0
			}
			lcd.mapline(lcd.windowMap, byte(x),
				byte(xoff), byte(-lcd.wy))
		}
	}

	if lcd.spriteEnable {
		lcd.oamline()
	}

	lcd.flushline()
}

func (lcd *display) mapline(map1 bool, x, xoff, yoff byte) {
	y := lcd.ly + yoff
	for ; x < displayW; x++ {
		b := lcd.mapAt(map1, int(x+xoff), int(y))
		lcd.oamLineMask[x] |= b
		lcd.lineBuf[x] = lcd.bgp[b]
	}
}

func (lcd *display) flushline() {
	// Do some simple run-length counting to reduce the number of
	// FillRect calls we need to make.
	scale := uint16(lcd.config.Scale)
	r := &sdl.Rect{0, int16(lcd.ly) * int16(scale), scale, scale}
	cur := lcd.lineBuf[0]
	for x := 1; x < displayW; x++ {
		b := lcd.lineBuf[x]
		if b != cur {
			lcd.FillRect(r, lcd.pal[cur])
			cur = b
			r.X = int16(x) * int16(scale)
			r.W = scale
		} else {
			r.W += scale
		}
	}
	lcd.FillRect(r, lcd.pal[cur])
}

// oamline draws up to 10 sprites on the current scanline
func (lcd *display) oamline() {
	// TODO sprite priorities for overlapping sprites at
	// different x-coordinates (lower x-coordinate wins)

	count := 0
	for idx := 0; idx < 0xA0 && count < 10; idx += 4 {
		y := int(lcd.oam[idx]) - 16
		x := int(lcd.oam[idx+1]) - 8
		tile := int(lcd.oam[idx+2])
		info := lcd.oam[idx+3]

		h := 8
		if lcd.spriteSize {
			h = 16
			tile &= 0xFE
		}

		if int(lcd.ly) < y || int(lcd.ly) >= y+h {
			continue
		}
		count++
		if x == -8 || x >= 168 {
			continue
		}

		lcd.spriteLine(tile, x, y, h, info)
	}
}

func (lcd *display) spriteLine(tile, x, y, h int, info byte) {
	masked := info&0x80 == 0x80
	yflip := info&0x40 == 0x40
	xflip := info&0x20 == 0x20
	palidx := (info >> 4) & 1

	tiley := int(lcd.ly) - y
	if yflip {
		tiley = h - 1 - tiley
	}
	tile = tile*16 + tiley*2

	for i := 0; i < 8; i++ {
		xi := byte(x + i)
		if xi >= displayW {
			if xi > 248 { // i.e., xi < 0
				continue
			}
			return
		}
		if masked && lcd.oamLineMask[xi] != 0 {
			continue
		}
		bit := uint(i)
		if !xflip {
			bit = uint(7 - i)
		}
		px := (lcd.vram[tile] >> bit) & 1
		px |= ((lcd.vram[tile+1] >> bit) & 1) << 1
		if px != 0 {
			lcd.lineBuf[xi] = lcd.obp[palidx][px]
		}
	}
}

func (lcd *display) mapAt(map1 bool, x, y int) byte {
	idx := (y/tileH)*mapW + x/tileW
	if map1 {
		idx += 0x1C00
	} else {
		idx += 0x1800
	}
	tile := lcd.vram[idx]
	if lcd.tileData {
		idx = int(tile) * 16
	} else {
		idx = 0x1000 + int(int8(tile))*16
	}
	idx += (y % tileH) * 2
	bit := uint(tileW - 1 - x%tileW)
	px := (lcd.vram[idx] >> bit) & 1
	px |= ((lcd.vram[idx+1] >> bit) & 1) << 1
	return px
}

func calcMode(t int, ly byte) byte {
	if ly >= displayH {
		return modeVBlank
	}
	t %= scanlineTicks
	if t < oamTicks {
		return modeOAM
	}
	if t < oamTicks+vramTicks {
		return modeVRAM
	}
	return modeHBlank
}
<commit_msg>Correct a mistake in some documentation<commit_after>// Copyright 2011 Kevin Bulusek. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gameboy

import (
	"⚛sdl"
	"time"
)

const (
	modeHBlank = byte(iota)
	modeVBlank
	modeOAM
	modeVRAM
)

const (
	tilesX   = 20
	tilesY   = 18
	tileW    = 8
	tileH    = 8
	mapW     = 32
	mapH     = 32
	displayW = tileW * tilesX
	displayH = tileH * tilesY

	hblankTicks = 204 / 4
	vblankTicks = 4560 / 4
	oamTicks    = 80 / 4
	vramTicks   = 172 / 4

	scanlineTicks = oamTicks + vramTicks + hblankTicks
	refreshTicks  = scanlineTicks*displayH + vblankTicks
)

type display struct {
	*memory
	*sdl.Surface
	pal       [4]uint32
	frameTime int64
	screenW   int
	screenH   int

	clock int

	// LCDC flags
	enable       bool
	windowMap    bool
	windowEnable bool
	tileData     bool
	bgMap        bool
	spriteSize   bool
	spriteEnable bool
	bgEnable     bool

	// STAT flags
	lycInterrupt    bool
	oamInterrupt    bool
	vblankInterrupt bool
	hblankInterrupt bool
	mode            byte

	// LCD registers
	ly       byte
	scy, scx byte
	wy, wx   byte

	bgp [4]byte
	obp [2][4]byte

	// Scanlines are rendered to here first, and then drawn to the
	// display - rather than each layer accessing the display
	// separately (which is slow).
	lineBuf [displayW]byte

	// When rendering a scanline this is zeroed out, then
	// bitwise-ORed with the pixels from the BG and window. This
	// is then used to lookup which pixels can be painted in
	// sprites that are to be obscured by the background layers.
	oamLineMask [displayW]byte
}

func newDisplay(m *memory) *display {
	sdl.WM_SetCaption(m.rom.title(), "")
	flags := uint32(sdl.DOUBLEBUF)
	if m.config.Fullscreen {
		flags |= sdl.FULLSCREEN
	}
	lcd := display{memory: m}
	lcd.screenW = displayW * m.config.Scale
	lcd.screenH = displayH * m.config.Scale
	lcd.Surface = sdl.SetVideoMode(lcd.screenW, lcd.screenH, 0, flags)
	lcd.pal[0] = sdl.MapRGBA(lcd.Format, 0x9B, 0xBC, 0x0F, 0)
	lcd.pal[1] = sdl.MapRGBA(lcd.Format, 0x8B, 0xAC, 0x0F, 0)
	lcd.pal[2] = sdl.MapRGBA(lcd.Format, 0x30, 0x62, 0x30, 0)
	lcd.pal[3] = sdl.MapRGBA(lcd.Format, 0x0F, 0x38, 0x0F, 0)
	sdl.ShowCursor(sdl.DISABLE)
	lcd.FillRect(nil, lcd.pal[0])
	lcd.Flip()
	lcd.frameTime = time.Nanoseconds()
	return &lcd
}

func (lcd *display) toggleFullScreen() {
	sdl.WM_ToggleFullScreen(lcd.Surface)
}

func (lcd *display) step(t int) {
	lcd.clock += t
	if lcd.clock >= refreshTicks {
		lcd.clock -= refreshTicks
	}

	lcd.ly = byte(lcd.clock / scanlineTicks)
	lcd.hram[portLY-0xFF00] = lcd.ly
	mode := calcMode(lcd.clock, lcd.ly)
	if mode == lcd.mode {
		return
	}
	lcd.mode = mode

	stat := lcd.readPort(portSTAT)&^3 | mode
	irq := lcd.readPort(portIF)

	switch mode {
	case modeOAM:
		if lcd.oamInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
		if lyc := lcd.readPort(portLYC); lcd.ly-1 == lyc {
			stat |= 0x04
			if lcd.lycInterrupt {
				lcd.writePort(portIF, irq|0x02)
			}
		} else {
			stat &^= 0x04
		}
	case modeVRAM:
		if lcd.enable {
			lcd.scanline()
		}
	case modeHBlank:
		if lcd.hblankInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
	case modeVBlank:
		if lcd.vblankInterrupt {
			irq |= 0x02
		}
		lcd.writePort(portIF, irq|0x01)
		lcd.Flip()

		// while audio is playing, we let it control the
		// emulation speed
		if !lcd.audio.enable {
			lcd.delay()
		}
	}

	lcd.writePort(portSTAT, stat)
}

func (lcd *display) delay() {
	now := time.Nanoseconds()
	delta := now - lcd.frameTime
	target := 16742706 - delta
	if target > 0 {
		time.Sleep(target)
	}
	lcd.frameTime = time.Nanoseconds()
}

func (lcd *display) scanline() {
	for i := 0; i < displayW; i++ {
		lcd.oamLineMask[i] = 0
		lcd.lineBuf[i] = 0
	}

	if lcd.bgEnable {
		lcd.mapline(lcd.bgMap, byte(0), lcd.scx, lcd.scy)
	}

	if lcd.windowEnable {
		if lcd.wx < 167 && lcd.wy < 144 && lcd.ly >= lcd.wy {
			x := int(lcd.wx) - 7
			xoff := -x
			if x < 0 {
				x = 0
			}
			lcd.mapline(lcd.windowMap, byte(x),
				byte(xoff), byte(-lcd.wy))
		}
	}

	if lcd.spriteEnable {
		lcd.oamline()
	}

	lcd.flushline()
}

func (lcd *display) mapline(map1 bool, x, xoff, yoff byte) {
	y := lcd.ly + yoff
	for ; x < displayW; x++ {
		b := lcd.mapAt(map1, int(x+xoff), int(y))
		lcd.oamLineMask[x] |= b
		lcd.lineBuf[x] = lcd.bgp[b]
	}
}

func (lcd *display) flushline() {
	// Do some simple run-length counting to reduce the number of
	// FillRect calls we need to make.
	scale := uint16(lcd.config.Scale)
	r := &sdl.Rect{0, int16(lcd.ly) * int16(scale), scale, scale}
	cur := lcd.lineBuf[0]
	for x := 1; x < displayW; x++ {
		b := lcd.lineBuf[x]
		if b != cur {
			lcd.FillRect(r, lcd.pal[cur])
			cur = b
			r.X = int16(x) * int16(scale)
			r.W = scale
		} else {
			r.W += scale
		}
	}
	lcd.FillRect(r, lcd.pal[cur])
}

// oamline draws up to 10 sprites on the current scanline
func (lcd *display) oamline() {
	// TODO sprite priorities for overlapping sprites at
	// different x-coordinates (lower x-coordinate wins)

	count := 0
	for idx := 0; idx < 0xA0 && count < 10; idx += 4 {
		y := int(lcd.oam[idx]) - 16
		x := int(lcd.oam[idx+1]) - 8
		tile := int(lcd.oam[idx+2])
		info := lcd.oam[idx+3]

		h := 8
		if lcd.spriteSize {
			h = 16
			tile &= 0xFE
		}

		if int(lcd.ly) < y || int(lcd.ly) >= y+h {
			continue
		}
		count++
		if x == -8 || x >= 168 {
			continue
		}

		lcd.spriteLine(tile, x, y, h, info)
	}
}

func (lcd *display) spriteLine(tile, x, y, h int, info byte) {
	masked := info&0x80 == 0x80
	yflip := info&0x40 == 0x40
	xflip := info&0x20 == 0x20
	palidx := (info >> 4) & 1

	tiley := int(lcd.ly) - y
	if yflip {
		tiley = h - 1 - tiley
	}
	tile = tile*16 + tiley*2

	for i := 0; i < 8; i++ {
		xi := byte(x + i)
		if xi >= displayW {
			if xi > 248 { // i.e., xi < 0
				continue
			}
			return
		}
		if masked && lcd.oamLineMask[xi] != 0 {
			continue
		}
		bit := uint(i)
		if !xflip {
			bit = uint(7 - i)
		}
		px := (lcd.vram[tile] >> bit) & 1
		px |= ((lcd.vram[tile+1] >> bit) & 1) << 1
		if px != 0 {
			lcd.lineBuf[xi] = lcd.obp[palidx][px]
		}
	}
}

func (lcd *display) mapAt(map1 bool, x, y int) byte {
	idx := (y/tileH)*mapW + x/tileW
	if map1 {
		idx += 0x1C00
	} else {
		idx += 0x1800
	}
	tile := lcd.vram[idx]
	if lcd.tileData {
		idx = int(tile) * 16
	} else {
		idx = 0x1000 + int(int8(tile))*16
	}
	idx += (y % tileH) * 2
	bit := uint(tileW - 1 - x%tileW)
	px := (lcd.vram[idx] >> bit) & 1
	px |= ((lcd.vram[idx+1] >> bit) & 1) << 1
	return px
}

func calcMode(t int, ly byte) byte {
	if ly >= displayH {
		return modeVBlank
	}
	t %= scanlineTicks
	if t < oamTicks {
		return modeOAM
	}
	if t < oamTicks+vramTicks {
		return modeVRAM
	}
	return modeHBlank
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nfs

import (
	"github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/golang/glog"
)

type nfsDriver struct {
	name    string
	nodeID  string
	version string

	endpoint string

	//ids *identityServer
	ns    *nodeServer
	cap   []*csi.VolumeCapability_AccessMode
	cscap []*csi.ControllerServiceCapability
}

const (
	driverName = "csi-nfsplugin"
)

var (
	version = "1.0.0-rc2"
)

func NewNFSdriver(nodeID, endpoint string) *nfsDriver {
	glog.Infof("Driver: %v version: %v", driverName, version)

	n := &nfsDriver{
		name:     driverName,
		version:  version,
		nodeID:   nodeID,
		endpoint: endpoint,
	}

	n.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER})
	// NFS plugin does not support ControllerServiceCapability now.
	// If support is added, it should set to appropriate
	// ControllerServiceCapability RPC types.
	n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN})

	return n
}

func NewNodeServer(n *nfsDriver) *nodeServer {
	return &nodeServer{
		Driver: n,
	}
}

func (n *nfsDriver) Run() {
	s := NewNonBlockingGRPCServer()
	s.Start(n.endpoint,
		NewDefaultIdentityServer(n),
		// NFS plugin has not implemented ControllerServer
		// using default controllerserver.
		NewControllerServer(n),
		NewNodeServer(n))
	s.Wait()
}

func (n *nfsDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode {
	var vca []*csi.VolumeCapability_AccessMode
	for _, c := range vc {
		glog.Infof("Enabling volume access mode: %v", c.String())
		vca = append(vca, &csi.VolumeCapability_AccessMode{Mode: c})
	}
	n.cap = vca
	return vca
}

func (n *nfsDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) {
	var csc []*csi.ControllerServiceCapability

	for _, c := range cl {
		glog.Infof("Enabling controller service capability: %v", c.String())
		csc = append(csc, NewControllerServiceCapability(c))
	}

	n.cscap = csc

	return
}
<commit_msg>Add more access modes to volume capabilities<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nfs

import (
	"github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/golang/glog"
)

type nfsDriver struct {
	name    string
	nodeID  string
	version string

	endpoint string

	//ids *identityServer
	ns    *nodeServer
	cap   map[csi.VolumeCapability_AccessMode_Mode]bool
	cscap []*csi.ControllerServiceCapability
}

const (
	driverName = "csi-nfsplugin"
)

var (
	version = "1.0.0-rc2"
)

func NewNFSdriver(nodeID, endpoint string) *nfsDriver {
	glog.Infof("Driver: %v version: %v", driverName, version)

	n := &nfsDriver{
		name:     driverName,
		version:  version,
		nodeID:   nodeID,
		endpoint: endpoint,
		cap:      map[csi.VolumeCapability_AccessMode_Mode]bool{},
	}

	vcam := []csi.VolumeCapability_AccessMode_Mode{
		csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
		csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY,
		csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY,
		csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER,
		csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
	}
	n.AddVolumeCapabilityAccessModes(vcam)

	// NFS plugin does not support ControllerServiceCapability now.
	// If support is added, it should set to appropriate
	// ControllerServiceCapability RPC types.
	n.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN})

	return n
}

func NewNodeServer(n *nfsDriver) *nodeServer {
	return &nodeServer{
		Driver: n,
	}
}

func (n *nfsDriver) Run() {
	s := NewNonBlockingGRPCServer()
	s.Start(n.endpoint,
		NewDefaultIdentityServer(n),
		// NFS plugin has not implemented ControllerServer
		// using default controllerserver.
		NewControllerServer(n),
		NewNodeServer(n))
	s.Wait()
}

func (n *nfsDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode {
	var vca []*csi.VolumeCapability_AccessMode
	for _, c := range vc {
		glog.Infof("Enabling volume access mode: %v", c.String())
		vca = append(vca, &csi.VolumeCapability_AccessMode{Mode: c})
		n.cap[c] = true
	}
	return vca
}

func (n *nfsDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) {
	var csc []*csi.ControllerServiceCapability

	for _, c := range cl {
		glog.Infof("Enabling controller service capability: %v", c.String())
		csc = append(csc, NewControllerServiceCapability(c))
	}

	n.cscap = csc

	return
}
<|endoftext|>


<commit_before>package expect

import (
	"flag"
	"fmt"
	"github.com/wsxiaoys/terminal/color"
	"os"
	"reflect"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var (
	showStdout = flag.Bool("vv", false, "Regular expression selecting which tests to run")
	matchFlag  = flag.String("m", "", "Regular expression selecting which tests to run")
	pattern    *regexp.Regexp
	runner     *Runner
	stdout     = os.Stdout
	silentOut  *os.File
)

func init() {
	flag.Parse()
	pattern = regexp.MustCompile("(?i)" + *matchFlag)
	if *showStdout == true {
		silentOut = stdout
	}
	os.Stdout = silentOut
}

func Expectify(suite interface{}, t *testing.T) {
	var name string
	defer func() {
		if err := recover(); err != nil {
			os.Stdout = stdout
			color.Printf("@R 💣  %-75s\n", name)
			panic(err)
		}
	}()

	tp := reflect.TypeOf(suite)
	sv := reflect.ValueOf(suite)
	count := tp.NumMethod()

	runner = &Runner{
		results: make([]*result, 0, 10),
	}
	announced := false
	for i := 0; i < count; i++ {
		method := tp.Method(i)
		name = method.Name
		if pattern.MatchString(name) == false || method.Type.NumIn() != 1 {
			continue
		}
		os.Stdout = stdout
		res := runner.Start(name)
		method.Func.Call([]reflect.Value{sv})
		if runner.End() == false || testing.Verbose() {
			if announced == false {
				color.Printf("\n@!%s@|\n", sv.Elem().Type().String())
				announced = true
			}
			res.Report()
		}
		os.Stdout = silentOut
	}
	if runner.Passed() == false {
		os.Stdout = stdout
		fmt.Println("")
		os.Stdout = silentOut
		t.Fail()
	}
}

type Runner struct {
	results []*result
	current *result
}

func (r *Runner) Start(name string) *result {
	r.current = &result{
		method:   name,
		start:    time.Now(),
		failures: make([]*Failure, 0, 3),
	}
	r.results = append(r.results, r.current)
	return r.current
}

func (r *Runner) End() bool {
	r.current.end = time.Now()
	passed := r.current.Passed()
	r.current = nil
	return passed
}

func (r *Runner) Passed() bool {
	for _, result := range r.results {
		if result.Passed() == false {
			return false
		}
	}
	return true
}

func (r *Runner) Skip(format string, args ...interface{}) {
	if r.current != nil {
		r.current.Skip(format, args...)
	}
}

func (r *Runner) Errorf(format string, args ...interface{}) {
	file := "???"
	line := 1
	ok := false
	for i := 3; i < 10; i++ {
		_, file, line, ok = runtime.Caller(i)
		if ok == false || strings.HasSuffix(file, "_test.go") {
			break
		}
	}

	if ok {
		if index := strings.LastIndex(file, "/"); index >= 0 {
			file = file[index+1:]
		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
			file = file[index+1:]
		}
	}

	failure := &Failure{
		message:  fmt.Sprintf(format, args...),
		location: fmt.Sprintf("%s:%d", file, line),
	}
	r.current.failures = append(r.current.failures, failure)
}

func (r *Runner) AdditionalInfo(format string, args ...interface{}) {
	if r.current != nil {
		r.current.AdditionalInfo(format, args...)
	}
}

type result struct {
	method      string
	failures    []*Failure
	start       time.Time
	end         time.Time
	skipMessage string
	skip        bool
}

type Failure struct {
	message    string
	location   string
	additional string
}

func (r *result) Skip(format string, args ...interface{}) {
	r.skip = true
	r.skipMessage = fmt.Sprintf(format, args...)
}

func (r *result) Passed() bool {
	return r.skip || len(r.failures) == 0
}

func (r *result) AdditionalInfo(format string, args ...interface{}) {
	l := len(r.failures)
	if l > 0 {
		r.failures[l-1].additional = fmt.Sprintf(format, args...)
	}
}

func (r *result) Report() {
	info := fmt.Sprintf(" %-70s%dms", r.method, r.end.Sub(r.start).Nanoseconds()/1000000)
	if r.skip {
		color.Println(" @y⸚" + info)
		color.Println("   @." + r.skipMessage)
	} else if r.Passed() {
		color.Println(" @g✓" + info)
	} else {
		color.Println(" @r×" + info)
		for _, failure := range r.failures {
			color.Printf("   @.%-40s%s\n", failure.location, failure.message)
			if len(failure.additional) > 0 {
				color.Printf("   @.%s\n", failure.additional)
			}
		}
	}
}
<commit_msg>improve display of additional error messages<commit_after>package expect

import (
	"flag"
	"fmt"
	"github.com/wsxiaoys/terminal/color"
	"os"
	"reflect"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var (
	showStdout = flag.Bool("vv", false, "Regular expression selecting which tests to run")
	matchFlag  = flag.String("m", "", "Regular expression selecting which tests to run")
	pattern    *regexp.Regexp
	runner     *Runner
	stdout     = os.Stdout
	silentOut  *os.File
)

func init() {
	flag.Parse()
	pattern = regexp.MustCompile("(?i)" + *matchFlag)
	if *showStdout == true {
		silentOut = stdout
	}
	os.Stdout = silentOut
}

func Expectify(suite interface{}, t *testing.T) {
	var name string
	defer func() {
		if err := recover(); err != nil {
			os.Stdout = stdout
			color.Printf("@R 💣  %-75s\n", name)
			panic(err)
		}
	}()

	tp := reflect.TypeOf(suite)
	sv := reflect.ValueOf(suite)
	count := tp.NumMethod()

	runner = &Runner{
		results: make([]*result, 0, 10),
	}
	announced := false
	for i := 0; i < count; i++ {
		method := tp.Method(i)
		name = method.Name
		if pattern.MatchString(name) == false || method.Type.NumIn() != 1 {
			continue
		}
		os.Stdout = stdout
		res := runner.Start(name)
		method.Func.Call([]reflect.Value{sv})
		if runner.End() == false || testing.Verbose() {
			if announced == false {
				color.Printf("\n@!%s@|\n", sv.Elem().Type().String())
				announced = true
			}
			res.Report()
		}
		os.Stdout = silentOut
	}
	if runner.Passed() == false {
		os.Stdout = stdout
		fmt.Println("")
		os.Stdout = silentOut
		t.Fail()
	}
}

type Runner struct {
	results []*result
	current *result
}

func (r *Runner) Start(name string) *result {
	r.current = &result{
		method:   name,
		start:    time.Now(),
		failures: make([]*Failure, 0, 3),
	}
	r.results = append(r.results, r.current)
	return r.current
}

func (r *Runner) End() bool {
	r.current.end = time.Now()
	passed := r.current.Passed()
	r.current = nil
	return passed
}

func (r *Runner) Passed() bool {
	for _, result := range r.results {
		if result.Passed() == false {
			return false
		}
	}
	return true
}

func (r *Runner) Skip(format string, args ...interface{}) {
	if r.current != nil {
		r.current.Skip(format, args...)
	}
}

func (r *Runner) Errorf(format string, args ...interface{}) {
	file := "???"
	line := 1
	ok := false
	for i := 3; i < 10; i++ {
		_, file, line, ok = runtime.Caller(i)
		if ok == false || strings.HasSuffix(file, "_test.go") {
			break
		}
	}

	if ok {
		if index := strings.LastIndex(file, "/"); index >= 0 {
			file = file[index+1:]
		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
			file = file[index+1:]
		}
	}

	failure := &Failure{
		message:  fmt.Sprintf(format, args...),
		location: fmt.Sprintf("%s:%d", file, line),
	}
	r.current.failures = append(r.current.failures, failure)
}

func (r *Runner) AdditionalInfo(format string, args ...interface{}) {
	if r.current != nil {
		r.current.AdditionalInfo(format, args...)
	}
}

type result struct {
	method      string
	failures    []*Failure
	start       time.Time
	end         time.Time
	skipMessage string
	skip        bool
}

type Failure struct {
	message    string
	location   string
	additional string
}

func (r *result) Skip(format string, args ...interface{}) {
	r.skip = true
	r.skipMessage = fmt.Sprintf(format, args...)
}

func (r *result) Passed() bool {
	return r.skip || len(r.failures) == 0
}

func (r *result) AdditionalInfo(format string, args ...interface{}) {
	l := len(r.failures)
	if l > 0 {
		r.failures[l-1].additional = fmt.Sprintf(format, args...)
	}
}

func (r *result) Report() {
	info := fmt.Sprintf(" %-70s%dms", r.method, r.end.Sub(r.start).Nanoseconds()/1000000)
	if r.skip {
		color.Println(" @y⸚" + info)
		color.Println("   @." + r.skipMessage)
	} else if r.Passed() {
		color.Println(" @g✓" + info)
	} else {
		color.Println(" @r×" + info)
		for _, failure := range r.failures {
			color.Printf("   @.%-40s%s", failure.location, failure.message)
			if len(failure.additional) > 0 {
				color.Printf(" | @r%s", failure.additional)
			}
			fmt.Println("")
		}
	}
}
<|endoftext|>


<commit_before>package jobs

import (
	"sync"
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Try contains details of an attempt to run
// a job.
type Try struct {
	// Runner is the name of the runner that tried to
	// run this job.
	Runner string
	// When is a timestamp of when the attempt took place.
	When time.Time
	// Err is the error that was returned by JobFunc.
	Err string
}

// JobFunc is the function that gets called for each
// job.
type JobFunc func(job *J) error

// Runner runs jobs.
type Runner struct {
	c        *mgo.Collection
	fn       JobFunc
	stop     chan struct{}
	Interval time.Duration
	err      error
	stoponce sync.Once
	name     string
	kind     string
}

// Start starts the process.
func (r *Runner) Start() error {
	r.stop = make(chan struct{})
	r.stoponce = sync.Once{}
	go func() {
		var job *J
	outside:
		for {
			iter := r.c.Find(bson.M{
				"status": bson.M{"$in": []interface{}{StatusNew, StatusWaiting}},
				"runat":  bson.M{"$lte": time.Now()},
				"kind":   r.kind,
			}).Sort("created").Iter()
			for iter.Next(&job) {

				var err error
				var changeInfo *mgo.ChangeInfo
				change := mgo.Change{
					Update:    bson.M{"$set": bson.M{"status": StatusWorking}},
					ReturnNew: true,
				}
				if changeInfo, err = r.c.Find(bson.M{
					"_id":    job.ID,
					"status": bson.M{"$in": []interface{}{StatusNew, StatusWaiting}},
				}).Apply(change, &job); err != nil {
					if err == mgo.ErrNotFound {
						// skip this one - someone else is dealing with it
						continue
					}
					r.err = err
					break
				}
				if changeInfo.Updated != 1 {
					// skip this one - someone else is dealing with it
					continue
				}

				jobErr := r.fn(job)

				// record this attempt
				try := &Try{
					When:   time.Now(),
					Runner: r.name,
				}
				job.Tries = append(job.Tries, try)

				if jobErr != nil {

					try.Err = jobErr.Error()

					job.RunAt = time.Now().Add(job.RetryInterval)
					job.Retries--
					job.Status = StatusWaiting
					if job.Retries == 0 {
						job.Status = StatusFailed
					}

				} else {
					// success
					job.Status = StatusSuccess
				}

				if err := r.c.UpdateId(job.ID, job); err != nil {
					r.err = err
					break
				}

			}
			if err := iter.Close(); err != nil {
				r.err = err
			}
			if r.err != nil {
				r.Stop()
			}
			select {
			case <-r.stop:
				// stop
				break outside
			case <-time.After(r.Interval):
				// carry on
			}
		}

	}()

	return nil
}

// Stop stops the runner. Callers should then block on StopChan()
// to be notified of when the runner has stopped.
func (r *Runner) Stop() {
	r.stoponce.Do(func() {
		close(r.stop)
	})
}

// StopChan is a channel that gets closed when the runner
// has stopped. Callers should block on this after calling
// Stop to ensure the runner has properly stopped.
//     <-runner.StopChan()
func (r *Runner) StopChan() <-chan struct{} { return r.stop }

// Err is the last error that occurred.
func (r *Runner) Err() error { return r.err }

// Name is the name of the runner.
func (r *Runner) Name() string { return r.name }

// Kind is the name of the types of jobs this runner will process.
func (r *Runner) Kind() string { return r.kind }

// NewRunner makes a new Runner capable of running jobs.
func NewRunner(name string, c *mgo.Collection, kind string, fn JobFunc) *Runner {
	return &Runner{
		c:        c,
		fn:       fn,
		Interval: 500 * time.Millisecond,
		name:     name,
		kind:     kind,
	}
}
<commit_msg>added info on NewRunner<commit_after>package jobs

import (
	"sync"
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// Try contains details of an attempt to run
// a job.
type Try struct {
	// Runner is the name of the runner that tried to
	// run this job.
	Runner string
	// When is a timestamp of when the attempt took place.
	When time.Time
	// Err is the error that was returned by JobFunc.
	Err string
}

// JobFunc is the function that gets called for each
// job.
type JobFunc func(job *J) error

// Runner runs jobs.
type Runner struct {
	c        *mgo.Collection
	fn       JobFunc
	stop     chan struct{}
	Interval time.Duration
	err      error
	stoponce sync.Once
	name     string
	kind     string
}

// Start starts the process.
func (r *Runner) Start() error {
	r.stop = make(chan struct{})
	r.stoponce = sync.Once{}
	go func() {
		var job *J
	outside:
		for {
			iter := r.c.Find(bson.M{
				"status": bson.M{"$in": []interface{}{StatusNew, StatusWaiting}},
				"runat":  bson.M{"$lte": time.Now()},
				"kind":   r.kind,
			}).Sort("created").Iter()
			for iter.Next(&job) {

				var err error
				var changeInfo *mgo.ChangeInfo
				change := mgo.Change{
					Update:    bson.M{"$set": bson.M{"status": StatusWorking}},
					ReturnNew: true,
				}
				if changeInfo, err = r.c.Find(bson.M{
					"_id":    job.ID,
					"status": bson.M{"$in": []interface{}{StatusNew, StatusWaiting}},
				}).Apply(change, &job); err != nil {
					if err == mgo.ErrNotFound {
						// skip this one - someone else is dealing with it
						continue
					}
					r.err = err
					break
				}
				if changeInfo.Updated != 1 {
					// skip this one - someone else is dealing with it
					continue
				}

				jobErr := r.fn(job)

				// record this attempt
				try := &Try{
					When:   time.Now(),
					Runner: r.name,
				}
				job.Tries = append(job.Tries, try)

				if jobErr != nil {

					try.Err = jobErr.Error()

					job.RunAt = time.Now().Add(job.RetryInterval)
					job.Retries--
					job.Status = StatusWaiting
					if job.Retries == 0 {
						job.Status = StatusFailed
					}

				} else {
					// success
					job.Status = StatusSuccess
				}

				if err := r.c.UpdateId(job.ID, job); err != nil {
					r.err = err
					break
				}

			}
			if err := iter.Close(); err != nil {
				r.err = err
			}
			if r.err != nil {
				r.Stop()
			}
			select {
			case <-r.stop:
				// stop
				break outside
			case <-time.After(r.Interval):
				// carry on
			}
		}

	}()

	return nil
}

// Stop stops the runner. Callers should then block on StopChan()
// to be notified of when the runner has stopped.
func (r *Runner) Stop() {
	r.stoponce.Do(func() {
		close(r.stop)
	})
}

// StopChan is a channel that gets closed when the runner
// has stopped. Callers should block on this after calling
// Stop to ensure the runner has properly stopped.
//     <-runner.StopChan()
func (r *Runner) StopChan() <-chan struct{} { return r.stop }

// Err is the last error that occurred.
func (r *Runner) Err() error { return r.err }

// Name is the name of the runner.
func (r *Runner) Name() string { return r.name }

// Kind is the name of the types of jobs this runner will process.
func (r *Runner) Kind() string { return r.kind }

// NewRunner makes a new Runner capable of running jobs. The name should be unique
// across a system. The mgo.Collection is where the job records live in MongoDB.
// The kind string should match J.Kind values for jobs that this runner should execute.
func NewRunner(name string, c *mgo.Collection, kind string, fn JobFunc) *Runner {
	return &Runner{
		c:        c,
		fn:       fn,
		Interval: 500 * time.Millisecond,
		name:     name,
		kind:     kind,
	}
}
<|endoftext|>


<commit_before>package forge

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/sclevine/forge/engine"
	"github.com/sclevine/forge/engine/docker/term"
)

const runScript = `
	set -eo pipefail
	if [[ -d /tmp/local ]]; then
		rsync -a --delete /tmp/local/ /home/vcap/app
    fi
	exec /packs/launcher "$1"
`

var bytesPattern = regexp.MustCompile(`(?i)^(-?\d+)([KMGT])B?$`)

type Runner struct {
	Logs   io.Writer
	TTY    engine.TTY
	Loader Loader
	engine Engine
}

type RunConfig struct {
	Droplet       engine.Stream
	Stack         string
	AppDir        string
	Shell         bool
	Restart       <-chan time.Time
	Color         Colorizer
	AppConfig     *AppConfig
	NetworkConfig *NetworkConfig
}

func NewRunner(engine Engine) *Runner {
	return &Runner{
		Logs: os.Stdout,
		TTY: &term.TTY{
			In:  os.Stdin,
			Out: os.Stdout,
		},
		Loader: noopLoader{},
		engine: engine,
	}
}

func (r *Runner) Run(config *RunConfig) (status int64, err error) {
	if err := r.pull(config.Stack); err != nil {
		return 0, err
	}

	var binds []string
	if config.AppDir != "" {
		binds = []string{config.AppDir + ":/tmp/local"}
	}
	containerConfig, err := r.buildConfig(config.AppConfig, config.NetworkConfig, binds, config.Stack)
	if err != nil {
		return 0, err
	}
	contr, err := r.engine.NewContainer(containerConfig)
	if err != nil {
		return 0, err
	}
	defer contr.Close()

	if err := contr.StreamTarTo(config.Droplet, "/home/vcap"); err != nil {
		return 0, err
	}
	color := config.Color("[%s] ", config.AppConfig.Name)
	if !config.Shell {
		return contr.Start(color, r.Logs, config.Restart)
	}
	if err := contr.Background(); err != nil {
		return 0, err
	}
	return 0, contr.Shell(r.TTY, "/packs/shell")
}

func (r *Runner) pull(stack string) error {
	return r.Loader.Loading("Image", r.engine.NewImage().Pull(stack))
}

func (r *Runner) buildConfig(app *AppConfig, net *NetworkConfig, binds []string, stack string) (*engine.ContainerConfig, error) {
	var disk, mem int64
	var err error
	env := map[string]string{}

	if app.Name != "" {
		env["PACK_APP_NAME"] = app.Name
	}

	if app.DiskQuota != "" {
		disk, err = toMegabytes(app.DiskQuota)
		if err != nil {
			return nil, err
		}
		env["PACK_APP_DISK"] = fmt.Sprintf("%d", disk)
	}
	if app.Memory != "" {
		mem, err = toMegabytes(app.Memory)
		if err != nil {
			return nil, err
		}
		env["PACK_APP_MEM"] = fmt.Sprintf("%d", mem)
	}

	if app.Services != nil {
		vcapServices, err := json.Marshal(app.Services)
		if err != nil {
			return nil, err
		}
		env["VCAP_SERVICES"] = string(vcapServices)
	}

	return &engine.ContainerConfig{
		Name:       app.Name,
		Hostname:   app.Name,
		Env:        mapToEnv(mergeMaps(env, app.RunningEnv, app.Env)),
		Image:      stack,
		WorkingDir: "/home/vcap/app",
		Entrypoint: []string{"/bin/bash", "-c", runScript, app.Command},

		Binds:        binds,
		NetContainer: net.ContainerID,
		HostIP:       net.HostIP,
		HostPort:     net.HostPort,
		Memory:       mem * 1024 * 1024,
		DiskQuota:    disk * 1024 * 1024,
	}, nil
}

func toMegabytes(s string) (int64, error) {
	parts := bytesPattern.FindStringSubmatch(strings.TrimSpace(s))
	if len(parts) < 3 {
		return 0, fmt.Errorf("invalid byte unit format: %s", s)
	}

	value, err := strconv.ParseInt(parts[1], 10, 0)
	if err != nil {
		return 0, fmt.Errorf("invalid byte number format: %s", s)
	}

	const (
		kilobyte = 1024
		megabyte = 1024 * kilobyte
		gigabyte = 1024 * megabyte
		terabyte = 1024 * gigabyte
	)

	var bytes int64
	switch strings.ToUpper(parts[2]) {
	case "T":
		bytes = value * terabyte
	case "G":
		bytes = value * gigabyte
	case "M":
		bytes = value * megabyte
	case "K":
		bytes = value * kilobyte
	}
	return bytes / megabyte, nil
}

func mergeMaps(maps ...map[string]string) map[string]string {
	merged := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			merged[k] = v
		}
	}
	return merged
}

func mapToEnv(env map[string]string) []string {
	var out []string
	for k, v := range env {
		out = append(out, fmt.Sprintf("%s=%s", k, v))
	}
	return out
}
<commit_msg>Added new configuration options for runner<commit_after>package forge

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/sclevine/forge/engine"
	"github.com/sclevine/forge/engine/docker/term"
)

const runScript = `
	set -eo pipefail
	if [[ -d /tmp/local ]]; then
		rsync -a --delete /tmp/local/ /home/vcap/app
    fi
	exec /packs/launcher "$1"
`

var bytesPattern = regexp.MustCompile(`(?i)^(-?\d+)([KMGT])B?$`)

type Runner struct {
	Logs   io.Writer
	TTY    engine.TTY
	Loader Loader
	engine Engine
}

type RunConfig struct {
	Droplet       engine.Stream
	Stack         string
	AppDir        string
	Shell         bool
	Restart       <-chan time.Time
	Color         Colorizer
	AppConfig     *AppConfig
	NetworkConfig *NetworkConfig
	SkipStackPull bool
	HomeDir       string
}

func NewRunner(engine Engine) *Runner {
	return &Runner{
		Logs: os.Stdout,
		TTY: &term.TTY{
			In:  os.Stdin,
			Out: os.Stdout,
		},
		Loader: noopLoader{},
		engine: engine,
	}
}

func (r *Runner) Run(config *RunConfig) (status int64, err error) {
	if config.SkipStackPull == false {
		if err := r.pull(config.Stack); err != nil {
			return 0, err
		}
	}

	var binds []string
	if config.AppDir != "" {
		binds = []string{config.AppDir + ":/tmp/local"}
	}
	containerConfig, err := r.buildConfig(config.AppConfig, config.NetworkConfig, binds, config.Stack)
	if err != nil {
		return 0, err
	}
	contr, err := r.engine.NewContainer(containerConfig)
	if err != nil {
		return 0, err
	}
	defer contr.Close()

	homeDir := config.HomeDir
	if homeDir == "" {
		homeDir = "/home/vcap"
	}
	if err := contr.StreamTarTo(config.Droplet, homeDir); err != nil {
		return 0, err
	}
	color := config.Color("[%s] ", config.AppConfig.Name)
	if !config.Shell {
		return contr.Start(color, r.Logs, config.Restart)
	}
	if err := contr.Background(); err != nil {
		return 0, err
	}
	return 0, contr.Shell(r.TTY, "/packs/shell")
}

func (r *Runner) pull(stack string) error {
	return r.Loader.Loading("Image", r.engine.NewImage().Pull(stack))
}

func (r *Runner) buildConfig(app *AppConfig, net *NetworkConfig, binds []string, stack string) (*engine.ContainerConfig, error) {
	var disk, mem int64
	var err error
	env := map[string]string{}

	if app.Name != "" {
		env["PACK_APP_NAME"] = app.Name
	}

	if app.DiskQuota != "" {
		disk, err = toMegabytes(app.DiskQuota)
		if err != nil {
			return nil, err
		}
		env["PACK_APP_DISK"] = fmt.Sprintf("%d", disk)
	}
	if app.Memory != "" {
		mem, err = toMegabytes(app.Memory)
		if err != nil {
			return nil, err
		}
		env["PACK_APP_MEM"] = fmt.Sprintf("%d", mem)
	}

	if app.Services != nil {
		vcapServices, err := json.Marshal(app.Services)
		if err != nil {
			return nil, err
		}
		env["VCAP_SERVICES"] = string(vcapServices)
	}

	return &engine.ContainerConfig{
		Name:       app.Name,
		Hostname:   app.Name,
		Env:        mapToEnv(mergeMaps(env, app.RunningEnv, app.Env)),
		Image:      stack,
		WorkingDir: "/home/vcap/app",
		Entrypoint: []string{"/bin/bash", "-c", runScript, app.Command},

		Binds:        binds,
		NetContainer: net.ContainerID,
		HostIP:       net.HostIP,
		HostPort:     net.HostPort,
		Memory:       mem * 1024 * 1024,
		DiskQuota:    disk * 1024 * 1024,
	}, nil
}

func toMegabytes(s string) (int64, error) {
	parts := bytesPattern.FindStringSubmatch(strings.TrimSpace(s))
	if len(parts) < 3 {
		return 0, fmt.Errorf("invalid byte unit format: %s", s)
	}

	value, err := strconv.ParseInt(parts[1], 10, 0)
	if err != nil {
		return 0, fmt.Errorf("invalid byte number format: %s", s)
	}

	const (
		kilobyte = 1024
		megabyte = 1024 * kilobyte
		gigabyte = 1024 * megabyte
		terabyte = 1024 * gigabyte
	)

	var bytes int64
	switch strings.ToUpper(parts[2]) {
	case "T":
		bytes = value * terabyte
	case "G":
		bytes = value * gigabyte
	case "M":
		bytes = value * megabyte
	case "K":
		bytes = value * kilobyte
	}
	return bytes / megabyte, nil
}

func mergeMaps(maps ...map[string]string) map[string]string {
	merged := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			merged[k] = v
		}
	}
	return merged
}

func mapToEnv(env map[string]string) []string {
	var out []string
	for k, v := range env {
		out = append(out, fmt.Sprintf("%s=%s", k, v))
	}
	return out
}
<|endoftext|>


<commit_before>package executor

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"github.com/hashicorp/go-multierror"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/driver/environment"
	"github.com/hashicorp/nomad/client/driver/spawn"
	cstructs "github.com/hashicorp/nomad/client/driver/structs"
	"github.com/hashicorp/nomad/helper/args"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// A mapping of directories on the host OS to attempt to embed inside each
	// task's chroot.
	chrootEnv = map[string]string{
		"/bin":     "/bin",
		"/etc":     "/etc",
		"/lib":     "/lib",
		"/lib32":   "/lib32",
		"/lib64":   "/lib64",
		"/usr/bin": "/usr/bin",
		"/usr/lib": "/usr/lib",
	}
)

func NewExecutor() Executor {
	return NewLinuxExecutor()
}

func NewLinuxExecutor() Executor {
	return &LinuxExecutor{}
}

// Linux executor is designed to run on linux kernel 2.8+.
type LinuxExecutor struct {
	cmd  exec.Cmd
	user *user.User

	// Isolation configurations.
	groups   *cgroupConfig.Cgroup
	taskName string
	taskDir  string
	allocDir string

	// Spawn process.
	spawn *spawn.Spawner
}

func (e *LinuxExecutor) Command() *exec.Cmd {
	return &e.cmd
}

func (e *LinuxExecutor) Limit(resources *structs.Resources) error {
	if resources == nil {
		return errNoResources
	}

	return e.configureCgroups(resources)
}

// execLinuxID contains the necessary information to reattach to an executed
// process and cleanup the created cgroups.
type ExecLinuxID struct {
	Groups  *cgroupConfig.Cgroup
	Spawn   *spawn.Spawner
	TaskDir string
}

func (e *LinuxExecutor) Open(id string) error {
	// De-serialize the ID.
	dec := json.NewDecoder(strings.NewReader(id))
	var execID ExecLinuxID
	if err := dec.Decode(&execID); err != nil {
		return fmt.Errorf("Failed to parse id: %v", err)
	}

	// Setup the executor.
	e.groups = execID.Groups
	e.spawn = execID.Spawn
	e.taskDir = execID.TaskDir
	return e.spawn.Valid()
}

func (e *LinuxExecutor) ID() (string, error) {
	if e.groups == nil || e.spawn == nil || e.taskDir == "" {
		return "", fmt.Errorf("LinuxExecutor not properly initialized.")
	}

	// Build the ID.
	id := ExecLinuxID{
		Groups:  e.groups,
		Spawn:   e.spawn,
		TaskDir: e.taskDir,
	}

	var buffer bytes.Buffer
	enc := json.NewEncoder(&buffer)
	if err := enc.Encode(id); err != nil {
		return "", fmt.Errorf("Failed to serialize id: %v", err)
	}

	return buffer.String(), nil
}

// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *LinuxExecutor) runAs(userid string) error {
	u, err := user.Lookup(userid)
	if err != nil {
		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
	}

	// Convert the uid and gid
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
	}
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
	}

	// Set the command to run as that user and group.
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	if e.cmd.SysProcAttr.Credential == nil {
		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
	}
	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)

	return nil
}

func (e *LinuxExecutor) Start() error {
	// Run as "nobody" user so we don't leak root privilege to the spawned
	// process.
	if err := e.runAs("nobody"); err != nil {
		return err
	}

	// Parse the commands arguments and replace instances of Nomad environment
	// variables.
	envVars, err := environment.ParseFromList(e.cmd.Env)
	if err != nil {
		return err
	}

	e.cmd.Path = args.ReplaceEnv(e.cmd.Path, envVars.Map())
	e.cmd.Args = args.ParseAndReplace(e.cmd.Args, envVars.Map())

	spawnState := filepath.Join(e.allocDir, fmt.Sprintf("%s_%s", e.taskName, "exit_status"))
	e.spawn = spawn.NewSpawner(spawnState)
	e.spawn.SetCommand(&e.cmd)
	e.spawn.SetChroot(e.taskDir)
	e.spawn.SetLogs(&spawn.Logs{
		Stdout: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)),
		Stderr: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)),
		Stdin:  os.DevNull,
	})

	enterCgroup := func(pid int) error {
		// Join the spawn-daemon to the cgroup.
		manager := e.getCgroupManager(e.groups)

		// Apply will place the spawn dameon into the created cgroups.
		if err := manager.Apply(pid); err != nil {
			return fmt.Errorf("Failed to join spawn-daemon to the cgroup (%+v): %v", e.groups, err)
		}

		return nil
	}

	return e.spawn.Spawn(enterCgroup)
}

// Wait waits til the user process exits and returns an error on non-zero exit
// codes. Wait also cleans up the task directory and created cgroups.
func (e *LinuxExecutor) Wait() *cstructs.WaitResult {
	errs := new(multierror.Error)
	res := e.spawn.Wait()
	if res.Err != nil {
		errs = multierror.Append(errs, res.Err)
	}

	if err := e.destroyCgroup(); err != nil {
		errs = multierror.Append(errs, err)
	}

	if err := e.cleanTaskDir(); err != nil {
		errs = multierror.Append(errs, err)
	}

	res.Err = errs.ErrorOrNil()
	return res
}

// Shutdown sends the user process an interrupt signal indicating that it is
// about to be forcefully shutdown in sometime
func (e *LinuxExecutor) Shutdown() error {
	proc, err := os.FindProcess(e.spawn.UserPid)
	if err != nil {
		return fmt.Errorf("Failed to find user processes %v: %v", e.spawn.UserPid, err)
	}

	return proc.Signal(os.Interrupt)
}

// ForceStop immediately exits the user process and cleans up both the task
// directory and the cgroups.
func (e *LinuxExecutor) ForceStop() error {
	errs := new(multierror.Error)
	if err := e.destroyCgroup(); err != nil {
		errs = multierror.Append(errs, err)
	}

	if err := e.cleanTaskDir(); err != nil {
		errs = multierror.Append(errs, err)
	}

	return errs.ErrorOrNil()
}

// Task Directory related functions.

// ConfigureTaskDir creates the necessary directory structure for a proper
// chroot. cleanTaskDir should be called after.
func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error {
	e.taskName = taskName
	e.allocDir = alloc.AllocDir

	taskDir, ok := alloc.TaskDirs[taskName]
	if !ok {
		fmt.Errorf("Couldn't find task directory for task %v", taskName)
	}
	e.taskDir = taskDir

	if err := alloc.MountSharedDir(taskName); err != nil {
		return err
	}

	if err := alloc.Embed(taskName, chrootEnv); err != nil {
		return err
	}

	// Mount dev
	dev := filepath.Join(taskDir, "dev")
	if !e.pathExists(dev) {
		if err := os.Mkdir(dev, 0777); err != nil {
			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
		}

		if err := syscall.Mount("", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
		}
	}

	// Mount proc
	proc := filepath.Join(taskDir, "proc")
	if !e.pathExists(proc) {
		if err := os.Mkdir(proc, 0777); err != nil {
			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
		}

		if err := syscall.Mount("", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
		}
	}

	// Set the tasks AllocDir environment variable.
	env, err := environment.ParseFromList(e.cmd.Env)
	if err != nil {
		return err
	}
	env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName))
	env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal))
	e.cmd.Env = env.List()

	return nil
}

// pathExists is a helper function to check if the path exists.
func (e *LinuxExecutor) pathExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *LinuxExecutor) cleanTaskDir() error {
	// Unmount dev.
	errs := new(multierror.Error)
	dev := filepath.Join(e.taskDir, "dev")
	if e.pathExists(dev) {
		if err := syscall.Unmount(dev, 0); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
		}

		if err := os.RemoveAll(dev); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
		}
	}

	// Unmount proc.
	proc := filepath.Join(e.taskDir, "proc")
	if e.pathExists(proc) {
		if err := syscall.Unmount(proc, 0); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
		}

		if err := os.RemoveAll(proc); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
		}
	}

	return errs.ErrorOrNil()
}

// Cgroup related functions.

// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
	e.groups = &cgroupConfig.Cgroup{}
	e.groups.Resources = &cgroupConfig.Resources{}
	e.groups.Name = structs.GenerateUUID()

	// TODO: verify this is needed for things like network access
	e.groups.Resources.AllowAllDevices = true

	if resources.MemoryMB > 0 {
		// Total amount of memory allowed to consume
		e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
		// Disable swap to avoid issues on the machine
		e.groups.Resources.MemorySwap = int64(-1)
	}

	if resources.CPU < 2 {
		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
	}

	// Set the relative CPU shares for this cgroup.
	e.groups.Resources.CpuShares = int64(resources.CPU)

	if resources.IOPS != 0 {
		// Validate it is in an acceptable range.
		if resources.IOPS < 10 || resources.IOPS > 1000 {
			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
		}

		e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
	}

	return nil
}

// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host.
func (e *LinuxExecutor) destroyCgroup() error {
	if e.groups == nil {
		return errors.New("Can't destroy: cgroup configuration empty")
	}

	manager := e.getCgroupManager(e.groups)
	pids, err := manager.GetPids()
	if err != nil {
		return fmt.Errorf("Failed to get pids in the cgroup %v: %v", e.groups.Name, err)
	}

	errs := new(multierror.Error)
	for _, pid := range pids {
		process, err := os.FindProcess(pid)
		if err != nil {
			multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err))
			continue
		}

		if err := process.Kill(); err != nil {
			multierror.Append(errs, fmt.Errorf("Failed to kill Pid %v: %v", pid, err))
			continue
		}
	}

	// Remove the cgroup.
	if err := manager.Destroy(); err != nil {
		multierror.Append(errs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
	}

	if len(errs.Errors) != 0 {
		return fmt.Errorf("Failed to destroy cgroup: %v", errs)
	}

	return nil
}

// getCgroupManager returns the correct libcontainer cgroup manager.
func (e *LinuxExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
	var manager cgroups.Manager
	manager = &cgroupFs.Manager{Cgroups: groups}
	if systemd.UseSystemd() {
		manager = &systemd.Manager{Cgroups: groups}
	}
	return manager
}
<commit_msg>linux/exec: add /usr/share to ChrootEnv<commit_after>package executor

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"github.com/hashicorp/go-multierror"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/driver/environment"
	"github.com/hashicorp/nomad/client/driver/spawn"
	cstructs "github.com/hashicorp/nomad/client/driver/structs"
	"github.com/hashicorp/nomad/helper/args"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// A mapping of directories on the host OS to attempt to embed inside each
	// task's chroot.
	chrootEnv = map[string]string{
		"/bin":       "/bin",
		"/etc":       "/etc",
		"/lib":       "/lib",
		"/lib32":     "/lib32",
		"/lib64":     "/lib64",
		"/usr/bin":   "/usr/bin",
		"/usr/lib":   "/usr/lib",
		"/usr/share": "/usr/share",
	}
)

func NewExecutor() Executor {
	return NewLinuxExecutor()
}

func NewLinuxExecutor() Executor {
	return &LinuxExecutor{}
}

// Linux executor is designed to run on linux kernel 2.8+.
type LinuxExecutor struct {
	cmd  exec.Cmd
	user *user.User

	// Isolation configurations.
	groups   *cgroupConfig.Cgroup
	taskName string
	taskDir  string
	allocDir string

	// Spawn process.
	spawn *spawn.Spawner
}

func (e *LinuxExecutor) Command() *exec.Cmd {
	return &e.cmd
}

func (e *LinuxExecutor) Limit(resources *structs.Resources) error {
	if resources == nil {
		return errNoResources
	}

	return e.configureCgroups(resources)
}

// execLinuxID contains the necessary information to reattach to an executed
// process and cleanup the created cgroups.
type ExecLinuxID struct {
	Groups  *cgroupConfig.Cgroup
	Spawn   *spawn.Spawner
	TaskDir string
}

func (e *LinuxExecutor) Open(id string) error {
	// De-serialize the ID.
	dec := json.NewDecoder(strings.NewReader(id))
	var execID ExecLinuxID
	if err := dec.Decode(&execID); err != nil {
		return fmt.Errorf("Failed to parse id: %v", err)
	}

	// Setup the executor.
	e.groups = execID.Groups
	e.spawn = execID.Spawn
	e.taskDir = execID.TaskDir
	return e.spawn.Valid()
}

func (e *LinuxExecutor) ID() (string, error) {
	if e.groups == nil || e.spawn == nil || e.taskDir == "" {
		return "", fmt.Errorf("LinuxExecutor not properly initialized.")
	}

	// Build the ID.
	id := ExecLinuxID{
		Groups:  e.groups,
		Spawn:   e.spawn,
		TaskDir: e.taskDir,
	}

	var buffer bytes.Buffer
	enc := json.NewEncoder(&buffer)
	if err := enc.Encode(id); err != nil {
		return "", fmt.Errorf("Failed to serialize id: %v", err)
	}

	return buffer.String(), nil
}

// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *LinuxExecutor) runAs(userid string) error {
	u, err := user.Lookup(userid)
	if err != nil {
		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
	}

	// Convert the uid and gid
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
	}
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
	}

	// Set the command to run as that user and group.
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	if e.cmd.SysProcAttr.Credential == nil {
		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
	}
	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)

	return nil
}

func (e *LinuxExecutor) Start() error {
	// Run as "nobody" user so we don't leak root privilege to the spawned
	// process.
	if err := e.runAs("nobody"); err != nil {
		return err
	}

	// Parse the commands arguments and replace instances of Nomad environment
	// variables.
	envVars, err := environment.ParseFromList(e.cmd.Env)
	if err != nil {
		return err
	}

	e.cmd.Path = args.ReplaceEnv(e.cmd.Path, envVars.Map())
	e.cmd.Args = args.ParseAndReplace(e.cmd.Args, envVars.Map())

	spawnState := filepath.Join(e.allocDir, fmt.Sprintf("%s_%s", e.taskName, "exit_status"))
	e.spawn = spawn.NewSpawner(spawnState)
	e.spawn.SetCommand(&e.cmd)
	e.spawn.SetChroot(e.taskDir)
	e.spawn.SetLogs(&spawn.Logs{
		Stdout: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)),
		Stderr: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)),
		Stdin:  os.DevNull,
	})

	enterCgroup := func(pid int) error {
		// Join the spawn-daemon to the cgroup.
		manager := e.getCgroupManager(e.groups)

		// Apply will place the spawn dameon into the created cgroups.
		if err := manager.Apply(pid); err != nil {
			return fmt.Errorf("Failed to join spawn-daemon to the cgroup (%+v): %v", e.groups, err)
		}

		return nil
	}

	return e.spawn.Spawn(enterCgroup)
}

// Wait waits til the user process exits and returns an error on non-zero exit
// codes. Wait also cleans up the task directory and created cgroups.
func (e *LinuxExecutor) Wait() *cstructs.WaitResult {
	errs := new(multierror.Error)
	res := e.spawn.Wait()
	if res.Err != nil {
		errs = multierror.Append(errs, res.Err)
	}

	if err := e.destroyCgroup(); err != nil {
		errs = multierror.Append(errs, err)
	}

	if err := e.cleanTaskDir(); err != nil {
		errs = multierror.Append(errs, err)
	}

	res.Err = errs.ErrorOrNil()
	return res
}

// Shutdown sends the user process an interrupt signal indicating that it is
// about to be forcefully shutdown in sometime
func (e *LinuxExecutor) Shutdown() error {
	proc, err := os.FindProcess(e.spawn.UserPid)
	if err != nil {
		return fmt.Errorf("Failed to find user processes %v: %v", e.spawn.UserPid, err)
	}

	return proc.Signal(os.Interrupt)
}

// ForceStop immediately exits the user process and cleans up both the task
// directory and the cgroups.
func (e *LinuxExecutor) ForceStop() error {
	errs := new(multierror.Error)
	if err := e.destroyCgroup(); err != nil {
		errs = multierror.Append(errs, err)
	}

	if err := e.cleanTaskDir(); err != nil {
		errs = multierror.Append(errs, err)
	}

	return errs.ErrorOrNil()
}

// Task Directory related functions.

// ConfigureTaskDir creates the necessary directory structure for a proper
// chroot. cleanTaskDir should be called after.
func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error {
	e.taskName = taskName
	e.allocDir = alloc.AllocDir

	taskDir, ok := alloc.TaskDirs[taskName]
	if !ok {
		fmt.Errorf("Couldn't find task directory for task %v", taskName)
	}
	e.taskDir = taskDir

	if err := alloc.MountSharedDir(taskName); err != nil {
		return err
	}

	if err := alloc.Embed(taskName, chrootEnv); err != nil {
		return err
	}

	// Mount dev
	dev := filepath.Join(taskDir, "dev")
	if !e.pathExists(dev) {
		if err := os.Mkdir(dev, 0777); err != nil {
			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
		}

		if err := syscall.Mount("", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
		}
	}

	// Mount proc
	proc := filepath.Join(taskDir, "proc")
	if !e.pathExists(proc) {
		if err := os.Mkdir(proc, 0777); err != nil {
			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
		}

		if err := syscall.Mount("", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
		}
	}

	// Set the tasks AllocDir environment variable.
	env, err := environment.ParseFromList(e.cmd.Env)
	if err != nil {
		return err
	}
	env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName))
	env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal))
	e.cmd.Env = env.List()

	return nil
}

// pathExists is a helper function to check if the path exists.
func (e *LinuxExecutor) pathExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *LinuxExecutor) cleanTaskDir() error {
	// Unmount dev.
	errs := new(multierror.Error)
	dev := filepath.Join(e.taskDir, "dev")
	if e.pathExists(dev) {
		if err := syscall.Unmount(dev, 0); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
		}

		if err := os.RemoveAll(dev); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
		}
	}

	// Unmount proc.
	proc := filepath.Join(e.taskDir, "proc")
	if e.pathExists(proc) {
		if err := syscall.Unmount(proc, 0); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
		}

		if err := os.RemoveAll(proc); err != nil {
			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
		}
	}

	return errs.ErrorOrNil()
}

// Cgroup related functions.

// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
	e.groups = &cgroupConfig.Cgroup{}
	e.groups.Resources = &cgroupConfig.Resources{}
	e.groups.Name = structs.GenerateUUID()

	// TODO: verify this is needed for things like network access
	e.groups.Resources.AllowAllDevices = true

	if resources.MemoryMB > 0 {
		// Total amount of memory allowed to consume
		e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
		// Disable swap to avoid issues on the machine
		e.groups.Resources.MemorySwap = int64(-1)
	}

	if resources.CPU < 2 {
		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
	}

	// Set the relative CPU shares for this cgroup.
	e.groups.Resources.CpuShares = int64(resources.CPU)

	if resources.IOPS != 0 {
		// Validate it is in an acceptable range.
		if resources.IOPS < 10 || resources.IOPS > 1000 {
			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
		}

		e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
	}

	return nil
}

// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host.
func (e *LinuxExecutor) destroyCgroup() error {
	if e.groups == nil {
		return errors.New("Can't destroy: cgroup configuration empty")
	}

	manager := e.getCgroupManager(e.groups)
	pids, err := manager.GetPids()
	if err != nil {
		return fmt.Errorf("Failed to get pids in the cgroup %v: %v", e.groups.Name, err)
	}

	errs := new(multierror.Error)
	for _, pid := range pids {
		process, err := os.FindProcess(pid)
		if err != nil {
			multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err))
			continue
		}

		if err := process.Kill(); err != nil {
			multierror.Append(errs, fmt.Errorf("Failed to kill Pid %v: %v", pid, err))
			continue
		}
	}

	// Remove the cgroup.
	if err := manager.Destroy(); err != nil {
		multierror.Append(errs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
	}

	if len(errs.Errors) != 0 {
		return fmt.Errorf("Failed to destroy cgroup: %v", errs)
	}

	return nil
}

// getCgroupManager returns the correct libcontainer cgroup manager.
func (e *LinuxExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
	var manager cgroups.Manager
	manager = &cgroupFs.Manager{Cgroups: groups}
	if systemd.UseSystemd() {
		manager = &systemd.Manager{Cgroups: groups}
	}
	return manager
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"github.com/voxelbrain/goptions"
)

var (
	options = struct {
		Server string        `goptions:"-s, --server, description='Pixelpixel server to push to'"`
		Help   goptions.Help `goptions:"-h, --help, description='Show this help'"`
		goptions.Verbs
		Upload struct{} `goptions:"upload"`
		Logs   struct{} `goptions:"logs"`
		Format struct{} `goptions:"format"`
	}{
		Server: "localhost:8080",
	}
)

const (
	keyFile = ".key"
)

func main() {
	fs := goptions.NewFlagSet("picli", &options)
	err := fs.Parse(os.Args[1:])
	options.Server = validateServer(options.Server)
	key := prepareKey()

	switch options.Verbs {
	case "upload":
		format()
		upload(key)
	case "logs":
		logs(key)
	case "format":
		format()
	default:
		if err != nil {
			fmt.Printf("Error: %s\n", err)
		}
		fs.PrintHelp(os.Stderr)
		return
	}
}

func makeApiCall(method, subpath string, body io.Reader) (int, string, error) {
	url := path.Join(options.Server, "pixels") + "/"
	url = slashify(url + subpath)
	req, _ := http.NewRequest(method, url, body)
	resp, err := http.DefaultClient.Do(req)
	buf := &bytes.Buffer{}

	code := -1
	if resp != nil {
		defer resp.Body.Close()
		code = resp.StatusCode
		io.Copy(buf, resp.Body)
	}

	// TODO: Sadly, this seems to be necassary. Can't seem to get it
	// working without it. I always get `malformed HTTP response ""`
	http.DefaultTransport.(*http.Transport).CloseIdleConnections()

	return code, buf.String(), err
}

func prepareKey() string {
	f, err := os.Open(keyFile)
	if err != nil {
		return ""
	}
	defer f.Close()
	key, err := ioutil.ReadAll(f)
	if err != nil {
		return ""
	}
	skey := strings.TrimSpace(string(key))
	code, _, err := makeApiCall("GET", "/"+skey, nil)
	if err == nil && code == http.StatusOK {
		return skey
	}
	return ""
}

func slashify(url string) string {
	url = path.Clean(url)
	if !strings.HasSuffix(url, "/") {
		url = url + "/"
	}
	return "http://" + strings.TrimPrefix(url, "/")
}

const (
	DEFAULT_REMOTE_SERVER = "pixelpixel.haxigon.com"
)

func validateServer(server string) string {
	t := time.AfterFunc(3*time.Second, func() {
		log.Fatalf("Attempt to connect to server timed out")
	})
	defer func() {
		t.Stop()
	}()

	resp, err := http.Get("http://" + path.Join(server, "handshake"))
	if err == nil && readAll(resp.Body) == "PIXELPIXEL OK" {
		return server
	}

	resp, err = http.Get("http://" + path.Join(DEFAULT_REMOTE_SERVER, "handshake"))
	if err == nil && readAll(resp.Body) == "PIXELPIXEL OK" {
		return DEFAULT_REMOTE_SERVER
	}
	log.Fatalf("Could not connect to server")
	return ""
}

func readAll(r io.Reader) string {
	data, _ := ioutil.ReadAll(r)
	return string(data)
}
<commit_msg>Dont require valid server for --help flag<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"github.com/voxelbrain/goptions"
)

var (
	options = struct {
		Server string        `goptions:"-s, --server, description='Pixelpixel server to push to'"`
		Help   goptions.Help `goptions:"-h, --help, description='Show this help'"`
		goptions.Verbs
		Upload struct{} `goptions:"upload"`
		Logs   struct{} `goptions:"logs"`
		Format struct{} `goptions:"format"`
	}{
		Server: "localhost:8080",
	}
)

const (
	keyFile = ".key"
)

func main() {
	fs := goptions.NewFlagSet("picli", &options)
	err := fs.Parse(os.Args[1:])
	if err == goptions.ErrHelpRequest {
		fs.PrintHelp(os.Stderr)
		return
	}

	options.Server = validateServer(options.Server)
	key := prepareKey()

	switch options.Verbs {
	case "upload":
		format()
		upload(key)
	case "logs":
		logs(key)
	case "format":
		format()
	default:
		if err != nil {
			fmt.Printf("Error: %s\n", err)
		}
		fs.PrintHelp(os.Stderr)
		return
	}
}

func makeApiCall(method, subpath string, body io.Reader) (int, string, error) {
	url := path.Join(options.Server, "pixels") + "/"
	url = slashify(url + subpath)
	req, _ := http.NewRequest(method, url, body)
	resp, err := http.DefaultClient.Do(req)
	buf := &bytes.Buffer{}

	code := -1
	if resp != nil {
		defer resp.Body.Close()
		code = resp.StatusCode
		io.Copy(buf, resp.Body)
	}

	// TODO: Sadly, this seems to be necassary. Can't seem to get it
	// working without it. I always get `malformed HTTP response ""`
	http.DefaultTransport.(*http.Transport).CloseIdleConnections()

	return code, buf.String(), err
}

func prepareKey() string {
	f, err := os.Open(keyFile)
	if err != nil {
		return ""
	}
	defer f.Close()
	key, err := ioutil.ReadAll(f)
	if err != nil {
		return ""
	}
	skey := strings.TrimSpace(string(key))
	code, _, err := makeApiCall("GET", "/"+skey, nil)
	if err == nil && code == http.StatusOK {
		return skey
	}
	return ""
}

func slashify(url string) string {
	url = path.Clean(url)
	if !strings.HasSuffix(url, "/") {
		url = url + "/"
	}
	return "http://" + strings.TrimPrefix(url, "/")
}

const (
	DEFAULT_REMOTE_SERVER = "pixelpixel.haxigon.com"
)

func validateServer(server string) string {
	t := time.AfterFunc(3*time.Second, func() {
		log.Fatalf("Attempt to connect to server timed out")
	})
	defer func() {
		t.Stop()
	}()

	resp, err := http.Get("http://" + path.Join(server, "handshake"))
	if err == nil && readAll(resp.Body) == "PIXELPIXEL OK" {
		return server
	}

	resp, err = http.Get("http://" + path.Join(DEFAULT_REMOTE_SERVER, "handshake"))
	if err == nil && readAll(resp.Body) == "PIXELPIXEL OK" {
		return DEFAULT_REMOTE_SERVER
	}
	log.Fatalf("Could not connect to server")
	return ""
}

func readAll(r io.Reader) string {
	data, _ := ioutil.ReadAll(r)
	return string(data)
}
<|endoftext|>


<commit_before>/*
Package sample implements some useful functions to process samples from
statistical populations. The standard Go `float64` type is used in all
computations.
*/
package sample

import (
	"errors"
	"math"
)

// ErrBesselNeedsTwo is returned whenever the sample length is less than 2
// sample points, as required by Bessel's correction.
//
// ErrNotAPercentile is returned when the confidence level for
// MeanConfidenceIntervals is not in the percentile range.
var (
	ErrBesselNeedsTwo     = errors.New("Bessel's correction needs at least two sample points")
	ErrNotAPercentile     = errors.New("number is not a percentile (0<=x<=1)")
	ErrNotPositiveInteger = errors.New("number is not a positive integer (0<n)")
)

// Sample zero values are not safe, use the New function to initialize Sample
// types.
type Sample struct {
	data []float64
	// memoization:
	mean *float64
	sd   *float64
	se   *float64
}

// New returns an Sample value initialized with a *copy* of its parameter and a
// nil error on success. It returns nil and ErrBesselNeedsTwo if the sample
// length is smaller than 2.
//
// A copy of the data is used internally to protect it from future
// modifications, allowing for memoizaton of already computed statistical
// values.
func New(data []float64) (*Sample, error) {
	if len(data) < 2 {
		return nil, ErrBesselNeedsTwo
	}
	sample := new(Sample)
	sample.data = make([]float64, len(data))
	copy(sample.data, data)
	return sample, nil
}

// Mean computes the sample mean of a population sample or returns its
// previously computed value.
func (s *Sample) Mean() float64 {
	if s.mean != nil {
		return *s.mean
	}
	s.mean = new(float64)

	sum := 0.0
	for i := range s.data {
		sum += s.data[i]
	}
	*s.mean = sum / float64(len(s.data))

	return *s.mean
}

// StandardDeviation computes the sample-based unbiased estimation of the
// standard deviation of a population or returns its previously computed value.
//
// This implementation uses the Bessel's correction and is downward biassed as
// per Jensen's inequality.
//
// It is calculated as sqrt(1/(N-1) sum_i_N(x_i-mean(x))).
func (s *Sample) StandardDeviation() float64 {
	if s.sd != nil {
		return *s.sd
	}
	s.sd = new(float64)

	s.Mean()

	sum := 0.0
	var diff float64
	for _, sp := range s.data {
		diff = sp - *s.mean
		sum += diff * diff
	}
	*s.sd = math.Sqrt(sum / float64(len(s.data)-1))

	return *s.sd
}

// StandardError returns the standard deviation of the sampling
// distribution of the mean, also known as the "Standard Error of the
// Mean".
func (s *Sample) StandardError() float64 {
	if s.se != nil {
		return *s.se
	}
	s.se = new(float64)

	s.StandardDeviation()

	*s.se = *s.sd / math.Sqrt(float64(len(s.data)))

	return *s.se
}
<commit_msg>Add some TODOs<commit_after>/*
Package sample implements some useful functions to process samples from
statistical populations. The standard Go `float64` type is used in all
computations.
*/
package sample

import (
	"errors"
	"math"
)

// ErrBesselNeedsTwo is returned whenever the sample length is less than 2
// sample points, as required by Bessel's correction.
//
// ErrNotAPercentile is returned when the confidence level for
// MeanConfidenceIntervals is not in the percentile range.
var (
	ErrBesselNeedsTwo     = errors.New("Bessel's correction needs at least two sample points")
	ErrNotAPercentile     = errors.New("number is not a percentile (0<=x<=1)")
	ErrNotPositiveInteger = errors.New("number is not a positive integer (0<n)")
)

// Sample zero values are not safe, use the New function to initialize Sample
// types.
type Sample struct {
	data []float64
	// memoization:
	mean *float64
	sd   *float64
	se   *float64
}

// New returns an Sample value initialized with a *copy* of its parameter and a
// nil error on success. It returns nil and ErrBesselNeedsTwo if the sample
// length is smaller than 2.
//
// A copy of the data is used internally to protect it from future
// modifications, allowing for memoizaton of already computed statistical
// values.
func New(data []float64) (*Sample, error) {
	if len(data) < 2 {
		return nil, ErrBesselNeedsTwo
	}
	sample := new(Sample)
	sample.data = make([]float64, len(data))
	copy(sample.data, data)
	return sample, nil
}

// Mean computes the sample mean of a population sample or returns its
// previously computed value.
func (s *Sample) Mean() float64 {
	if s.mean != nil {
		return *s.mean
	}
	s.mean = new(float64)

	// TODO: the sum can be done concurrently
	sum := 0.0
	for i := range s.data {
		sum += s.data[i]
	}
	*s.mean = sum / float64(len(s.data))

	return *s.mean
}

// StandardDeviation computes the sample-based unbiased estimation of the
// standard deviation of a population or returns its previously computed value.
//
// This implementation uses the Bessel's correction and is downward biassed as
// per Jensen's inequality.
//
// It is calculated as sqrt(1/(N-1) sum_i_N(x_i-mean(x))).
func (s *Sample) StandardDeviation() float64 {
	if s.sd != nil {
		return *s.sd
	}
	s.sd = new(float64)

	s.Mean()

	// TODO: this sum can be done concurrently
	sum := 0.0
	var diff float64
	for _, sp := range s.data {
		diff = sp - *s.mean
		sum += diff * diff
	}
	*s.sd = math.Sqrt(sum / float64(len(s.data)-1))

	return *s.sd
}

// StandardError returns the standard deviation of the sampling
// distribution of the mean, also known as the "Standard Error of the
// Mean".
func (s *Sample) StandardError() float64 {
	if s.se != nil {
		return *s.se
	}
	s.se = new(float64)

	s.StandardDeviation()

	*s.se = *s.sd / math.Sqrt(float64(len(s.data)))

	return *s.se
}
<|endoftext|>


<commit_before>package horizon

import (
	"net/http"

	"github.com/jagregory/halgo"
	"github.com/stellar/go-horizon/render/hal"
)

// RootResource is the initial map of links into the api.
type RootResource struct {
	halgo.Links
}

var globalRootResource = RootResource{
	Links: halgo.Links{}.
		Self("/").
		Link("account", "/accounts/{address}").
		Link("account_transactions", "/accounts/{address}/transactions{?cursor,limit,order}").
		Link("transaction", "/transactions/{hash}").
		Link("transactions", "/transactions{?cursor,limit,order}").
		Link("orderbook", "/orderbooks{?base_type,base_code,base_issuer,counter_type,counter_code,counter_issuer}").
		Link("metrics", "/metrics").
		Link("friendbot", "/friendbot{?addr}"),
}

func rootAction(w http.ResponseWriter, r *http.Request) {
	hal.Render(w, globalRootResource)
}
<commit_msg>Correct root resource link to real order_book route<commit_after>package horizon

import (
	"net/http"

	"github.com/jagregory/halgo"
	"github.com/stellar/go-horizon/render/hal"
)

// RootResource is the initial map of links into the api.
type RootResource struct {
	halgo.Links
}

var globalRootResource = RootResource{
	Links: halgo.Links{}.
		Self("/").
		Link("account", "/accounts/{address}").
		Link("account_transactions", "/accounts/{address}/transactions{?cursor,limit,order}").
		Link("transaction", "/transactions/{hash}").
		Link("transactions", "/transactions{?cursor,limit,order}").
		Link("order_book", "/order_book{?selling_type,selling_code,selling_issuer,buying_type,buying_code,buying_issuer}").
		Link("metrics", "/metrics").
		Link("friendbot", "/friendbot{?addr}"),
}

func rootAction(w http.ResponseWriter, r *http.Request) {
	hal.Render(w, globalRootResource)
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package watch provides a sub tool for watching a running pipeline.
package watch

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"github.com/googlegenomics/pipelines-tools/pipelines/internal/common"
	genomics "google.golang.org/api/genomics/v2alpha1"
)

func Invoke(ctx context.Context, service *genomics.Service, project string, arguments []string) error {
	if len(arguments) < 1 {
		return errors.New("missing operation name")
	}

	name := common.ExpandOperationName(project, arguments[0])
	result, err := watch(ctx, service, name)
	if err != nil {
		return fmt.Errorf("watching pipeline: %v", err)
	}

	if status, ok := result.(*genomics.Status); ok {
		return fmt.Errorf("executing pipeline: %s", status.Message)
	}

	fmt.Println("Pipeline execution completed")
	return nil
}

func watch(ctx context.Context, service *genomics.Service, name string) (interface{}, error) {
	var events []*genomics.Event
	for {
		lro, err := service.Projects.Operations.Get(name).Context(ctx).Do()
		if err != nil {
			return nil, fmt.Errorf("getting operation status: %v", err)
		}

		var metadata genomics.Metadata
		if err := json.Unmarshal(lro.Metadata, &metadata); err != nil {
			return nil, fmt.Errorf("parsing metadata: %v", err)
		}

		if len(events) != len(metadata.Events) {
			for i := len(metadata.Events) - len(events) - 1; i >= 0; i-- {
				timestamp, _ := time.Parse(time.RFC3339Nano, metadata.Events[i].Timestamp)
				fmt.Println(timestamp.Format("15:04:05"), metadata.Events[i].Description)
			}
			events = metadata.Events
		}

		if lro.Done {
			if lro.Error != nil {
				return lro.Error, nil
			}
			return lro.Response, nil
		}

		time.Sleep(5 * time.Second)
	}
}
<commit_msg>Add --details flag to watch<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package watch provides a sub tool for watching a running pipeline.
package watch

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"time"

	"github.com/googlegenomics/pipelines-tools/pipelines/internal/common"
	genomics "google.golang.org/api/genomics/v2alpha1"
)

var (
	flags = flag.NewFlagSet("", flag.ExitOnError)

	details = flags.Bool("details", false, "show event details")
)

func Invoke(ctx context.Context, service *genomics.Service, project string, arguments []string) error {
	names := common.ParseFlags(flags, arguments)
	if len(names) < 1 {
		return errors.New("missing operation name")
	}

	name := common.ExpandOperationName(project, names[0])
	result, err := watch(ctx, service, name)
	if err != nil {
		return fmt.Errorf("watching pipeline: %v", err)
	}

	if status, ok := result.(*genomics.Status); ok {
		return fmt.Errorf("executing pipeline: %s", status.Message)
	}

	fmt.Println("Pipeline execution completed")
	return nil
}

func watch(ctx context.Context, service *genomics.Service, name string) (interface{}, error) {
	var events []*genomics.Event
	for {
		lro, err := service.Projects.Operations.Get(name).Context(ctx).Do()
		if err != nil {
			return nil, fmt.Errorf("getting operation status: %v", err)
		}

		var metadata genomics.Metadata
		if err := json.Unmarshal(lro.Metadata, &metadata); err != nil {
			return nil, fmt.Errorf("parsing metadata: %v", err)
		}

		if len(events) != len(metadata.Events) {
			for i := len(metadata.Events) - len(events) - 1; i >= 0; i-- {
				timestamp, _ := time.Parse(time.RFC3339Nano, metadata.Events[i].Timestamp)
				fmt.Println(timestamp.Format("15:04:05"), metadata.Events[i].Description)

				if *details {
					fmt.Println(string(metadata.Events[i].Details))
				}
			}
			events = metadata.Events
		}

		if lro.Done {
			if lro.Error != nil {
				return lro.Error, nil
			}
			return lro.Response, nil
		}

		time.Sleep(5 * time.Second)
	}
}
<|endoftext|>