<commit_before>package player

import (
	"bytes"
	"expvar"
	"log"
	"io"
	"math"
	"net"
	"os"
	"sync"

	"chunkymonkey/entity"
	. "chunkymonkey/interfaces"
	"chunkymonkey/inventory"
	"chunkymonkey/itemtype"
	"chunkymonkey/proto"
	"chunkymonkey/slot"
	. "chunkymonkey/types"
)

var (
	expVarPlayerConnectionCount    *expvar.Int
	expVarPlayerDisconnectionCount *expvar.Int
)

const StanceNormal = 1.62

func init() {
	expVarPlayerConnectionCount = expvar.NewInt("player-connection-count")
	expVarPlayerDisconnectionCount = expvar.NewInt("player-disconnection-count")
}

type Player struct {
	entity.Entity
	game      IGame
	conn      net.Conn
	name      string
	position  AbsXyz
	look      LookDegrees
	chunkSubs chunkSubscriptions

	cursor       slot.Slot // Item being moved by mouse cursor.
	inventory    inventory.PlayerInventory
	curWindow    inventory.IWindow
	nextWindowId WindowId

	mainQueue chan func(IPlayer)
	txQueue   chan []byte
	lock      sync.Mutex
}

func StartPlayer(game IGame, conn net.Conn, name string) {
	player := &Player{
		game:     game,
		conn:     conn,
		name:     name,
		position: *game.GetStartPosition(),
		look:     LookDegrees{0, 0},

		curWindow:    nil,
		nextWindowId: WindowIdFreeMin,

		mainQueue: make(chan func(IPlayer), 128),
		txQueue:   make(chan []byte, 128),
	}

	player.chunkSubs.Init(player)

	player.cursor.Init()
	player.inventory.Init(player.EntityId, player, game.GetGameRules())

	game.Enqueue(func(game IGame) {
		game.AddPlayer(player)
		buf := &bytes.Buffer{}
		// TODO pass proper dimension. This is low priority, because there is
		// currently no way to update the client's dimension after login.
		proto.ServerWriteLogin(buf, player.EntityId, 0, DimensionNormal)
		proto.WriteSpawnPosition(buf, player.position.ToBlockXyz())
		player.TransmitPacket(buf.Bytes())
		player.start()
	})
}

func (player *Player) GetEntityId() EntityId {
	return player.EntityId
}

func (player *Player) GetEntity() *entity.Entity {
	return &player.Entity
}

func (player *Player) LockedGetChunkPosition() *ChunkXz {
	player.lock.Lock()
	defer player.lock.Unlock()
	return player.position.ToChunkXz()
}

func (player *Player) IsWithin(p1, p2 *ChunkXz) bool {
	p := player.position.ToChunkXz()
	return (p.X >= p1.X && p.X <= p2.X &&
		p.Z >= p1.Z && p.Z <= p2.Z)
}

func (player *Player) GetName() string {
	return player.name
}

func (player *Player) SendSpawn(writer io.Writer) (err os.Error) {
	heldSlot, _ := player.inventory.HeldItem()

	err = proto.WriteNamedEntitySpawn(
		writer,
		player.EntityId, player.name,
		player.position.ToAbsIntXyz(),
		player.look.ToLookBytes(),
		heldSlot.GetItemTypeId(),
	)
	if err != nil {
		return
	}
	return player.inventory.SendFullEquipmentUpdate(writer)
}

func (player *Player) GetHeldItemType() *itemtype.ItemType {
	slot, _ := player.inventory.HeldItem()
	return slot.ItemType
}

func (player *Player) TakeOneHeldItem(into *slot.Slot) {
	player.inventory.TakeOneHeldItem(into)
}

func (player *Player) start() {
	go player.receiveLoop()
	go player.transmitLoop()
	go player.mainLoop()
}

// Start of packet handling code
// Note: any packet handlers that could change the player state or read a
// changeable state must use player.lock

func (player *Player) PacketKeepAlive() {
}

func (player *Player) PacketChatMessage(message string) {
	player.game.Enqueue(func(game IGame) { game.SendChatMessage(message) })
}

func (player *Player) PacketEntityAction(entityId EntityId, action EntityAction) {
}

func (player *Player) PacketUseEntity(user EntityId, target EntityId, leftClick bool) {
}

func (player *Player) PacketRespawn() {
}

func (player *Player) PacketPlayer(onGround bool) {
}

func (player *Player) PacketPlayerPosition(position *AbsXyz, stance AbsCoord, onGround bool) {
	player.lock.Lock()
	defer player.lock.Unlock()

	var delta = AbsXyz{position.X - player.position.X,
		position.Y - player.position.Y,
		position.Z - player.position.Z}
	distance := math.Sqrt(float64(delta.X*delta.X + delta.Y*delta.Y + delta.Z*delta.Z))
	if distance > 10 {
		log.Printf("Discarding player position that is too far removed (%.2f, %.2f, %.2f)",
			position.X, position.Y, position.Z)
		return
	}
	player.position = *position
	player.chunkSubs.Move(position, nil)

	// TODO: Should keep track of when players enter/leave their mutual radius
	// of "awareness". I.e a client should receive a RemoveEntity packet when
	// the player walks out of range, and no longer receive WriteEntityTeleport
	// packets for them. The converse should happen when players come in range
	// of each other.

	buf := &bytes.Buffer{}
	proto.WriteEntityTeleport(
		buf,
		player.EntityId,
		player.position.ToAbsIntXyz(),
		player.look.ToLookBytes())

	player.game.Enqueue(func(game IGame) {
		game.MulticastPacket(buf.Bytes(), player)
	})
}

func (player *Player) PacketPlayerLook(look *LookDegrees, onGround bool) {
	player.lock.Lock()
	defer player.lock.Unlock()

	// TODO input validation
	player.look = *look

	buf := &bytes.Buffer{}
	proto.WriteEntityLook(buf, player.EntityId, look.ToLookBytes())

	player.game.Enqueue(func(game IGame) {
		game.MulticastPacket(buf.Bytes(), player)
	})
}

func (player *Player) PacketPlayerBlockHit(status DigStatus, blockLoc *BlockXyz, face Face) {
	// TODO validate that the player is actually somewhere near the block

	// TODO validate that the player has dug long enough to stop speed
	// hacking (based on block type and tool used - non-trivial).

	if face != FaceNull {
		chunkLoc, subLoc := blockLoc.ToChunkLocal()

		player.game.Enqueue(func(game IGame) {
			chunk := game.GetChunkManager().Get(chunkLoc)

			if chunk == nil {
				return
			}

			chunk.Enqueue(func(chunk IChunk) {
				chunk.PlayerBlockHit(player, subLoc, status)
			})
		})
	} else {
		// TODO player dropped item
	}
}

func (player *Player) PacketPlayerBlockInteract(itemId ItemTypeId, blockLoc *BlockXyz, face Face, amount ItemCount, uses ItemData) {
	if face < FaceMinValid || face > FaceMaxValid {
		// TODO sometimes FaceNull means something. This case should be covered.
		log.Printf("Player/PacketPlayerBlockInteract: invalid face %d", face)
		return
	}

	placeChunkLoc, _ := blockLoc.ToChunkLocal()

	player.game.Enqueue(func(game IGame) {
		chunk := game.GetChunkManager().Get(placeChunkLoc)

		if chunk == nil {
			return
		}

		chunk.Enqueue(func(chunk IChunk) {
			chunk.PlayerBlockInteract(player, blockLoc, face)
		})
	})
}

func (player *Player) PacketHoldingChange(slotId SlotId) {
	player.lock.Lock()
	defer player.lock.Unlock()
	player.inventory.SetHolding(slotId)
}

func (player *Player) PacketEntityAnimation(entityId EntityId, animation EntityAnimation) {
}

func (player *Player) PacketUnknown0x1b(field1, field2 float32, field3, field4 bool, field5, field6 float32) {
}

func (player *Player) PacketWindowClose(windowId WindowId) {
	player.lock.Lock()
	defer player.lock.Unlock()

	if player.curWindow != nil && player.curWindow.GetWindowId() == windowId {
		player.curWindow.Finalize(false)
	}
}

func (player *Player) PacketWindowClick(windowId WindowId, slotId SlotId, rightClick bool, txId TxId, shiftClick bool, itemId ItemTypeId, amount ItemCount, uses ItemData) {
	player.lock.Lock()
	defer player.lock.Unlock()

	// Note that the parameters itemId, amount and uses are all currently
	// ignored. The item(s) involved are worked out from the server-side data.

	// Determine which inventory window is involved.
	// TODO support for more windows

	var clickedWindow inventory.IWindow
	if windowId == WindowIdInventory {
		clickedWindow = &player.inventory
	} else if player.curWindow != nil && player.curWindow.GetWindowId() == windowId {
		clickedWindow = player.curWindow
	} else {
		log.Printf(
			"Warning: ignored window click on unknown window ID %d",
			windowId)
	}

	buf := &bytes.Buffer{}
	accepted := false

	if clickedWindow != nil {
		accepted = clickedWindow.Click(slotId, &player.cursor, rightClick, shiftClick)

		// We send slot updates in case we have custom max counts that differ
		// from the client's own model.
		player.cursor.SendUpdate(buf, WindowIdCursor, SlotIdCursor)
	}

	// Inform client of operation status.
	proto.WriteWindowTransaction(buf, windowId, txId, accepted)

	player.TransmitPacket(buf.Bytes())
}

func (player *Player) PacketWindowTransaction(windowId WindowId, txId TxId, accepted bool) {
	// TODO investigate when this packet is sent from the client and what it
	// means when it does get sent.
}

func (player *Player) PacketSignUpdate(position *BlockXyz, lines [4]string) {
}

func (player *Player) PacketDisconnect(reason string) {
	log.Printf("Player %s disconnected reason=%s", player.name, reason)
	player.game.Enqueue(func(game IGame) {
		game.RemovePlayer(player)
	})
	player.txQueue <- nil
	player.mainQueue <- nil
	player.conn.Close()
}

func (player *Player) receiveLoop() {
	for {
		err := proto.ServerReadPacket(player.conn, player)
		if err != nil {
			if err != os.EOF {
				log.Print("ReceiveLoop failed: ", err.String())
			}
			return
		}
	}
}

// End of packet handling code

func (player *Player) transmitLoop() {
	for {
		bs := <-player.txQueue

		if bs == nil {
			return // txQueue closed
		}

		_, err := player.conn.Write(bs)
		if err != nil {
			if err != os.EOF {
				log.Print("TransmitLoop failed: ", err.String())
			}
			return
		}
	}
}

func (player *Player) TransmitPacket(packet []byte) {
	if packet == nil {
		return // skip empty packets
	}
	player.txQueue <- packet
}

func (player *Player) runQueuedCall(f func(IPlayer)) {
	player.lock.Lock()
	defer player.lock.Unlock()
	f(player)
}

func (player *Player) mainLoop() {
	expVarPlayerConnectionCount.Add(1)
	defer func() {
		expVarPlayerDisconnectionCount.Add(1)
		player.chunkSubs.clear()
	}()

	player.postLogin()

	for {
		f := <-player.mainQueue
		if f == nil {
			return
		}
		player.runQueuedCall(f)
	}
}

// Enqueue queues a function to run with the player lock within the player's
// mainloop.
func (player *Player) Enqueue(f func(IPlayer)) {
	if f == nil {
		return
	}
	player.mainQueue <- f
}

// WithLock runs a function with the player lock within the calling goroutine.
func (player *Player) WithLock(f func(IPlayer)) {
	player.lock.Lock()
	defer player.lock.Unlock()
	f(player)
}

// Used to receive items picked up from chunks. It is synchronous so that the
// passed item can be looked at by the caller afterwards to see if it has been
// consumed.
func (player *Player) OfferItem(item *slot.Slot) {
	player.lock.Lock()
	defer player.lock.Unlock()

	player.inventory.PutItem(item)

	return
}

// OpenWindow queues a request that the player opens the given window type.
// TODO this should be passed an appropriate *Inventory for inventories that
// are tied to the world (particularly for chests).
func (player *Player) OpenWindow(invTypeId InvTypeId) {
	player.Enqueue(func(_ IPlayer) {
		player.closeCurrentWindow(true)
		window := player.inventory.NewWindow(invTypeId, player.nextWindowId)

		buf := &bytes.Buffer{}
		if err := window.WriteWindowOpen(buf); err != nil {
			window.Finalize(false)
			return
		}
		if err := window.WriteWindowItems(buf); err != nil {
			window.Finalize(false)
			return
		}
		player.TransmitPacket(buf.Bytes())

		player.curWindow = window
		if player.nextWindowId >= WindowIdFreeMax {
			player.nextWindowId = WindowIdFreeMin
		} else {
			player.nextWindowId++
		}
	})
}

// closeCurrentWindow closes any open window. It must be called with
// player.lock held.
func (player *Player) closeCurrentWindow(sendClosePacket bool) {
	if player.curWindow != nil {
		player.curWindow.Finalize(sendClosePacket)
	}
	player.curWindow = nil
}

// Blocks until essential login packets have been transmitted.
func (player *Player) postLogin() {
	nearbySent := func() {
		player.lock.Lock()
		defer player.lock.Unlock()

		// Send player start position etc.
		buf := &bytes.Buffer{}
		proto.ServerWritePlayerPositionLook(
			buf, &player.position, &player.look,
			player.position.Y+StanceNormal, false)

		player.inventory.WriteWindowItems(buf)

		player.TransmitPacket(buf.Bytes())
	}

	player.chunkSubs.Move(&player.position, nearbySent)
}
<commit_msg>Added check for opening nil window.<commit_after>package player

import (
	"bytes"
	"expvar"
	"log"
	"io"
	"math"
	"net"
	"os"
	"sync"

	"chunkymonkey/entity"
	. "chunkymonkey/interfaces"
	"chunkymonkey/inventory"
	"chunkymonkey/itemtype"
	"chunkymonkey/proto"
	"chunkymonkey/slot"
	. "chunkymonkey/types"
)

var (
	expVarPlayerConnectionCount    *expvar.Int
	expVarPlayerDisconnectionCount *expvar.Int
)

const StanceNormal = 1.62

func init() {
	expVarPlayerConnectionCount = expvar.NewInt("player-connection-count")
	expVarPlayerDisconnectionCount = expvar.NewInt("player-disconnection-count")
}

type Player struct {
	entity.Entity
	game      IGame
	conn      net.Conn
	name      string
	position  AbsXyz
	look      LookDegrees
	chunkSubs chunkSubscriptions

	cursor       slot.Slot // Item being moved by mouse cursor.
	inventory    inventory.PlayerInventory
	curWindow    inventory.IWindow
	nextWindowId WindowId

	mainQueue chan func(IPlayer)
	txQueue   chan []byte
	lock      sync.Mutex
}

func StartPlayer(game IGame, conn net.Conn, name string) {
	player := &Player{
		game:     game,
		conn:     conn,
		name:     name,
		position: *game.GetStartPosition(),
		look:     LookDegrees{0, 0},

		curWindow:    nil,
		nextWindowId: WindowIdFreeMin,

		mainQueue: make(chan func(IPlayer), 128),
		txQueue:   make(chan []byte, 128),
	}

	player.chunkSubs.Init(player)

	player.cursor.Init()
	player.inventory.Init(player.EntityId, player, game.GetGameRules())

	game.Enqueue(func(game IGame) {
		game.AddPlayer(player)
		buf := &bytes.Buffer{}
		// TODO pass proper dimension. This is low priority, because there is
		// currently no way to update the client's dimension after login.
		proto.ServerWriteLogin(buf, player.EntityId, 0, DimensionNormal)
		proto.WriteSpawnPosition(buf, player.position.ToBlockXyz())
		player.TransmitPacket(buf.Bytes())
		player.start()
	})
}

func (player *Player) GetEntityId() EntityId {
	return player.EntityId
}

func (player *Player) GetEntity() *entity.Entity {
	return &player.Entity
}

func (player *Player) LockedGetChunkPosition() *ChunkXz {
	player.lock.Lock()
	defer player.lock.Unlock()
	return player.position.ToChunkXz()
}

func (player *Player) IsWithin(p1, p2 *ChunkXz) bool {
	p := player.position.ToChunkXz()
	return (p.X >= p1.X && p.X <= p2.X &&
		p.Z >= p1.Z && p.Z <= p2.Z)
}

func (player *Player) GetName() string {
	return player.name
}

func (player *Player) SendSpawn(writer io.Writer) (err os.Error) {
	heldSlot, _ := player.inventory.HeldItem()

	err = proto.WriteNamedEntitySpawn(
		writer,
		player.EntityId, player.name,
		player.position.ToAbsIntXyz(),
		player.look.ToLookBytes(),
		heldSlot.GetItemTypeId(),
	)
	if err != nil {
		return
	}
	return player.inventory.SendFullEquipmentUpdate(writer)
}

func (player *Player) GetHeldItemType() *itemtype.ItemType {
	slot, _ := player.inventory.HeldItem()
	return slot.ItemType
}

func (player *Player) TakeOneHeldItem(into *slot.Slot) {
	player.inventory.TakeOneHeldItem(into)
}

func (player *Player) start() {
	go player.receiveLoop()
	go player.transmitLoop()
	go player.mainLoop()
}

// Start of packet handling code
// Note: any packet handlers that could change the player state or read a
// changeable state must use player.lock

func (player *Player) PacketKeepAlive() {
}

func (player *Player) PacketChatMessage(message string) {
	player.game.Enqueue(func(game IGame) { game.SendChatMessage(message) })
}

func (player *Player) PacketEntityAction(entityId EntityId, action EntityAction) {
}

func (player *Player) PacketUseEntity(user EntityId, target EntityId, leftClick bool) {
}

func (player *Player) PacketRespawn() {
}

func (player *Player) PacketPlayer(onGround bool) {
}

func (player *Player) PacketPlayerPosition(position *AbsXyz, stance AbsCoord, onGround bool) {
	player.lock.Lock()
	defer player.lock.Unlock()

	var delta = AbsXyz{position.X - player.position.X,
		position.Y - player.position.Y,
		position.Z - player.position.Z}
	distance := math.Sqrt(float64(delta.X*delta.X + delta.Y*delta.Y + delta.Z*delta.Z))
	if distance > 10 {
		log.Printf("Discarding player position that is too far removed (%.2f, %.2f, %.2f)",
			position.X, position.Y, position.Z)
		return
	}
	player.position = *position
	player.chunkSubs.Move(position, nil)

	// TODO: Should keep track of when players enter/leave their mutual radius
	// of "awareness". I.e a client should receive a RemoveEntity packet when
	// the player walks out of range, and no longer receive WriteEntityTeleport
	// packets for them. The converse should happen when players come in range
	// of each other.

	buf := &bytes.Buffer{}
	proto.WriteEntityTeleport(
		buf,
		player.EntityId,
		player.position.ToAbsIntXyz(),
		player.look.ToLookBytes())

	player.game.Enqueue(func(game IGame) {
		game.MulticastPacket(buf.Bytes(), player)
	})
}

func (player *Player) PacketPlayerLook(look *LookDegrees, onGround bool) {
	player.lock.Lock()
	defer player.lock.Unlock()

	// TODO input validation
	player.look = *look

	buf := &bytes.Buffer{}
	proto.WriteEntityLook(buf, player.EntityId, look.ToLookBytes())

	player.game.Enqueue(func(game IGame) {
		game.MulticastPacket(buf.Bytes(), player)
	})
}

func (player *Player) PacketPlayerBlockHit(status DigStatus, blockLoc *BlockXyz, face Face) {
	// TODO validate that the player is actually somewhere near the block

	// TODO validate that the player has dug long enough to stop speed
	// hacking (based on block type and tool used - non-trivial).

	if face != FaceNull {
		chunkLoc, subLoc := blockLoc.ToChunkLocal()

		player.game.Enqueue(func(game IGame) {
			chunk := game.GetChunkManager().Get(chunkLoc)

			if chunk == nil {
				return
			}

			chunk.Enqueue(func(chunk IChunk) {
				chunk.PlayerBlockHit(player, subLoc, status)
			})
		})
	} else {
		// TODO player dropped item
	}
}

func (player *Player) PacketPlayerBlockInteract(itemId ItemTypeId, blockLoc *BlockXyz, face Face, amount ItemCount, uses ItemData) {
	if face < FaceMinValid || face > FaceMaxValid {
		// TODO sometimes FaceNull means something. This case should be covered.
		log.Printf("Player/PacketPlayerBlockInteract: invalid face %d", face)
		return
	}

	placeChunkLoc, _ := blockLoc.ToChunkLocal()

	player.game.Enqueue(func(game IGame) {
		chunk := game.GetChunkManager().Get(placeChunkLoc)

		if chunk == nil {
			return
		}

		chunk.Enqueue(func(chunk IChunk) {
			chunk.PlayerBlockInteract(player, blockLoc, face)
		})
	})
}

func (player *Player) PacketHoldingChange(slotId SlotId) {
	player.lock.Lock()
	defer player.lock.Unlock()
	player.inventory.SetHolding(slotId)
}

func (player *Player) PacketEntityAnimation(entityId EntityId, animation EntityAnimation) {
}

func (player *Player) PacketUnknown0x1b(field1, field2 float32, field3, field4 bool, field5, field6 float32) {
}

func (player *Player) PacketWindowClose(windowId WindowId) {
	player.lock.Lock()
	defer player.lock.Unlock()

	if player.curWindow != nil && player.curWindow.GetWindowId() == windowId {
		player.curWindow.Finalize(false)
	}
}

func (player *Player) PacketWindowClick(windowId WindowId, slotId SlotId, rightClick bool, txId TxId, shiftClick bool, itemId ItemTypeId, amount ItemCount, uses ItemData) {
	player.lock.Lock()
	defer player.lock.Unlock()

	// Note that the parameters itemId, amount and uses are all currently
	// ignored. The item(s) involved are worked out from the server-side data.

	// Determine which inventory window is involved.
	// TODO support for more windows

	var clickedWindow inventory.IWindow
	if windowId == WindowIdInventory {
		clickedWindow = &player.inventory
	} else if player.curWindow != nil && player.curWindow.GetWindowId() == windowId {
		clickedWindow = player.curWindow
	} else {
		log.Printf(
			"Warning: ignored window click on unknown window ID %d",
			windowId)
	}

	buf := &bytes.Buffer{}
	accepted := false

	if clickedWindow != nil {
		accepted = clickedWindow.Click(slotId, &player.cursor, rightClick, shiftClick)

		// We send slot updates in case we have custom max counts that differ
		// from the client's own model.
		player.cursor.SendUpdate(buf, WindowIdCursor, SlotIdCursor)
	}

	// Inform client of operation status.
	proto.WriteWindowTransaction(buf, windowId, txId, accepted)

	player.TransmitPacket(buf.Bytes())
}

func (player *Player) PacketWindowTransaction(windowId WindowId, txId TxId, accepted bool) {
	// TODO investigate when this packet is sent from the client and what it
	// means when it does get sent.
}

func (player *Player) PacketSignUpdate(position *BlockXyz, lines [4]string) {
}

func (player *Player) PacketDisconnect(reason string) {
	log.Printf("Player %s disconnected reason=%s", player.name, reason)
	player.game.Enqueue(func(game IGame) {
		game.RemovePlayer(player)
	})
	player.txQueue <- nil
	player.mainQueue <- nil
	player.conn.Close()
}

func (player *Player) receiveLoop() {
	for {
		err := proto.ServerReadPacket(player.conn, player)
		if err != nil {
			if err != os.EOF {
				log.Print("ReceiveLoop failed: ", err.String())
			}
			return
		}
	}
}

// End of packet handling code

func (player *Player) transmitLoop() {
	for {
		bs := <-player.txQueue

		if bs == nil {
			return // txQueue closed
		}

		_, err := player.conn.Write(bs)
		if err != nil {
			if err != os.EOF {
				log.Print("TransmitLoop failed: ", err.String())
			}
			return
		}
	}
}

func (player *Player) TransmitPacket(packet []byte) {
	if packet == nil {
		return // skip empty packets
	}
	player.txQueue <- packet
}

func (player *Player) runQueuedCall(f func(IPlayer)) {
	player.lock.Lock()
	defer player.lock.Unlock()
	f(player)
}

func (player *Player) mainLoop() {
	expVarPlayerConnectionCount.Add(1)
	defer func() {
		expVarPlayerDisconnectionCount.Add(1)
		player.chunkSubs.clear()
	}()

	player.postLogin()

	for {
		f := <-player.mainQueue
		if f == nil {
			return
		}
		player.runQueuedCall(f)
	}
}

// Enqueue queues a function to run with the player lock within the player's
// mainloop.
func (player *Player) Enqueue(f func(IPlayer)) {
	if f == nil {
		return
	}
	player.mainQueue <- f
}

// WithLock runs a function with the player lock within the calling goroutine.
func (player *Player) WithLock(f func(IPlayer)) {
	player.lock.Lock()
	defer player.lock.Unlock()
	f(player)
}

// Used to receive items picked up from chunks. It is synchronous so that the
// passed item can be looked at by the caller afterwards to see if it has been
// consumed.
func (player *Player) OfferItem(item *slot.Slot) {
	player.lock.Lock()
	defer player.lock.Unlock()

	player.inventory.PutItem(item)

	return
}

// OpenWindow queues a request that the player opens the given window type.
// TODO this should be passed an appropriate *Inventory for inventories that
// are tied to the world (particularly for chests).
func (player *Player) OpenWindow(invTypeId InvTypeId) {
	player.Enqueue(func(_ IPlayer) {
		player.closeCurrentWindow(true)
		window := player.inventory.NewWindow(invTypeId, player.nextWindowId)
		if window == nil {
			return
		}

		buf := &bytes.Buffer{}
		if err := window.WriteWindowOpen(buf); err != nil {
			window.Finalize(false)
			return
		}
		if err := window.WriteWindowItems(buf); err != nil {
			window.Finalize(false)
			return
		}
		player.TransmitPacket(buf.Bytes())

		player.curWindow = window
		if player.nextWindowId >= WindowIdFreeMax {
			player.nextWindowId = WindowIdFreeMin
		} else {
			player.nextWindowId++
		}
	})
}

// closeCurrentWindow closes any open window. It must be called with
// player.lock held.
func (player *Player) closeCurrentWindow(sendClosePacket bool) {
	if player.curWindow != nil {
		player.curWindow.Finalize(sendClosePacket)
	}
	player.curWindow = nil
}

// Blocks until essential login packets have been transmitted.
func (player *Player) postLogin() {
	nearbySent := func() {
		player.lock.Lock()
		defer player.lock.Unlock()

		// Send player start position etc.
		buf := &bytes.Buffer{}
		proto.ServerWritePlayerPositionLook(
			buf, &player.position, &player.look,
			player.position.Y+StanceNormal, false)

		player.inventory.WriteWindowItems(buf)

		player.TransmitPacket(buf.Bytes())
	}

	player.chunkSubs.Move(&player.position, nearbySent)
}
<|endoftext|>


<commit_before>// This file is part of *kellner*
//
// Copyright (C) 2015, Travelping GmbH <copyright@travelping.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// the Identity-folder contains a bunch of subfolders. the name of each subfolder is
//
// root/
//      ipk-folder1/
//      ipk-folder2/
//      ipk-folder3/
//
// id-root/
//         client-id-1/
//                     ipk-folder1           (empty file, maps request "/ipk-folder1" to
//                                            /root/ipk-folder1 )
//                     special [ipk-folder2] (text file, containing "ipk-folder2",
//                                            maps request "/special" to /root/ipk-folder2 )
//
type clientIDMuxer struct {
	Folder string         // folder to use for lookup client-id-requests
	Muxer  *http.ServeMux // hold the real worker
}

// looks up the first certificate to get the client-id. based upon the client-id
// we lookup the client-directory and, based upon the request, the mapping to the
// real handler.
func (muxer *clientIDMuxer) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
		writeError(http.StatusUnauthorized, w, r)
		return
	}

	cert := r.TLS.PeerCertificates[0]
	clientID := clientIDByName(&cert.Subject)

	// TODO: do we want this?
	w.Header().Set("Kellner-Client-Id", clientID)

	// TODO: decide how we treat "/" requests ...
	// we could deliver opkg.conf containing all the valid repos
	// or something like that?
	if r.URL.Path == "/" {
		writeError(http.StatusForbidden, w, r)
		return
	}

	fs := fsFileProbe{}
	mapFile, mappedPath, fi, err := findMappingFile(r.URL.Path, muxer.Folder, clientID, ",", &fs)

	if fi == nil || err != nil {
		http.NotFound(w, r)
		return
	}

	// TODO: decide how to treat a directory
	if fi.IsDir() {
		log.Println("fi is a directory, is 404 ok?")
		http.NotFound(w, r)
		return
	}

	// if the file is not empty, the content defines the mapping
	if fi.Size() > 0 {
		content, err := ioutil.ReadFile(mapFile)
		if err != nil {
			log.Printf("warning: reading %q yields %v", mapFile, err)
			writeError(http.StatusInternalServerError, w, r)
			return
		}
		mappedPath = string(bytes.TrimSpace(content))
	}

	mappedRequest := *r
	mappedRequest.URL, _ = url.Parse(r.URL.String())
	mappedRequest.URL.Path = cleanPath(path.Join(mappedPath, path.Base(r.URL.Path)))
	mappedRequest.RequestURI = mappedRequest.URL.Path

	handler, matchingPattern := muxer.Muxer.Handler(&mappedRequest)

	r.Header.Add(_ExtraLogKey,
		fmt.Sprintf("mappedRequest %q: %s => %s (based on matching handler for %q",
			clientID, r.URL.Path, mappedRequest.URL.Path, matchingPattern))

	handler.ServeHTTP(w, &mappedRequest)
}

func findMappingFile(name, folder, id, sep string, fs fileProbe) (mapFile string, needle string, fi os.FileInfo, err error) {

	var (
		needleFile = path.Clean(name)
		needleDir  = path.Dir(needleFile)
	)

	for id != "" {

		base := filepath.Join(folder, id)
		needle = needleFile

		// first try  /cdir/request/file.ipk
		// then       /cdir/request
		//
		// both should find the mapping file /cdir/request
		// TODO:      /cidr/request/subfolder/file.ipk
		mapFile = filepath.Join(base, needle)
		if fi, err = fs.Stat(mapFile); err != nil {
			needle = needleDir
			mapFile = filepath.Join(base, needle)
			fi, err = fs.Stat(mapFile)
		}

		if fi != nil && !fi.IsDir() {
			break
		}

		pos := strings.LastIndexAny(id, sep)
		if pos <= 0 {
			break
		}
		id = id[:pos]
	}

	return mapFile, needle, fi, err
}

func writeError(code int, w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%d %q for %s\n\n", code, http.StatusText(code), r.URL.Path)
}

// copy of http.cleanPath(); we dont want mux.Handler() return a redirect-handler.
// see code of how (*http.ServeMux)(Handler(r *http.Request)) is implemented
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<commit_msg>cosmetic: add closing ) to output<commit_after>// This file is part of *kellner*
//
// Copyright (C) 2015, Travelping GmbH <copyright@travelping.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// the Identity-folder contains a bunch of subfolders. the name of each subfolder is
//
// root/
//      ipk-folder1/
//      ipk-folder2/
//      ipk-folder3/
//
// id-root/
//         client-id-1/
//                     ipk-folder1           (empty file, maps request "/ipk-folder1" to
//                                            /root/ipk-folder1 )
//                     special [ipk-folder2] (text file, containing "ipk-folder2",
//                                            maps request "/special" to /root/ipk-folder2 )
//
type clientIDMuxer struct {
	Folder string         // folder to use for lookup client-id-requests
	Muxer  *http.ServeMux // hold the real worker
}

// looks up the first certificate to get the client-id. based upon the client-id
// we lookup the client-directory and, based upon the request, the mapping to the
// real handler.
func (muxer *clientIDMuxer) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
		writeError(http.StatusUnauthorized, w, r)
		return
	}

	cert := r.TLS.PeerCertificates[0]
	clientID := clientIDByName(&cert.Subject)

	// TODO: do we want this?
	w.Header().Set("Kellner-Client-Id", clientID)

	// TODO: decide how we treat "/" requests ...
	// we could deliver opkg.conf containing all the valid repos
	// or something like that?
	if r.URL.Path == "/" {
		writeError(http.StatusForbidden, w, r)
		return
	}

	fs := fsFileProbe{}
	mapFile, mappedPath, fi, err := findMappingFile(r.URL.Path, muxer.Folder, clientID, ",", &fs)

	if fi == nil || err != nil {
		http.NotFound(w, r)
		return
	}

	// TODO: decide how to treat a directory
	if fi.IsDir() {
		log.Println("fi is a directory, is 404 ok?")
		http.NotFound(w, r)
		return
	}

	// if the file is not empty, the content defines the mapping
	if fi.Size() > 0 {
		content, err := ioutil.ReadFile(mapFile)
		if err != nil {
			log.Printf("warning: reading %q yields %v", mapFile, err)
			writeError(http.StatusInternalServerError, w, r)
			return
		}
		mappedPath = string(bytes.TrimSpace(content))
	}

	mappedRequest := *r
	mappedRequest.URL, _ = url.Parse(r.URL.String())
	mappedRequest.URL.Path = cleanPath(path.Join(mappedPath, path.Base(r.URL.Path)))
	mappedRequest.RequestURI = mappedRequest.URL.Path

	handler, matchingPattern := muxer.Muxer.Handler(&mappedRequest)

	r.Header.Add(_ExtraLogKey,
		fmt.Sprintf("mappedRequest %q: %s => %s (based on matching handler for %q)",
			clientID, r.URL.Path, mappedRequest.URL.Path, matchingPattern))

	handler.ServeHTTP(w, &mappedRequest)
}

func findMappingFile(name, folder, id, sep string, fs fileProbe) (mapFile string, needle string, fi os.FileInfo, err error) {

	var (
		needleFile = path.Clean(name)
		needleDir  = path.Dir(needleFile)
	)

	for id != "" {

		base := filepath.Join(folder, id)
		needle = needleFile

		// first try  /cdir/request/file.ipk
		// then       /cdir/request
		//
		// both should find the mapping file /cdir/request
		// TODO:      /cidr/request/subfolder/file.ipk
		mapFile = filepath.Join(base, needle)
		if fi, err = fs.Stat(mapFile); err != nil {
			needle = needleDir
			mapFile = filepath.Join(base, needle)
			fi, err = fs.Stat(mapFile)
		}

		if fi != nil && !fi.IsDir() {
			break
		}

		pos := strings.LastIndexAny(id, sep)
		if pos <= 0 {
			break
		}
		id = id[:pos]
	}

	return mapFile, needle, fi, err
}

func writeError(code int, w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(code)
	fmt.Fprintf(w, "%d %q for %s\n\n", code, http.StatusText(code), r.URL.Path)
}

// copy of http.cleanPath(); we dont want mux.Handler() return a redirect-handler.
// see code of how (*http.ServeMux)(Handler(r *http.Request)) is implemented
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<|endoftext|>


<commit_before>package imgio

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	"io"
	"sync"
)

type rwImage struct {
	img draw.Image
	gen PointsSequenceGenerator
	prw PointReadWriter
	mux sync.RWMutex

	byteCursor int
}

func NewImage(img draw.Image, gen PointsSequenceGenerator, prw PointReadWriter) *rwImage {
	return &rwImage{
		img: img,
		gen: gen,
		prw: prw,
	}
}

// Read implements io.Reader interface
func (i *rwImage) Read(p []byte) (n int, err error) {
	i.mux.RLock()
	defer i.mux.RUnlock()

	if !i.gen.Valid() {
		return 0, io.EOF
	}

	if len(p) == 0 {
		return 0, nil
	}

	for {
		if !i.gen.Valid() {
			return n, io.EOF
		}
		if n >= len(p) {
			return
		}

		point := i.gen.Current()
		color := i.img.At(point.X, point.Y)
		buff, nBytesRead := i.prw.Read(i.byteCursor, color, point)

		if len(p)-n >= nBytesRead {
			copy(p[n:], buff[:nBytesRead])
			n += nBytesRead
			i.gen.Next()
			i.byteCursor = 0
		} else {
			end := nBytesRead - len(p) + n
			copy(p[n:], buff[:end])
			n += end
			i.byteCursor = end + 1
			return
		}
	}

	return
}

var ErrOverflow = errors.New("Overflow")

// Write implements io.Writer interface
func (i *rwImage) Write(p []byte) (n int, err error) {
	if len(p) == 0 {
		return 0, nil
	}

	i.mux.Lock()
	defer i.mux.Unlock()

	if !i.gen.Valid() {
		return 0, ErrOverflow
	}

	for {
		if len(p) == 0 {
			return n, nil
		}
		if !i.gen.Valid() {
			return n, ErrOverflow
		}

		point := i.gen.Current()
		srcColor := i.img.At(point.X, point.Y)
		color, writtenBytes := i.prw.Write(p, i.byteCursor, srcColor, point)
		i.img.Set(point.X, point.Y, color)
		i.gen.Next()
		n += writtenBytes
		p = p[writtenBytes:]
		if i.prw.Size(point) > int64(writtenBytes) {
			i.byteCursor = writtenBytes + 1
		}
	}

	return
}

func (i *rwImage) Seek(offset int64, whence int) (int64, error) {
	return 0, nil
}

func (i *rwImage) Size() (size int64) {
	i.gen.Rewind()
	defer i.gen.Rewind()
	for i.gen.Valid() {
		point := i.gen.Current()
		size += i.prw.Size(point)
		i.gen.Next()
	}
	return
}

// ColorModel implements image.Image interface
func (i *rwImage) ColorModel() color.Model {
	return i.img.ColorModel()
}

// Bounds implements image.Image interface
func (i *rwImage) Bounds() image.Rectangle {
	return i.img.Bounds()
}

// At  implements image.Image interface
func (i *rwImage) At(x, y int) color.Color {
	return i.img.At(x, y)
}
<commit_msg>Create Image interface<commit_after>package imgio

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	"io"
	"sync"
)

type Image interface {
	io.ReadWriteSeeker
}

type rwImage struct {
	img draw.Image
	gen PointsSequenceGenerator
	prw PointReadWriter
	mux sync.RWMutex

	byteCursor int
}

func NewImage(img draw.Image, gen PointsSequenceGenerator, prw PointReadWriter) *rwImage {
	return &rwImage{
		img: img,
		gen: gen,
		prw: prw,
	}
}

// Read implements io.Reader interface
func (i *rwImage) Read(p []byte) (n int, err error) {
	i.mux.RLock()
	defer i.mux.RUnlock()

	if !i.gen.Valid() {
		return 0, io.EOF
	}

	if len(p) == 0 {
		return 0, nil
	}

	for {
		if !i.gen.Valid() {
			return n, io.EOF
		}
		if n >= len(p) {
			return
		}

		point := i.gen.Current()
		buff, nBytesRead := i.prw.Read(i.byteCursor, i.img.At(point.X, point.Y), point)

		if len(p)-n >= nBytesRead {
			copy(p[n:], buff[:nBytesRead])
			n += nBytesRead
			i.gen.Next()
			i.byteCursor = 0
		} else {
			end := nBytesRead - len(p) + n
			copy(p[n:], buff[:end])
			n += end
			i.byteCursor = end + 1
			return
		}
	}

	return
}

var ErrOverflow = errors.New("Overflow")

// Write implements io.Writer interface
func (i *rwImage) Write(p []byte) (n int, err error) {
	if len(p) == 0 {
		return 0, nil
	}

	i.mux.Lock()
	defer i.mux.Unlock()

	if !i.gen.Valid() {
		return 0, ErrOverflow
	}

	for {
		if len(p) == 0 {
			return n, nil
		}
		if !i.gen.Valid() {
			return n, ErrOverflow
		}

		point := i.gen.Current()
		srcColor := i.img.At(point.X, point.Y)
		c, writtenBytes := i.prw.Write(p, i.byteCursor, srcColor, point)
		i.img.Set(point.X, point.Y, c)
		i.gen.Next()
		n += writtenBytes
		p = p[writtenBytes:]
		if i.prw.Size(point) > int64(writtenBytes) {
			i.byteCursor = writtenBytes + 1
		}
	}

	return
}

func (i *rwImage) Seek(offset int64, whence int) (int64, error) {
	return 0, nil
}

func (i *rwImage) Size() (size int64) {
	i.gen.Rewind()
	defer i.gen.Rewind()
	for i.gen.Valid() {
		point := i.gen.Current()
		size += i.prw.Size(point)
		i.gen.Next()
	}
	return
}

// ColorModel implements image.Image interface
func (i *rwImage) ColorModel() color.Model {
	return i.img.ColorModel()
}

// Bounds implements image.Image interface
func (i *rwImage) Bounds() image.Rectangle {
	return i.img.Bounds()
}

// At  implements image.Image interface
func (i *rwImage) At(x, y int) color.Color {
	return i.img.At(x, y)
}
<|endoftext|>


<commit_before>package clw11

/*
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
*/
import "C"

type (
	ChannelOrder C.cl_channel_order
	ChannelType  C.cl_channel_type
)

const (
	R         ChannelOrder = C.CL_R
	A         ChannelOrder = C.CL_A
	RG        ChannelOrder = C.CL_RG
	RA        ChannelOrder = C.CL_RA
	RGB       ChannelOrder = C.CL_RGB
	RGBA      ChannelOrder = C.CL_RGBA
	BGRA      ChannelOrder = C.CL_BGRA
	ARGB      ChannelOrder = C.CL_ARGB
	Intensity ChannelOrder = C.CL_INTENSITY
	Luminance ChannelOrder = C.CL_LUMINANCE
	Rx        ChannelOrder = C.CL_Rx
	RGx       ChannelOrder = C.CL_RGx
	RGBx      ChannelOrder = C.CL_RGBx
)

const (
	SnormInt8      ChannelType = C.CL_SNORM_INT8
	SnormInt16     ChannelType = C.CL_SNORM_INT16
	UnormInt8      ChannelType = C.CL_UNORM_INT8
	UnormInt16     ChannelType = C.CL_UNORM_INT16
	UnormShort565  ChannelType = C.CL_UNORM_SHORT_565
	UnormShort555  ChannelType = C.CL_UNORM_SHORT_555
	UnormInt101010 ChannelType = C.CL_UNORM_INT_101010
	SignedInt8     ChannelType = C.CL_SIGNED_INT8
	SignedInt16    ChannelType = C.CL_SIGNED_INT16
	SignedInt32    ChannelType = C.CL_SIGNED_INT32
	UnsignedInt8   ChannelType = C.CL_UNSIGNED_INT8
	UnsignedInt16  ChannelType = C.CL_UNSIGNED_INT16
	UnsignedInt32  ChannelType = C.CL_UNSIGNED_INT32
	HalfFloat      ChannelType = C.CL_HALF_FLOAT
	Float          ChannelType = C.CL_FLOAT
)
<commit_msg>Initial image code.<commit_after>package clw11

/*
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
*/
import "C"
import "unsafe"

type (
	ImageFormat  C.cl_image_format
	ChannelOrder C.cl_channel_order
	ChannelType  C.cl_channel_type
)

const (
	R         ChannelOrder = C.CL_R
	A         ChannelOrder = C.CL_A
	RG        ChannelOrder = C.CL_RG
	RA        ChannelOrder = C.CL_RA
	RGB       ChannelOrder = C.CL_RGB
	RGBA      ChannelOrder = C.CL_RGBA
	BGRA      ChannelOrder = C.CL_BGRA
	ARGB      ChannelOrder = C.CL_ARGB
	Intensity ChannelOrder = C.CL_INTENSITY
	Luminance ChannelOrder = C.CL_LUMINANCE
	Rx        ChannelOrder = C.CL_Rx
	RGx       ChannelOrder = C.CL_RGx
	RGBx      ChannelOrder = C.CL_RGBx
)

const (
	SnormInt8      ChannelType = C.CL_SNORM_INT8
	SnormInt16     ChannelType = C.CL_SNORM_INT16
	UnormInt8      ChannelType = C.CL_UNORM_INT8
	UnormInt16     ChannelType = C.CL_UNORM_INT16
	UnormShort565  ChannelType = C.CL_UNORM_SHORT_565
	UnormShort555  ChannelType = C.CL_UNORM_SHORT_555
	UnormInt101010 ChannelType = C.CL_UNORM_INT_101010
	SignedInt8     ChannelType = C.CL_SIGNED_INT8
	SignedInt16    ChannelType = C.CL_SIGNED_INT16
	SignedInt32    ChannelType = C.CL_SIGNED_INT32
	UnsignedInt8   ChannelType = C.CL_UNSIGNED_INT8
	UnsignedInt16  ChannelType = C.CL_UNSIGNED_INT16
	UnsignedInt32  ChannelType = C.CL_UNSIGNED_INT32
	HalfFloat      ChannelType = C.CL_HALF_FLOAT
	Float          ChannelType = C.CL_FLOAT
)

func CreateImage2D(context Context, flags MemFlags, image_format *ImageFormat, image_width, image_height,
	image_row_pitch Size, host_ptr unsafe.Pointer) (Mem, error) {

	var err C.cl_int

	mem := C.clCreateImage2D(context, C.cl_mem_flags(flags), (*C.cl_image_format)(image_format), C.size_t(image_width),
		C.size_t(image_height), C.size_t(image_row_pitch), host_ptr, &err)

	return Mem(mem), toError(err)
}
<|endoftext|>


<commit_before>// build !ocr

package docconv

import (
	"fmt"
	"io"
)

func ConvertImage(r io.Reader) (string, map[string]string, error) {
	return "", nil, fmt.Errorf("docconv not built with `ocr` build tag")
}
<commit_msg>fix build<commit_after>// +build !ocr

package docconv

import (
	"fmt"
	"io"
)

func ConvertImage(r io.Reader) (string, map[string]string, error) {
	return "", nil, fmt.Errorf("docconv not built with `ocr` build tag")
}
<|endoftext|>


<commit_before>package parquet_go

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"reflect"
)

func BitNum(num uint64) uint64 { //the number of bits needed by the num; 0 needs 0, 1 need 1, 2 need 2, 3 need 2 ....
	var bitn int32 = 63
	for (bitn >= 0) && (((uint64(1) << uint32(bitn)) & num) == 0) {
		bitn--
	}
	return uint64(bitn + 1)
}

func ToInt64(nums []Interface) []int64 { //convert bool/int values to int64 values
	ln := len(nums)
	res := make([]int64, ln)
	tk := reflect.TypeOf(nums[0]).Kind()
	for i := 0; i < ln; i++ {
		if tk == reflect.Bool {
			if nums[i].(bool) {
				res[i] = 1
			} else {
				res[i] = 0
			}
		} else {
			res[i] = int64(reflect.ValueOf(nums[i]).Int())
		}

	}
	return res
}

func WritePlain(src []Interface) []byte {
	ln := len(src)
	if ln <= 0 {
		return []byte{}
	}

	dataType := reflect.TypeOf(src[0])

	if dataType.Kind() == reflect.Bool { //parquet.Type_BOOLEAN
		//		return WriteBitPacked(src, 1)
		srcTmp := make([]bool, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(bool)
		}
		return WriteBoolean(srcTmp)

	} else if dataType.Kind() == reflect.Int32 { //parquet.Type_INT32
		srcTmp := make([]int32, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(int32)
		}
		return WritePlainInt32(srcTmp)

	} else if dataType.Kind() == reflect.Int64 { //parquet.Type_INT64
		srcTmp := make([]int64, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(int64)
		}
		return WritePlainInt64(srcTmp)

	} else if dataType.Kind() == reflect.Float32 { // parquet.Type_FLOAT
		srcTmp := make([]float32, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(float32)
		}
		return WritePlainFloat32(srcTmp)

	} else if dataType.Kind() == reflect.Float64 { // parquet.Type_DOUBLE
		srcTmp := make([]float64, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(float64)
		}
		return WritePlainFloat64(srcTmp)

	} else if dataType.Kind() == reflect.String { // parquet.Type_BYTE_ARRAY
		srcTmp := make([][]byte, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = []byte(src[i].(string))
		}
		return WritePlainByteArray(srcTmp)

	} else if dataType.Kind() == reflect.Slice && dataType.Name() == "INT96" { //parquet.INT96
		srcTmp := make([]INT96, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(INT96)
		}
		return WritePlainInt96(srcTmp)

	} else {
		return nil
	}
}

func WriteBOOLEAN(nums []BOOLEAN) []byte {
	ln := len(nums)
	byteNum := (ln + 7) / 8
	res := make([]byte, byteNum)
	for i := 0; i < ln; i++ {
		if nums[i] {
			res[i/8] = res[i/8] | (1 << uint32(i%8))
		}
	}
	return res
}

func WritePlainINT32(nums []INT32) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainINT64(nums []INT64) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*8)
	b.Read(res)
	return res
}

func WritePlainINT96(nums []INT96) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		for j := 0; j < len(nums[i]); j++ {
			binary.Write(bufWriter, binary.LittleEndian, &nums[i][j])
		}
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*12)
	b.Read(res)
	return res
}

func WritePlainFLOAT(nums []FLOAT) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainDOUBLE(nums []DOUBLE) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*8)
	b.Read(res)
	return res
}

func WritePlainBYTE_ARRAY(arrays []BYTE_ARRAY) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)

	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(arrays[i]))
		binary.Write(bufWriter, binary.LittleEndian, &ln)
		bufWriter.Write([]byte(arrays[i]))
		size += 4 + ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WritePlainFIXED_LEN_BYTE_ARRAY(arrays []FIXED_LEN_BYTE_ARRAY) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(arrays[i]))
		bufWriter.Write([]byte(arrays[i]))
		size += ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WriteUTF8(arrays []UTF8) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(arrays[i]))
		binary.Write(bufWriter, binary.LittleEndian, &ln)
		bufWriter.Write([]byte(arrays[i]))
		size += 4 + ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WriteINT_8(arrays []INT_8) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WriteUnsignedVarInt(num uint32) []byte {
	byteNum := (BitNum(uint64(num)) + 6) / 7
	if byteNum == 0 {
		return make([]byte, 1)
	}
	res := make([]byte, byteNum)

	numTmp := num
	for i := 0; i < int(byteNum); i++ {
		res[i] = byte(numTmp & uint32(0x7F))
		res[i] = res[i] | byte(0x80)
		numTmp = numTmp >> 7
	}
	res[byteNum-1] &= byte(0x7F)
	return res
}

func WriteRLE(val int32, cnt int32, bitWidth int32) []byte {
	header := cnt << 1
	byteNum := (bitWidth + 7) / 8

	headerBuf := WriteUnsignedVarInt(uint32(header))
	bufTmp := make([]int32, 1)
	bufTmp[0] = val
	valBuf := WritePlainInt32(bufTmp)

	res := make([]byte, int64(len(headerBuf))+int64(byteNum))
	copy(res[0:], headerBuf)
	copy(res[len(headerBuf):], valBuf[0:byteNum])

	return res
}

func WriteBitPacked(vals []Interface, bitWidth int64) []byte {
	ln := len(vals)
	if ln <= 0 {
		return nil
	}
	valsInt := ToInt64(vals)

	header := ((ln/8)<<1 | 1)
	headerBuf := WriteUnsignedVarInt(uint32(header))

	valBuf := make([]byte, 0)

	i := 0
	var resCur int64 = 0
	var resCurNeedBits int64 = 8
	var used int64 = 0
	var left int64 = bitWidth - used
	val := int64(valsInt[i])
	for i < ln {
		if left >= resCurNeedBits {
			resCur |= ((val >> uint64(used)) & ((1 << uint64(resCurNeedBits)) - 1)) << uint64(8-resCurNeedBits)
			valBuf = append(valBuf, byte(resCur))
			left -= resCurNeedBits
			used += resCurNeedBits

			resCurNeedBits = 8
			resCur = 0

			if left <= 0 && (i+1) < ln {
				i += 1
				val = int64(valsInt[i])
				left = bitWidth
				used = 0
			}

		} else {
			resCur |= (val >> uint64(used)) << uint64(8-resCurNeedBits)
			i += 1

			if i < ln {
				val = int64(valsInt[i])
			}
			resCurNeedBits -= left

			left = bitWidth
			used = 0
		}
	}

	res := make([]byte, 0)
	res = append(res, headerBuf...)
	res = append(res, valBuf...)
	return res
}
<commit_msg>adding EncodingWrite<commit_after>package parquet_go

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"reflect"
)

func BitNum(num uint64) uint64 { //the number of bits needed by the num; 0 needs 0, 1 need 1, 2 need 2, 3 need 2 ....
	var bitn int32 = 63
	for (bitn >= 0) && (((uint64(1) << uint32(bitn)) & num) == 0) {
		bitn--
	}
	return uint64(bitn + 1)
}

func ToInt64(nums []Interface) []int64 { //convert bool/int values to int64 values
	ln := len(nums)
	res := make([]int64, ln)
	tk := reflect.TypeOf(nums[0]).Kind()
	for i := 0; i < ln; i++ {
		if tk == reflect.Bool {
			if nums[i].(bool) {
				res[i] = 1
			} else {
				res[i] = 0
			}
		} else {
			res[i] = int64(reflect.ValueOf(nums[i]).Int())
		}

	}
	return res
}

func WritePlain(src []Interface) []byte {
	ln := len(src)
	if ln <= 0 {
		return []byte{}
	}

	dataType := reflect.TypeOf(src[0])

	if dataType.Kind() == reflect.Bool { //parquet.Type_BOOLEAN
		//		return WriteBitPacked(src, 1)
		srcTmp := make([]bool, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(bool)
		}
		return WriteBoolean(srcTmp)

	} else if dataType.Kind() == reflect.Int32 { //parquet.Type_INT32
		srcTmp := make([]int32, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(int32)
		}
		return WritePlainInt32(srcTmp)

	} else if dataType.Kind() == reflect.Int64 { //parquet.Type_INT64
		srcTmp := make([]int64, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(int64)
		}
		return WritePlainInt64(srcTmp)

	} else if dataType.Kind() == reflect.Float32 { // parquet.Type_FLOAT
		srcTmp := make([]float32, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(float32)
		}
		return WritePlainFloat32(srcTmp)

	} else if dataType.Kind() == reflect.Float64 { // parquet.Type_DOUBLE
		srcTmp := make([]float64, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(float64)
		}
		return WritePlainFloat64(srcTmp)

	} else if dataType.Kind() == reflect.String { // parquet.Type_BYTE_ARRAY
		srcTmp := make([][]byte, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = []byte(src[i].(string))
		}
		return WritePlainByteArray(srcTmp)

	} else if dataType.Kind() == reflect.Slice && dataType.Name() == "INT96" { //parquet.INT96
		srcTmp := make([]INT96, ln)
		for i := 0; i < ln; i++ {
			srcTmp[i] = src[i].(INT96)
		}
		return WritePlainInt96(srcTmp)

	} else {
		return nil
	}
}

func WritePlainBOOLEAN(nums []BOOLEAN) []byte {
	ln := len(nums)
	byteNum := (ln + 7) / 8
	res := make([]byte, byteNum)
	for i := 0; i < ln; i++ {
		if nums[i] {
			res[i/8] = res[i/8] | (1 << uint32(i%8))
		}
	}
	return res
}

func WritePlainINT32(nums []INT32) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainINT64(nums []INT64) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*8)
	b.Read(res)
	return res
}

func WritePlainINT96(nums []INT96) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		for j := 0; j < len(nums[i]); j++ {
			binary.Write(bufWriter, binary.LittleEndian, &nums[i][j])
		}
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*12)
	b.Read(res)
	return res
}

func WritePlainINT_8(nums []INT_8) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainINT_16(nums []INT_16) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainINT_32(nums []INT_32) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainINT_64(nums []INT_64) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*8)
	b.Read(res)
	return res
}

func WritePlainUINT_8(nums []UINT_8) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainUINT_16(nums []UINT_16) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainUINT_32(nums []UINT_32) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainUINT_64(nums []UINT_64) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*8)
	b.Read(res)
	return res
}

func WritePlainFLOAT(nums []FLOAT) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*4)
	b.Read(res)
	return res
}

func WritePlainDOUBLE(nums []DOUBLE) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &nums[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(nums)*8)
	b.Read(res)
	return res
}

func WritePlainUTF8(utf8s []UTF8) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(utf8s[i]))
		binary.Write(bufWriter, binary.LittleEndian, &ln)
		bufWriter.Write([]byte(utf8s[i]))
		size += 4 + ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WritePlainDATE(dates []DATE) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &dates[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(dates)*4)
	b.Read(res)
	return res
}

func WritePlainTIME_MILLIS(times []TIME_MILLIS) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &times[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(times)*4)
	b.Read(res)
	return res
}

func WritePlainTIME_MICROS(times []TIME_MICROS) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &times[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(times)*8)
	b.Read(res)
	return res
}

func WritePlainTIMESTAMP_MILLIS(times []TIMESTAMP_MILLIS) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &times[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(times)*8)
	b.Read(res)
	return res
}

func WritePlainTIMESTAMP_MICROS(times []TIMESTAMP_MICROS) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	for i := 0; i < len(nums); i++ {
		binary.Write(bufWriter, binary.LittleEndian, &times[i])
	}
	bufWriter.Flush()
	res := make([]byte, len(times)*8)
	b.Read(res)
	return res
}

func WritePlainINTERVAL(intervals []INTERVAL) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(intervals[i]))
		bufWriter.Write([]byte(intervals[i]))
		size += ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WritePlainDECIMAL(decimals []DECIMAL) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(arrays[i]))
		binary.Write(bufWriter, binary.LittleEndian, &ln)
		bufWriter.Write([]byte(decimals[i]))
		size += 4 + ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WritePlainBYTE_ARRAY(arrays []BYTE_ARRAY) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(arrays[i]))
		binary.Write(bufWriter, binary.LittleEndian, &ln)
		bufWriter.Write([]byte(arrays[i]))
		size += 4 + ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WritePlainFIXED_LEN_BYTE_ARRAY(arrays []FIXED_LEN_BYTE_ARRAY) []byte {
	var b bytes.Buffer
	bufWriter := bufio.NewWriter(&b)
	var size uint32 = 0
	cnt := len(arrays)
	for i := 0; i < int(cnt); i++ {
		ln := uint32(len(arrays[i]))
		bufWriter.Write([]byte(arrays[i]))
		size += ln
	}
	bufWriter.Flush()
	res := make([]byte, size)
	b.Read(res)
	return res
}

func WriteUnsignedVarInt(num uint32) []byte {
	byteNum := (BitNum(uint64(num)) + 6) / 7
	if byteNum == 0 {
		return make([]byte, 1)
	}
	res := make([]byte, byteNum)

	numTmp := num
	for i := 0; i < int(byteNum); i++ {
		res[i] = byte(numTmp & uint32(0x7F))
		res[i] = res[i] | byte(0x80)
		numTmp = numTmp >> 7
	}
	res[byteNum-1] &= byte(0x7F)
	return res
}

func WriteRLE(val int32, cnt int32, bitWidth int32) []byte {
	header := cnt << 1
	byteNum := (bitWidth + 7) / 8

	headerBuf := WriteUnsignedVarInt(uint32(header))
	bufTmp := make([]int32, 1)
	bufTmp[0] = val
	valBuf := WritePlainInt32(bufTmp)

	res := make([]byte, int64(len(headerBuf))+int64(byteNum))
	copy(res[0:], headerBuf)
	copy(res[len(headerBuf):], valBuf[0:byteNum])

	return res
}

func WriteBitPacked(vals []Interface, bitWidth int64) []byte {
	ln := len(vals)
	if ln <= 0 {
		return nil
	}
	valsInt := ToInt64(vals)

	header := ((ln/8)<<1 | 1)
	headerBuf := WriteUnsignedVarInt(uint32(header))

	valBuf := make([]byte, 0)

	i := 0
	var resCur int64 = 0
	var resCurNeedBits int64 = 8
	var used int64 = 0
	var left int64 = bitWidth - used
	val := int64(valsInt[i])
	for i < ln {
		if left >= resCurNeedBits {
			resCur |= ((val >> uint64(used)) & ((1 << uint64(resCurNeedBits)) - 1)) << uint64(8-resCurNeedBits)
			valBuf = append(valBuf, byte(resCur))
			left -= resCurNeedBits
			used += resCurNeedBits

			resCurNeedBits = 8
			resCur = 0

			if left <= 0 && (i+1) < ln {
				i += 1
				val = int64(valsInt[i])
				left = bitWidth
				used = 0
			}

		} else {
			resCur |= (val >> uint64(used)) << uint64(8-resCurNeedBits)
			i += 1

			if i < ln {
				val = int64(valsInt[i])
			}
			resCurNeedBits -= left

			left = bitWidth
			used = 0
		}
	}

	res := make([]byte, 0)
	res = append(res, headerBuf...)
	res = append(res, valBuf...)
	return res
}
<|endoftext|>


<commit_before>// Copyright 2011 Alexander Orlov <alexander.orlov@loxal.net>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commander

import (
	"testing"
	"http"

	"appengine"
	"appengine/datastore"
)

type webCmdTest struct {
	cmd, restCall string
}

var webCmdTests = []webCmdTest{
	webCmdTest{"c", "https://mail.google.com/mail/?shva=1#compose"},
	webCmdTest{"t", "http://twitter.com"},
	webCmdTest{"sem", "https://github.com/loxal/Sem"},
	webCmdTest{"verp", "https://github.com/loxal/Verp"},
	webCmdTest{"lox", "https://github.com/loxal/Lox"},
}

type cmdTest struct {
    name, restCall, desc string
}

var cmdTests = []cmdTest {
	cmdTest{"c", "https://mail.google.com/mail/?shva=1#compose", "Compose Gmail"},
	cmdTest{"t", "http://twitter.com", "Twitter"},
	cmdTest{"sem", "https://github.com/loxal/Sem", "GitHub: Sem Project"},
	cmdTest{"verp", "https://github.com/loxal/Verp", "GitHub: Verp Project"},
	cmdTest{"lox", "https://github.com/loxal/Lox", "GitHub: Lox Project"},
}

func TestCmdCreation(t *testing.T) {

    cmd := &Cmd {
        Name: "blub",
        RESTcall: "blab",
        Desc: "....",
    }

    c := appengine.NewContext(&http.Request{Header: make(http.Header)})
    c.Logf("%#v", c)
    c.Logf("%#v", cmd)
    datastore.Put(c, datastore.NewIncompleteKey("Cmd"), cmd)
}
<commit_msg>updated to API3<commit_after>// Copyright 2011 Alexander Orlov <alexander.orlov@loxal.net>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commander

import (
	"testing"
	"http"

	"appengine"
	"appengine/datastore"
)

type webCmdTest struct {
	cmd, restCall string
}

var webCmdTests = []webCmdTest{
	webCmdTest{"c", "https://mail.google.com/mail/?shva=1#compose"},
	webCmdTest{"t", "http://twitter.com"},
	webCmdTest{"sem", "https://github.com/loxal/Sem"},
	webCmdTest{"verp", "https://github.com/loxal/Verp"},
	webCmdTest{"lox", "https://github.com/loxal/Lox"},
}

type cmdTest struct {
	name, restCall, desc string
}

var cmdTests = []cmdTest{
	cmdTest{"c", "https://mail.google.com/mail/?shva=1#compose", "Compose Gmail"},
	cmdTest{"t", "http://twitter.com", "Twitter"},
	cmdTest{"sem", "https://github.com/loxal/Sem", "GitHub: Sem Project"},
	cmdTest{"verp", "https://github.com/loxal/Verp", "GitHub: Verp Project"},
	cmdTest{"lox", "https://github.com/loxal/Lox", "GitHub: Lox Project"},
}

func TestCmdCreation(t *testing.T) {

	cmd := &Cmd{
		Name:     "blub",
		RESTcall: "blab",
		Desc:     "....",
	}

	c := appengine.NewContext(&http.Request{Header: make(http.Header)})
	c.Logf("%#v", c)
	c.Logf("%#v", cmd)
	datastore.Put(c, datastore.NewIncompleteKey(c, "Cmd", nil), cmd)
}
<|endoftext|>


<commit_before>package images // import "github.com/docker/docker/daemon/images"

import (
	"time"

	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/system"
	"github.com/pkg/errors"
)

// LookupImage looks up an image by name and returns it as an ImageInspect
// structure.
func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) {
	img, err := i.GetImage(name, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "no such image: %s", name)
	}
	if !system.IsOSSupported(img.OperatingSystem()) {
		return nil, system.ErrNotSupportedOperatingSystem
	}
	refs := i.referenceStore.References(img.ID().Digest())
	repoTags := []string{}
	repoDigests := []string{}
	for _, ref := range refs {
		switch ref.(type) {
		case reference.NamedTagged:
			repoTags = append(repoTags, reference.FamiliarString(ref))
		case reference.Canonical:
			repoDigests = append(repoDigests, reference.FamiliarString(ref))
		}
	}

	var size int64
	var layerMetadata map[string]string
	layerID := img.RootFS.ChainID()
	if layerID != "" {
		l, err := i.layerStore.Get(layerID)
		if err != nil {
			return nil, err
		}
		defer layer.ReleaseAndLog(i.layerStore, l)
		size = l.Size()
		layerMetadata, err = l.Metadata()
		if err != nil {
			return nil, err
		}
	}

	comment := img.Comment
	if len(comment) == 0 && len(img.History) > 0 {
		comment = img.History[len(img.History)-1].Comment
	}

	lastUpdated, err := i.imageStore.GetLastUpdated(img.ID())
	if err != nil {
		return nil, err
	}

	imageInspect := &types.ImageInspect{
		ID:              img.ID().String(),
		RepoTags:        repoTags,
		RepoDigests:     repoDigests,
		Parent:          img.Parent.String(),
		Comment:         comment,
		Created:         img.Created.Format(time.RFC3339Nano),
		Container:       img.Container,
		ContainerConfig: &img.ContainerConfig,
		DockerVersion:   img.DockerVersion,
		Author:          img.Author,
		Config:          img.Config,
		Architecture:    img.Architecture,
		Variant:         img.Variant,
		Os:              img.OperatingSystem(),
		OsVersion:       img.OSVersion,
		Size:            size,
		VirtualSize:     size, // TODO: field unused, deprecate
		RootFS:          rootFSToAPIType(img.RootFS),
		Metadata: types.ImageMetadata{
			LastTagTime: lastUpdated,
		},
	}

	imageInspect.GraphDriver.Name = i.layerStore.DriverName()
	imageInspect.GraphDriver.Data = layerMetadata

	return imageInspect, nil
}

func rootFSToAPIType(rootfs *image.RootFS) types.RootFS {
	var layers []string
	for _, l := range rootfs.DiffIDs {
		layers = append(layers, l.String())
	}
	return types.RootFS{
		Type:   rootfs.Type,
		Layers: layers,
	}
}
<commit_msg>daemon/images: ImageService.LookupImage(): minor cleanup<commit_after>package images // import "github.com/docker/docker/daemon/images"

import (
	"time"

	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/system"
	"github.com/pkg/errors"
)

// LookupImage looks up an image by name and returns it as an ImageInspect
// structure.
func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) {
	img, err := i.GetImage(name, nil)
	if err != nil {
		return nil, errors.Wrapf(err, "no such image: %s", name)
	}
	if !system.IsOSSupported(img.OperatingSystem()) {
		return nil, system.ErrNotSupportedOperatingSystem
	}
	refs := i.referenceStore.References(img.ID().Digest())
	repoTags := []string{}
	repoDigests := []string{}
	for _, ref := range refs {
		switch ref.(type) {
		case reference.NamedTagged:
			repoTags = append(repoTags, reference.FamiliarString(ref))
		case reference.Canonical:
			repoDigests = append(repoDigests, reference.FamiliarString(ref))
		}
	}

	var size int64
	var layerMetadata map[string]string
	layerID := img.RootFS.ChainID()
	if layerID != "" {
		l, err := i.layerStore.Get(layerID)
		if err != nil {
			return nil, err
		}
		defer layer.ReleaseAndLog(i.layerStore, l)
		size = l.Size()
		layerMetadata, err = l.Metadata()
		if err != nil {
			return nil, err
		}
	}

	comment := img.Comment
	if len(comment) == 0 && len(img.History) > 0 {
		comment = img.History[len(img.History)-1].Comment
	}

	lastUpdated, err := i.imageStore.GetLastUpdated(img.ID())
	if err != nil {
		return nil, err
	}

	return &types.ImageInspect{
		ID:              img.ID().String(),
		RepoTags:        repoTags,
		RepoDigests:     repoDigests,
		Parent:          img.Parent.String(),
		Comment:         comment,
		Created:         img.Created.Format(time.RFC3339Nano),
		Container:       img.Container,
		ContainerConfig: &img.ContainerConfig,
		DockerVersion:   img.DockerVersion,
		Author:          img.Author,
		Config:          img.Config,
		Architecture:    img.Architecture,
		Variant:         img.Variant,
		Os:              img.OperatingSystem(),
		OsVersion:       img.OSVersion,
		Size:            size,
		VirtualSize:     size, // TODO: field unused, deprecate
		GraphDriver: types.GraphDriverData{
			Name: i.layerStore.DriverName(),
			Data: layerMetadata,
		},
		RootFS: rootFSToAPIType(img.RootFS),
		Metadata: types.ImageMetadata{
			LastTagTime: lastUpdated,
		},
	}, nil
}

func rootFSToAPIType(rootfs *image.RootFS) types.RootFS {
	var layers []string
	for _, l := range rootfs.DiffIDs {
		layers = append(layers, l.String())
	}
	return types.RootFS{
		Type:   rootfs.Type,
		Layers: layers,
	}
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

// For ease of implementation, our database's interface matches Ethereum's
// database implementation. This was to allow use to use Geth code as is for the
// EVM chain.

package database

// Batch is a write-only database that commits changes to its host database
// when Write is called. A batch cannot be used concurrently.
type Batch interface {
	KeyValueWriter

	// Size retrieves the amount of data queued up for writing.
	Size() int

	// Write flushes any accumulated data to disk.
	Write() error

	// Reset resets the batch for reuse.
	Reset()

	// Replay replays the batch contents.
	Replay(w KeyValueWriter) error

	// Inner returns a Batch writing to the inner database, if one exists. If
	// this batch is already writing to the base DB, then itself should be
	// returned.
	Inner() Batch
}

// Batcher wraps the NewBatch method of a backing data store.
type Batcher interface {
	// NewBatch creates a write-only database that buffers changes to its host db
	// until a final write is called.
	NewBatch() Batch
}
<commit_msg>added comment to descript batch size better<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

// For ease of implementation, our database's interface matches Ethereum's
// database implementation. This was to allow use to use Geth code as is for the
// EVM chain.

package database

// Batch is a write-only database that commits changes to its host database
// when Write is called. A batch cannot be used concurrently.
type Batch interface {
	KeyValueWriter

	// Size retrieves the amount of data queued up for writing, this includes
	// the keys, values, and deleted keys.
	Size() int

	// Write flushes any accumulated data to disk.
	Write() error

	// Reset resets the batch for reuse.
	Reset()

	// Replay replays the batch contents.
	Replay(w KeyValueWriter) error

	// Inner returns a Batch writing to the inner database, if one exists. If
	// this batch is already writing to the base DB, then itself should be
	// returned.
	Inner() Batch
}

// Batcher wraps the NewBatch method of a backing data store.
type Batcher interface {
	// NewBatch creates a write-only database that buffers changes to its host db
	// until a final write is called.
	NewBatch() Batch
}
<|endoftext|>


<commit_before>package datalayer

import (
	"encoding/json"
	"os"
	"path/filepath"
	"regexp"
)

const MAX_VOLUME_NAME_LENGTH int = 40

// ClusterHQ data layer, naive vfs (directory-based) implementation

func ValidVolumeName(volumeName string) bool {
	var validVolumeRegex = regexp.MustCompile(`^[a-zA-Z]+[a-zA-Z0-9-]*$`)
	return validVolumeRegex.MatchString(volumeName) && len(volumeName) <= MAX_VOLUME_NAME_LENGTH
}

func VolumeExists(basePath string, volumeName string) bool {
	volumePath := filepath.FromSlash(basePath + "/" + volumeName)
	_, err := os.Stat(volumePath)
	return err == nil
}

func setActiveVolume(basePath, volumeName string) error {
	currentVolumeJsonPath := filepath.FromSlash(basePath + "/" + "current_volume.json")
	currentVolumeContent := make(map[string]string)
	currentVolumeContent["current_volume"] = volumeName
	// Create or update this file
	file, err := os.Create(currentVolumeJsonPath)
	if err != nil {
		return err
	}
	encoder := json.NewEncoder(file)
	encoder.Encode(currentVolumeContent)
	file.Sync()
	return nil
}

func CreateVolume(basePath string, volumeName string) error {
	volumePath := filepath.FromSlash(basePath + "/" + volumeName)
	// TODO Factor this into a data layer object.
	os.MkdirAll(volumePath, 0777) // XXX SEC
	err := setActiveVolume(basePath, volumeName)
	if err != nil {
		return err
	}
	return nil
}
func CreateVariant(basePath, volumeName, variantName string) error {
	// XXX Variants are meant to be tagged commits???
	variantPath := filepath.FromSlash(basePath + "/" + volumeName + "/branches/master")
	os.MkdirAll(variantPath, 0777) // XXX SEC
	return nil
}

func SwitchVolume(basePath, volumeName string) error {
	return nil
}
<commit_msg>Use composite literal for map instead of making and assigning<commit_after>package datalayer

import (
	"encoding/json"
	"os"
	"path/filepath"
	"regexp"
)

const MAX_VOLUME_NAME_LENGTH int = 40

// ClusterHQ data layer, naive vfs (directory-based) implementation

func ValidVolumeName(volumeName string) bool {
	var validVolumeRegex = regexp.MustCompile(`^[a-zA-Z]+[a-zA-Z0-9-]*$`)
	return validVolumeRegex.MatchString(volumeName) && len(volumeName) <= MAX_VOLUME_NAME_LENGTH
}

func VolumeExists(basePath string, volumeName string) bool {
	volumePath := filepath.FromSlash(basePath + "/" + volumeName)
	_, err := os.Stat(volumePath)
	return err == nil
}

func setActiveVolume(basePath, volumeName string) error {
	currentVolumeJsonPath := filepath.FromSlash(basePath + "/" + "current_volume.json")
	currentVolumeContent := map[string]string{
		"current_volume": volumeName,
	}
	// Create or update this file
	file, err := os.Create(currentVolumeJsonPath)
	if err != nil {
		return err
	}
	encoder := json.NewEncoder(file)
	encoder.Encode(currentVolumeContent)
	file.Sync()
	return nil
}

func CreateVolume(basePath string, volumeName string) error {
	volumePath := filepath.FromSlash(basePath + "/" + volumeName)
	// TODO Factor this into a data layer object.
	os.MkdirAll(volumePath, 0777) // XXX SEC
	err := setActiveVolume(basePath, volumeName)
	if err != nil {
		return err
	}
	return nil
}
func CreateVariant(basePath, volumeName, variantName string) error {
	// XXX Variants are meant to be tagged commits???
	variantPath := filepath.FromSlash(basePath + "/" + volumeName + "/branches/master")
	os.MkdirAll(variantPath, 0777) // XXX SEC
	return nil
}

func SwitchVolume(basePath, volumeName string) error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func serve() {
	http.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		username := r.Form.Get("username")
		if !isValidUsername(username) {
			// BAD: a request parameter is incorporated without validation into the response
			fmt.Fprintf(w, "%q is an unknown user", username)
		} else {
			// TODO: do something exciting
		}
	})
	http.ListenAndServe(":80", nil)
}

func encode(s string) ([]byte, error) {

	return json.Marshal(s)

}

func ServeJsonIndirect(w http.ResponseWriter, r http.Request) {

	tainted := r.Header.Get("Origin")
	noLongerTainted, _ := encode(tainted)
	w.Write(noLongerTainted)

}

func ServeJsonDirect(w http.ResponseWriter, r http.Request) {

	tainted := r.Header.Get("Origin")
	noLongerTainted, _ := json.Marshal(tainted)
	w.Write(noLongerTainted)

}

func ErrTest(w http.ResponseWriter, r http.Request) {

	cookie, err := r.Cookie("somecookie")
	w.Write([]byte(fmt.Sprintf("Cookie result: %v", cookie)))   // BAD: Cookie's value is user-controlled
	w.Write([]byte(fmt.Sprintf("Cookie check error: %v", err))) // GOOD: Cookie's err return is harmless

	file, header, err := r.FormFile("someFile")
	content, err2 := ioutil.ReadAll(file)
	w.Write([]byte(fmt.Sprintf("File content: %v", content)))      // BAD: file content is user-controlled
	w.Write([]byte(fmt.Sprintf("File name: %v", header.Filename))) // BAD: file header is user-controlled
	w.Write([]byte(fmt.Sprintf("FormFile error: %v", err)))        // GOOD: FormFile's err return is harmless
	w.Write([]byte(fmt.Sprintf("FormFile error: %v", err2)))       // GOOD: ReadAll's err return is harmless

	reader, err := r.MultipartReader()
	part, err2 := reader.NextPart()
	partName := part.FileName()
	byteSlice := make([]byte, 100)
	part.Read(byteSlice)

	w.Write([]byte(fmt.Sprintf("Part name: %v", partName)))         // BAD: part name is user-controlled
	w.Write(byteSlice)                                              // BAD: part contents are user-controlled
	w.Write([]byte(fmt.Sprintf("MultipartReader error: %v", err)))  // GOOD: MultipartReader's err return is harmless
	w.Write([]byte(fmt.Sprintf("MultipartReader error: %v", err2))) // GOOD: NextPart's err return is harmless

}
<commit_msg>Add test case for map extraction<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func serve() {
	http.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		username := r.Form.Get("username")
		if !isValidUsername(username) {
			// BAD: a request parameter is incorporated without validation into the response
			fmt.Fprintf(w, "%q is an unknown user", username)
		} else {
			// TODO: do something exciting
		}
	})
	http.ListenAndServe(":80", nil)
}

func encode(s string) ([]byte, error) {

	return json.Marshal(s)

}

func ServeJsonIndirect(w http.ResponseWriter, r http.Request) {

	tainted := r.Header.Get("Origin")
	noLongerTainted, _ := encode(tainted)
	w.Write(noLongerTainted)

}

func ServeJsonDirect(w http.ResponseWriter, r http.Request) {

	tainted := r.Header.Get("Origin")
	noLongerTainted, _ := json.Marshal(tainted)
	w.Write(noLongerTainted)

}

func ErrTest(w http.ResponseWriter, r http.Request) {

	cookie, err := r.Cookie("somecookie")
	w.Write([]byte(fmt.Sprintf("Cookie result: %v", cookie)))   // BAD: Cookie's value is user-controlled
	w.Write([]byte(fmt.Sprintf("Cookie check error: %v", err))) // GOOD: Cookie's err return is harmless

	file, header, err := r.FormFile("someFile")
	content, err2 := ioutil.ReadAll(file)
	w.Write([]byte(fmt.Sprintf("File content: %v", content)))      // BAD: file content is user-controlled
	w.Write([]byte(fmt.Sprintf("File name: %v", header.Filename))) // BAD: file header is user-controlled
	w.Write([]byte(fmt.Sprintf("FormFile error: %v", err)))        // GOOD: FormFile's err return is harmless
	w.Write([]byte(fmt.Sprintf("FormFile error: %v", err2)))       // GOOD: ReadAll's err return is harmless

	reader, err := r.MultipartReader()
	part, err2 := reader.NextPart()
	partName := part.FileName()
	byteSlice := make([]byte, 100)
	part.Read(byteSlice)

	w.Write([]byte(fmt.Sprintf("Part name: %v", partName)))         // BAD: part name is user-controlled
	w.Write(byteSlice)                                              // BAD: part contents are user-controlled
	w.Write([]byte(fmt.Sprintf("MultipartReader error: %v", err)))  // GOOD: MultipartReader's err return is harmless
	w.Write([]byte(fmt.Sprintf("MultipartReader error: %v", err2))) // GOOD: NextPart's err return is harmless
}

func QueryMapTest(w http.ResponseWriter, r http.Request) {
	keys, ok := r.URL.Query()["data_id"]
	if ok && len(keys[0]) > 0 {
		key := keys[0]
		w.Write([]byte(key))
	}
}
<|endoftext|>


<commit_before>package cluster

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"
)

const (
	// Force-refresh the state of the node this often.
	stateRefreshPeriod = 30 * time.Second

	// Timeout for requests sent out to the node.
	requestTimeout = 10 * time.Second
)

func NewNode(addr string) *Node {
	e := &Node{
		Addr:       addr,
		Labels:     make(map[string]string),
		ch:         make(chan bool),
		containers: make(map[string]*Container),
		healthy:    true,
	}
	return e
}

type Node struct {
	sync.RWMutex

	ID     string
	IP     string
	Addr   string
	Name   string
	Cpus   int64
	Memory int64
	Labels map[string]string

	ch           chan bool
	containers   map[string]*Container
	client       dockerclient.Client
	eventHandler EventHandler
	healthy      bool
}

// Connect will initialize a connection to the Docker daemon running on the
// host, gather machine specs (memory, cpu, ...) and monitor state changes.
func (n *Node) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClientTimeout(n.Addr, config, time.Duration(requestTimeout))
	if err != nil {
		return err
	}

	addr, err := net.ResolveIPAddr("ip4", strings.Split(c.URL.Host, ":")[0])
	if err != nil {
		return err
	}
	n.IP = addr.IP.String()

	return n.connectClient(c)
}

func (n *Node) connectClient(client dockerclient.Client) error {
	n.client = client

	// Fetch the engine labels.
	if err := n.updateSpecs(); err != nil {
		n.client = nil
		return err
	}

	// Force a state update before returning.
	if err := n.refreshContainers(); err != nil {
		n.client = nil
		return err
	}

	// Start the update loop.
	go n.refreshLoop()

	// Start monitoring events from the Node.
	n.client.StartMonitorEvents(n.handler)

	return nil
}

// IsConnected returns true if the engine is connected to a remote docker API
func (n *Node) IsConnected() bool {
	return n.client != nil
}

func (n *Node) IsHealthy() bool {
	return n.healthy
}

// Gather node specs (CPU, memory, constraints, ...).
func (n *Node) updateSpecs() error {
	info, err := n.client.Info()
	if err != nil {
		return err
	}
	// Older versions of Docker don't expose the ID field and are not supported
	// by Swarm.  Catch the error ASAP and refuse to connect.
	if len(info.ID) == 0 {
		return fmt.Errorf("Node %s is running an unsupported version of Docker Engine. Please upgrade.", n.Addr)
	}
	n.ID = info.ID
	n.Name = info.Name
	n.Cpus = info.NCPU
	n.Memory = info.MemTotal
	n.Labels = map[string]string{
		"storagedriver":   info.Driver,
		"executiondriver": info.ExecutionDriver,
		"kernelversion":   info.KernelVersion,
		"operatingsystem": info.OperatingSystem,
	}
	for _, label := range info.Labels {
		kv := strings.SplitN(label, "=", 2)
		n.Labels[kv[0]] = kv[1]
	}
	return nil
}

// Refresh the list and status of containers running on the node.
func (n *Node) refreshContainers() error {
	containers, err := n.client.ListContainers(true, false, "")
	if err != nil {
		return err
	}

	n.Lock()
	defer n.Unlock()

	merged := make(map[string]*Container)
	for _, c := range containers {
		merged, err = n.updateContainer(c, merged)
		if err != nil {
			log.Errorf("[%s/%s] Unable to update state of %s", n.ID, n.Name, c.Id)
		}
	}

	n.containers = merged

	log.Debugf("[%s/%s] Updated state", n.ID, n.Name)
	return nil
}

// Refresh the status of a container running on the node.
func (n *Node) refreshContainer(ID string) error {
	containers, err := n.client.ListContainers(true, false, fmt.Sprintf("{%q:[%q]}", "id", ID))
	if err != nil {
		return err
	}

	if len(containers) > 1 {
		// We expect one container, if we get more than one, trigger a full refresh.
		return n.refreshContainers()
	}

	n.Lock()
	defer n.Unlock()

	if len(containers) == 0 {
		// The container doesn't exist on the node, remove it.
		delete(n.containers, ID)
		return nil
	}

	_, err = n.updateContainer(containers[0], n.containers)
	return err
}

func (n *Node) updateContainer(c dockerclient.Container, containers map[string]*Container) (map[string]*Container, error) {
	if current, exists := n.containers[c.Id]; exists {
		// The container exists. Update its state.
		current.Container = c
		containers[current.Id] = current
	} else {
		// This is a brand new container.
		container := &Container{}
		container.Container = c
		container.node = n

		info, err := n.client.InspectContainer(c.Id)
		if err != nil {
			return containers, err
		}
		container.Info = *info

		// real CpuShares -> nb of CPUs
		container.Info.Config.CpuShares = container.Info.Config.CpuShares / 100.0 * n.Cpus

		containers[container.Id] = container
	}
	return containers, nil
}

func (n *Node) refreshContainersAsync() {
	n.ch <- true
}

func (n *Node) refreshLoop() {
	for {
		var err error
		select {
		case <-n.ch:
			err = n.refreshContainers()
		case <-time.After(stateRefreshPeriod):
			err = n.refreshContainers()
		}

		if err != nil {
			n.healthy = false
			log.Errorf("[%s/%s] Flagging node as dead. Updated state failed: %v", n.ID, n.Name, err)
		} else {
			if !n.healthy {
				log.Infof("[%s/%s] Node came back to life. Hooray!", n.ID, n.Name)
			}
			n.healthy = true
		}
	}
}

// Return the sum of memory reserved by containers.
func (n *Node) ReservedMemory() int64 {
	var r int64 = 0
	n.RLock()
	for _, c := range n.containers {
		r += c.Info.Config.Memory
	}
	n.RUnlock()
	return r
}

// Return the sum of CPUs reserved by containers.
func (n *Node) ReservedCpus() int64 {
	var r int64 = 0
	n.RLock()
	for _, c := range n.containers {
		r += c.Info.Config.CpuShares
	}
	n.RUnlock()
	return r
}

func (n *Node) Create(config *dockerclient.ContainerConfig, name string, pullImage bool) (*Container, error) {
	var (
		err    error
		id     string
		client = n.client
	)

	newConfig := *config

	// nb of CPUs -> real CpuShares
	newConfig.CpuShares = config.CpuShares * 100 / n.Cpus

	if id, err = client.CreateContainer(&newConfig, name); err != nil {
		// If the error is other than not found, abort immediately.
		if err != dockerclient.ErrNotFound {
			return nil, err
		}
		// Otherwise, try to pull the image...
		if err = n.Pull(config.Image); err != nil {
			return nil, err
		}
		// ...And try again.
		if id, err = client.CreateContainer(&newConfig, name); err != nil {
			return nil, err
		}
	}

	// Register the container immediately while waiting for a state refresh.
	// Force a state refresh to pick up the newly created container.
	n.refreshContainer(id)

	n.RLock()
	defer n.RUnlock()

	return n.containers[id], nil
}

func (n *Node) ListImages() ([]string, error) {
	images, err := n.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

// Destroy and remove a container from the node.
func (n *Node) Destroy(container *Container, force bool) error {
	if err := n.client.RemoveContainer(container.Id, force); err != nil {
		return err
	}

	// Remove the container from the state. Eventually, the state refresh loop
	// will rewrite this.
	n.Lock()
	defer n.Unlock()
	delete(n.containers, container.Id)

	return nil
}

func (n *Node) Pull(image string) error {
	if err := n.client.PullImage(image, nil); err != nil {
		return err
	}
	return nil
}

// Register an event handler.
func (n *Node) Events(h EventHandler) error {
	if n.eventHandler != nil {
		return errors.New("event handler already set")
	}
	n.eventHandler = h
	return nil
}

func (n *Node) Containers() []*Container {
	containers := []*Container{}
	n.RLock()
	for _, container := range n.containers {
		containers = append(containers, container)
	}
	n.RUnlock()
	return containers
}

func (n *Node) String() string {
	return fmt.Sprintf("node %s addr %s", n.ID, n.Addr)
}

func (n *Node) handler(ev *dockerclient.Event, args ...interface{}) {
	// Something changed - refresh our internal state.
	n.refreshContainer(ev.Id)

	// If there is no event handler registered, abort right now.
	if n.eventHandler == nil {
		return
	}

	event := &Event{
		Node: n,
	}
	event.Event = *ev

	n.eventHandler.Handle(event)
}

// Inject a container into the internal state.
func (n *Node) AddContainer(container *Container) error {
	n.Lock()
	defer n.Unlock()

	if _, ok := n.containers[container.Id]; ok {
		return errors.New("container already exists")
	}
	n.containers[container.Id] = container
	return nil
}

// Remove a container from the internal test.
func (n *Node) RemoveContainer(container *Container) error {
	n.Lock()
	defer n.Unlock()

	if _, ok := n.containers[container.Id]; !ok {
		return errors.New("container not found")
	}
	delete(n.containers, container.Id)
	return nil
}

// Wipes the internal container state.
func (n *Node) CleanupContainers() {
	n.Lock()
	n.containers = make(map[string]*Container)
	n.Unlock()
}
<commit_msg>make use of the "pullImage" parameter of node.Create method<commit_after>package cluster

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"
)

const (
	// Force-refresh the state of the node this often.
	stateRefreshPeriod = 30 * time.Second

	// Timeout for requests sent out to the node.
	requestTimeout = 10 * time.Second
)

func NewNode(addr string) *Node {
	e := &Node{
		Addr:       addr,
		Labels:     make(map[string]string),
		ch:         make(chan bool),
		containers: make(map[string]*Container),
		healthy:    true,
	}
	return e
}

type Node struct {
	sync.RWMutex

	ID     string
	IP     string
	Addr   string
	Name   string
	Cpus   int64
	Memory int64
	Labels map[string]string

	ch           chan bool
	containers   map[string]*Container
	client       dockerclient.Client
	eventHandler EventHandler
	healthy      bool
}

// Connect will initialize a connection to the Docker daemon running on the
// host, gather machine specs (memory, cpu, ...) and monitor state changes.
func (n *Node) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClientTimeout(n.Addr, config, time.Duration(requestTimeout))
	if err != nil {
		return err
	}

	addr, err := net.ResolveIPAddr("ip4", strings.Split(c.URL.Host, ":")[0])
	if err != nil {
		return err
	}
	n.IP = addr.IP.String()

	return n.connectClient(c)
}

func (n *Node) connectClient(client dockerclient.Client) error {
	n.client = client

	// Fetch the engine labels.
	if err := n.updateSpecs(); err != nil {
		n.client = nil
		return err
	}

	// Force a state update before returning.
	if err := n.refreshContainers(); err != nil {
		n.client = nil
		return err
	}

	// Start the update loop.
	go n.refreshLoop()

	// Start monitoring events from the Node.
	n.client.StartMonitorEvents(n.handler)

	return nil
}

// IsConnected returns true if the engine is connected to a remote docker API
func (n *Node) IsConnected() bool {
	return n.client != nil
}

func (n *Node) IsHealthy() bool {
	return n.healthy
}

// Gather node specs (CPU, memory, constraints, ...).
func (n *Node) updateSpecs() error {
	info, err := n.client.Info()
	if err != nil {
		return err
	}
	// Older versions of Docker don't expose the ID field and are not supported
	// by Swarm.  Catch the error ASAP and refuse to connect.
	if len(info.ID) == 0 {
		return fmt.Errorf("Node %s is running an unsupported version of Docker Engine. Please upgrade.", n.Addr)
	}
	n.ID = info.ID
	n.Name = info.Name
	n.Cpus = info.NCPU
	n.Memory = info.MemTotal
	n.Labels = map[string]string{
		"storagedriver":   info.Driver,
		"executiondriver": info.ExecutionDriver,
		"kernelversion":   info.KernelVersion,
		"operatingsystem": info.OperatingSystem,
	}
	for _, label := range info.Labels {
		kv := strings.SplitN(label, "=", 2)
		n.Labels[kv[0]] = kv[1]
	}
	return nil
}

// Refresh the list and status of containers running on the node.
func (n *Node) refreshContainers() error {
	containers, err := n.client.ListContainers(true, false, "")
	if err != nil {
		return err
	}

	n.Lock()
	defer n.Unlock()

	merged := make(map[string]*Container)
	for _, c := range containers {
		merged, err = n.updateContainer(c, merged)
		if err != nil {
			log.Errorf("[%s/%s] Unable to update state of %s", n.ID, n.Name, c.Id)
		}
	}

	n.containers = merged

	log.Debugf("[%s/%s] Updated state", n.ID, n.Name)
	return nil
}

// Refresh the status of a container running on the node.
func (n *Node) refreshContainer(ID string) error {
	containers, err := n.client.ListContainers(true, false, fmt.Sprintf("{%q:[%q]}", "id", ID))
	if err != nil {
		return err
	}

	if len(containers) > 1 {
		// We expect one container, if we get more than one, trigger a full refresh.
		return n.refreshContainers()
	}

	n.Lock()
	defer n.Unlock()

	if len(containers) == 0 {
		// The container doesn't exist on the node, remove it.
		delete(n.containers, ID)
		return nil
	}

	_, err = n.updateContainer(containers[0], n.containers)
	return err
}

func (n *Node) updateContainer(c dockerclient.Container, containers map[string]*Container) (map[string]*Container, error) {
	if current, exists := n.containers[c.Id]; exists {
		// The container exists. Update its state.
		current.Container = c
		containers[current.Id] = current
	} else {
		// This is a brand new container.
		container := &Container{}
		container.Container = c
		container.node = n

		info, err := n.client.InspectContainer(c.Id)
		if err != nil {
			return containers, err
		}
		container.Info = *info

		// real CpuShares -> nb of CPUs
		container.Info.Config.CpuShares = container.Info.Config.CpuShares / 100.0 * n.Cpus

		containers[container.Id] = container
	}
	return containers, nil
}

func (n *Node) refreshContainersAsync() {
	n.ch <- true
}

func (n *Node) refreshLoop() {
	for {
		var err error
		select {
		case <-n.ch:
			err = n.refreshContainers()
		case <-time.After(stateRefreshPeriod):
			err = n.refreshContainers()
		}

		if err != nil {
			n.healthy = false
			log.Errorf("[%s/%s] Flagging node as dead. Updated state failed: %v", n.ID, n.Name, err)
		} else {
			if !n.healthy {
				log.Infof("[%s/%s] Node came back to life. Hooray!", n.ID, n.Name)
			}
			n.healthy = true
		}
	}
}

// Return the sum of memory reserved by containers.
func (n *Node) ReservedMemory() int64 {
	var r int64 = 0
	n.RLock()
	for _, c := range n.containers {
		r += c.Info.Config.Memory
	}
	n.RUnlock()
	return r
}

// Return the sum of CPUs reserved by containers.
func (n *Node) ReservedCpus() int64 {
	var r int64 = 0
	n.RLock()
	for _, c := range n.containers {
		r += c.Info.Config.CpuShares
	}
	n.RUnlock()
	return r
}

func (n *Node) Create(config *dockerclient.ContainerConfig, name string, pullImage bool) (*Container, error) {
	var (
		err    error
		id     string
		client = n.client
	)

	newConfig := *config

	// nb of CPUs -> real CpuShares
	newConfig.CpuShares = config.CpuShares * 100 / n.Cpus

	if id, err = client.CreateContainer(&newConfig, name); err != nil {
		// If the error is other than not found, abort immediately.
		if err != dockerclient.ErrNotFound || !pullImage {
			return nil, err
		}
		// Otherwise, try to pull the image...
		if err = n.Pull(config.Image); err != nil {
			return nil, err
		}
		// ...And try again.
		if id, err = client.CreateContainer(&newConfig, name); err != nil {
			return nil, err
		}
	}

	// Register the container immediately while waiting for a state refresh.
	// Force a state refresh to pick up the newly created container.
	n.refreshContainer(id)

	n.RLock()
	defer n.RUnlock()

	return n.containers[id], nil
}

func (n *Node) ListImages() ([]string, error) {
	images, err := n.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

// Destroy and remove a container from the node.
func (n *Node) Destroy(container *Container, force bool) error {
	if err := n.client.RemoveContainer(container.Id, force); err != nil {
		return err
	}

	// Remove the container from the state. Eventually, the state refresh loop
	// will rewrite this.
	n.Lock()
	defer n.Unlock()
	delete(n.containers, container.Id)

	return nil
}

func (n *Node) Pull(image string) error {
	if err := n.client.PullImage(image, nil); err != nil {
		return err
	}
	return nil
}

// Register an event handler.
func (n *Node) Events(h EventHandler) error {
	if n.eventHandler != nil {
		return errors.New("event handler already set")
	}
	n.eventHandler = h
	return nil
}

func (n *Node) Containers() []*Container {
	containers := []*Container{}
	n.RLock()
	for _, container := range n.containers {
		containers = append(containers, container)
	}
	n.RUnlock()
	return containers
}

func (n *Node) String() string {
	return fmt.Sprintf("node %s addr %s", n.ID, n.Addr)
}

func (n *Node) handler(ev *dockerclient.Event, args ...interface{}) {
	// Something changed - refresh our internal state.
	n.refreshContainer(ev.Id)

	// If there is no event handler registered, abort right now.
	if n.eventHandler == nil {
		return
	}

	event := &Event{
		Node: n,
	}
	event.Event = *ev

	n.eventHandler.Handle(event)
}

// Inject a container into the internal state.
func (n *Node) AddContainer(container *Container) error {
	n.Lock()
	defer n.Unlock()

	if _, ok := n.containers[container.Id]; ok {
		return errors.New("container already exists")
	}
	n.containers[container.Id] = container
	return nil
}

// Remove a container from the internal test.
func (n *Node) RemoveContainer(container *Container) error {
	n.Lock()
	defer n.Unlock()

	if _, ok := n.containers[container.Id]; !ok {
		return errors.New("container not found")
	}
	delete(n.containers, container.Id)
	return nil
}

// Wipes the internal container state.
func (n *Node) CleanupContainers() {
	n.Lock()
	n.containers = make(map[string]*Container)
	n.Unlock()
}
<|endoftext|>