<commit_before>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cache

import (
	"errors"
	"github.com/lpabon/godbc"
	"github.com/pblcache/pblcache/message"
	"sync"
)

type Cache struct {
	stats             *cachestats
	cachemap          *CacheMap
	addressmap        map[uint64]uint64
	blocks, blocksize uint64
	pipeline          chan *message.Message
	lock              sync.Mutex
}

var (
	ErrNotFound  = errors.New("None of the blocks where found")
	ErrSomeFound = errors.New("Only some of the blocks where found")
	ErrPending   = errors.New("New messages where created and are pending")
)

func NewCache(blocks, blocksize uint64, pipeline chan *message.Message) *Cache {

	godbc.Require(blocks > 0)
	godbc.Require(pipeline != nil)

	cache := &Cache{}
	cache.blocks = blocks
	cache.pipeline = pipeline
	cache.blocksize = blocksize

	cache.stats = &cachestats{}
	cache.cachemap = NewCacheMap(cache.blocks)
	cache.addressmap = make(map[uint64]uint64)

	godbc.Ensure(cache.blocks > 0)
	godbc.Ensure(cache.cachemap != nil)
	godbc.Ensure(cache.addressmap != nil)
	godbc.Ensure(cache.stats != nil)

	return cache
}

func (c *Cache) Close() {

}

func (c *Cache) Invalidate(io *message.IoPkt) error {
	c.lock.Lock()
	defer c.lock.Unlock()

	for block := 0; block < io.Nblocks; block++ {
		c.invalidate(io.Offset + uint64(block)*c.blocksize)
	}

	return nil
}

func (c *Cache) Put(msg *message.Message) error {

	c.lock.Lock()
	defer c.lock.Unlock()

	io := msg.IoPkt()

	if io.Nblocks > 1 {
		for block := 0; block < io.Nblocks; block++ {
			m := message.NewMsgPut()
			m.RetChan = msg.RetChan

			mio := m.IoPkt()
			mio.Offset = io.Offset + uint64(block)*c.blocksize
			mio.Buffer = io.Buffer[uint64(block)*c.blocksize : uint64(block)*c.blocksize+c.blocksize]
			mio.BlockNum = c.put(io.Offset)
			mio.Nblocks = 1

			// Send to next one in line
			c.pipeline <- m
		}
	} else {
		io.BlockNum = c.put(io.Offset)
		c.pipeline <- msg
	}

	return nil
}

func (c *Cache) Get(msg *message.Message) (*message.HitmapPkt, error) {

	c.lock.Lock()
	defer c.lock.Unlock()

	io := msg.IoPkt()
	hitmap := make([]bool, io.Nblocks)
	hits := 0

	// Create a message
	var m *message.Message
	var mblock uint64
	for block := uint64(0); block < uint64(io.Nblocks); block++ {
		// Get
		buffer_offset := block * c.blocksize
		current_offset := io.Offset + buffer_offset
		if index, ok := c.get(current_offset); ok {
			hitmap[block] = true
			hits++

			// Check if we already have a message ready
			if m == nil {

				// This is the first message, so let's set it up
				m = c.create_get_submsg(msg,
					current_offset,
					buffer_offset,
					index,
					io.Buffer[buffer_offset:(block+1)*c.blocksize])
				mblock = block
			} else {
				// Let's check what block we are using starting from the block
				// setup by the message
				numblocks := block - mblock

				// If the next block is available on the log after this block, then
				// we can optimize the read by reading a larger amount from the log.
				if m.IoPkt().BlockNum+numblocks == index && hitmap[block-1] == true {
					// It is the next in both the cache and storage device
					mio := m.IoPkt()
					mio.Buffer = io.Buffer[mblock*c.blocksize : (mblock+numblocks+1)*c.blocksize]
					mio.Nblocks++
				} else {
					// Send the previous one
					c.pipeline <- m

					// This is the first message, so let's set it up
					m = c.create_get_submsg(msg,
						current_offset,
						buffer_offset,
						index,
						io.Buffer[buffer_offset:(block+1)*c.blocksize])
					mblock = block
				}

			}
		}
	}

	// Check if we have one more message
	if m != nil {
		c.pipeline <- m
	}

	if hits > 0 {
		hitmappkt := &message.HitmapPkt{
			Hitmap: hitmap,
			Hits:   hits,
		}
		return hitmappkt, nil
	} else {
		return nil, ErrNotFound
	}
}

func (c *Cache) create_get_submsg(msg *message.Message,
	offset, buffer_offset, blocknum uint64,
	buffer []byte) *message.Message {

	m := message.NewMsgGet()
	m.RetChan = msg.RetChan
	m.Priv = msg.Priv

	// Set IoPkt
	mio := m.IoPkt()
	mio.Offset = offset
	mio.Buffer = buffer
	mio.BlockNum = blocknum

	return m
}

func (c *Cache) invalidate(key uint64) bool {
	c.stats.invalidation()

	if index, ok := c.addressmap[key]; ok {
		c.stats.invalidateHit()

		c.cachemap.Free(index)
		delete(c.addressmap, key)

		return true
	}

	return false
}

func (c *Cache) put(key uint64) (index uint64) {

	var (
		evictkey uint64
		evict    bool
	)

	c.stats.insertion()

	if index, evictkey, evict = c.cachemap.Insert(key); evict {
		c.stats.eviction()
		delete(c.addressmap, evictkey)
	}

	c.addressmap[key] = index

	return
}

func (c *Cache) get(key uint64) (index uint64, ok bool) {

	c.stats.read()

	if index, ok = c.addressmap[key]; ok {
		c.stats.readHit()
		c.cachemap.Using(index)
	}

	return
}

func (c *Cache) String() string {
	return c.stats.stats().String()
}

func (c *Cache) Stats() *CacheStats {
	return c.stats.stats()
}

func (c *Cache) StatsClear() {
	c.stats.clear()
}
<commit_msg>Added some comments<commit_after>//
// Copyright (c) 2014 The pblcache Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cache

import (
	"errors"
	"github.com/lpabon/godbc"
	"github.com/pblcache/pblcache/message"
	"sync"
)

type Cache struct {
	stats             *cachestats
	cachemap          *CacheMap
	addressmap        map[uint64]uint64
	blocks, blocksize uint64
	pipeline          chan *message.Message
	lock              sync.Mutex
}

var (
	ErrNotFound  = errors.New("None of the blocks where found")
	ErrSomeFound = errors.New("Only some of the blocks where found")
	ErrPending   = errors.New("New messages where created and are pending")
)

func NewCache(blocks, blocksize uint64, pipeline chan *message.Message) *Cache {

	godbc.Require(blocks > 0)
	godbc.Require(pipeline != nil)

	cache := &Cache{}
	cache.blocks = blocks
	cache.pipeline = pipeline
	cache.blocksize = blocksize

	cache.stats = &cachestats{}
	cache.cachemap = NewCacheMap(cache.blocks)
	cache.addressmap = make(map[uint64]uint64)

	godbc.Ensure(cache.blocks > 0)
	godbc.Ensure(cache.cachemap != nil)
	godbc.Ensure(cache.addressmap != nil)
	godbc.Ensure(cache.stats != nil)

	return cache
}

func (c *Cache) Close() {

}

func (c *Cache) Invalidate(io *message.IoPkt) error {
	c.lock.Lock()
	defer c.lock.Unlock()

	for block := 0; block < io.Nblocks; block++ {
		c.invalidate(io.Offset + uint64(block)*c.blocksize)
	}

	return nil
}

func (c *Cache) Put(msg *message.Message) error {

	c.lock.Lock()
	defer c.lock.Unlock()

	io := msg.IoPkt()

	if io.Nblocks > 1 {
		for block := 0; block < io.Nblocks; block++ {
			// It does not matter that we send small blocks to the Log, since
			// it will buffer them before sending them out to the cache device
			//
			// We do need to send each one sperately now so that the cache
			// policy hopefully aligns them one after the other.
			//
			m := message.NewMsgPut()
			m.RetChan = msg.RetChan

			mio := m.IoPkt()
			mio.Offset = io.Offset + uint64(block)*c.blocksize
			mio.Buffer = io.Buffer[uint64(block)*c.blocksize : uint64(block)*c.blocksize+c.blocksize]
			mio.BlockNum = c.put(io.Offset)
			mio.Nblocks = 1

			// Send to next one in line
			c.pipeline <- m
		}
	} else {
		io.BlockNum = c.put(io.Offset)
		c.pipeline <- msg
	}

	return nil
}

func (c *Cache) Get(msg *message.Message) (*message.HitmapPkt, error) {

	c.lock.Lock()
	defer c.lock.Unlock()

	io := msg.IoPkt()
	hitmap := make([]bool, io.Nblocks)
	hits := 0

	// Create a message
	var m *message.Message
	var mblock uint64
	for block := uint64(0); block < uint64(io.Nblocks); block++ {
		// Get
		buffer_offset := block * c.blocksize
		current_offset := io.Offset + buffer_offset
		if index, ok := c.get(current_offset); ok {
			hitmap[block] = true
			hits++

			// Check if we already have a message ready
			if m == nil {

				// This is the first message, so let's set it up
				m = c.create_get_submsg(msg,
					current_offset,
					buffer_offset,
					index,
					io.Buffer[buffer_offset:(block+1)*c.blocksize])
				mblock = block
			} else {
				// Let's check what block we are using starting from the block
				// setup by the message
				numblocks := block - mblock

				// If the next block is available on the log after this block, then
				// we can optimize the read by reading a larger amount from the log.
				if m.IoPkt().BlockNum+numblocks == index && hitmap[block-1] == true {
					// It is the next in both the cache and storage device
					mio := m.IoPkt()
					mio.Buffer = io.Buffer[mblock*c.blocksize : (mblock+numblocks+1)*c.blocksize]
					mio.Nblocks++
				} else {
					// Send the previous one
					c.pipeline <- m

					// This is the first message, so let's set it up
					m = c.create_get_submsg(msg,
						current_offset,
						buffer_offset,
						index,
						io.Buffer[buffer_offset:(block+1)*c.blocksize])
					mblock = block
				}

			}
		}
	}

	// Check if we have one more message
	if m != nil {
		c.pipeline <- m
	}

	if hits > 0 {
		hitmappkt := &message.HitmapPkt{
			Hitmap: hitmap,
			Hits:   hits,
		}
		return hitmappkt, nil
	} else {
		return nil, ErrNotFound
	}
}

func (c *Cache) create_get_submsg(msg *message.Message,
	offset, buffer_offset, blocknum uint64,
	buffer []byte) *message.Message {

	m := message.NewMsgGet()
	m.RetChan = msg.RetChan
	m.Priv = msg.Priv

	// Set IoPkt
	mio := m.IoPkt()
	mio.Offset = offset
	mio.Buffer = buffer
	mio.BlockNum = blocknum

	return m
}

func (c *Cache) invalidate(key uint64) bool {
	c.stats.invalidation()

	if index, ok := c.addressmap[key]; ok {
		c.stats.invalidateHit()

		c.cachemap.Free(index)
		delete(c.addressmap, key)

		return true
	}

	return false
}

func (c *Cache) put(key uint64) (index uint64) {

	var (
		evictkey uint64
		evict    bool
	)

	c.stats.insertion()

	if index, evictkey, evict = c.cachemap.Insert(key); evict {
		c.stats.eviction()
		delete(c.addressmap, evictkey)
	}

	c.addressmap[key] = index

	return
}

func (c *Cache) get(key uint64) (index uint64, ok bool) {

	c.stats.read()

	if index, ok = c.addressmap[key]; ok {
		c.stats.readHit()
		c.cachemap.Using(index)
	}

	return
}

func (c *Cache) String() string {
	return c.stats.stats().String()
}

func (c *Cache) Stats() *CacheStats {
	return c.stats.stats()
}

func (c *Cache) StatsClear() {
	c.stats.clear()
}
<|endoftext|>


<commit_before>package cache

import (
	"container/list"
	"imposm3/cache/binary"
	"imposm3/element"
	"sort"
	"sync"
)

type byId []element.Node

func (s byId) Len() int           { return len(s) }
func (s byId) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s byId) Less(i, j int) bool { return s[i].Id < s[j].Id }

func packNodes(nodes []element.Node) *binary.DeltaCoords {
	var lastLon, lastLat int64
	var lon, lat int64
	var lastId int64
	ids := make([]int64, len(nodes))
	lons := make([]int64, len(nodes))
	lats := make([]int64, len(nodes))

	i := 0
	for _, nd := range nodes {
		lon = int64(binary.CoordToInt(nd.Long))
		lat = int64(binary.CoordToInt(nd.Lat))
		ids[i] = nd.Id - lastId
		lons[i] = lon - lastLon
		lats[i] = lat - lastLat

		lastId = nd.Id
		lastLon = lon
		lastLat = lat
		i++
	}
	return &binary.DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
}

func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []element.Node {
	if len(deltaCoords.Ids) > cap(nodes) {
		nodes = make([]element.Node, len(deltaCoords.Ids))
	} else {
		nodes = nodes[:len(deltaCoords.Ids)]
	}

	var lastLon, lastLat int64
	var lon, lat int64
	var lastId, id int64

	for i := 0; i < len(deltaCoords.Ids); i++ {
		id = lastId + deltaCoords.Ids[i]
		lon = lastLon + deltaCoords.Lons[i]
		lat = lastLat + deltaCoords.Lats[i]
		nodes[i] = element.Node{
			OSMElem: element.OSMElem{Id: int64(id)},
			Long:    binary.IntToCoord(uint32(lon)),
			Lat:     binary.IntToCoord(uint32(lat)),
		}

		lastId = id
		lastLon = lon
		lastLat = lat
	}
	return nodes
}

type coordsBunch struct {
	sync.Mutex
	id         int64
	coords     []element.Node
	elem       *list.Element
	needsWrite bool
}

func (b *coordsBunch) GetCoord(id int64) (*element.Node, error) {
	idx := sort.Search(len(b.coords), func(i int) bool {
		return b.coords[i].Id >= id
	})
	if idx < len(b.coords) && b.coords[idx].Id == id {
		return &b.coords[idx], nil
	}
	return nil, NotFound
}

func (b *coordsBunch) DeleteCoord(id int64) {
	idx := sort.Search(len(b.coords), func(i int) bool {
		return b.coords[i].Id >= id
	})
	if idx < len(b.coords) && b.coords[idx].Id == id {
		b.coords = append(b.coords[:idx], b.coords[idx+1:]...)
	}
}

type DeltaCoordsCache struct {
	cache
	lruList      *list.List
	table        map[int64]*coordsBunch
	capacity     int64
	linearImport bool
	mu           sync.Mutex
	bunchSize    int64
	readOnly     bool
}

func newDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
	coordsCache := DeltaCoordsCache{}
	coordsCache.options = &globalCacheOptions.Coords.cacheOptions
	err := coordsCache.open(path)
	if err != nil {
		return nil, err
	}
	coordsCache.bunchSize = int64(globalCacheOptions.Coords.BunchSize)
	coordsCache.lruList = list.New()
	coordsCache.table = make(map[int64]*coordsBunch)
	// mem req for cache approx. capacity*bunchSize*40
	coordsCache.capacity = int64(globalCacheOptions.Coords.BunchCacheCapacity)
	return &coordsCache, nil
}

func (self *DeltaCoordsCache) SetLinearImport(v bool) {
	self.linearImport = v
}

func (self *DeltaCoordsCache) Flush() {
	for bunchId, bunch := range self.table {
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
	}
}
func (self *DeltaCoordsCache) Close() {
	self.Flush()
	self.cache.Close()
}

func (self *DeltaCoordsCache) SetReadOnly(val bool) {
	self.readOnly = val
}

func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
	bunchId := self.getBunchId(id)
	bunch, err := self.getBunch(bunchId)
	if err != nil {
		return nil, err
	}
	if self.readOnly {
		bunch.Unlock()
	} else {
		defer bunch.Unlock()
	}
	return bunch.GetCoord(id)
}

func (self *DeltaCoordsCache) DeleteCoord(id int64) error {
	bunchId := self.getBunchId(id)
	bunch, err := self.getBunch(bunchId)
	if err != nil {
		return err
	}
	defer bunch.Unlock()
	bunch.DeleteCoord(id)
	bunch.needsWrite = true
	return nil
}

func (self *DeltaCoordsCache) FillWay(way *element.Way) error {
	if way == nil {
		return nil
	}
	way.Nodes = make([]element.Node, len(way.Refs))

	var err error
	var bunch *coordsBunch
	var bunchId, lastBunchId int64
	lastBunchId = -1

	for i, id := range way.Refs {
		bunchId = self.getBunchId(id)
		// re-use bunches
		if bunchId != lastBunchId {
			if bunch != nil {
				bunch.Unlock()
			}
			bunch, err = self.getBunch(bunchId)
			if err != nil {
				return err
			}
		}
		lastBunchId = bunchId

		nd, err := bunch.GetCoord(id)
		if err != nil {
			bunch.Unlock()
			return err
		}
		way.Nodes[i] = *nd
	}
	if bunch != nil {
		bunch.Unlock()
	}
	return nil
}

func removeSkippedNodes(nodes []element.Node) []element.Node {
	insertPoint := 0
	for i := 0; i < len(nodes); i++ {
		if i != insertPoint {
			nodes[insertPoint] = nodes[i]
		}
		if nodes[i].Id != SKIP {
			insertPoint += 1
		}
	}
	return nodes[:insertPoint]
}

// PutCoords puts nodes into cache.
// nodes need to be sorted by Id.
func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
	var start, currentBunchId int64
	nodes = removeSkippedNodes(nodes)
	if len(nodes) == 0 {
		// skipped all nodes
		return nil
	}
	currentBunchId = self.getBunchId(nodes[0].Id)
	start = 0
	totalNodes := len(nodes)
	for i, node := range nodes {
		bunchId := self.getBunchId(node.Id)
		if bunchId != currentBunchId {
			if self.linearImport && int64(i) > self.bunchSize && int64(i) < int64(totalNodes)-self.bunchSize {
				// no need to handle concurrent updates to the same
				// bunch if we are not at the boundary of a self.bunchSize
				self.putCoordsPacked(currentBunchId, nodes[start:i])
			} else {
				bunch, err := self.getBunch(currentBunchId)
				if err != nil {
					return err
				}
				bunch.coords = append(bunch.coords, nodes[start:i]...)
				sort.Sort(byId(bunch.coords))
				bunch.needsWrite = true
				bunch.Unlock()
			}
			currentBunchId = bunchId
			start = int64(i)
		}
	}
	bunch, err := self.getBunch(currentBunchId)
	if err != nil {
		return err
	}
	bunch.coords = append(bunch.coords, nodes[start:]...)
	sort.Sort(byId(bunch.coords))
	bunch.needsWrite = true
	bunch.Unlock()
	return nil
}

var (
	freeBuffer = make(chan []byte, 4)
)

func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) error {
	keyBuf := idToKeyBuf(bunchId)

	if len(nodes) == 0 {
		return p.db.Delete(p.wo, keyBuf)
	}

	var data []byte
	select {
	case data = <-freeBuffer:
	default:
	}

	data = binary.MarshalDeltaNodes(nodes, data)

	err := p.db.Put(p.wo, keyBuf, data)
	if err != nil {
		return err
	}

	select {
	case freeBuffer <- data:
	default:
	}

	return nil
}

func (p *DeltaCoordsCache) getCoordsPacked(bunchId int64, nodes []element.Node) ([]element.Node, error) {
	keyBuf := idToKeyBuf(bunchId)

	data, err := p.db.Get(p.ro, keyBuf)
	if err != nil {
		return nil, err
	}
	if data == nil {
		// clear before returning
		return nodes[:0], nil
	}
	nodes, err = binary.UnmarshalDeltaNodes(data, nodes)
	if err != nil {
		return nil, err
	}

	return nodes, nil
}

func (self *DeltaCoordsCache) getBunchId(nodeId int64) int64 {
	return nodeId / self.bunchSize
}

var (
	freeNodes = make(chan []element.Node, 4)
)

func (self *DeltaCoordsCache) getBunch(bunchId int64) (*coordsBunch, error) {
	self.mu.Lock()
	bunch, ok := self.table[bunchId]
	var nodes []element.Node
	needsGet := false
	if !ok {
		elem := self.lruList.PushFront(bunchId)
		select {
		case nodes = <-freeNodes:
		default:
			nodes = make([]element.Node, 0, self.bunchSize)
		}
		bunch = &coordsBunch{id: bunchId, coords: nil, elem: elem}
		needsGet = true
		self.table[bunchId] = bunch
	} else {
		self.lruList.MoveToFront(bunch.elem)
	}
	bunch.Lock()
	self.CheckCapacity()
	self.mu.Unlock()

	if needsGet {
		nodes, err := self.getCoordsPacked(bunchId, nodes)
		if err != nil {
			return nil, err
		}
		bunch.coords = nodes
	}

	return bunch, nil
}

func (self *DeltaCoordsCache) CheckCapacity() {
	for int64(len(self.table)) > self.capacity {
		elem := self.lruList.Back()
		bunchId := self.lruList.Remove(elem).(int64)
		bunch := self.table[bunchId]
		bunch.elem = nil
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
		select {
		case freeNodes <- bunch.coords:
		default:
		}
		delete(self.table, bunchId)
	}
}

func (self *DeltaCoordsCache) FirstRefIsCached(refs []int64) bool {
	if len(refs) <= 0 {
		return false
	}
	_, err := self.GetCoord(refs[0])
	if err != nil {
		return false
	}
	return true
}
<commit_msg>initialize coordsBunch map with capacity size<commit_after>package cache

import (
	"container/list"
	"imposm3/cache/binary"
	"imposm3/element"
	"sort"
	"sync"
)

type byId []element.Node

func (s byId) Len() int           { return len(s) }
func (s byId) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s byId) Less(i, j int) bool { return s[i].Id < s[j].Id }

func packNodes(nodes []element.Node) *binary.DeltaCoords {
	var lastLon, lastLat int64
	var lon, lat int64
	var lastId int64
	ids := make([]int64, len(nodes))
	lons := make([]int64, len(nodes))
	lats := make([]int64, len(nodes))

	i := 0
	for _, nd := range nodes {
		lon = int64(binary.CoordToInt(nd.Long))
		lat = int64(binary.CoordToInt(nd.Lat))
		ids[i] = nd.Id - lastId
		lons[i] = lon - lastLon
		lats[i] = lat - lastLat

		lastId = nd.Id
		lastLon = lon
		lastLat = lat
		i++
	}
	return &binary.DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
}

func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []element.Node {
	if len(deltaCoords.Ids) > cap(nodes) {
		nodes = make([]element.Node, len(deltaCoords.Ids))
	} else {
		nodes = nodes[:len(deltaCoords.Ids)]
	}

	var lastLon, lastLat int64
	var lon, lat int64
	var lastId, id int64

	for i := 0; i < len(deltaCoords.Ids); i++ {
		id = lastId + deltaCoords.Ids[i]
		lon = lastLon + deltaCoords.Lons[i]
		lat = lastLat + deltaCoords.Lats[i]
		nodes[i] = element.Node{
			OSMElem: element.OSMElem{Id: int64(id)},
			Long:    binary.IntToCoord(uint32(lon)),
			Lat:     binary.IntToCoord(uint32(lat)),
		}

		lastId = id
		lastLon = lon
		lastLat = lat
	}
	return nodes
}

type coordsBunch struct {
	sync.Mutex
	id         int64
	coords     []element.Node
	elem       *list.Element
	needsWrite bool
}

func (b *coordsBunch) GetCoord(id int64) (*element.Node, error) {
	idx := sort.Search(len(b.coords), func(i int) bool {
		return b.coords[i].Id >= id
	})
	if idx < len(b.coords) && b.coords[idx].Id == id {
		return &b.coords[idx], nil
	}
	return nil, NotFound
}

func (b *coordsBunch) DeleteCoord(id int64) {
	idx := sort.Search(len(b.coords), func(i int) bool {
		return b.coords[i].Id >= id
	})
	if idx < len(b.coords) && b.coords[idx].Id == id {
		b.coords = append(b.coords[:idx], b.coords[idx+1:]...)
	}
}

type DeltaCoordsCache struct {
	cache
	lruList      *list.List
	table        map[int64]*coordsBunch
	capacity     int64
	linearImport bool
	mu           sync.Mutex
	bunchSize    int64
	readOnly     bool
}

func newDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
	coordsCache := DeltaCoordsCache{}
	coordsCache.options = &globalCacheOptions.Coords.cacheOptions
	err := coordsCache.open(path)
	if err != nil {
		return nil, err
	}
	coordsCache.bunchSize = int64(globalCacheOptions.Coords.BunchSize)
	coordsCache.lruList = list.New()
	// mem req for cache approx. capacity*bunchSize*40
	coordsCache.capacity = int64(globalCacheOptions.Coords.BunchCacheCapacity)
	coordsCache.table = make(map[int64]*coordsBunch, coordsCache.capacity)
	return &coordsCache, nil
}

func (self *DeltaCoordsCache) SetLinearImport(v bool) {
	self.linearImport = v
}

func (self *DeltaCoordsCache) Flush() {
	for bunchId, bunch := range self.table {
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
	}
}
func (self *DeltaCoordsCache) Close() {
	self.Flush()
	self.cache.Close()
}

func (self *DeltaCoordsCache) SetReadOnly(val bool) {
	self.readOnly = val
}

func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
	bunchId := self.getBunchId(id)
	bunch, err := self.getBunch(bunchId)
	if err != nil {
		return nil, err
	}
	if self.readOnly {
		bunch.Unlock()
	} else {
		defer bunch.Unlock()
	}
	return bunch.GetCoord(id)
}

func (self *DeltaCoordsCache) DeleteCoord(id int64) error {
	bunchId := self.getBunchId(id)
	bunch, err := self.getBunch(bunchId)
	if err != nil {
		return err
	}
	defer bunch.Unlock()
	bunch.DeleteCoord(id)
	bunch.needsWrite = true
	return nil
}

func (self *DeltaCoordsCache) FillWay(way *element.Way) error {
	if way == nil {
		return nil
	}
	way.Nodes = make([]element.Node, len(way.Refs))

	var err error
	var bunch *coordsBunch
	var bunchId, lastBunchId int64
	lastBunchId = -1

	for i, id := range way.Refs {
		bunchId = self.getBunchId(id)
		// re-use bunches
		if bunchId != lastBunchId {
			if bunch != nil {
				bunch.Unlock()
			}
			bunch, err = self.getBunch(bunchId)
			if err != nil {
				return err
			}
		}
		lastBunchId = bunchId

		nd, err := bunch.GetCoord(id)
		if err != nil {
			bunch.Unlock()
			return err
		}
		way.Nodes[i] = *nd
	}
	if bunch != nil {
		bunch.Unlock()
	}
	return nil
}

func removeSkippedNodes(nodes []element.Node) []element.Node {
	insertPoint := 0
	for i := 0; i < len(nodes); i++ {
		if i != insertPoint {
			nodes[insertPoint] = nodes[i]
		}
		if nodes[i].Id != SKIP {
			insertPoint += 1
		}
	}
	return nodes[:insertPoint]
}

// PutCoords puts nodes into cache.
// nodes need to be sorted by Id.
func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
	var start, currentBunchId int64
	nodes = removeSkippedNodes(nodes)
	if len(nodes) == 0 {
		// skipped all nodes
		return nil
	}
	currentBunchId = self.getBunchId(nodes[0].Id)
	start = 0
	totalNodes := len(nodes)
	for i, node := range nodes {
		bunchId := self.getBunchId(node.Id)
		if bunchId != currentBunchId {
			if self.linearImport && int64(i) > self.bunchSize && int64(i) < int64(totalNodes)-self.bunchSize {
				// no need to handle concurrent updates to the same
				// bunch if we are not at the boundary of a self.bunchSize
				self.putCoordsPacked(currentBunchId, nodes[start:i])
			} else {
				bunch, err := self.getBunch(currentBunchId)
				if err != nil {
					return err
				}
				bunch.coords = append(bunch.coords, nodes[start:i]...)
				sort.Sort(byId(bunch.coords))
				bunch.needsWrite = true
				bunch.Unlock()
			}
			currentBunchId = bunchId
			start = int64(i)
		}
	}
	bunch, err := self.getBunch(currentBunchId)
	if err != nil {
		return err
	}
	bunch.coords = append(bunch.coords, nodes[start:]...)
	sort.Sort(byId(bunch.coords))
	bunch.needsWrite = true
	bunch.Unlock()
	return nil
}

var (
	freeBuffer = make(chan []byte, 4)
)

func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) error {
	keyBuf := idToKeyBuf(bunchId)

	if len(nodes) == 0 {
		return p.db.Delete(p.wo, keyBuf)
	}

	var data []byte
	select {
	case data = <-freeBuffer:
	default:
	}

	data = binary.MarshalDeltaNodes(nodes, data)

	err := p.db.Put(p.wo, keyBuf, data)
	if err != nil {
		return err
	}

	select {
	case freeBuffer <- data:
	default:
	}

	return nil
}

func (p *DeltaCoordsCache) getCoordsPacked(bunchId int64, nodes []element.Node) ([]element.Node, error) {
	keyBuf := idToKeyBuf(bunchId)

	data, err := p.db.Get(p.ro, keyBuf)
	if err != nil {
		return nil, err
	}
	if data == nil {
		// clear before returning
		return nodes[:0], nil
	}
	nodes, err = binary.UnmarshalDeltaNodes(data, nodes)
	if err != nil {
		return nil, err
	}

	return nodes, nil
}

func (self *DeltaCoordsCache) getBunchId(nodeId int64) int64 {
	return nodeId / self.bunchSize
}

var (
	freeNodes = make(chan []element.Node, 4)
)

func (self *DeltaCoordsCache) getBunch(bunchId int64) (*coordsBunch, error) {
	self.mu.Lock()
	bunch, ok := self.table[bunchId]
	var nodes []element.Node
	needsGet := false
	if !ok {
		elem := self.lruList.PushFront(bunchId)
		select {
		case nodes = <-freeNodes:
		default:
			nodes = make([]element.Node, 0, self.bunchSize)
		}
		bunch = &coordsBunch{id: bunchId, coords: nil, elem: elem}
		needsGet = true
		self.table[bunchId] = bunch
	} else {
		self.lruList.MoveToFront(bunch.elem)
	}
	bunch.Lock()
	self.CheckCapacity()
	self.mu.Unlock()

	if needsGet {
		nodes, err := self.getCoordsPacked(bunchId, nodes)
		if err != nil {
			return nil, err
		}
		bunch.coords = nodes
	}

	return bunch, nil
}

func (self *DeltaCoordsCache) CheckCapacity() {
	for int64(len(self.table)) > self.capacity {
		elem := self.lruList.Back()
		bunchId := self.lruList.Remove(elem).(int64)
		bunch := self.table[bunchId]
		bunch.elem = nil
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
		select {
		case freeNodes <- bunch.coords:
		default:
		}
		delete(self.table, bunchId)
	}
}

func (self *DeltaCoordsCache) FirstRefIsCached(refs []int64) bool {
	if len(refs) <= 0 {
		return false
	}
	_, err := self.GetCoord(refs[0])
	if err != nil {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

// ServerName of the current box
var ServerName string

type VHost struct {
	Hostname         string
	FileListLocation string
	Files            []*File
}

type File struct {
	Path           string
	Sha256Expected string
	Sha256Actual   string
	LastModified   time.Time
	LastChecked    time.Time
	Cached         bool
}

type StatusBoard struct {
	Status       []*WorkerStatus
	BadFiles     []string
	Checks       int
	Misses       int
	BadRequests  int
	BadChecksums int

	mu sync.Mutex
}

type WorkerStatus struct {
	Current string
	Status  string
}

type FileChannel chan *File

var Status StatusBoard

type FileStatus struct {
	Path        string
	BadChecksum bool
	BadRequest  bool
	ReadError   bool
	Miss        bool
}

func addStatus(fs *FileStatus) {
	// log.Printf("adding status: %#v\n", fs)
	Status.mu.Lock()
	defer Status.mu.Unlock()
	Status.Checks++

	if fs.BadChecksum {
		Status.BadChecksums++
		Status.BadFiles = append(Status.BadFiles, fs.Path)
	}
	if fs.BadRequest {
		Status.BadRequests++
	}
	if fs.Miss {
		Status.Misses++
	}
}

func statusPrinter() {
	for {

		Status.mu.Lock()

		// terminal.Stdout.Reset()
		// terminal.Stdout.Clear()

		log.Printf("Files: %6d  Misses: %4d  BadRequest: %d  Checksums: %d\n",
			Status.Checks, Status.Misses,
			Status.BadRequests, Status.BadChecksums,
		)

		Status.mu.Unlock()
		time.Sleep(4 * time.Second)
	}
}

func main() {

	var err error

	ServerName, err = os.Hostname()
	if err != nil {
		log.Fatalln("Could not get hostname", err)
	}

	runtime.GOMAXPROCS(6)

	vhost := new(VHost)
	vhost.FileListLocation = "http://storage-hc.dal01.netdna.com/sha256-small.txt"
	vhost.FileListLocation = "http://storage-hc.dal01.netdna.com/sha256.txt"

	vhost.Hostname = "hcinstall.tera-online.com"

	log.Println("Getting file list")
	err = getFileList(vhost)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println("Got file list")

	workQueue := make(FileChannel)

	nworkers := 10

	Status.Status = make([]*WorkerStatus, nworkers)

	for n := 0; n < nworkers; n++ {
		log.Println("starting worker", n)
		Status.Status[n] = new(WorkerStatus)
		go Worker(n, vhost, workQueue)
	}

	go statusPrinter()

	for i, _ := range vhost.Files {
		// log.Printf("File: %#v\n", file)
		workQueue <- vhost.Files[i]
	}

	for n := 0; n < nworkers; n++ {
		log.Println("closing workers", n)
		workQueue <- nil
	}

	time.Sleep(5 * time.Second)
	log.Println("exiting")

	for _, path := range Status.BadFiles {
		fmt.Println(path)
	}

}

func Worker(id int, vhost *VHost, in FileChannel) {

	client := &http.Client{}

	for {
		file := <-in
		// log.Printf("%d FILE: %#v\n", id, file)
		if file == nil {
			log.Println(id, "got nil file")
			break
		}

		getFile(id, client, vhost.Hostname, file)
	}
}

func getFile(id int, client *http.Client, hostname string, file *File) {

	fs := new(FileStatus)
	fs.Path = file.Path
	defer addStatus(fs)

	// log.Printf("%d Getting file '%s'\n", id, file.Path)
	updateStatus(id, file.Path, "GET'ing")

	host := "localhost"
	// host = "flex02.lax04.netdna.com"

	url := "http://" + host + file.Path
	req, err := http.NewRequest("GET", "http://"+host+file.Path, nil)
	if err != nil {
		log.Fatalf("Could not create for %s: %s", url, err)
	}

	req.Host = hostname

	// log.Println("REQUEST", req)

	resp, err := client.Do(req)
	if err != nil {
		fs.ReadError = true
		log.Printf("Error fetching %s: %s\n", url, err)
		return
	}
	defer resp.Body.Close()

	// log.Println("File", url)
	// log.Println("Status", resp.StatusCode, resp.Status)
	// log.Println("Headers", resp.Header)

	if resp.StatusCode != 200 {
		fs.BadRequest = true
		log.Printf("No 200 response for %s: %d\n", file.Path, resp.StatusCode)
		return
	}

	cacheStatus := resp.Header.Get("X-Cache")

	if cacheStatus != "HIT" {
		fs.Miss = true
		if cacheStatus == "" {
			cacheStatus = "[no cache status]"
		}
		log.Printf("%s: %s\n", file.Path, cacheStatus)
	}

	sha := sha256.New()
	size, err := io.Copy(sha, resp.Body)
	if err != nil {
		fs.ReadError = true
		log.Printf("%d Could not read file '%s': %s", id, file.Path, err)
		return
	}

	file.Sha256Actual = hex.EncodeToString(sha.Sum(nil))

	if file.Sha256Actual != file.Sha256Expected {
		fs.BadChecksum = true
		log.Printf("%d Wrong SHA256 for '%s' (size %d)\n", id, file.Path, size)
	} else {
		// log.Println("Ok!")
	}
}

func updateStatus(id int, path, status string) {
	Status.mu.Lock()
	defer Status.mu.Unlock()

	if len(path) > 0 {
		Status.Status[id].Current = path
	}

	if len(status) > 0 {
		Status.Status[id].Status = status
	}
}

func getFileList(vhost *VHost) error {
	url := vhost.FileListLocation
	resp, err := http.Get(url)
	if err != nil {
		return fmt.Errorf("Could not get url %v: %v", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Could not get file list: %d %s", resp.StatusCode, err)
	}

	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		shaPath := strings.SplitN(scanner.Text(), "  .", 2)
		file := new(File)
		file.Sha256Expected = shaPath[0]
		file.Path = shaPath[1]

		vhost.Files = append(vhost.Files, file)
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	return nil

}
<commit_msg>Limit number of CPUs used to half of what's available or 6<commit_after>package main

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"
)

// ServerName of the current box
var ServerName string

type VHost struct {
	Hostname         string
	FileListLocation string
	Files            []*File
}

type File struct {
	Path           string
	Sha256Expected string
	Sha256Actual   string
	LastModified   time.Time
	LastChecked    time.Time
	Cached         bool
}

type StatusBoard struct {
	Status       []*WorkerStatus
	BadFiles     []string
	Checks       int
	Misses       int
	BadRequests  int
	BadChecksums int

	mu sync.Mutex
}

type WorkerStatus struct {
	Current string
	Status  string
}

type FileChannel chan *File

var Status StatusBoard

type FileStatus struct {
	Path        string
	BadChecksum bool
	BadRequest  bool
	ReadError   bool
	Miss        bool
}

func addStatus(fs *FileStatus) {
	// log.Printf("adding status: %#v\n", fs)
	Status.mu.Lock()
	defer Status.mu.Unlock()
	Status.Checks++

	if fs.BadChecksum {
		Status.BadChecksums++
		Status.BadFiles = append(Status.BadFiles, fs.Path)
	}
	if fs.BadRequest {
		Status.BadRequests++
	}
	if fs.Miss {
		Status.Misses++
	}
}

func statusPrinter() {
	for {

		Status.mu.Lock()

		// terminal.Stdout.Reset()
		// terminal.Stdout.Clear()

		log.Printf("Files: %6d  Misses: %4d  BadRequest: %d  Checksums: %d\n",
			Status.Checks, Status.Misses,
			Status.BadRequests, Status.BadChecksums,
		)

		Status.mu.Unlock()
		time.Sleep(4 * time.Second)
	}
}

func main() {

	var err error

	ServerName, err = os.Hostname()
	if err != nil {
		log.Fatalln("Could not get hostname", err)
	}

	ncpus := runtime.NumCPU()

	ncpus /= 2
	if ncpus > 6 {
		ncpus = 6
	}

	log.Printf("Using up to %d CPUs for sha256'ing\n", ncpus)
	runtime.GOMAXPROCS(ncpus)

	vhost := new(VHost)
	vhost.FileListLocation = "http://storage-hc.dal01.netdna.com/sha256-small.txt"
	vhost.FileListLocation = "http://storage-hc.dal01.netdna.com/sha256.txt"

	vhost.Hostname = "hcinstall.tera-online.com"

	log.Println("Getting file list")
	err = getFileList(vhost)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println("Got file list")

	workQueue := make(FileChannel)

	nworkers := 10

	Status.Status = make([]*WorkerStatus, nworkers)

	for n := 0; n < nworkers; n++ {
		log.Println("starting worker", n)
		Status.Status[n] = new(WorkerStatus)
		go Worker(n, vhost, workQueue)
	}

	go statusPrinter()

	for i, _ := range vhost.Files {
		// log.Printf("File: %#v\n", file)
		workQueue <- vhost.Files[i]
	}

	for n := 0; n < nworkers; n++ {
		log.Println("closing workers", n)
		workQueue <- nil
	}

	time.Sleep(5 * time.Second)
	log.Println("exiting")

	for _, path := range Status.BadFiles {
		fmt.Println(path)
	}

}

func Worker(id int, vhost *VHost, in FileChannel) {

	client := &http.Client{}

	for {
		file := <-in
		// log.Printf("%d FILE: %#v\n", id, file)
		if file == nil {
			log.Println(id, "got nil file")
			break
		}

		getFile(id, client, vhost.Hostname, file)
	}
}

func getFile(id int, client *http.Client, hostname string, file *File) {

	fs := new(FileStatus)
	fs.Path = file.Path
	defer addStatus(fs)

	// log.Printf("%d Getting file '%s'\n", id, file.Path)
	updateStatus(id, file.Path, "GET'ing")

	host := "localhost"
	// host = "flex02.lax04.netdna.com"

	url := "http://" + host + file.Path
	req, err := http.NewRequest("GET", "http://"+host+file.Path, nil)
	if err != nil {
		log.Fatalf("Could not create for %s: %s", url, err)
	}

	req.Host = hostname

	// log.Println("REQUEST", req)

	resp, err := client.Do(req)
	if err != nil {
		fs.ReadError = true
		log.Printf("Error fetching %s: %s\n", url, err)
		return
	}
	defer resp.Body.Close()

	// log.Println("File", url)
	// log.Println("Status", resp.StatusCode, resp.Status)
	// log.Println("Headers", resp.Header)

	if resp.StatusCode != 200 {
		fs.BadRequest = true
		log.Printf("No 200 response for %s: %d\n", file.Path, resp.StatusCode)
		return
	}

	cacheStatus := resp.Header.Get("X-Cache")

	if cacheStatus != "HIT" {
		fs.Miss = true
		if cacheStatus == "" {
			cacheStatus = "[no cache status]"
		}
		log.Printf("%s: %s\n", file.Path, cacheStatus)
	}

	sha := sha256.New()
	size, err := io.Copy(sha, resp.Body)
	if err != nil {
		fs.ReadError = true
		log.Printf("%d Could not read file '%s': %s", id, file.Path, err)
		return
	}

	file.Sha256Actual = hex.EncodeToString(sha.Sum(nil))

	if file.Sha256Actual != file.Sha256Expected {
		fs.BadChecksum = true
		log.Printf("%d Wrong SHA256 for '%s' (size %d)\n", id, file.Path, size)
	} else {
		// log.Println("Ok!")
	}
}

func updateStatus(id int, path, status string) {
	Status.mu.Lock()
	defer Status.mu.Unlock()

	if len(path) > 0 {
		Status.Status[id].Current = path
	}

	if len(status) > 0 {
		Status.Status[id].Status = status
	}
}

func getFileList(vhost *VHost) error {
	url := vhost.FileListLocation
	resp, err := http.Get(url)
	if err != nil {
		return fmt.Errorf("Could not get url %v: %v", url, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Could not get file list: %d %s", resp.StatusCode, err)
	}

	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		shaPath := strings.SplitN(scanner.Text(), "  .", 2)
		file := new(File)
		file.Sha256Expected = shaPath[0]
		file.Path = shaPath[1]

		vhost.Files = append(vhost.Files, file)
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	return nil

}
<|endoftext|>


<commit_before>package render

import (
	"github.com/akavel/polyclip-go"
	"github.com/oakmound/oak/alg"
	"github.com/oakmound/oak/alg/floatgeom"
)

// A DrawPolygon is used to determine whether elements should be drawn, defining
// a polygonal area for what things should be visible.
type DrawPolygon struct {
	usingDrawPolygon bool
	drawPolygon      []floatgeom.Point2
	dims             floatgeom.Rect2
	rectangular      bool
}

// SetDrawPolygon sets the draw polygon and flags that draw functions
// should check for containment in the polygon before drawing elements
// Deprecated: use SetPolygon instead
func (dp *DrawPolygon) SetDrawPolygon(p polyclip.Polygon) {
	// get []floatgeom.Point2
	poly := make([]floatgeom.Point2, 0, len(p))
	for _, c := range p {
		for _, pt := range c {
			poly = append(poly, floatgeom.Point2{pt.X, pt.Y})
		}
	}
	dp.SetPolygon(poly)
}

// SetPolygon sets the draw polygon and flags that draw functions
// should check for containment in the polygon before drawing elements.
func (dp *DrawPolygon) SetPolygon(poly []floatgeom.Point2) {
	dp.usingDrawPolygon = true
	dp.dims = floatgeom.NewBoundingRect2(poly...)
	dp.drawPolygon = poly
	if isRectangular(poly...) {
		dp.rectangular = true
	}
}

func isRectangular(pts ...floatgeom.Point2) bool {
	last := pts[len(pts)-1]
	for _, pt := range pts {
		// The last point needs to share an x or y value with this point
		if !alg.F64eq(pt.X(), last.X()) && !alg.F64eq(pt.Y(), last.Y()) {
			return false
		}
		last = pt
	}
	return true
}

// ClearDrawPolygon will stop checking the set draw polygon for whether elements
// should be drawn to screen. If SetDrawPolygon was not called before this was
// called, this does nothing.
// This may in the future be called at the start of new scenes.
func (dp *DrawPolygon) ClearDrawPolygon() {
	dp.usingDrawPolygon = false
	dp.dims = floatgeom.Rect2{}
	dp.rectangular = false
}

// DrawPolygonDim returns the dimensions of this draw polygon, or (0,0)->(0,0)
// if there is no draw polygon in use.
func (dp *DrawPolygon) DrawPolygonDim() floatgeom.Rect2 {
	return dp.dims
}

// InDrawPolygon returns whehter a coordinate and dimension set should be drawn
// given the draw polygon
func (dp *DrawPolygon) InDrawPolygon(xi, yi, x2i, y2i int) bool {
	if dp.usingDrawPolygon {
		x := float64(xi)
		y := float64(yi)
		x2 := float64(x2i)
		y2 := float64(y2i)

		dx := dp.dims.Min.X()
		dy := dp.dims.Min.Y()
		dx2 := dp.dims.Max.X()
		dy2 := dp.dims.Max.Y()

		dimOverlap := false
		if x > dx {
			if x < dx2 {
				dimOverlap = true
			}
		} else {
			if dx < x2 {
				dimOverlap = true
			}
		}
		if y > dy {
			if y < dy2 {
				dimOverlap = true
			}
		} else {
			if dy < y2 {
				dimOverlap = true
			}
		}
		if !dimOverlap {
			return false
		}
		if dp.rectangular {
			return true
		}
		r := floatgeom.NewRect2(x, y, x2, y2)
		diags := [][2]floatgeom.Point2{
			{
				{r.Min.X(), r.Max.Y()},
				{r.Max.X(), r.Min.Y()},
			}, {
				r.Min,
				r.Max,
			},
		}
		last := dp.drawPolygon[len(dp.drawPolygon)-1]
		for i := 0; i < len(dp.drawPolygon); i++ {
			next := dp.drawPolygon[i]
			if r.Contains(next) {
				return true
			}
			// Checking line segment from last to next
			for _, diag := range diags {
				if orient(diag[0], diag[1], last) != orient(diag[0], diag[1], next) &&
					orient(next, last, diag[0]) != orient(next, last, diag[1]) {
					return true
				}
			}

			last = next
		}
		return false
	}
	return true
}

func orient(p1, p2, p3 floatgeom.Point2) int8 {
	val := (p2.Y()-p1.Y())*(p3.X()-p2.X()) -
		(p2.X()-p1.X())*(p3.Y()-p2.Y())
	switch {
	case val < 0:
		return 2
	case val > 0:
		return 1
	default:
		return 0
	}
}
<commit_msg>SetPolygon simplification<commit_after>package render

import (
	"github.com/akavel/polyclip-go"
	"github.com/oakmound/oak/alg"
	"github.com/oakmound/oak/alg/floatgeom"
)

// A DrawPolygon is used to determine whether elements should be drawn, defining
// a polygonal area for what things should be visible.
type DrawPolygon struct {
	usingDrawPolygon bool
	drawPolygon      []floatgeom.Point2
	dims             floatgeom.Rect2
	rectangular      bool
}

// SetDrawPolygon sets the draw polygon and flags that draw functions
// should check for containment in the polygon before drawing elements
// Deprecated: use SetPolygon instead
func (dp *DrawPolygon) SetDrawPolygon(p polyclip.Polygon) {
	// get []floatgeom.Point2
	poly := make([]floatgeom.Point2, 0, len(p))
	for _, c := range p {
		for _, pt := range c {
			poly = append(poly, floatgeom.Point2{pt.X, pt.Y})
		}
	}
	dp.SetPolygon(poly)
}

// SetPolygon sets the draw polygon and flags that draw functions
// should check for containment in the polygon before drawing elements.
func (dp *DrawPolygon) SetPolygon(poly []floatgeom.Point2) {
	dp.usingDrawPolygon = true
	dp.dims = floatgeom.NewBoundingRect2(poly...)
	dp.drawPolygon = poly
	dp.rectangular = isRectangular(poly...)
}

func isRectangular(pts ...floatgeom.Point2) bool {
	last := pts[len(pts)-1]
	for _, pt := range pts {
		// The last point needs to share an x or y value with this point
		if !alg.F64eq(pt.X(), last.X()) && !alg.F64eq(pt.Y(), last.Y()) {
			return false
		}
		last = pt
	}
	return true
}

// ClearDrawPolygon will stop checking the set draw polygon for whether elements
// should be drawn to screen. If SetDrawPolygon was not called before this was
// called, this does nothing.
// This may in the future be called at the start of new scenes.
func (dp *DrawPolygon) ClearDrawPolygon() {
	dp.usingDrawPolygon = false
	dp.dims = floatgeom.Rect2{}
	dp.rectangular = false
}

// DrawPolygonDim returns the dimensions of this draw polygon, or (0,0)->(0,0)
// if there is no draw polygon in use.
func (dp *DrawPolygon) DrawPolygonDim() floatgeom.Rect2 {
	return dp.dims
}

// InDrawPolygon returns whehter a coordinate and dimension set should be drawn
// given the draw polygon
func (dp *DrawPolygon) InDrawPolygon(xi, yi, x2i, y2i int) bool {
	if dp.usingDrawPolygon {
		x := float64(xi)
		y := float64(yi)
		x2 := float64(x2i)
		y2 := float64(y2i)

		dx := dp.dims.Min.X()
		dy := dp.dims.Min.Y()
		dx2 := dp.dims.Max.X()
		dy2 := dp.dims.Max.Y()

		dimOverlap := false
		if x > dx {
			if x < dx2 {
				dimOverlap = true
			}
		} else {
			if dx < x2 {
				dimOverlap = true
			}
		}
		if y > dy {
			if y < dy2 {
				dimOverlap = true
			}
		} else {
			if dy < y2 {
				dimOverlap = true
			}
		}
		if !dimOverlap {
			return false
		}
		if dp.rectangular {
			return true
		}
		r := floatgeom.NewRect2(x, y, x2, y2)
		diags := [][2]floatgeom.Point2{
			{
				{r.Min.X(), r.Max.Y()},
				{r.Max.X(), r.Min.Y()},
			}, {
				r.Min,
				r.Max,
			},
		}
		last := dp.drawPolygon[len(dp.drawPolygon)-1]
		for i := 0; i < len(dp.drawPolygon); i++ {
			next := dp.drawPolygon[i]
			if r.Contains(next) {
				return true
			}
			// Checking line segment from last to next
			for _, diag := range diags {
				if orient(diag[0], diag[1], last) != orient(diag[0], diag[1], next) &&
					orient(next, last, diag[0]) != orient(next, last, diag[1]) {
					return true
				}
			}

			last = next
		}
		return false
	}
	return true
}

func orient(p1, p2, p3 floatgeom.Point2) int8 {
	val := (p2.Y()-p1.Y())*(p3.X()-p2.X()) -
		(p2.X()-p1.X())*(p3.Y()-p2.Y())
	switch {
	case val < 0:
		return 2
	case val > 0:
		return 1
	default:
		return 0
	}
}
<|endoftext|>


<commit_before>package fsrepo

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"sync"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	repo "github.com/jbenet/go-ipfs/repo"
	"github.com/jbenet/go-ipfs/repo/common"
	config "github.com/jbenet/go-ipfs/repo/config"
	component "github.com/jbenet/go-ipfs/repo/fsrepo/component"
	counter "github.com/jbenet/go-ipfs/repo/fsrepo/counter"
	lockfile "github.com/jbenet/go-ipfs/repo/fsrepo/lock"
	serialize "github.com/jbenet/go-ipfs/repo/fsrepo/serialize"
	dir "github.com/jbenet/go-ipfs/thirdparty/dir"
	u "github.com/jbenet/go-ipfs/util"
	util "github.com/jbenet/go-ipfs/util"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
)

var (

	// packageLock must be held to while performing any operation that modifies an
	// FSRepo's state field. This includes Init, Open, Close, and Remove.
	packageLock sync.Mutex // protects openersCounter and lockfiles
	// lockfiles holds references to the Closers that ensure that repos are
	// only accessed by one process at a time.
	lockfiles map[string]io.Closer
	// openersCounter prevents the fsrepo from being removed while there exist open
	// FSRepo handles. It also ensures that the Init is atomic.
	//
	// packageLock also protects numOpenedRepos
	//
	// If an operation is used when repo is Open and the operation does not
	// change the repo's state, the package lock does not need to be acquired.
	openersCounter *counter.Openers
)

func init() {
	openersCounter = counter.NewOpenersCounter()
	lockfiles = make(map[string]io.Closer)
}

// FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple
// callers.
type FSRepo struct {
	// state is the FSRepo's state (unopened, opened, closed)
	state state
	// path is the file-system path
	path string
	// config is set on Open, guarded by packageLock
	config *config.Config

	// TODO test
	datastoreComponent component.DatastoreComponent
	eventlogComponent  component.EventlogComponent
}

var _ repo.Repo = (*FSRepo)(nil)

type componentBuilder struct {
	Init          component.Initializer
	IsInitialized component.InitializationChecker
	OpenHandler   func(*FSRepo) error
}

// At returns a handle to an FSRepo at the provided |path|.
func At(repoPath string) *FSRepo {
	// This method must not have side-effects.
	return &FSRepo{
		path:  path.Clean(repoPath),
		state: unopened, // explicitly set for clarity
	}
}

// ConfigAt returns an error if the FSRepo at the given path is not
// initialized. This function allows callers to read the config file even when
// another process is running and holding the lock.
func ConfigAt(repoPath string) (*config.Config, error) {

	// packageLock must be held to ensure that the Read is atomic.
	packageLock.Lock()
	defer packageLock.Unlock()

	configFilename, err := config.Filename(repoPath)
	if err != nil {
		return nil, err
	}
	return serialize.Load(configFilename)
}

// configIsInitialized returns true if the repo is initialized at
// provided |path|.
func configIsInitialized(path string) bool {
	configFilename, err := config.Filename(path)
	if err != nil {
		return false
	}
	if !util.FileExists(configFilename) {
		return false
	}
	return true
}

func initConfig(path string, conf *config.Config) error {
	if configIsInitialized(path) {
		return nil
	}
	configFilename, err := config.Filename(path)
	if err != nil {
		return err
	}
	// initialization is the one time when it's okay to write to the config
	// without reading the config from disk and merging any user-provided keys
	// that may exist.
	if err := serialize.WriteConfigFile(configFilename, conf); err != nil {
		return err
	}
	return nil
}

// Init initializes a new FSRepo at the given path with the provided config.
// TODO add support for custom datastores.
func Init(path string, conf *config.Config) error {

	// packageLock must be held to ensure that the repo is not initialized more
	// than once.
	packageLock.Lock()
	defer packageLock.Unlock()

	if isInitializedUnsynced(path) {
		return nil
	}

	if err := initConfig(path, conf); err != nil {
		return err
	}

	for _, b := range componentBuilders() {
		if err := b.Init(path, conf); err != nil {
			return err
		}
	}
	return nil
}

// Remove recursively removes the FSRepo at |path|.
func Remove(repoPath string) error {
	repoPath = path.Clean(repoPath)

	// packageLock must be held to ensure that the repo is not removed while
	// being accessed by others.
	packageLock.Lock()
	defer packageLock.Unlock()

	if openersCounter.NumOpeners(repoPath) != 0 {
		return errors.New("repo in use")
	}
	return os.RemoveAll(repoPath)
}

// LockedByOtherProcess returns true if the FSRepo is locked by another
// process. If true, then the repo cannot be opened by this process.
func LockedByOtherProcess(repoPath string) bool {
	repoPath = path.Clean(repoPath)

	// packageLock must be held to check the number of openers.
	packageLock.Lock()
	defer packageLock.Unlock()

	// NB: the lock is only held when repos are Open
	return lockfile.Locked(repoPath) && openersCounter.NumOpeners(repoPath) == 0
}

// openConfig returns an error if the config file is not present.
func (r *FSRepo) openConfig() error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	conf, err := serialize.Load(configFilename)
	if err != nil {
		return err
	}
	r.config = conf
	return nil
}

// Open returns an error if the repo is not initialized.
func (r *FSRepo) Open() error {

	// packageLock must be held to make sure that the repo is not destroyed by
	// another caller. It must not be released until initialization is complete
	// and the number of openers is incremeneted.
	packageLock.Lock()
	defer packageLock.Unlock()

	expPath, err := u.TildeExpansion(r.path)
	if err != nil {
		return err
	}
	r.path = expPath

	if r.state != unopened {
		return debugerror.Errorf("repo is %s", r.state)
	}
	if !isInitializedUnsynced(r.path) {
		return debugerror.New("ipfs not initialized, please run 'ipfs init'")
	}
	// check repo path, then check all constituent parts.
	// TODO acquire repo lock
	// TODO if err := initCheckDir(logpath); err != nil { // }
	if err := dir.Writable(r.path); err != nil {
		return err
	}

	if err := r.openConfig(); err != nil {
		return err
	}

	for _, b := range componentBuilders() {
		if err := b.OpenHandler(r); err != nil {
			return err
		}
	}

	return r.transitionToOpened()
}

// Close closes the FSRepo, releasing held resources.
func (r *FSRepo) Close() error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	for _, closer := range r.components() {
		if err := closer.Close(); err != nil {
			return err
		}
	}
	return r.transitionToClosed()
}

// Config returns the FSRepo's config. This method must not be called if the
// repo is not open.
//
// Result when not Open is undefined. The method may panic if it pleases.
func (r *FSRepo) Config() *config.Config {

	// It is not necessary to hold the package lock since the repo is in an
	// opened state. The package lock is _not_ meant to ensure that the repo is
	// thread-safe. The package lock is only meant to guard againt removal and
	// coordinate the lockfile. However, we provide thread-safety to keep
	// things simple.
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		panic(fmt.Sprintln("repo is", r.state))
	}
	return r.config
}

// setConfigUnsynced is for private use.
func (r *FSRepo) setConfigUnsynced(updated *config.Config) error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	// to avoid clobbering user-provided keys, must read the config from disk
	// as a map, write the updated struct values to the map and write the map
	// to disk.
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(configFilename, &mapconf); err != nil {
		return err
	}
	m, err := config.ToMap(updated)
	if err != nil {
		return err
	}
	for k, v := range m {
		mapconf[k] = v
	}
	if err := serialize.WriteConfigFile(configFilename, mapconf); err != nil {
		return err
	}
	*r.config = *updated // copy so caller cannot modify this private config
	return nil
}

// SetConfig updates the FSRepo's config.
func (r *FSRepo) SetConfig(updated *config.Config) error {

	// packageLock is held to provide thread-safety.
	packageLock.Lock()
	defer packageLock.Unlock()

	return r.setConfigUnsynced(updated)
}

// GetConfigKey retrieves only the value of a particular key.
func (r *FSRepo) GetConfigKey(key string) (interface{}, error) {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return nil, debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return nil, err
	}
	var cfg map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &cfg); err != nil {
		return nil, err
	}
	return common.MapGetKV(cfg, key)
}

// SetConfigKey writes the value of a particular key.
func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	switch v := value.(type) {
	case string:
		if i, err := strconv.Atoi(v); err == nil {
			value = i
		}
	}
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &mapconf); err != nil {
		return err
	}
	if err := common.MapSetKV(mapconf, key, value); err != nil {
		return err
	}
	conf, err := config.FromMap(mapconf)
	if err != nil {
		return err
	}
	if err := serialize.WriteConfigFile(filename, mapconf); err != nil {
		return err
	}
	return r.setConfigUnsynced(conf) // TODO roll this into this method
}

// Datastore returns a repo-owned datastore. If FSRepo is Closed, return value
// is undefined.
func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
	packageLock.Lock()
	d := r.datastoreComponent.Datastore()
	packageLock.Unlock()
	return d
}

var _ io.Closer = &FSRepo{}
var _ repo.Repo = &FSRepo{}

// IsInitialized returns true if the repo is initialized at provided |path|.
func IsInitialized(path string) bool {
	// packageLock is held to ensure that another caller doesn't attempt to
	// Init or Remove the repo while this call is in progress.
	packageLock.Lock()
	defer packageLock.Unlock()

	return isInitializedUnsynced(path)
}

// private methods below this point. NB: packageLock must held by caller.

// isInitializedUnsynced reports whether the repo is initialized. Caller must
// hold the packageLock.
func isInitializedUnsynced(path string) bool {
	if !configIsInitialized(path) {
		return false
	}
	for _, b := range componentBuilders() {
		if !b.IsInitialized(path) {
			return false
		}
	}
	return true
}

// transitionToOpened manages the state transition to |opened|. Caller must hold
// the package mutex.
func (r *FSRepo) transitionToOpened() error {
	r.state = opened
	if countBefore := openersCounter.NumOpeners(r.path); countBefore == 0 { // #first
		closer, err := lockfile.Lock(r.path)
		if err != nil {
			return err
		}
		lockfiles[r.path] = closer
	}
	return openersCounter.AddOpener(r.path)
}

// transitionToClosed manages the state transition to |closed|. Caller must
// hold the package mutex.
func (r *FSRepo) transitionToClosed() error {
	r.state = closed
	if err := openersCounter.RemoveOpener(r.path); err != nil {
		return err
	}
	if countAfter := openersCounter.NumOpeners(r.path); countAfter == 0 {
		closer, ok := lockfiles[r.path]
		if !ok {
			return errors.New("package error: lockfile is not held")
		}
		if err := closer.Close(); err != nil {
			return err
		}
		delete(lockfiles, r.path)
	}
	return nil
}

// components returns the FSRepo's constituent components
func (r *FSRepo) components() []component.Component {
	return []component.Component{
		&r.datastoreComponent,
	}
}

func componentBuilders() []componentBuilder {
	return []componentBuilder{

		// DatastoreComponent
		componentBuilder{
			Init:          component.InitDatastoreComponent,
			IsInitialized: component.DatastoreComponentIsInitialized,
			OpenHandler: func(r *FSRepo) error {
				c := component.DatastoreComponent{}
				c.SetPath(r.path)
				if err := c.Open(r.config); err != nil {
					return err
				}
				r.datastoreComponent = c
				return nil
			},
		},

		// EventlogComponent
		componentBuilder{
			Init:          component.InitEventlogComponent,
			IsInitialized: component.EventlogComponentIsInitialized,
			OpenHandler: func(r *FSRepo) error {
				c := component.EventlogComponent{}
				c.SetPath(r.path)
				if err := c.Open(r.config); err != nil {
					return err
				}
				r.eventlogComponent = c
				return nil
			},
		},
	}
}
<commit_msg>Rename variable to avoid collision with the path package<commit_after>package fsrepo

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"sync"

	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	repo "github.com/jbenet/go-ipfs/repo"
	"github.com/jbenet/go-ipfs/repo/common"
	config "github.com/jbenet/go-ipfs/repo/config"
	component "github.com/jbenet/go-ipfs/repo/fsrepo/component"
	counter "github.com/jbenet/go-ipfs/repo/fsrepo/counter"
	lockfile "github.com/jbenet/go-ipfs/repo/fsrepo/lock"
	serialize "github.com/jbenet/go-ipfs/repo/fsrepo/serialize"
	dir "github.com/jbenet/go-ipfs/thirdparty/dir"
	u "github.com/jbenet/go-ipfs/util"
	util "github.com/jbenet/go-ipfs/util"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
)

var (

	// packageLock must be held to while performing any operation that modifies an
	// FSRepo's state field. This includes Init, Open, Close, and Remove.
	packageLock sync.Mutex // protects openersCounter and lockfiles
	// lockfiles holds references to the Closers that ensure that repos are
	// only accessed by one process at a time.
	lockfiles map[string]io.Closer
	// openersCounter prevents the fsrepo from being removed while there exist open
	// FSRepo handles. It also ensures that the Init is atomic.
	//
	// packageLock also protects numOpenedRepos
	//
	// If an operation is used when repo is Open and the operation does not
	// change the repo's state, the package lock does not need to be acquired.
	openersCounter *counter.Openers
)

func init() {
	openersCounter = counter.NewOpenersCounter()
	lockfiles = make(map[string]io.Closer)
}

// FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple
// callers.
type FSRepo struct {
	// state is the FSRepo's state (unopened, opened, closed)
	state state
	// path is the file-system path
	path string
	// config is set on Open, guarded by packageLock
	config *config.Config

	// TODO test
	datastoreComponent component.DatastoreComponent
	eventlogComponent  component.EventlogComponent
}

var _ repo.Repo = (*FSRepo)(nil)

type componentBuilder struct {
	Init          component.Initializer
	IsInitialized component.InitializationChecker
	OpenHandler   func(*FSRepo) error
}

// At returns a handle to an FSRepo at the provided |path|.
func At(repoPath string) *FSRepo {
	// This method must not have side-effects.
	return &FSRepo{
		path:  path.Clean(repoPath),
		state: unopened, // explicitly set for clarity
	}
}

// ConfigAt returns an error if the FSRepo at the given path is not
// initialized. This function allows callers to read the config file even when
// another process is running and holding the lock.
func ConfigAt(repoPath string) (*config.Config, error) {

	// packageLock must be held to ensure that the Read is atomic.
	packageLock.Lock()
	defer packageLock.Unlock()

	configFilename, err := config.Filename(repoPath)
	if err != nil {
		return nil, err
	}
	return serialize.Load(configFilename)
}

// configIsInitialized returns true if the repo is initialized at
// provided |path|.
func configIsInitialized(path string) bool {
	configFilename, err := config.Filename(path)
	if err != nil {
		return false
	}
	if !util.FileExists(configFilename) {
		return false
	}
	return true
}

func initConfig(path string, conf *config.Config) error {
	if configIsInitialized(path) {
		return nil
	}
	configFilename, err := config.Filename(path)
	if err != nil {
		return err
	}
	// initialization is the one time when it's okay to write to the config
	// without reading the config from disk and merging any user-provided keys
	// that may exist.
	if err := serialize.WriteConfigFile(configFilename, conf); err != nil {
		return err
	}
	return nil
}

// Init initializes a new FSRepo at the given path with the provided config.
// TODO add support for custom datastores.
func Init(repoPath string, conf *config.Config) error {

	// packageLock must be held to ensure that the repo is not initialized more
	// than once.
	packageLock.Lock()
	defer packageLock.Unlock()

	if isInitializedUnsynced(repoPath) {
		return nil
	}

	if err := initConfig(repoPath, conf); err != nil {
		return err
	}

	for _, b := range componentBuilders() {
		if err := b.Init(repoPath, conf); err != nil {
			return err
		}
	}
	return nil
}

// Remove recursively removes the FSRepo at |path|.
func Remove(repoPath string) error {
	repoPath = path.Clean(repoPath)

	// packageLock must be held to ensure that the repo is not removed while
	// being accessed by others.
	packageLock.Lock()
	defer packageLock.Unlock()

	if openersCounter.NumOpeners(repoPath) != 0 {
		return errors.New("repo in use")
	}
	return os.RemoveAll(repoPath)
}

// LockedByOtherProcess returns true if the FSRepo is locked by another
// process. If true, then the repo cannot be opened by this process.
func LockedByOtherProcess(repoPath string) bool {
	repoPath = path.Clean(repoPath)

	// packageLock must be held to check the number of openers.
	packageLock.Lock()
	defer packageLock.Unlock()

	// NB: the lock is only held when repos are Open
	return lockfile.Locked(repoPath) && openersCounter.NumOpeners(repoPath) == 0
}

// openConfig returns an error if the config file is not present.
func (r *FSRepo) openConfig() error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	conf, err := serialize.Load(configFilename)
	if err != nil {
		return err
	}
	r.config = conf
	return nil
}

// Open returns an error if the repo is not initialized.
func (r *FSRepo) Open() error {

	// packageLock must be held to make sure that the repo is not destroyed by
	// another caller. It must not be released until initialization is complete
	// and the number of openers is incremeneted.
	packageLock.Lock()
	defer packageLock.Unlock()

	expPath, err := u.TildeExpansion(r.path)
	if err != nil {
		return err
	}
	r.path = expPath

	if r.state != unopened {
		return debugerror.Errorf("repo is %s", r.state)
	}
	if !isInitializedUnsynced(r.path) {
		return debugerror.New("ipfs not initialized, please run 'ipfs init'")
	}
	// check repo path, then check all constituent parts.
	// TODO acquire repo lock
	// TODO if err := initCheckDir(logpath); err != nil { // }
	if err := dir.Writable(r.path); err != nil {
		return err
	}

	if err := r.openConfig(); err != nil {
		return err
	}

	for _, b := range componentBuilders() {
		if err := b.OpenHandler(r); err != nil {
			return err
		}
	}

	return r.transitionToOpened()
}

// Close closes the FSRepo, releasing held resources.
func (r *FSRepo) Close() error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	for _, closer := range r.components() {
		if err := closer.Close(); err != nil {
			return err
		}
	}
	return r.transitionToClosed()
}

// Config returns the FSRepo's config. This method must not be called if the
// repo is not open.
//
// Result when not Open is undefined. The method may panic if it pleases.
func (r *FSRepo) Config() *config.Config {

	// It is not necessary to hold the package lock since the repo is in an
	// opened state. The package lock is _not_ meant to ensure that the repo is
	// thread-safe. The package lock is only meant to guard againt removal and
	// coordinate the lockfile. However, we provide thread-safety to keep
	// things simple.
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		panic(fmt.Sprintln("repo is", r.state))
	}
	return r.config
}

// setConfigUnsynced is for private use.
func (r *FSRepo) setConfigUnsynced(updated *config.Config) error {
	configFilename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	// to avoid clobbering user-provided keys, must read the config from disk
	// as a map, write the updated struct values to the map and write the map
	// to disk.
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(configFilename, &mapconf); err != nil {
		return err
	}
	m, err := config.ToMap(updated)
	if err != nil {
		return err
	}
	for k, v := range m {
		mapconf[k] = v
	}
	if err := serialize.WriteConfigFile(configFilename, mapconf); err != nil {
		return err
	}
	*r.config = *updated // copy so caller cannot modify this private config
	return nil
}

// SetConfig updates the FSRepo's config.
func (r *FSRepo) SetConfig(updated *config.Config) error {

	// packageLock is held to provide thread-safety.
	packageLock.Lock()
	defer packageLock.Unlock()

	return r.setConfigUnsynced(updated)
}

// GetConfigKey retrieves only the value of a particular key.
func (r *FSRepo) GetConfigKey(key string) (interface{}, error) {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return nil, debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return nil, err
	}
	var cfg map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &cfg); err != nil {
		return nil, err
	}
	return common.MapGetKV(cfg, key)
}

// SetConfigKey writes the value of a particular key.
func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
	packageLock.Lock()
	defer packageLock.Unlock()

	if r.state != opened {
		return debugerror.Errorf("repo is %s", r.state)
	}

	filename, err := config.Filename(r.path)
	if err != nil {
		return err
	}
	switch v := value.(type) {
	case string:
		if i, err := strconv.Atoi(v); err == nil {
			value = i
		}
	}
	var mapconf map[string]interface{}
	if err := serialize.ReadConfigFile(filename, &mapconf); err != nil {
		return err
	}
	if err := common.MapSetKV(mapconf, key, value); err != nil {
		return err
	}
	conf, err := config.FromMap(mapconf)
	if err != nil {
		return err
	}
	if err := serialize.WriteConfigFile(filename, mapconf); err != nil {
		return err
	}
	return r.setConfigUnsynced(conf) // TODO roll this into this method
}

// Datastore returns a repo-owned datastore. If FSRepo is Closed, return value
// is undefined.
func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
	packageLock.Lock()
	d := r.datastoreComponent.Datastore()
	packageLock.Unlock()
	return d
}

var _ io.Closer = &FSRepo{}
var _ repo.Repo = &FSRepo{}

// IsInitialized returns true if the repo is initialized at provided |path|.
func IsInitialized(path string) bool {
	// packageLock is held to ensure that another caller doesn't attempt to
	// Init or Remove the repo while this call is in progress.
	packageLock.Lock()
	defer packageLock.Unlock()

	return isInitializedUnsynced(path)
}

// private methods below this point. NB: packageLock must held by caller.

// isInitializedUnsynced reports whether the repo is initialized. Caller must
// hold the packageLock.
func isInitializedUnsynced(path string) bool {
	if !configIsInitialized(path) {
		return false
	}
	for _, b := range componentBuilders() {
		if !b.IsInitialized(path) {
			return false
		}
	}
	return true
}

// transitionToOpened manages the state transition to |opened|. Caller must hold
// the package mutex.
func (r *FSRepo) transitionToOpened() error {
	r.state = opened
	if countBefore := openersCounter.NumOpeners(r.path); countBefore == 0 { // #first
		closer, err := lockfile.Lock(r.path)
		if err != nil {
			return err
		}
		lockfiles[r.path] = closer
	}
	return openersCounter.AddOpener(r.path)
}

// transitionToClosed manages the state transition to |closed|. Caller must
// hold the package mutex.
func (r *FSRepo) transitionToClosed() error {
	r.state = closed
	if err := openersCounter.RemoveOpener(r.path); err != nil {
		return err
	}
	if countAfter := openersCounter.NumOpeners(r.path); countAfter == 0 {
		closer, ok := lockfiles[r.path]
		if !ok {
			return errors.New("package error: lockfile is not held")
		}
		if err := closer.Close(); err != nil {
			return err
		}
		delete(lockfiles, r.path)
	}
	return nil
}

// components returns the FSRepo's constituent components
func (r *FSRepo) components() []component.Component {
	return []component.Component{
		&r.datastoreComponent,
	}
}

func componentBuilders() []componentBuilder {
	return []componentBuilder{

		// DatastoreComponent
		componentBuilder{
			Init:          component.InitDatastoreComponent,
			IsInitialized: component.DatastoreComponentIsInitialized,
			OpenHandler: func(r *FSRepo) error {
				c := component.DatastoreComponent{}
				c.SetPath(r.path)
				if err := c.Open(r.config); err != nil {
					return err
				}
				r.datastoreComponent = c
				return nil
			},
		},

		// EventlogComponent
		componentBuilder{
			Init:          component.InitEventlogComponent,
			IsInitialized: component.EventlogComponentIsInitialized,
			OpenHandler: func(r *FSRepo) error {
				c := component.EventlogComponent{}
				c.SetPath(r.path)
				if err := c.Open(r.config); err != nil {
					return err
				}
				r.eventlogComponent = c
				return nil
			},
		},
	}
}
<|endoftext|>


<commit_before>package canal

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"

	"github.com/fusionrsrch/go-mysql/client"
	"github.com/fusionrsrch/go-mysql/dump"
	"github.com/fusionrsrch/go-mysql/mysql"
	"github.com/fusionrsrch/go-mysql/replication"
	"github.com/fusionrsrch/go-mysql/schema"
	"github.com/siddontang/go/log"
	"github.com/siddontang/go/sync2"
)

var errCanalClosed = errors.New("canal was closed")

// Canal can sync your MySQL data into everywhere, like Elasticsearch, Redis, etc...
// MySQL must open row format for binlog
type Canal struct {
	m sync.Mutex

	cfg *Config

	master     *masterInfo
	dumper     *dump.Dumper
	dumpDoneCh chan struct{}
	syncer     *replication.BinlogSyncer

	rsLock     sync.Mutex
	rsHandlers []RowsEventHandler

	connLock sync.Mutex
	conn     *client.Conn

	wg sync.WaitGroup

	tableLock sync.Mutex
	tables    map[string]*schema.Table

	quit   chan struct{}
	closed sync2.AtomicBool
}

func NewCanal(cfg *Config) (*Canal, error) {
	c := new(Canal)
	c.cfg = cfg
	c.closed.Set(false)
	c.quit = make(chan struct{})

	os.MkdirAll(cfg.DataDir, 0755)

	c.dumpDoneCh = make(chan struct{})
	c.rsHandlers = make([]RowsEventHandler, 0, 4)
	c.tables = make(map[string]*schema.Table)

	var err error
	if c.master, err = loadMasterInfo(c.masterInfoPath()); err != nil {
		return nil, err
	} else if len(c.master.Addr) != 0 && c.master.Addr != c.cfg.Addr {
		log.Infof("MySQL addr %s in old master.info, but new %s, reset", c.master.Addr, c.cfg.Addr)
		// may use another MySQL, reset
		c.master = &masterInfo{}
	}

	c.master.Addr = c.cfg.Addr

	if err := c.prepareDumper(); err != nil {
		return nil, err
	}

	if err = c.prepareSyncer(); err != nil {
		return nil, err
	}

	if err := c.checkBinlogRowFormat(); err != nil {
		return nil, err
	}

	return c, nil
}

func (c *Canal) prepareDumper() error {
	var err error
	if c.dumper, err = dump.NewDumper(c.cfg.Dump.ExecutionPath,
		c.cfg.Addr, c.cfg.User, c.cfg.Password); err != nil {
		if err != exec.ErrNotFound {
			return err
		}
		//no mysqldump, use binlog only
		c.dumper = nil
		return nil
	}

	dbs := c.cfg.Dump.Databases
	tables := c.cfg.Dump.Tables
	tableDB := c.cfg.Dump.TableDB

	if len(tables) == 0 {
		c.dumper.AddDatabases(dbs...)
	} else {
		c.dumper.AddTables(tableDB, tables...)
	}

	for _, ignoreTable := range c.cfg.Dump.IgnoreTables {
		if seps := strings.Split(ignoreTable, ","); len(seps) == 2 {
			c.dumper.AddIgnoreTables(seps[0], seps[1])
		}
	}

	c.dumper.SetErrOut(ioutil.Discard)
	return nil
}

func (c *Canal) Start() error {
	c.wg.Add(1)
	go c.run()

	return nil
}

func (c *Canal) run() error {
	defer c.wg.Done()

	if err := c.tryDump(); err != nil {
		log.Errorf("canal dump mysql err: %v", err)
		return err
	}

	close(c.dumpDoneCh)

	if err := c.startSyncBinlog(); err != nil {
		if !c.isClosed() {
			log.Errorf("canal start sync binlog err: %v", err)
		}
		return err
	}

	return nil
}

func (c *Canal) isClosed() bool {
	return c.closed.Get()
}

func (c *Canal) Close() {
	log.Infof("close canal")

	c.m.Lock()
	defer c.m.Unlock()

	if c.isClosed() {
		return
	}

	c.closed.Set(true)

	close(c.quit)

	c.connLock.Lock()
	c.conn.Close()
	c.conn = nil
	c.connLock.Unlock()

	if c.syncer != nil {
		c.syncer.Close()
		c.syncer = nil
	}

	c.master.Close()

	c.wg.Wait()
}

func (c *Canal) WaitDumpDone() <-chan struct{} {
	return c.dumpDoneCh
}

func (c *Canal) GetTable(db string, table string) (*schema.Table, error) {
	key := fmt.Sprintf("%s.%s", db, table)
	c.tableLock.Lock()
	t, ok := c.tables[key]
	c.tableLock.Unlock()

	if ok {
		return t, nil
	}

	t, err := schema.NewTable(c, db, table)
	if err != nil {
		return nil, err
	}

	c.tableLock.Lock()
	c.tables[key] = t
	c.tableLock.Unlock()

	return t, nil
}

// Check MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB
func (c *Canal) CheckBinlogRowImage(image string) error {
	// need to check MySQL binlog row image? full, minimal or noblob?
	// now only log
	if c.cfg.Flavor == mysql.MySQLFlavor {
		if res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_row_image"`); err != nil {
			return err
		} else {
			// MySQL has binlog row image from 5.6, so older will return empty
			rowImage, _ := res.GetString(0, 1)
			if rowImage != "" && !strings.EqualFold(rowImage, image) {
				return fmt.Errorf("MySQL uses %s binlog row image, but we want %s", rowImage, image)
			}
		}
	}

	return nil
}

func (c *Canal) checkBinlogRowFormat() error {
	res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_format";`)
	if err != nil {
		return err
	} else if f, _ := res.GetString(0, 1); f != "ROW" {
		return fmt.Errorf("binlog must ROW format, but %s now", f)
	}

	return nil
}

func (c *Canal) prepareSyncer() error {
	c.syncer = replication.NewBinlogSyncer(c.cfg.ServerID, c.cfg.Flavor)

	seps := strings.Split(c.cfg.Addr, ":")
	if len(seps) != 2 {
		return fmt.Errorf("invalid mysql addr format %s, must host:port", c.cfg.Addr)
	}

	port, err := strconv.ParseUint(seps[1], 10, 16)
	if err != nil {
		return err
	}

	if err = c.syncer.RegisterSlave(seps[0], uint16(port), c.cfg.User, c.cfg.Password); err != nil {
		return err
	}
	return nil
}

func (c *Canal) masterInfoPath() string {
	return path.Join(c.cfg.DataDir, "master.info")
}

// Execute a SQL
func (c *Canal) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err error) {
	c.connLock.Lock()
	defer c.connLock.Unlock()

	retryNum := 3
	for i := 0; i < retryNum; i++ {
		if c.conn == nil {
			c.conn, err = client.Connect(c.cfg.Addr, c.cfg.User, c.cfg.Password, "")
			if err != nil {
				return nil, err
			}
		}

		rr, err = c.conn.Execute(cmd, args...)
		if err != nil && err != mysql.ErrBadConn {
			return
		} else if err == mysql.ErrBadConn {
			c.conn.Close()
			c.conn = nil
			continue
		} else {
			return
		}
	}
	return
}

func (c *Canal) SyncedPosition() mysql.Position {
	return c.master.Pos()
}
<commit_msg>Update<commit_after>package canal

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"

	"github.com/fusionrsrch/go-mysql/client"
	"github.com/fusionrsrch/go-mysql/dump"
	"github.com/fusionrsrch/go-mysql/mysql"
	"github.com/fusionrsrch/go-mysql/replication"
	"github.com/fusionrsrch/go-mysql/schema"
	"github.com/siddontang/go/log"
	"github.com/siddontang/go/sync2"
)

var errCanalClosed = errors.New("canal was closed")

// Canal can sync your MySQL data into everywhere, like Elasticsearch, Redis, etc...
// MySQL must open row format for binlog
type Canal struct {
	m sync.Mutex

	cfg *Config

	master     *masterInfo
	dumper     *dump.Dumper
	dumpDoneCh chan struct{}
	syncer     *replication.BinlogSyncer

	rsLock     sync.Mutex
	rsHandlers []RowsEventHandler

	connLock sync.Mutex
	conn     *client.Conn

	wg sync.WaitGroup

	tableLock sync.Mutex
	tables    map[string]*schema.Table

	quit   chan struct{}
	closed sync2.AtomicBool
}

func NewCanal(cfg *Config) (*Canal, error) {
	c := new(Canal)
	c.cfg = cfg
	c.closed.Set(false)
	c.quit = make(chan struct{})

	os.MkdirAll(cfg.DataDir, 0755)

	c.dumpDoneCh = make(chan struct{})
	c.rsHandlers = make([]RowsEventHandler, 0, 4)
	c.tables = make(map[string]*schema.Table)

	var err error
	if c.master, err = loadMasterInfo(c.masterInfoPath()); err != nil {
		return nil, err
	} else if len(c.master.Addr) != 0 && c.master.Addr != c.cfg.Addr {
		log.Infof("MySQL addr %s in old master.info, but new %s, reset", c.master.Addr, c.cfg.Addr)
		// may use another MySQL, reset
		c.master = &masterInfo{}
	}

	c.master.Addr = c.cfg.Addr

	if err := c.prepareDumper(); err != nil {
		return nil, err
	}

	if err = c.prepareSyncer(); err != nil {
		return nil, err
	}

	if err := c.checkBinlogRowFormat(); err != nil {
		return nil, err
	}

	return c, nil
}

func (c *Canal) prepareDumper() error {
	var err error
	if c.dumper, err = dump.NewDumper(c.cfg.Dump.ExecutionPath,
		c.cfg.Addr, c.cfg.User, c.cfg.Password); err != nil {
		if err != exec.ErrNotFound {
			return err
		}
		//no mysqldump, use binlog only
		c.dumper = nil
		return nil
	}

	dbs := c.cfg.Dump.Databases
	tables := c.cfg.Dump.Tables
	tableDB := c.cfg.Dump.TableDB

	if len(tables) == 0 {
		c.dumper.AddDatabases(dbs...)
	} else {
		c.dumper.AddTables(tableDB, tables...)
	}

	for _, ignoreTable := range c.cfg.Dump.IgnoreTables {
		if seps := strings.Split(ignoreTable, ","); len(seps) == 2 {
			c.dumper.AddIgnoreTables(seps[0], seps[1])
		}
	}

	c.dumper.SetErrOut(ioutil.Discard)
	return nil
}

func (c *Canal) Start() error {
	c.wg.Add(1)
	go c.run()

	return nil
}

func (c *Canal) run() error {
	defer c.wg.Done()

	if err := c.tryDump(); err != nil {
		log.Errorf("canal dump mysql err: %v", err)
		return err
	}

	close(c.dumpDoneCh)

	if err := c.startSyncBinlog(); err != nil {
		if !c.isClosed() {
			log.Errorf("canal start sync binlog err: %v", err)
		}
		return err
	}

	return nil
}

func (c *Canal) isClosed() bool {
	return c.closed.Get()
}

func (c *Canal) Close() {
	log.Infof("close canal")

	c.m.Lock()
	defer c.m.Unlock()

	if c.isClosed() {
		return
	}

	c.closed.Set(true)

	close(c.quit)

	c.connLock.Lock()
	c.conn.Close()
	c.conn = nil
	c.connLock.Unlock()

	if c.syncer != nil {
		c.syncer.Close()
		c.syncer = nil
	}

	c.master.Close()

	c.wg.Wait()
}

func (c *Canal) WaitDumpDone() <-chan struct{} {
	return c.dumpDoneCh
}

func (c *Canal) GetTable(db string, table string) (*schema.Table, error) {
	key := fmt.Sprintf("%s.%s", db, table)
	c.tableLock.Lock()
	t, ok := c.tables[key]
	c.tableLock.Unlock()

	if ok {
		return t, nil
	}

	t, err := schema.NewTable(c, db, table)
	if err != nil {
		return nil, err
	}

	c.tableLock.Lock()
	c.tables[key] = t
	c.tableLock.Unlock()

	return t, nil
}

// Check MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB
func (c *Canal) CheckBinlogRowImage(image string) error {
	fmt.Println(" CheckBinlogRowImage")
	// need to check MySQL binlog row image? full, minimal or noblob?
	// now only log
	if c.cfg.Flavor == mysql.MySQLFlavor {
		if res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_row_image"`); err != nil {
			return err
		} else {
			// MySQL has binlog row image from 5.6, so older will return empty
			rowImage, _ := res.GetString(0, 1)
			if rowImage != "" && !strings.EqualFold(rowImage, image) {
				return fmt.Errorf("MySQL uses %s binlog row image, but we want %s", rowImage, image)
			}
		}
	}

	return nil
}

func (c *Canal) checkBinlogRowFormat() error {
	fmt.Println(" checkBinlogRowFormat")
	res, err := c.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_format";`)
	if err != nil {
		return err
	} else if f, _ := res.GetString(0, 1); f != "ROW" {
		return fmt.Errorf("binlog must ROW format, but %s now", f)
	}

	return nil
}

func (c *Canal) prepareSyncer() error {
	c.syncer = replication.NewBinlogSyncer(c.cfg.ServerID, c.cfg.Flavor)

	seps := strings.Split(c.cfg.Addr, ":")
	if len(seps) != 2 {
		return fmt.Errorf("invalid mysql addr format %s, must host:port", c.cfg.Addr)
	}

	port, err := strconv.ParseUint(seps[1], 10, 16)
	if err != nil {
		return err
	}

	if err = c.syncer.RegisterSlave(seps[0], uint16(port), c.cfg.User, c.cfg.Password); err != nil {
		return err
	}
	return nil
}

func (c *Canal) masterInfoPath() string {
	return path.Join(c.cfg.DataDir, "master.info")
}

// Execute a SQL
func (c *Canal) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err error) {
	c.connLock.Lock()
	defer c.connLock.Unlock()

	retryNum := 3
	for i := 0; i < retryNum; i++ {
		if c.conn == nil {
			c.conn, err = client.Connect(c.cfg.Addr, c.cfg.User, c.cfg.Password, "")
			if err != nil {
				return nil, err
			}
		}

		rr, err = c.conn.Execute(cmd, args...)
		if err != nil && err != mysql.ErrBadConn {
			return
		} else if err == mysql.ErrBadConn {
			c.conn.Close()
			c.conn = nil
			continue
		} else {
			return
		}
	}
	return
}

func (c *Canal) SyncedPosition() mysql.Position {
	return c.master.Pos()
}
<|endoftext|>