<commit_before>package syscalls

import (
	"fmt"
	"github.com/lunixbochs/struc"
	"os"
	"strings"
	"syscall"

	"../models"
)

func errno(err error) uint64 {
	if err != nil {
		return uint64(err.(syscall.Errno))
	}
	return 0
}

type U models.Usercorn

type Syscall struct {
	Func func(u U, a []uint64) uint64
	Args []int
	Ret  int
}

func exit(u U, a []uint64) uint64 {
	code := int(a[0])
	u.Exit(code)
	return 0
}

func read(u U, a []uint64) uint64 {
	fd, buf, size := int(a[0]), a[1], a[2]
	tmp := make([]byte, size)
	n, _ := syscall.Read(fd, tmp)
	u.MemWrite(buf, tmp[:n])
	return uint64(n)
}

func write(u U, a []uint64) uint64 {
	fd, buf, size := int(a[0]), a[1], a[2]
	mem, _ := u.MemRead(buf, size)
	n, _ := syscall.Write(fd, mem)
	return uint64(n)
}

func open(u U, a []uint64) uint64 {
	path, _ := u.Mem().ReadStrAt(a[0])
	if strings.Contains(path, "/lib/") {
		path = u.PrefixPath(path, false)
	}
	mode, flags := int(a[1]), uint32(a[2])
	fd, _ := syscall.Open(path, mode, flags)
	return uint64(fd)
}

func _close(u U, a []uint64) uint64 {
	fd := int(a[0])
	syscall.Close(fd)
	return 0
}

func lseek(u U, a []uint64) uint64 {
	fd, offset, whence := int(a[0]), int64(a[1]), int(a[2])
	off, _ := syscall.Seek(fd, offset, whence)
	return uint64(off)
}

func mmap(u U, a []uint64) uint64 {
	addr_hint, size, prot, flags, fd, off := a[0], a[1], a[2], a[3], int(int32(a[4])), int64(a[5])
	prot, flags = flags, prot // ignore go error
	addr, _ := u.Mmap(addr_hint, size)
	if fd > 0 {
		fd2, _ := syscall.Dup(fd)
		f := os.NewFile(uintptr(fd2), "")
		f.Seek(off, 0)
		tmp := make([]byte, size)
		n, _ := f.Read(tmp)
		u.MemWrite(addr, tmp[:n])
	}
	return uint64(addr)
}

func munmap(u U, a []uint64) uint64 {
	return 0
}

func mprotect(u U, a []uint64) uint64 {
	return 0
}

func brk(u U, a []uint64) uint64 {
	// TODO: return is Linux specific
	addr := a[0]
	ret, _ := u.Brk(addr)
	return ret
}

func fstat(u U, a []uint64) uint64 {
	fd, buf := int(a[0]), a[1]
	var stat syscall.Stat_t
	if err := syscall.Fstat(fd, &stat); err != nil {
		return errno(err)
	}
	if err := struc.Pack(u.Mem().StreamAt(buf), &stat); err != nil {
		panic(err)
	}
	return 0
}

func stat(u U, a []uint64) uint64 {
	path, _ := u.Mem().ReadStrAt(a[0])
	if strings.Contains(path, "/lib/") {
		path = u.PrefixPath(path, false)
	}
	buf := a[1]
	var stat syscall.Stat_t
	if err := syscall.Stat(path, &stat); err != nil {
		return errno(err)
	}
	if err := struc.Pack(u.Mem().StreamAt(buf), &stat); err != nil {
		panic(err)
	}
	return 0
}

func getcwd(u U, a []uint64) uint64 {
	buf, size := a[0], a[1]
	wd, _ := os.Getwd()
	if uint64(len(wd)) > size {
		wd = wd[:size]
	}
	u.MemWrite(buf, []byte(wd))
	return 0
}

func access(u U, a []uint64) uint64 {
	// TODO: portability
	path, _ := u.Mem().ReadStrAt(a[0])
	amode := uint32(a[1])
	err := syscall.Access(path, amode)
	return errno(err)
}

func readv(u U, a []uint64) uint64 {
	fd, iov, count := int(a[0]), a[1], a[2]
	for vec := range iovecIter(u.Mem().StreamAt(iov), count, int(u.Bits()), u.ByteOrder()) {
		tmp := make([]byte, vec.Len)
		n, _ := syscall.Read(fd, tmp)
		if n <= 0 {
			break
		}
		u.MemWrite(vec.Base, tmp[:n])
	}
	return 0
}

func writev(u U, a []uint64) uint64 {
	fd, iov, count := int(a[0]), a[1], a[2]
	for vec := range iovecIter(u.Mem().StreamAt(iov), count, int(u.Bits()), u.ByteOrder()) {
		data, _ := u.MemRead(vec.Base, vec.Len)
		syscall.Write(fd, data)
	}
	return 0
}

func getuid(u U, a []uint64) uint64 {
	return uint64(os.Getuid())
}

func getgid(u U, a []uint64) uint64 {
	return uint64(os.Getgid())
}

func dup2(u U, a []uint64) uint64 {
	return errno(syscall.Dup2(int(a[0]), int(a[1])))
}

func stub(u U, a []uint64) uint64 {
	return 0
}

type A []int

var syscalls = map[string]Syscall{
	"exit":       {exit, A{INT}, INT},
	"exit_group": {exit, A{INT}, INT},
	// "fork": {fork, A{}, INT},
	"read":     {read, A{FD, OBUF, LEN}, INT},
	"write":    {write, A{FD, BUF, LEN}, INT},
	"open":     {open, A{STR, INT, INT}, FD},
	"close":    {_close, A{FD}, INT},
	"lseek":    {lseek, A{FD, OFF, INT}, INT},
	"mmap":     {mmap, A{PTR, LEN, INT, INT, FD, OFF}, PTR},
	"mmap2":    {mmap, A{PTR, LEN, INT, INT, FD, OFF}, PTR},
	"munmap":   {munmap, A{PTR, LEN}, INT},
	"mprotect": {mprotect, A{PTR, LEN, INT}, INT},
	"brk":      {brk, A{PTR}, PTR},
	"fstat":    {fstat, A{FD, PTR}, INT},
	"stat":     {fstat, A{STR, PTR}, INT},
	"getcwd":   {getcwd, A{PTR, LEN}, INT},
	"access":   {access, A{STR, INT}, INT},
	"readv":    {readv, A{FD, PTR, INT}, INT},
	"writev":   {writev, A{FD, PTR, INT}, INT},
	"getuid":   {getuid, A{}, INT},
	"getgid":   {getgid, A{}, INT},
	"dup2":     {dup2, A{INT, INT}, INT},

	// stubs
	"ioctl":          {stub, A{}, INT},
	"rt_sigprocmask": {stub, A{}, INT},
}

func Call(u models.Usercorn, num int, name string, getArgs func(n int) ([]uint64, error), strace bool) (uint64, error) {
	s, ok := syscalls[name]
	if !ok {
		panic(fmt.Errorf("Unknown syscall: %s", name))
	}
	args, err := getArgs(len(s.Args))
	if err != nil {
		return 0, err
	}
	if strace {
		Trace(u, name, args)
	}
	ret := s.Func(u, args)
	if strace {
		TraceRet(u, name, args, ret)
	}
	return ret, nil
}
<commit_msg>add readlink<commit_after>package syscalls

import (
	"fmt"
	"github.com/lunixbochs/struc"
	"os"
	"strings"
	"syscall"

	"../models"
)

func errno(err error) uint64 {
	if err != nil {
		return uint64(err.(syscall.Errno))
	}
	return 0
}

const (
	UINT64_MAX = 0xFFFFFFFFFFFFFFFF
)

type U models.Usercorn

type Syscall struct {
	Func func(u U, a []uint64) uint64
	Args []int
	Ret  int
}

func exit(u U, a []uint64) uint64 {
	code := int(a[0])
	u.Exit(code)
	return 0
}

func read(u U, a []uint64) uint64 {
	fd, buf, size := int(a[0]), a[1], a[2]
	tmp := make([]byte, size)
	n, _ := syscall.Read(fd, tmp)
	u.MemWrite(buf, tmp[:n])
	return uint64(n)
}

func write(u U, a []uint64) uint64 {
	fd, buf, size := int(a[0]), a[1], a[2]
	mem, _ := u.MemRead(buf, size)
	n, _ := syscall.Write(fd, mem)
	return uint64(n)
}

func open(u U, a []uint64) uint64 {
	path, _ := u.Mem().ReadStrAt(a[0])
	if strings.Contains(path, "/lib/") {
		path = u.PrefixPath(path, false)
	}
	mode, flags := int(a[1]), uint32(a[2])
	fd, _ := syscall.Open(path, mode, flags)
	return uint64(fd)
}

func _close(u U, a []uint64) uint64 {
	fd := int(a[0])
	syscall.Close(fd)
	return 0
}

func lseek(u U, a []uint64) uint64 {
	fd, offset, whence := int(a[0]), int64(a[1]), int(a[2])
	off, _ := syscall.Seek(fd, offset, whence)
	return uint64(off)
}

func mmap(u U, a []uint64) uint64 {
	addr_hint, size, prot, flags, fd, off := a[0], a[1], a[2], a[3], int(int32(a[4])), int64(a[5])
	prot, flags = flags, prot // ignore go error
	addr, _ := u.Mmap(addr_hint, size)
	if fd > 0 {
		fd2, _ := syscall.Dup(fd)
		f := os.NewFile(uintptr(fd2), "")
		f.Seek(off, 0)
		tmp := make([]byte, size)
		n, _ := f.Read(tmp)
		u.MemWrite(addr, tmp[:n])
	}
	return uint64(addr)
}

func munmap(u U, a []uint64) uint64 {
	return 0
}

func mprotect(u U, a []uint64) uint64 {
	return 0
}

func brk(u U, a []uint64) uint64 {
	// TODO: return is Linux specific
	addr := a[0]
	ret, _ := u.Brk(addr)
	return ret
}

func fstat(u U, a []uint64) uint64 {
	fd, buf := int(a[0]), a[1]
	var stat syscall.Stat_t
	if err := syscall.Fstat(fd, &stat); err != nil {
		return errno(err)
	}
	if err := struc.Pack(u.Mem().StreamAt(buf), &stat); err != nil {
		panic(err)
	}
	return 0
}

func stat(u U, a []uint64) uint64 {
	path, _ := u.Mem().ReadStrAt(a[0])
	if strings.Contains(path, "/lib/") {
		path = u.PrefixPath(path, false)
	}
	buf := a[1]
	var stat syscall.Stat_t
	if err := syscall.Stat(path, &stat); err != nil {
		return errno(err)
	}
	if err := struc.Pack(u.Mem().StreamAt(buf), &stat); err != nil {
		panic(err)
	}
	return 0
}

func getcwd(u U, a []uint64) uint64 {
	buf, size := a[0], a[1]
	wd, _ := os.Getwd()
	if uint64(len(wd)) > size {
		wd = wd[:size]
	}
	u.MemWrite(buf, []byte(wd))
	return 0
}

func access(u U, a []uint64) uint64 {
	// TODO: portability
	path, _ := u.Mem().ReadStrAt(a[0])
	amode := uint32(a[1])
	err := syscall.Access(path, amode)
	return errno(err)
}

func readv(u U, a []uint64) uint64 {
	fd, iov, count := int(a[0]), a[1], a[2]
	for vec := range iovecIter(u.Mem().StreamAt(iov), count, int(u.Bits()), u.ByteOrder()) {
		tmp := make([]byte, vec.Len)
		n, _ := syscall.Read(fd, tmp)
		if n <= 0 {
			break
		}
		u.MemWrite(vec.Base, tmp[:n])
	}
	return 0
}

func writev(u U, a []uint64) uint64 {
	fd, iov, count := int(a[0]), a[1], a[2]
	for vec := range iovecIter(u.Mem().StreamAt(iov), count, int(u.Bits()), u.ByteOrder()) {
		data, _ := u.MemRead(vec.Base, vec.Len)
		syscall.Write(fd, data)
	}
	return 0
}

func getuid(u U, a []uint64) uint64 {
	return uint64(os.Getuid())
}

func getgid(u U, a []uint64) uint64 {
	return uint64(os.Getgid())
}

func dup2(u U, a []uint64) uint64 {
	return errno(syscall.Dup2(int(a[0]), int(a[1])))
}

func readlink(u U, a []uint64) uint64 {
	path, _ := u.Mem().ReadStrAt(a[0])
	bufsz := a[2]
	name, err := os.Readlink(path)
	if err != nil {
		return UINT64_MAX
	}
	if len(name) > int(bufsz)-1 {
		name = name[:bufsz-1]
	}
	u.Mem().WriteAt([]byte(name+"\x00"), a[1])
	return uint64(len(name))
}

func stub(u U, a []uint64) uint64 {
	return 0
}

type A []int

var syscalls = map[string]Syscall{
	"exit":       {exit, A{INT}, INT},
	"exit_group": {exit, A{INT}, INT},
	// "fork": {fork, A{}, INT},
	"read":     {read, A{FD, OBUF, LEN}, INT},
	"write":    {write, A{FD, BUF, LEN}, INT},
	"open":     {open, A{STR, INT, INT}, FD},
	"close":    {_close, A{FD}, INT},
	"lseek":    {lseek, A{FD, OFF, INT}, INT},
	"mmap":     {mmap, A{PTR, LEN, INT, INT, FD, OFF}, PTR},
	"mmap2":    {mmap, A{PTR, LEN, INT, INT, FD, OFF}, PTR},
	"munmap":   {munmap, A{PTR, LEN}, INT},
	"mprotect": {mprotect, A{PTR, LEN, INT}, INT},
	"brk":      {brk, A{PTR}, PTR},
	"fstat":    {fstat, A{FD, PTR}, INT},
	"stat":     {fstat, A{STR, PTR}, INT},
	"getcwd":   {getcwd, A{PTR, LEN}, INT},
	"access":   {access, A{STR, INT}, INT},
	"readv":    {readv, A{FD, PTR, INT}, INT},
	"writev":   {writev, A{FD, PTR, INT}, INT},
	"getuid":   {getuid, A{}, INT},
	"getgid":   {getgid, A{}, INT},
	"dup2":     {dup2, A{INT, INT}, INT},
	"readlink": {readlink, A{STR, STR, INT}, INT},

	// stubs
	"ioctl":          {stub, A{}, INT},
	"rt_sigprocmask": {stub, A{}, INT},
}

func Call(u models.Usercorn, num int, name string, getArgs func(n int) ([]uint64, error), strace bool) (uint64, error) {
	s, ok := syscalls[name]
	if !ok {
		panic(fmt.Errorf("Unknown syscall: %s", name))
	}
	args, err := getArgs(len(s.Args))
	if err != nil {
		return 0, err
	}
	if strace {
		Trace(u, name, args)
	}
	ret := s.Func(u, args)
	if strace {
		TraceRet(u, name, args, ret)
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package topology

import (
	"code.google.com/p/weed-fs/go/directory"
	"code.google.com/p/weed-fs/go/sequence"
	"code.google.com/p/weed-fs/go/storage"
	"errors"
	"io/ioutil"
	"math/rand"
)

type Topology struct {
	NodeImpl

	//transient vid~servers mapping for each replication type
	replicaType2VolumeLayout []*VolumeLayout

	pulse int64

	volumeSizeLimit uint64

	sequence sequence.Sequencer

	chanDeadDataNodes      chan *DataNode
	chanRecoveredDataNodes chan *DataNode
	chanFullVolumes        chan storage.VolumeInfo

	configuration *Configuration
}

func NewTopology(id string, confFile string, dirname string, sequenceFilename string, volumeSizeLimit uint64, pulse int) (*Topology, error) {
	t := &Topology{}
	t.id = NodeId(id)
	t.nodeType = "Topology"
	t.NodeImpl.value = t
	t.children = make(map[NodeId]Node)
	t.replicaType2VolumeLayout = make([]*VolumeLayout, storage.LengthRelicationType)
	t.pulse = int64(pulse)
	t.volumeSizeLimit = volumeSizeLimit

	t.sequence = sequence.NewSequencer(dirname, sequenceFilename)

	t.chanDeadDataNodes = make(chan *DataNode)
	t.chanRecoveredDataNodes = make(chan *DataNode)
	t.chanFullVolumes = make(chan storage.VolumeInfo)

	err := t.loadConfiguration(confFile)

	return t, err
}

func (t *Topology) loadConfiguration(configurationFile string) error {
	b, e := ioutil.ReadFile(configurationFile)
	if e == nil {
		t.configuration, e = NewConfiguration(b)
		return e
	}
	return nil
}

func (t *Topology) Lookup(vid storage.VolumeId) []*DataNode {
	for _, vl := range t.replicaType2VolumeLayout {
		if vl != nil {
			if list := vl.Lookup(vid); list != nil {
				return list
			}
		}
	}
	return nil
}

func (t *Topology) RandomlyReserveOneVolume() (bool, *DataNode, *storage.VolumeId) {
	if t.FreeSpace() <= 0 {
		return false, nil, nil
	}
	vid := t.NextVolumeId()
	ret, node := t.ReserveOneVolume(rand.Intn(t.FreeSpace()), vid)
	return ret, node, &vid
}

func (t *Topology) RandomlyReserveOneVolumeExcept(except []Node) (bool, *DataNode, *storage.VolumeId) {
	freeSpace := t.FreeSpace()
	for _, node := range except {
		freeSpace -= node.FreeSpace()
	}
	if freeSpace <= 0 {
		return false, nil, nil
	}
	vid := t.NextVolumeId()
	ret, node := t.ReserveOneVolume(rand.Intn(freeSpace), vid)
	return ret, node, &vid
}

func (t *Topology) NextVolumeId() storage.VolumeId {
	vid := t.GetMaxVolumeId()
	return vid.Next()
}

func (t *Topology) PickForWrite(repType storage.ReplicationType, count int) (string, int, *DataNode, error) {
	replicationTypeIndex := repType.GetReplicationLevelIndex()
	if t.replicaType2VolumeLayout[replicationTypeIndex] == nil {
		t.replicaType2VolumeLayout[replicationTypeIndex] = NewVolumeLayout(repType, t.volumeSizeLimit, t.pulse)
	}
	vid, count, datanodes, err := t.replicaType2VolumeLayout[replicationTypeIndex].PickForWrite(count)
	if err != nil {
		return "", 0, nil, errors.New("No writable volumes avalable!")
	}
	fileId, count := t.sequence.NextFileId(count)
	return directory.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes.Head(), nil
}

func (t *Topology) GetVolumeLayout(repType storage.ReplicationType) *VolumeLayout {
	replicationTypeIndex := repType.GetReplicationLevelIndex()
	if t.replicaType2VolumeLayout[replicationTypeIndex] == nil {
		t.replicaType2VolumeLayout[replicationTypeIndex] = NewVolumeLayout(repType, t.volumeSizeLimit, t.pulse)
	}
	return t.replicaType2VolumeLayout[replicationTypeIndex]
}

func (t *Topology) RegisterVolumeLayout(v *storage.VolumeInfo, dn *DataNode) {
	t.GetVolumeLayout(v.RepType).RegisterVolume(v, dn)
}

func (t *Topology) RegisterVolumes(init bool, volumeInfos []storage.VolumeInfo, ip string, port int, publicUrl string, maxVolumeCount int) {
	dcName, rackName := t.configuration.Locate(ip)
	dc := t.GetOrCreateDataCenter(dcName)
	rack := dc.GetOrCreateRack(rackName)
	dn := rack.FindDataNode(ip, port)
	if init && dn != nil {
		t.UnRegisterDataNode(dn)
	}
	dn = rack.GetOrCreateDataNode(ip, port, publicUrl, maxVolumeCount)
	for _, v := range volumeInfos {
		dn.AddOrUpdateVolume(v)
		t.RegisterVolumeLayout(&v, dn)
	}
}

func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
	for _, c := range t.Children() {
		dc := c.(*DataCenter)
		if string(dc.Id()) == dcName {
			return dc
		}
	}
	dc := NewDataCenter(dcName)
	t.LinkChildNode(dc)
	return dc
}
<commit_msg>avoid error when missing configuration file<commit_after>package topology

import (
	"code.google.com/p/weed-fs/go/directory"
	"code.google.com/p/weed-fs/go/sequence"
	"code.google.com/p/weed-fs/go/storage"
	"errors"
	"io/ioutil"
	"math/rand"
)

type Topology struct {
	NodeImpl

	//transient vid~servers mapping for each replication type
	replicaType2VolumeLayout []*VolumeLayout

	pulse int64

	volumeSizeLimit uint64

	sequence sequence.Sequencer

	chanDeadDataNodes      chan *DataNode
	chanRecoveredDataNodes chan *DataNode
	chanFullVolumes        chan storage.VolumeInfo

	configuration *Configuration
}

func NewTopology(id string, confFile string, dirname string, sequenceFilename string, volumeSizeLimit uint64, pulse int) (*Topology, error) {
	t := &Topology{}
	t.id = NodeId(id)
	t.nodeType = "Topology"
	t.NodeImpl.value = t
	t.children = make(map[NodeId]Node)
	t.replicaType2VolumeLayout = make([]*VolumeLayout, storage.LengthRelicationType)
	t.pulse = int64(pulse)
	t.volumeSizeLimit = volumeSizeLimit

	t.sequence = sequence.NewSequencer(dirname, sequenceFilename)

	t.chanDeadDataNodes = make(chan *DataNode)
	t.chanRecoveredDataNodes = make(chan *DataNode)
	t.chanFullVolumes = make(chan storage.VolumeInfo)

	err := t.loadConfiguration(confFile)

	return t, err
}

func (t *Topology) loadConfiguration(configurationFile string) error {
	b, e := ioutil.ReadFile(configurationFile)
	if e == nil {
		t.configuration, e = NewConfiguration(b)
		return e
	}else{
    log.Println("Using default configurations.")
	}
	return nil
}

func (t *Topology) Lookup(vid storage.VolumeId) []*DataNode {
	for _, vl := range t.replicaType2VolumeLayout {
		if vl != nil {
			if list := vl.Lookup(vid); list != nil {
				return list
			}
		}
	}
	return nil
}

func (t *Topology) RandomlyReserveOneVolume() (bool, *DataNode, *storage.VolumeId) {
	if t.FreeSpace() <= 0 {
		return false, nil, nil
	}
	vid := t.NextVolumeId()
	ret, node := t.ReserveOneVolume(rand.Intn(t.FreeSpace()), vid)
	return ret, node, &vid
}

func (t *Topology) RandomlyReserveOneVolumeExcept(except []Node) (bool, *DataNode, *storage.VolumeId) {
	freeSpace := t.FreeSpace()
	for _, node := range except {
		freeSpace -= node.FreeSpace()
	}
	if freeSpace <= 0 {
		return false, nil, nil
	}
	vid := t.NextVolumeId()
	ret, node := t.ReserveOneVolume(rand.Intn(freeSpace), vid)
	return ret, node, &vid
}

func (t *Topology) NextVolumeId() storage.VolumeId {
	vid := t.GetMaxVolumeId()
	return vid.Next()
}

func (t *Topology) PickForWrite(repType storage.ReplicationType, count int) (string, int, *DataNode, error) {
	replicationTypeIndex := repType.GetReplicationLevelIndex()
	if t.replicaType2VolumeLayout[replicationTypeIndex] == nil {
		t.replicaType2VolumeLayout[replicationTypeIndex] = NewVolumeLayout(repType, t.volumeSizeLimit, t.pulse)
	}
	vid, count, datanodes, err := t.replicaType2VolumeLayout[replicationTypeIndex].PickForWrite(count)
	if err != nil {
		return "", 0, nil, errors.New("No writable volumes avalable!")
	}
	fileId, count := t.sequence.NextFileId(count)
	return directory.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes.Head(), nil
}

func (t *Topology) GetVolumeLayout(repType storage.ReplicationType) *VolumeLayout {
	replicationTypeIndex := repType.GetReplicationLevelIndex()
	if t.replicaType2VolumeLayout[replicationTypeIndex] == nil {
		t.replicaType2VolumeLayout[replicationTypeIndex] = NewVolumeLayout(repType, t.volumeSizeLimit, t.pulse)
	}
	return t.replicaType2VolumeLayout[replicationTypeIndex]
}

func (t *Topology) RegisterVolumeLayout(v *storage.VolumeInfo, dn *DataNode) {
	t.GetVolumeLayout(v.RepType).RegisterVolume(v, dn)
}

func (t *Topology) RegisterVolumes(init bool, volumeInfos []storage.VolumeInfo, ip string, port int, publicUrl string, maxVolumeCount int) {
	dcName, rackName := t.configuration.Locate(ip)
	dc := t.GetOrCreateDataCenter(dcName)
	rack := dc.GetOrCreateRack(rackName)
	dn := rack.FindDataNode(ip, port)
	if init && dn != nil {
		t.UnRegisterDataNode(dn)
	}
	dn = rack.GetOrCreateDataNode(ip, port, publicUrl, maxVolumeCount)
	for _, v := range volumeInfos {
		dn.AddOrUpdateVolume(v)
		t.RegisterVolumeLayout(&v, dn)
	}
}

func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
	for _, c := range t.Children() {
		dc := c.(*DataCenter)
		if string(dc.Id()) == dcName {
			return dc
		}
	}
	dc := NewDataCenter(dcName)
	t.LinkChildNode(dc)
	return dc
}
<|endoftext|>


<commit_before>package main

import (
	"container/ring"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	_ "net"
	"net/http"
	"regexp"
	"runtime"
	"time"
)

type RSSResult struct {
	XMLName xml.Name `xml:"rss"`
	Titles  []string `xml:"channel>item>title"`
}
type CacheItem struct {
	topic string
	ring  *ring.Ring
}

var cachemap = map[string]*CacheItem{
	"top":           &CacheItem{"", ring.New(10)},
	"world":         &CacheItem{"w", ring.New(10)},
	"US":            &CacheItem{"n", ring.New(10)},
	"business":      &CacheItem{"b", ring.New(10)},
	"technology":    &CacheItem{"t", ring.New(10)},
	"entertainment": &CacheItem{"e", ring.New(10)},
	"sports":        &CacheItem{"s", ring.New(10)},
	"science":       &CacheItem{"snc", ring.New(10)},
	"health":        &CacheItem{"m", ring.New(10)},
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func get_news(key string, topic string) {
	rssurl := "https://news.google.com/news?pz=1&cf=all&ned=us&hl=en&output=rss&topic=" + topic
	log.Printf("Getting news to cachemap[%s] from %s\n", key, rssurl)
	r, _ := http.Get(rssurl)
	defer r.Body.Close()
	b, _ := ioutil.ReadAll(r.Body)
	//ioutil.WriteFile("news.xml", b, 0666) // write rss to disk
	c := regexp.MustCompile("(.+) - [^-]+$")
	//d, _ := xml.NewDecoder(r.Body)
	v := RSSResult{}
	_ = xml.Unmarshal(b, &v)
	for i, _ := range v.Titles { // reverse
		t := v.Titles[len(v.Titles)-1-i]
		st := c.FindAllStringSubmatch(t, -1)
		var s string
		if st == nil {
			s = t
		} else {
			s = st[0][1]
		}
		//fmt.Println(s)
		dup := false
		for _, v := range cachemap {
			v.ring.Do(func(o interface{}) { // check if already exists
				if dup || o == nil {
					return
				}
				if o == s {
					dup = true
				}
			})
		}
		if dup {
			fmt.Printf("%s is removed because of duplication\n", s)
			continue
		}
		cachemap[key].addRing(s)
		fmt.Println(s)
	}
}

func (cacheItem *CacheItem) addRing(value interface{}) {
	cacheItem.ring = cacheItem.ring.Prev()
	cacheItem.ring.Value = value
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU()) // use all physical cores

	http.HandleFunc("/news", newshandler)
	go http.ListenAndServe(":81", nil)

	for {
		log.Println("Refreshing cachemap")
		for k, v := range cachemap {
			get_news(k, v.topic)
		}

		/* //debug purpose code
		for k, v := range cachemap {
			v.ring.Do(func(o interface{}) {
				fmt.Println("cache", k, ":", o)
			})
		} */
		time.Sleep(15 * time.Minute)
	}
}

func newshandler(w http.ResponseWriter, r *http.Request) {
	ori := r.Header.Get("Origin")
	w.Header().Set("Access-Control-Allow-Origin", ori)
	w.Header().Set("Access-Control-Allow-Methods", "GET")
	m := make(map[string][]string)
	for k, v := range cachemap {
		v.ring.Do(func(o interface{}) {
			if o != nil {
				m[k] = append(m[k], o.(string))
			}
		})
	}
	json.NewEncoder(w).Encode(m)
	log.Printf("%s is retrieving cache\n", r.RemoteAddr)
}
<commit_msg>JSON output cache as well<commit_after>package main

import (
	"container/ring"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	_ "net"
	"net/http"
	"regexp"
	"runtime"
	"time"
)

type RSSResult struct {
	XMLName xml.Name `xml:"rss"`
	Titles  []string `xml:"channel>item>title"`
}
type CacheItem struct {
	topic string
	ring  *ring.Ring
}

var cachemap = map[string]*CacheItem{
	"top":           &CacheItem{"", ring.New(10)},
	"world":         &CacheItem{"w", ring.New(10)},
	"US":            &CacheItem{"n", ring.New(10)},
	"business":      &CacheItem{"b", ring.New(10)},
	"technology":    &CacheItem{"t", ring.New(10)},
	"entertainment": &CacheItem{"e", ring.New(10)},
	"sports":        &CacheItem{"s", ring.New(10)},
	"science":       &CacheItem{"snc", ring.New(10)},
	"health":        &CacheItem{"m", ring.New(10)},
}
var cache = make([]byte, 1)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func get_news(key string, topic string) {
	rssurl := "https://news.google.com/news?pz=1&cf=all&ned=us&hl=en&output=rss&topic=" + topic
	log.Printf("Getting news to cachemap[%s] from %s\n", key, rssurl)
	r, _ := http.Get(rssurl)
	defer r.Body.Close()
	b, _ := ioutil.ReadAll(r.Body)
	//ioutil.WriteFile("news.xml", b, 0666) // write rss to disk
	c := regexp.MustCompile("(.+) - [^-]+$")
	//d, _ := xml.NewDecoder(r.Body)
	v := RSSResult{}
	_ = xml.Unmarshal(b, &v)
	for i, _ := range v.Titles { // reverse
		t := v.Titles[len(v.Titles)-1-i]
		st := c.FindAllStringSubmatch(t, -1)
		var s string
		if st == nil {
			s = t
		} else {
			s = st[0][1]
		}
		//fmt.Println(s)
		dup := false
		for _, v := range cachemap {
			v.ring.Do(func(o interface{}) { // check if already exists
				if dup || o == nil {
					return
				}
				if o == s {
					dup = true
				}
			})
		}
		if dup {
			fmt.Printf("%s is removed because of duplication\n", s)
			continue
		}
		cachemap[key].addRing(s)
		fmt.Println(s)
	}
}

func (cacheItem *CacheItem) addRing(value interface{}) {
	cacheItem.ring = cacheItem.ring.Prev()
	cacheItem.ring.Value = value
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU()) // use all physical cores

	http.HandleFunc("/news", newshandler)
	go http.ListenAndServe(":81", nil)

	for {
		log.Println("Refreshing cachemap")
		for k, v := range cachemap {
			get_news(k, v.topic)
		}
		update_cache()
		/* //debug purpose code
		for k, v := range cachemap {
			v.ring.Do(func(o interface{}) {
				fmt.Println("cache", k, ":", o)
			})
		} */
		time.Sleep(15 * time.Minute)
	}
}

func update_cache() {
	m := make(map[string][]string)
	for k, v := range cachemap {
		v.ring.Do(func(o interface{}) {
			if o != nil {
				m[k] = append(m[k], o.(string))
			}
		})
	}
	cache, _ = json.Marshal(m)
}

func newshandler(w http.ResponseWriter, r *http.Request) {
	ori := r.Header.Get("Origin")
	w.Header().Set("Access-Control-Allow-Origin", ori)
	w.Header().Set("Access-Control-Allow-Methods", "GET")
	//json.NewEncoder(w).Encode(m)
	log.Printf("%s is retrieving cache\n", r.RemoteAddr)
	w.Write(cache)
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"regexp"
)

const (
	globalLinkTemplate             = "projects/%s/global/%s/%s"
	globalLinkBasePattern          = "projects/(.+)/global/%s/(.+)"
	zonalLinkTemplate              = "projects/%s/zones/%s/%s/%s"
	zonalLinkBasePattern           = "projects/(.+)/zones/(.+)/%s/(.+)"
	zonalPartialLinkBasePattern    = "zones/(.+)/%s/(.+)"
	regionalLinkTemplate           = "projects/%s/regions/%s/%s/%s"
	regionalLinkBasePattern        = "projects/(.+)/regions/(.+)/%s/(.+)"
	regionalPartialLinkBasePattern = "regions/(.+)/%s/(.+)"
	organizationLinkTemplate       = "organizations/%s/%s/%s"
	organizationBasePattern        = "organizations/(.+)/%s/(.+)"
)

// ------------------------------------------------------------
// Field helpers
// ------------------------------------------------------------

func ParseNetworkFieldValue(network string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("networks", network, "project", d, config, true)
}

func ParseSubnetworkFieldValue(subnetwork string, d TerraformResourceData, config *Config) (*RegionalFieldValue, error) {
	return parseRegionalFieldValue("subnetworks", subnetwork, "project", "region", "zone", d, config, true)
}

func ParseSubnetworkFieldValueWithProjectField(subnetwork, projectField string, d TerraformResourceData, config *Config) (*RegionalFieldValue, error) {
	return parseRegionalFieldValue("subnetworks", subnetwork, projectField, "region", "zone", d, config, true)
}

func ParseSslCertificateFieldValue(sslCertificate string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("sslCertificates", sslCertificate, "project", d, config, false)
}

func ParseHttpHealthCheckFieldValue(healthCheck string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("httpHealthChecks", healthCheck, "project", d, config, false)
}

func ParseDiskFieldValue(disk string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("disks", disk, "project", "zone", d, config, false)
}

func ParseRegionDiskFieldValue(disk string, d TerraformResourceData, config *Config) (*RegionalFieldValue, error) {
	return parseRegionalFieldValue("disks", disk, "project", "region", "zone", d, config, false)
}

func ParseOrganizationCustomRoleName(role string) (*OrganizationFieldValue, error) {
	return parseOrganizationFieldValue("roles", role, false)
}

func ParseAcceleratorFieldValue(accelerator string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("acceleratorTypes", accelerator, "project", "zone", d, config, false)
}

func ParseMachineTypesFieldValue(machineType string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("machineTypes", machineType, "project", "zone", d, config, false)
}

func ParseInstanceGroupFieldValue(instanceGroup string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("instanceGroups", instanceGroup, "project", "zone", d, config, false)
}

func ParseInstanceTemplateFieldValue(instanceTemplate string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("instanceTemplates", instanceTemplate, "project", d, config, false)
}

func ParseSecurityPolicyFieldValue(securityPolicy string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("securityPolicies", securityPolicy, "project", d, config, true)
}

// ------------------------------------------------------------
// Base helpers used to create helpers for specific fields.
// ------------------------------------------------------------

type GlobalFieldValue struct {
	Project string
	Name    string

	resourceType string
}

func (f GlobalFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(globalLinkTemplate, f.Project, f.resourceType, f.Name)
}

// Parses a global field supporting 5 different formats:
// - https://www.googleapis.com/compute/ANY_VERSION/projects/{my_project}/global/{resource_type}/{resource_name}
// - projects/{my_project}/global/{resource_type}/{resource_name}
// - global/{resource_type}/{resource_name}
// - resource_name
// - "" (empty string). RelativeLink() returns empty if isEmptyValid is true.
//
// If the project is not specified, it first tries to get the project from the `projectSchemaField` and then fallback on the default project.
func parseGlobalFieldValue(resourceType, fieldValue, projectSchemaField string, d TerraformResourceData, config *Config, isEmptyValid bool) (*GlobalFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &GlobalFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The global field for resource %s cannot be empty", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(globalLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &GlobalFieldValue{
			Project: parts[1],
			Name:    parts[2],

			resourceType: resourceType,
		}, nil
	}

	project, err := getProjectFromSchema(projectSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	return &GlobalFieldValue{
		Project: project,
		Name:    GetResourceNameFromSelfLink(fieldValue),

		resourceType: resourceType,
	}, nil
}

type ZonalFieldValue struct {
	Project string
	Zone    string
	Name    string

	resourceType string
}

func (f ZonalFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(zonalLinkTemplate, f.Project, f.Zone, f.resourceType, f.Name)
}

// Parses a zonal field supporting 5 different formats:
// - https://www.googleapis.com/compute/ANY_VERSION/projects/{my_project}/zones/{zone}/{resource_type}/{resource_name}
// - projects/{my_project}/zones/{zone}/{resource_type}/{resource_name}
// - zones/{zone}/{resource_type}/{resource_name}
// - resource_name
// - "" (empty string). RelativeLink() returns empty if isEmptyValid is true.
//
// If the project is not specified, it first tries to get the project from the `projectSchemaField` and then fallback on the default project.
// If the zone is not specified, it takes the value of `zoneSchemaField`.
func parseZonalFieldValue(resourceType, fieldValue, projectSchemaField, zoneSchemaField string, d TerraformResourceData, config *Config, isEmptyValid bool) (*ZonalFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &ZonalFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The zonal field for resource %s cannot be empty.", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(zonalLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &ZonalFieldValue{
			Project:      parts[1],
			Zone:         parts[2],
			Name:         parts[3],
			resourceType: resourceType,
		}, nil
	}

	project, err := getProjectFromSchema(projectSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	r = regexp.MustCompile(fmt.Sprintf(zonalPartialLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &ZonalFieldValue{
			Project:      project,
			Zone:         parts[1],
			Name:         parts[2],
			resourceType: resourceType,
		}, nil
	}

	if len(zoneSchemaField) == 0 {
		return nil, fmt.Errorf("Invalid field format. Got '%s', expected format '%s'", fieldValue, fmt.Sprintf(globalLinkTemplate, "{project}", resourceType, "{name}"))
	}

	zone, ok := d.GetOk(zoneSchemaField)
	if !ok {
		zone = config.Zone
		if zone == "" {
			return nil, fmt.Errorf("A zone must be specified")
		}
	}

	return &ZonalFieldValue{
		Project:      project,
		Zone:         zone.(string),
		Name:         GetResourceNameFromSelfLink(fieldValue),
		resourceType: resourceType,
	}, nil
}

func getProjectFromSchema(projectSchemaField string, d TerraformResourceData, config *Config) (string, error) {
	res, ok := d.GetOk(projectSchemaField)
	if ok && projectSchemaField != "" {
		return res.(string), nil
	}
	if config.Project != "" {
		return config.Project, nil
	}
	return "", fmt.Errorf("%s: required field is not set", projectSchemaField)
}

type OrganizationFieldValue struct {
	OrgId string
	Name  string

	resourceType string
}

func (f OrganizationFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(organizationLinkTemplate, f.OrgId, f.resourceType, f.Name)
}

// Parses an organization field with the following formats:
// - organizations/{my_organizations}/{resource_type}/{resource_name}
func parseOrganizationFieldValue(resourceType, fieldValue string, isEmptyValid bool) (*OrganizationFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &OrganizationFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The organization field for resource %s cannot be empty", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(organizationBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &OrganizationFieldValue{
			OrgId: parts[1],
			Name:  parts[2],

			resourceType: resourceType,
		}, nil
	}

	return nil, fmt.Errorf("Invalid field format. Got '%s', expected format '%s'", fieldValue, fmt.Sprintf(organizationLinkTemplate, "{org_id}", resourceType, "{name}"))
}

type RegionalFieldValue struct {
	Project string
	Region  string
	Name    string

	resourceType string
}

func (f RegionalFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(regionalLinkTemplate, f.Project, f.Region, f.resourceType, f.Name)
}

// Parses a regional field supporting 5 different formats:
// - https://www.googleapis.com/compute/ANY_VERSION/projects/{my_project}/regions/{region}/{resource_type}/{resource_name}
// - projects/{my_project}/regions/{region}/{resource_type}/{resource_name}
// - regions/{region}/{resource_type}/{resource_name}
// - resource_name
// - "" (empty string). RelativeLink() returns empty if isEmptyValid is true.
//
// If the project is not specified, it first tries to get the project from the `projectSchemaField` and then fallback on the default project.
// If the region is not specified, see function documentation for `getRegionFromSchema`.
func parseRegionalFieldValue(resourceType, fieldValue, projectSchemaField, regionSchemaField, zoneSchemaField string, d TerraformResourceData, config *Config, isEmptyValid bool) (*RegionalFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &RegionalFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The regional field for resource %s cannot be empty.", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(regionalLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &RegionalFieldValue{
			Project:      parts[1],
			Region:       parts[2],
			Name:         parts[3],
			resourceType: resourceType,
		}, nil
	}

	project, err := getProjectFromSchema(projectSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	r = regexp.MustCompile(fmt.Sprintf(regionalPartialLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &RegionalFieldValue{
			Project:      project,
			Region:       parts[1],
			Name:         parts[2],
			resourceType: resourceType,
		}, nil
	}

	region, err := getRegionFromSchema(regionSchemaField, zoneSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	return &RegionalFieldValue{
		Project:      project,
		Region:       region,
		Name:         GetResourceNameFromSelfLink(fieldValue),
		resourceType: resourceType,
	}, nil
}

// Infers the region based on the following (in order of priority):
// - `regionSchemaField` in resource schema
// - region extracted from the `zoneSchemaField` in resource schema
// - provider-level region
// - region extracted from the provider-level zone
func getRegionFromSchema(regionSchemaField, zoneSchemaField string, d TerraformResourceData, config *Config) (string, error) {
	if v, ok := d.GetOk(regionSchemaField); ok && regionSchemaField != "" {
		return GetResourceNameFromSelfLink(v.(string)), nil
	}
	if v, ok := d.GetOk(zoneSchemaField); ok && zoneSchemaField != "" {
		return getRegionFromZone(v.(string)), nil
	}
	if config.Region != "" {
		return config.Region, nil
	}
	if config.Zone != "" {
		return getRegionFromZone(config.Zone), nil
	}

	return "", fmt.Errorf("Cannot determine region: set in this resource, or set provider-level 'region' or 'zone'.")
}
<commit_msg>Upstream instance iam<commit_after>package google

import (
	"fmt"
	"regexp"
)

const (
	globalLinkTemplate             = "projects/%s/global/%s/%s"
	globalLinkBasePattern          = "projects/(.+)/global/%s/(.+)"
	zonalLinkTemplate              = "projects/%s/zones/%s/%s/%s"
	zonalLinkBasePattern           = "projects/(.+)/zones/(.+)/%s/(.+)"
	zonalPartialLinkBasePattern    = "zones/(.+)/%s/(.+)"
	regionalLinkTemplate           = "projects/%s/regions/%s/%s/%s"
	regionalLinkBasePattern        = "projects/(.+)/regions/(.+)/%s/(.+)"
	regionalPartialLinkBasePattern = "regions/(.+)/%s/(.+)"
	organizationLinkTemplate       = "organizations/%s/%s/%s"
	organizationBasePattern        = "organizations/(.+)/%s/(.+)"
)

// ------------------------------------------------------------
// Field helpers
// ------------------------------------------------------------

func ParseNetworkFieldValue(network string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("networks", network, "project", d, config, true)
}

func ParseSubnetworkFieldValue(subnetwork string, d TerraformResourceData, config *Config) (*RegionalFieldValue, error) {
	return parseRegionalFieldValue("subnetworks", subnetwork, "project", "region", "zone", d, config, true)
}

func ParseSubnetworkFieldValueWithProjectField(subnetwork, projectField string, d TerraformResourceData, config *Config) (*RegionalFieldValue, error) {
	return parseRegionalFieldValue("subnetworks", subnetwork, projectField, "region", "zone", d, config, true)
}

func ParseSslCertificateFieldValue(sslCertificate string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("sslCertificates", sslCertificate, "project", d, config, false)
}

func ParseHttpHealthCheckFieldValue(healthCheck string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("httpHealthChecks", healthCheck, "project", d, config, false)
}

func ParseDiskFieldValue(disk string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("disks", disk, "project", "zone", d, config, false)
}

func ParseRegionDiskFieldValue(disk string, d TerraformResourceData, config *Config) (*RegionalFieldValue, error) {
	return parseRegionalFieldValue("disks", disk, "project", "region", "zone", d, config, false)
}

func ParseOrganizationCustomRoleName(role string) (*OrganizationFieldValue, error) {
	return parseOrganizationFieldValue("roles", role, false)
}

func ParseAcceleratorFieldValue(accelerator string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("acceleratorTypes", accelerator, "project", "zone", d, config, false)
}

func ParseMachineTypesFieldValue(machineType string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("machineTypes", machineType, "project", "zone", d, config, false)
}

func ParseInstanceFieldValue(instance string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("instances", instance, "project", "zone", d, config, false)
}

func ParseInstanceGroupFieldValue(instanceGroup string, d TerraformResourceData, config *Config) (*ZonalFieldValue, error) {
	return parseZonalFieldValue("instanceGroups", instanceGroup, "project", "zone", d, config, false)
}

func ParseInstanceTemplateFieldValue(instanceTemplate string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("instanceTemplates", instanceTemplate, "project", d, config, false)
}

func ParseSecurityPolicyFieldValue(securityPolicy string, d TerraformResourceData, config *Config) (*GlobalFieldValue, error) {
	return parseGlobalFieldValue("securityPolicies", securityPolicy, "project", d, config, true)
}

// ------------------------------------------------------------
// Base helpers used to create helpers for specific fields.
// ------------------------------------------------------------

type GlobalFieldValue struct {
	Project string
	Name    string

	resourceType string
}

func (f GlobalFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(globalLinkTemplate, f.Project, f.resourceType, f.Name)
}

// Parses a global field supporting 5 different formats:
// - https://www.googleapis.com/compute/ANY_VERSION/projects/{my_project}/global/{resource_type}/{resource_name}
// - projects/{my_project}/global/{resource_type}/{resource_name}
// - global/{resource_type}/{resource_name}
// - resource_name
// - "" (empty string). RelativeLink() returns empty if isEmptyValid is true.
//
// If the project is not specified, it first tries to get the project from the `projectSchemaField` and then fallback on the default project.
func parseGlobalFieldValue(resourceType, fieldValue, projectSchemaField string, d TerraformResourceData, config *Config, isEmptyValid bool) (*GlobalFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &GlobalFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The global field for resource %s cannot be empty", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(globalLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &GlobalFieldValue{
			Project: parts[1],
			Name:    parts[2],

			resourceType: resourceType,
		}, nil
	}

	project, err := getProjectFromSchema(projectSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	return &GlobalFieldValue{
		Project: project,
		Name:    GetResourceNameFromSelfLink(fieldValue),

		resourceType: resourceType,
	}, nil
}

type ZonalFieldValue struct {
	Project string
	Zone    string
	Name    string

	resourceType string
}

func (f ZonalFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(zonalLinkTemplate, f.Project, f.Zone, f.resourceType, f.Name)
}

// Parses a zonal field supporting 5 different formats:
// - https://www.googleapis.com/compute/ANY_VERSION/projects/{my_project}/zones/{zone}/{resource_type}/{resource_name}
// - projects/{my_project}/zones/{zone}/{resource_type}/{resource_name}
// - zones/{zone}/{resource_type}/{resource_name}
// - resource_name
// - "" (empty string). RelativeLink() returns empty if isEmptyValid is true.
//
// If the project is not specified, it first tries to get the project from the `projectSchemaField` and then fallback on the default project.
// If the zone is not specified, it takes the value of `zoneSchemaField`.
func parseZonalFieldValue(resourceType, fieldValue, projectSchemaField, zoneSchemaField string, d TerraformResourceData, config *Config, isEmptyValid bool) (*ZonalFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &ZonalFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The zonal field for resource %s cannot be empty.", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(zonalLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &ZonalFieldValue{
			Project:      parts[1],
			Zone:         parts[2],
			Name:         parts[3],
			resourceType: resourceType,
		}, nil
	}

	project, err := getProjectFromSchema(projectSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	r = regexp.MustCompile(fmt.Sprintf(zonalPartialLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &ZonalFieldValue{
			Project:      project,
			Zone:         parts[1],
			Name:         parts[2],
			resourceType: resourceType,
		}, nil
	}

	if len(zoneSchemaField) == 0 {
		return nil, fmt.Errorf("Invalid field format. Got '%s', expected format '%s'", fieldValue, fmt.Sprintf(globalLinkTemplate, "{project}", resourceType, "{name}"))
	}

	zone, ok := d.GetOk(zoneSchemaField)
	if !ok {
		zone = config.Zone
		if zone == "" {
			return nil, fmt.Errorf("A zone must be specified")
		}
	}

	return &ZonalFieldValue{
		Project:      project,
		Zone:         zone.(string),
		Name:         GetResourceNameFromSelfLink(fieldValue),
		resourceType: resourceType,
	}, nil
}

func getProjectFromSchema(projectSchemaField string, d TerraformResourceData, config *Config) (string, error) {
	res, ok := d.GetOk(projectSchemaField)
	if ok && projectSchemaField != "" {
		return res.(string), nil
	}
	if config.Project != "" {
		return config.Project, nil
	}
	return "", fmt.Errorf("%s: required field is not set", projectSchemaField)
}

type OrganizationFieldValue struct {
	OrgId string
	Name  string

	resourceType string
}

func (f OrganizationFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(organizationLinkTemplate, f.OrgId, f.resourceType, f.Name)
}

// Parses an organization field with the following formats:
// - organizations/{my_organizations}/{resource_type}/{resource_name}
func parseOrganizationFieldValue(resourceType, fieldValue string, isEmptyValid bool) (*OrganizationFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &OrganizationFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The organization field for resource %s cannot be empty", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(organizationBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &OrganizationFieldValue{
			OrgId: parts[1],
			Name:  parts[2],

			resourceType: resourceType,
		}, nil
	}

	return nil, fmt.Errorf("Invalid field format. Got '%s', expected format '%s'", fieldValue, fmt.Sprintf(organizationLinkTemplate, "{org_id}", resourceType, "{name}"))
}

type RegionalFieldValue struct {
	Project string
	Region  string
	Name    string

	resourceType string
}

func (f RegionalFieldValue) RelativeLink() string {
	if len(f.Name) == 0 {
		return ""
	}

	return fmt.Sprintf(regionalLinkTemplate, f.Project, f.Region, f.resourceType, f.Name)
}

// Parses a regional field supporting 5 different formats:
// - https://www.googleapis.com/compute/ANY_VERSION/projects/{my_project}/regions/{region}/{resource_type}/{resource_name}
// - projects/{my_project}/regions/{region}/{resource_type}/{resource_name}
// - regions/{region}/{resource_type}/{resource_name}
// - resource_name
// - "" (empty string). RelativeLink() returns empty if isEmptyValid is true.
//
// If the project is not specified, it first tries to get the project from the `projectSchemaField` and then fallback on the default project.
// If the region is not specified, see function documentation for `getRegionFromSchema`.
func parseRegionalFieldValue(resourceType, fieldValue, projectSchemaField, regionSchemaField, zoneSchemaField string, d TerraformResourceData, config *Config, isEmptyValid bool) (*RegionalFieldValue, error) {
	if len(fieldValue) == 0 {
		if isEmptyValid {
			return &RegionalFieldValue{resourceType: resourceType}, nil
		}
		return nil, fmt.Errorf("The regional field for resource %s cannot be empty.", resourceType)
	}

	r := regexp.MustCompile(fmt.Sprintf(regionalLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &RegionalFieldValue{
			Project:      parts[1],
			Region:       parts[2],
			Name:         parts[3],
			resourceType: resourceType,
		}, nil
	}

	project, err := getProjectFromSchema(projectSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	r = regexp.MustCompile(fmt.Sprintf(regionalPartialLinkBasePattern, resourceType))
	if parts := r.FindStringSubmatch(fieldValue); parts != nil {
		return &RegionalFieldValue{
			Project:      project,
			Region:       parts[1],
			Name:         parts[2],
			resourceType: resourceType,
		}, nil
	}

	region, err := getRegionFromSchema(regionSchemaField, zoneSchemaField, d, config)
	if err != nil {
		return nil, err
	}

	return &RegionalFieldValue{
		Project:      project,
		Region:       region,
		Name:         GetResourceNameFromSelfLink(fieldValue),
		resourceType: resourceType,
	}, nil
}

// Infers the region based on the following (in order of priority):
// - `regionSchemaField` in resource schema
// - region extracted from the `zoneSchemaField` in resource schema
// - provider-level region
// - region extracted from the provider-level zone
func getRegionFromSchema(regionSchemaField, zoneSchemaField string, d TerraformResourceData, config *Config) (string, error) {
	if v, ok := d.GetOk(regionSchemaField); ok && regionSchemaField != "" {
		return GetResourceNameFromSelfLink(v.(string)), nil
	}
	if v, ok := d.GetOk(zoneSchemaField); ok && zoneSchemaField != "" {
		return getRegionFromZone(v.(string)), nil
	}
	if config.Region != "" {
		return config.Region, nil
	}
	if config.Zone != "" {
		return getRegionFromZone(config.Zone), nil
	}

	return "", fmt.Errorf("Cannot determine region: set in this resource, or set provider-level 'region' or 'zone'.")
}
<|endoftext|>


<commit_before>package similarity

// An item that has been rated.
type Item struct {
	Name  string
	Value float64
}

// Similarity is a similarity storage and retrieval engine.
type Similarity struct {
	data map[string][]Item
}

// Create a new Similarity engine.
func NewSimilarity() *Similarity {
	return &Similarity{make(map[string][]Item)}
}

// Add an Item to the engine with a key.
func (sim *Similarity) Add(key string, item Item) {
	sim.data[key] = append(sim.data[key], item)
}

// Get all the keys in this Similarity.
func (sim *Similarity) Keys() []string {
	keys := make([]string, 0, len(sim.data))
	for k, _ := range(sim.data) {
		keys = append(keys, k)
	}
	return keys
}
<commit_msg>Added RWMutex for read and write locking.<commit_after>package similarity

import (
	"sync"
)

// An item that has been rated.
type Item struct {
	Name  string
	Value float64
}

// Similarity is a similarity storage and retrieval engine.
type Similarity struct {
	sync.RWMutex
	data map[string][]Item
}

// Create a new Similarity engine.
func NewSimilarity() *Similarity {
	return &Similarity{make(map[string][]Item)}
}

// Add an Item to the engine with a key.
func (sim *Similarity) Add(key string, item Item) {
	sim.Lock()
	sim.data[key] = append(sim.data[key], item)
	sim.Unlock()
}

// Get all the keys in this Similarity.
func (sim *Similarity) Keys() []string {
	sim.RLock()
	keys := make([]string, 0, len(sim.data))
	for k, _ := range sim.data {
		keys = append(keys, k)
	}
	sim.RUnlock()
	return keys
}
<|endoftext|>


<commit_before>// Package lib contains functions useful for most programs.
package lib

import (
	"fmt"
	"os"
	"path/filepath"
)

var progname = filepath.Base(os.Args[0])

// Warnx displays a formatted error message to standard error, à la
// warnx(3).
func Warnx(format string, a ...interface{}) (int, error) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += "\n"
	return fmt.Fprintf(os.Stderr, format, a...)
}

// Warn displays a formatted error message to standard output,
// appending the error string, à la warn(3).
func Warn(err error, format string, a ...interface{}) (int, error) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += ": %v\n"
	a = append(a, err)
	return fmt.Fprintf(os.Stderr, format, a...)
}

// Errx displays a formatted error message to standard error and exits
// with the status code from `exit`, à la errx(3).
func Errx(exit int, format string, a ...interface{}) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += "\n"
	fmt.Fprintf(os.Stderr, format, a...)
	os.Exit(exit)
}

// Err displays a formatting error message to standard error,
// appending the error string, and exits with the status code from
// `exit`, à la err(3).
func Err(exit int, err error, format string, a ...interface{}) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += ": %v\n"
	a = append(a, err)
	fmt.Fprintf(os.Stderr, format, a...)
	os.Exit(exit)
}

// CheckFatal calls Err if err isn't nil.
func CheckFatal(err error, format string, a ...interface{}) {
	if err == nil {
		return
	}

	Err(ExitFailure, err, format, a...)
}
<commit_msg>Add Itoa.<commit_after>// Package lib contains functions useful for most programs.
package lib

import (
	"fmt"
	"os"
	"path/filepath"
)

var progname = filepath.Base(os.Args[0])

// Warnx displays a formatted error message to standard error, à la
// warnx(3).
func Warnx(format string, a ...interface{}) (int, error) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += "\n"
	return fmt.Fprintf(os.Stderr, format, a...)
}

// Warn displays a formatted error message to standard output,
// appending the error string, à la warn(3).
func Warn(err error, format string, a ...interface{}) (int, error) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += ": %v\n"
	a = append(a, err)
	return fmt.Fprintf(os.Stderr, format, a...)
}

// Errx displays a formatted error message to standard error and exits
// with the status code from `exit`, à la errx(3).
func Errx(exit int, format string, a ...interface{}) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += "\n"
	fmt.Fprintf(os.Stderr, format, a...)
	os.Exit(exit)
}

// Err displays a formatting error message to standard error,
// appending the error string, and exits with the status code from
// `exit`, à la err(3).
func Err(exit int, err error, format string, a ...interface{}) {
	format = fmt.Sprintf("[%s] %s", progname, format)
	format += ": %v\n"
	a = append(a, err)
	fmt.Fprintf(os.Stderr, format, a...)
	os.Exit(exit)
}

// CheckFatal calls Err if err isn't nil.
func CheckFatal(err error, format string, a ...interface{}) {
	if err == nil {
		return
	}

	Err(ExitFailure, err, format, a...)
}

// Itoa provides cheap integer to fixed-width decimal ASCII.  Give a
// negative width to avoid zero-padding. Adapted from the 'itoa'
// function in the log/log.go file in the standard library.
func Itoa(i int, wid int) string {
	// Assemble decimal in reverse order.
	var b [20]byte
	bp := len(b) - 1
	for i >= 10 || wid > 1 {
		wid--
		q := i / 10
		b[bp] = byte('0' + i - q*10)
		bp--
		i = q
	}
	// i < 10
	b[bp] = byte('0' + i)
	return string(b[bp:])
}
<|endoftext|>


<commit_before>package main

import (
	"C"
	"log"

	"amp_cache_url"
)

//export fib
func fib(n int) int {
	if n < 2 {
		return n
	}
	return fib(n-2) + fib(n-1)
}

func init() {
}

func main() {
}
<commit_msg>implement lib.so<commit_after>package main

import (
	"C"

	"github.com/blueoceans/amp-cache-url"
)

func init() {
}

func main() {
}

//export isCacheUrl
func isCacheUrl(rawurl string) bool {
	return amp_cache_url.IsCacheUrl(rawurl)
}

//export getCacheUrl
func getCacheUrl(rawurl string) (string, error) {
	return amp_cache_url.GetCacheUrl(rawurl)
}

//export getOriginUrl
func getOriginUrl(rawurl string) (string, error) {
	return amp_cache_url.GetOriginUrl(rawurl)
}
<|endoftext|>


<commit_before>package handler

import "testing"

var fakeHandler = Conplicity{}

func TestSetup(t *testing.T) {
	fakeHandler.Setup("noversion")

	// Check Hostname
	if fakeHandler.Hostname == "" {
		t.Fatal("Hostname should not be nil")
	}

	// Check Client
	expectedEndpoint := "unix:///var/run/docker.sock"
	gotEndpoint := fakeHandler.Client.Endpoint()
	if gotEndpoint != expectedEndpoint {
		t.Fatalf("Expected %s, got %s", expectedEndpoint, gotEndpoint)
	}
}
<commit_msg>Test loglevel setting<commit_after>package handler

import (
	"testing"

	log "github.com/Sirupsen/logrus"
)

var fakeHandler = Conplicity{}

func TestSetup(t *testing.T) {
	fakeHandler.Setup("noversion")

	// Check Hostname
	if fakeHandler.Hostname == "" {
		t.Fatal("Hostname should not be nil")
	}

	// Check Client
	expectedEndpoint := "unix:///var/run/docker.sock"
	gotEndpoint := fakeHandler.Client.Endpoint()
	if gotEndpoint != expectedEndpoint {
		t.Fatalf("Expected %s, got %s", expectedEndpoint, gotEndpoint)
	}

	// Check default Loglevel
	if l := log.GetLevel(); l != log.InfoLevel {
		t.Fatalf("Expected %v loglevel by default, got %v", log.InfoLevel, l)
	}

	// Check setting Loglevel
	fakeHandler.Config.Loglevel = "debug"
	fakeHandler.setupLoglevel()
	if l := log.GetLevel(); l != log.DebugLevel {
		t.Fatalf("Expected %v loglevel, got %v", log.DebugLevel, l)
	}

	// Check setting Loglevel to wrong value
	fakeHandler.Config.Loglevel = "wrong"
	err := fakeHandler.setupLoglevel()
	if err == nil {
		t.Fatal("Expected setupLoglevel to fail")
	}
}
<|endoftext|>


<commit_before>package hashmap

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"io"
	"testing"
)

func TestMapWithBkts(t *testing.T) {
	bkts := make([]*Entry, 3, 3)
	_, err := NewWithBkts(bkts)
	if err == nil {
		t.Fatalf("Buckets size of %d should have failed\n", len(bkts))
	}
	bkts = make([]*Entry, 8, 8)
	_, err = NewWithBkts(bkts)
	if err != nil {
		t.Fatalf("Buckets size of %d should have succeeded\n", len(bkts))
	}
}

var foo = []byte("foo")
var bar = []byte("bar")
var baz = []byte("baz")
var med = []byte("foo.bar.baz")
var sub = []byte("apcera.continuum.router.foo.bar.baz")

func TestHashMapBasics(t *testing.T) {
	h := New()

	if h.used != 0 {
		t.Fatalf("Wrong number of entries: %d vs 0\n", h.used)
	}
	h.Set(foo, bar)
	if h.used != 1 {
		t.Fatalf("Wrong number of entries: %d vs 1\n", h.used)
	}
	if v := h.Get(foo).([]byte); !bytes.Equal(v, bar) {
		t.Fatalf("Did not receive correct answer: '%s' vs '%s'\n", bar, v)
	}
	h.Remove(foo)
	if h.used != 0 {
		t.Fatalf("Wrong number of entries: %d vs 0\n", h.used)
	}
	if v := h.Get(foo); v != nil {
		t.Fatal("Did not receive correct answer, should be nil")
	}
}

const (
	INS  = 100
	EXP  = 128
	REM  = 75
	EXP2 = 64
)

func TestGrowing(t *testing.T) {
	h := New()

	if len(h.bkts) != _BSZ {
		t.Fatalf("Initial bucket size is wrong: %d vs %d\n", len(h.bkts), _BSZ)
	}
	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}
	if len(h.bkts) != EXP {
		t.Fatalf("Expanded bucket size is wrong: %d vs %d\n", len(h.bkts), EXP)
	}
}

func TestHashMapCollisions(t *testing.T) {
	h := New()
	h.rsz = false

	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}
	if len(h.bkts) != _BSZ {
		t.Fatalf("Bucket size is wrong: %d vs %d\n", len(h.bkts), _BSZ)
	}
	h.grow()
	if len(h.bkts) != 2*_BSZ {
		t.Fatalf("Bucket size is wrong: %d vs %d\n", len(h.bkts), 2*_BSZ)
	}
	ti := 32
	tg := h.Get(toks[ti]).([]byte)
	if !bytes.Equal(tg, toks[ti]) {
		t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[ti])
	}

	h.Remove(toks[99])
	rg := h.Get(toks[99])
	if rg != nil {
		t.Fatalf("After remove should have been nil! '%s'\n", rg.([]byte))
	}
}

func TestAll(t *testing.T) {
	h := New()
	h.Set([]byte("1"), 1)
	h.Set([]byte("2"), 1)
	h.Set([]byte("3"), 1)
	all := h.All()
	if len(all) != 3 {
		t.Fatalf("Expected All() to return 3, but got %d\n", len(all))
	}
	allkeys := h.AllKeys()
	if len(allkeys) != 3 {
		t.Fatalf("Expected All() to return 3, but got %d\n", len(allkeys))
	}
}

func TestSetDoesReplaceOnExisting(t *testing.T) {
	h := New()
	k := []byte("key")
	h.Set(k, "foo")
	h.Set(k, "bar")
	all := h.All()
	if len(all) != 1 {
		t.Fatalf("Set should replace, expected 1 vs %d\n", len(all))
	}
	s, ok := all[0].(string)
	if !ok {
		t.Fatalf("Value is incorrect: %v\n", all[0].(string))
	}
	if s != "bar" {
		t.Fatalf("Value is incorrect, expected 'bar' vs '%s'\n", s)
	}
}

func TestCollision(t *testing.T) {
	h := New()
	k1 := []byte("999")
	k2 := []byte("1000")
	h.Set(k1, "foo")
	h.Set(k2, "bar")
	all := h.All()
	if len(all) != 2 {
		t.Fatalf("Expected 2 vs %d\n", len(all))
	}
	if h.Get(k1) == nil {
		t.Fatalf("Failed to get '999'\n")
	}
}

func TestHashMapStats(t *testing.T) {
	h := New()
	h.rsz = false

	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}

	s := h.Stats()
	if s.NumElements != INS {
		t.Fatalf("NumElements incorrect: %d vs %d\n", s.NumElements, INS)
	}
	if s.NumBuckets != _BSZ {
		t.Fatalf("NumBuckets incorrect: %d vs %d\n", s.NumBuckets, _BSZ)
	}
	if s.AvgChain > 13 || s.AvgChain < 12 {
		t.Fatalf("AvgChain out of bounds: %f vs %f\n", s.AvgChain, 12.5)
	}
	if s.LongChain > 25 {
		t.Fatalf("LongChain out of bounds: %d vs %d\n", s.LongChain, 22)
	}
}

func TestShrink(t *testing.T) {
	h := New()

	if len(h.bkts) != _BSZ {
		t.Fatalf("Initial bucket size is wrong: %d vs %d\n", len(h.bkts), _BSZ)
	}
	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}
	if len(h.bkts) != EXP {
		t.Fatalf("Expanded bucket size is wrong: %d vs %d\n", len(h.bkts), EXP)
	}
	for i := 0; i < REM; i++ {
		h.Remove(toks[i])
	}
	if len(h.bkts) != EXP2 {
		t.Fatalf("Shrunk bucket size is wrong: %d vs %d\n", len(h.bkts), EXP2)
	}
}

func TestFalseLookup(t *testing.T) {
	h := New()
	// DW + W
	h.Set([]byte("cache.test.0"), "foo")
	v := h.Get([]byte("cache.test.1"))
	if v != nil {
		t.Fatalf("Had a match when did not expect one!\n")
	}
	// DW + W + 3
	h.Set([]byte("cache.test.1234"), "foo")
	v = h.Get([]byte("cache.test.0000"))
	if v != nil {
		t.Fatalf("Had a match when did not expect one!\n")
	}
}

func Benchmark_GoMap___GetSmallKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := make(map[string][]byte)
	m["foo"] = bar
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m["foo"]
	}
}

func Benchmark_HashMap_GetSmallKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	m.Set(foo, bar)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m.Get(foo)
	}
}

func Benchmark_GoMap____GetMedKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	ts := string(med)
	m := make(map[string][]byte)
	m[ts] = bar
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m[ts]
	}
}

func Benchmark_HashMap__GetMedKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	m.Set(sub, bar)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m.Get(med)
	}
}

func Benchmark_GoMap____GetLrgKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	ts := string(sub)
	m := make(map[string][]byte)
	m[ts] = bar
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m[ts]
	}
}

func Benchmark_HashMap__GetLrgKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	m.Set(sub, bar)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m.Get(sub)
	}
}

func Benchmark_HashMap_______Set(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		m.Set(foo, bar)
	}
}
<commit_msg>Test RemoveRandom()<commit_after>package hashmap

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"io"
	"testing"
)

func TestMapWithBkts(t *testing.T) {
	bkts := make([]*Entry, 3, 3)
	_, err := NewWithBkts(bkts)
	if err == nil {
		t.Fatalf("Buckets size of %d should have failed\n", len(bkts))
	}
	bkts = make([]*Entry, 8, 8)
	_, err = NewWithBkts(bkts)
	if err != nil {
		t.Fatalf("Buckets size of %d should have succeeded\n", len(bkts))
	}
}

var foo = []byte("foo")
var bar = []byte("bar")
var baz = []byte("baz")
var med = []byte("foo.bar.baz")
var sub = []byte("apcera.continuum.router.foo.bar.baz")

func TestHashMapBasics(t *testing.T) {
	h := New()

	if h.used != 0 {
		t.Fatalf("Wrong number of entries: %d vs 0\n", h.used)
	}
	h.Set(foo, bar)
	if h.used != 1 {
		t.Fatalf("Wrong number of entries: %d vs 1\n", h.used)
	}
	if v := h.Get(foo).([]byte); !bytes.Equal(v, bar) {
		t.Fatalf("Did not receive correct answer: '%s' vs '%s'\n", bar, v)
	}
	h.Remove(foo)
	if h.used != 0 {
		t.Fatalf("Wrong number of entries: %d vs 0\n", h.used)
	}
	if v := h.Get(foo); v != nil {
		t.Fatal("Did not receive correct answer, should be nil")
	}
}

const (
	INS  = 100
	EXP  = 128
	REM  = 75
	EXP2 = 64
)

func TestGrowing(t *testing.T) {
	h := New()

	if len(h.bkts) != _BSZ {
		t.Fatalf("Initial bucket size is wrong: %d vs %d\n", len(h.bkts), _BSZ)
	}
	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}
	if len(h.bkts) != EXP {
		t.Fatalf("Expanded bucket size is wrong: %d vs %d\n", len(h.bkts), EXP)
	}
}

func TestHashMapCollisions(t *testing.T) {
	h := New()
	h.rsz = false

	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}
	if len(h.bkts) != _BSZ {
		t.Fatalf("Bucket size is wrong: %d vs %d\n", len(h.bkts), _BSZ)
	}
	h.grow()
	if len(h.bkts) != 2*_BSZ {
		t.Fatalf("Bucket size is wrong: %d vs %d\n", len(h.bkts), 2*_BSZ)
	}
	ti := 32
	tg := h.Get(toks[ti]).([]byte)
	if !bytes.Equal(tg, toks[ti]) {
		t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[ti])
	}

	h.Remove(toks[99])
	rg := h.Get(toks[99])
	if rg != nil {
		t.Fatalf("After remove should have been nil! '%s'\n", rg.([]byte))
	}
}

func TestAll(t *testing.T) {
	h := New()
	h.Set([]byte("1"), 1)
	h.Set([]byte("2"), 1)
	h.Set([]byte("3"), 1)
	all := h.All()
	if len(all) != 3 {
		t.Fatalf("Expected All() to return 3, but got %d\n", len(all))
	}
	allkeys := h.AllKeys()
	if len(allkeys) != 3 {
		t.Fatalf("Expected All() to return 3, but got %d\n", len(allkeys))
	}
}

func TestSetDoesReplaceOnExisting(t *testing.T) {
	h := New()
	k := []byte("key")
	h.Set(k, "foo")
	h.Set(k, "bar")
	all := h.All()
	if len(all) != 1 {
		t.Fatalf("Set should replace, expected 1 vs %d\n", len(all))
	}
	s, ok := all[0].(string)
	if !ok {
		t.Fatalf("Value is incorrect: %v\n", all[0].(string))
	}
	if s != "bar" {
		t.Fatalf("Value is incorrect, expected 'bar' vs '%s'\n", s)
	}
}

func TestCollision(t *testing.T) {
	h := New()
	k1 := []byte("999")
	k2 := []byte("1000")
	h.Set(k1, "foo")
	h.Set(k2, "bar")
	all := h.All()
	if len(all) != 2 {
		t.Fatalf("Expected 2 vs %d\n", len(all))
	}
	if h.Get(k1) == nil {
		t.Fatalf("Failed to get '999'\n")
	}
}

func TestHashMapStats(t *testing.T) {
	h := New()
	h.rsz = false

	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}

	s := h.Stats()
	if s.NumElements != INS {
		t.Fatalf("NumElements incorrect: %d vs %d\n", s.NumElements, INS)
	}
	if s.NumBuckets != _BSZ {
		t.Fatalf("NumBuckets incorrect: %d vs %d\n", s.NumBuckets, _BSZ)
	}
	if s.AvgChain > 13 || s.AvgChain < 12 {
		t.Fatalf("AvgChain out of bounds: %f vs %f\n", s.AvgChain, 12.5)
	}
	if s.LongChain > 25 {
		t.Fatalf("LongChain out of bounds: %d vs %d\n", s.LongChain, 22)
	}
}

func TestShrink(t *testing.T) {
	h := New()

	if len(h.bkts) != _BSZ {
		t.Fatalf("Initial bucket size is wrong: %d vs %d\n", len(h.bkts), _BSZ)
	}
	// Create _INBOX style end tokens
	var toks [INS][]byte
	for i, _ := range toks {
		u := make([]byte, 13)
		io.ReadFull(rand.Reader, u)
		toks[i] = []byte(hex.EncodeToString(u))
		h.Set(toks[i], toks[i])
		tg := h.Get(toks[i]).([]byte)
		if !bytes.Equal(tg, toks[i]) {
			t.Fatalf("Did not match properly, '%s' vs '%s'\n", tg, toks[i])
		}
	}
	if len(h.bkts) != EXP {
		t.Fatalf("Expanded bucket size is wrong: %d vs %d\n", len(h.bkts), EXP)
	}
	for i := 0; i < REM; i++ {
		h.Remove(toks[i])
	}
	if len(h.bkts) != EXP2 {
		t.Fatalf("Shrunk bucket size is wrong: %d vs %d\n", len(h.bkts), EXP2)
	}
}

func TestFalseLookup(t *testing.T) {
	h := New()
	// DW + W
	h.Set([]byte("cache.test.0"), "foo")
	v := h.Get([]byte("cache.test.1"))
	if v != nil {
		t.Fatalf("Had a match when did not expect one!\n")
	}
	// DW + W + 3
	h.Set([]byte("cache.test.1234"), "foo")
	v = h.Get([]byte("cache.test.0000"))
	if v != nil {
		t.Fatalf("Had a match when did not expect one!\n")
	}
}

func TestRemoveRandom(t *testing.T) {
	h := New()
	h.RemoveRandom()

	h.Set(foo, "1")
	h.Set(bar, "1")
	h.Set(baz, "1")

	if h.Count() != 3 {
		t.Fatalf("Expected 3 members, got %d\n", h.Count())
	}

	h.RemoveRandom()

	if h.Count() != 2 {
		t.Fatalf("Expected 2 members, got %d\n", h.Count())
	}
}

func Benchmark_GoMap___GetSmallKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := make(map[string][]byte)
	m["foo"] = bar
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m["foo"]
	}
}

func Benchmark_HashMap_GetSmallKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	m.Set(foo, bar)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m.Get(foo)
	}
}

func Benchmark_GoMap____GetMedKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	ts := string(med)
	m := make(map[string][]byte)
	m[ts] = bar
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m[ts]
	}
}

func Benchmark_HashMap__GetMedKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	m.Set(sub, bar)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m.Get(med)
	}
}

func Benchmark_GoMap____GetLrgKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	ts := string(sub)
	m := make(map[string][]byte)
	m[ts] = bar
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m[ts]
	}
}

func Benchmark_HashMap__GetLrgKey(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	m.Set(sub, bar)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_ = m.Get(sub)
	}
}

func Benchmark_HashMap_______Set(b *testing.B) {
	b.StopTimer()
	b.SetBytes(1)
	m := New()
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		m.Set(foo, bar)
	}
}
<|endoftext|>