<commit_before>package buffer

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

func BenchmarkMemory(b *testing.B) {
	buf := New(32 * 1024)
	for i := 0; i < b.N; i++ {
		io.Copy(buf, io.LimitReader(rand.Reader, 32*1024))
		io.Copy(ioutil.Discard, buf)
	}
}

func TestPool(t *testing.T) {
	pool := NewPool(func() Buffer { return New(10) })
	buf := pool.Get()
	buf.Write([]byte("hello world"))
	pool.Put(buf)
}

func TestMemPool(t *testing.T) {
	pool := NewMemPool(10)
	buf := pool.Get()
	if n, err := buf.Write([]byte("hello world")); n != 10 {
		t.Errorf("wrote incorrect amount")
	} else if err == nil {
		t.Errorf("should have been a shortwrite error here")
	}
}

func TestFilePool(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("there should have been a panic here!")
		}
	}()

	pool := NewFilePool(1024, "::~_bad_dir_~::")
	pool.Get()
}

func TestOverflow(t *testing.T) {
	buf := NewMulti(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))

	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}

	if !bytes.Equal(data, []byte("Hello")) {
		t.Errorf("Expected Hello got %s", string(data))
	}

}

func TestWriteAt(t *testing.T) {
	var b BufferAt

	b = New(5)
	BufferAtTester(t, b)

	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	b = NewFile(5, file)
	BufferAtTester(t, b)

}

func BufferAtTester(t *testing.T, b BufferAt) {
	b.WriteAt([]byte("abc"), 0)
	Compare(t, b, "abc")

	b.WriteAt([]byte("abc"), 1)
	Compare(t, b, "aabc")

	b.WriteAt([]byte("abc"), 2)
	Compare(t, b, "aaabc")

	b.WriteAt([]byte("abc"), 3)
	Compare(t, b, "aaaab")

	b.Read(make([]byte, 2))

	Compare(t, b, "aab")
	b.Reset()
}

func Compare(t *testing.T, b BufferAt, s string) {
	data := make([]byte, b.Len())
	n, _ := b.ReadAt(data, 0)
	if string(data[:n]) != s {
		t.Error("Mismatch:", string(data[:n]), s)
	}
}

func TestRing(t *testing.T) {
	ring := NewRing(New(3))
	if ring.Len() != 0 {
		t.Errorf("Ring non-empty start!")
	}

	if ring.Cap() != MAXINT64 {
		t.Errorf("Ring has < max capacity")
	}

	ring.Write([]byte("abc"))
	if ring.Len() != 3 {
		t.Errorf("expected ring len == 3")
	}

	ring.Write([]byte("de"))
	if ring.Len() != 3 {
		t.Errorf("expected riing len == 3")
	}

	data := make([]byte, 12)
	if n, err := ring.Read(data); err != nil {
		t.Error(err.Error())
	} else {
		if !bytes.Equal(data[:n], []byte("cde")) {
			t.Errorf("expected cde, got %s", data[:n])
		}
	}

	if ring.Len() != 0 {
		t.Errorf("ring should now be empty")
	}

	ring.Write([]byte("hello"))
	ring.Reset()

	if ring.Len() != 0 {
		t.Errorf("ring should still be empty")
	}
}

func TestGob(t *testing.T) {
	str := "HelloWorld"

	buf := NewUnboundedBuffer(2, 2)
	buf.Write([]byte(str))
	b := bytes.NewBuffer(nil)
	var test Buffer = buf
	if err := gob.NewEncoder(b).Encode(&test); err != nil {
		t.Error(err.Error())
		return
	}
	var buffer Buffer
	if err := gob.NewDecoder(b).Decode(&buffer); err != nil {
		t.Error(err.Error())
		return
	}
	data := make([]byte, len(str))
	if _, err := buffer.Read(data); err != nil && err != io.EOF {
		t.Error(err.Error())
	}
	if !bytes.Equal(data, []byte(str)) {
		t.Error("Gob Recover Failed... " + string(data))
	}
	buffer.Reset()
}

func TestDiscard(t *testing.T) {
	buf := NewDiscard()
	if buf.Cap() != MAXINT64 {
		t.Errorf("cap isn't infinite")
	}
	buf.Write([]byte("hello"))
	if buf.Len() != 0 {
		t.Errorf("buf should always be empty")
	}
}

func TestList(t *testing.T) {
	mem := New(10)
	ory := New(10)
	mem.Write([]byte("Hello"))
	ory.Write([]byte("world"))

	buf := BufferList([]Buffer{mem, ory, NewDiscard()})
	if buf.Len() != 10 {
		t.Errorf("incorrect sum of lengths")
	}
	if buf.Cap() != MAXINT64 {
		t.Errorf("incorrect sum of caps")
	}

	buf.Reset()
	if buf.Len() != 0 {
		t.Errorf("buffer should be empty")
	}
}

func TestSpill(t *testing.T) {
	buf := NewSpill(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))
	data := make([]byte, 12)
	n, _ := buf.Read(data)
	if !bytes.Equal(data[:n], []byte("Hello")) {
		t.Error("ReadAt Failed. " + string(data[:n]))
	}
}

func TestSpill2(t *testing.T) {
	buf := NewSpill(New(5), nil)

	if buf.Cap() != MAXINT64 {
		t.Errorf("cap isn't infinite")
	}

	buf.Write([]byte("Hello World"))
	data := make([]byte, 12)
	n, _ := buf.Read(data)
	if !bytes.Equal(data[:n], []byte("Hello")) {
		t.Error("ReadAt Failed. " + string(data[:n]))
	}
}

func TestFile(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewFile(1024, file)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()

	buf = NewFile(3, file)
	buf.Write([]byte("abc"))
	buf.Read(make([]byte, 1))
	buf.Write([]byte("a"))
	d, _ := ioutil.ReadAll(buf)
	if !bytes.Equal(d, []byte("bca")) {
		t.Error("back and forth error!")
	}
}

func TestMem(t *testing.T) {
	buf := New(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()
	if n, err := buf.WriteAt([]byte("hello"), 1); err == nil || n != 0 {
		t.Errorf("write should have failed")
	}
}

func TestFilePartition(t *testing.T) {
	buf := NewPartition(NewFilePool(1024, ""))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	buf.Reset()
}

func TestMulti(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewMulti(New(5), New(5), NewFile(500, file), NewPartition(NewFilePool(1024, "")))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
	buf.Reset()
}

func TestMulti2(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewMulti(New(5), New(5), NewFile(500, file))
	checkCap(t, buf, 510)
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
	buf.Reset()
}

func runPerfectSeries(t *testing.T, buf Buffer) {
	checkEmpty(t, buf)
	simple(t, buf)

	max := int64(1024)
	isPerfectMatch(t, buf, 0)
	for i := int64(1); i < max; i *= 2 {
		isPerfectMatch(t, buf, i)
	}
	isPerfectMatch(t, buf, max)
}

func simple(t *testing.T, buf Buffer) {
	buf.Write([]byte("hello world"))
	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("hello world"), data) {
		t.Error("Hello world failed.")
	}

	buf.Write([]byte("hello world"))
	data = make([]byte, 3)
	buf.Read(data)
	buf.Write([]byte(" yolo"))
	data, err = ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("lo world yolo"), data) {
		t.Error("Buffer crossing error :(", string(data))
	}
}

func buildOutputs(t *testing.T, buf Buffer, size int64) (wrote []byte, read []byte) {
	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ = ioutil.ReadAll(tee)
	read, _ = ioutil.ReadAll(buf)

	return wrote, read
}

func isPerfectMatch(t *testing.T, buf Buffer, size int64) {
	wrote, read := buildOutputs(t, buf, size)
	if !bytes.Equal(wrote, read) {
		t.Error("Buffer should have matched")
	}
}

func checkEmpty(t *testing.T, buf Buffer) {
	if !Empty(buf) {
		t.Error("Buffer should start empty!")
	}
}

func checkCap(t *testing.T, buf Buffer, correctCap int64) {
	if buf.Cap() != correctCap {
		t.Error("Buffer cap is incorrect", buf.Cap(), correctCap)
	}
}
<commit_msg>added a non-infinite list test<commit_after>package buffer

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

func BenchmarkMemory(b *testing.B) {
	buf := New(32 * 1024)
	for i := 0; i < b.N; i++ {
		io.Copy(buf, io.LimitReader(rand.Reader, 32*1024))
		io.Copy(ioutil.Discard, buf)
	}
}

func TestPool(t *testing.T) {
	pool := NewPool(func() Buffer { return New(10) })
	buf := pool.Get()
	buf.Write([]byte("hello world"))
	pool.Put(buf)
}

func TestMemPool(t *testing.T) {
	pool := NewMemPool(10)
	buf := pool.Get()
	if n, err := buf.Write([]byte("hello world")); n != 10 {
		t.Errorf("wrote incorrect amount")
	} else if err == nil {
		t.Errorf("should have been a shortwrite error here")
	}
}

func TestFilePool(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("there should have been a panic here!")
		}
	}()

	pool := NewFilePool(1024, "::~_bad_dir_~::")
	pool.Get()
}

func TestOverflow(t *testing.T) {
	buf := NewMulti(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))

	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}

	if !bytes.Equal(data, []byte("Hello")) {
		t.Errorf("Expected Hello got %s", string(data))
	}

}

func TestWriteAt(t *testing.T) {
	var b BufferAt

	b = New(5)
	BufferAtTester(t, b)

	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	b = NewFile(5, file)
	BufferAtTester(t, b)

}

func BufferAtTester(t *testing.T, b BufferAt) {
	b.WriteAt([]byte("abc"), 0)
	Compare(t, b, "abc")

	b.WriteAt([]byte("abc"), 1)
	Compare(t, b, "aabc")

	b.WriteAt([]byte("abc"), 2)
	Compare(t, b, "aaabc")

	b.WriteAt([]byte("abc"), 3)
	Compare(t, b, "aaaab")

	b.Read(make([]byte, 2))

	Compare(t, b, "aab")
	b.Reset()
}

func Compare(t *testing.T, b BufferAt, s string) {
	data := make([]byte, b.Len())
	n, _ := b.ReadAt(data, 0)
	if string(data[:n]) != s {
		t.Error("Mismatch:", string(data[:n]), s)
	}
}

func TestRing(t *testing.T) {
	ring := NewRing(New(3))
	if ring.Len() != 0 {
		t.Errorf("Ring non-empty start!")
	}

	if ring.Cap() != MAXINT64 {
		t.Errorf("Ring has < max capacity")
	}

	ring.Write([]byte("abc"))
	if ring.Len() != 3 {
		t.Errorf("expected ring len == 3")
	}

	ring.Write([]byte("de"))
	if ring.Len() != 3 {
		t.Errorf("expected riing len == 3")
	}

	data := make([]byte, 12)
	if n, err := ring.Read(data); err != nil {
		t.Error(err.Error())
	} else {
		if !bytes.Equal(data[:n], []byte("cde")) {
			t.Errorf("expected cde, got %s", data[:n])
		}
	}

	if ring.Len() != 0 {
		t.Errorf("ring should now be empty")
	}

	ring.Write([]byte("hello"))
	ring.Reset()

	if ring.Len() != 0 {
		t.Errorf("ring should still be empty")
	}
}

func TestGob(t *testing.T) {
	str := "HelloWorld"

	buf := NewUnboundedBuffer(2, 2)
	buf.Write([]byte(str))
	b := bytes.NewBuffer(nil)
	var test Buffer = buf
	if err := gob.NewEncoder(b).Encode(&test); err != nil {
		t.Error(err.Error())
		return
	}
	var buffer Buffer
	if err := gob.NewDecoder(b).Decode(&buffer); err != nil {
		t.Error(err.Error())
		return
	}
	data := make([]byte, len(str))
	if _, err := buffer.Read(data); err != nil && err != io.EOF {
		t.Error(err.Error())
	}
	if !bytes.Equal(data, []byte(str)) {
		t.Error("Gob Recover Failed... " + string(data))
	}
	buffer.Reset()
}

func TestDiscard(t *testing.T) {
	buf := NewDiscard()
	if buf.Cap() != MAXINT64 {
		t.Errorf("cap isn't infinite")
	}
	buf.Write([]byte("hello"))
	if buf.Len() != 0 {
		t.Errorf("buf should always be empty")
	}
}

func TestList(t *testing.T) {
	mem := New(10)
	ory := New(10)
	mem.Write([]byte("Hello"))
	ory.Write([]byte("world"))

	buf := BufferList([]Buffer{mem, ory, NewDiscard()})
	if buf.Len() != 10 {
		t.Errorf("incorrect sum of lengths")
	}
	if buf.Cap() != MAXINT64 {
		t.Errorf("incorrect sum of caps")
	}

	buf.Reset()
	if buf.Len() != 0 {
		t.Errorf("buffer should be empty")
	}
}

func TestList2(t *testing.T) {
	mem := New(10)
	ory := New(10)
	mem.Write([]byte("Hello"))
	ory.Write([]byte("world"))

	buf := BufferList([]Buffer{mem, ory})
	if buf.Len() != 10 {
		t.Errorf("incorrect sum of lengths")
	}
	if buf.Cap() != 20 {
		t.Errorf("incorrect sum of caps")
	}

	buf.Reset()
	if buf.Len() != 0 {
		t.Errorf("buffer should be empty")
	}
}

func TestSpill(t *testing.T) {
	buf := NewSpill(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))
	data := make([]byte, 12)
	n, _ := buf.Read(data)
	if !bytes.Equal(data[:n], []byte("Hello")) {
		t.Error("ReadAt Failed. " + string(data[:n]))
	}
}

func TestSpill2(t *testing.T) {
	buf := NewSpill(New(5), nil)

	if buf.Cap() != MAXINT64 {
		t.Errorf("cap isn't infinite")
	}

	buf.Write([]byte("Hello World"))
	data := make([]byte, 12)
	n, _ := buf.Read(data)
	if !bytes.Equal(data[:n], []byte("Hello")) {
		t.Error("ReadAt Failed. " + string(data[:n]))
	}
}

func TestFile(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewFile(1024, file)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()

	buf = NewFile(3, file)
	buf.Write([]byte("abc"))
	buf.Read(make([]byte, 1))
	buf.Write([]byte("a"))
	d, _ := ioutil.ReadAll(buf)
	if !bytes.Equal(d, []byte("bca")) {
		t.Error("back and forth error!")
	}
}

func TestMem(t *testing.T) {
	buf := New(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()
	if n, err := buf.WriteAt([]byte("hello"), 1); err == nil || n != 0 {
		t.Errorf("write should have failed")
	}
}

func TestFilePartition(t *testing.T) {
	buf := NewPartition(NewFilePool(1024, ""))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	buf.Reset()
}

func TestMulti(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewMulti(New(5), New(5), NewFile(500, file), NewPartition(NewFilePool(1024, "")))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
	buf.Reset()
}

func TestMulti2(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewMulti(New(5), New(5), NewFile(500, file))
	checkCap(t, buf, 510)
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
	buf.Reset()
}

func runPerfectSeries(t *testing.T, buf Buffer) {
	checkEmpty(t, buf)
	simple(t, buf)

	max := int64(1024)
	isPerfectMatch(t, buf, 0)
	for i := int64(1); i < max; i *= 2 {
		isPerfectMatch(t, buf, i)
	}
	isPerfectMatch(t, buf, max)
}

func simple(t *testing.T, buf Buffer) {
	buf.Write([]byte("hello world"))
	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("hello world"), data) {
		t.Error("Hello world failed.")
	}

	buf.Write([]byte("hello world"))
	data = make([]byte, 3)
	buf.Read(data)
	buf.Write([]byte(" yolo"))
	data, err = ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("lo world yolo"), data) {
		t.Error("Buffer crossing error :(", string(data))
	}
}

func buildOutputs(t *testing.T, buf Buffer, size int64) (wrote []byte, read []byte) {
	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ = ioutil.ReadAll(tee)
	read, _ = ioutil.ReadAll(buf)

	return wrote, read
}

func isPerfectMatch(t *testing.T, buf Buffer, size int64) {
	wrote, read := buildOutputs(t, buf, size)
	if !bytes.Equal(wrote, read) {
		t.Error("Buffer should have matched")
	}
}

func checkEmpty(t *testing.T, buf Buffer) {
	if !Empty(buf) {
		t.Error("Buffer should start empty!")
	}
}

func checkCap(t *testing.T, buf Buffer, correctCap int64) {
	if buf.Cap() != correctCap {
		t.Error("Buffer cap is incorrect", buf.Cap(), correctCap)
	}
}
<|endoftext|>


<commit_before>/* Copyright (C) 2013 CompleteDB LLC.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with PubSubSQL.  If not, see <http://www.gnu.org/licenses/>.
 */

package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"io"
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"path/filepath"
)

var OS = ""          //windows,linux
var ARCH = ""        //32,64
var VS_PLATFORM = "" //set automatically
var GOROOT = ""      // must match ARCH
var PATH_SEPARATOR = ""
var PATH_SLASH = ""
var VERSION = "1.0.0"

func main() {
	start()
	//
	buildServer()
	buildService()
	copyRootFiles()
	createArchive()
	//
	done()
}

// server

func buildServer() {
	emptyln();
	print("Building pubsubsql server...")
	bin := "build/pubsubsql/bin/"
	cd("..")
	rm(serverFileName())
	execute("go", "build")
	cp(serverFileName(), bin + serverFileName(), true)
	cd("build")
	success()
}

func serverFileName() string {
	switch OS {
		case "windows":
			return "pubsubsql.exe"
		default:
			return "pubsubsql"
	}
}

// service installer

func buildService() {
	emptyln();
	print("Building service/installer...")
	cd("../service/" + OS)
	switch OS {
		case "linux":
			buildServiceLinux()
		default:
			buildServiceWindows()
	}
	cd("../../build")
	success()
}

func buildServiceWindows() {
	bin := "../../build/pubsubsql/bin/"
	execute("msbuild.exe", "/t:Clean,Build",  "/p:Configuration=Release", "/p:Platform=" + VS_PLATFORM)
	svc := "pubsubsqlsvc.exe"
	cp(svc, bin + svc, false)
}

func buildServiceLinux() {
	m := "m64"
	if ARCH == "32" {
		m = "m32"
	}
	bin := "../../build/pubsubsql/bin/"
	execute("make", "ARCH=" + m)
	svc := "pubsubsqlsvc"
	cp(svc, bin + svc, true)
}

// copy README LICENSE etc..

func copyRootFiles() {
	emptyln();
	print("Coping root files...")
	cp("../LICENSE", "./pubsubsql/LICENSE", false)
	success()
}

// create archive

func createArchive() {
	emptyln();
	print("Archiving files...")
	switch OS {
		case "linux":
			targz(getarchname() + ".tar.gz", "./pubsubsql")
		case "windows":
			dozip(getarchname() + ".zip", "./pubsubsql")
	}
	success()
}

// helpers

func print(str string, v ...interface{}) {
	fmt.Printf(str, v...)
	fmt.Println("")
}

func fail(str string, v ...interface{}) {
	print("ERROR: " + str, v...)
	os.Exit(1)
}

func emptyln() {
	fmt.Println("")
}

func success() {
	print("SUCCESS")
}

func start() {
	// read flags
	flag.StringVar(&OS, "OS", "windows", "Operating System (linux,windows)")
	flag.StringVar(&ARCH, "ARCH", "", "Architecture (32,64)")
	flag.StringVar(&GOROOT, "GOROOT", "", "Go root directory")
	flag.Parse()
	print("Usage")
	flag.PrintDefaults()

	print("BUILD STARTED")
	emptyln();
	// check OS
	switch OS {
		case "windows":
			PATH_SEPARATOR = ";"
			PATH_SLASH = "\\";
		case "linux":
			PATH_SEPARATOR = ":"
			PATH_SLASH = "/";
		default:
			fail("Unkown os %v", OS)
	}

	// set up go build env
	setenv("GOROOT", GOROOT)
	path := getenv("PATH")
	setenv("PATH", GOROOT + PATH_SLASH + "bin" + PATH_SEPARATOR + path)

	// check ARCH
	switch ARCH {
		case "32":
			setenv("GOARCH", "386")
			VS_PLATFORM = "Win32"
		case "64":
			setenv("GOARCH", "amd64")
			VS_PLATFORM = "x64"
		default:
			fail("Unkown architecture %v", ARCH)

	}

	// display current go env variables
	execute("go", "env")
	print("Preparing staging area...")
	prepareStagingArea();
	success()
}

func done() {
	emptyln();
	print("BUILD SUCCEEDED")
}

func prepareStagingArea() {
	rm("pubsubsql")
	mkdir("./pubsubsql/bin")
}

func mkdir(path string) {
	err := os.MkdirAll(path, os.ModeDir | os.ModePerm)
	if err != nil {
		fail("Failed to create directory: %v error: %v", path, err)
	}
}

func cd(path string) {
	err := os.Chdir(path)
	if err != nil {
		fail("Failed to change directory: %v error: %v", path, err)
	}
}

func rm(path string) {
	err := os.RemoveAll(path)
	if err != nil {
		fail("Fialed to remove path: %s error: %v", path, err)
	}
}

func execute(name string, arg ...string) {
	cmd := exec.Command(name, arg ...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		fail("Failed to execute command %v", err)
	}
}

func setenv(key string, value string) {
	err := os.Setenv(key, value)
	if err != nil {
		fail("Failed to set environment variable key:%v, value:% error:%v", key, value, err)
	}
}

func getenv(key string) string {
	return os.Getenv(key)
}

func copyFile(src string, dst string, execute bool)  (err error) {
    srcFile, err := os.Open(src)
    if err != nil {
		return err
    }
    defer srcFile.Close()

    dstFile, err := os.Create(dst)
    if err != nil {
        return
    }
    defer dstFile.Close()

	if OS == "linux" {
		err = dstFile.Chmod(os.ModePerm)
		if err != nil {
			return
		}
	}

    _, err = io.Copy(dstFile, srcFile)
	return err
}

func cp(src string, dst string, execute bool) {
	err := copyFile(src, dst, execute)
	if err != nil {
		fail("Failed to copy file %v", err)
	}
}

func open(path string) *os.File {
	file, err := os.Open(path)
	if err != nil {
		fail("Failed to open file %v error %v", path, err)
	}
	return file
}

func create(path string) *os.File {
	file, err := os.Create(path)
	if err != nil {
		fail("Failed to create file %v error %v", path, err)
	}
	return file
}

func getarchname() string {
	name := "pubsubsql-v" + VERSION + "-" + OS + "-"
	switch ARCH {
	case "32":
		name += "386"
	case "64":
		name += "amd64"
	}
	return name
}

func targz(archiveFile string, dir string) {
	// file
	file := create(archiveFile)
	defer file.Close()
	// gzip
	gzipWriter := gzip.NewWriter(file)
	defer gzipWriter.Close()
	// tar
	tarWriter := tar.NewWriter(gzipWriter)
	defer tarWriter.Close()
	//
	walk := func (path string, fileInfo os.FileInfo, err error) error {
		if fileInfo.Mode().IsDir() {
			return nil
		}
		if err != nil {
			fail("Failed to traverse directory structure %v", err)
		}
		print(path)
		fileToWrite := open(path)
		defer fileToWrite.Close()
		header, err := tar.FileInfoHeader(fileInfo, path)
		header.Name = path
		if err != nil {
			fail("Failed to create tar header from file info %v", err)
		}
		err = tarWriter.WriteHeader(header)
		if err != nil {
			fail("Failed to write tar header %v", err)
		}
		_, err = io.Copy(tarWriter, fileToWrite)
		return nil
	}
	//
	err := filepath.Walk(dir, walk)
	if err != nil {
		fail("Failed to traverse directory %v %v", dir, err)
	}
}

func dozip(archiveFile string, dir string) {
	// file
	file := create(archiveFile)
	defer file.Close()
	// zip
	zipWriter := zip.NewWriter(file)
	defer zipWriter.Close()
	//
	walk := func (path string, fileInfo os.FileInfo, err error) error {
		if fileInfo.Mode().IsDir() {
			return nil
		}
		if err != nil {
			fail("Failed to traverse directory structure %v", err)
		}
		print(path)
		fileToWrite := open(path)
		var w io.Writer
		w, err = zipWriter.Create(path)
		if err != nil {
			fail("Failed to create zip writer %v", err)
		}
		_, err = io.Copy(w, fileToWrite)
		return nil
	}
	//
	err := filepath.Walk(dir, walk)
	if err != nil {
		fail("Failed to traverse directory %v %v", dir, err)
	}
}

<commit_msg>build add go client<commit_after>/* Copyright (C) 2013 CompleteDB LLC.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with PubSubSQL.  If not, see <http://www.gnu.org/licenses/>.
 */

package main

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
)

var OS = ""          //windows,linux
var ARCH = ""        //32,64
var VS_PLATFORM = "" //set automatically
var GOROOT = ""      // must match ARCH
var PATH_SEPARATOR = ""
var PATH_SLASH = ""
var VERSION = "1.0.0"

func main() {
	start()
	//
	buildServer()
	buildService()
	copyRootFiles()
	copyGo();
	createArchive()
	//
	done()
}

// server

func buildServer() {
	emptyln()
	print("Building pubsubsql server...")
	bin := "build/pubsubsql/bin/"
	cd("..")
	rm(serverFileName())
	execute("go", "build")
	cp(serverFileName(), bin+serverFileName(), true)
	cd("build")
	success()
}

func serverFileName() string {
	switch OS {
	case "windows":
		return "pubsubsql.exe"
	default:
		return "pubsubsql"
	}
}

// service installer

func buildService() {
	emptyln()
	print("Building service/installer...")
	cd("../service/" + OS)
	switch OS {
	case "linux":
		buildServiceLinux()
	default:
		buildServiceWindows()
	}
	cd("../../build")
	success()
}

func buildServiceWindows() {
	bin := "../../build/pubsubsql/bin/"
	execute("msbuild.exe", "/t:Clean,Build", "/p:Configuration=Release", "/p:Platform="+VS_PLATFORM)
	svc := "pubsubsqlsvc.exe"
	cp(svc, bin+svc, false)
}

func buildServiceLinux() {
	m := "m64"
	if ARCH == "32" {
		m = "m32"
	}
	bin := "../../build/pubsubsql/bin/"
	execute("make", "ARCH="+m)
	svc := "pubsubsqlsvc"
	cp(svc, bin+svc, true)
}

// copy README LICENSE etc..

func copyRootFiles() {
	emptyln()
	print("Copying root files...")
	cp("../LICENSE", "./pubsubsql/LICENSE", false)
	success()
}

func copyGo() {
	emptyln()
	print("Copying go files...")
	mkdir("./pubsubsql/go/bin")
	mkdir("./pubsubsql/go/src/github.com/PubSubSQL/client")
	mkdir("./pubsubsql/go/pkg")
	cp("../../client/client.go", "./pubsubsql/go/src/github.com/PubSubSQL/client/client.go", false)
	cp("../../client/client_test.go", "./pubsubsql/go/src/github.com/PubSubSQL/client/client_test.go", false)
	cp("../../client/netheader.go", "./pubsubsql/go/src/github.com/PubSubSQL/client/netheader.go", false)
	cp("../../client/netheader_test.go", "./pubsubsql/go/src/github.com/PubSubSQL/client/netheader_test.go", false)
	cp("../../client/nethelper.go", "./pubsubsql/go/src/github.com/PubSubSQL/client/nethelper.go", false)
	success()
}

// create archive

func createArchive() {
	emptyln()
	print("Archiving files...")
	switch OS {
	case "linux":
		targz(getarchname()+".tar.gz", "./pubsubsql")
	case "windows":
		dozip(getarchname()+".zip", "./pubsubsql")
	}
	success()
}

// helpers

func print(str string, v ...interface{}) {
	fmt.Printf(str, v...)
	fmt.Println("")
}

func fail(str string, v ...interface{}) {
	print("ERROR: "+str, v...)
	os.Exit(1)
}

func emptyln() {
	fmt.Println("")
}

func success() {
	print("SUCCESS")
}

func start() {
	// read flags
	flag.StringVar(&OS, "OS", "windows", "Operating System (linux,windows)")
	flag.StringVar(&ARCH, "ARCH", "", "Architecture (32,64)")
	flag.StringVar(&GOROOT, "GOROOT", "", "Go root directory")
	flag.Parse()
	print("Usage")
	flag.PrintDefaults()

	print("BUILD STARTED")
	emptyln()
	// check OS
	switch OS {
	case "windows":
		PATH_SEPARATOR = ";"
		PATH_SLASH = "\\"
	case "linux":
		PATH_SEPARATOR = ":"
		PATH_SLASH = "/"
	default:
		fail("Unkown os %v", OS)
	}

	// set up go build env
	setenv("GOROOT", GOROOT)
	path := getenv("PATH")
	setenv("PATH", GOROOT+PATH_SLASH+"bin"+PATH_SEPARATOR+path)

	// check ARCH
	switch ARCH {
	case "32":
		setenv("GOARCH", "386")
		VS_PLATFORM = "Win32"
	case "64":
		setenv("GOARCH", "amd64")
		VS_PLATFORM = "x64"
	default:
		fail("Unkown architecture %v", ARCH)

	}

	// display current go env variables
	execute("go", "env")
	print("Preparing staging area...")
	prepareStagingArea()
	success()
}

func done() {
	emptyln()
	print("BUILD SUCCEEDED")
}

func prepareStagingArea() {
	rm("pubsubsql")
	mkdir("./pubsubsql/bin")
}

func mkdir(path string) {
	err := os.MkdirAll(path, os.ModeDir|os.ModePerm)
	if err != nil {
		fail("Failed to create directory: %v error: %v", path, err)
	}
}

func cd(path string) {
	err := os.Chdir(path)
	if err != nil {
		fail("Failed to change directory: %v error: %v", path, err)
	}
}

func rm(path string) {
	err := os.RemoveAll(path)
	if err != nil {
		fail("Fialed to remove path: %s error: %v", path, err)
	}
}

func execute(name string, arg ...string) {
	cmd := exec.Command(name, arg...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		fail("Failed to execute command %v", err)
	}
}

func setenv(key string, value string) {
	err := os.Setenv(key, value)
	if err != nil {
		fail("Failed to set environment variable key:%v, value:% error:%v", key, value, err)
	}
}

func getenv(key string) string {
	return os.Getenv(key)
}

func copyFile(src string, dst string, execute bool) (err error) {
	srcFile, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFile.Close()

	dstFile, err := os.Create(dst)
	if err != nil {
		return
	}
	defer dstFile.Close()

	if OS == "linux" {
		err = dstFile.Chmod(os.ModePerm)
		if err != nil {
			return
		}
	}

	_, err = io.Copy(dstFile, srcFile)
	return err
}

func cp(src string, dst string, execute bool) {
	err := copyFile(src, dst, execute)
	if err != nil {
		fail("Failed to copy file %v", err)
	}
}

func open(path string) *os.File {
	file, err := os.Open(path)
	if err != nil {
		fail("Failed to open file %v error %v", path, err)
	}
	return file
}

func create(path string) *os.File {
	file, err := os.Create(path)
	if err != nil {
		fail("Failed to create file %v error %v", path, err)
	}
	return file
}

func getarchname() string {
	name := "pubsubsql-v" + VERSION + "-" + OS + "-"
	switch ARCH {
	case "32":
		name += "386"
	case "64":
		name += "amd64"
	}
	return name
}

func targz(archiveFile string, dir string) {
	// file
	file := create(archiveFile)
	defer file.Close()
	// gzip
	gzipWriter := gzip.NewWriter(file)
	defer gzipWriter.Close()
	// tar
	tarWriter := tar.NewWriter(gzipWriter)
	defer tarWriter.Close()
	//
	walk := func(path string, fileInfo os.FileInfo, err error) error {
		if fileInfo.Mode().IsDir() {
			return nil
		}
		if err != nil {
			fail("Failed to traverse directory structure %v", err)
		}
		print(path)
		fileToWrite := open(path)
		defer fileToWrite.Close()
		header, err := tar.FileInfoHeader(fileInfo, path)
		header.Name = path
		if err != nil {
			fail("Failed to create tar header from file info %v", err)
		}
		err = tarWriter.WriteHeader(header)
		if err != nil {
			fail("Failed to write tar header %v", err)
		}
		_, err = io.Copy(tarWriter, fileToWrite)
		return nil
	}
	//
	err := filepath.Walk(dir, walk)
	if err != nil {
		fail("Failed to traverse directory %v %v", dir, err)
	}
}

func dozip(archiveFile string, dir string) {
	// file
	file := create(archiveFile)
	defer file.Close()
	// zip
	zipWriter := zip.NewWriter(file)
	defer zipWriter.Close()
	//
	walk := func(path string, fileInfo os.FileInfo, err error) error {
		if fileInfo.Mode().IsDir() {
			return nil
		}
		if err != nil {
			fail("Failed to traverse directory structure %v", err)
		}
		print(path)
		fileToWrite := open(path)
		var w io.Writer
		w, err = zipWriter.Create(path)
		if err != nil {
			fail("Failed to create zip writer %v", err)
		}
		_, err = io.Copy(w, fileToWrite)
		return nil
	}
	//
	err := filepath.Walk(dir, walk)
	if err != nil {
		fail("Failed to traverse directory %v %v", dir, err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

type Build struct{}

func main() {
	build := new(Build)
	build.darwin()
}

func (b *Build) darwin() {
	version := "1.0.0"
	scriptDir := "./darwin/scripts"
	installRoot := "./root"
	tempDest, err := ioutil.TempDir("", "tempDest")
	if err != nil {
		return
	}
	defer os.RemoveAll(tempDest)

	cmdPkg := exec.Command("pkgbuild",
		"--identifier", "com.koding.kd.pkg",
		"--version", version,
		"--scripts", scriptDir,
		"--root", installRoot,
		"--install-location", "/",
		tempDest+"/com.koding.kd.pkg", // used for next step, also set up for distribution.xml
	)

	res, err := cmdPkg.CombinedOutput()
	if err != nil {
		fmt.Println("res, err", string(res), err)
		return
	}

	distribution := "./darwin/Distribution.xml"
	resources := "./darwin/Resources"
	targetFile := "koding-kd-tool.pkg"

	cmdBuild := exec.Command("productbuild",
		"--distribution", distribution,
		"--resources", resources,
		"--package-path", tempDest,
		targetFile,
	)

	res, err = cmdBuild.CombinedOutput()
	if err != nil {
		fmt.Println("res, err", string(res), err)
		return
	}

	fmt.Println("everything is ok")

}
<commit_msg>kite/build: make it modular that it supports other OS'es too<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
)

type Build struct{}

func main() {
	build := new(Build)
	build.do()
}

func (b *Build) do() {
	switch runtime.GOOS {
	case "darwin":
		b.darwin()
	default:
		fmt.Printf("not supported os: %s.\n", runtime.GOOS)
	}
}

// darwin is building a new .pkg installer for darwin based OS'es.
func (b *Build) darwin() {
	version := "1.0.0"
	scriptDir := "./darwin/scripts"
	installRoot := "./root"
	tempDest, err := ioutil.TempDir("", "tempDest")
	if err != nil {
		return
	}
	defer os.RemoveAll(tempDest)

	cmdPkg := exec.Command("pkgbuild",
		"--identifier", "com.koding.kd.pkg",
		"--version", version,
		"--scripts", scriptDir,
		"--root", installRoot,
		"--install-location", "/",
		tempDest+"/com.koding.kd.pkg", // used for next step, also set up for distribution.xml
	)

	res, err := cmdPkg.CombinedOutput()
	if err != nil {
		fmt.Println("res, err", string(res), err)
		return
	}

	distribution := "./darwin/Distribution.xml"
	resources := "./darwin/Resources"
	targetFile := "koding-kd-tool.pkg"

	cmdBuild := exec.Command("productbuild",
		"--distribution", distribution,
		"--resources", resources,
		"--package-path", tempDest,
		targetFile,
	)

	res, err = cmdBuild.CombinedOutput()
	if err != nil {
		fmt.Println("res, err", string(res), err)
		return
	}

	fmt.Println("everything is ok")

}
<|endoftext|>


<commit_before>package events

import (
	"sync"

	. "github.com/eris-ltd/eris-db/Godeps/_workspace/src/github.com/tendermint/tendermint/common"
	"github.com/eris-ltd/eris-db/Godeps/_workspace/src/github.com/tendermint/tendermint/types"
)

// reactors and other modules should export
// this interface to become eventable
type Eventable interface {
	SetFireable(Fireable)
}

// an event switch or cache implements fireable
type Fireable interface {
	FireEvent(event string, data types.EventData)
}

type EventSwitch struct {
	BaseService

	mtx        sync.RWMutex
	eventCells map[string]*eventCell
	listeners  map[string]*eventListener
}

func NewEventSwitch() *EventSwitch {
	evsw := &EventSwitch{}
	evsw.BaseService = *NewBaseService(log, "EventSwitch", evsw)
	return evsw
}

func (evsw *EventSwitch) OnStart() error {
	evsw.BaseService.OnStart()
	evsw.eventCells = make(map[string]*eventCell)
	evsw.listeners = make(map[string]*eventListener)
	return nil
}

func (evsw *EventSwitch) OnStop() {
	evsw.BaseService.OnStop()
	evsw.eventCells = nil
	evsw.listeners = nil
}

func (evsw *EventSwitch) AddListenerForEvent(listenerID, event string, cb eventCallback) {
	// Get/Create eventCell and listener
	evsw.mtx.Lock()
	eventCell := evsw.eventCells[event]
	if eventCell == nil {
		eventCell = newEventCell()
		evsw.eventCells[event] = eventCell
	}
	listener := evsw.listeners[listenerID]
	if listener == nil {
		listener = newEventListener(listenerID)
		evsw.listeners[listenerID] = listener
	}
	evsw.mtx.Unlock()

	// Add event and listener
	eventCell.AddListener(listenerID, cb)
	listener.AddEvent(event)
}

func (evsw *EventSwitch) RemoveListener(listenerID string) {
	// Get and remove listener
	evsw.mtx.RLock()
	listener := evsw.listeners[listenerID]
	delete(evsw.listeners, listenerID)
	evsw.mtx.RUnlock()

	if listener == nil {
		return
	}

	// Remove callback for each event.
	listener.SetRemoved()
	for _, event := range listener.GetEvents() {
		evsw.RemoveListenerForEvent(event, listenerID)
	}
}

func (evsw *EventSwitch) RemoveListenerForEvent(event string, listenerID string) {
	// Get eventCell
	evsw.mtx.Lock()
	eventCell := evsw.eventCells[event]
	evsw.mtx.Unlock()

	if eventCell == nil {
		return
	}

	// Remove listenerID from eventCell
	numListeners := eventCell.RemoveListener(listenerID)

	// Maybe garbage collect eventCell.
	if numListeners == 0 {
		// Lock again and double check.
		evsw.mtx.Lock()      // OUTER LOCK
		eventCell.mtx.Lock() // INNER LOCK
		if len(eventCell.listeners) == 0 {
			delete(evsw.eventCells, event)
		}
		eventCell.mtx.Unlock() // INNER LOCK
		evsw.mtx.Unlock()      // OUTER LOCK
	}
}

func (evsw *EventSwitch) FireEvent(event string, data types.EventData) {
	// Get the eventCell
	evsw.mtx.RLock()
	eventCell := evsw.eventCells[event]
	evsw.mtx.RUnlock()

	if eventCell == nil {
		return
	}

	// Fire event for all listeners in eventCell
	eventCell.FireEvent(data)
}

//-----------------------------------------------------------------------------

// eventCell handles keeping track of listener callbacks for a given event.
type eventCell struct {
	mtx       sync.RWMutex
	listeners map[string]eventCallback
}

func newEventCell() *eventCell {
	return &eventCell{
		listeners: make(map[string]eventCallback),
	}
}

func (cell *eventCell) AddListener(listenerID string, cb eventCallback) {
	cell.mtx.Lock()
	cell.listeners[listenerID] = cb
	cell.mtx.Unlock()
}

func (cell *eventCell) RemoveListener(listenerID string) int {
	cell.mtx.Lock()
	delete(cell.listeners, listenerID)
	numListeners := len(cell.listeners)
	cell.mtx.Unlock()
	return numListeners
}

func (cell *eventCell) FireEvent(data types.EventData) {
	cell.mtx.RLock()
	for _, listener := range cell.listeners {
		listener(data) // XXX: make sure this doesn't block on removing listeners!
	}
	cell.mtx.RUnlock()
}

//-----------------------------------------------------------------------------

type eventCallback func(data types.EventData)

type eventListener struct {
	id string

	mtx     sync.RWMutex
	removed bool
	events  []string
}

func newEventListener(id string) *eventListener {
	return &eventListener{
		id:      id,
		removed: false,
		events:  nil,
	}
}

func (evl *eventListener) AddEvent(event string) {
	evl.mtx.Lock()
	defer evl.mtx.Unlock()

	if evl.removed {
		return
	}
	evl.events = append(evl.events, event)
}

func (evl *eventListener) GetEvents() []string {
	evl.mtx.RLock()
	defer evl.mtx.RUnlock()

	events := make([]string, len(evl.events))
	copy(events, evl.events)
	return events
}

func (evl *eventListener) SetRemoved() {
	evl.mtx.Lock()
	defer evl.mtx.Unlock()
	evl.removed = true
}
<commit_msg>fixes #143; godeps/tendermint/event.go: claim full lock on delete listener; refer to github.com/tendermint/go-events pr#4 for more details<commit_after>package events

import (
	"sync"

	. "github.com/eris-ltd/eris-db/Godeps/_workspace/src/github.com/tendermint/tendermint/common"
	"github.com/eris-ltd/eris-db/Godeps/_workspace/src/github.com/tendermint/tendermint/types"
)

// reactors and other modules should export
// this interface to become eventable
type Eventable interface {
	SetFireable(Fireable)
}

// an event switch or cache implements fireable
type Fireable interface {
	FireEvent(event string, data types.EventData)
}

type EventSwitch struct {
	BaseService

	mtx        sync.RWMutex
	eventCells map[string]*eventCell
	listeners  map[string]*eventListener
}

func NewEventSwitch() *EventSwitch {
	evsw := &EventSwitch{}
	evsw.BaseService = *NewBaseService(log, "EventSwitch", evsw)
	return evsw
}

func (evsw *EventSwitch) OnStart() error {
	evsw.BaseService.OnStart()
	evsw.eventCells = make(map[string]*eventCell)
	evsw.listeners = make(map[string]*eventListener)
	return nil
}

func (evsw *EventSwitch) OnStop() {
	evsw.BaseService.OnStop()
	evsw.eventCells = nil
	evsw.listeners = nil
}

func (evsw *EventSwitch) AddListenerForEvent(listenerID, event string, cb eventCallback) {
	// Get/Create eventCell and listener
	evsw.mtx.Lock()
	eventCell := evsw.eventCells[event]
	if eventCell == nil {
		eventCell = newEventCell()
		evsw.eventCells[event] = eventCell
	}
	listener := evsw.listeners[listenerID]
	if listener == nil {
		listener = newEventListener(listenerID)
		evsw.listeners[listenerID] = listener
	}
	evsw.mtx.Unlock()

	// Add event and listener
	eventCell.AddListener(listenerID, cb)
	listener.AddEvent(event)
}

func (evsw *EventSwitch) RemoveListener(listenerID string) {
	// Get and remove listener
	evsw.mtx.RLock()
	listener := evsw.listeners[listenerID]
	evsw.mtx.RUnlock()
	if listener == nil {
		return
	}

	evsw.mtx.Lock()
	delete(evsw.listeners, listenerID)
	evsw.mtx.Unlock()

	// Remove callback for each event.
	listener.SetRemoved()
	for _, event := range listener.GetEvents() {
		evsw.RemoveListenerForEvent(event, listenerID)
	}
}

func (evsw *EventSwitch) RemoveListenerForEvent(event string, listenerID string) {
	// Get eventCell
	evsw.mtx.Lock()
	eventCell := evsw.eventCells[event]
	evsw.mtx.Unlock()

	if eventCell == nil {
		return
	}

	// Remove listenerID from eventCell
	numListeners := eventCell.RemoveListener(listenerID)

	// Maybe garbage collect eventCell.
	if numListeners == 0 {
		// Lock again and double check.
		evsw.mtx.Lock()      // OUTER LOCK
		eventCell.mtx.Lock() // INNER LOCK
		if len(eventCell.listeners) == 0 {
			delete(evsw.eventCells, event)
		}
		eventCell.mtx.Unlock() // INNER LOCK
		evsw.mtx.Unlock()      // OUTER LOCK
	}
}

func (evsw *EventSwitch) FireEvent(event string, data types.EventData) {
	// Get the eventCell
	evsw.mtx.RLock()
	eventCell := evsw.eventCells[event]
	evsw.mtx.RUnlock()

	if eventCell == nil {
		return
	}

	// Fire event for all listeners in eventCell
	eventCell.FireEvent(data)
}

//-----------------------------------------------------------------------------

// eventCell handles keeping track of listener callbacks for a given event.
type eventCell struct {
	mtx       sync.RWMutex
	listeners map[string]eventCallback
}

func newEventCell() *eventCell {
	return &eventCell{
		listeners: make(map[string]eventCallback),
	}
}

func (cell *eventCell) AddListener(listenerID string, cb eventCallback) {
	cell.mtx.Lock()
	cell.listeners[listenerID] = cb
	cell.mtx.Unlock()
}

func (cell *eventCell) RemoveListener(listenerID string) int {
	cell.mtx.Lock()
	delete(cell.listeners, listenerID)
	numListeners := len(cell.listeners)
	cell.mtx.Unlock()
	return numListeners
}

func (cell *eventCell) FireEvent(data types.EventData) {
	cell.mtx.RLock()
	for _, listener := range cell.listeners {
		listener(data) // XXX: make sure this doesn't block on removing listeners!
	}
	cell.mtx.RUnlock()
}

//-----------------------------------------------------------------------------

type eventCallback func(data types.EventData)

type eventListener struct {
	id string

	mtx     sync.RWMutex
	removed bool
	events  []string
}

func newEventListener(id string) *eventListener {
	return &eventListener{
		id:      id,
		removed: false,
		events:  nil,
	}
}

func (evl *eventListener) AddEvent(event string) {
	evl.mtx.Lock()
	defer evl.mtx.Unlock()

	if evl.removed {
		return
	}
	evl.events = append(evl.events, event)
}

func (evl *eventListener) GetEvents() []string {
	evl.mtx.RLock()
	defer evl.mtx.RUnlock()

	events := make([]string, len(evl.events))
	copy(events, evl.events)
	return events
}

func (evl *eventListener) SetRemoved() {
	evl.mtx.Lock()
	defer evl.mtx.Unlock()
	evl.removed = true
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2019 Red Hat, Inc.
 *
 */

package tests_test

import (
	"encoding/xml"
	"flag"
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("MultiQueue", func() {
	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	var vmi *v1.VirtualMachineInstance

	BeforeEach(func() {
		tests.BeforeTestCleanup()
		vmi = tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
	})

	Context("MultiQueue Behavior", func() {

		It("Should honor multiQueue requests", func() {
			numCpus := 3

			multiQueue := true
			vmi.Spec.Domain.Devices.BlockMultiQueue = &multiQueue
			cpuReq := resource.MustParse(fmt.Sprintf("%d", numCpus))
			vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceCPU] = cpuReq

			tests.AddEphemeralDisk(vmi, "disk1", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))

			By("Creating VMI with 2 disks, 3 CPUs and multi-queue enabled")
			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			By("Waiting for VMI to start")
			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			By("Fetching VMI from cluster")
			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			By("Verifying VMI")
			newCpuReq := newVMI.Spec.Domain.Resources.Requests[k8sv1.ResourceCPU]
			Expect(int(newCpuReq.Value())).To(Equal(numCpus))
			Expect(*newVMI.Spec.Domain.Devices.BlockMultiQueue).To(BeTrue())

			By("Fetching Domain XML from running pod")
			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			By("Ensuring each disk has three queues assigned")
			for _, disk := range domSpec.Devices.Disks {
				Expect(int(*disk.Driver.Queues)).To(Equal(numCpus))
			}
		})

	})
})
<commit_msg>Add test_id reference to test<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2019 Red Hat, Inc.
 *
 */

package tests_test

import (
	"encoding/xml"
	"flag"
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("MultiQueue", func() {
	flag.Parse()

	virtClient, err := kubecli.GetKubevirtClient()
	tests.PanicOnError(err)

	var vmi *v1.VirtualMachineInstance

	BeforeEach(func() {
		tests.BeforeTestCleanup()
		vmi = tests.NewRandomVMIWithEphemeralDisk(tests.ContainerDiskFor(tests.ContainerDiskAlpine))
	})

	Context("MultiQueue Behavior", func() {

		It("[test_id:959][rfe_id:2065] Should honor multiQueue requests", func() {
			numCpus := 3

			multiQueue := true
			vmi.Spec.Domain.Devices.BlockMultiQueue = &multiQueue
			cpuReq := resource.MustParse(fmt.Sprintf("%d", numCpus))
			vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceCPU] = cpuReq

			tests.AddEphemeralDisk(vmi, "disk1", "virtio", tests.ContainerDiskFor(tests.ContainerDiskCirros))

			By("Creating VMI with 2 disks, 3 CPUs and multi-queue enabled")
			vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
			Expect(err).ToNot(HaveOccurred())

			By("Waiting for VMI to start")
			tests.WaitForSuccessfulVMIStart(vmi)

			getOptions := metav1.GetOptions{}
			var newVMI *v1.VirtualMachineInstance

			By("Fetching VMI from cluster")
			newVMI, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.Name, &getOptions)
			Expect(err).ToNot(HaveOccurred())

			By("Verifying VMI")
			newCpuReq := newVMI.Spec.Domain.Resources.Requests[k8sv1.ResourceCPU]
			Expect(int(newCpuReq.Value())).To(Equal(numCpus))
			Expect(*newVMI.Spec.Domain.Devices.BlockMultiQueue).To(BeTrue())

			By("Fetching Domain XML from running pod")
			domain, err := tests.GetRunningVirtualMachineInstanceDomainXML(virtClient, vmi)
			Expect(err).ToNot(HaveOccurred())
			domSpec := &api.DomainSpec{}
			Expect(xml.Unmarshal([]byte(domain), domSpec)).To(Succeed())

			By("Ensuring each disk has three queues assigned")
			for _, disk := range domSpec.Devices.Disks {
				Expect(int(*disk.Driver.Queues)).To(Equal(numCpus))
			}
		})

	})
})
<|endoftext|>


<commit_before>package main

import(
	"github.com/Crackerz/goSocketServer"
	"code.google.com/p/go.net/websocket"
)

type Node struct {
	uniq_id int
	socket goSocketServer.Socket
}

func NewNode(socket *goSocketServer.Socket) Node {
	node := Node{socket.GetId(),*socket}
	node.Write("obj = "+Server.program)
	Server.pendingNodes <-&node
	return node
}

func (n *Node) Handle() {
	n.socket.Handle()
}

func (n *Node) Write(text string) {
	n.socket.WriteString(text)
}
<commit_msg>Post merge cleanup (fixing mistakes I made)<commit_after>package main

import(
	"github.com/Crackerz/goSocketServer"
)

type Node struct {
	uniq_id int
	socket goSocketServer.Socket
}

func NewNode(socket *goSocketServer.Socket) Node {
	node := Node{socket.GetId(),*socket}
	node.Write("obj = "+Server.program)
	Server.pendingNodes <-&node
	return node
}

func (n *Node) Handle() {
	n.socket.Handle()
}

func (n *Node) Write(text string) {
	n.socket.WriteString(text)
}
<|endoftext|>


<commit_before>package flowgraph

import (
	"github.com/vectaport/fgbase"
)

// Node interface for flowgraph nodes that are connected by flowgraph edges
type Node interface {
	// Tracef for debug trace printing.  Use atomic log mechanism.
	Tracef(format string, v ...interface{})

	// LogError for logging of error messages.  Use atomic log mechanism.
	LogError(format string, v ...interface{})

	// Name returns the node name
	Name() string

	// Source returns upstream edge by index
	Source(n int) Edge

	// Destination returns downstream edge by index
	Destination(n int) Edge

	// FindSource returns upstream edge by name
	FindSource(name string) Edge

	// FindDestination returns downstream edge by name
	FindDestination(name string) Edge

	// AddSource adds a list of source edges
	AddSource(e ...Edge)

	// AddDestination adds a list of destination edges
	AddDestination(e ...Edge)

	// NumSource returns the number of upstream edges
	NumSource() int

	// NumDestination returns the number of downstream edges
	NumDestination() int

	// Auxiliary returns auxiliary storage used by
	// underlying implementation for storing state
	Auxiliary() interface{}

	// Base returns the value that implements this node
	// The type of this value identifies the implementation.
	Base() interface{}
}

// implementation of Node
type node struct {
	base *fgbase.Node
}

// Tracef for debug trace printing.  Uses atomic log mechanism.
func (n node) Tracef(format string, v ...interface{}) {
	n.base.Tracef(format, v)
}

// LogError for logging of error messages.  Uses atomic log mechanism.
func (n node) LogError(format string, v ...interface{}) {
	n.base.LogError(format, v)
}

// Name returns the node name
func (n node) Name() string {
	return n.base.Name
}

// Source returns upstream edge by index
func (n node) Source(i int) Edge {
	return edge{n.base.Srcs[i]}
}

// Destination returns downstream edge by index
func (n node) Destination(i int) Edge {
	return edge{n.base.Dsts[i]}
}

// FindSource returns upstream edge by name
func (n node) FindSource(name string) Edge {
	return edge{n.base.FindSrc(name)}
}

// FindDestination returns downstream edge by name
func (n node) FindDestination(name string) Edge {
	return edge{n.base.FindDst(name)}
}

// AddSource adds a list of source edges
func (n node) AddSource(e ...Edge) {
	for _, ev := range e {
		n.base.Srcs = append(n.base.Srcs, ev.Base().(*fgbase.Edge))
	}
}

// AddDestination adds a list of destination edges
func (n node) AddDestination(e ...Edge) {
	for _, ev := range e {
		n.base.Dsts = append(n.base.Dsts, ev.Base().(*fgbase.Edge))
	}
}

// NumSource returns the number of upstream edges
func (n node) NumSource() int {
	return len(n.base.Srcs)
}

// NumDestination returns the number of downstream edges
func (n node) NumDestination() int {
	return len(n.base.Dsts)
}

// Auxiliary returns auxiliary storage for this node used by
// the underlying implementation for storing state
func (n node) Auxiliary() interface{} {
	return n.base.Aux
}

// Base returns the value that implements this edge
// The type of this value identifies the implementation.
func (n node) Base() interface{} {
	return n.base
}
<commit_msg>add naming of sources and destination of a node<commit_after>package flowgraph

import (
	"github.com/vectaport/fgbase"
)

// Node interface for flowgraph nodes that are connected by flowgraph edges
type Node interface {
	// Tracef for debug trace printing.  Use atomic log mechanism.
	Tracef(format string, v ...interface{})

	// LogError for logging of error messages.  Use atomic log mechanism.
	LogError(format string, v ...interface{})

	// Name returns the node name
	Name() string

	// Source returns upstream edge by index
	Source(n int) Edge

	// Destination returns downstream edge by index
	Destination(n int) Edge

	// FindSource returns upstream edge by name
	FindSource(name string) Edge

	// FindDestination returns downstream edge by name
	FindDestination(name string) Edge

	// AddSource adds a list of source edges
	AddSource(e ...Edge)

	// AddDestination adds a list of destination edges
	AddDestination(e ...Edge)

	// NumSource returns the number of upstream edges
	NumSource() int

	// NumDestination returns the number of downstream edges
	NumDestination() int

	// SetSourceNames names the sources
	SetSourceNames(nm ...string)

	// SetDestinationNames names the destinations
	SetDestinationNames(nm ...string)

	// SourceNames returns the names of the sources
	SourceNames() []string

	// DestinationNames returns the names of the destinations
	DestinationNames() []string

	// Auxiliary returns auxiliary storage used by
	// underlying implementation for storing state
	Auxiliary() interface{}

	// Base returns the value that implements this node
	// The type of this value identifies the implementation.
	Base() interface{}
}

// implementation of Node
type node struct {
	base *fgbase.Node
}

// Tracef for debug trace printing.  Uses atomic log mechanism.
func (n node) Tracef(format string, v ...interface{}) {
	n.base.Tracef(format, v)
}

// LogError for logging of error messages.  Uses atomic log mechanism.
func (n node) LogError(format string, v ...interface{}) {
	n.base.LogError(format, v)
}

// Name returns the node name
func (n node) Name() string {
	return n.base.Name
}

// Source returns upstream edge by index
func (n node) Source(i int) Edge {
	return edge{n.base.Srcs[i]}
}

// Destination returns downstream edge by index
func (n node) Destination(i int) Edge {
	return edge{n.base.Dsts[i]}
}

// FindSource returns upstream edge by name
func (n node) FindSource(name string) Edge {
	return edge{n.base.FindSrc(name)}
}

// FindDestination returns downstream edge by name
func (n node) FindDestination(name string) Edge {
	return edge{n.base.FindDst(name)}
}

// AddSource adds a list of source edges
func (n node) AddSource(e ...Edge) {
	for _, ev := range e {
		n.base.Srcs = append(n.base.Srcs, ev.Base().(*fgbase.Edge))
	}
}

// AddDestination adds a list of destination edges
func (n node) AddDestination(e ...Edge) {
	for _, ev := range e {
		n.base.Dsts = append(n.base.Dsts, ev.Base().(*fgbase.Edge))
	}
}

// NumSource returns the number of upstream edges
func (n node) NumSource() int {
	return len(n.base.Srcs)
}

// NumDestination returns the number of downstream edges
func (n node) NumDestination() int {
	return len(n.base.Dsts)
}

// SetSourceNames names the sources
func (n node) SetSourceNames(nm ...string) {
	for _, v := range nm {
		n.base.SrcNames = append(n.base.SrcNames, v)
	}
}

// SetDestinationNames names the destinations
func (n node) SetDestinationNames(nm ...string) {
	for _, v := range nm {
		n.base.DstNames = append(n.base.DstNames, v)
	}
}

// SourceNames returns the names of the sources
func (n node) SourceNames() []string {
	return n.base.SrcNames
}

// DestinationNames returns the names of the destinatiopns
func (n node) DestinationNames() []string {
	return n.base.DstNames
}

// Auxiliary returns auxiliary storage for this node used by
// the underlying implementation for storing state
func (n node) Auxiliary() interface{} {
	return n.base.Aux
}

// Base returns the value that implements this edge
// The type of this value identifies the implementation.
func (n node) Base() interface{} {
	return n.base
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"github.com/jesand/stats/channel/bsc"
	"github.com/jesand/stats/dist"
	"github.com/jesand/stats/factor"
	"github.com/jesand/stats/variable"
	"math"
)

// Create a new MultipleBSCModel
func NewMultipleBSCModel() *MultipleBSCModel {
	return &MultipleBSCModel{
		Inputs:      make(map[string]*variable.DiscreteRV),
		Channels:    make(map[string]*bsc.BSC),
		FactorGraph: factor.NewFactorGraph(),
		SoftInputs:  true,
	}
}

// A noisy channel model which explains a data set as passing through one of
// a set of BSC channels with unknown noise rates. For instance, this can be
// used to infer the answers of independent binary-valued crowdsourcing
// questions.
type MultipleBSCModel struct {

	// The variables sent over the noisy channels
	Inputs map[string]*variable.DiscreteRV

	// The posterior probability that each latent input variable is true
	InputScores map[string]float64

	// Whether to use soft or hard assignments to inputs during inference
	SoftInputs bool

	// The noisy channels
	Channels map[string]*bsc.BSC

	// A factor graph relating inputs to outputs
	FactorGraph *factor.FactorGraph
}

// Adds a new BSC to the model with the given name and noise rate.
func (model *MultipleBSCModel) AddChannel(name string, noise float64) {
	model.Channels[name] = bsc.NewBSC(noise)
}

// Ask whether a given channel exists
func (model MultipleBSCModel) HasChannel(name string) bool {
	_, ok := model.Channels[name]
	return ok
}

// Ask whether a given input exists
func (model MultipleBSCModel) HasInput(name string) bool {
	_, ok := model.Inputs[name]
	return ok
}

// Adds a new observation to the model for the given channel and input. If the
// input is new, it will be created automatically.
func (model *MultipleBSCModel) AddObservation(input, channel string, value bool) {
	inputVar, ok := model.Inputs[input]
	if !ok {
		inputVar = variable.NewDiscreteRV(0, dist.BooleanSpace)
		model.Inputs[input] = inputVar
	}
	ch := model.Channels[channel]

	// Add a worker noise factor to explain the assessment
	outcome := dist.BooleanSpace.BoolOutcome(value)
	model.FactorGraph.AddFactor(ch.Factor(inputVar, variable.NewDiscreteRV(
		outcome, dist.BooleanSpace)))
}

// Score the model, using the current parameter values
func (model MultipleBSCModel) Score() float64 {
	return model.FactorGraph.Score()
}

// Train noise rates and input values using expectation maximization.
func (model *MultipleBSCModel) EM(maxRounds int, tolerance float64,
	callback func(model *MultipleBSCModel, round int, stage string)) {

	var (
		round        int
		initialScore = model.Score()
		thisRound    = initialScore
		lastRound    = thisRound - 1.0
		softScores   = make(map[*variable.DiscreteRV]float64)
	)

	if callback != nil {
		callback(model, round, "Initial")
	}
	for round = 1; (maxRounds == 0 || round <= maxRounds) &&
		thisRound-lastRound > tolerance; round++ {

		// Update input
		for _, input := range model.Inputs {
			input.Set(0)
			ifFalse := math.Exp(model.FactorGraph.ScoreVar(input))
			input.Set(1)
			ifTrue := math.Exp(model.FactorGraph.ScoreVar(input))
			if ifFalse > ifTrue {
				input.Set(0)
			}
			if model.SoftInputs {
				if ifTrue == 0 {
					softScores[input] = 1e-6
				} else if ifFalse == 0 {
					softScores[input] = 1 - 1e-6
				} else {
					softScores[input] = ifTrue / (ifTrue + ifFalse)
				}
			} else {
				softScores[input] = input.Val()
			}
		}
		if callback != nil {
			callback(model, round, "input")
		}

		// Update noise rates
		thisRound2, lastRound2 := thisRound, lastRound
		for r2 := 1; (maxRounds == 0 || r2 <= maxRounds) &&
			thisRound2-lastRound2 > tolerance; r2++ {
			for _, ch := range model.Channels {
				var sum, count float64
				for _, factor := range model.FactorGraph.AdjToVariable(ch.NoiseRate) {
					if ch, ok := factor.(*bsc.BSCFactor); ok {
						count++
						qi := softScores[ch.Input]
						if ch.Output.Val() == 1 {
							sum += qi
						} else {
							sum += 1 - qi
						}
					}
				}
				if sum == 0 {
					ch.NoiseRate.Set(1e-3)
				} else if sum == count {
					ch.NoiseRate.Set(1 - 1e-3)
				} else {
					ch.NoiseRate.Set(sum / count)
				}
			}
			if callback != nil {
				callback(model, round, "noise")
			}

			lastRound2, thisRound2 = thisRound2, model.Score()
		}

		lastRound, thisRound = thisRound, model.Score()
	}

	model.InputScores = make(map[string]float64)
	for name, input := range model.Inputs {
		input.Set(0)
		ifFalse := math.Exp(model.FactorGraph.ScoreVar(input))
		input.Set(1)
		ifTrue := math.Exp(model.FactorGraph.ScoreVar(input))
		if ifFalse > ifTrue {
			input.Set(0)
		}
		model.InputScores[name] = ifTrue / (ifTrue + ifFalse)
	}
	for name, ch := range model.Channels {
		fmt.Println(name, "noise:", ch.NoiseRate.Val())
	}
	if callback != nil {
		callback(model, 0, "Final")
	}
}
<commit_msg>removed superfluous print statements<commit_after>package model

import (
	"github.com/jesand/stats/channel/bsc"
	"github.com/jesand/stats/dist"
	"github.com/jesand/stats/factor"
	"github.com/jesand/stats/variable"
	"math"
)

// Create a new MultipleBSCModel
func NewMultipleBSCModel() *MultipleBSCModel {
	return &MultipleBSCModel{
		Inputs:      make(map[string]*variable.DiscreteRV),
		Channels:    make(map[string]*bsc.BSC),
		FactorGraph: factor.NewFactorGraph(),
		SoftInputs:  true,
	}
}

// A noisy channel model which explains a data set as passing through one of
// a set of BSC channels with unknown noise rates. For instance, this can be
// used to infer the answers of independent binary-valued crowdsourcing
// questions.
type MultipleBSCModel struct {

	// The variables sent over the noisy channels
	Inputs map[string]*variable.DiscreteRV

	// The posterior probability that each latent input variable is true
	InputScores map[string]float64

	// Whether to use soft or hard assignments to inputs during inference
	SoftInputs bool

	// The noisy channels
	Channels map[string]*bsc.BSC

	// A factor graph relating inputs to outputs
	FactorGraph *factor.FactorGraph
}

// Adds a new BSC to the model with the given name and noise rate.
func (model *MultipleBSCModel) AddChannel(name string, noise float64) {
	model.Channels[name] = bsc.NewBSC(noise)
}

// Ask whether a given channel exists
func (model MultipleBSCModel) HasChannel(name string) bool {
	_, ok := model.Channels[name]
	return ok
}

// Ask whether a given input exists
func (model MultipleBSCModel) HasInput(name string) bool {
	_, ok := model.Inputs[name]
	return ok
}

// Adds a new observation to the model for the given channel and input. If the
// input is new, it will be created automatically.
func (model *MultipleBSCModel) AddObservation(input, channel string, value bool) {
	inputVar, ok := model.Inputs[input]
	if !ok {
		inputVar = variable.NewDiscreteRV(0, dist.BooleanSpace)
		model.Inputs[input] = inputVar
	}
	ch := model.Channels[channel]

	// Add a worker noise factor to explain the assessment
	outcome := dist.BooleanSpace.BoolOutcome(value)
	model.FactorGraph.AddFactor(ch.Factor(inputVar, variable.NewDiscreteRV(
		outcome, dist.BooleanSpace)))
}

// Score the model, using the current parameter values
func (model MultipleBSCModel) Score() float64 {
	return model.FactorGraph.Score()
}

// Train noise rates and input values using expectation maximization.
func (model *MultipleBSCModel) EM(maxRounds int, tolerance float64,
	callback func(model *MultipleBSCModel, round int, stage string)) {

	var (
		round        int
		initialScore = model.Score()
		thisRound    = initialScore
		lastRound    = thisRound - 1.0
		softScores   = make(map[*variable.DiscreteRV]float64)
	)

	if callback != nil {
		callback(model, round, "Initial")
	}
	for round = 1; (maxRounds == 0 || round <= maxRounds) &&
		thisRound-lastRound > tolerance; round++ {

		// Update input
		for _, input := range model.Inputs {
			input.Set(0)
			ifFalse := math.Exp(model.FactorGraph.ScoreVar(input))
			input.Set(1)
			ifTrue := math.Exp(model.FactorGraph.ScoreVar(input))
			if ifFalse > ifTrue {
				input.Set(0)
			}
			if model.SoftInputs {
				if ifTrue == 0 {
					softScores[input] = 1e-6
				} else if ifFalse == 0 {
					softScores[input] = 1 - 1e-6
				} else {
					softScores[input] = ifTrue / (ifTrue + ifFalse)
				}
			} else {
				softScores[input] = input.Val()
			}
		}
		if callback != nil {
			callback(model, round, "input")
		}

		// Update noise rates
		thisRound2, lastRound2 := thisRound, lastRound
		for r2 := 1; (maxRounds == 0 || r2 <= maxRounds) &&
			thisRound2-lastRound2 > tolerance; r2++ {
			for _, ch := range model.Channels {
				var sum, count float64
				for _, factor := range model.FactorGraph.AdjToVariable(ch.NoiseRate) {
					if ch, ok := factor.(*bsc.BSCFactor); ok {
						count++
						qi := softScores[ch.Input]
						if ch.Output.Val() == 1 {
							sum += qi
						} else {
							sum += 1 - qi
						}
					}
				}
				if sum == 0 {
					ch.NoiseRate.Set(1e-3)
				} else if sum == count {
					ch.NoiseRate.Set(1 - 1e-3)
				} else {
					ch.NoiseRate.Set(sum / count)
				}
			}
			if callback != nil {
				callback(model, round, "noise")
			}

			lastRound2, thisRound2 = thisRound2, model.Score()
		}

		lastRound, thisRound = thisRound, model.Score()
	}

	model.InputScores = make(map[string]float64)
	for name, input := range model.Inputs {
		input.Set(0)
		ifFalse := math.Exp(model.FactorGraph.ScoreVar(input))
		input.Set(1)
		ifTrue := math.Exp(model.FactorGraph.ScoreVar(input))
		if ifFalse > ifTrue {
			input.Set(0)
		}
		model.InputScores[name] = ifTrue / (ifTrue + ifFalse)
	}
	if callback != nil {
		callback(model, 0, "Final")
	}
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/jinzhu/gorm"
	"github.com/kleister/kleister-api/config"
	"github.com/vincent-petithory/dataurl"
)

// VersionFiles is simply a collection of version file structs.
type VersionFiles []*VersionFile

// VersionFile represents a version file model definition.
type VersionFile struct {
	ID          int64            `json:"-" gorm:"primary_key"`
	VersionID   int64            `json:"-"`
	Version     *Pack            `json:"-"`
	Slug        string           `json:"slug" sql:"unique_index"`
	ContentType string           `json:"content_type"`
	Path        string           `json:"-" sql:"-"`
	URL         string           `json:"url" sql:"-"`
	MD5         string           `json:"md5" gorm:"column:md5"`
	Upload      *dataurl.DataURL `json:"upload,omitempty" sql:"-"`
	CreatedAt   time.Time        `json:"-"`
	UpdatedAt   time.Time        `json:"-"`
}

// AfterFind invokes required after loading a record from the database.
func (u *VersionFile) AfterFind(db *gorm.DB) {
	u.SetURL()
}

// BeforeSave invokes required actions before persisting.
func (u *VersionFile) BeforeSave(db *gorm.DB) error {
	if u.Slug == "" {
		for i := 0; true; i++ {
			u.Slug = RandomHash()

			notFound := db.Where(
				"slug = ?",
				u.Slug,
			).Not(
				"id",
				u.ID,
			).First(
				&VersionFile{},
			).RecordNotFound()

			if notFound {
				break
			}
		}
	}

	if u.Upload != nil {
		u.MD5, u.ContentType = AttachmentHash(u.Upload)
	}

	return nil
}

// AfterSave invokes required actions after persisting.
func (u *VersionFile) AfterSave(db *gorm.DB) error {
	if u.Upload != nil {
		var (
			err  error
			dest string
		)

		if config.S3.Enabled {
			dest = u.RelativePath()
		} else {
			dest, err = u.AbsolutePath()

			if err != nil {
				return err
			}
		}

		if err := AttachmentUpload(dest, u.ContentType, u.Upload.Data); err != nil {
			return fmt.Errorf("Failed to store version. %s", err)
		}
	}

	return nil
}

// AfterDelete invokes required actions after deletion.
func (u *VersionFile) AfterDelete(tx *gorm.DB) error {
	var (
		err  error
		dest string
	)

	if config.S3.Enabled {
		dest = u.RelativePath()
	} else {
		dest, err = u.AbsolutePath()

		if err != nil {
			return err
		}
	}

	if err := AttachmentDelete(dest); err != nil {
		return fmt.Errorf("Failed to remove version. %s", err)
	}

	return nil
}

// Validate does some validation to be able to store the record.
func (u *VersionFile) Validate(db *gorm.DB) {
	if u.Upload != nil {
		if isInvalidVersionFileType(u.Upload.MediaType.String()) {
			db.AddError(fmt.Errorf("Invalid file media type"))
		}
	}
}

// AbsolutePath generates the absolute path to the file.
func (u *VersionFile) AbsolutePath() (string, error) {
	if config.Server.Storage == "" {
		return "", fmt.Errorf("Missing storage path for version")
	}

	return path.Join(
		config.Server.Storage,
		u.RelativePath(),
	), nil
}

// RelativePath generates the relative path to the file.
func (u *VersionFile) RelativePath() string {
	return path.Join(
		"version",
		u.Slug,
	)
}

// SetURL generates the absolute URL to access this file.
func (u *VersionFile) SetURL() {
	u.URL = AttachmentURL(
		u.RelativePath(),
	)
}

func isInvalidVersionFileType(mediaType string) bool {
	logrus.Debugf("Got %s version file media type", mediaType)
	return false
}
<commit_msg>Validate only zips for version uploads<commit_after>package model

import (
	"fmt"
	"path"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/jinzhu/gorm"
	"github.com/kleister/kleister-api/config"
	"github.com/vincent-petithory/dataurl"
)

// VersionFiles is simply a collection of version file structs.
type VersionFiles []*VersionFile

// VersionFile represents a version file model definition.
type VersionFile struct {
	ID          int64            `json:"-" gorm:"primary_key"`
	VersionID   int64            `json:"-"`
	Version     *Pack            `json:"-"`
	Slug        string           `json:"slug" sql:"unique_index"`
	ContentType string           `json:"content_type"`
	Path        string           `json:"-" sql:"-"`
	URL         string           `json:"url" sql:"-"`
	MD5         string           `json:"md5" gorm:"column:md5"`
	Upload      *dataurl.DataURL `json:"upload,omitempty" sql:"-"`
	CreatedAt   time.Time        `json:"-"`
	UpdatedAt   time.Time        `json:"-"`
}

// AfterFind invokes required after loading a record from the database.
func (u *VersionFile) AfterFind(db *gorm.DB) {
	u.SetURL()
}

// BeforeSave invokes required actions before persisting.
func (u *VersionFile) BeforeSave(db *gorm.DB) error {
	if u.Slug == "" {
		for i := 0; true; i++ {
			u.Slug = RandomHash()

			notFound := db.Where(
				"slug = ?",
				u.Slug,
			).Not(
				"id",
				u.ID,
			).First(
				&VersionFile{},
			).RecordNotFound()

			if notFound {
				break
			}
		}
	}

	if u.Upload != nil {
		u.MD5, u.ContentType = AttachmentHash(u.Upload)
	}

	return nil
}

// AfterSave invokes required actions after persisting.
func (u *VersionFile) AfterSave(db *gorm.DB) error {
	if u.Upload != nil {
		var (
			err  error
			dest string
		)

		if config.S3.Enabled {
			dest = u.RelativePath()
		} else {
			dest, err = u.AbsolutePath()

			if err != nil {
				return err
			}
		}

		if err := AttachmentUpload(dest, u.ContentType, u.Upload.Data); err != nil {
			return fmt.Errorf("Failed to store version. %s", err)
		}
	}

	return nil
}

// AfterDelete invokes required actions after deletion.
func (u *VersionFile) AfterDelete(tx *gorm.DB) error {
	var (
		err  error
		dest string
	)

	if config.S3.Enabled {
		dest = u.RelativePath()
	} else {
		dest, err = u.AbsolutePath()

		if err != nil {
			return err
		}
	}

	if err := AttachmentDelete(dest); err != nil {
		return fmt.Errorf("Failed to remove version. %s", err)
	}

	return nil
}

// Validate does some validation to be able to store the record.
func (u *VersionFile) Validate(db *gorm.DB) {
	if u.Upload != nil {
		if isInvalidVersionFileType(u.Upload.MediaType.String()) {
			db.AddError(fmt.Errorf("Invalid file media type"))
		}
	}
}

// AbsolutePath generates the absolute path to the file.
func (u *VersionFile) AbsolutePath() (string, error) {
	if config.Server.Storage == "" {
		return "", fmt.Errorf("Missing storage path for version")
	}

	return path.Join(
		config.Server.Storage,
		u.RelativePath(),
	), nil
}

// RelativePath generates the relative path to the file.
func (u *VersionFile) RelativePath() string {
	return path.Join(
		"version",
		u.Slug,
	)
}

// SetURL generates the absolute URL to access this file.
func (u *VersionFile) SetURL() {
	u.URL = AttachmentURL(
		u.RelativePath(),
	)
}

func isInvalidVersionFileType(mediaType string) bool {
	switch mediaType {
	case "application/zip":
		logrus.Debugf("Detected valid %s media type", mediaType)
		return false
	}

	logrus.Debugf("Failed to validate %s media type", mediaType)
	return true
}
<|endoftext|>


<commit_before>package data_models

import (
	"github.com/nttdots/go-dots/dots_server/db"
	"github.com/nttdots/go-dots/dots_server/db_models/data"
	log "github.com/sirupsen/logrus"
	types "github.com/nttdots/go-dots/dots_common/types/data"
)

type Vendor struct {
	Id            int64
	ClientId      int64
	VendorId      int
	AttackMapping []AttackMapping
}

type AttackMapping struct {
	Id         int64
	AttackId   int
	AttackName string
}

type Vendors []Vendor

// New vendor-mapping
func NewVendorMapping(clientId int64, bodyData types.Vendor) Vendor {
	vendor := Vendor{}
	vendor.ClientId = clientId
	vendor.VendorId = int(*bodyData.VendorId)
	for _, v := range bodyData.AttackMapping {
		attackMapping := AttackMapping{}
		attackMapping.AttackId   = int(*v.AttackId)
		attackMapping.AttackName = *v.AttackName
		vendor.AttackMapping = append(vendor.AttackMapping, attackMapping)
	}
	return vendor
}

// Insert vendor-mapping into DB
func (vendor *Vendor) Save(tx *db.Tx) error {
	v := data_db_models.VendorMapping{}
	v.Id           = vendor.Id
	v.DataClientId = vendor.ClientId
	v.VendorId     = vendor.VendorId

	if v.Id == 0 {
		// Register vendor-mapping
		_, err := tx.Session.Insert(&v)
		if err != nil {
			log.WithError(err).Error("Insert() vendor-mapping failed.")
			return err
		}
	} else {
		// Update vendor-mapping
		_, err := tx.Session.ID(v.Id).Update(&v)
		if err != nil {
			log.WithError(err).Errorf("Update() vendor-mapping failed.")
			return err
		}
		// Delete attack-mapping
		_, err =tx.Session.Delete(&data_db_models.AttackMapping{VendorMappingId: vendor.Id})
		if err != nil {
			log.WithError(err).Errorf("Delete() attack-mapping failed.")
			return err
		}
	}
	// Register attack-mapping
	for _, attack := range vendor.AttackMapping {
		a := data_db_models.AttackMapping{}
		a.VendorMappingId = v.Id
		a.AttackId        = attack.AttackId
		a.AttackName      = attack.AttackName
		_, err := tx.Session.Insert(&a)
		if err != nil {
			log.WithError(err).Error("Insert() attack-mapping failed.")
			return err
		}
	}
	return nil
}

// Find vendor-mapping by vendor-id
func FindVendorByVendorId(tx *db.Tx, clientId int64, vendorId int) (Vendor, error) {
	vendor := Vendor{}
	dbVendor := data_db_models.VendorMapping{}
	_, err := tx.Session.Where("data_client_id = ? AND vendor_id = ?", clientId, vendorId).Get(&dbVendor)
	if err != nil {
		return vendor, err
	}
	vendor.Id = dbVendor.Id
	vendor.ClientId = dbVendor.DataClientId
	vendor.VendorId = dbVendor.VendorId
	// Get attack-mapping
	dbAttackList := []data_db_models.AttackMapping{}
	err = tx.Session.Where("vendor_mapping_id = ?", dbVendor.Id).Find(&dbAttackList)
	if err != nil {
		log.WithError(err).Error("Get() attack-mapping failed.")
		return vendor, err
	}
	for _, dbAttack := range dbAttackList {
		attack := AttackMapping{
			Id:         dbAttack.Id,
			AttackId:   dbAttack.AttackId,
			AttackName: dbAttack.AttackName,
		}
		vendor.AttackMapping = append(vendor.AttackMapping, attack)
	}
	return vendor, nil
}

// Find vendor-mapping by client-id
func FindVendorMappingByClientId(tx *db.Tx, clientId *int64) (Vendors, error) {
	vendorList := make(Vendors, 0)
	// Get vendor-mapping
	dbVendorList := []data_db_models.VendorMapping{}
	if clientId != nil {
		err := tx.Session.Where("data_client_id = ?", clientId).Find(&dbVendorList)
		if err != nil {
			log.WithError(err).Error("Get() vendor-mapping failed.")
			return nil, err
		}
	} else {
		err := tx.Session.Where("data_client_id = ?", 0).Find(&dbVendorList)
		if err != nil {
			log.WithError(err).Error("Get() vendor-mapping failed.")
			return nil, err
		}
	}
	for _, dbVendor := range dbVendorList {
		vendor := Vendor{}
		vendor.ClientId = dbVendor.DataClientId
		vendor.VendorId = dbVendor.VendorId
		// Get attack-mapping
		dbAttackList := []data_db_models.AttackMapping{}
		err := tx.Session.Where("vendor_mapping_id = ?", dbVendor.Id).Find(&dbAttackList)
		if err != nil {
			log.WithError(err).Error("Get() attack-mapping failed.")
			return nil, err
		}
		for _, dbAttack := range dbAttackList {
			attack := AttackMapping{
				Id:         dbAttack.Id,
				AttackId:   dbAttack.AttackId,
				AttackName: dbAttack.AttackName,
			}
			vendor.AttackMapping = append(vendor.AttackMapping, attack)
		}
		vendorList = append(vendorList, vendor)
	}
	return vendorList, nil
}

// Convert vendor-mapping to types vendor-mapping
func (vendors Vendors) ToTypesVendorMapping(depth *int) (*types.VendorMapping) {
	vendorList := types.VendorMapping{}
	if depth == nil || *depth > 2 {
		for _, v := range vendors {
			vendor := types.Vendor{}
			vendorId := uint32(v.VendorId)
			vendor.VendorId = &vendorId
			if depth == nil || *depth > 3 {
				for _, a := range v.AttackMapping {
					attack := types.AttackMapping{}
					attackId := uint32(a.AttackId)
					attack.AttackId   = &attackId
					attack.AttackName = &a.AttackName
					vendor.AttackMapping = append(vendor.AttackMapping, attack)
				}
			} else {
				vendor.AttackMapping = []types.AttackMapping{}
			}
			vendorList.Vendor = append(vendorList.Vendor, vendor)
		}
	} else {
		vendorList.Vendor = []types.Vendor{}
	}
	return &vendorList
}<commit_msg>Handle fix wrong attack-name when get issue #294<commit_after>package data_models

import (
	"github.com/nttdots/go-dots/dots_server/db"
	"github.com/nttdots/go-dots/dots_server/db_models/data"
	log "github.com/sirupsen/logrus"
	types "github.com/nttdots/go-dots/dots_common/types/data"
)

type Vendor struct {
	Id            int64
	ClientId      int64
	VendorId      int
	AttackMapping []AttackMapping
}

type AttackMapping struct {
	Id         int64
	AttackId   int
	AttackName string
}

type Vendors []Vendor

// New vendor-mapping
func NewVendorMapping(clientId int64, bodyData types.Vendor) Vendor {
	vendor := Vendor{}
	vendor.ClientId = clientId
	vendor.VendorId = int(*bodyData.VendorId)
	for _, v := range bodyData.AttackMapping {
		attackMapping := AttackMapping{}
		attackMapping.AttackId   = int(*v.AttackId)
		attackMapping.AttackName = *v.AttackName
		vendor.AttackMapping = append(vendor.AttackMapping, attackMapping)
	}
	return vendor
}

// Insert vendor-mapping into DB
func (vendor *Vendor) Save(tx *db.Tx) error {
	v := data_db_models.VendorMapping{}
	v.Id           = vendor.Id
	v.DataClientId = vendor.ClientId
	v.VendorId     = vendor.VendorId

	if v.Id == 0 {
		// Register vendor-mapping
		_, err := tx.Session.Insert(&v)
		if err != nil {
			log.WithError(err).Error("Insert() vendor-mapping failed.")
			return err
		}
	} else {
		// Update vendor-mapping
		_, err := tx.Session.ID(v.Id).Update(&v)
		if err != nil {
			log.WithError(err).Errorf("Update() vendor-mapping failed.")
			return err
		}
		// Delete attack-mapping
		_, err =tx.Session.Delete(&data_db_models.AttackMapping{VendorMappingId: vendor.Id})
		if err != nil {
			log.WithError(err).Errorf("Delete() attack-mapping failed.")
			return err
		}
	}
	// Register attack-mapping
	for _, attack := range vendor.AttackMapping {
		a := data_db_models.AttackMapping{}
		a.VendorMappingId = v.Id
		a.AttackId        = attack.AttackId
		a.AttackName      = attack.AttackName
		_, err := tx.Session.Insert(&a)
		if err != nil {
			log.WithError(err).Error("Insert() attack-mapping failed.")
			return err
		}
	}
	return nil
}

// Find vendor-mapping by vendor-id
func FindVendorByVendorId(tx *db.Tx, clientId int64, vendorId int) (Vendor, error) {
	vendor := Vendor{}
	dbVendor := data_db_models.VendorMapping{}
	_, err := tx.Session.Where("data_client_id = ? AND vendor_id = ?", clientId, vendorId).Get(&dbVendor)
	if err != nil {
		return vendor, err
	}
	vendor.Id = dbVendor.Id
	vendor.ClientId = dbVendor.DataClientId
	vendor.VendorId = dbVendor.VendorId
	// Get attack-mapping
	dbAttackList := []data_db_models.AttackMapping{}
	err = tx.Session.Where("vendor_mapping_id = ?", dbVendor.Id).Find(&dbAttackList)
	if err != nil {
		log.WithError(err).Error("Get() attack-mapping failed.")
		return vendor, err
	}
	for _, dbAttack := range dbAttackList {
		attack := AttackMapping{
			Id:         dbAttack.Id,
			AttackId:   dbAttack.AttackId,
			AttackName: dbAttack.AttackName,
		}
		vendor.AttackMapping = append(vendor.AttackMapping, attack)
	}
	return vendor, nil
}

// Find vendor-mapping by client-id
func FindVendorMappingByClientId(tx *db.Tx, clientId *int64) (Vendors, error) {
	vendorList := make(Vendors, 0)
	// Get vendor-mapping
	dbVendorList := []data_db_models.VendorMapping{}
	if clientId != nil {
		err := tx.Session.Where("data_client_id = ?", clientId).Find(&dbVendorList)
		if err != nil {
			log.WithError(err).Error("Get() vendor-mapping failed.")
			return nil, err
		}
	} else {
		err := tx.Session.Where("data_client_id = ?", 0).Find(&dbVendorList)
		if err != nil {
			log.WithError(err).Error("Get() vendor-mapping failed.")
			return nil, err
		}
	}
	for _, dbVendor := range dbVendorList {
		vendor := Vendor{}
		vendor.ClientId = dbVendor.DataClientId
		vendor.VendorId = dbVendor.VendorId
		// Get attack-mapping
		dbAttackList := []data_db_models.AttackMapping{}
		err := tx.Session.Where("vendor_mapping_id = ?", dbVendor.Id).Find(&dbAttackList)
		if err != nil {
			log.WithError(err).Error("Get() attack-mapping failed.")
			return nil, err
		}
		for _, dbAttack := range dbAttackList {
			attack := AttackMapping{
				Id:         dbAttack.Id,
				AttackId:   dbAttack.AttackId,
				AttackName: dbAttack.AttackName,
			}
			vendor.AttackMapping = append(vendor.AttackMapping, attack)
		}
		vendorList = append(vendorList, vendor)
	}
	return vendorList, nil
}

// Convert vendor-mapping to types vendor-mapping
func (vendors Vendors) ToTypesVendorMapping(depth *int) (*types.VendorMapping) {
	vendorList := types.VendorMapping{}
	if depth == nil || *depth > 2 {
		for _, v := range vendors {
			vendor := types.Vendor{}
			vendorId := uint32(v.VendorId)
			vendor.VendorId = &vendorId
			if depth == nil || *depth > 3 {
				for _, a := range v.AttackMapping {
					attack := types.AttackMapping{}
					attackId := uint32(a.AttackId)
					attackName := a.AttackName
					attack.AttackId   = &attackId
					attack.AttackName = &attackName
					vendor.AttackMapping = append(vendor.AttackMapping, attack)
				}
			} else {
				vendor.AttackMapping = []types.AttackMapping{}
			}
			vendorList.Vendor = append(vendorList.Vendor, vendor)
		}
	} else {
		vendorList.Vendor = []types.Vendor{}
	}
	return &vendorList
}<|endoftext|>