<commit_before>package scipipe

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"sync"
	"time"
)

// ======= InformationPacket ========

// InformationPacket contains information and helper methods for a physical file on a
// normal disk.
type InformationPacket struct {
	path      string
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	auditInfo *AuditInfo
	SubStream *FilePort
}

// Create new InformationPacket "object"
func NewInformationPacket(path string) *InformationPacket {
	ip := new(InformationPacket)
	ip.path = path
	ip.lock = new(sync.Mutex)
	ip.SubStream = NewFilePort()
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ip.buffer = bytes.NewBuffer(buf)
	return ip
}

// Get the (final) path of the physical file
func (ip *InformationPacket) GetPath() string {
	return ip.path
}

// Get the temporary path of the physical file
func (ip *InformationPacket) GetTempPath() string {
	return ip.path + ".tmp"
}

// Get the path to use when a FIFO file is used instead of a normal file
func (ip *InformationPacket) GetFifoPath() string {
	return ip.path + ".fifo"
}

// Open the file and return a file handle (*os.File)
func (ip *InformationPacket) Open() *os.File {
	f, err := os.Open(ip.GetPath())
	Check(err, "Could not open file: "+ip.GetPath())
	return f
}

// Open the temp file and return a file handle (*os.File)
func (ip *InformationPacket) OpenTemp() *os.File {
	f, err := os.Open(ip.GetTempPath())
	Check(err, "Could not open temp file: "+ip.GetTempPath())
	return f
}

// Open the file for writing return a file handle (*os.File)
func (ip *InformationPacket) OpenWriteTemp() *os.File {
	f, err := os.Create(ip.GetTempPath())
	Check(err, "Could not open temp file for writing: "+ip.GetTempPath())
	return f
}

// Read the whole content of the file and return as a byte array ([]byte)
func (ip *InformationPacket) Read() []byte {
	dat, err := ioutil.ReadFile(ip.GetPath())
	Check(err, "Could not open file for reading: "+ip.GetPath())
	return dat
}

// Write a byte array ([]byte) to the file (first to its temp path, and then atomize)
func (ip *InformationPacket) WriteTempFile(dat []byte) {
	err := ioutil.WriteFile(ip.GetTempPath(), dat, 0644)
	Check(err, "Could not write to temp file: "+ip.GetTempPath())
}

// Change from the temporary file name to the final file name
func (ip *InformationPacket) Atomize() {
	Debug.Println("InformationPacket: Atomizing", ip.GetTempPath(), "->", ip.GetPath())
	sleepingDurationSec := 5
	Debug.Printf("Sleeping for %d seconds before atomizing ...\n", sleepingDurationSec)
	time.Sleep(time.Duration(sleepingDurationSec) * time.Second)
	ip.lock.Lock()
	out, err := exec.Command("sh", "-c", "mv "+ip.GetTempPath()+" "+ip.path).CombinedOutput()
	Check(err, "Could not rename file: "+ip.GetTempPath()+". Output: "+string(out))
	ip.lock.Unlock()
	Debug.Println("InformationPacket: Done atomizing", ip.GetTempPath(), "->", ip.GetPath())
}

// Create FIFO file for the InformationPacket
func (ip *InformationPacket) CreateFifo() {
	ip.lock.Lock()
	cmd := "mkfifo " + ip.GetFifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)

	if _, err := os.Stat(ip.GetFifoPath()); err == nil {
		Warning.Println("FIFO already exists, so not creating a new one:", ip.GetFifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		Check(err, "Could not execute command: "+cmd)
	}

	ip.lock.Unlock()
}

// Remove the FIFO file, if it exists
func (ip *InformationPacket) RemoveFifo() {
	// FIXME: Shouldn't we check first whether the fifo exists?
	ip.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ip.GetFifoPath()).Output()
	Check(err, "Could not delete fifo file: "+ip.GetFifoPath())
	Debug.Println("Removed FIFO output: ", output)
	ip.lock.Unlock()
}

// Check if the file exists (at its final file name)
func (ip *InformationPacket) Exists() bool {
	exists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.GetPath()); err == nil {
		exists = true
	}
	ip.lock.Unlock()
	return exists
}

func (ip *InformationPacket) GetAuditInfo() *AuditInfo {
	defer ip.lock.Unlock()
	ip.lock.Lock()
	return ip.auditInfo
}

func (ip *InformationPacket) SetAuditInfo(ai *AuditInfo) {
	ip.lock.Lock()
	ip.auditInfo = ai
	ip.lock.Unlock()
}

func (ip *InformationPacket) WriteAuditLogToFile() {
	auditInfo := ip.GetAuditInfo()
	auditInfoJson, jsonErr := json.MarshalIndent(auditInfo, "", "    ")
	Check(jsonErr, "Could not marshall JSON")
	writeErr := ioutil.WriteFile(ip.GetPath()+".audit.json", auditInfoJson, 0644)
	Check(writeErr, "Could not write audit file: "+ip.GetPath())
}

// ======= IPQueue =======

// IPQueue is initialized by a set of strings with file paths, and from that
// will return instantiated InformationPacket on its Out-port, when run.
type IPQueue struct {
	Process
	Out       *FilePort
	FilePaths []string
}

// Initialize a new IPQueue component from a list of file paths
func NewIPQueue(filePaths ...string) (fq *IPQueue) {
	fq = &IPQueue{
		Out:       NewFilePort(),
		FilePaths: filePaths,
	}
	return
}

// Execute the IPQueue, returning instantiated InformationPacket
func (ipq *IPQueue) Run() {
	defer ipq.Out.Close()
	for _, fp := range ipq.FilePaths {
		ipq.Out.Chan <- NewInformationPacket(fp)
	}
}

// Check if the IPQueue outport is connected
func (ipq *IPQueue) IsConnected() bool {
	return ipq.Out.IsConnected()
}

func (snk *Sink) deleteInPortAtKey(i int) {
	if snk.inPorts != nil {
		if snk.inPorts[i] != nil {
			snk.inPorts = append(snk.inPorts[:i], snk.inPorts[i+1:]...)
		} else {
			Warning.Println("Inport %d does not exist, in sink")
		}
	} else {
		Warning.Println("Inports array not initialized!")
	}
}
<commit_msg>Sleep: 5->10<commit_after>package scipipe

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"sync"
	"time"
)

// ======= InformationPacket ========

// InformationPacket contains information and helper methods for a physical file on a
// normal disk.
type InformationPacket struct {
	path      string
	buffer    *bytes.Buffer
	doStream  bool
	lock      *sync.Mutex
	auditInfo *AuditInfo
	SubStream *FilePort
}

// Create new InformationPacket "object"
func NewInformationPacket(path string) *InformationPacket {
	ip := new(InformationPacket)
	ip.path = path
	ip.lock = new(sync.Mutex)
	ip.SubStream = NewFilePort()
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ip.buffer = bytes.NewBuffer(buf)
	return ip
}

// Get the (final) path of the physical file
func (ip *InformationPacket) GetPath() string {
	return ip.path
}

// Get the temporary path of the physical file
func (ip *InformationPacket) GetTempPath() string {
	return ip.path + ".tmp"
}

// Get the path to use when a FIFO file is used instead of a normal file
func (ip *InformationPacket) GetFifoPath() string {
	return ip.path + ".fifo"
}

// Open the file and return a file handle (*os.File)
func (ip *InformationPacket) Open() *os.File {
	f, err := os.Open(ip.GetPath())
	Check(err, "Could not open file: "+ip.GetPath())
	return f
}

// Open the temp file and return a file handle (*os.File)
func (ip *InformationPacket) OpenTemp() *os.File {
	f, err := os.Open(ip.GetTempPath())
	Check(err, "Could not open temp file: "+ip.GetTempPath())
	return f
}

// Open the file for writing return a file handle (*os.File)
func (ip *InformationPacket) OpenWriteTemp() *os.File {
	f, err := os.Create(ip.GetTempPath())
	Check(err, "Could not open temp file for writing: "+ip.GetTempPath())
	return f
}

// Read the whole content of the file and return as a byte array ([]byte)
func (ip *InformationPacket) Read() []byte {
	dat, err := ioutil.ReadFile(ip.GetPath())
	Check(err, "Could not open file for reading: "+ip.GetPath())
	return dat
}

// Write a byte array ([]byte) to the file (first to its temp path, and then atomize)
func (ip *InformationPacket) WriteTempFile(dat []byte) {
	err := ioutil.WriteFile(ip.GetTempPath(), dat, 0644)
	Check(err, "Could not write to temp file: "+ip.GetTempPath())
}

// Change from the temporary file name to the final file name
func (ip *InformationPacket) Atomize() {
	Debug.Println("InformationPacket: Atomizing", ip.GetTempPath(), "->", ip.GetPath())
	sleepingDurationSec := 10
	Debug.Printf("Sleeping for %d seconds before atomizing ...\n", sleepingDurationSec)
	time.Sleep(time.Duration(sleepingDurationSec) * time.Second)
	ip.lock.Lock()
	out, err := exec.Command("sh", "-c", "mv "+ip.GetTempPath()+" "+ip.path).CombinedOutput()
	Check(err, "Could not rename file: "+ip.GetTempPath()+". Output: "+string(out))
	ip.lock.Unlock()
	Debug.Println("InformationPacket: Done atomizing", ip.GetTempPath(), "->", ip.GetPath())
}

// Create FIFO file for the InformationPacket
func (ip *InformationPacket) CreateFifo() {
	ip.lock.Lock()
	cmd := "mkfifo " + ip.GetFifoPath()
	Debug.Println("Now creating FIFO with command:", cmd)

	if _, err := os.Stat(ip.GetFifoPath()); err == nil {
		Warning.Println("FIFO already exists, so not creating a new one:", ip.GetFifoPath())
	} else {
		_, err := exec.Command("bash", "-c", cmd).Output()
		Check(err, "Could not execute command: "+cmd)
	}

	ip.lock.Unlock()
}

// Remove the FIFO file, if it exists
func (ip *InformationPacket) RemoveFifo() {
	// FIXME: Shouldn't we check first whether the fifo exists?
	ip.lock.Lock()
	output, err := exec.Command("bash", "-c", "rm "+ip.GetFifoPath()).Output()
	Check(err, "Could not delete fifo file: "+ip.GetFifoPath())
	Debug.Println("Removed FIFO output: ", output)
	ip.lock.Unlock()
}

// Check if the file exists (at its final file name)
func (ip *InformationPacket) Exists() bool {
	exists := false
	ip.lock.Lock()
	if _, err := os.Stat(ip.GetPath()); err == nil {
		exists = true
	}
	ip.lock.Unlock()
	return exists
}

func (ip *InformationPacket) GetAuditInfo() *AuditInfo {
	defer ip.lock.Unlock()
	ip.lock.Lock()
	return ip.auditInfo
}

func (ip *InformationPacket) SetAuditInfo(ai *AuditInfo) {
	ip.lock.Lock()
	ip.auditInfo = ai
	ip.lock.Unlock()
}

func (ip *InformationPacket) WriteAuditLogToFile() {
	auditInfo := ip.GetAuditInfo()
	auditInfoJson, jsonErr := json.MarshalIndent(auditInfo, "", "    ")
	Check(jsonErr, "Could not marshall JSON")
	writeErr := ioutil.WriteFile(ip.GetPath()+".audit.json", auditInfoJson, 0644)
	Check(writeErr, "Could not write audit file: "+ip.GetPath())
}

// ======= IPQueue =======

// IPQueue is initialized by a set of strings with file paths, and from that
// will return instantiated InformationPacket on its Out-port, when run.
type IPQueue struct {
	Process
	Out       *FilePort
	FilePaths []string
}

// Initialize a new IPQueue component from a list of file paths
func NewIPQueue(filePaths ...string) (fq *IPQueue) {
	fq = &IPQueue{
		Out:       NewFilePort(),
		FilePaths: filePaths,
	}
	return
}

// Execute the IPQueue, returning instantiated InformationPacket
func (ipq *IPQueue) Run() {
	defer ipq.Out.Close()
	for _, fp := range ipq.FilePaths {
		ipq.Out.Chan <- NewInformationPacket(fp)
	}
}

// Check if the IPQueue outport is connected
func (ipq *IPQueue) IsConnected() bool {
	return ipq.Out.IsConnected()
}

func (snk *Sink) deleteInPortAtKey(i int) {
	if snk.inPorts != nil {
		if snk.inPorts[i] != nil {
			snk.inPorts = append(snk.inPorts[:i], snk.inPorts[i+1:]...)
		} else {
			Warning.Println("Inport %d does not exist, in sink")
		}
	} else {
		Warning.Println("Inports array not initialized!")
	}
}
<|endoftext|>


<commit_before>package node

import (
	"sync"
	"testing"

	"github.com/anycable/anycable-go/ws"
	"github.com/stretchr/testify/assert"
)

func TestSendRaceConditions(t *testing.T) {
	node := NewMockNode()
	var wg sync.WaitGroup

	for i := 1; i <= 10; i++ {
		session := NewMockSession("123", &node)

		go func() {
			for {
				session.conn.Read() // nolint:errcheck
			}
		}()

		wg.Add(2)
		go func() {
			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("hi!")})
				wg.Done()
			}()

			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
				wg.Done()
			}()
		}()

		wg.Add(2)
		go func() {
			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
				wg.Done()
			}()

			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("why")})
				wg.Done()
			}()
		}()
	}

	wg.Wait()
}

func TestSessionSend(t *testing.T) {
	node := NewMockNode()
	session := NewMockSession("123", &node)

	go func() {
		for i := 1; i <= 10; i++ {
			session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
		}
	}()

	for i := 1; i <= 10; i++ {
		_, err := session.conn.Read()
		assert.Nil(t, err)
	}
}

func TestSessionDisconnect(t *testing.T) {
	node := NewMockNode()
	session := NewMockSession("123", &node)
	session.closed = false
	session.Connected = true

	go func() {
		session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
		session.Disconnect("test", 1042)
	}()

	// Message frame
	_, err := session.conn.Read()
	assert.Nil(t, err)

	// Close frame
	_, err = session.conn.Read()
	assert.Nil(t, err)

	assert.True(t, session.closed)
}
<commit_msg>fix(tests): race condition<commit_after>package node

import (
	"sync"
	"testing"

	"github.com/anycable/anycable-go/ws"
	"github.com/stretchr/testify/assert"
)

func TestSendRaceConditions(t *testing.T) {
	node := NewMockNode()
	var wg sync.WaitGroup

	for i := 1; i <= 10; i++ {
		session := NewMockSession("123", &node)

		go func() {
			for {
				session.conn.Read() // nolint:errcheck
			}
		}()

		wg.Add(2)
		go func() {
			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("hi!")})
				wg.Done()
			}()

			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
				wg.Done()
			}()
		}()

		wg.Add(2)
		go func() {
			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
				wg.Done()
			}()

			go func() {
				session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("why")})
				wg.Done()
			}()
		}()
	}

	wg.Wait()
}

func TestSessionSend(t *testing.T) {
	node := NewMockNode()
	session := NewMockSession("123", &node)

	go func() {
		for i := 1; i <= 10; i++ {
			session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
		}
	}()

	for i := 1; i <= 10; i++ {
		_, err := session.conn.Read()
		assert.Nil(t, err)
	}
}

func TestSessionDisconnect(t *testing.T) {
	node := NewMockNode()
	session := NewMockSession("123", &node)
	session.closed = false
	session.Connected = true

	go func() {
		session.sendFrame(&ws.SentFrame{FrameType: ws.TextFrame, Payload: []byte("bye")})
		session.Disconnect("test", 1042)
	}()

	// Message frame
	_, err := session.conn.Read()
	assert.Nil(t, err)

	// Close frame
	_, err = session.conn.Read()
	assert.Nil(t, err)
}
<|endoftext|>


<commit_before>// Package beta provides algorithms for working with beta distributions.
//
// https://en.wikipedia.org/wiki/Beta_distribution
package beta

import (
	"math"
)

// Self represents a particular distribution from the family.
type Self struct {
	α float64
	β float64
	a float64
	b float64
}

// New returns a beta distribution with α and β on [a, b].
func New(α, β, a, b float64) *Self {
	return &Self{α, β, a, b}
}

// CDF evaluates the CDF of the distribution.
func (s *Self) CDF(points []float64) []float64 {
	values := make([]float64, len(points))

	α, β, k, b := s.α, s.β, s.b-s.a, s.a

	for i, x := range points {
		values[i] = betaInc(α, β, (x-b)/k)
	}

	return values
}

func betaInc(α, β, x float64) float64 {
	// Author: John Burkardt
	// Source: http://people.sc.fsu.edu/~jburkardt/c_src/prob/prob.html

	const (
		max = 1000
		tol = 1.0e-07
	)

	if x <= 0 {
		return 0
	} else if 1 <= x {
		return 1
	}

	var flip bool
	var i, ns int
	var cx, pp, psq, qq, rx, temp, term, value, xx float64

	psq = α + β

	if α < (α+β)*x {
		xx = 1 - x
		cx = x
		pp, qq = β, α
		flip = true
	} else {
		xx = x
		cx = 1 - x
		pp, qq = α, β
	}

	term = 1
	i = 1
	value = 1

	ns = int(qq + cx*(α+β))
	rx = xx / cx

	temp = qq - float64(i)
	if ns == 0 {
		rx = xx
	}

	// FIXME: Might not converge. Panic?
	for k := 0; k < max; k++ {
		term = term * temp * rx / (pp + float64(i))
		value = value + term
		temp = math.Abs(term)

		if temp <= tol && temp <= tol*value {
			break
		}

		i = i + 1
		ns = ns - 1

		if 0 <= ns {
			temp = qq - float64(i)
			if ns == 0 {
				rx = xx
			}
		} else {
			temp = psq
			psq = psq + 1
		}
	}

	value = value * math.Exp(pp*math.Log(xx)+(qq-1)*math.Log(cx)) / (beta(α, β) * pp)

	if flip {
		return 1 - value
	} else {
		return value
	}
}

func beta(x, y float64) float64 {
	z, _ := math.Lgamma(x + y)
	x, _ = math.Lgamma(x)
	y, _ = math.Lgamma(y)

	return math.Exp(x + y - z)
}
<commit_msg>Trying to make sense of beta.CDF<commit_after>// Package beta provides algorithms for working with beta distributions.
//
// https://en.wikipedia.org/wiki/Beta_distribution
package beta

import (
	"math"
)

// Self represents a particular distribution from the family.
type Self struct {
	α float64
	β float64
	a float64
	b float64
}

// New returns a beta distribution with α and β on [a, b].
func New(α, β, a, b float64) *Self {
	return &Self{α, β, a, b}
}

// CDF evaluates the CDF of the distribution.
func (s *Self) CDF(points []float64) []float64 {
	values := make([]float64, len(points))

	α, β, k, b := s.α, s.β, s.b-s.a, s.a

	for i, x := range points {
		values[i] = betaInc(α, β, (x-b)/k)
	}

	return values
}

func betaInc(α, β, x float64) float64 {
	// Author: John Burkardt
	// Source: http://people.sc.fsu.edu/~jburkardt/c_src/prob/prob.html

	const (
		max = 1000
		tol = 1.0e-07
	)

	if x <= 0 {
		return 0
	} else if 1 <= x {
		return 1
	}

	αx, βx := x, 1 - x

	var flip bool
	if α < (α+β)*x {
		α, αx, β, βx = β, βx, α, αx
		flip = true
	}

	// https://en.wikipedia.org/wiki/Integration_by_reduction_formulae
	rx := αx
	n := int(β + βx*(α+β))
	if n != 0 {
		rx /= βx
	}

	value := 1.0

	// FIXME: Might not converge. Panic?
	for i, temp, psq, term := 1, β - 1, α + β, 1.0; i <= max; {
		term = term * temp * rx / (α + float64(i))

		value += term
		temp = math.Abs(term)

		if temp <= tol && temp <= tol*value {
			break
		}

		i++
		n--

		if 0 <= n {
			temp = β - float64(i)
			if n == 0 {
				rx = αx
			}
		} else {
			temp = psq
			psq += 1
		}
	}

	// http://dlmf.nist.gov/8.17#v
	value = value * math.Exp(α*math.Log(αx)+(β-1)*math.Log(βx)) / (beta(α, β) * α)

	if flip {
		return 1 - value
	} else {
		return value
	}
}

func beta(x, y float64) float64 {
	z, _ := math.Lgamma(x + y)
	x, _ = math.Lgamma(x)
	y, _ = math.Lgamma(y)

	return math.Exp(x + y - z)
}
<|endoftext|>


<commit_before>// A simple low level wrapper around the OpenCL 1.1 C API.
package clw11

/*
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
*/
import "C"

type (
	Bool  C.cl_bool
	Int   C.cl_int
	Uint  C.cl_uint
	Ulong C.cl_ulong
	Size  C.size_t
)

const (
	True  = Bool(C.CL_TRUE)
	False = Bool(C.CL_FALSE)
)
<commit_msg>Updated package documentation.<commit_after>// A simple wrapper around the OpenCL 1.1 C API.
package clw11

/*
#define CL_USE_DEPRECATED_OPENCL_1_1_APIS
#ifdef __APPLE__
#include "OpenCL/opencl.h"
#else
#include "CL/opencl.h"
#endif
*/
import "C"

type (
	Bool  C.cl_bool
	Int   C.cl_int
	Uint  C.cl_uint
	Ulong C.cl_ulong
	Size  C.size_t
)

const (
	True  = Bool(C.CL_TRUE)
	False = Bool(C.CL_FALSE)
)
<|endoftext|>


<commit_before>package sarama

import (
	"bytes"
	"code.google.com/p/snappy-go/snappy"
	"encoding/binary"
	_ "fmt"
)

var snappyMagic = []byte{130, 83, 78, 65, 80, 80, 89, 0}

// SnappyEncode encodes binary data
func SnappyEncode(src []byte) ([]byte, error) {
	return snappy.Encode(nil, src)
}

// SnappyDecode decodes snappy data
func SnappyDecode(src []byte) ([]byte, error) {
	if bytes.Equal(src[:8], snappyMagic) {
		pos := uint32(16)
		max := uint32(len(src))
		dst := make([]byte, 0)
		for pos < max {
			size := binary.BigEndian.Uint32(src[pos : pos+4])
			pos = pos + 4
			chunk, err := snappy.Decode(nil, src[pos:pos+size])
			if err != nil {
				return nil, err
			}
			pos = pos + size
			dst = append(dst, chunk...)
		}
		return dst, nil
	}
	return snappy.Decode(nil, src)
}
<commit_msg>Changed snappy dependency to point to SD github clone<commit_after>package sarama

import (
	"bytes"
	"github.com/scalingdata/snappy-go/snappy"
	"encoding/binary"
	_ "fmt"
)

var snappyMagic = []byte{130, 83, 78, 65, 80, 80, 89, 0}

// SnappyEncode encodes binary data
func SnappyEncode(src []byte) ([]byte, error) {
	return snappy.Encode(nil, src)
}

// SnappyDecode decodes snappy data
func SnappyDecode(src []byte) ([]byte, error) {
	if bytes.Equal(src[:8], snappyMagic) {
		pos := uint32(16)
		max := uint32(len(src))
		dst := make([]byte, 0)
		for pos < max {
			size := binary.BigEndian.Uint32(src[pos : pos+4])
			pos = pos + 4
			chunk, err := snappy.Decode(nil, src[pos:pos+size])
			if err != nil {
				return nil, err
			}
			pos = pos + size
			dst = append(dst, chunk...)
		}
		return dst, nil
	}
	return snappy.Decode(nil, src)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package text offers functions to draw texts on an Ebiten's image.
//
// For the example using a TTF font, see font package in the examples.
package text

import (
	"image"
	"image/color"
	"math"
	"reflect"

	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"

	"github.com/hajimehoshi/ebiten"
	emath "github.com/hajimehoshi/ebiten/internal/math"
	"github.com/hajimehoshi/ebiten/internal/sync"
)

var (
	monotonicClock int64
)

func now() int64 {
	monotonicClock++
	return monotonicClock
}

var (
	faces = map[font.Face]struct{}{}
)

func fontFaceToFace(f font.Face) font.Face {
	if _, ok := faces[f]; ok {
		return f
	}
	// If the (DeepEqual-ly) same font exists,
	// reuse this to avoid to consume a lot of cache (#498).
	for key := range faces {
		if reflect.DeepEqual(key, f) {
			return key
		}
	}
	faces[f] = struct{}{}
	return f
}

var (
	// Use pointers to avoid copying on browsers.
	charBounds = map[font.Face]map[rune]*fixed.Rectangle26_6{}
)

type char struct {
	face   font.Face
	rune   rune
	atlasG int
}

func (c *char) bounds() *fixed.Rectangle26_6 {
	if m, ok := charBounds[c.face]; ok {
		if b, ok := m[c.rune]; ok {
			return b
		}
	} else {
		charBounds[c.face] = map[rune]*fixed.Rectangle26_6{}
	}
	b, _, _ := c.face.GlyphBounds(c.rune)
	charBounds[c.face][c.rune] = &b
	return &b
}

func (c *char) size() (fixed.Int26_6, fixed.Int26_6) {
	b := c.bounds()
	return b.Max.X - b.Min.X, b.Max.Y - b.Min.Y
}

func (c *char) empty() bool {
	x, y := c.size()
	return x == 0 || y == 0
}

func (c *char) atlasGroup() int {
	if c.atlasG != 0 {
		return c.atlasG
	}

	x, y := c.size()
	w, h := x.Ceil(), y.Ceil()
	t := w
	if t < h {
		t = h
	}

	// Different images for small runes are inefficient.
	// Let's use a same texture atlas for typical character sizes.
	if t < 32 {
		return 32
	}
	c.atlasG = emath.NextPowerOf2Int(t)
	return c.atlasG
}

type glyph struct {
	char  char
	index int
	atime int64
}

func fixed26_6ToFloat64(x fixed.Int26_6) float64 {
	return float64(x) / (1 << 6)
}

var (
	colorMCache = map[color.Color]ebiten.ColorM{}
)

func (g *glyph) draw(dst *ebiten.Image, x, y fixed.Int26_6, clr color.Color) {
	cr, cg, cb, ca := clr.RGBA()
	if ca == 0 {
		return
	}

	b := g.char.bounds()
	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(fixed26_6ToFloat64(x+b.Min.X), fixed26_6ToFloat64(y+b.Min.Y))

	cm, ok := colorMCache[clr]
	if !ok {
		rf := float64(cr) / float64(ca)
		gf := float64(cg) / float64(ca)
		bf := float64(cb) / float64(ca)
		af := float64(ca) / 0xffff
		cm.Reset()
		cm.Scale(rf, gf, bf, af)
		colorMCache[clr] = cm
	}
	op.ColorM = cm

	a := atlases[g.char.face][g.char.atlasGroup()]
	sx, sy := a.at(g)
	r := image.Rect(sx, sy, sx+a.glyphSize, sy+a.glyphSize)
	op.SourceRect = &r

	dst.DrawImage(a.image, op)
}

var (
	atlases = map[font.Face]map[int]*atlas{}
)

type atlas struct {
	// image is the back-end image to hold glyph cache.
	image *ebiten.Image

	// tmpImage is the temporary image as a renderer source for glyph.
	tmpImage *ebiten.Image

	// glyphSize is the size of one glyph in the cache.
	// This value is always power of 2.
	glyphSize int

	runeToGlyph map[rune]*glyph
}

func (a *atlas) at(glyph *glyph) (int, int) {
	if a.glyphSize != glyph.char.atlasGroup() {
		panic("not reached")
	}
	w, _ := a.image.Size()
	xnum := w / a.glyphSize
	x, y := glyph.index%xnum, glyph.index/xnum
	return x * a.glyphSize, y * a.glyphSize
}

func (a *atlas) maxGlyphNum() int {
	w, h := a.image.Size()
	xnum := w / a.glyphSize
	ynum := h / a.glyphSize
	return xnum * ynum
}

func (a *atlas) appendGlyph(char char, now int64) *glyph {
	g := &glyph{
		char:  char,
		atime: now,
	}
	if len(a.runeToGlyph) == a.maxGlyphNum() {
		var oldest *glyph
		t := int64(math.MaxInt64)
		for _, g := range a.runeToGlyph {
			if g.atime < t {
				t = g.atime
				oldest = g
			}
		}
		if oldest == nil {
			panic("not reached")
		}
		idx := oldest.index
		delete(a.runeToGlyph, oldest.char.rune)

		g.index = idx
	} else {
		g.index = len(a.runeToGlyph)
	}
	a.runeToGlyph[g.char.rune] = g
	a.draw(g)
	return g
}

func (a *atlas) draw(glyph *glyph) {
	if a.tmpImage == nil {
		a.tmpImage, _ = ebiten.NewImage(a.glyphSize, a.glyphSize, ebiten.FilterNearest)
	}

	dst := image.NewRGBA(image.Rect(0, 0, a.glyphSize, a.glyphSize))
	d := font.Drawer{
		Dst:  dst,
		Src:  image.White,
		Face: glyph.char.face,
	}
	b := glyph.char.bounds()
	d.Dot = fixed.Point26_6{-b.Min.X, -b.Min.Y}
	d.DrawString(string(glyph.char.rune))
	a.tmpImage.ReplacePixels(dst.Pix)

	op := &ebiten.DrawImageOptions{}
	x, y := a.at(glyph)
	op.GeoM.Translate(float64(x), float64(y))
	op.CompositeMode = ebiten.CompositeModeCopy
	a.image.DrawImage(a.tmpImage, op)

	a.tmpImage.Clear()
}

func getGlyphFromCache(face font.Face, r rune, now int64) *glyph {
	ch := char{
		face: face,
		rune: r,
	}
	var at *atlas
	if m, ok := atlases[face]; ok {
		a, ok := m[ch.atlasGroup()]
		if ok {
			g, ok := a.runeToGlyph[r]
			if ok {
				g.atime = now
				return g
			}
		}
		at = a
	} else {
		atlases[face] = map[int]*atlas{}
	}

	if ch.empty() {
		// The glyph doesn't have its size but might have valid 'advance' parameter
		// when ch is e.g. space (U+0020).
		return &glyph{
			char:  ch,
			atime: now,
		}
	}

	if at == nil {
		// Don't use ebiten.MaxImageSize here.
		// It's because the back-end image pixels will be restored from GPU
		// whenever a new glyph is rendered on the image, and restoring cost is
		// expensive if the image is big.
		// The back-end image is updated a temporary image, and the temporary image is
		// always cleared after used. This means that there is no clue to restore
		// the back-end image without reading from GPU
		// (see the package 'restorable' implementation).
		//
		// TODO: How about making a new function for 'flagile' image?
		const size = 1024
		i, _ := ebiten.NewImage(size, size, ebiten.FilterNearest)
		at = &atlas{
			image:       i,
			glyphSize:   ch.atlasGroup(),
			runeToGlyph: map[rune]*glyph{},
		}
		atlases[face][ch.atlasGroup()] = at
	}

	return at.appendGlyph(ch, now)
}

var textM sync.Mutex

// Draw draws a given text on a given destination image dst.
//
// face is the font for text rendering.
// (x, y) represents a 'dot' (period) position.
// Be careful that this doesn't represent left-upper corner position.
// clr is the color for text rendering.
//
// Glyphs used for rendering are cached in least-recently-used way.
// It is OK to call this function with a same text and a same face at every frame in terms of performance.
//
// This function is concurrent-safe.
func Draw(dst *ebiten.Image, text string, face font.Face, x, y int, clr color.Color) {
	textM.Lock()

	n := now()
	fx := fixed.I(x)
	prevC := rune(-1)

	runes := []rune(text)
	for _, c := range runes {
		if prevC >= 0 {
			fx += face.Kern(prevC, c)
		}
		fa := fontFaceToFace(face)
		if g := getGlyphFromCache(fa, c, n); g != nil {
			if !g.char.empty() {
				g.draw(dst, fx, fixed.I(y), clr)
			}
			a, _ := face.GlyphAdvance(c)
			fx += a
		}
		prevC = c
	}

	textM.Unlock()
}
<commit_msg>text: Limit color matrices cache size<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package text offers functions to draw texts on an Ebiten's image.
//
// For the example using a TTF font, see font package in the examples.
package text

import (
	"image"
	"image/color"
	"math"
	"reflect"

	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"

	"github.com/hajimehoshi/ebiten"
	emath "github.com/hajimehoshi/ebiten/internal/math"
	"github.com/hajimehoshi/ebiten/internal/sync"
)

var (
	monotonicClock int64
)

func now() int64 {
	monotonicClock++
	return monotonicClock
}

var (
	faces = map[font.Face]struct{}{}
)

func fontFaceToFace(f font.Face) font.Face {
	if _, ok := faces[f]; ok {
		return f
	}
	// If the (DeepEqual-ly) same font exists,
	// reuse this to avoid to consume a lot of cache (#498).
	for key := range faces {
		if reflect.DeepEqual(key, f) {
			return key
		}
	}
	faces[f] = struct{}{}
	return f
}

var (
	// Use pointers to avoid copying on browsers.
	charBounds = map[font.Face]map[rune]*fixed.Rectangle26_6{}
)

type char struct {
	face   font.Face
	rune   rune
	atlasG int
}

func (c *char) bounds() *fixed.Rectangle26_6 {
	if m, ok := charBounds[c.face]; ok {
		if b, ok := m[c.rune]; ok {
			return b
		}
	} else {
		charBounds[c.face] = map[rune]*fixed.Rectangle26_6{}
	}
	b, _, _ := c.face.GlyphBounds(c.rune)
	charBounds[c.face][c.rune] = &b
	return &b
}

func (c *char) size() (fixed.Int26_6, fixed.Int26_6) {
	b := c.bounds()
	return b.Max.X - b.Min.X, b.Max.Y - b.Min.Y
}

func (c *char) empty() bool {
	x, y := c.size()
	return x == 0 || y == 0
}

func (c *char) atlasGroup() int {
	if c.atlasG != 0 {
		return c.atlasG
	}

	x, y := c.size()
	w, h := x.Ceil(), y.Ceil()
	t := w
	if t < h {
		t = h
	}

	// Different images for small runes are inefficient.
	// Let's use a same texture atlas for typical character sizes.
	if t < 32 {
		return 32
	}
	c.atlasG = emath.NextPowerOf2Int(t)
	return c.atlasG
}

type glyph struct {
	char  char
	index int
	atime int64
}

func fixed26_6ToFloat64(x fixed.Int26_6) float64 {
	return float64(x) / (1 << 6)
}

type colorMCacheEntry struct {
	m     ebiten.ColorM
	atime int64
}

var (
	colorMCache = map[color.Color]*colorMCacheEntry{}
)

func (g *glyph) draw(dst *ebiten.Image, x, y fixed.Int26_6, clr color.Color) {
	cr, cg, cb, ca := clr.RGBA()
	if ca == 0 {
		return
	}

	b := g.char.bounds()
	op := &ebiten.DrawImageOptions{}
	op.GeoM.Translate(fixed26_6ToFloat64(x+b.Min.X), fixed26_6ToFloat64(y+b.Min.Y))

	e, ok := colorMCache[clr]
	if ok {
		e.atime = now()
	} else {
		if len(colorMCache) >= 256 {
			var oldest color.Color
			t := int64(math.MaxInt64)
			for key, e := range colorMCache {
				if e.atime < t {
					t = g.atime
					oldest = key
				}
			}
			delete(colorMCache, oldest)
		}
		cm := ebiten.ColorM{}
		rf := float64(cr) / float64(ca)
		gf := float64(cg) / float64(ca)
		bf := float64(cb) / float64(ca)
		af := float64(ca) / 0xffff
		cm.Scale(rf, gf, bf, af)
		e = &colorMCacheEntry{
			m:     cm,
			atime: now(),
		}
		colorMCache[clr] = e
	}
	op.ColorM = e.m

	a := atlases[g.char.face][g.char.atlasGroup()]
	sx, sy := a.at(g)
	r := image.Rect(sx, sy, sx+a.glyphSize, sy+a.glyphSize)
	op.SourceRect = &r

	dst.DrawImage(a.image, op)
}

var (
	atlases = map[font.Face]map[int]*atlas{}
)

type atlas struct {
	// image is the back-end image to hold glyph cache.
	image *ebiten.Image

	// tmpImage is the temporary image as a renderer source for glyph.
	tmpImage *ebiten.Image

	// glyphSize is the size of one glyph in the cache.
	// This value is always power of 2.
	glyphSize int

	runeToGlyph map[rune]*glyph
}

func (a *atlas) at(glyph *glyph) (int, int) {
	if a.glyphSize != glyph.char.atlasGroup() {
		panic("not reached")
	}
	w, _ := a.image.Size()
	xnum := w / a.glyphSize
	x, y := glyph.index%xnum, glyph.index/xnum
	return x * a.glyphSize, y * a.glyphSize
}

func (a *atlas) maxGlyphNum() int {
	w, h := a.image.Size()
	xnum := w / a.glyphSize
	ynum := h / a.glyphSize
	return xnum * ynum
}

func (a *atlas) appendGlyph(char char, now int64) *glyph {
	g := &glyph{
		char:  char,
		atime: now,
	}
	if len(a.runeToGlyph) == a.maxGlyphNum() {
		var oldest *glyph
		t := int64(math.MaxInt64)
		for _, g := range a.runeToGlyph {
			if g.atime < t {
				t = g.atime
				oldest = g
			}
		}
		if oldest == nil {
			panic("not reached")
		}
		idx := oldest.index
		delete(a.runeToGlyph, oldest.char.rune)

		g.index = idx
	} else {
		g.index = len(a.runeToGlyph)
	}
	a.runeToGlyph[g.char.rune] = g
	a.draw(g)
	return g
}

func (a *atlas) draw(glyph *glyph) {
	if a.tmpImage == nil {
		a.tmpImage, _ = ebiten.NewImage(a.glyphSize, a.glyphSize, ebiten.FilterNearest)
	}

	dst := image.NewRGBA(image.Rect(0, 0, a.glyphSize, a.glyphSize))
	d := font.Drawer{
		Dst:  dst,
		Src:  image.White,
		Face: glyph.char.face,
	}
	b := glyph.char.bounds()
	d.Dot = fixed.Point26_6{-b.Min.X, -b.Min.Y}
	d.DrawString(string(glyph.char.rune))
	a.tmpImage.ReplacePixels(dst.Pix)

	op := &ebiten.DrawImageOptions{}
	x, y := a.at(glyph)
	op.GeoM.Translate(float64(x), float64(y))
	op.CompositeMode = ebiten.CompositeModeCopy
	a.image.DrawImage(a.tmpImage, op)

	a.tmpImage.Clear()
}

func getGlyphFromCache(face font.Face, r rune, now int64) *glyph {
	ch := char{
		face: face,
		rune: r,
	}
	var at *atlas
	if m, ok := atlases[face]; ok {
		a, ok := m[ch.atlasGroup()]
		if ok {
			g, ok := a.runeToGlyph[r]
			if ok {
				g.atime = now
				return g
			}
		}
		at = a
	} else {
		atlases[face] = map[int]*atlas{}
	}

	if ch.empty() {
		// The glyph doesn't have its size but might have valid 'advance' parameter
		// when ch is e.g. space (U+0020).
		return &glyph{
			char:  ch,
			atime: now,
		}
	}

	if at == nil {
		// Don't use ebiten.MaxImageSize here.
		// It's because the back-end image pixels will be restored from GPU
		// whenever a new glyph is rendered on the image, and restoring cost is
		// expensive if the image is big.
		// The back-end image is updated a temporary image, and the temporary image is
		// always cleared after used. This means that there is no clue to restore
		// the back-end image without reading from GPU
		// (see the package 'restorable' implementation).
		//
		// TODO: How about making a new function for 'flagile' image?
		const size = 1024
		i, _ := ebiten.NewImage(size, size, ebiten.FilterNearest)
		at = &atlas{
			image:       i,
			glyphSize:   ch.atlasGroup(),
			runeToGlyph: map[rune]*glyph{},
		}
		atlases[face][ch.atlasGroup()] = at
	}

	return at.appendGlyph(ch, now)
}

var textM sync.Mutex

// Draw draws a given text on a given destination image dst.
//
// face is the font for text rendering.
// (x, y) represents a 'dot' (period) position.
// Be careful that this doesn't represent left-upper corner position.
// clr is the color for text rendering.
//
// Glyphs used for rendering are cached in least-recently-used way.
// It is OK to call this function with a same text and a same face at every frame in terms of performance.
//
// This function is concurrent-safe.
func Draw(dst *ebiten.Image, text string, face font.Face, x, y int, clr color.Color) {
	textM.Lock()

	n := now()
	fx := fixed.I(x)
	prevC := rune(-1)

	runes := []rune(text)
	for _, c := range runes {
		if prevC >= 0 {
			fx += face.Kern(prevC, c)
		}
		fa := fontFaceToFace(face)
		if g := getGlyphFromCache(fa, c, n); g != nil {
			if !g.char.empty() {
				g.draw(dst, fx, fixed.I(y), clr)
			}
			a, _ := face.GlyphAdvance(c)
			fx += a
		}
		prevC = c
	}

	textM.Unlock()
}
<|endoftext|>


<commit_before>package sqli

import "database/sql"

// DB is a wrapper around the database/sql database handle to provide the
// query scanner interface functions.
type DB struct {
	*sql.DB
}

// Row is an interface representing a row in the database that provides a Scan method.
// This interface is primiarly intended to represent a *sql.Row or *sql.Rows from the
// database/sql package.
type Row interface {
	Scan(dest ...interface{}) error
}

// Scanner is an interface to be used in scanning a database row handle.
type Scanner interface {
	Scan(row Row) error
}

// ScannerFn satifies the Scanner interface for a function type. The function will
// be called for each row.
type ScannerFn func(Row) error

// Scan calls the scanner function and returns its result.
func (s ScannerFn) Scan(row Row) error { return s(row) }

// Open opens a database specified by the database driver name and data source name.
// The arguments are passed to the database/sql Open function to retrive the
// database handle it provides.
func Open(driverName, dataSourceName string) (*DB, error) {
	db, err := sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return &DB{db}, nil
}

// Query provides a wrapper around the database/sql Query function which provides
// boilerplate around looping through the returned rows. The Scanner Scan method from
// the provided scanner is called on each interation of the row cursor loop.
//
// Query calls Scan on each row, which can be used like:
//
//  func (p *peopleScanner) Scan(row sqli.Row) error {
//      var person Person
//      if err := row.Scan(&person.Name); err != nil {
//           return err
//      }
//      p.people = append(p.people, &person)
//      return nil
//  }
func (db *DB) Query(scanner Scanner, query string, args ...interface{}) error {
	rows, err := db.DB.Query(query, args...)
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		err := scanner.Scan(rows)
		if err != nil {
			return err
		}
	}
	if err := rows.Err(); err != nil {
		return err
	}
	return rows.Close()
}

// QueryIn operates like Query but expands arguments for IN clauses.
// IN arguments are to be passed as slices (e.g. []int{1, 2, 3}).
func (db *DB) QueryIn(scanner Scanner, query string, args ...interface{}) error {
	query, args = In(query, args...)
	return db.Query(scanner, query, args...)
}

// QueryRow operates like Query but on a single row. The Scanner Scan function will
// be called only once.
func (db *DB) QueryRow(scanner Scanner, query string, args ...interface{}) error {
	return scanner.Scan(db.DB.QueryRow(query, args...))
}
<commit_msg>Update query methods to use context<commit_after>package sqli

import (
	"context"
	"database/sql"
)

// DB is a wrapper around the database/sql database handle to provide the
// query scanner interface functions.
type DB struct {
	*sql.DB
}

// Row is an interface representing a row in the database that provides a Scan method.
// This interface is primiarly intended to represent a *sql.Row or *sql.Rows from the
// database/sql package.
type Row interface {
	Scan(dest ...interface{}) error
}

// Scanner is an interface to be used in scanning a database row handle.
type Scanner interface {
	Scan(row Row) error
}

// ScannerFn satifies the Scanner interface for a function type. The function will
// be called for each row.
type ScannerFn func(Row) error

// Scan calls the scanner function and returns its result.
func (s ScannerFn) Scan(row Row) error { return s(row) }

// Open opens a database specified by the database driver name and data source name.
// The arguments are passed to the database/sql Open function to retrive the
// database handle it provides.
func Open(driverName, dataSourceName string) (*DB, error) {
	db, err := sql.Open(driverName, dataSourceName)
	if err != nil {
		return nil, err
	}
	return &DB{db}, nil
}

// Query provides a wrapper around the database/sql Query function which provides
// boilerplate around looping through the returned rows. The Scanner Scan method from
// the provided scanner is called on each interation of the row cursor loop.
//
// Query calls Scan on each row, which can be used like:
//
//  func (p *peopleScanner) Scan(row sqli.Row) error {
//      var person Person
//      if err := row.Scan(&person.Name); err != nil {
//           return err
//      }
//      p.people = append(p.people, &person)
//      return nil
//  }
func (db *DB) Query(scanner Scanner, query string, args ...interface{}) error {
	return db.QueryContext(context.Background(), scanner, query, args...)
}

// QueryIn operates like Query but expands arguments for IN clauses.
// IN arguments are to be passed as slices (e.g. []int{1, 2, 3}).
func (db *DB) QueryIn(scanner Scanner, query string, args ...interface{}) error {
	return db.QueryInContext(context.Background(), scanner, query, args...)
}

// QueryRow operates like Query but on a single row. The Scanner Scan function will
// be called only once.
func (db *DB) QueryRow(scanner Scanner, query string, args ...interface{}) error {
	return db.QueryRowContext(context.Background(), scanner, query, args...)
}

// QueryContext performs a query utilizing the supplied context.
func (db *DB) QueryContext(ctx context.Context, scanner Scanner, query string, args ...interface{}) error {
	rows, err := db.DB.QueryContext(ctx, query, args...)
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		err := scanner.Scan(rows)
		if err != nil {
			return err
		}
	}
	if err := rows.Err(); err != nil {
		return err
	}
	return rows.Close()
}

// QueryInContext performs an in query utilizing the supplied context.
func (db *DB) QueryInContext(ctx context.Context, scanner Scanner, query string, args ...interface{}) error {
	query, args = In(query, args...)
	return db.QueryContext(ctx, scanner, query, args...)
}

// QueryRowContext performs a query on a single row utlizing the supplied context.
func (db *DB) QueryRowContext(ctx context.Context, scanner Scanner, query string, args ...interface{}) error {
	return scanner.Scan(db.DB.QueryRowContext(ctx, query, args...))
}
<|endoftext|>


<commit_before>package vcap

import (
	"encoding/json"
	"os"
	"strconv"
	"time"
)

type VCAP struct {
	Application struct {
		ID            string   `json:"application_id"`
		Name          string   `json:"application_name"`
		Version       string   `json:"application_version"`
		InstanceID    string   `json:"instance_id"`
		InstanceIndex int      `json:"instance_index"`
		Host          string   `json:"host"`
		Port          int      `json:"port"`
		Users         string   `json:"users"`
		URIs          []string `json:"application_uris"`
		Limits        struct {
			Memory int `json:"mem"`
			Disk   int `json:"disk"`
			Files  int `json:"fds"`
		} `json:"limits"`
		Started *Timestamp `json:"started_at_timestamp"`
		State   *Timestamp `json:"state_timestamp"`
	}
	Host     string
	Port     int
	Services map[string][]struct {
		Name        string            `json:"name"`
		Label       string            `json:"label"`
		Tags        []string          `json:"tags"`
		Plan        string            `json:"plan"`
		Credentials map[string]string `json:"credentials"`
	}
}

type Timestamp time.Time

func (t *Timestamp) String() string {
	return time.Time(*t).String()
}

func (t *Timestamp) UnmarshalJSON(b []byte) error {
	ts, err := strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		return err
	}

	*t = Timestamp(time.Unix(ts, 0))
	return nil
}

func New() (*VCAP, error) {
	vcap := &VCAP{}

	if app := os.Getenv("VCAP_APPLICATION"); app != "" {
		if err := json.Unmarshal([]byte(app), &(vcap.Application)); err != nil {
			return nil, err
		}
	}

	if serv := os.Getenv("VCAP_SERVICES"); serv != "" {
		if err := json.Unmarshal([]byte(serv), &(vcap.Services)); err != nil {
			return nil, err
		}
	}

	vcap.Host = os.Getenv("VCAP_APP_HOST")
	if port := os.Getenv("VCAP_APP_PORT"); port != "" {
		p, err := strconv.ParseInt(port, 10, 32)
		if err != nil {
			return nil, err
		}
		vcap.Port = int(p)
	}

	// set some defaults in case of local development / missing VCAP_APPLICATION
	if vcap.Application.ID == "" {
		vcap.Application.ID = "123-456-789"
	}
	if vcap.Application.Name == "" {
		vcap.Application.Name = "devapp"
	}
	if vcap.Application.InstanceID == "" {
		vcap.Application.InstanceID = "987-654-321"
	}
	if vcap.Application.InstanceIndex == 0 {
		vcap.Application.InstanceIndex = 1
	}
	if vcap.Application.Host == "" {
		vcap.Application.Host = "localhost"
	}
	if vcap.Application.Port == 0 {
		vcap.Application.Port = 4000
	}
	if vcap.Host == "" {
		vcap.Host = "localhost"
	}
	if vcap.Port == 0 {
		vcap.Port = 4000
	}

	return vcap, nil
}
<commit_msg>correctly parse service credentials<commit_after>package vcap

import (
	"encoding/json"
	"os"
	"strconv"
	"time"
)

type VCAP struct {
	Application struct {
		ID            string   `json:"application_id"`
		Name          string   `json:"application_name"`
		Version       string   `json:"application_version"`
		InstanceID    string   `json:"instance_id"`
		InstanceIndex int      `json:"instance_index"`
		Host          string   `json:"host"`
		Port          int      `json:"port"`
		Users         string   `json:"users"`
		URIs          []string `json:"application_uris"`
		Limits        struct {
			Memory int `json:"mem"`
			Disk   int `json:"disk"`
			Files  int `json:"fds"`
		} `json:"limits"`
		Started *Timestamp `json:"started_at_timestamp"`
		State   *Timestamp `json:"state_timestamp"`
	}
	Host     string
	Port     int
	Services map[string][]Service
}

type Service struct {
	Name        string                 `json:"name"`
	Label       string                 `json:"label"`
	Tags        []string               `json:"tags"`
	Plan        string                 `json:"plan"`
	Credentials map[string]interface{} `json:"credentials"`
}

type Timestamp time.Time

func (t *Timestamp) String() string {
	return time.Time(*t).String()
}

func (t *Timestamp) UnmarshalJSON(b []byte) error {
	ts, err := strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		return err
	}

	*t = Timestamp(time.Unix(ts, 0))
	return nil
}

func New() (*VCAP, error) {
	vcap := &VCAP{}

	if app := os.Getenv("VCAP_APPLICATION"); app != "" {
		if err := json.Unmarshal([]byte(app), &(vcap.Application)); err != nil {
			return nil, err
		}
	}

	if serv := os.Getenv("VCAP_SERVICES"); serv != "" {
		if err := json.Unmarshal([]byte(serv), &(vcap.Services)); err != nil {
			return nil, err
		}
	}

	vcap.Host = os.Getenv("VCAP_APP_HOST")
	if port := os.Getenv("VCAP_APP_PORT"); port != "" {
		p, err := strconv.ParseInt(port, 10, 32)
		if err != nil {
			return nil, err
		}
		vcap.Port = int(p)
	}

	// set some defaults in case of local development / missing VCAP_APPLICATION
	if vcap.Application.ID == "" {
		vcap.Application.ID = "123-456-789"
	}
	if vcap.Application.Name == "" {
		vcap.Application.Name = "devapp"
	}
	if vcap.Application.InstanceID == "" {
		vcap.Application.InstanceID = "987-654-321"
	}
	if vcap.Application.InstanceIndex == 0 {
		vcap.Application.InstanceIndex = 1
	}
	if vcap.Application.Host == "" {
		vcap.Application.Host = "localhost"
	}
	if vcap.Application.Port == 0 {
		vcap.Application.Port = 4000
	}
	if vcap.Host == "" {
		vcap.Host = "localhost"
	}
	if vcap.Port == 0 {
		vcap.Port = 4000
	}

	return vcap, nil
}

func (v *VCAP) GetService(name string) *Service {
	for _, services := range v.Services {
		for _, service := range services {
			if service.Name == name {
				return &service
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package zeromq

import (
	"errors"

	"github.com/zeromq/gozmtp"
)

type Socket interface {
	Recv() ([]byte, error)
	Send([]byte) error
}

type socket struct{}

var (
	ClientSocketType = zmtp.ClientSocketType
	ServerSocketType = zmtp.ServerSocketType

	NullSecurityMechanism  = zmtp.NullSecurityMechanismType
	PlainSecurityMechanism = zmtp.PlainSecurityMechanismType
	CurveSecurityMechanism = zmtp.CurveSecurityMechanismType

	ErrNotImplemented = errors.New("not implemented")
)

func NewSecurityNull() *zmtp.SecurityNull {
	return &zmtp.SecurityNull{}
}

func NewSocket(socktype zmtp.SocketType, endpoint string, mechanism zmtp.SecurityMechanism) (Socket, error) {
	return &socket{}, ErrNotImplemented
}

func NewClient(endpoint string, mechanism zmtp.SecurityMechanism) (Socket, error) {
	return NewSocket(ClientSocketType, endpoint, mechanism)
}

func NewServer(endpoint string, mechanism zmtp.SecurityMechanism) (Socket, error) {
	return NewSocket(ServerSocketType, endpoint, mechanism)
}

func (s *socket) Recv() ([]byte, error) {
	var msg []byte
	return msg, ErrNotImplemented
}

func (s *socket) Send([]byte) error {
	return ErrNotImplemented
}
<commit_msg>Problem: security names did not match zmtp versions<commit_after>package zeromq

import (
	"errors"

	"github.com/zeromq/gozmtp"
)

type Socket interface {
	Recv() ([]byte, error)
	Send([]byte) error
}

type socket struct{}

var (
	ClientSocketType = zmtp.ClientSocketType
	ServerSocketType = zmtp.ServerSocketType

	NullSecurityMechanismType  = zmtp.NullSecurityMechanismType
	PlainSecurityMechanismType = zmtp.PlainSecurityMechanismType
	CurveSecurityMechanismTyp  = zmtp.CurveSecurityMechanismType

	ErrNotImplemented = errors.New("not implemented")
)

func NewSecurityNull() *zmtp.SecurityNull {
	return zmtp.NewSecurityNull()
}

func NewSocket(socktype zmtp.SocketType, endpoint string, mechanism zmtp.SecurityMechanism) (Socket, error) {
	return &socket{}, ErrNotImplemented
}

func NewClient(endpoint string, mechanism zmtp.SecurityMechanism) (Socket, error) {
	return NewSocket(ClientSocketType, endpoint, mechanism)
}

func NewServer(endpoint string, mechanism zmtp.SecurityMechanism) (Socket, error) {
	return NewSocket(ServerSocketType, endpoint, mechanism)
}

func (s *socket) Recv() ([]byte, error) {
	var msg []byte
	return msg, ErrNotImplemented
}

func (s *socket) Send([]byte) error {
	return ErrNotImplemented
}
<|endoftext|>


<commit_before>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cvefeed

import (
	"bytes"
	"sync"
	"unsafe"

	"github.com/facebookincubator/flog"
	"github.com/facebookincubator/nvdtools/wfn"
)

const cacheEvictPercentage = 0.1 // every eviction cycle invalidates this part of cache size at once

// Index maps the CPEs to the entries in the NVD feed they mentioned in
type Index map[string][]Vuln

// NewIndex creates new Index from a slice of CVE entries
func NewIndex(d Dictionary) Index {
	idx := Index{}
	for _, entry := range d {
		set := map[string]bool{}
		for _, cpe := range entry.Config() {
			// Can happen, for instance, when the feed contains illegal binding of CPE name. Unfortunately, it happens to NVD,
			// e.g. embedded ? in cpe:2.3:a:disney:where\\'s_my_perry?_free:1.5.1:*:*:*:*:android:*:* of CVE-2014-5606
			if cpe == nil {
				continue
			}
			product := cpe.Product
			if wfn.HasWildcard(product) {
				product = wfn.Any
			}
			if !set[product] {
				set[product] = true
				idx[product] = append(idx[product], entry)
			}
		}
	}
	return idx
}

// MatchResult stores CVE and a slice of CPEs that matched it
type MatchResult struct {
	CVE  Vuln
	CPEs []*wfn.Attributes
}

// cachedCVEs stores cached CVEs, a channel to signal if the value is ready
type cachedCVEs struct {
	res           []MatchResult
	ready         chan struct{}
	size          int64
	evictionIndex int // position in eviction queue
}

// updateResSize calculates the size of cached MatchResult and assigns it to cves.size
func (cves *cachedCVEs) updateResSize(key string) {
	cves.size = int64(int(unsafe.Sizeof(key)) + len(key))
	if cves == nil {
		return
	}
	cves.size += int64(unsafe.Sizeof(cves.res))
	for i := range cves.res {
		cves.size += int64(unsafe.Sizeof(cves.res[i].CVE))
		for _, attr := range cves.res[i].CPEs {
			cves.size += int64(len(attr.Part)) + int64(unsafe.Sizeof(attr.Part))
			cves.size += int64(len(attr.Vendor)) + int64(unsafe.Sizeof(attr.Vendor))
			cves.size += int64(len(attr.Product)) + int64(unsafe.Sizeof(attr.Product))
			cves.size += int64(len(attr.Version)) + int64(unsafe.Sizeof(attr.Version))
			cves.size += int64(len(attr.Update)) + int64(unsafe.Sizeof(attr.Update))
			cves.size += int64(len(attr.Edition)) + int64(unsafe.Sizeof(attr.Edition))
			cves.size += int64(len(attr.SWEdition)) + int64(unsafe.Sizeof(attr.SWEdition))
			cves.size += int64(len(attr.TargetHW)) + int64(unsafe.Sizeof(attr.TargetHW))
			cves.size += int64(len(attr.Other)) + int64(unsafe.Sizeof(attr.Other))
			cves.size += int64(len(attr.Language)) + int64(unsafe.Sizeof(attr.Language))
		}
	}
}

// Cache caches CVEs for known CPEs
type Cache struct {
	data           map[string]*cachedCVEs
	evictionQ      *evictionQueue
	mu             sync.Mutex
	Dict           Dictionary
	Idx            Index
	RequireVersion bool  // ignore matching specifications that have Version == ANY
	MaxSize        int64 // maximum size of the cache, 0 -- unlimited, -1 -- no caching
	size           int64 // current size of the cache
}

// NewCache creates new Cache instance with dictionary dict.
func NewCache(dict Dictionary) *Cache {
	return &Cache{Dict: dict, evictionQ: new(evictionQueue)}
}

// SetRequireVersion sets if the instance of cache fails matching the dictionary
// records without Version attribute of CPE name.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetRequireVersion(requireVersion bool) *Cache {
	c.RequireVersion = requireVersion
	return c
}

// SetMaxSize sets maximum size of the cache to some pre-defined value,
// size of 0 disables eviction (makes the cache grow indefinitely),
// negative size disables caching.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetMaxSize(size int64) *Cache {
	c.MaxSize = size
	return c
}

// Get returns slice of CVEs for CPE names from cpes parameter;
// if CVEs aren't cached (and the feature is enabled) it finds them in cveDict and caches the results
func (c *Cache) Get(cpes []*wfn.Attributes) []MatchResult {
	// negative max size of the cache disables caching
	if c.MaxSize < 0 {
		return c.match(cpes)
	}

	// otherwise, let's get to the business
	key := cacheKey(cpes)
	c.mu.Lock()
	if c.data == nil {
		c.data = make(map[string]*cachedCVEs)
	}
	cves := c.data[key]
	if cves != nil {
		// value is being computed, wait till ready
		c.mu.Unlock()
		<-cves.ready
		c.mu.Lock() // TODO: XXX: ugly, consider using atomic.Value instead
		cves.evictionIndex = c.evictionQ.touch(cves.evictionIndex)
		c.mu.Unlock()
		return cves.res
	}
	// first request; the goroutine that sent it computes the value
	cves = &cachedCVEs{ready: make(chan struct{})}
	c.data[key] = cves
	c.mu.Unlock()
	// now other requests for same key wait on the channel, and the requests for the different keys aren't blocked
	cves.res = c.match(cpes)
	cves.updateResSize(key)
	c.mu.Lock()
	c.size += cves.size
	if c.MaxSize != 0 && c.size > c.MaxSize {
		c.evict(int64(cacheEvictPercentage * float64(c.MaxSize)))
	}
	cves.evictionIndex = c.evictionQ.push(key)
	c.mu.Unlock()
	close(cves.ready)
	return cves.res
}

// match will return all match results based on the given cpes
func (c *Cache) match(cpes []*wfn.Attributes) []MatchResult {
	d := c.Dict
	if c.Idx != nil {
		d = c.dictFromIndex(cpes)
	}
	return c.matchDict(cpes, d)
}

// dictFromIndex creates CVE dictionary from entries indexed by CPE names
func (c *Cache) dictFromIndex(cpes []*wfn.Attributes) Dictionary {
	d := Dictionary{}
	if c.Idx == nil {
		return d
	}

	knownEntries := map[Vuln]bool{}
	addVulns := func(product string) {
		for _, vuln := range c.Idx[product] {
			if !knownEntries[vuln] {
				knownEntries[vuln] = true
				d[vuln.ID()] = vuln
			}
		}
	}

	for _, cpe := range cpes {
		if cpe == nil { // should never happen
			flog.Warning("nil CPE in list")
			continue
		}
		if cpe.Product != wfn.Any {
			addVulns(cpe.Product)
		}
	}
	addVulns(wfn.Any)

	return d
}

// match matches the CPE names against internal vulnerability dictionary and returns a slice of matching resutls
func (c *Cache) matchDict(cpes []*wfn.Attributes, dict Dictionary) (results []MatchResult) {
	for _, v := range dict {
		if matches := v.Match(cpes, c.RequireVersion); len(matches) > 0 {
			results = append(results, MatchResult{v, matches})
		}
	}
	return results
}

// evict the least recently used records untile nbytes of capacity is achieved or no more records left.
// It is not concurrency-safe, c.mu should be locked before calling it.
func (c *Cache) evict(nbytes int64) {
	for c.size+nbytes > c.MaxSize {
		key := c.evictionQ.pop()
		cd, ok := c.data[key]
		if !ok { // should not happen
			panic("attempted to evict non-existent record")
		}
		c.size -= cd.size
		delete(c.data, key)
	}
}

func cacheKey(cpes []*wfn.Attributes) string {
	var out bytes.Buffer
	for _, cpe := range cpes {
		if cpe == nil {
			continue
		}
		out.WriteString(cpe.Part)
		out.WriteByte('^')
		out.WriteString(cpe.Vendor)
		out.WriteByte('^')
		out.WriteString(cpe.Product)
		out.WriteByte('^')
		out.WriteString(cpe.Version)
		out.WriteByte('^')
		out.WriteString(cpe.Update)
		out.WriteByte('^')
		out.WriteString(cpe.Edition)
		out.WriteByte('^')
		out.WriteString(cpe.SWEdition)
		out.WriteByte('^')
		out.WriteString(cpe.TargetSW)
		out.WriteByte('^')
		out.WriteString(cpe.TargetHW)
		out.WriteByte('^')
		out.WriteString(cpe.Other)
		out.WriteByte('^')
		out.WriteString(cpe.Language)
		out.WriteByte('#')
	}
	return out.String()
}
<commit_msg>[cvfeed] Provide utility to compute the CVE cache hit ratio<commit_after>// Copyright (c) Facebook, Inc. and its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cvefeed

import (
	"bytes"
	"sync"
	"sync/atomic"
	"unsafe"

	"github.com/facebookincubator/flog"
	"github.com/facebookincubator/nvdtools/wfn"
)

const cacheEvictPercentage = 0.1 // every eviction cycle invalidates this part of cache size at once

// Index maps the CPEs to the entries in the NVD feed they mentioned in
type Index map[string][]Vuln

// NewIndex creates new Index from a slice of CVE entries
func NewIndex(d Dictionary) Index {
	idx := Index{}
	for _, entry := range d {
		set := map[string]bool{}
		for _, cpe := range entry.Config() {
			// Can happen, for instance, when the feed contains illegal binding of CPE name. Unfortunately, it happens to NVD,
			// e.g. embedded ? in cpe:2.3:a:disney:where\\'s_my_perry?_free:1.5.1:*:*:*:*:android:*:* of CVE-2014-5606
			if cpe == nil {
				continue
			}
			product := cpe.Product
			if wfn.HasWildcard(product) {
				product = wfn.Any
			}
			if !set[product] {
				set[product] = true
				idx[product] = append(idx[product], entry)
			}
		}
	}
	return idx
}

// MatchResult stores CVE and a slice of CPEs that matched it
type MatchResult struct {
	CVE  Vuln
	CPEs []*wfn.Attributes
}

// cachedCVEs stores cached CVEs, a channel to signal if the value is ready
type cachedCVEs struct {
	res           []MatchResult
	ready         chan struct{}
	size          int64
	evictionIndex int // position in eviction queue
}

// updateResSize calculates the size of cached MatchResult and assigns it to cves.size
func (cves *cachedCVEs) updateResSize(key string) {
	cves.size = int64(int(unsafe.Sizeof(key)) + len(key))
	if cves == nil {
		return
	}
	cves.size += int64(unsafe.Sizeof(cves.res))
	for i := range cves.res {
		cves.size += int64(unsafe.Sizeof(cves.res[i].CVE))
		for _, attr := range cves.res[i].CPEs {
			cves.size += int64(len(attr.Part)) + int64(unsafe.Sizeof(attr.Part))
			cves.size += int64(len(attr.Vendor)) + int64(unsafe.Sizeof(attr.Vendor))
			cves.size += int64(len(attr.Product)) + int64(unsafe.Sizeof(attr.Product))
			cves.size += int64(len(attr.Version)) + int64(unsafe.Sizeof(attr.Version))
			cves.size += int64(len(attr.Update)) + int64(unsafe.Sizeof(attr.Update))
			cves.size += int64(len(attr.Edition)) + int64(unsafe.Sizeof(attr.Edition))
			cves.size += int64(len(attr.SWEdition)) + int64(unsafe.Sizeof(attr.SWEdition))
			cves.size += int64(len(attr.TargetHW)) + int64(unsafe.Sizeof(attr.TargetHW))
			cves.size += int64(len(attr.Other)) + int64(unsafe.Sizeof(attr.Other))
			cves.size += int64(len(attr.Language)) + int64(unsafe.Sizeof(attr.Language))
		}
	}
}

// Cache caches CVEs for known CPEs
type Cache struct {
	data           map[string]*cachedCVEs
	evictionQ      *evictionQueue
	mu             sync.Mutex
	Dict           Dictionary
	Idx            Index
	RequireVersion bool  // ignore matching specifications that have Version == ANY
	MaxSize        int64 // maximum size of the cache, 0 -- unlimited, -1 -- no caching
	size           int64 // current size of the cache

	// Used to compute the hit ratio
	numLookups int64
	numHits    int64
}

// NewCache creates new Cache instance with dictionary dict.
func NewCache(dict Dictionary) *Cache {
	return &Cache{Dict: dict, evictionQ: new(evictionQueue)}
}

// SetRequireVersion sets if the instance of cache fails matching the dictionary
// records without Version attribute of CPE name.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetRequireVersion(requireVersion bool) *Cache {
	c.RequireVersion = requireVersion
	return c
}

// SetMaxSize sets maximum size of the cache to some pre-defined value,
// size of 0 disables eviction (makes the cache grow indefinitely),
// negative size disables caching.
// Returns a pointer to the instance of Cache, for easy chaining.
func (c *Cache) SetMaxSize(size int64) *Cache {
	c.MaxSize = size
	return c
}

// Get returns slice of CVEs for CPE names from cpes parameter;
// if CVEs aren't cached (and the feature is enabled) it finds them in cveDict and caches the results
func (c *Cache) Get(cpes []*wfn.Attributes) []MatchResult {
	atomic.AddInt64(&c.numLookups, 1)

	// negative max size of the cache disables caching
	if c.MaxSize < 0 {
		return c.match(cpes)
	}

	// otherwise, let's get to the business
	key := cacheKey(cpes)
	c.mu.Lock()
	if c.data == nil {
		c.data = make(map[string]*cachedCVEs)
	}
	cves := c.data[key]
	if cves != nil {
		atomic.AddInt64(&c.numHits, 1)

		// value is being computed, wait till ready
		c.mu.Unlock()
		<-cves.ready
		c.mu.Lock() // TODO: XXX: ugly, consider using atomic.Value instead
		cves.evictionIndex = c.evictionQ.touch(cves.evictionIndex)
		c.mu.Unlock()
		return cves.res
	}
	// first request; the goroutine that sent it computes the value
	cves = &cachedCVEs{ready: make(chan struct{})}
	c.data[key] = cves
	c.mu.Unlock()
	// now other requests for same key wait on the channel, and the requests for the different keys aren't blocked
	cves.res = c.match(cpes)
	cves.updateResSize(key)
	c.mu.Lock()
	c.size += cves.size
	if c.MaxSize != 0 && c.size > c.MaxSize {
		c.evict(int64(cacheEvictPercentage * float64(c.MaxSize)))
	}
	cves.evictionIndex = c.evictionQ.push(key)
	c.mu.Unlock()
	close(cves.ready)
	return cves.res
}

// match will return all match results based on the given cpes
func (c *Cache) match(cpes []*wfn.Attributes) []MatchResult {
	d := c.Dict
	if c.Idx != nil {
		d = c.dictFromIndex(cpes)
	}
	return c.matchDict(cpes, d)
}

// dictFromIndex creates CVE dictionary from entries indexed by CPE names
func (c *Cache) dictFromIndex(cpes []*wfn.Attributes) Dictionary {
	d := Dictionary{}
	if c.Idx == nil {
		return d
	}

	knownEntries := map[Vuln]bool{}
	addVulns := func(product string) {
		for _, vuln := range c.Idx[product] {
			if !knownEntries[vuln] {
				knownEntries[vuln] = true
				d[vuln.ID()] = vuln
			}
		}
	}

	for _, cpe := range cpes {
		if cpe == nil { // should never happen
			flog.Warning("nil CPE in list")
			continue
		}
		if cpe.Product != wfn.Any {
			addVulns(cpe.Product)
		}
	}
	addVulns(wfn.Any)

	return d
}

// match matches the CPE names against internal vulnerability dictionary and returns a slice of matching resutls
func (c *Cache) matchDict(cpes []*wfn.Attributes, dict Dictionary) (results []MatchResult) {
	for _, v := range dict {
		if matches := v.Match(cpes, c.RequireVersion); len(matches) > 0 {
			results = append(results, MatchResult{v, matches})
		}
	}
	return results
}

// evict the least recently used records untile nbytes of capacity is achieved or no more records left.
// It is not concurrency-safe, c.mu should be locked before calling it.
func (c *Cache) evict(nbytes int64) {
	for c.size+nbytes > c.MaxSize {
		key := c.evictionQ.pop()
		cd, ok := c.data[key]
		if !ok { // should not happen
			panic("attempted to evict non-existent record")
		}
		c.size -= cd.size
		delete(c.data, key)
	}
}

func cacheKey(cpes []*wfn.Attributes) string {
	var out bytes.Buffer
	for _, cpe := range cpes {
		if cpe == nil {
			continue
		}
		out.WriteString(cpe.Part)
		out.WriteByte('^')
		out.WriteString(cpe.Vendor)
		out.WriteByte('^')
		out.WriteString(cpe.Product)
		out.WriteByte('^')
		out.WriteString(cpe.Version)
		out.WriteByte('^')
		out.WriteString(cpe.Update)
		out.WriteByte('^')
		out.WriteString(cpe.Edition)
		out.WriteByte('^')
		out.WriteString(cpe.SWEdition)
		out.WriteByte('^')
		out.WriteString(cpe.TargetSW)
		out.WriteByte('^')
		out.WriteString(cpe.TargetHW)
		out.WriteByte('^')
		out.WriteString(cpe.Other)
		out.WriteByte('^')
		out.WriteString(cpe.Language)
		out.WriteByte('#')
	}
	return out.String()
}

// HitRatio returns the cache hit ratio, the number of cache hits to the number
// of lookups, as a percentage.
func (c *Cache) HitRatio() float64 {
	if c.numLookups == 0 {
		return 0
	}
	return float64(c.numHits) / float64(c.numLookups) * 100
}
<|endoftext|>


<commit_before>/*
Tiny web server to handle slack outgoing webhook and push the data to resque
*/
package main

import (
	"fmt"
	"flag"
	"net/http"
	"net/url"
	"encoding/json"
	"strings"
	"github.com/fzzy/radix/redis"
	"github.com/glacjay/goini"
)

// dictionary of string
type Dict map[string]bool
func StringToDict(s string, d Dict) {
	for _, v := range strings.Split(s, ",") {
		d[v] = true
	}
	return
}

// access rule
type AccessRule struct {
	Name string
	Users Dict
	Policy bool
	Class string
	Queue string
	Allow_msg string
	Deny_msg string
}

type AccessList map[string]*AccessRule

// some default global variable
var (
	PORT string = ":8765"
	API_PATH string = "/api"
	REDIS string = "localhost:6379"
	CLASS string = "SlackOPS"
	QUEUE string = "slackops"
	TOKENS Dict = make(Dict)
	VERBOSE bool = false
	FLAGS Dict = make(Dict)
	ACCESS_LIST AccessList = make(AccessList)
	DEFAULT_ALLOW_RESPONSE_TEXT = "please wait"
	DEFAULT_DENY_RESPONSE_TEXT = "task denied"
	UNDEFINED_JOB_CAN_PASS = true
	err error
)

func check_flag(f *flag.Flag) {
	FLAGS[f.Name] = true
}

func main() {
	// config parameter
	p_port := flag.String("p", "8765", "listen port")
	p_redis := flag.String("r", "localhost:6379", "redis host")
	p_class := flag.String("c", "SlackOPS", "resque class")
	p_queue := flag.String("q", "slackops", "resque queue")
	p_tokens := flag.String("t", "", "slack tokens, split by a comma (,)")
	p_verbose := flag.Bool("v", false, "verbose")
	p_config := flag.String("C", "", "configuration file")
	flag.Parse()

	// mark parsed flags
	flag.Visit(check_flag)

	// read config file
	if FLAGS["C"] {
		config, err := ini.Load(*p_config)
		if err != nil {
			fmt.Println("error:", err)
			return
		}

		// load general config
		c_port, exist := config.GetString("general", "port")
		if exist {
			PORT = ":" + c_port
		}
		c_redis, exist := config.GetString("general", "redis")
		if exist {
			REDIS = c_redis
		}
		c_class, exist := config.GetString("general", "class")
		if exist {
			CLASS = c_class
		}
		c_queue, exist := config.GetString("general", "queue")
		if exist {
			QUEUE = c_queue
		}
		c_tokens, exist := config.GetString("general", "tokens")
		if exist {
			StringToDict(c_tokens, TOKENS)
		}
		c_verbose, exist := config.GetBool("general", "verbose")
		if exist {
			VERBOSE = c_verbose
		}

		// load access list
		for _, s := range config.GetSections() {
			if strings.HasPrefix(s, "job: ") {
				// read users and policy (mandatory)
				users_string, e_users := config.GetString(s, "users")
				policy_string, e_policy := config.GetString(s, "policy")
				if !(e_users && e_policy) {
					fmt.Printf("Skipping %s\nError: policy and users are mandatory in a job", s)
					continue
				}

				// read optional config
				allow_msg, _ := config.GetString(s, "allow_msg")
				deny_msg, _ := config.GetString(s, "deny_msg")
				class, _ := config.GetString(s, "class")
				queue, _ := config.GetString(s, "queue")

				// parse users and policy
				var policy bool
				if policy_string == "allow" {
					policy = true
				} else if policy_string == "deny" {
					policy = false
				} else {
					fmt.Printf("Skipping %s\nError: policy must be either 'allow' or 'deny'", s)
					continue
				}
				users := make(Dict)
				StringToDict(users_string, users)

				// create AccessRule
				ar := AccessRule{
					Name: strings.TrimPrefix(s, "job: "),
					Users: users,
					Policy: policy,
					Class: class,
					Queue: queue,
					Allow_msg: allow_msg,
					Deny_msg: deny_msg,
				}
				ACCESS_LIST[ar.Name] = &ar
			}
		}
	}

	// override configuration with flags
	if FLAGS["p"] {
		PORT = ":" + *p_port
	}
	if FLAGS["r"] {
		REDIS = *p_redis
	}
	if FLAGS["c"] {
		CLASS = *p_class
	}
	if FLAGS["q"] {
		QUEUE = *p_queue
	}
	if FLAGS["t"] {
		StringToDict(*p_tokens, TOKENS)
	}
	if FLAGS["v"] {
		VERBOSE = *p_verbose
	}

	if VERBOSE {
		fmt.Printf("Listening on port %s using resque at %s\n", PORT, REDIS)
		fmt.Printf("Accepting tokens:\n")
		for t, _ := range TOKENS {
			fmt.Printf("+ %s\n", t)
		}
		fmt.Printf("Jobs will be pushed to queue %s, class %s\n", QUEUE, CLASS)
	}

	// connect to redis and add queue
	rcon, err := redis.Dial("tcp", REDIS)
	if err != nil {
		fmt.Println("error:", err)
		return
	}
	rcon.Cmd("SADD", "resque:queues", QUEUE)
	// add queue optionally specified in jobs
	for k, v := range ACCESS_LIST {
		if len(v.Queue) > 0 {
			rcon.Cmd("SADD", "resque:queues", v.Queue)
			if VERBOSE {
				fmt.Printf("Job '%s' will be pushed to queue %s", k, v.Queue)
				if len(v.Class) > 0 {
					fmt.Printf(", class %s", v.Class)
				}
				fmt.Printf("\n")
			}
		}
	}
	rcon.Close()

	// start web app
	http.HandleFunc(API_PATH, apiHandler)
	err = http.ListenAndServe(PORT, nil)
	if err != nil {
		fmt.Println("error:", err)
		return
	}
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	type Response struct {
		Text string `json:"text"`
	}

	type Job struct {
		Class string `json:"class"`
		Args map[string]string `json:"args"`
	}

	// accept only POST
	if r.Method != "POST" {
		return
	}

	// read and parse request body content
	body := make([]byte, r.ContentLength)
	r.Body.Read(body)
	data, err := url.ParseQuery(string(body))
	if err != nil {
		fmt.Println("error:", err)
	}

	// check token
	if ! TOKENS[data.Get("token")] {
		return
	}

	// get request and user name
	request := strings.TrimPrefix(data.Get("text"), data.Get("trigger_word"))
	request = strings.TrimLeft(request, " ")
	request = strings.TrimRight(request, " ")
	user := data.Get("user_name")

	// filter with access list
	var pass bool = UNDEFINED_JOB_CAN_PASS
	var response_text string
	if UNDEFINED_JOB_CAN_PASS {
		response_text = DEFAULT_ALLOW_RESPONSE_TEXT
	} else {
		response_text = DEFAULT_DENY_RESPONSE_TEXT
	}
	var class string = CLASS
	var queue string = "resque:queue:" + QUEUE

	// if job is defined
	if ACCESS_LIST[request] != nil {
		// if user in deny list
		if (!ACCESS_LIST[request].Policy && ACCESS_LIST[request].Users[user]) {
			pass = false

		// if user not in allow list
		} else if (ACCESS_LIST[request].Policy && !ACCESS_LIST[request].Users[user]) {
			pass = false

		// if user in allow list
		} else if (ACCESS_LIST[request].Policy && ACCESS_LIST[request].Users[user]) {
			pass = true

		// if user not in deny list
		} else if (!ACCESS_LIST[request].Policy && !ACCESS_LIST[request].Users[user]) {
			pass = true
		}

		// if user is allowed
		if pass {
			// choose response text
			if len(ACCESS_LIST[request].Allow_msg) > 0 {
				response_text = ACCESS_LIST[request].Allow_msg
			} else {
				response_text = DEFAULT_ALLOW_RESPONSE_TEXT
			}

			// choose class
			if len(ACCESS_LIST[request].Class) > 0 {
				class = ACCESS_LIST[request].Class
			} else {
				class = CLASS
			}

			// choose queue
			if len(ACCESS_LIST[request].Queue) > 0 {
				queue = "resque:queue:" + ACCESS_LIST[request].Queue
			} else {
				queue = "resque:queue:" + QUEUE
			}

		// if user is denied
		} else {
			if len(ACCESS_LIST[request].Deny_msg) > 0 {
				response_text = ACCESS_LIST[request].Deny_msg
			} else {
				response_text = DEFAULT_DENY_RESPONSE_TEXT
			}
		}

	} else {
		// undefined job
		if pass {
			response_text = DEFAULT_ALLOW_RESPONSE_TEXT
			class = CLASS
			queue = "resque:queue:" + QUEUE
		} else {
			response_text = DEFAULT_DENY_RESPONSE_TEXT
		}
	}

	if pass {
		// make connection to redis
		rcon, err := redis.Dial("tcp", REDIS)
		if err != nil {
			fmt.Println("error:", err)
		}

		// create job
		job := Job{
			Class: class,
			Args: map[string]string {
			"request": request,
			"user": user,
			"channel": data.Get("channel_name"),
			"timestamp": data.Get("timestamp"),
			},
		}
		json_job, err := json.Marshal(job)
		if err != nil {
			fmt.Println("error:", err)
		}
		// push job to resque
		rcon.Cmd("RPUSH", queue, string(json_job))
		rcon.Close()
	}

	// response to slack
	res := Response{
		Text: "@" + user + ": " + response_text,
	}
	json_res, err := json.Marshal(res)
	if err != nil {
		fmt.Println("error:", err)
	}
	w.Header().Set("Content-type", "application/json")
	fmt.Fprintf(w, string(json_res))
}
<commit_msg>more config<commit_after>/*
Tiny web server to handle slack outgoing webhook and push the data to resque
*/
package main

import (
	"fmt"
	"flag"
	"net/http"
	"net/url"
	"encoding/json"
	"strings"
	"github.com/fzzy/radix/redis"
	"github.com/glacjay/goini"
)

// dictionary of string
type Dict map[string]bool
func StringToDict(s string, d Dict) {
	for _, v := range strings.Split(s, ",") {
		d[v] = true
	}
	return
}

// access rule
type AccessRule struct {
	Name string
	Users Dict
	Policy bool
	Class string
	Queue string
	Allow_msg string
	Deny_msg string
}

type AccessList map[string]*AccessRule

// some default global variable
var (
	PORT string = ":8765"
	API_PATH string = "/api"
	REDIS string = "localhost:6379"
	CLASS string = "SlackOPS"
	QUEUE string = "slackops"
	TOKENS Dict = make(Dict)
	VERBOSE bool = false
	FLAGS Dict = make(Dict)
	ACCESS_LIST AccessList = make(AccessList)
	DEFAULT_ALLOW_RESPONSE_TEXT = "please wait"
	DEFAULT_DENY_RESPONSE_TEXT = "task denied"
	UNDEFINED_JOB_CAN_PASS = true
	err error
)

func check_flag(f *flag.Flag) {
	FLAGS[f.Name] = true
}

func main() {
	// config parameter
	p_port := flag.String("p", "8765", "listen port")
	p_redis := flag.String("r", "localhost:6379", "redis host")
	p_class := flag.String("c", "SlackOPS", "resque class")
	p_queue := flag.String("q", "slackops", "resque queue")
	p_tokens := flag.String("t", "", "slack tokens, split by a comma (,)")
	p_verbose := flag.Bool("v", false, "verbose")
	p_config := flag.String("C", "", "configuration file")
	p_undefined_job_can_pass := flag.Bool("undefined-job-can-pass", false, "Undefined job can pass or not")
	p_default_allow_response_text := flag.String("default-allow-response-text", "please wait", "Specify default allow response text")
	p_default_deny_response_text := flag.String("default-deny-response-text", "task denied", "Specify default deny response text")
	flag.Parse()

	// mark parsed flags
	flag.Visit(check_flag)

	// read config file
	if FLAGS["C"] {
		config, err := ini.Load(*p_config)
		if err != nil {
			fmt.Println("error:", err)
			return
		}

		// load general config
		c_port, exist := config.GetString("general", "port")
		if exist {
			PORT = ":" + c_port
		}
		c_redis, exist := config.GetString("general", "redis")
		if exist {
			REDIS = c_redis
		}
		c_class, exist := config.GetString("general", "class")
		if exist {
			CLASS = c_class
		}
		c_queue, exist := config.GetString("general", "queue")
		if exist {
			QUEUE = c_queue
		}
		c_tokens, exist := config.GetString("general", "tokens")
		if exist {
			StringToDict(c_tokens, TOKENS)
		}
		c_verbose, exist := config.GetBool("general", "verbose")
		if exist {
			VERBOSE = c_verbose
		}
		c_undefined_job_can_pass, exist := config.GetBool("general", "undefined_job_can_pass")
		if exist {
			UNDEFINED_JOB_CAN_PASS = c_undefined_job_can_pass
		}
		c_default_allow_response_text, exist := config.GetString("general", "default_allow_response_text")
		if exist {
			DEFAULT_ALLOW_RESPONSE_TEXT = c_default_allow_response_text
		}
		c_default_deny_response_text, exist := config.GetString("general", "default_deny_response_text")
		if exist {
			DEFAULT_DENY_RESPONSE_TEXT = c_default_deny_response_text
		}

		// load access list
		for _, s := range config.GetSections() {
			if strings.HasPrefix(s, "job: ") {
				// read users and policy (mandatory)
				users_string, e_users := config.GetString(s, "users")
				policy_string, e_policy := config.GetString(s, "policy")
				if !(e_users && e_policy) {
					fmt.Printf("Skipping %s\nError: policy and users are mandatory in a job", s)
					continue
				}

				// read optional config
				allow_msg, _ := config.GetString(s, "allow_msg")
				deny_msg, _ := config.GetString(s, "deny_msg")
				class, _ := config.GetString(s, "class")
				queue, _ := config.GetString(s, "queue")

				// parse users and policy
				var policy bool
				if policy_string == "allow" {
					policy = true
				} else if policy_string == "deny" {
					policy = false
				} else {
					fmt.Printf("Skipping %s\nError: policy must be either 'allow' or 'deny'", s)
					continue
				}
				users := make(Dict)
				StringToDict(users_string, users)

				// create AccessRule
				ar := AccessRule{
					Name: strings.TrimPrefix(s, "job: "),
					Users: users,
					Policy: policy,
					Class: class,
					Queue: queue,
					Allow_msg: allow_msg,
					Deny_msg: deny_msg,
				}
				ACCESS_LIST[ar.Name] = &ar
			}
		}
	}

	// override configuration with flags
	if FLAGS["p"] {
		PORT = ":" + *p_port
	}
	if FLAGS["r"] {
		REDIS = *p_redis
	}
	if FLAGS["c"] {
		CLASS = *p_class
	}
	if FLAGS["q"] {
		QUEUE = *p_queue
	}
	if FLAGS["t"] {
		StringToDict(*p_tokens, TOKENS)
	}
	if FLAGS["v"] {
		VERBOSE = *p_verbose
	}
	if FLAGS["undefined-job-can-pass"] {
		UNDEFINED_JOB_CAN_PASS = *p_undefined_job_can_pass
	}
	if FLAGS["default-allow-response-text"] {
		DEFAULT_ALLOW_RESPONSE_TEXT = *p_default_allow_response_text
	}
	if FLAGS["default-deny-response-text"] {
		DEFAULT_DENY_RESPONSE_TEXT = *p_default_deny_response_text
	}

	if VERBOSE {
		fmt.Printf("Listening on port %s using resque at %s\n", PORT, REDIS)
		fmt.Printf("Accepting tokens:\n")
		for t, _ := range TOKENS {
			fmt.Printf("+ %s\n", t)
		}
		fmt.Printf("Jobs will be pushed to queue %s, class %s\n", QUEUE, CLASS)
	}

	// connect to redis and add queue
	rcon, err := redis.Dial("tcp", REDIS)
	if err != nil {
		fmt.Println("error:", err)
		return
	}
	rcon.Cmd("SADD", "resque:queues", QUEUE)
	// add queue optionally specified in jobs
	for k, v := range ACCESS_LIST {
		if len(v.Queue) > 0 {
			rcon.Cmd("SADD", "resque:queues", v.Queue)
			if VERBOSE {
				fmt.Printf("Job '%s' will be pushed to queue %s", k, v.Queue)
				if len(v.Class) > 0 {
					fmt.Printf(", class %s", v.Class)
				}
				fmt.Printf("\n")
			}
		}
	}
	rcon.Close()

	// start web app
	http.HandleFunc(API_PATH, apiHandler)
	err = http.ListenAndServe(PORT, nil)
	if err != nil {
		fmt.Println("error:", err)
		return
	}
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
	type Response struct {
		Text string `json:"text"`
	}

	type Job struct {
		Class string `json:"class"`
		Args map[string]string `json:"args"`
	}

	// accept only POST
	if r.Method != "POST" {
		return
	}

	// read and parse request body content
	body := make([]byte, r.ContentLength)
	r.Body.Read(body)
	data, err := url.ParseQuery(string(body))
	if err != nil {
		fmt.Println("error:", err)
	}

	// check token
	if ! TOKENS[data.Get("token")] {
		return
	}

	// get request and user name
	request := strings.TrimPrefix(data.Get("text"), data.Get("trigger_word"))
	request = strings.TrimLeft(request, " ")
	request = strings.TrimRight(request, " ")
	user := data.Get("user_name")

	// filter with access list
	var pass bool = UNDEFINED_JOB_CAN_PASS
	var response_text string
	if UNDEFINED_JOB_CAN_PASS {
		response_text = DEFAULT_ALLOW_RESPONSE_TEXT
	} else {
		response_text = DEFAULT_DENY_RESPONSE_TEXT
	}
	var class string = CLASS
	var queue string = "resque:queue:" + QUEUE

	// if job is defined
	if ACCESS_LIST[request] != nil {
		// if user in deny list
		if (!ACCESS_LIST[request].Policy && ACCESS_LIST[request].Users[user]) {
			pass = false

		// if user not in allow list
		} else if (ACCESS_LIST[request].Policy && !ACCESS_LIST[request].Users[user]) {
			pass = false

		// if user in allow list
		} else if (ACCESS_LIST[request].Policy && ACCESS_LIST[request].Users[user]) {
			pass = true

		// if user not in deny list
		} else if (!ACCESS_LIST[request].Policy && !ACCESS_LIST[request].Users[user]) {
			pass = true
		}

		// if user is allowed
		if pass {
			// choose response text
			if len(ACCESS_LIST[request].Allow_msg) > 0 {
				response_text = ACCESS_LIST[request].Allow_msg
			} else {
				response_text = DEFAULT_ALLOW_RESPONSE_TEXT
			}

			// choose class
			if len(ACCESS_LIST[request].Class) > 0 {
				class = ACCESS_LIST[request].Class
			} else {
				class = CLASS
			}

			// choose queue
			if len(ACCESS_LIST[request].Queue) > 0 {
				queue = "resque:queue:" + ACCESS_LIST[request].Queue
			} else {
				queue = "resque:queue:" + QUEUE
			}

		// if user is denied
		} else {
			if len(ACCESS_LIST[request].Deny_msg) > 0 {
				response_text = ACCESS_LIST[request].Deny_msg
			} else {
				response_text = DEFAULT_DENY_RESPONSE_TEXT
			}
		}

	} else {
		// undefined job
		if pass {
			response_text = DEFAULT_ALLOW_RESPONSE_TEXT
			class = CLASS
			queue = "resque:queue:" + QUEUE
		} else {
			response_text = DEFAULT_DENY_RESPONSE_TEXT
		}
	}

	if pass {
		// make connection to redis
		rcon, err := redis.Dial("tcp", REDIS)
		if err != nil {
			fmt.Println("error:", err)
		}

		// create job
		job := Job{
			Class: class,
			Args: map[string]string {
			"request": request,
			"user": user,
			"channel": data.Get("channel_name"),
			"timestamp": data.Get("timestamp"),
			},
		}
		json_job, err := json.Marshal(job)
		if err != nil {
			fmt.Println("error:", err)
		}
		// push job to resque
		rcon.Cmd("RPUSH", queue, string(json_job))
		rcon.Close()
	}

	// response to slack
	res := Response{
		Text: "@" + user + ": " + response_text,
	}
	json_res, err := json.Marshal(res)
	if err != nil {
		fmt.Println("error:", err)
	}
	w.Header().Set("Content-type", "application/json")
	fmt.Fprintf(w, string(json_res))
}
<|endoftext|>