<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gles

import (
	"context"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/resolve/dependencygraph"
)

type uniformKey struct {
	program  *Program
	location UniformLocation
	count    GLsizei
}

func (k uniformKey) Parent() dependencygraph.StateKey { return uniformGroupKey{k.program} }

type uniformGroupKey struct {
	program *Program
}

func (k uniformGroupKey) Parent() dependencygraph.StateKey { return nil }

type vertexAttribKey struct {
	vertexArray *VertexArray
	location    AttributeLocation
}

func (k vertexAttribKey) Parent() dependencygraph.StateKey {
	return vertexAttribGroupKey{k.vertexArray}
}

type vertexAttribGroupKey struct {
	vertexArray *VertexArray
}

func (k vertexAttribGroupKey) Parent() dependencygraph.StateKey { return nil }

type renderbufferDataKey struct {
	renderbuffer *Renderbuffer
}

func (k renderbufferDataKey) Parent() dependencygraph.StateKey { return nil }

type renderbufferSubDataKey struct {
	renderbuffer *Renderbuffer
	region       Rect
}

func (k renderbufferSubDataKey) Parent() dependencygraph.StateKey {
	return renderbufferDataKey{k.renderbuffer}
}

type textureDataKey struct {
	texture *Texture
	id      TextureId // For debugging, as 0 is not unique identifier.
}

func (k textureDataKey) Parent() dependencygraph.StateKey { return nil }

type textureSizeKey struct {
	texture *Texture
	id      TextureId // For debugging, as 0 is not unique identifier.
}

func (k textureSizeKey) Parent() dependencygraph.StateKey { return nil }

type eglImageDataKey struct {
	image *EGLImage
}

func (k eglImageDataKey) Parent() dependencygraph.StateKey { return nil }

type eglImageSizeKey struct {
	image *EGLImage
}

func (k eglImageSizeKey) Parent() dependencygraph.StateKey { return nil }

type GlesDependencyGraphBehaviourProvider struct {
}

func newGlesDependencyGraphBehaviourProvider() *GlesDependencyGraphBehaviourProvider {
	return &GlesDependencyGraphBehaviourProvider{}
}

// GetBehaviourForAtom returns state reads/writes that the given atom performs.
//
// Writes: Write dependencies keep atoms alive. Each atom must correctly report
// all its writes or it must set the keep-alive flag. If a write is missing
// then the liveness analysis will remove the atom since it seems unneeded.
// It is fine to omit related/overloaded commands that write to the same state
// as long as they are marked as keep-alive.
//
// Reads: For each state write, all commands that could possibly read it must be
// implemented. This makes it more difficult to do only partial implementations.
// It is fine to overestimate reads, or to read parent state (i.e. superset).
//
func (*GlesDependencyGraphBehaviourProvider) GetBehaviourForAtom(
	ctx context.Context, s *api.State, id api.CmdID, cmd api.Cmd, g *dependencygraph.DependencyGraph) dependencygraph.AtomBehaviour {
	b := dependencygraph.AtomBehaviour{}
	c := GetContext(s, cmd.Thread())
	if c != nil && c.Info.Initialized {
		_, isEglSwapBuffers := cmd.(*EglSwapBuffers)
		_, isEglSwapBuffersWithDamageKHR := cmd.(*EglSwapBuffersWithDamageKHR)
		if isEglSwapBuffers || isEglSwapBuffersWithDamageKHR {
			// Get default renderbuffers
			fb := c.Objects.Framebuffers[0]
			color := fb.ColorAttachments[0].Renderbuffer
			depth := fb.DepthAttachment.Renderbuffer
			stencil := fb.StencilAttachment.Renderbuffer
			if !c.Info.PreserveBuffersOnSwap {
				b.Write(g, renderbufferDataKey{color})
			}
			b.Write(g, renderbufferDataKey{depth})
			b.Write(g, renderbufferDataKey{stencil})
		} else if cmd.CmdFlags().IsDrawCall() {
			b.Read(g, uniformGroupKey{c.Bound.Program})
			b.Read(g, vertexAttribGroupKey{c.Bound.VertexArray})
			for _, stateKey := range getAllUsedTextureData(ctx, cmd, s, c) {
				b.Read(g, stateKey)
			}
			fb := c.Bound.DrawFramebuffer
			for _, att := range fb.ColorAttachments {
				b.Modify(g, getAttachmentData(g, c, att))
			}
			b.Modify(g, getAttachmentData(g, c, fb.DepthAttachment))
			b.Modify(g, getAttachmentData(g, c, fb.StencilAttachment))
			// TODO: Write transform feedback buffers.
		} else {
			switch cmd := cmd.(type) {
			case *GlClear:
				fb := c.Bound.DrawFramebuffer
				if (cmd.Mask & GLbitfield_GL_COLOR_BUFFER_BIT) != 0 {
					for _, att := range fb.ColorAttachments {
						b.Read(g, getAttachmentSize(g, c, att))
						b.Write(g, getAttachmentData(g, c, att))
					}
				}
				if (cmd.Mask & GLbitfield_GL_DEPTH_BUFFER_BIT) != 0 {
					b.Read(g, getAttachmentSize(g, c, fb.DepthAttachment))
					b.Write(g, getAttachmentData(g, c, fb.DepthAttachment))
				}
				if (cmd.Mask & GLbitfield_GL_STENCIL_BUFFER_BIT) != 0 {
					b.Read(g, getAttachmentSize(g, c, fb.StencilAttachment))
					b.Write(g, getAttachmentData(g, c, fb.StencilAttachment))
				}
			case *GlCopyImageSubData:
				// TODO: This assumes whole-image copy.  Handle sub-range copies.
				if cmd.SrcTarget == GLenum_GL_RENDERBUFFER {
					b.Read(g, renderbufferDataKey{c.Objects.Shared.Renderbuffers[RenderbufferId(cmd.SrcName)]})
				} else {
					data, size := c.Objects.Shared.Textures[TextureId(cmd.SrcName)].getTextureDataAndSize()
					b.Read(g, data)
					b.Read(g, size)
				}
				if cmd.DstTarget == GLenum_GL_RENDERBUFFER {
					b.Write(g,
						renderbufferDataKey{c.Objects.Shared.Renderbuffers[RenderbufferId(cmd.DstName)]})
				} else {
					data, size := c.Objects.Shared.Textures[TextureId(cmd.DstName)].getTextureDataAndSize()
					b.Write(g, data)
					b.Write(g, size)
				}
			case *GlFramebufferTexture2D:
				b.Read(g, textureSizeKey{c.Objects.Shared.Textures[cmd.Texture], cmd.Texture})
				b.KeepAlive = true // Changes untracked state
			case *GlCompressedTexImage2D:
				texData, texSize := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
				b.Write(g, texSize)
			case *GlCompressedTexSubImage2D:
				texData, _ := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
			case *GlTexImage2D:
				texData, texSize := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
				b.Write(g, texSize)
			case *GlTexSubImage2D:
				texData, _ := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
			case *GlUniform1fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniform2fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniform3fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniform4fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniformMatrix4fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlVertexAttribPointer:
				b.Write(g, vertexAttribKey{c.Bound.VertexArray, cmd.Location})
			default:
				// Force all unhandled atoms to be kept alive.
				b.KeepAlive = true
			}
		}
	} else /* c == nil */ {
		b.KeepAlive = true
	}
	if err := cmd.Mutate(ctx, s, nil /* builder */); err != nil {
		log.W(ctx, "Command %v %v: %v", id, cmd, err)
		return dependencygraph.AtomBehaviour{Aborted: true}
	}
	return b
}

func getAllUsedTextureData(ctx context.Context, cmd api.Cmd, s *api.State, c *Context) (stateKeys []dependencygraph.StateKey) {
	// Look for samplers used by the current program.
	if prog := c.Bound.Program; prog != nil {
		for _, activeUniform := range prog.ActiveUniforms {
			// Optimization - skip the two most common types which we know are not samplers.
			if activeUniform.Type != GLenum_GL_FLOAT_VEC4 && activeUniform.Type != GLenum_GL_FLOAT_MAT4 {
				target, _ := subGetTextureTargetFromSamplerType(ctx, cmd, nil, s, GetState(s), cmd.Thread(), nil, activeUniform.Type)
				if target == GLenum_GL_NONE {
					continue // Not a sampler type
				}
				for i := 0; i < int(activeUniform.ArraySize); i++ {
					uniform := prog.Uniforms[activeUniform.Location+UniformLocation(i)]
					units := AsU32ˢ(uniform.Value, s.MemoryLayout).Read(ctx, cmd, s, nil)
					if len(units) == 0 {
						units = []uint32{0} // The uniform was not set, so use default value.
					}
					for _, unit := range units {
						if tu := c.Objects.TextureUnits[TextureUnitId(unit)]; tu != nil {
							texData, _ := getTextureDataAndSize(ctx, cmd, s, tu, target)
							stateKeys = append(stateKeys, texData)
						}
					}
				}
			}
		}
	}
	return
}

func getTextureDataAndSize(ctx context.Context, cmd api.Cmd, s *api.State, unit *TextureUnit, target GLenum) (dependencygraph.StateKey, dependencygraph.StateKey) {
	tex, err := subGetBoundTextureForUnit(ctx, cmd, nil, s, GetState(s), cmd.Thread(), nil, unit, target)
	if tex == nil || err != nil {
		log.E(ctx, "Can not find texture %v in unit %v", target, unit)
		return nil, nil
	}
	return tex.getTextureDataAndSize()
}

func (tex *Texture) getTextureDataAndSize() (dependencygraph.StateKey, dependencygraph.StateKey) {
	if tex.EGLImage != nil {
		return eglImageDataKey{tex.EGLImage}, eglImageSizeKey{tex.EGLImage}
	} else {
		return textureDataKey{tex, tex.ID}, textureSizeKey{tex, tex.ID}
	}
}

func getAttachmentData(g *dependencygraph.DependencyGraph, c *Context, att FramebufferAttachment) (key dependencygraph.StateKey) {
	if att.Type == GLenum_GL_RENDERBUFFER {
		rb := att.Renderbuffer
		if rb != nil && rb.InternalFormat != GLenum_GL_NONE {
			scissor := c.Pixel.Scissor
			fullBox := Rect{Width: rb.Width, Height: rb.Height}
			if scissor.Test == GLboolean_GL_TRUE && scissor.Box != fullBox {
				key = renderbufferSubDataKey{rb, scissor.Box}
			} else {
				key = renderbufferDataKey{rb}
			}
		}
	}
	if att.Type == GLenum_GL_TEXTURE {
		tex := att.Texture
		if tex != nil {
			// TODO: We should handle scissor here as well.
			if tex.EGLImage != nil {
				key = eglImageDataKey{tex.EGLImage}
			} else {
				key = textureDataKey{tex, tex.ID}
			}
		}
	}
	if key != nil {
		g.SetRoot(key)
	}
	return
}

func getAttachmentSize(g *dependencygraph.DependencyGraph, c *Context, att FramebufferAttachment) (key dependencygraph.StateKey) {
	if att.Type == GLenum_GL_TEXTURE {
		tex := att.Texture
		if tex != nil {
			if tex.EGLImage != nil {
				key = eglImageSizeKey{tex.EGLImage}
			} else {
				key = textureSizeKey{tex, tex.ID}
			}
		}
	}
	if key != nil {
		g.SetRoot(key)
	}
	return
}
<commit_msg>Add dependency tracking for other clear variants<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gles

import (
	"context"

	"github.com/google/gapid/core/log"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/resolve/dependencygraph"
)

type uniformKey struct {
	program  *Program
	location UniformLocation
	count    GLsizei
}

func (k uniformKey) Parent() dependencygraph.StateKey { return uniformGroupKey{k.program} }

type uniformGroupKey struct {
	program *Program
}

func (k uniformGroupKey) Parent() dependencygraph.StateKey { return nil }

type vertexAttribKey struct {
	vertexArray *VertexArray
	location    AttributeLocation
}

func (k vertexAttribKey) Parent() dependencygraph.StateKey {
	return vertexAttribGroupKey{k.vertexArray}
}

type vertexAttribGroupKey struct {
	vertexArray *VertexArray
}

func (k vertexAttribGroupKey) Parent() dependencygraph.StateKey { return nil }

type renderbufferDataKey struct {
	renderbuffer *Renderbuffer
}

func (k renderbufferDataKey) Parent() dependencygraph.StateKey { return nil }

type renderbufferSubDataKey struct {
	renderbuffer *Renderbuffer
	region       Rect
}

func (k renderbufferSubDataKey) Parent() dependencygraph.StateKey {
	return renderbufferDataKey{k.renderbuffer}
}

type textureDataKey struct {
	texture *Texture
	id      TextureId // For debugging, as 0 is not unique identifier.
}

func (k textureDataKey) Parent() dependencygraph.StateKey { return nil }

type textureSizeKey struct {
	texture *Texture
	id      TextureId // For debugging, as 0 is not unique identifier.
}

func (k textureSizeKey) Parent() dependencygraph.StateKey { return nil }

type eglImageDataKey struct {
	image *EGLImage
}

func (k eglImageDataKey) Parent() dependencygraph.StateKey { return nil }

type eglImageSizeKey struct {
	image *EGLImage
}

func (k eglImageSizeKey) Parent() dependencygraph.StateKey { return nil }

type GlesDependencyGraphBehaviourProvider struct {
}

func newGlesDependencyGraphBehaviourProvider() *GlesDependencyGraphBehaviourProvider {
	return &GlesDependencyGraphBehaviourProvider{}
}

// GetBehaviourForAtom returns state reads/writes that the given atom performs.
//
// Writes: Write dependencies keep atoms alive. Each atom must correctly report
// all its writes or it must set the keep-alive flag. If a write is missing
// then the liveness analysis will remove the atom since it seems unneeded.
// It is fine to omit related/overloaded commands that write to the same state
// as long as they are marked as keep-alive.
//
// Reads: For each state write, all commands that could possibly read it must be
// implemented. This makes it more difficult to do only partial implementations.
// It is fine to overestimate reads, or to read parent state (i.e. superset).
//
func (*GlesDependencyGraphBehaviourProvider) GetBehaviourForAtom(
	ctx context.Context, s *api.State, id api.CmdID, cmd api.Cmd, g *dependencygraph.DependencyGraph) dependencygraph.AtomBehaviour {
	b := dependencygraph.AtomBehaviour{}
	c := GetContext(s, cmd.Thread())
	if c != nil && c.Info.Initialized {
		_, isEglSwapBuffers := cmd.(*EglSwapBuffers)
		_, isEglSwapBuffersWithDamageKHR := cmd.(*EglSwapBuffersWithDamageKHR)
		if isEglSwapBuffers || isEglSwapBuffersWithDamageKHR {
			// Get default renderbuffers
			fb := c.Objects.Framebuffers[0]
			color := fb.ColorAttachments[0].Renderbuffer
			depth := fb.DepthAttachment.Renderbuffer
			stencil := fb.StencilAttachment.Renderbuffer
			if !c.Info.PreserveBuffersOnSwap {
				b.Write(g, renderbufferDataKey{color})
			}
			b.Write(g, renderbufferDataKey{depth})
			b.Write(g, renderbufferDataKey{stencil})
		} else if cmd.CmdFlags().IsDrawCall() {
			b.Read(g, uniformGroupKey{c.Bound.Program})
			b.Read(g, vertexAttribGroupKey{c.Bound.VertexArray})
			for _, stateKey := range getAllUsedTextureData(ctx, cmd, s, c) {
				b.Read(g, stateKey)
			}
			c.Bound.DrawFramebuffer.ForEachAttachment(func(name GLenum, att FramebufferAttachment) {
				data, size := att.dataAndSize(g, c)
				b.Read(g, size)
				b.Modify(g, data)
			})
			// TODO: Write transform feedback buffers.
		} else if cmd.CmdFlags().IsClear() {
			switch cmd := cmd.(type) {
			case *GlClearBufferfi:
				clearBuffer(g, &b, cmd.Buffer, cmd.Drawbuffer, c)
			case *GlClearBufferfv:
				clearBuffer(g, &b, cmd.Buffer, cmd.Drawbuffer, c)
			case *GlClearBufferiv:
				clearBuffer(g, &b, cmd.Buffer, cmd.Drawbuffer, c)
			case *GlClearBufferuiv:
				clearBuffer(g, &b, cmd.Buffer, cmd.Drawbuffer, c)
			case *GlClear:
				if (cmd.Mask & GLbitfield_GL_COLOR_BUFFER_BIT) != 0 {
					for i, _ := range c.Bound.DrawFramebuffer.ColorAttachments {
						clearBuffer(g, &b, GLenum_GL_COLOR, i, c)
					}
				}
				if (cmd.Mask & GLbitfield_GL_DEPTH_BUFFER_BIT) != 0 {
					clearBuffer(g, &b, GLenum_GL_DEPTH, 0, c)
				}
				if (cmd.Mask & GLbitfield_GL_STENCIL_BUFFER_BIT) != 0 {
					clearBuffer(g, &b, GLenum_GL_STENCIL, 0, c)
				}
			default:
				log.E(ctx, "Unknown clear command: %v", cmd)
			}
		} else {
			switch cmd := cmd.(type) {
			case *GlCopyImageSubData:
				// TODO: This assumes whole-image copy.  Handle sub-range copies.
				if cmd.SrcTarget == GLenum_GL_RENDERBUFFER {
					b.Read(g, renderbufferDataKey{c.Objects.Shared.Renderbuffers[RenderbufferId(cmd.SrcName)]})
				} else {
					data, size := c.Objects.Shared.Textures[TextureId(cmd.SrcName)].dataAndSize()
					b.Read(g, data)
					b.Read(g, size)
				}
				if cmd.DstTarget == GLenum_GL_RENDERBUFFER {
					b.Write(g,
						renderbufferDataKey{c.Objects.Shared.Renderbuffers[RenderbufferId(cmd.DstName)]})
				} else {
					data, size := c.Objects.Shared.Textures[TextureId(cmd.DstName)].dataAndSize()
					b.Write(g, data)
					b.Write(g, size)
				}
			case *GlFramebufferTexture2D:
				b.Read(g, textureSizeKey{c.Objects.Shared.Textures[cmd.Texture], cmd.Texture})
				b.KeepAlive = true // Changes untracked state
			case *GlCompressedTexImage2D:
				texData, texSize := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
				b.Write(g, texSize)
			case *GlCompressedTexSubImage2D:
				texData, _ := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
			case *GlTexImage2D:
				texData, texSize := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
				b.Write(g, texSize)
			case *GlTexSubImage2D:
				texData, _ := getTextureDataAndSize(ctx, cmd, s, c.Bound.TextureUnit, cmd.Target)
				b.Modify(g, texData)
			case *GlUniform1fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniform2fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniform3fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniform4fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlUniformMatrix4fv:
				b.Write(g, uniformKey{c.Bound.Program, cmd.Location, cmd.Count})
			case *GlVertexAttribPointer:
				b.Write(g, vertexAttribKey{c.Bound.VertexArray, cmd.Location})
			default:
				// Force all unhandled atoms to be kept alive.
				b.KeepAlive = true
			}
		}
	} else /* c == nil */ {
		b.KeepAlive = true
	}
	if err := cmd.Mutate(ctx, s, nil /* builder */); err != nil {
		log.W(ctx, "Command %v %v: %v", id, cmd, err)
		return dependencygraph.AtomBehaviour{Aborted: true}
	}
	return b
}

func clearBuffer(g *dependencygraph.DependencyGraph, b *dependencygraph.AtomBehaviour, buffer GLenum, index GLint, c *Context) {
	var data, size dependencygraph.StateKey
	switch buffer {
	case GLenum_GL_COLOR:
		data, size = c.Bound.DrawFramebuffer.ColorAttachments[index].dataAndSize(g, c)
	case GLenum_GL_DEPTH:
		data, size = c.Bound.DrawFramebuffer.DepthAttachment.dataAndSize(g, c)
	case GLenum_GL_STENCIL:
		data, size = c.Bound.DrawFramebuffer.StencilAttachment.dataAndSize(g, c)
	}
	b.Read(g, size)
	b.Write(g, data)
}

func getAllUsedTextureData(ctx context.Context, cmd api.Cmd, s *api.State, c *Context) (stateKeys []dependencygraph.StateKey) {
	// Look for samplers used by the current program.
	if prog := c.Bound.Program; prog != nil {
		for _, activeUniform := range prog.ActiveUniforms {
			// Optimization - skip the two most common types which we know are not samplers.
			if activeUniform.Type != GLenum_GL_FLOAT_VEC4 && activeUniform.Type != GLenum_GL_FLOAT_MAT4 {
				target, _ := subGetTextureTargetFromSamplerType(ctx, cmd, nil, s, GetState(s), cmd.Thread(), nil, activeUniform.Type)
				if target == GLenum_GL_NONE {
					continue // Not a sampler type
				}
				for i := 0; i < int(activeUniform.ArraySize); i++ {
					uniform := prog.Uniforms[activeUniform.Location+UniformLocation(i)]
					units := AsU32ˢ(uniform.Value, s.MemoryLayout).Read(ctx, cmd, s, nil)
					if len(units) == 0 {
						units = []uint32{0} // The uniform was not set, so use default value.
					}
					for _, unit := range units {
						if tu := c.Objects.TextureUnits[TextureUnitId(unit)]; tu != nil {
							texData, _ := getTextureDataAndSize(ctx, cmd, s, tu, target)
							stateKeys = append(stateKeys, texData)
						}
					}
				}
			}
		}
	}
	return
}

func getTextureDataAndSize(ctx context.Context, cmd api.Cmd, s *api.State, unit *TextureUnit, target GLenum) (dependencygraph.StateKey, dependencygraph.StateKey) {
	tex, err := subGetBoundTextureForUnit(ctx, cmd, nil, s, GetState(s), cmd.Thread(), nil, unit, target)
	if tex == nil || err != nil {
		log.E(ctx, "Can not find texture %v in unit %v", target, unit)
		return nil, nil
	}
	return tex.dataAndSize()
}

func (tex *Texture) dataAndSize() (dependencygraph.StateKey, dependencygraph.StateKey) {
	if tex.EGLImage != nil {
		return eglImageDataKey{tex.EGLImage}, eglImageSizeKey{tex.EGLImage}
	} else {
		return textureDataKey{tex, tex.ID}, textureSizeKey{tex, tex.ID}
	}
}

func (att FramebufferAttachment) dataAndSize(g *dependencygraph.DependencyGraph, c *Context) (dataKey dependencygraph.StateKey, sizeKey dependencygraph.StateKey) {
	if att.Type == GLenum_GL_RENDERBUFFER {
		rb := att.Renderbuffer
		if rb != nil && rb.InternalFormat != GLenum_GL_NONE {
			scissor := c.Pixel.Scissor
			fullBox := Rect{Width: rb.Width, Height: rb.Height}
			if scissor.Test == GLboolean_GL_TRUE && scissor.Box != fullBox {
				dataKey, sizeKey = renderbufferSubDataKey{rb, scissor.Box}, nil
			} else {
				dataKey, sizeKey = renderbufferDataKey{rb}, nil
			}
		}
	}
	if att.Type == GLenum_GL_TEXTURE {
		tex := att.Texture
		if tex != nil {
			// TODO: We should handle scissor here as well.
			dataKey, sizeKey = tex.dataAndSize()
		}
	}
	if dataKey != nil {
		g.SetRoot(dataKey)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
    // . "github.com/levythu/gurgling"
    . "logs"
    "config"
    "errors"
    "strconv"
    . "storage"
    . "storage/general"
)

type CDNServerConf struct {
    confName string

    srcHost string
    dstHost string
    extends map[string]interface{}

    storage StorageAPI
}

var confList []*CDNServerConf

func InitAllServerConf() error {
    var anonyCount=0
    confList=[]*CDNServerConf{}
    if list, ok:=config.Conf["servers"].([]interface{}); !ok {
        return errors.New("Incorrect tag 'servers'")
    } else {
        for _, oe:=range list {
            var e map[string]interface{}
            if e, ok=oe.(map[string]interface{}); !ok {
                continue
            }

            var newConf=&CDNServerConf{}
            var ok2 bool
            if newConf.confName, ok2=e["name"].(string); !ok2 {
                newConf.confName="Anonymous#"+strconv.Itoa(anonyCount)
                anonyCount++
            }
            if newConf.srcHost, ok2=e["srcHost"].(string); !ok2 {
                L.Warn(newConf.confName, "has incorrect tag 'srcHost'. Ignore it.")
                continue
            }
            if newConf.dstHost, ok2=e["dstHost"].(string); !ok2 {
                L.Warn(newConf.confName, "has incorrect tag 'dstHost'. Ignore it.")
                continue
            }
            if newConf.extends, ok2=e["extends"].(map[string]interface{}); !ok2 {
                newConf.extends=map[string]interface{}{}
            }

            if storageType, ok2:=e["storage"].(string); !ok2 {
                newConf.storage=GetStorageAPI("simple-mem", newConf.extends)
            } else {
                newConf.storage=GetStorageAPI(storageType, newConf.extends)
                if newConf.storage==nil {
                    L.Warn(newConf.confName, "has incorrect tag 'storage'. Ignore it.")
                    continue
                }
            }

            L.Log(newConf.confName, "has set up!")
            confList=append(confList, newConf)
        }
    }

    return nil
}
<commit_msg>refine<commit_after>package main

import (
    // . "github.com/levythu/gurgling"
    . "logs"
    "config"
    "errors"
    "strconv"
    . "storage"
    . "storage/general"
)

type CDNServerConf struct {
    confName string

    srcHost string
    dstHost string
    extends map[string]interface{}

    storage StorageAPI
}

var confList []*CDNServerConf

func InitAllServerConf() error {
    var anonyCount=0
    confList=[]*CDNServerConf{}
    if list, ok:=config.Conf["servers"].([]interface{}); !ok {
        return errors.New("Incorrect tag 'servers'")
    } else {
        for _, oe:=range list {
            var e map[string]interface{}
            if e, ok=oe.(map[string]interface{}); !ok {
                continue
            }

            var newConf=&CDNServerConf{}
            var ok2 bool
            if newConf.confName, ok2=e["name"].(string); !ok2 {
                newConf.confName="Anonymous#"+strconv.Itoa(anonyCount)
                anonyCount++
            }
            if newConf.srcHost, ok2=e["srcHost"].(string); !ok2 {
                L.Warn(newConf.confName, "has incorrect tag 'srcHost'. Ignore it.")
                continue
            }
            if newConf.dstHost, ok2=e["dstHost"].(string); !ok2 {
                L.Warn(newConf.confName, "has incorrect tag 'dstHost'. Ignore it.")
                continue
            }
            if newConf.extends, ok2=e["extends"].(map[string]interface{}); !ok2 {
                newConf.extends=map[string]interface{}{}
            }

            if storageType, ok2:=e["storage"].(string); !ok2 {
                newConf.storage=GetStorageAPI("simple-mem", newConf.extends)
            } else {
                newConf.storage=GetStorageAPI(storageType, newConf.extends)
                if newConf.storage==nil {
                    L.Warn(newConf.confName, "has incorrect tag 'storage'. Ignore it.")
                    continue
                }
            }

            L.Log("<"+newConf.confName+">", "has set up!")
            confList=append(confList, newConf)
        }
    }

    return nil
}
<|endoftext|>


<commit_before>package eth

import (
	"bytes"
	"container/list"
	"fmt"
	"math"
	"math/big"
	"sync"
	"time"

	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
)

type block struct {
	peer      *Peer
	block     *ethchain.Block
	reqAt     time.Time
	requested int
}

type BlockPool struct {
	mut sync.Mutex

	eth *Ethereum

	hashPool [][]byte
	pool     map[string]*block

	td   *big.Int
	quit chan bool
}

func NewBlockPool(eth *Ethereum) *BlockPool {
	return &BlockPool{
		eth:  eth,
		pool: make(map[string]*block),
		td:   ethutil.Big0,
		quit: make(chan bool),
	}
}

func (self *BlockPool) Len() int {
	return len(self.hashPool)
}

func (self *BlockPool) HasLatestHash() bool {
	return self.pool[string(self.eth.BlockChain().CurrentBlock.Hash())] != nil
}

func (self *BlockPool) HasCommonHash(hash []byte) bool {
	return self.eth.BlockChain().GetBlock(hash) != nil
}

func (self *BlockPool) AddHash(hash []byte) {
	if self.pool[string(hash)] == nil {
		self.pool[string(hash)] = &block{nil, nil, time.Now(), 0}

		self.hashPool = append([][]byte{hash}, self.hashPool...)
	}
}

func (self *BlockPool) SetBlock(b *ethchain.Block, peer *Peer) {
	hash := string(b.Hash())

	if self.pool[hash] == nil && !self.eth.BlockChain().HasBlock(b.Hash()) {
		self.hashPool = append(self.hashPool, b.Hash())
		self.pool[hash] = &block{peer, b, time.Now(), 0}
	} else if self.pool[hash] != nil {
		self.pool[hash].block = b
	}
}

func (self *BlockPool) getParent(block *ethchain.Block) *ethchain.Block {
	for _, item := range self.pool {
		if item.block != nil {
			if bytes.Compare(item.block.Hash(), block.PrevHash) == 0 {
				return item.block
			}
		}
	}

	return nil
}

func (self *BlockPool) GetChainFromBlock(block *ethchain.Block) ethchain.Blocks {
	var blocks ethchain.Blocks

	for b := block; b != nil; b = self.getParent(b) {
		blocks = append(ethchain.Blocks{b}, blocks...)
	}

	return blocks
}

func (self *BlockPool) CheckLinkAndProcess(f func(block *ethchain.Block)) {

	var blocks ethchain.Blocks
	for _, item := range self.pool {
		if item.block != nil {
			blocks = append(blocks, item.block)
		}
	}

	ethchain.BlockBy(ethchain.Number).Sort(blocks)
	for _, block := range blocks {
		if self.eth.BlockChain().HasBlock(block.PrevHash) {
			f(block)

			hash := block.Hash()
			self.hashPool = ethutil.DeleteFromByteSlice(self.hashPool, hash)
			delete(self.pool, string(hash))
		}

	}
}

func (self *BlockPool) Take(amount int, peer *Peer) (hashes [][]byte) {
	self.mut.Lock()
	defer self.mut.Unlock()

	num := int(math.Min(float64(amount), float64(len(self.pool))))
	j := 0
	for i := 0; i < len(self.hashPool) && j < num; i++ {
		hash := string(self.hashPool[i])
		item := self.pool[hash]
		if item != nil && item.block == nil &&
			(item.peer == nil ||
				((time.Since(item.reqAt) > 5*time.Second && item.peer != peer) && self.eth.peers.Len() > 1) || // multiple peers
				(time.Since(item.reqAt) > 5*time.Second && self.eth.peers.Len() == 1) /* single peer*/) {
			self.pool[hash].peer = peer
			self.pool[hash].reqAt = time.Now()
			self.pool[hash].requested++

			hashes = append(hashes, self.hashPool[i])
			j++
		}
	}

	return
}

func (self *BlockPool) Start() {
	go self.update()
}

func (self *BlockPool) Stop() {
	close(self.quit)
}

func (self *BlockPool) update() {
	serviceTimer := time.NewTicker(100 * time.Millisecond)
	procTimer := time.NewTicker(500 * time.Millisecond)
out:
	for {
		select {
		case <-self.quit:
			break out
		case <-serviceTimer.C:
			// Clean up hashes that can't be fetched
			done := true
			eachPeer(self.eth.peers, func(p *Peer, v *list.Element) {
				if p.statusKnown && p.FetchingHashes() {
					done = false
				}
			})

			if done {
				eachPeer(self.eth.peers, func(p *Peer, v *list.Element) {
					if p.statusKnown {
						hashes := self.Take(100, p)
						if len(hashes) > 0 {
							p.FetchBlocks(hashes)
							if len(hashes) == 1 {
								fmt.Printf("last hash = %x\n", hashes[0])
							} else {
								fmt.Println("Requesting", len(hashes), "of", p)
							}
						}
					}
				})
			}
		case <-procTimer.C:
			var err error
			self.CheckLinkAndProcess(func(block *ethchain.Block) {
				err = self.eth.StateManager().Process(block, false)
			})

			if err != nil {
				peerlogger.Infoln(err)
			}
		}
	}
}
<commit_msg>Changed the block fetching code and hash distribution<commit_after>package eth

import (
	"bytes"
	"container/list"
	"math"
	"math/big"
	"sync"
	"time"

	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethlog"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/eth-go/ethwire"
)

var poollogger = ethlog.NewLogger("[BPOOL]")

type block struct {
	from      *Peer
	peer      *Peer
	block     *ethchain.Block
	reqAt     time.Time
	requested int
}

type BlockPool struct {
	mut sync.Mutex

	eth *Ethereum

	hashPool [][]byte
	pool     map[string]*block

	td   *big.Int
	quit chan bool

	ChainLength, BlocksProcessed int
}

func NewBlockPool(eth *Ethereum) *BlockPool {
	return &BlockPool{
		eth:  eth,
		pool: make(map[string]*block),
		td:   ethutil.Big0,
		quit: make(chan bool),
	}
}

func (self *BlockPool) Len() int {
	return len(self.hashPool)
}

func (self *BlockPool) HasLatestHash() bool {
	return self.pool[string(self.eth.BlockChain().CurrentBlock.Hash())] != nil
}

func (self *BlockPool) HasCommonHash(hash []byte) bool {
	return self.eth.BlockChain().GetBlock(hash) != nil
}

func (self *BlockPool) AddHash(hash []byte, peer *Peer) {
	if self.pool[string(hash)] == nil {
		self.pool[string(hash)] = &block{peer, nil, nil, time.Now(), 0}

		self.hashPool = append([][]byte{hash}, self.hashPool...)
	}
}

func (self *BlockPool) SetBlock(b *ethchain.Block, peer *Peer) {
	hash := string(b.Hash())

	if self.pool[hash] == nil && !self.eth.BlockChain().HasBlock(b.Hash()) {
		self.hashPool = append(self.hashPool, b.Hash())
		self.pool[hash] = &block{peer, peer, b, time.Now(), 0}
	} else if self.pool[hash] != nil {
		self.pool[hash].block = b
	}

	self.BlocksProcessed++
}

func (self *BlockPool) getParent(block *ethchain.Block) *ethchain.Block {
	for _, item := range self.pool {
		if item.block != nil {
			if bytes.Compare(item.block.Hash(), block.PrevHash) == 0 {
				return item.block
			}
		}
	}

	return nil
}

func (self *BlockPool) GetChainFromBlock(block *ethchain.Block) ethchain.Blocks {
	var blocks ethchain.Blocks

	for b := block; b != nil; b = self.getParent(b) {
		blocks = append(ethchain.Blocks{b}, blocks...)
	}

	return blocks
}

func (self *BlockPool) Blocks() (blocks ethchain.Blocks) {
	for _, item := range self.pool {
		if item.block != nil {
			blocks = append(blocks, item.block)
		}
	}

	return
}

func (self *BlockPool) ProcessCanonical(f func(block *ethchain.Block)) (procAmount int) {
	blocks := self.Blocks()

	ethchain.BlockBy(ethchain.Number).Sort(blocks)
	for _, block := range blocks {
		if self.eth.BlockChain().HasBlock(block.PrevHash) {
			procAmount++

			f(block)

			hash := block.Hash()
			self.hashPool = ethutil.DeleteFromByteSlice(self.hashPool, hash)
			delete(self.pool, string(hash))
		}

	}

	return
}

func (self *BlockPool) DistributeHashes() {
	var (
		peerLen = self.eth.peers.Len()
		amount  = 200 * peerLen
		dist    = make(map[*Peer][][]byte)
	)

	num := int(math.Min(float64(amount), float64(len(self.pool))))
	for i, j := 0, 0; i < len(self.hashPool) && j < num; i++ {
		hash := self.hashPool[i]
		item := self.pool[string(hash)]

		if item != nil && item.block == nil {
			var peer *Peer
			lastFetchFailed := time.Since(item.reqAt) > 5*time.Second

			// Handle failed requests
			if lastFetchFailed && item.requested > 0 && item.peer != nil {
				if item.requested < 100 {
					// Select peer the hash was retrieved off
					peer = item.from
				} else {
					// Remove it
					self.hashPool = ethutil.DeleteFromByteSlice(self.hashPool, hash)
					delete(self.pool, string(hash))
				}
			} else if lastFetchFailed || item.peer == nil {
				// Find a suitable, available peer
				eachPeer(self.eth.peers, func(p *Peer, v *list.Element) {
					if peer == nil && len(dist[p]) < amount/peerLen {
						peer = p
					}
				})
			}

			if peer != nil {
				item.reqAt = time.Now()
				item.peer = peer
				item.requested++

				dist[peer] = append(dist[peer], hash)
			}
		}
	}

	for peer, hashes := range dist {
		peer.FetchBlocks(hashes)
	}
}

func (self *BlockPool) Start() {
	go self.update()
}

func (self *BlockPool) Stop() {
	close(self.quit)
}

func (self *BlockPool) update() {
	serviceTimer := time.NewTicker(100 * time.Millisecond)
	procTimer := time.NewTicker(500 * time.Millisecond)
out:
	for {
		select {
		case <-self.quit:
			break out
		case <-serviceTimer.C:
			// Check if we're catching up. If not distribute the hashes to
			// the peers and download the blockchain
			done := true
			eachPeer(self.eth.peers, func(p *Peer, v *list.Element) {
				if p.statusKnown && p.FetchingHashes() {
					done = false
				}
			})

			if done && len(self.hashPool) > 0 {
				self.DistributeHashes()
			}

			if self.ChainLength < len(self.hashPool) {
				self.ChainLength = len(self.hashPool)
			}
		case <-procTimer.C:
			// XXX We can optimize this lifting this on to a new goroutine.
			// We'd need to make sure that the pools are properly protected by a mutex
			amount := self.ProcessCanonical(func(block *ethchain.Block) {
				err := self.eth.StateManager().Process(block, false)
				if err != nil {
					poollogger.Infoln(err)
				}
			})

			// Do not propagate to the network on catchups
			if amount == 1 {
				block := self.eth.BlockChain().CurrentBlock
				self.eth.Broadcast(ethwire.MsgBlockTy, []interface{}{block.Value().Val})
			}
		}
	}
}
<|endoftext|>


<commit_before>package aeunittest

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"

	//"fmt"

	gorillacontext "github.com/gorilla/context"
	"golang.org/x/net/context"
)

type (
	TestCase struct {
		Name             string
		RequestBody      string
		HttpVerb         string
		Uri              string
		WantStatusCode   int
		WantResponseBody string
		Skip             bool
		//
		*testing.T
		context.Context
		http.Handler
	}

	TestCases []TestCase
)

func (tc TestCase) Run1() (ResponseBody []byte) { // compare response code only and return response body received

	if tc.Skip {
		tc.Log("Skipped test case:", tc.Name)
		return
	}

	// prepare request
	body := strings.NewReader(tc.RequestBody)
	req, err := http.NewRequest(tc.HttpVerb, tc.Uri, body) //inst.NewRequest("GET", goalUrl, body) //
	if err != nil {
		tc.Error(err)
	}

	gorillacontext.Set(req, "Context", tc.Context)

	// prepare response writer
	record := httptest.NewRecorder()

	// make the request
	tc.ServeHTTP(record, req)

	//tc.Log(record.Body)

	if got := record.Code; tc.WantStatusCode != got {
		tc.Error(tc.Name, ": Status Code: wanted ", tc.WantStatusCode, " but got ", got)
	}

	return record.Body.Bytes()

}

func (tc TestCase) Run() {

	if tc.Skip {
		tc.Log("Skipped test case:", tc.Name)
		return
	}

	g1 := tc.Run1()
	var got interface{}

	if err := json.Unmarshal(g1, &got); err != nil {
		tc.Error("Want Response Body invalid format: ", err.Error())
	}

	var want interface{}

	if err := json.Unmarshal([]byte(tc.WantResponseBody), &want); err != nil {
		tc.Error("Want Response Body invalid format: ", err.Error())
	}

	//tc.Log("Want:", want)
	//tc.Log("Got: ", got)
	//tc.Log("DeepEqual:", reflect.DeepEqual(got, want))

	if !reflect.DeepEqual(got, want) {
		tc.Error(tc.Name, ": Response Body : wanted ", tc.WantResponseBody, " but got ", string(g1))
	}

}

// Load method loads the given test cases data from the flat file into the TestCases slice object.
// if header is provided, will parse the data accordign the header, otherwise, the default order will be used.
func (tcs *TestCases) Load(filePath string, delim rune, hasHeader bool) error {

	file, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	r := csv.NewReader(file)
	r.Comma = delim
	r.Comment = '#'

	records, err := r.ReadAll()
	if err != nil {
		return err
	}

	var name, requestBody, httpVerb, uri, wantStatusCode, wantResponseBody, skip int

	for i, row := range records {

		if i == 0 && hasHeader == true {
			for j, c := range row {
				//find the position of each field
				//fmt.Println("###", j, c)
				switch c {
				case "Name":
					name = j
				case "RequestBody":
					requestBody = j
				case "HttpVerb":
					httpVerb = j
				case "Uri":
					uri = j
				case "WantStatusCode":
					wantStatusCode = j
				case "WantResponseBody":
					wantResponseBody = j
				case "Skip":
					skip = j
				}
			}
			continue

		} else if i == 0 {
			name = 0
			requestBody = 1
			httpVerb = 2
			uri = 3
			wantStatusCode = 4
			wantResponseBody = 5
			skip = 6
		}

		if len(row[name])+len(row[requestBody])+len(row[httpVerb])+len(row[uri])+len(row[wantStatusCode])+
			len(row[wantResponseBody])+len(row[skip]) == 0 {
			continue // if all the fields are blank, then skip
		} else if len(row[name]) == 0 || len(row[httpVerb]) == 0 || len(row[uri]) == 0 || len(row[wantStatusCode]) == 0 {
			return errors.New("Missing manditory information in row " + strconv.Itoa(i) + " (" + row[name] + ")") // if only manditory fields are blank, then error out
		}

		tc := TestCase{}
		tc.Name = row[name]
		tc.RequestBody = row[requestBody]
		tc.HttpVerb = row[httpVerb]
		tc.Uri = row[uri]
		tc.WantResponseBody = row[wantResponseBody]
		switch strings.ToUpper(row[skip]) {
		case "YES", "TRUE", "1":
			tc.Skip = true
		case "NO", "FALSE", "0":
			tc.Skip = false
		}

		statusCode, err := strconv.Atoi(row[wantStatusCode][0:3]) // take the first 3 digits of the status code. skip the rest, the information that user might put like in '200 OK' or '400 Bad Request'
		if err != nil {
			return err
		}
		tc.WantStatusCode = statusCode
		*tcs = append(*tcs, tc)
	}

	return nil
}
<commit_msg>Code cleanup: - code cleanup - standardization of names - replacing the Go1.7 specific testing package method names with the names specific to this library, to avoid conflict when used with Go 1.7<commit_after>package aeunittest

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"

	//"fmt"

	gorillacontext "github.com/gorilla/context"
	"golang.org/x/net/context"
)

type (
	TestCase struct {
		Name             string
		RequestBody      string
		HttpVerb         string
		Uri              string
		WantStatusCode   int
		WantResponseBody string
		SkipFlag         bool
		//
		*testing.T
		context.Context
		http.Handler
	}

	TestCases []TestCase
)

func (tc TestCase) RunCheckStatusCode() (ResponseBody []byte) { // compare response status code only and return response body received
	// prepare response writer
	record := httptest.NewRecorder()

	//tc.Run(tc.Name, func(t *testing.T) { //Note: removed the code related to Subtests as it looks like Appengine SDK is not yet using Go 1.7 as of 2016 Dec.
	if tc.SkipFlag {
		tc.Log("Skipped test case:", tc.Name)
		return
	}

	// prepare request
	body := strings.NewReader(tc.RequestBody)
	req, err := http.NewRequest(tc.HttpVerb, tc.Uri, body)
	if err != nil {
		tc.Error(err)
	}

	gorillacontext.Set(req, "Context", tc.Context)

	// make the request
	tc.ServeHTTP(record, req)

	//tc.Log(record.Body)

	if got := record.Code; tc.WantStatusCode != got {
		tc.Error(tc.Name, ": Status Code: wanted ", tc.WantStatusCode, " but got ", got)
	}
	//})
	return record.Body.Bytes()
}

func (tc TestCase) RunCase() {
	//tc.Run(tc.Name, func(t *testing.T) {
	if tc.SkipFlag {
		tc.Log("Skipped test case:", tc.Name)
		//return
	}

	// execute test case to check the status code and capture the response body
	gotResponseBody := tc.RunCheckStatusCode()

	// compare the 'got' with 'want', and report if not matching

	var got interface{}
	if err := json.Unmarshal(gotResponseBody, &got); err != nil {
		tc.Error(tc.Name, ": Got Response Body invalid format: \n", string(gotResponseBody), "\n", err.Error())
	}

	var want interface{}
	if err := json.Unmarshal([]byte(tc.WantResponseBody), &want); err != nil {
		tc.Error(tc.Name, ": Want Response Body invalid format: \n", tc.WantResponseBody, "\n", err.Error())
	}

	if !reflect.DeepEqual(got, want) {
		tc.Error(tc.Name, ": Response Body : wanted ", tc.WantResponseBody, " but got ", string(gotResponseBody))
	}
	//})
}

// Load method loads the given test cases data from the flat file into the TestCases slice object.
// if header is provided, will parse the data accordign the header, otherwise, the default order will be used.
func (tcs *TestCases) Load(filePath string, delim rune, hasHeader bool) error {

	file, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer file.Close()

	r := csv.NewReader(file)
	r.Comma = delim
	r.Comment = '#'

	records, err := r.ReadAll()
	if err != nil {
		return err
	}

	var name, requestBody, httpVerb, uri, wantStatusCode, wantResponseBody, skip int

	for i, row := range records {

		if i == 0 && hasHeader == true {
			for j, c := range row {
				//find the position of each field
				//fmt.Println("###", j, c)
				switch c {
				case "Name":
					name = j
				case "RequestBody":
					requestBody = j
				case "HttpVerb":
					httpVerb = j
				case "Uri":
					uri = j
				case "WantStatusCode":
					wantStatusCode = j
				case "WantResponseBody":
					wantResponseBody = j
				case "Skip":
					skip = j
				}
			}
			continue

		} else if i == 0 {
			name = 0
			requestBody = 1
			httpVerb = 2
			uri = 3
			wantStatusCode = 4
			wantResponseBody = 5
			skip = 6
		}

		if len(row[name])+len(row[requestBody])+len(row[httpVerb])+len(row[uri])+len(row[wantStatusCode])+
			len(row[wantResponseBody])+len(row[skip]) == 0 {
			continue // if all the fields are blank, then skip
		} else if len(row[name]) == 0 || len(row[httpVerb]) == 0 || len(row[uri]) == 0 || len(row[wantStatusCode]) == 0 {
			return errors.New("Missing manditory information in row " + strconv.Itoa(i) + " (" + row[name] + ")") // if only manditory fields are blank, then error out
		}

		tc := TestCase{}
		tc.Name = row[name]
		tc.RequestBody = row[requestBody]
		tc.HttpVerb = row[httpVerb]
		tc.Uri = row[uri]
		tc.WantResponseBody = row[wantResponseBody]
		switch strings.ToUpper(row[skip]) {
		case "YES", "TRUE", "1":
			tc.SkipFlag = true
		case "NO", "FALSE", "0":
			tc.SkipFlag = false
		}

		statusCode, err := strconv.Atoi(row[wantStatusCode][0:3]) // take the first 3 digits of the status code. skip the rest, the information that user might put like in '200 OK' or '400 Bad Request'
		if err != nil {
			return err
		}
		tc.WantStatusCode = statusCode
		*tcs = append(*tcs, tc)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"server"
)

func main() {

	var port int

	flag.IntVar(&port, "Port", 8080, "Port the server listens to")

	flag.Parse()

	game.Open()
	defer game.Close()
	server.Run(port)

}
<commit_msg>removed game references<commit_after>package main

import (
	"flag"
	"server"
)

func main() {

	var port int

	flag.IntVar(&port, "Port", 8080, "Port the server listens to")

	flag.Parse()
	server.Run(port)

}
<|endoftext|>


<commit_before>package common

import (
	"strings"
	"time"
)

func parseTime(formatted string) (time.Time, error) {
	var layouts = []string{
		"Mon, _2 Jan 2006 15:04:05 MST",
		"Mon, _2 Jan 2006 15:04:05 -0700",
		time.ANSIC,
		time.UnixDate,
		time.RubyDate,
		time.RFC822,
		time.RFC822Z,
		time.RFC850,
		time.RFC1123,
		time.RFC1123Z,
		time.RFC3339,
		time.RFC3339Nano,
		"2006-01-02T15:04:05-0700",
		"Mon, 2, Jan 2006 15:4",
		"02 Jan 2006 15:04:05 MST",
	}

	var t time.Time
	var err error
	formatted = strings.TrimSpace(formatted)

	for _, layout := range layouts {
		t, err = time.Parse(layout, formatted)
		if !t.IsZero() {
			break
		}
	}
	return t, err
}
<commit_msg>More time formats, without seconds<commit_after>package common

import (
	"strings"
	"time"
)

func parseTime(formatted string) (time.Time, error) {
	var layouts = []string{
		"Mon, _2 Jan 2006 15:04:05 MST",
		"Mon, _2 Jan 2006 15:04:05 -0700",
		time.ANSIC,
		time.UnixDate,
		time.RubyDate,
		time.RFC822,
		time.RFC822Z,
		time.RFC850,
		time.RFC1123,
		time.RFC1123Z,
		time.RFC3339,
		time.RFC3339Nano,
		"2006-01-02T15:04:05-0700",
		"2006-01-02T15:04-0700",
		"Mon, 2, Jan 2006 15:4",
		"02 Jan 2006 15:04:05 MST",
		"2006-01-02 15:04:05-0700",
		"2006-01-02 15:04-0700",
	}

	var t time.Time
	var err error
	formatted = strings.TrimSpace(formatted)

	for _, layout := range layouts {
		t, err = time.Parse(layout, formatted)
		if !t.IsZero() {
			break
		}
	}
	return t, err
}
<|endoftext|>


<commit_before>package upload

import (
	"bytes"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/chadweimer/gomp/modules/conf"
	"github.com/julienschmidt/httprouter"
)

// S3Driver is an implementation of Driver that uses the Amazon S3.
type S3Driver struct {
	cfg *conf.Config
}

// NewS3Driver constucts a S3Driver.
func NewS3Driver(cfg *conf.Config) S3Driver {
	return S3Driver{cfg: cfg}
}

// Save creates or overrites a file with the provided binary data.
func (u S3Driver) Save(filePath string, data []byte) error {
	svc := s3.New(session.New())

	bucket := u.cfg.UploadPath
	key := filepath.ToSlash(filePath)
	contentType := http.DetectContentType(data)
	_, err := svc.PutObject(&s3.PutObjectInput{
		Body:        bytes.NewReader(data),
		ContentType: &contentType,
		Bucket:      &bucket,
		Key:         &key,
	})
	return err
}

// Delete deletes the file with the specified key, if it exists.
func (u S3Driver) Delete(key string) error {
	svc := s3.New(session.New())

	bucket := u.cfg.UploadPath
	key = filepath.ToSlash(key)

	_, err := svc.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &bucket,
		Key:    &key,
	})
	return err
}

// DeleteAll deletes all files with the specified key prefix.
func (u S3Driver) DeleteAll(keyPrefix string) error {
	svc := s3.New(session.New())

	bucket := u.cfg.UploadPath
	keyPrefix = filepath.ToSlash(keyPrefix)

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &keyPrefix,
	})
	if err != nil {
		return err
	}
	for _, object := range listOutput.Contents {
		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
			Bucket: &bucket,
			Key:    object.Key,
		})
		if err != nil {
			return err
		}
	}

	return nil
}

// List retrieves information about all uploaded files with the specified key prefix.
func (u S3Driver) List(keyPrefix string) ([]FileInfo, error) {
	svc := s3.New(session.New())

	var fileInfos []FileInfo
	bucket := u.cfg.UploadPath
	prefix := filepath.ToSlash(filepath.Join(keyPrefix, "images"))

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &prefix,
	})
	if err != nil {
		return fileInfos, err
	}

	for _, object := range listOutput.Contents {
		urlSegments := strings.Split(*object.Key, "/")
		name := urlSegments[len(urlSegments)-1]
		origURL := "/uploads/" + *object.Key

		thumbKey := strings.Replace(*object.Key, "/images/", "/thumbs/", 1)
		thumbURL := "/uploads/" + thumbKey

		fileInfo := FileInfo{
			Name:         name,
			URL:          origURL,
			ThumbnailURL: thumbURL,
		}
		fileInfos = append(fileInfos, fileInfo)
	}

	return fileInfos, nil
}

// HandleS3Uploads returns a handler that serves static files from Amazon S3.
// If the file does not exist on the S3, a 404 is returned.
func HandleS3Uploads(bucket string) httprouter.Handle {
	return func(resp http.ResponseWriter, req *http.Request, p httprouter.Params) {
		filePath := p.ByName("filepath")
		// if the path is '/', move along because we'll just get bucket information
		if filePath == "/" {
			http.NotFound(resp, req)
			return
		}

		svc := s3.New(session.New())

		// Build the GetObject request, passing along conditional GET parameters
		getReq := s3.GetObjectInput{
			Bucket: &bucket,
			Key:    &filePath,
		}
		passThroughReqHeaders(&getReq, req.Header)

		// Make the request and check the error
		getResp, err := svc.GetObject(&getReq)
		if getResp != nil && getResp.Body != nil {
			defer getResp.Body.Close()
		}
		if err != nil {
			log.Print(err.Error())
			if reqerr, ok := err.(awserr.RequestFailure); ok {
				if reqerr.StatusCode() == http.StatusNotFound {
					http.Error(resp, reqerr.Message(), http.StatusNotFound)
					return
				}
				if reqerr.StatusCode() != http.StatusNotModified {
					http.Error(resp, reqerr.Message(), http.StatusInternalServerError)
					return
				}
			}
		}

		var buf []byte
		// If we got content, read it, including associated headers
		if getResp.ContentLength != nil && *getResp.ContentLength > 0 {
			buf, err = ioutil.ReadAll(getResp.Body)
			if err != nil {
				http.Error(resp, err.Error(), http.StatusInternalServerError)
				return
			}
		}
		passThroughRespHeaders(getResp, resp.Header())

		// Serve up the file to the client
		http.ServeContent(resp, req, filePath, *getResp.LastModified, bytes.NewReader(buf))
	}
}

func passThroughReqHeaders(getReq *s3.GetObjectInput, reqHeader http.Header) {
	if ifModSince, err := time.Parse(http.TimeFormat, reqHeader.Get("If-Modified-Since")); err == nil {
		getReq.IfModifiedSince = &ifModSince
	}
	if ifNoneMatch := reqHeader.Get("If-None-Match"); ifNoneMatch != "" {
		getReq.IfNoneMatch = &ifNoneMatch
	}
}

func passThroughRespHeaders(getResp *s3.GetObjectOutput, respHeader http.Header) {
	if getResp.ContentType != nil && *getResp.ContentType != "" {
		respHeader.Set("Content-Type", *getResp.ContentType)
	}
	if getResp.ContentEncoding != nil && *getResp.ContentEncoding != "" {
		respHeader.Set("Content-Encoding", *getResp.ContentEncoding)
	}
	if getResp.ETag != nil && *getResp.ETag != "" {
		respHeader.Set("ETag", *getResp.ETag)
	}
}
<commit_msg>Fake a ReadSeeker for s3, which should lower memory needs<commit_after>package upload

import (
	"bytes"
	"io"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/chadweimer/gomp/modules/conf"
	"github.com/julienschmidt/httprouter"
)

// S3Driver is an implementation of Driver that uses the Amazon S3.
type S3Driver struct {
	cfg *conf.Config
}

type readSizer struct {
	io.Reader
	size int64
}

func (r readSizer) Seek(offset int64, whence int) (int64, error) {
	if whence == io.SeekEnd {
		return r.size, nil
	}

	return 0, nil
}

// NewS3Driver constucts a S3Driver.
func NewS3Driver(cfg *conf.Config) S3Driver {
	return S3Driver{cfg: cfg}
}

// Save creates or overrites a file with the provided binary data.
func (u S3Driver) Save(filePath string, data []byte) error {
	svc := s3.New(session.New())

	bucket := u.cfg.UploadPath
	key := filepath.ToSlash(filePath)
	contentType := http.DetectContentType(data)
	_, err := svc.PutObject(&s3.PutObjectInput{
		Body:        bytes.NewReader(data),
		ContentType: &contentType,
		Bucket:      &bucket,
		Key:         &key,
	})
	return err
}

// Delete deletes the file with the specified key, if it exists.
func (u S3Driver) Delete(key string) error {
	svc := s3.New(session.New())

	bucket := u.cfg.UploadPath
	key = filepath.ToSlash(key)

	_, err := svc.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &bucket,
		Key:    &key,
	})
	return err
}

// DeleteAll deletes all files with the specified key prefix.
func (u S3Driver) DeleteAll(keyPrefix string) error {
	svc := s3.New(session.New())

	bucket := u.cfg.UploadPath
	keyPrefix = filepath.ToSlash(keyPrefix)

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &keyPrefix,
	})
	if err != nil {
		return err
	}
	for _, object := range listOutput.Contents {
		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
			Bucket: &bucket,
			Key:    object.Key,
		})
		if err != nil {
			return err
		}
	}

	return nil
}

// List retrieves information about all uploaded files with the specified key prefix.
func (u S3Driver) List(keyPrefix string) ([]FileInfo, error) {
	svc := s3.New(session.New())

	var fileInfos []FileInfo
	bucket := u.cfg.UploadPath
	prefix := filepath.ToSlash(filepath.Join(keyPrefix, "images"))

	listOutput, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
		Bucket: &bucket,
		Prefix: &prefix,
	})
	if err != nil {
		return fileInfos, err
	}

	for _, object := range listOutput.Contents {
		urlSegments := strings.Split(*object.Key, "/")
		name := urlSegments[len(urlSegments)-1]
		origURL := "/uploads/" + *object.Key

		thumbKey := strings.Replace(*object.Key, "/images/", "/thumbs/", 1)
		thumbURL := "/uploads/" + thumbKey

		fileInfo := FileInfo{
			Name:         name,
			URL:          origURL,
			ThumbnailURL: thumbURL,
		}
		fileInfos = append(fileInfos, fileInfo)
	}

	return fileInfos, nil
}

// HandleS3Uploads returns a handler that serves static files from Amazon S3.
// If the file does not exist on the S3, a 404 is returned.
func HandleS3Uploads(bucket string) httprouter.Handle {
	return func(resp http.ResponseWriter, req *http.Request, p httprouter.Params) {
		filePath := p.ByName("filepath")
		// if the path is '/', move along because we'll just get bucket information
		if filePath == "/" {
			http.NotFound(resp, req)
			return
		}

		svc := s3.New(session.New())

		// Build the GetObject request, passing along conditional GET parameters
		getReq := s3.GetObjectInput{
			Bucket: &bucket,
			Key:    &filePath,
		}
		passThroughReqHeaders(&getReq, req.Header)

		// Make the request and check the error
		getResp, err := svc.GetObject(&getReq)
		if getResp != nil && getResp.Body != nil {
			defer getResp.Body.Close()
		}
		if err != nil {
			log.Print(err.Error())
			if reqerr, ok := err.(awserr.RequestFailure); ok {
				if reqerr.StatusCode() == http.StatusNotFound {
					http.Error(resp, reqerr.Message(), http.StatusNotFound)
					return
				}
				if reqerr.StatusCode() != http.StatusNotModified {
					http.Error(resp, reqerr.Message(), http.StatusInternalServerError)
					return
				}
			}
		}

		var readSeeker io.ReadSeeker
		// If we got content, read it, including associated headers
		if getResp.ContentLength != nil && *getResp.ContentLength > 0 {
			readSeeker = readSizer{getResp.Body, *getResp.ContentLength}
		} else {
			readSeeker = readSizer{getResp.Body, 0}
		}
		passThroughRespHeaders(getResp, resp.Header())

		// Serve up the file to the client
		http.ServeContent(resp, req, filePath, *getResp.LastModified, readSeeker)
	}
}

func passThroughReqHeaders(getReq *s3.GetObjectInput, reqHeader http.Header) {
	if ifModSince, err := time.Parse(http.TimeFormat, reqHeader.Get("If-Modified-Since")); err == nil {
		getReq.IfModifiedSince = &ifModSince
	}
	if ifNoneMatch := reqHeader.Get("If-None-Match"); ifNoneMatch != "" {
		getReq.IfNoneMatch = &ifNoneMatch
	}
}

func passThroughRespHeaders(getResp *s3.GetObjectOutput, respHeader http.Header) {
	if getResp.ContentType != nil && *getResp.ContentType != "" {
		respHeader.Set("Content-Type", *getResp.ContentType)
	}
	if getResp.ContentEncoding != nil && *getResp.ContentEncoding != "" {
		respHeader.Set("Content-Encoding", *getResp.ContentEncoding)
	}
	if getResp.ETag != nil && *getResp.ETag != "" {
		respHeader.Set("ETag", *getResp.ETag)
	}
}
<|endoftext|>


<commit_before>package bloom

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"testing"
)

func TestBasic(t *testing.T) {
	f := New(1000, 4)
	n1 := []byte("Bess")
	n2 := []byte("Jane")
	n3 := []byte("Emma")
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestBasicUint32(t *testing.T) {
	f := New(1000, 4)
	n1 := make([]byte, 4)
	n2 := make([]byte, 4)
	n3 := make([]byte, 4)
	n4 := make([]byte, 4)
	binary.BigEndian.PutUint32(n1, 100)
	binary.BigEndian.PutUint32(n2, 101)
	binary.BigEndian.PutUint32(n3, 102)
	binary.BigEndian.PutUint32(n4, 103)
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	f.Test(n4)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestDirect20_5(t *testing.T) {
	n := uint(10000)
	k := uint(5)
	load := uint(20)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestDirect15_10(t *testing.T) {
	n := uint(10000)
	k := uint(10)
	load := uint(15)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestEstimated10_0001(t *testing.T) {
	n := uint(10000)
	fp := 0.0001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestEstimated10_001(t *testing.T) {
	n := uint(10000)
	fp := 0.001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestMarshalUnmarshalJSON(t *testing.T) {
	f := New(1000, 4)
	data, err := json.Marshal(f)
	if err != nil {
		t.Fatal(err.Error())
	}

	var g BloomFilter
	err = json.Unmarshal(data, &g)
	if err != nil {
		t.Fatal(err.Error())
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
}

func TestWriteToReadFrom(t *testing.T) {
	var b bytes.Buffer
	f := New(1000, 4)
	_, err := f.WriteTo(&b)
	if err != nil {
		t.Fatal(err)
	}

	g := New(1000, 1)
	_, err = g.ReadFrom(&b)
	if err != nil {
		t.Fatal(err)
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}

	g.Test([]byte(""))
}

func TestReadWriteBinary(t *testing.T) {
	f := New(1000, 4)
	var buf bytes.Buffer
	bytesWritten, err := f.WriteTo(&buf)
	if err != nil {
		t.Fatal(err.Error())
	}
	if bytesWritten != int64(buf.Len()) {
		t.Errorf("incorrect write length %d != %d", bytesWritten, buf.Len())
	}

	var g BloomFilter
	bytesRead, err := g.ReadFrom(&buf)
	if err != nil {
		t.Fatal(err.Error())
	}
	if bytesRead != bytesWritten {
		t.Errorf("read unexpected number of bytes %d != %d", bytesRead, bytesWritten)
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
}

func TestEncodeDecodeGob(t *testing.T) {
	f := New(1000, 4)
	f.Add([]byte("one"))
	f.Add([]byte("two"))
	f.Add([]byte("three"))
	var buf bytes.Buffer
	err := gob.NewEncoder(&buf).Encode(f)
	if err != nil {
		t.Fatal(err.Error())
	}

	var g BloomFilter
	err = gob.NewDecoder(&buf).Decode(&g)
	if err != nil {
		t.Fatal(err.Error())
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
	if !g.Test([]byte("three")) {
		t.Errorf("missing value 'three'")
	}
	if !g.Test([]byte("two")) {
		t.Errorf("missing value 'two'")
	}
	if !g.Test([]byte("one")) {
		t.Errorf("missing value 'one'")
	}
}

func BenchmarkDirect(b *testing.B) {
	n := uint(10000)
	max_k := uint(10)
	max_load := uint(20)
	fmt.Printf("m/n")
	for k := uint(2); k <= max_k; k++ {
		fmt.Printf("\tk=%v", k)
	}
	fmt.Println()
	for load := uint(2); load <= max_load; load++ {
		fmt.Print(load)
		for k := uint(2); k <= max_k; k++ {
			f := New(n*load, k)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%f", fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkEstimated(b *testing.B) {
	for n := uint(5000); n <= 50000; n += 5000 {
		fmt.Printf("%v", n)
		for fp := 0.1; fp >= 0.00001; fp /= 10.0 {
			fmt.Printf("\t%f", fp)
			m, k := estimateParameters(n, fp)
			f := NewWithEstimates(n, fp)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%v\t%v\t%f", m, k, fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkSeparateTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.Test(key)
		f.Add(key)
	}
}

func BenchmarkCombinedTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.TestAndAdd(key)
	}
}
<commit_msg>Adding concurrency test designed by @cenkalti<commit_after>package bloom

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"testing"
	"runtime"
	"sync"
)

func TestConcurrent(t *testing.T) {
	gmp := runtime.GOMAXPROCS(2)
	defer runtime.GOMAXPROCS(gmp)

	f := New(1000, 4)
	n1 := []byte("Bess")
	n2 := []byte("Jane")
	f.Add(n1)
	f.Add(n2)

	var wg sync.WaitGroup
	const try = 1000
	var err1, err2 error

	wg.Add(1)
	go func() {
		for i := 0; i < try; i++ {
			n1b := f.Test(n1)
			if !n1b {
				err1 = fmt.Errorf("%v should be in.", n1)
				break
			}
		}
		wg.Done()
	}()

	wg.Add(1)
	go func() {
		for i := 0; i < try; i++ {
			n2b := f.Test(n2)
			if !n2b {
				err2 = fmt.Errorf("%v should be in.", n2)
				break
			}
		}
		wg.Done()
	}()

	wg.Wait()

	if err1 != nil {
		t.Fatal(err1)
	}
	if err2 != nil {
		t.Fatal(err2)
	}
}

func TestBasic(t *testing.T) {
	f := New(1000, 4)
	n1 := []byte("Bess")
	n2 := []byte("Jane")
	n3 := []byte("Emma")
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestBasicUint32(t *testing.T) {
	f := New(1000, 4)
	n1 := make([]byte, 4)
	n2 := make([]byte, 4)
	n3 := make([]byte, 4)
	n4 := make([]byte, 4)
	binary.BigEndian.PutUint32(n1, 100)
	binary.BigEndian.PutUint32(n2, 101)
	binary.BigEndian.PutUint32(n3, 102)
	binary.BigEndian.PutUint32(n4, 103)
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	f.Test(n4)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestDirect20_5(t *testing.T) {
	n := uint(10000)
	k := uint(5)
	load := uint(20)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestDirect15_10(t *testing.T) {
	n := uint(10000)
	k := uint(10)
	load := uint(15)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestEstimated10_0001(t *testing.T) {
	n := uint(10000)
	fp := 0.0001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestEstimated10_001(t *testing.T) {
	n := uint(10000)
	fp := 0.001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestMarshalUnmarshalJSON(t *testing.T) {
	f := New(1000, 4)
	data, err := json.Marshal(f)
	if err != nil {
		t.Fatal(err.Error())
	}

	var g BloomFilter
	err = json.Unmarshal(data, &g)
	if err != nil {
		t.Fatal(err.Error())
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
}

func TestWriteToReadFrom(t *testing.T) {
	var b bytes.Buffer
	f := New(1000, 4)
	_, err := f.WriteTo(&b)
	if err != nil {
		t.Fatal(err)
	}

	g := New(1000, 1)
	_, err = g.ReadFrom(&b)
	if err != nil {
		t.Fatal(err)
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}

	g.Test([]byte(""))
}

func TestReadWriteBinary(t *testing.T) {
	f := New(1000, 4)
	var buf bytes.Buffer
	bytesWritten, err := f.WriteTo(&buf)
	if err != nil {
		t.Fatal(err.Error())
	}
	if bytesWritten != int64(buf.Len()) {
		t.Errorf("incorrect write length %d != %d", bytesWritten, buf.Len())
	}

	var g BloomFilter
	bytesRead, err := g.ReadFrom(&buf)
	if err != nil {
		t.Fatal(err.Error())
	}
	if bytesRead != bytesWritten {
		t.Errorf("read unexpected number of bytes %d != %d", bytesRead, bytesWritten)
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
}

func TestEncodeDecodeGob(t *testing.T) {
	f := New(1000, 4)
	f.Add([]byte("one"))
	f.Add([]byte("two"))
	f.Add([]byte("three"))
	var buf bytes.Buffer
	err := gob.NewEncoder(&buf).Encode(f)
	if err != nil {
		t.Fatal(err.Error())
	}

	var g BloomFilter
	err = gob.NewDecoder(&buf).Decode(&g)
	if err != nil {
		t.Fatal(err.Error())
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid k value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
	if !g.Test([]byte("three")) {
		t.Errorf("missing value 'three'")
	}
	if !g.Test([]byte("two")) {
		t.Errorf("missing value 'two'")
	}
	if !g.Test([]byte("one")) {
		t.Errorf("missing value 'one'")
	}
}

func BenchmarkDirect(b *testing.B) {
	n := uint(10000)
	max_k := uint(10)
	max_load := uint(20)
	fmt.Printf("m/n")
	for k := uint(2); k <= max_k; k++ {
		fmt.Printf("\tk=%v", k)
	}
	fmt.Println()
	for load := uint(2); load <= max_load; load++ {
		fmt.Print(load)
		for k := uint(2); k <= max_k; k++ {
			f := New(n*load, k)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%f", fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkEstimated(b *testing.B) {
	for n := uint(5000); n <= 50000; n += 5000 {
		fmt.Printf("%v", n)
		for fp := 0.1; fp >= 0.00001; fp /= 10.0 {
			fmt.Printf("\t%f", fp)
			m, k := estimateParameters(n, fp)
			f := NewWithEstimates(n, fp)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%v\t%v\t%f", m, k, fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkSeparateTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.Test(key)
		f.Add(key)
	}
}

func BenchmarkCombinedTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.TestAndAdd(key)
	}
}
<|endoftext|>


<commit_before>package bolt

import (
	"bytes"
	"context"
	"encoding/json"

	bolt "github.com/coreos/bbolt"
	platform "github.com/influxdata/influxdb"
)

var (
	macroBucket    = []byte("macrosv1")
	macroOrgsIndex = []byte("macroorgsv1")
)

func (c *Client) initializeMacros(ctx context.Context, tx *bolt.Tx) error {
	if _, err := tx.CreateBucketIfNotExists([]byte(macroBucket)); err != nil {
		return err
	}
	if _, err := tx.CreateBucketIfNotExists(macroOrgsIndex); err != nil {
		return err
	}
	return nil
}

func decodeMacroOrgsIndexKey(indexKey []byte) (orgID platform.ID, macroID platform.ID, err error) {
	if len(indexKey) != 2*platform.IDLength {
		return 0, 0, &platform.Error{
			Code: platform.EInvalid,
			Msg:  "malformed macro orgs index key (please report this error)",
		}
	}

	if err := (&orgID).Decode(indexKey[:platform.IDLength]); err != nil {
		return 0, 0, &platform.Error{
			Code: platform.EInvalid,
			Msg:  "bad org id",
			Err:  platform.ErrInvalidID,
		}
	}

	if err := (&macroID).Decode(indexKey[platform.IDLength:]); err != nil {
		return 0, 0, &platform.Error{
			Code: platform.EInvalid,
			Msg:  "bad macro id",
			Err:  platform.ErrInvalidID,
		}
	}

	return orgID, macroID, nil
}

func (c *Client) findOrganizationMacros(ctx context.Context, tx *bolt.Tx, orgID platform.ID) ([]*platform.Macro, error) {
	// TODO(leodido): support find options
	cur := tx.Bucket(macroOrgsIndex).Cursor()
	prefix, err := orgID.Encode()
	if err != nil {
		return nil, err
	}

	macros := []*platform.Macro{}
	for k, _ := cur.Seek(prefix); bytes.HasPrefix(k, prefix); k, _ = cur.Next() {
		_, id, err := decodeMacroOrgsIndexKey(k)
		if err != nil {
			return nil, err
		}

		m, err := c.findMacroByID(ctx, tx, id)
		if err != nil {
			return nil, err
		}

		macros = append(macros, m)
	}

	return macros, nil
}

func (c *Client) findMacros(ctx context.Context, tx *bolt.Tx, filter platform.MacroFilter) ([]*platform.Macro, error) {
	if filter.OrganizationID != nil {
		return c.findOrganizationMacros(ctx, tx, *filter.OrganizationID)
	}

	if filter.Organization != nil {
		o, err := c.findOrganizationByName(ctx, tx, *filter.Organization)
		if err != nil {
			return nil, err
		}
		return c.findOrganizationMacros(ctx, tx, o.ID)
	}

	macros := []*platform.Macro{}
	filterFn := filterMacrosFn(filter)
	err := c.forEachMacro(ctx, tx, func(m *platform.Macro) bool {
		if filterFn(m) {
			macros = append(macros, m)
		}
		return true
	})

	if err != nil {
		return nil, err
	}

	return macros, nil
}

func filterMacrosFn(filter platform.MacroFilter) func(m *platform.Macro) bool {
	if filter.ID != nil {
		return func(m *platform.Macro) bool {
			return m.ID == *filter.ID
		}
	}

	if filter.OrganizationID != nil {
		return func(m *platform.Macro) bool {
			return m.OrganizationID == *filter.OrganizationID
		}
	}

	return func(m *platform.Macro) bool { return true }
}

// forEachMacro will iterate through all macros while fn returns true.
func (c *Client) forEachMacro(ctx context.Context, tx *bolt.Tx, fn func(*platform.Macro) bool) error {
	cur := tx.Bucket(macroBucket).Cursor()
	for k, v := cur.First(); k != nil; k, v = cur.Next() {
		m := &platform.Macro{}
		if err := json.Unmarshal(v, m); err != nil {
			return err
		}
		if !fn(m) {
			break
		}
	}

	return nil
}

// FindMacros returns all macros in the store
func (c *Client) FindMacros(ctx context.Context, filter platform.MacroFilter, opt ...platform.FindOptions) ([]*platform.Macro, error) {
	// todo(leodido) > handle find options
	op := getOp(platform.OpFindMacros)
	res := []*platform.Macro{}
	err := c.db.View(func(tx *bolt.Tx) error {
		macros, err := c.findMacros(ctx, tx, filter)
		if err != nil && platform.ErrorCode(err) != platform.ENotFound {
			return err
		}
		res = macros
		return nil
	})

	if err != nil {
		return nil, &platform.Error{
			Op:  op,
			Err: err,
		}
	}

	return res, nil
}

// FindMacroByID finds a single macro in the store by its ID
func (c *Client) FindMacroByID(ctx context.Context, id platform.ID) (*platform.Macro, error) {
	op := getOp(platform.OpFindMacroByID)
	var macro *platform.Macro
	err := c.db.View(func(tx *bolt.Tx) error {
		m, pe := c.findMacroByID(ctx, tx, id)
		if pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}
		macro = m
		return nil
	})
	if err != nil {
		return nil, err
	}

	return macro, nil
}

func (c *Client) findMacroByID(ctx context.Context, tx *bolt.Tx, id platform.ID) (*platform.Macro, error) {
	encID, err := id.Encode()
	if err != nil {
		return nil, &platform.Error{
			Code: platform.EInvalid,
			Err:  err,
		}
	}

	d := tx.Bucket(macroBucket).Get(encID)
	if d == nil {
		return nil, &platform.Error{
			Code: platform.ENotFound,
			Msg:  platform.ErrMacroNotFound,
		}
	}

	macro := &platform.Macro{}
	err = json.Unmarshal(d, &macro)
	if err != nil {
		return nil, &platform.Error{
			Err: err,
		}
	}

	return macro, nil
}

// CreateMacro creates a new macro and assigns it an ID
func (c *Client) CreateMacro(ctx context.Context, macro *platform.Macro) error {
	op := getOp(platform.OpCreateMacro)
	return c.db.Update(func(tx *bolt.Tx) error {
		macro.ID = c.IDGenerator.ID()

		if err := c.putMacroOrgsIndex(ctx, tx, macro); err != nil {
			return err
		}

		if pe := c.putMacro(ctx, tx, macro); pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}

		}
		return nil
	})
}

// ReplaceMacro puts a macro in the store
func (c *Client) ReplaceMacro(ctx context.Context, macro *platform.Macro) error {
	return c.db.Update(func(tx *bolt.Tx) error {
		if err := c.putMacroOrgsIndex(ctx, tx, macro); err != nil {
			return err
		}
		return c.putMacro(ctx, tx, macro)
	})
}

func encodeMacroOrgsIndex(macro *platform.Macro) ([]byte, error) {
	oID, err := macro.OrganizationID.Encode()
	if err != nil {
		return nil, &platform.Error{
			Err: err,
			Msg: "bad organization id",
		}
	}

	mID, err := macro.ID.Encode()
	if err != nil {
		return nil, &platform.Error{
			Err: err,
			Msg: "bad macro id",
		}
	}

	key := make([]byte, 0, platform.IDLength*2)
	key = append(key, oID...)
	key = append(key, mID...)

	return key, nil
}

func (c *Client) putMacroOrgsIndex(ctx context.Context, tx *bolt.Tx, macro *platform.Macro) error {
	key, err := encodeMacroOrgsIndex(macro)
	if err != nil {
		return err
	}

	if err := tx.Bucket(macroOrgsIndex).Put(key, nil); err != nil {
		return &platform.Error{
			Err: err,
		}
	}

	return nil
}

func (c *Client) removeMacroOrgsIndex(ctx context.Context, tx *bolt.Tx, macro *platform.Macro) error {
	key, err := encodeMacroOrgsIndex(macro)
	if err != nil {
		return err
	}

	if err := tx.Bucket(macroOrgsIndex).Delete(key); err != nil {
		return err
	}

	return nil
}

func (c *Client) putMacro(ctx context.Context, tx *bolt.Tx, macro *platform.Macro) error {
	m, err := json.Marshal(macro)
	if err != nil {
		return &platform.Error{
			Err: err,
		}
	}

	encID, err := macro.ID.Encode()
	if err != nil {
		return &platform.Error{
			Code: platform.EInvalid,
			Err:  err,
		}
	}

	if err := tx.Bucket(macroBucket).Put(encID, m); err != nil {
		return &platform.Error{
			Err: err,
		}
	}

	return nil
}

// UpdateMacro updates a single macro in the store with a changeset
func (c *Client) UpdateMacro(ctx context.Context, id platform.ID, update *platform.MacroUpdate) (*platform.Macro, error) {
	op := getOp(platform.OpUpdateMacro)
	var macro *platform.Macro
	err := c.db.Update(func(tx *bolt.Tx) error {
		m, pe := c.findMacroByID(ctx, tx, id)
		if pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}

		if err := update.Apply(m); err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}

		macro = m
		if pe = c.putMacro(ctx, tx, macro); pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}
		return nil
	})

	return macro, err
}

// DeleteMacro removes a single macro from the store by its ID
func (c *Client) DeleteMacro(ctx context.Context, id platform.ID) error {
	op := getOp(platform.OpDeleteMacro)
	return c.db.Update(func(tx *bolt.Tx) error {
		m, pe := c.findMacroByID(ctx, tx, id)
		if pe != nil {
			return pe
		}

		encID, err := id.Encode()
		if err != nil {
			return &platform.Error{
				Err: err,
			}
		}

		if err := c.removeMacroOrgsIndex(ctx, tx, m); err != nil {
			return platform.NewError(platform.WithErrorErr(err))
		}

		if err := tx.Bucket(macroBucket).Delete(encID); err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}

		return nil
	})
}
<commit_msg>chore(bolt): use platform.Error within ReplaceMacro and DeleteMacro methods<commit_after>package bolt

import (
	"bytes"
	"context"
	"encoding/json"

	bolt "github.com/coreos/bbolt"
	platform "github.com/influxdata/influxdb"
)

var (
	macroBucket    = []byte("macrosv1")
	macroOrgsIndex = []byte("macroorgsv1")
)

func (c *Client) initializeMacros(ctx context.Context, tx *bolt.Tx) error {
	if _, err := tx.CreateBucketIfNotExists([]byte(macroBucket)); err != nil {
		return err
	}
	if _, err := tx.CreateBucketIfNotExists(macroOrgsIndex); err != nil {
		return err
	}
	return nil
}

func decodeMacroOrgsIndexKey(indexKey []byte) (orgID platform.ID, macroID platform.ID, err error) {
	if len(indexKey) != 2*platform.IDLength {
		return 0, 0, &platform.Error{
			Code: platform.EInvalid,
			Msg:  "malformed macro orgs index key (please report this error)",
		}
	}

	if err := (&orgID).Decode(indexKey[:platform.IDLength]); err != nil {
		return 0, 0, &platform.Error{
			Code: platform.EInvalid,
			Msg:  "bad org id",
			Err:  platform.ErrInvalidID,
		}
	}

	if err := (&macroID).Decode(indexKey[platform.IDLength:]); err != nil {
		return 0, 0, &platform.Error{
			Code: platform.EInvalid,
			Msg:  "bad macro id",
			Err:  platform.ErrInvalidID,
		}
	}

	return orgID, macroID, nil
}

func (c *Client) findOrganizationMacros(ctx context.Context, tx *bolt.Tx, orgID platform.ID) ([]*platform.Macro, error) {
	// TODO(leodido): support find options
	cur := tx.Bucket(macroOrgsIndex).Cursor()
	prefix, err := orgID.Encode()
	if err != nil {
		return nil, err
	}

	macros := []*platform.Macro{}
	for k, _ := cur.Seek(prefix); bytes.HasPrefix(k, prefix); k, _ = cur.Next() {
		_, id, err := decodeMacroOrgsIndexKey(k)
		if err != nil {
			return nil, err
		}

		m, err := c.findMacroByID(ctx, tx, id)
		if err != nil {
			return nil, err
		}

		macros = append(macros, m)
	}

	return macros, nil
}

func (c *Client) findMacros(ctx context.Context, tx *bolt.Tx, filter platform.MacroFilter) ([]*platform.Macro, error) {
	if filter.OrganizationID != nil {
		return c.findOrganizationMacros(ctx, tx, *filter.OrganizationID)
	}

	if filter.Organization != nil {
		o, err := c.findOrganizationByName(ctx, tx, *filter.Organization)
		if err != nil {
			return nil, err
		}
		return c.findOrganizationMacros(ctx, tx, o.ID)
	}

	macros := []*platform.Macro{}
	filterFn := filterMacrosFn(filter)
	err := c.forEachMacro(ctx, tx, func(m *platform.Macro) bool {
		if filterFn(m) {
			macros = append(macros, m)
		}
		return true
	})

	if err != nil {
		return nil, err
	}

	return macros, nil
}

func filterMacrosFn(filter platform.MacroFilter) func(m *platform.Macro) bool {
	if filter.ID != nil {
		return func(m *platform.Macro) bool {
			return m.ID == *filter.ID
		}
	}

	if filter.OrganizationID != nil {
		return func(m *platform.Macro) bool {
			return m.OrganizationID == *filter.OrganizationID
		}
	}

	return func(m *platform.Macro) bool { return true }
}

// forEachMacro will iterate through all macros while fn returns true.
func (c *Client) forEachMacro(ctx context.Context, tx *bolt.Tx, fn func(*platform.Macro) bool) error {
	cur := tx.Bucket(macroBucket).Cursor()
	for k, v := cur.First(); k != nil; k, v = cur.Next() {
		m := &platform.Macro{}
		if err := json.Unmarshal(v, m); err != nil {
			return err
		}
		if !fn(m) {
			break
		}
	}

	return nil
}

// FindMacros returns all macros in the store
func (c *Client) FindMacros(ctx context.Context, filter platform.MacroFilter, opt ...platform.FindOptions) ([]*platform.Macro, error) {
	// todo(leodido) > handle find options
	op := getOp(platform.OpFindMacros)
	res := []*platform.Macro{}
	err := c.db.View(func(tx *bolt.Tx) error {
		macros, err := c.findMacros(ctx, tx, filter)
		if err != nil && platform.ErrorCode(err) != platform.ENotFound {
			return err
		}
		res = macros
		return nil
	})

	if err != nil {
		return nil, &platform.Error{
			Op:  op,
			Err: err,
		}
	}

	return res, nil
}

// FindMacroByID finds a single macro in the store by its ID
func (c *Client) FindMacroByID(ctx context.Context, id platform.ID) (*platform.Macro, error) {
	op := getOp(platform.OpFindMacroByID)
	var macro *platform.Macro
	err := c.db.View(func(tx *bolt.Tx) error {
		m, pe := c.findMacroByID(ctx, tx, id)
		if pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}
		macro = m
		return nil
	})
	if err != nil {
		return nil, err
	}

	return macro, nil
}

func (c *Client) findMacroByID(ctx context.Context, tx *bolt.Tx, id platform.ID) (*platform.Macro, error) {
	encID, err := id.Encode()
	if err != nil {
		return nil, &platform.Error{
			Code: platform.EInvalid,
			Err:  err,
		}
	}

	d := tx.Bucket(macroBucket).Get(encID)
	if d == nil {
		return nil, &platform.Error{
			Code: platform.ENotFound,
			Msg:  platform.ErrMacroNotFound,
		}
	}

	macro := &platform.Macro{}
	err = json.Unmarshal(d, &macro)
	if err != nil {
		return nil, &platform.Error{
			Err: err,
		}
	}

	return macro, nil
}

// CreateMacro creates a new macro and assigns it an ID
func (c *Client) CreateMacro(ctx context.Context, macro *platform.Macro) error {
	op := getOp(platform.OpCreateMacro)
	return c.db.Update(func(tx *bolt.Tx) error {
		macro.ID = c.IDGenerator.ID()

		if err := c.putMacroOrgsIndex(ctx, tx, macro); err != nil {
			return err
		}

		if pe := c.putMacro(ctx, tx, macro); pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}

		}
		return nil
	})
}

// ReplaceMacro puts a macro in the store
func (c *Client) ReplaceMacro(ctx context.Context, macro *platform.Macro) error {
	op := getOp(platform.OpReplaceMacro)
	return c.db.Update(func(tx *bolt.Tx) error {
		if err := c.putMacroOrgsIndex(ctx, tx, macro); err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}
		return c.putMacro(ctx, tx, macro)
	})
}

func encodeMacroOrgsIndex(macro *platform.Macro) ([]byte, error) {
	oID, err := macro.OrganizationID.Encode()
	if err != nil {
		return nil, &platform.Error{
			Err: err,
			Msg: "bad organization id",
		}
	}

	mID, err := macro.ID.Encode()
	if err != nil {
		return nil, &platform.Error{
			Err: err,
			Msg: "bad macro id",
		}
	}

	key := make([]byte, 0, platform.IDLength*2)
	key = append(key, oID...)
	key = append(key, mID...)

	return key, nil
}

func (c *Client) putMacroOrgsIndex(ctx context.Context, tx *bolt.Tx, macro *platform.Macro) error {
	key, err := encodeMacroOrgsIndex(macro)
	if err != nil {
		return err
	}

	if err := tx.Bucket(macroOrgsIndex).Put(key, nil); err != nil {
		return &platform.Error{
			Err: err,
		}
	}

	return nil
}

func (c *Client) removeMacroOrgsIndex(ctx context.Context, tx *bolt.Tx, macro *platform.Macro) error {
	key, err := encodeMacroOrgsIndex(macro)
	if err != nil {
		return err
	}

	if err := tx.Bucket(macroOrgsIndex).Delete(key); err != nil {
		return err
	}

	return nil
}

func (c *Client) putMacro(ctx context.Context, tx *bolt.Tx, macro *platform.Macro) error {
	m, err := json.Marshal(macro)
	if err != nil {
		return &platform.Error{
			Err: err,
		}
	}

	encID, err := macro.ID.Encode()
	if err != nil {
		return &platform.Error{
			Code: platform.EInvalid,
			Err:  err,
		}
	}

	if err := tx.Bucket(macroBucket).Put(encID, m); err != nil {
		return &platform.Error{
			Err: err,
		}
	}

	return nil
}

// UpdateMacro updates a single macro in the store with a changeset
func (c *Client) UpdateMacro(ctx context.Context, id platform.ID, update *platform.MacroUpdate) (*platform.Macro, error) {
	op := getOp(platform.OpUpdateMacro)
	var macro *platform.Macro
	err := c.db.Update(func(tx *bolt.Tx) error {
		m, pe := c.findMacroByID(ctx, tx, id)
		if pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}

		if err := update.Apply(m); err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}

		macro = m
		if pe = c.putMacro(ctx, tx, macro); pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}
		return nil
	})

	return macro, err
}

// DeleteMacro removes a single macro from the store by its ID
func (c *Client) DeleteMacro(ctx context.Context, id platform.ID) error {
	op := getOp(platform.OpDeleteMacro)
	return c.db.Update(func(tx *bolt.Tx) error {
		m, pe := c.findMacroByID(ctx, tx, id)
		if pe != nil {
			return &platform.Error{
				Op:  op,
				Err: pe,
			}
		}

		encID, err := id.Encode()
		if err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}

		if err := c.removeMacroOrgsIndex(ctx, tx, m); err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}

		if err := tx.Bucket(macroBucket).Delete(encID); err != nil {
			return &platform.Error{
				Op:  op,
				Err: err,
			}
		}

		return nil
	})
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"os"
	_ "rsc.io/sqlite"
	"testing"
)

func queryAndExpect(query string, expected int, db *sql.DB, t *testing.T) {
	var bookCount = len(getBooks(query, db))
	if bookCount != expected {
		t.Errorf("Expected %v books, found %v", expected, bookCount)
	}
}

func TestBasic(t *testing.T) {
	var db, err = initDb("./", "test.db")
	if err != nil {
		t.Fatalf("Error on db init: %v", err)
	}
	defer db.Close()

	queryAndExpect("", 0, db, t)

	var b = Book{}
	insert(&b, db)

	queryAndExpect("", 1, db, t)

	os.Remove("./test.db")
}

func TestQueries(t *testing.T) {
	var db, err = initDb("./", "test.db")
	if err != nil {
		t.Fatalf("Error on db init: %v", err)
	}
	defer db.Close()

	var b1 = Book{ID: 1, Title: "Foo", Author: "bar"}
	var b2 = Book{ID: 2, Title: "Ze'bra", Author: "bar"}
	var b3 = Book{ID: 3, Title: "Delta Spaces otherchars (subtitle goes here)", Author: "other"}
	insert(&b1, db)
	insert(&b2, db)
	insert(&b3, db)

	queryAndExpect("", 3, db, t)
	queryAndExpect("bar", 2, db, t)
	queryAndExpect("other", 1, db, t)
	queryAndExpect("a", 3, db, t)
	queryAndExpect("NO such $tring", 0, db, t)

	os.Remove("./test.db")
}
<commit_msg>Added tests for web handlers<commit_after>package main

import (
	"database/sql"
	"net/http"
	"net/http/httptest"
	"os"
	_ "rsc.io/sqlite"
	"testing"
)

func queryAndExpect(query string, expected int, db *sql.DB, t *testing.T) {
	var bookCount = len(getBooks(query, db))
	if bookCount != expected {
		t.Errorf("Expected %v books, found %v", expected, bookCount)
	}
}

func TestBasic(t *testing.T) {
	var db, err = initDb("./", "test.db")
	if err != nil {
		t.Fatalf("Error on db init: %v", err)
	}
	defer db.Close()

	queryAndExpect("", 0, db, t)

	var b = Book{}
	insert(&b, db)

	queryAndExpect("", 1, db, t)

	os.Remove("./test.db")
}

func TestQueries(t *testing.T) {
	var db, err = initDb("./", "test.db")
	if err != nil {
		t.Fatalf("Error on db init: %v", err)
	}
	defer db.Close()

	var b1 = Book{ID: 1, Title: "Foo", Author: "bar"}
	var b2 = Book{ID: 2, Title: "Ze'bra", Author: "bar"}
	var b3 = Book{ID: 3, Title: "Delta Spaces otherchars (subtitle goes here)", Author: "other"}
	insert(&b1, db)
	insert(&b2, db)
	insert(&b3, db)

	queryAndExpect("", 3, db, t)
	queryAndExpect("bar", 2, db, t)
	queryAndExpect("other", 1, db, t)
	queryAndExpect("a", 3, db, t)
	queryAndExpect("NO such $tring", 0, db, t)

	os.Remove("./test.db")
}

func TestAPIgetOk(t *testing.T) {
	DBName = "testgetok.db"
	defer os.Remove(DBName)

	req, err := http.NewRequest("GET", "/api/book", nil)
	if err != nil {
		t.Fatalf("Error setting up http request: %v", err)
	}

	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(webAPIBook)

	handler.ServeHTTP(rr, req)
	if status := rr.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
}

func TestAPIpostAndGet(t *testing.T) {
	DBName = "testpostandgetok.db"
	defer os.Remove(DBName)

	postReq := httptest.NewRequest("POST", "/api/book", nil)
	getReq := httptest.NewRequest("GET", "/api/book", nil)

	rr := httptest.NewRecorder()

	webAPIBook(rr, postReq)
	resp := rr.Result()
	if status := resp.StatusCode; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}

	webAPIBook(rr, getReq)
	resp = rr.Result()
	if status := resp.StatusCode; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
}

func TestAPImethodFail(t *testing.T) {
	DBName = "testmethodfail.db"
	defer os.Remove(DBName)

	req, err := http.NewRequest("PATCH", "/api/book", nil)
	if err != nil {
		t.Fatalf("Error setting up http request: %v", err)
	}

	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(webAPIBook)

	handler.ServeHTTP(rr, req)
	if status := rr.Code; status != http.StatusMethodNotAllowed {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
}
<|endoftext|>